From b683de4ad79242fdeebcae2afefb72c1530babe9 Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Sun, 29 Dec 2019 16:39:31 +0300 Subject: Rename directories for some crates from `syntax_x` to `rustc_x` `syntax_expand` -> `rustc_expand` `syntax_pos` -> `rustc_span` `syntax_ext` -> `rustc_builtin_macros` --- src/librustc_builtin_macros/Cargo.toml | 24 + src/librustc_builtin_macros/asm.rs | 289 ++++ src/librustc_builtin_macros/assert.rs | 137 ++ src/librustc_builtin_macros/cfg.rs | 54 + src/librustc_builtin_macros/cmdline_attrs.rs | 30 + src/librustc_builtin_macros/compile_error.rs | 20 + src/librustc_builtin_macros/concat.rs | 62 + src/librustc_builtin_macros/concat_idents.rs | 68 + src/librustc_builtin_macros/deriving/bounds.rs | 29 + src/librustc_builtin_macros/deriving/clone.rs | 225 +++ src/librustc_builtin_macros/deriving/cmp/eq.rs | 104 ++ src/librustc_builtin_macros/deriving/cmp/ord.rs | 113 ++ .../deriving/cmp/partial_eq.rs | 112 ++ .../deriving/cmp/partial_ord.rs | 302 ++++ src/librustc_builtin_macros/deriving/debug.rs | 137 ++ src/librustc_builtin_macros/deriving/decodable.rs | 225 +++ src/librustc_builtin_macros/deriving/default.rs | 83 + src/librustc_builtin_macros/deriving/encodable.rs | 287 ++++ .../deriving/generic/mod.rs | 1812 ++++++++++++++++++++ src/librustc_builtin_macros/deriving/generic/ty.rs | 283 +++ src/librustc_builtin_macros/deriving/hash.rs | 92 + src/librustc_builtin_macros/deriving/mod.rs | 171 ++ src/librustc_builtin_macros/env.rs | 88 + src/librustc_builtin_macros/format.rs | 1233 +++++++++++++ src/librustc_builtin_macros/format_foreign.rs | 827 +++++++++ .../format_foreign/printf/tests.rs | 145 ++ .../format_foreign/shell/tests.rs | 56 + src/librustc_builtin_macros/global_allocator.rs | 175 ++ src/librustc_builtin_macros/global_asm.rs | 64 + src/librustc_builtin_macros/lib.rs | 109 ++ src/librustc_builtin_macros/log_syntax.rs | 15 + src/librustc_builtin_macros/proc_macro_harness.rs | 463 +++++ src/librustc_builtin_macros/source_util.rs | 216 +++ .../standard_library_imports.rs | 85 + src/librustc_builtin_macros/test.rs | 439 +++++ src/librustc_builtin_macros/test_harness.rs | 366 ++++ src/librustc_builtin_macros/trace_macros.rs | 29 + src/librustc_builtin_macros/util.rs | 12 + src/librustc_expand/Cargo.toml | 23 + src/librustc_expand/base.rs | 1181 +++++++++++++ src/librustc_expand/build.rs | 663 +++++++ src/librustc_expand/expand.rs | 1725 +++++++++++++++++++ src/librustc_expand/lib.rs | 67 + src/librustc_expand/mbe.rs | 156 ++ src/librustc_expand/mbe/macro_check.rs | 627 +++++++ src/librustc_expand/mbe/macro_parser.rs | 930 ++++++++++ src/librustc_expand/mbe/macro_rules.rs | 1207 +++++++++++++ src/librustc_expand/mbe/quoted.rs | 248 +++ src/librustc_expand/mbe/transcribe.rs | 392 +++++ src/librustc_expand/mut_visit/tests.rs | 72 + src/librustc_expand/parse/lexer/tests.rs | 256 +++ src/librustc_expand/parse/tests.rs | 348 ++++ src/librustc_expand/placeholders.rs | 339 ++++ src/librustc_expand/proc_macro.rs | 239 +++ src/librustc_expand/proc_macro_server.rs | 689 ++++++++ src/librustc_expand/tests.rs | 1012 +++++++++++ src/librustc_expand/tokenstream/tests.rs | 110 ++ src/librustc_span/Cargo.toml | 21 + src/librustc_span/analyze_source_file.rs | 274 +++ src/librustc_span/analyze_source_file/tests.rs | 142 ++ src/librustc_span/caching_source_map_view.rs | 108 ++ src/librustc_span/edition.rs | 83 + src/librustc_span/fatal_error.rs | 26 + src/librustc_span/hygiene.rs | 850 +++++++++ src/librustc_span/lib.rs | 1672 ++++++++++++++++++ src/librustc_span/source_map.rs | 984 +++++++++++ src/librustc_span/source_map/tests.rs | 216 +++ src/librustc_span/span_encoding.rs | 140 ++ src/librustc_span/symbol.rs | 1213 +++++++++++++ src/librustc_span/symbol/tests.rs | 25 + src/librustc_span/tests.rs | 40 + src/libsyntax_expand/Cargo.toml | 23 - src/libsyntax_expand/base.rs | 1181 ------------- src/libsyntax_expand/build.rs | 663 ------- src/libsyntax_expand/expand.rs | 1725 ------------------- src/libsyntax_expand/lib.rs | 67 - src/libsyntax_expand/mbe.rs | 156 -- src/libsyntax_expand/mbe/macro_check.rs | 627 ------- src/libsyntax_expand/mbe/macro_parser.rs | 930 ---------- src/libsyntax_expand/mbe/macro_rules.rs | 1207 ------------- src/libsyntax_expand/mbe/quoted.rs | 248 --- src/libsyntax_expand/mbe/transcribe.rs | 392 ----- src/libsyntax_expand/mut_visit/tests.rs | 72 - src/libsyntax_expand/parse/lexer/tests.rs | 256 --- src/libsyntax_expand/parse/tests.rs | 348 ---- src/libsyntax_expand/placeholders.rs | 339 ---- src/libsyntax_expand/proc_macro.rs | 239 --- src/libsyntax_expand/proc_macro_server.rs | 689 -------- src/libsyntax_expand/tests.rs | 1012 ----------- src/libsyntax_expand/tokenstream/tests.rs | 110 -- src/libsyntax_ext/Cargo.toml | 24 - src/libsyntax_ext/asm.rs | 289 ---- src/libsyntax_ext/assert.rs | 137 -- src/libsyntax_ext/cfg.rs | 54 - src/libsyntax_ext/cmdline_attrs.rs | 30 - src/libsyntax_ext/compile_error.rs | 20 - src/libsyntax_ext/concat.rs | 62 - src/libsyntax_ext/concat_idents.rs | 68 - src/libsyntax_ext/deriving/bounds.rs | 29 - src/libsyntax_ext/deriving/clone.rs | 225 --- src/libsyntax_ext/deriving/cmp/eq.rs | 104 -- src/libsyntax_ext/deriving/cmp/ord.rs | 113 -- src/libsyntax_ext/deriving/cmp/partial_eq.rs | 112 -- src/libsyntax_ext/deriving/cmp/partial_ord.rs | 302 ---- src/libsyntax_ext/deriving/debug.rs | 137 -- src/libsyntax_ext/deriving/decodable.rs | 225 --- src/libsyntax_ext/deriving/default.rs | 83 - src/libsyntax_ext/deriving/encodable.rs | 287 ---- src/libsyntax_ext/deriving/generic/mod.rs | 1812 -------------------- src/libsyntax_ext/deriving/generic/ty.rs | 283 --- src/libsyntax_ext/deriving/hash.rs | 92 - src/libsyntax_ext/deriving/mod.rs | 171 -- src/libsyntax_ext/env.rs | 88 - src/libsyntax_ext/format.rs | 1233 ------------- src/libsyntax_ext/format_foreign.rs | 827 --------- src/libsyntax_ext/format_foreign/printf/tests.rs | 145 -- src/libsyntax_ext/format_foreign/shell/tests.rs | 56 - src/libsyntax_ext/global_allocator.rs | 175 -- src/libsyntax_ext/global_asm.rs | 64 - src/libsyntax_ext/lib.rs | 109 -- src/libsyntax_ext/log_syntax.rs | 15 - src/libsyntax_ext/proc_macro_harness.rs | 463 ----- src/libsyntax_ext/source_util.rs | 216 --- src/libsyntax_ext/standard_library_imports.rs | 85 - src/libsyntax_ext/test.rs | 439 ----- src/libsyntax_ext/test_harness.rs | 366 ---- src/libsyntax_ext/trace_macros.rs | 29 - src/libsyntax_ext/util.rs | 12 - src/libsyntax_pos/Cargo.toml | 21 - src/libsyntax_pos/analyze_source_file.rs | 274 --- src/libsyntax_pos/analyze_source_file/tests.rs | 142 -- src/libsyntax_pos/caching_source_map_view.rs | 108 -- src/libsyntax_pos/edition.rs | 83 - src/libsyntax_pos/fatal_error.rs | 26 - src/libsyntax_pos/hygiene.rs | 850 --------- src/libsyntax_pos/lib.rs | 1672 ------------------ src/libsyntax_pos/source_map.rs | 984 ----------- src/libsyntax_pos/source_map/tests.rs | 216 --- src/libsyntax_pos/span_encoding.rs | 140 -- src/libsyntax_pos/symbol.rs | 1213 ------------- src/libsyntax_pos/symbol/tests.rs | 25 - src/libsyntax_pos/tests.rs | 40 - 142 files changed, 25059 insertions(+), 25059 deletions(-) create mode 100644 src/librustc_builtin_macros/Cargo.toml create mode 100644 src/librustc_builtin_macros/asm.rs create mode 100644 src/librustc_builtin_macros/assert.rs create mode 100644 src/librustc_builtin_macros/cfg.rs create mode 100644 src/librustc_builtin_macros/cmdline_attrs.rs create mode 100644 src/librustc_builtin_macros/compile_error.rs create mode 100644 src/librustc_builtin_macros/concat.rs create mode 100644 src/librustc_builtin_macros/concat_idents.rs create mode 100644 src/librustc_builtin_macros/deriving/bounds.rs create mode 100644 src/librustc_builtin_macros/deriving/clone.rs create mode 100644 src/librustc_builtin_macros/deriving/cmp/eq.rs create mode 100644 src/librustc_builtin_macros/deriving/cmp/ord.rs create mode 100644 src/librustc_builtin_macros/deriving/cmp/partial_eq.rs create mode 100644 src/librustc_builtin_macros/deriving/cmp/partial_ord.rs create mode 100644 src/librustc_builtin_macros/deriving/debug.rs create mode 100644 src/librustc_builtin_macros/deriving/decodable.rs create mode 100644 src/librustc_builtin_macros/deriving/default.rs create mode 100644 src/librustc_builtin_macros/deriving/encodable.rs create mode 100644 src/librustc_builtin_macros/deriving/generic/mod.rs create mode 100644 src/librustc_builtin_macros/deriving/generic/ty.rs create mode 100644 src/librustc_builtin_macros/deriving/hash.rs create mode 100644 src/librustc_builtin_macros/deriving/mod.rs create mode 100644 src/librustc_builtin_macros/env.rs create mode 100644 src/librustc_builtin_macros/format.rs create mode 100644 src/librustc_builtin_macros/format_foreign.rs create mode 100644 src/librustc_builtin_macros/format_foreign/printf/tests.rs create mode 100644 src/librustc_builtin_macros/format_foreign/shell/tests.rs create mode 100644 src/librustc_builtin_macros/global_allocator.rs create mode 100644 src/librustc_builtin_macros/global_asm.rs create mode 100644 src/librustc_builtin_macros/lib.rs create mode 100644 src/librustc_builtin_macros/log_syntax.rs create mode 100644 src/librustc_builtin_macros/proc_macro_harness.rs create mode 100644 src/librustc_builtin_macros/source_util.rs create mode 100644 src/librustc_builtin_macros/standard_library_imports.rs create mode 100644 src/librustc_builtin_macros/test.rs create mode 100644 src/librustc_builtin_macros/test_harness.rs create mode 100644 src/librustc_builtin_macros/trace_macros.rs create mode 100644 src/librustc_builtin_macros/util.rs create mode 100644 src/librustc_expand/Cargo.toml create mode 100644 src/librustc_expand/base.rs create mode 100644 src/librustc_expand/build.rs create mode 100644 src/librustc_expand/expand.rs create mode 100644 src/librustc_expand/lib.rs create mode 100644 src/librustc_expand/mbe.rs create mode 100644 src/librustc_expand/mbe/macro_check.rs create mode 100644 src/librustc_expand/mbe/macro_parser.rs create mode 100644 src/librustc_expand/mbe/macro_rules.rs create mode 100644 src/librustc_expand/mbe/quoted.rs create mode 100644 src/librustc_expand/mbe/transcribe.rs create mode 100644 src/librustc_expand/mut_visit/tests.rs create mode 100644 src/librustc_expand/parse/lexer/tests.rs create mode 100644 src/librustc_expand/parse/tests.rs create mode 100644 src/librustc_expand/placeholders.rs create mode 100644 src/librustc_expand/proc_macro.rs create mode 100644 src/librustc_expand/proc_macro_server.rs create mode 100644 src/librustc_expand/tests.rs create mode 100644 src/librustc_expand/tokenstream/tests.rs create mode 100644 src/librustc_span/Cargo.toml create mode 100644 src/librustc_span/analyze_source_file.rs create mode 100644 src/librustc_span/analyze_source_file/tests.rs create mode 100644 src/librustc_span/caching_source_map_view.rs create mode 100644 src/librustc_span/edition.rs create mode 100644 src/librustc_span/fatal_error.rs create mode 100644 src/librustc_span/hygiene.rs create mode 100644 src/librustc_span/lib.rs create mode 100644 src/librustc_span/source_map.rs create mode 100644 src/librustc_span/source_map/tests.rs create mode 100644 src/librustc_span/span_encoding.rs create mode 100644 src/librustc_span/symbol.rs create mode 100644 src/librustc_span/symbol/tests.rs create mode 100644 src/librustc_span/tests.rs delete mode 100644 src/libsyntax_expand/Cargo.toml delete mode 100644 src/libsyntax_expand/base.rs delete mode 100644 src/libsyntax_expand/build.rs delete mode 100644 src/libsyntax_expand/expand.rs delete mode 100644 src/libsyntax_expand/lib.rs delete mode 100644 src/libsyntax_expand/mbe.rs delete mode 100644 src/libsyntax_expand/mbe/macro_check.rs delete mode 100644 src/libsyntax_expand/mbe/macro_parser.rs delete mode 100644 src/libsyntax_expand/mbe/macro_rules.rs delete mode 100644 src/libsyntax_expand/mbe/quoted.rs delete mode 100644 src/libsyntax_expand/mbe/transcribe.rs delete mode 100644 src/libsyntax_expand/mut_visit/tests.rs delete mode 100644 src/libsyntax_expand/parse/lexer/tests.rs delete mode 100644 src/libsyntax_expand/parse/tests.rs delete mode 100644 src/libsyntax_expand/placeholders.rs delete mode 100644 src/libsyntax_expand/proc_macro.rs delete mode 100644 src/libsyntax_expand/proc_macro_server.rs delete mode 100644 src/libsyntax_expand/tests.rs delete mode 100644 src/libsyntax_expand/tokenstream/tests.rs delete mode 100644 src/libsyntax_ext/Cargo.toml delete mode 100644 src/libsyntax_ext/asm.rs delete mode 100644 src/libsyntax_ext/assert.rs delete mode 100644 src/libsyntax_ext/cfg.rs delete mode 100644 src/libsyntax_ext/cmdline_attrs.rs delete mode 100644 src/libsyntax_ext/compile_error.rs delete mode 100644 src/libsyntax_ext/concat.rs delete mode 100644 src/libsyntax_ext/concat_idents.rs delete mode 100644 src/libsyntax_ext/deriving/bounds.rs delete mode 100644 src/libsyntax_ext/deriving/clone.rs delete mode 100644 src/libsyntax_ext/deriving/cmp/eq.rs delete mode 100644 src/libsyntax_ext/deriving/cmp/ord.rs delete mode 100644 src/libsyntax_ext/deriving/cmp/partial_eq.rs delete mode 100644 src/libsyntax_ext/deriving/cmp/partial_ord.rs delete mode 100644 src/libsyntax_ext/deriving/debug.rs delete mode 100644 src/libsyntax_ext/deriving/decodable.rs delete mode 100644 src/libsyntax_ext/deriving/default.rs delete mode 100644 src/libsyntax_ext/deriving/encodable.rs delete mode 100644 src/libsyntax_ext/deriving/generic/mod.rs delete mode 100644 src/libsyntax_ext/deriving/generic/ty.rs delete mode 100644 src/libsyntax_ext/deriving/hash.rs delete mode 100644 src/libsyntax_ext/deriving/mod.rs delete mode 100644 src/libsyntax_ext/env.rs delete mode 100644 src/libsyntax_ext/format.rs delete mode 100644 src/libsyntax_ext/format_foreign.rs delete mode 100644 src/libsyntax_ext/format_foreign/printf/tests.rs delete mode 100644 src/libsyntax_ext/format_foreign/shell/tests.rs delete mode 100644 src/libsyntax_ext/global_allocator.rs delete mode 100644 src/libsyntax_ext/global_asm.rs delete mode 100644 src/libsyntax_ext/lib.rs delete mode 100644 src/libsyntax_ext/log_syntax.rs delete mode 100644 src/libsyntax_ext/proc_macro_harness.rs delete mode 100644 src/libsyntax_ext/source_util.rs delete mode 100644 src/libsyntax_ext/standard_library_imports.rs delete mode 100644 src/libsyntax_ext/test.rs delete mode 100644 src/libsyntax_ext/test_harness.rs delete mode 100644 src/libsyntax_ext/trace_macros.rs delete mode 100644 src/libsyntax_ext/util.rs delete mode 100644 src/libsyntax_pos/Cargo.toml delete mode 100644 src/libsyntax_pos/analyze_source_file.rs delete mode 100644 src/libsyntax_pos/analyze_source_file/tests.rs delete mode 100644 src/libsyntax_pos/caching_source_map_view.rs delete mode 100644 src/libsyntax_pos/edition.rs delete mode 100644 src/libsyntax_pos/fatal_error.rs delete mode 100644 src/libsyntax_pos/hygiene.rs delete mode 100644 src/libsyntax_pos/lib.rs delete mode 100644 src/libsyntax_pos/source_map.rs delete mode 100644 src/libsyntax_pos/source_map/tests.rs delete mode 100644 src/libsyntax_pos/span_encoding.rs delete mode 100644 src/libsyntax_pos/symbol.rs delete mode 100644 src/libsyntax_pos/symbol/tests.rs delete mode 100644 src/libsyntax_pos/tests.rs diff --git a/src/librustc_builtin_macros/Cargo.toml b/src/librustc_builtin_macros/Cargo.toml new file mode 100644 index 00000000000..d73a9ea6cdb --- /dev/null +++ b/src/librustc_builtin_macros/Cargo.toml @@ -0,0 +1,24 @@ +[package] +authors = ["The Rust Project Developers"] +name = "syntax_ext" +version = "0.0.0" +edition = "2018" + +[lib] +name = "syntax_ext" +path = "lib.rs" +doctest = false + +[dependencies] +errors = { path = "../librustc_errors", package = "rustc_errors" } +fmt_macros = { path = "../libfmt_macros" } +log = "0.4" +rustc_data_structures = { path = "../librustc_data_structures" } +rustc_feature = { path = "../librustc_feature" } +rustc_parse = { path = "../librustc_parse" } +rustc_target = { path = "../librustc_target" } +smallvec = { version = "1.0", features = ["union", "may_dangle"] } +syntax = { path = "../libsyntax" } +syntax_expand = { path = "../libsyntax_expand" } +syntax_pos = { path = "../libsyntax_pos" } +rustc_error_codes = { path = "../librustc_error_codes" } diff --git a/src/librustc_builtin_macros/asm.rs b/src/librustc_builtin_macros/asm.rs new file mode 100644 index 00000000000..324bef9cbb8 --- /dev/null +++ b/src/librustc_builtin_macros/asm.rs @@ -0,0 +1,289 @@ +// Inline assembly support. +// +use State::*; + +use errors::{DiagnosticBuilder, PResult}; +use rustc_parse::parser::Parser; +use syntax::ast::{self, AsmDialect}; +use syntax::ptr::P; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::token::{self, Token}; +use syntax::tokenstream::{self, TokenStream}; +use syntax::{span_err, struct_span_err}; +use syntax_expand::base::*; +use syntax_pos::Span; + +use rustc_error_codes::*; + +enum State { + Asm, + Outputs, + Inputs, + Clobbers, + Options, + StateNone, +} + +impl State { + fn next(&self) -> State { + match *self { + Asm => Outputs, + Outputs => Inputs, + Inputs => Clobbers, + Clobbers => Options, + Options => StateNone, + StateNone => StateNone, + } + } +} + +const OPTIONS: &[Symbol] = &[sym::volatile, sym::alignstack, sym::intel]; + +pub fn expand_asm<'cx>( + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let mut inline_asm = match parse_inline_asm(cx, sp, tts) { + Ok(Some(inline_asm)) => inline_asm, + Ok(None) => return DummyResult::any(sp), + Err(mut err) => { + err.emit(); + return DummyResult::any(sp); + } + }; + + // If there are no outputs, the inline assembly is executed just for its side effects, + // so ensure that it is volatile + if inline_asm.outputs.is_empty() { + inline_asm.volatile = true; + } + + MacEager::expr(P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind: ast::ExprKind::InlineAsm(P(inline_asm)), + span: cx.with_def_site_ctxt(sp), + attrs: ast::AttrVec::new(), + })) +} + +fn parse_asm_str<'a>(p: &mut Parser<'a>) -> PResult<'a, Symbol> { + match p.parse_str_lit() { + Ok(str_lit) => Ok(str_lit.symbol_unescaped), + Err(opt_lit) => { + let span = opt_lit.map_or(p.token.span, |lit| lit.span); + let mut err = p.sess.span_diagnostic.struct_span_err(span, "expected string literal"); + err.span_label(span, "not a string literal"); + Err(err) + } + } +} + +fn parse_inline_asm<'a>( + cx: &mut ExtCtxt<'a>, + sp: Span, + tts: TokenStream, +) -> Result, DiagnosticBuilder<'a>> { + // Split the tts before the first colon, to avoid `asm!("x": y)` being + // parsed as `asm!(z)` with `z = "x": y` which is type ascription. + let first_colon = tts + .trees() + .position(|tt| match tt { + tokenstream::TokenTree::Token(Token { kind: token::Colon, .. }) + | tokenstream::TokenTree::Token(Token { kind: token::ModSep, .. }) => true, + _ => false, + }) + .unwrap_or(tts.len()); + let mut p = cx.new_parser_from_tts(tts.trees().skip(first_colon).collect()); + let mut asm = kw::Invalid; + let mut asm_str_style = None; + let mut outputs = Vec::new(); + let mut inputs = Vec::new(); + let mut clobs = Vec::new(); + let mut volatile = false; + let mut alignstack = false; + let mut dialect = AsmDialect::Att; + + let mut state = Asm; + + 'statement: loop { + match state { + Asm => { + if asm_str_style.is_some() { + // If we already have a string with instructions, + // ending up in Asm state again is an error. + return Err(struct_span_err!( + cx.parse_sess.span_diagnostic, + sp, + E0660, + "malformed inline assembly" + )); + } + // Nested parser, stop before the first colon (see above). + let mut p2 = cx.new_parser_from_tts(tts.trees().take(first_colon).collect()); + + if p2.token == token::Eof { + let mut err = + cx.struct_span_err(sp, "macro requires a string literal as an argument"); + err.span_label(sp, "string literal required"); + return Err(err); + } + + let expr = p2.parse_expr()?; + let (s, style) = + match expr_to_string(cx, expr, "inline assembly must be a string literal") { + Some((s, st)) => (s, st), + None => return Ok(None), + }; + + // This is most likely malformed. + if p2.token != token::Eof { + let mut extra_tts = p2.parse_all_token_trees()?; + extra_tts.extend(tts.trees().skip(first_colon)); + p = cx.new_parser_from_tts(extra_tts.into_iter().collect()); + } + + asm = s; + asm_str_style = Some(style); + } + Outputs => { + while p.token != token::Eof && p.token != token::Colon && p.token != token::ModSep { + if !outputs.is_empty() { + p.eat(&token::Comma); + } + + let constraint = parse_asm_str(&mut p)?; + + let span = p.prev_span; + + p.expect(&token::OpenDelim(token::Paren))?; + let expr = p.parse_expr()?; + p.expect(&token::CloseDelim(token::Paren))?; + + // Expands a read+write operand into two operands. + // + // Use '+' modifier when you want the same expression + // to be both an input and an output at the same time. + // It's the opposite of '=&' which means that the memory + // cannot be shared with any other operand (usually when + // a register is clobbered early.) + let constraint_str = constraint.as_str(); + let mut ch = constraint_str.chars(); + let output = match ch.next() { + Some('=') => None, + Some('+') => Some(Symbol::intern(&format!("={}", ch.as_str()))), + _ => { + span_err!( + cx, + span, + E0661, + "output operand constraint lacks '=' or '+'" + ); + None + } + }; + + let is_rw = output.is_some(); + let is_indirect = constraint_str.contains("*"); + outputs.push(ast::InlineAsmOutput { + constraint: output.unwrap_or(constraint), + expr, + is_rw, + is_indirect, + }); + } + } + Inputs => { + while p.token != token::Eof && p.token != token::Colon && p.token != token::ModSep { + if !inputs.is_empty() { + p.eat(&token::Comma); + } + + let constraint = parse_asm_str(&mut p)?; + + if constraint.as_str().starts_with("=") { + span_err!(cx, p.prev_span, E0662, "input operand constraint contains '='"); + } else if constraint.as_str().starts_with("+") { + span_err!(cx, p.prev_span, E0663, "input operand constraint contains '+'"); + } + + p.expect(&token::OpenDelim(token::Paren))?; + let input = p.parse_expr()?; + p.expect(&token::CloseDelim(token::Paren))?; + + inputs.push((constraint, input)); + } + } + Clobbers => { + while p.token != token::Eof && p.token != token::Colon && p.token != token::ModSep { + if !clobs.is_empty() { + p.eat(&token::Comma); + } + + let s = parse_asm_str(&mut p)?; + + if OPTIONS.iter().any(|&opt| s == opt) { + cx.span_warn(p.prev_span, "expected a clobber, found an option"); + } else if s.as_str().starts_with("{") || s.as_str().ends_with("}") { + span_err!( + cx, + p.prev_span, + E0664, + "clobber should not be surrounded by braces" + ); + } + + clobs.push(s); + } + } + Options => { + let option = parse_asm_str(&mut p)?; + + if option == sym::volatile { + // Indicates that the inline assembly has side effects + // and must not be optimized out along with its outputs. + volatile = true; + } else if option == sym::alignstack { + alignstack = true; + } else if option == sym::intel { + dialect = AsmDialect::Intel; + } else { + cx.span_warn(p.prev_span, "unrecognized option"); + } + + if p.token == token::Comma { + p.eat(&token::Comma); + } + } + StateNone => (), + } + + loop { + // MOD_SEP is a double colon '::' without space in between. + // When encountered, the state must be advanced twice. + match (&p.token.kind, state.next(), state.next().next()) { + (&token::Colon, StateNone, _) | (&token::ModSep, _, StateNone) => { + p.bump(); + break 'statement; + } + (&token::Colon, st, _) | (&token::ModSep, _, st) => { + p.bump(); + state = st; + } + (&token::Eof, ..) => break 'statement, + _ => break, + } + } + } + + Ok(Some(ast::InlineAsm { + asm, + asm_str_style: asm_str_style.unwrap(), + outputs, + inputs, + clobbers: clobs, + volatile, + alignstack, + dialect, + })) +} diff --git a/src/librustc_builtin_macros/assert.rs b/src/librustc_builtin_macros/assert.rs new file mode 100644 index 00000000000..331e9fa61d0 --- /dev/null +++ b/src/librustc_builtin_macros/assert.rs @@ -0,0 +1,137 @@ +use errors::{Applicability, DiagnosticBuilder}; + +use rustc_parse::parser::Parser; +use syntax::ast::{self, *}; +use syntax::print::pprust; +use syntax::ptr::P; +use syntax::symbol::{sym, Symbol}; +use syntax::token::{self, TokenKind}; +use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree}; +use syntax_expand::base::*; +use syntax_pos::{Span, DUMMY_SP}; + +pub fn expand_assert<'cx>( + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let Assert { cond_expr, custom_message } = match parse_assert(cx, sp, tts) { + Ok(assert) => assert, + Err(mut err) => { + err.emit(); + return DummyResult::any(sp); + } + }; + + // `core::panic` and `std::panic` are different macros, so we use call-site + // context to pick up whichever is currently in scope. + let sp = cx.with_call_site_ctxt(sp); + let tokens = custom_message.unwrap_or_else(|| { + TokenStream::from(TokenTree::token( + TokenKind::lit( + token::Str, + Symbol::intern(&format!( + "assertion failed: {}", + pprust::expr_to_string(&cond_expr).escape_debug() + )), + None, + ), + DUMMY_SP, + )) + }); + let args = P(MacArgs::Delimited(DelimSpan::from_single(sp), MacDelimiter::Parenthesis, tokens)); + let panic_call = Mac { + path: Path::from_ident(Ident::new(sym::panic, sp)), + args, + prior_type_ascription: None, + }; + let if_expr = cx.expr_if( + sp, + cx.expr(sp, ExprKind::Unary(UnOp::Not, cond_expr)), + cx.expr(sp, ExprKind::Mac(panic_call)), + None, + ); + MacEager::expr(if_expr) +} + +struct Assert { + cond_expr: P, + custom_message: Option, +} + +fn parse_assert<'a>( + cx: &mut ExtCtxt<'a>, + sp: Span, + stream: TokenStream, +) -> Result> { + let mut parser = cx.new_parser_from_tts(stream); + + if parser.token == token::Eof { + let mut err = cx.struct_span_err(sp, "macro requires a boolean expression as an argument"); + err.span_label(sp, "boolean expression required"); + return Err(err); + } + + let cond_expr = parser.parse_expr()?; + + // Some crates use the `assert!` macro in the following form (note extra semicolon): + // + // assert!( + // my_function(); + // ); + // + // Warn about semicolon and suggest removing it. Eventually, this should be turned into an + // error. + if parser.token == token::Semi { + let mut err = cx.struct_span_warn(sp, "macro requires an expression as an argument"); + err.span_suggestion( + parser.token.span, + "try removing semicolon", + String::new(), + Applicability::MaybeIncorrect, + ); + err.note("this is going to be an error in the future"); + err.emit(); + + parser.bump(); + } + + // Some crates use the `assert!` macro in the following form (note missing comma before + // message): + // + // assert!(true "error message"); + // + // Parse this as an actual message, and suggest inserting a comma. Eventually, this should be + // turned into an error. + let custom_message = + if let token::Literal(token::Lit { kind: token::Str, .. }) = parser.token.kind { + let mut err = cx.struct_span_warn(parser.token.span, "unexpected string literal"); + let comma_span = cx.source_map().next_point(parser.prev_span); + err.span_suggestion_short( + comma_span, + "try adding a comma", + ", ".to_string(), + Applicability::MaybeIncorrect, + ); + err.note("this is going to be an error in the future"); + err.emit(); + + parse_custom_message(&mut parser) + } else if parser.eat(&token::Comma) { + parse_custom_message(&mut parser) + } else { + None + }; + + if parser.token != token::Eof { + parser.expect_one_of(&[], &[])?; + unreachable!(); + } + + Ok(Assert { cond_expr, custom_message }) +} + +fn parse_custom_message(parser: &mut Parser<'_>) -> Option { + let ts = parser.parse_tokens(); + if !ts.is_empty() { Some(ts) } else { None } +} diff --git a/src/librustc_builtin_macros/cfg.rs b/src/librustc_builtin_macros/cfg.rs new file mode 100644 index 00000000000..7b1dbcc7762 --- /dev/null +++ b/src/librustc_builtin_macros/cfg.rs @@ -0,0 +1,54 @@ +/// The compiler code necessary to support the cfg! extension, which expands to +/// a literal `true` or `false` based on whether the given cfg matches the +/// current compilation environment. +use errors::DiagnosticBuilder; + +use syntax::ast; +use syntax::attr; +use syntax::token; +use syntax::tokenstream::TokenStream; +use syntax_expand::base::{self, *}; +use syntax_pos::Span; + +pub fn expand_cfg( + cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let sp = cx.with_def_site_ctxt(sp); + + match parse_cfg(cx, sp, tts) { + Ok(cfg) => { + let matches_cfg = attr::cfg_matches(&cfg, cx.parse_sess, cx.ecfg.features); + MacEager::expr(cx.expr_bool(sp, matches_cfg)) + } + Err(mut err) => { + err.emit(); + DummyResult::any(sp) + } + } +} + +fn parse_cfg<'a>( + cx: &mut ExtCtxt<'a>, + sp: Span, + tts: TokenStream, +) -> Result> { + let mut p = cx.new_parser_from_tts(tts); + + if p.token == token::Eof { + let mut err = cx.struct_span_err(sp, "macro requires a cfg-pattern as an argument"); + err.span_label(sp, "cfg-pattern required"); + return Err(err); + } + + let cfg = p.parse_meta_item()?; + + let _ = p.eat(&token::Comma); + + if !p.eat(&token::Eof) { + return Err(cx.struct_span_err(sp, "expected 1 cfg-pattern")); + } + + Ok(cfg) +} diff --git a/src/librustc_builtin_macros/cmdline_attrs.rs b/src/librustc_builtin_macros/cmdline_attrs.rs new file mode 100644 index 00000000000..1ce083112a8 --- /dev/null +++ b/src/librustc_builtin_macros/cmdline_attrs.rs @@ -0,0 +1,30 @@ +//! Attributes injected into the crate root from command line using `-Z crate-attr`. + +use syntax::ast::{self, AttrItem, AttrStyle}; +use syntax::attr::mk_attr; +use syntax::sess::ParseSess; +use syntax::token; +use syntax_expand::panictry; +use syntax_pos::FileName; + +pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -> ast::Crate { + for raw_attr in attrs { + let mut parser = rustc_parse::new_parser_from_source_str( + parse_sess, + FileName::cli_crate_attr_source_code(&raw_attr), + raw_attr.clone(), + ); + + let start_span = parser.token.span; + let AttrItem { path, args } = panictry!(parser.parse_attr_item()); + let end_span = parser.token.span; + if parser.token != token::Eof { + parse_sess.span_diagnostic.span_err(start_span.to(end_span), "invalid crate attribute"); + continue; + } + + krate.attrs.push(mk_attr(AttrStyle::Inner, path, args, start_span.to(end_span))); + } + + krate +} diff --git a/src/librustc_builtin_macros/compile_error.rs b/src/librustc_builtin_macros/compile_error.rs new file mode 100644 index 00000000000..394259fc67b --- /dev/null +++ b/src/librustc_builtin_macros/compile_error.rs @@ -0,0 +1,20 @@ +// The compiler code necessary to support the compile_error! extension. + +use syntax::tokenstream::TokenStream; +use syntax_expand::base::{self, *}; +use syntax_pos::Span; + +pub fn expand_compile_error<'cx>( + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let var = match get_single_str_from_tts(cx, sp, tts, "compile_error!") { + None => return DummyResult::any(sp), + Some(v) => v, + }; + + cx.span_err(sp, &var); + + DummyResult::any(sp) +} diff --git a/src/librustc_builtin_macros/concat.rs b/src/librustc_builtin_macros/concat.rs new file mode 100644 index 00000000000..0cc8e205ae9 --- /dev/null +++ b/src/librustc_builtin_macros/concat.rs @@ -0,0 +1,62 @@ +use syntax::ast; +use syntax::symbol::Symbol; +use syntax::tokenstream::TokenStream; +use syntax_expand::base::{self, DummyResult}; + +use std::string::String; + +pub fn expand_concat( + cx: &mut base::ExtCtxt<'_>, + sp: syntax_pos::Span, + tts: TokenStream, +) -> Box { + let es = match base::get_exprs_from_tts(cx, sp, tts) { + Some(e) => e, + None => return DummyResult::any(sp), + }; + let mut accumulator = String::new(); + let mut missing_literal = vec![]; + let mut has_errors = false; + for e in es { + match e.kind { + ast::ExprKind::Lit(ref lit) => match lit.kind { + ast::LitKind::Str(ref s, _) | ast::LitKind::Float(ref s, _) => { + accumulator.push_str(&s.as_str()); + } + ast::LitKind::Char(c) => { + accumulator.push(c); + } + ast::LitKind::Int(i, ast::LitIntType::Unsigned(_)) + | ast::LitKind::Int(i, ast::LitIntType::Signed(_)) + | ast::LitKind::Int(i, ast::LitIntType::Unsuffixed) => { + accumulator.push_str(&i.to_string()); + } + ast::LitKind::Bool(b) => { + accumulator.push_str(&b.to_string()); + } + ast::LitKind::Byte(..) | ast::LitKind::ByteStr(..) => { + cx.span_err(e.span, "cannot concatenate a byte string literal"); + } + ast::LitKind::Err(_) => { + has_errors = true; + } + }, + ast::ExprKind::Err => { + has_errors = true; + } + _ => { + missing_literal.push(e.span); + } + } + } + if missing_literal.len() > 0 { + let mut err = cx.struct_span_err(missing_literal, "expected a literal"); + err.note("only literals (like `\"foo\"`, `42` and `3.14`) can be passed to `concat!()`"); + err.emit(); + return DummyResult::any(sp); + } else if has_errors { + return DummyResult::any(sp); + } + let sp = cx.with_def_site_ctxt(sp); + base::MacEager::expr(cx.expr_str(sp, Symbol::intern(&accumulator))) +} diff --git a/src/librustc_builtin_macros/concat_idents.rs b/src/librustc_builtin_macros/concat_idents.rs new file mode 100644 index 00000000000..d870e858bea --- /dev/null +++ b/src/librustc_builtin_macros/concat_idents.rs @@ -0,0 +1,68 @@ +use syntax::ast; +use syntax::ptr::P; +use syntax::token::{self, Token}; +use syntax::tokenstream::{TokenStream, TokenTree}; +use syntax_expand::base::{self, *}; +use syntax_pos::symbol::Symbol; +use syntax_pos::Span; + +pub fn expand_concat_idents<'cx>( + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + if tts.is_empty() { + cx.span_err(sp, "concat_idents! takes 1 or more arguments."); + return DummyResult::any(sp); + } + + let mut res_str = String::new(); + for (i, e) in tts.into_trees().enumerate() { + if i & 1 == 1 { + match e { + TokenTree::Token(Token { kind: token::Comma, .. }) => {} + _ => { + cx.span_err(sp, "concat_idents! expecting comma."); + return DummyResult::any(sp); + } + } + } else { + match e { + TokenTree::Token(Token { kind: token::Ident(name, _), .. }) => { + res_str.push_str(&name.as_str()) + } + _ => { + cx.span_err(sp, "concat_idents! requires ident args."); + return DummyResult::any(sp); + } + } + } + } + + let ident = ast::Ident::new(Symbol::intern(&res_str), cx.with_call_site_ctxt(sp)); + + struct ConcatIdentsResult { + ident: ast::Ident, + } + + impl base::MacResult for ConcatIdentsResult { + fn make_expr(self: Box) -> Option> { + Some(P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind: ast::ExprKind::Path(None, ast::Path::from_ident(self.ident)), + span: self.ident.span, + attrs: ast::AttrVec::new(), + })) + } + + fn make_ty(self: Box) -> Option> { + Some(P(ast::Ty { + id: ast::DUMMY_NODE_ID, + kind: ast::TyKind::Path(None, ast::Path::from_ident(self.ident)), + span: self.ident.span, + })) + } + } + + Box::new(ConcatIdentsResult { ident }) +} diff --git a/src/librustc_builtin_macros/deriving/bounds.rs b/src/librustc_builtin_macros/deriving/bounds.rs new file mode 100644 index 00000000000..9793ac1ca08 --- /dev/null +++ b/src/librustc_builtin_macros/deriving/bounds.rs @@ -0,0 +1,29 @@ +use crate::deriving::generic::ty::*; +use crate::deriving::generic::*; +use crate::deriving::path_std; + +use syntax::ast::MetaItem; +use syntax_expand::base::{Annotatable, ExtCtxt}; +use syntax_pos::Span; + +pub fn expand_deriving_copy( + cx: &mut ExtCtxt<'_>, + span: Span, + mitem: &MetaItem, + item: &Annotatable, + push: &mut dyn FnMut(Annotatable), +) { + let trait_def = TraitDef { + span, + attributes: Vec::new(), + path: path_std!(cx, marker::Copy), + additional_bounds: Vec::new(), + generics: LifetimeBounds::empty(), + is_unsafe: false, + supports_unions: true, + methods: Vec::new(), + associated_types: Vec::new(), + }; + + trait_def.expand(cx, mitem, item, push); +} diff --git a/src/librustc_builtin_macros/deriving/clone.rs b/src/librustc_builtin_macros/deriving/clone.rs new file mode 100644 index 00000000000..171e4104c0a --- /dev/null +++ b/src/librustc_builtin_macros/deriving/clone.rs @@ -0,0 +1,225 @@ +use crate::deriving::generic::ty::*; +use crate::deriving::generic::*; +use crate::deriving::path_std; + +use syntax::ast::{self, Expr, GenericArg, Generics, ItemKind, MetaItem, VariantData}; +use syntax::ptr::P; +use syntax::symbol::{kw, sym, Symbol}; +use syntax_expand::base::{Annotatable, ExtCtxt}; +use syntax_pos::Span; + +pub fn expand_deriving_clone( + cx: &mut ExtCtxt<'_>, + span: Span, + mitem: &MetaItem, + item: &Annotatable, + push: &mut dyn FnMut(Annotatable), +) { + // check if we can use a short form + // + // the short form is `fn clone(&self) -> Self { *self }` + // + // we can use the short form if: + // - the item is Copy (unfortunately, all we can check is whether it's also deriving Copy) + // - there are no generic parameters (after specialization this limitation can be removed) + // if we used the short form with generics, we'd have to bound the generics with + // Clone + Copy, and then there'd be no Clone impl at all if the user fills in something + // that is Clone but not Copy. and until specialization we can't write both impls. + // - the item is a union with Copy fields + // Unions with generic parameters still can derive Clone because they require Copy + // for deriving, Clone alone is not enough. + // Whever Clone is implemented for fields is irrelevant so we don't assert it. + let bounds; + let substructure; + let is_shallow; + match *item { + Annotatable::Item(ref annitem) => match annitem.kind { + ItemKind::Struct(_, Generics { ref params, .. }) + | ItemKind::Enum(_, Generics { ref params, .. }) => { + let container_id = cx.current_expansion.id.expn_data().parent; + if cx.resolver.has_derive_copy(container_id) + && !params.iter().any(|param| match param.kind { + ast::GenericParamKind::Type { .. } => true, + _ => false, + }) + { + bounds = vec![]; + is_shallow = true; + substructure = combine_substructure(Box::new(|c, s, sub| { + cs_clone_shallow("Clone", c, s, sub, false) + })); + } else { + bounds = vec![]; + is_shallow = false; + substructure = + combine_substructure(Box::new(|c, s, sub| cs_clone("Clone", c, s, sub))); + } + } + ItemKind::Union(..) => { + bounds = vec![Literal(path_std!(cx, marker::Copy))]; + is_shallow = true; + substructure = combine_substructure(Box::new(|c, s, sub| { + cs_clone_shallow("Clone", c, s, sub, true) + })); + } + _ => { + bounds = vec![]; + is_shallow = false; + substructure = + combine_substructure(Box::new(|c, s, sub| cs_clone("Clone", c, s, sub))); + } + }, + + _ => cx.span_bug(span, "`#[derive(Clone)]` on trait item or impl item"), + } + + let inline = cx.meta_word(span, sym::inline); + let attrs = vec![cx.attribute(inline)]; + let trait_def = TraitDef { + span, + attributes: Vec::new(), + path: path_std!(cx, clone::Clone), + additional_bounds: bounds, + generics: LifetimeBounds::empty(), + is_unsafe: false, + supports_unions: true, + methods: vec![MethodDef { + name: "clone", + generics: LifetimeBounds::empty(), + explicit_self: borrowed_explicit_self(), + args: Vec::new(), + ret_ty: Self_, + attributes: attrs, + is_unsafe: false, + unify_fieldless_variants: false, + combine_substructure: substructure, + }], + associated_types: Vec::new(), + }; + + trait_def.expand_ext(cx, mitem, item, push, is_shallow) +} + +fn cs_clone_shallow( + name: &str, + cx: &mut ExtCtxt<'_>, + trait_span: Span, + substr: &Substructure<'_>, + is_union: bool, +) -> P { + fn assert_ty_bounds( + cx: &mut ExtCtxt<'_>, + stmts: &mut Vec, + ty: P, + span: Span, + helper_name: &str, + ) { + // Generate statement `let _: helper_name;`, + // set the expn ID so we can use the unstable struct. + let span = cx.with_def_site_ctxt(span); + let assert_path = cx.path_all( + span, + true, + cx.std_path(&[sym::clone, Symbol::intern(helper_name)]), + vec![GenericArg::Type(ty)], + ); + stmts.push(cx.stmt_let_type_only(span, cx.ty_path(assert_path))); + } + fn process_variant(cx: &mut ExtCtxt<'_>, stmts: &mut Vec, variant: &VariantData) { + for field in variant.fields() { + // let _: AssertParamIsClone; + assert_ty_bounds(cx, stmts, field.ty.clone(), field.span, "AssertParamIsClone"); + } + } + + let mut stmts = Vec::new(); + if is_union { + // let _: AssertParamIsCopy; + let self_ty = + cx.ty_path(cx.path_ident(trait_span, ast::Ident::with_dummy_span(kw::SelfUpper))); + assert_ty_bounds(cx, &mut stmts, self_ty, trait_span, "AssertParamIsCopy"); + } else { + match *substr.fields { + StaticStruct(vdata, ..) => { + process_variant(cx, &mut stmts, vdata); + } + StaticEnum(enum_def, ..) => { + for variant in &enum_def.variants { + process_variant(cx, &mut stmts, &variant.data); + } + } + _ => cx.span_bug( + trait_span, + &format!( + "unexpected substructure in \ + shallow `derive({})`", + name + ), + ), + } + } + stmts.push(cx.stmt_expr(cx.expr_deref(trait_span, cx.expr_self(trait_span)))); + cx.expr_block(cx.block(trait_span, stmts)) +} + +fn cs_clone( + name: &str, + cx: &mut ExtCtxt<'_>, + trait_span: Span, + substr: &Substructure<'_>, +) -> P { + let ctor_path; + let all_fields; + let fn_path = cx.std_path(&[sym::clone, sym::Clone, sym::clone]); + let subcall = |cx: &mut ExtCtxt<'_>, field: &FieldInfo<'_>| { + let args = vec![cx.expr_addr_of(field.span, field.self_.clone())]; + cx.expr_call_global(field.span, fn_path.clone(), args) + }; + + let vdata; + match *substr.fields { + Struct(vdata_, ref af) => { + ctor_path = cx.path(trait_span, vec![substr.type_ident]); + all_fields = af; + vdata = vdata_; + } + EnumMatching(.., variant, ref af) => { + ctor_path = cx.path(trait_span, vec![substr.type_ident, variant.ident]); + all_fields = af; + vdata = &variant.data; + } + EnumNonMatchingCollapsed(..) => { + cx.span_bug(trait_span, &format!("non-matching enum variants in `derive({})`", name,)) + } + StaticEnum(..) | StaticStruct(..) => { + cx.span_bug(trait_span, &format!("associated function in `derive({})`", name)) + } + } + + match *vdata { + VariantData::Struct(..) => { + let fields = all_fields + .iter() + .map(|field| { + let ident = match field.name { + Some(i) => i, + None => cx.span_bug( + trait_span, + &format!("unnamed field in normal struct in `derive({})`", name,), + ), + }; + let call = subcall(cx, field); + cx.field_imm(field.span, ident, call) + }) + .collect::>(); + + cx.expr_struct(trait_span, ctor_path, fields) + } + VariantData::Tuple(..) => { + let subcalls = all_fields.iter().map(|f| subcall(cx, f)).collect(); + let path = cx.expr_path(ctor_path); + cx.expr_call(trait_span, path, subcalls) + } + VariantData::Unit(..) => cx.expr_path(ctor_path), + } +} diff --git a/src/librustc_builtin_macros/deriving/cmp/eq.rs b/src/librustc_builtin_macros/deriving/cmp/eq.rs new file mode 100644 index 00000000000..f292ec0e428 --- /dev/null +++ b/src/librustc_builtin_macros/deriving/cmp/eq.rs @@ -0,0 +1,104 @@ +use crate::deriving::generic::ty::*; +use crate::deriving::generic::*; +use crate::deriving::path_std; + +use syntax::ast::{self, Expr, GenericArg, Ident, MetaItem}; +use syntax::ptr::P; +use syntax::symbol::{sym, Symbol}; +use syntax_expand::base::{Annotatable, ExtCtxt}; +use syntax_pos::Span; + +pub fn expand_deriving_eq( + cx: &mut ExtCtxt<'_>, + span: Span, + mitem: &MetaItem, + item: &Annotatable, + push: &mut dyn FnMut(Annotatable), +) { + let inline = cx.meta_word(span, sym::inline); + let hidden = syntax::attr::mk_nested_word_item(Ident::new(sym::hidden, span)); + let doc = syntax::attr::mk_list_item(Ident::new(sym::doc, span), vec![hidden]); + let attrs = vec![cx.attribute(inline), cx.attribute(doc)]; + let trait_def = TraitDef { + span, + attributes: Vec::new(), + path: path_std!(cx, cmp::Eq), + additional_bounds: Vec::new(), + generics: LifetimeBounds::empty(), + is_unsafe: false, + supports_unions: true, + methods: vec![MethodDef { + name: "assert_receiver_is_total_eq", + generics: LifetimeBounds::empty(), + explicit_self: borrowed_explicit_self(), + args: vec![], + ret_ty: nil_ty(), + attributes: attrs, + is_unsafe: false, + unify_fieldless_variants: true, + combine_substructure: combine_substructure(Box::new(|a, b, c| { + cs_total_eq_assert(a, b, c) + })), + }], + associated_types: Vec::new(), + }; + + super::inject_impl_of_structural_trait( + cx, + span, + item, + path_std!(cx, marker::StructuralEq), + push, + ); + + trait_def.expand_ext(cx, mitem, item, push, true) +} + +fn cs_total_eq_assert( + cx: &mut ExtCtxt<'_>, + trait_span: Span, + substr: &Substructure<'_>, +) -> P { + fn assert_ty_bounds( + cx: &mut ExtCtxt<'_>, + stmts: &mut Vec, + ty: P, + span: Span, + helper_name: &str, + ) { + // Generate statement `let _: helper_name;`, + // set the expn ID so we can use the unstable struct. + let span = cx.with_def_site_ctxt(span); + let assert_path = cx.path_all( + span, + true, + cx.std_path(&[sym::cmp, Symbol::intern(helper_name)]), + vec![GenericArg::Type(ty)], + ); + stmts.push(cx.stmt_let_type_only(span, cx.ty_path(assert_path))); + } + fn process_variant( + cx: &mut ExtCtxt<'_>, + stmts: &mut Vec, + variant: &ast::VariantData, + ) { + for field in variant.fields() { + // let _: AssertParamIsEq; + assert_ty_bounds(cx, stmts, field.ty.clone(), field.span, "AssertParamIsEq"); + } + } + + let mut stmts = Vec::new(); + match *substr.fields { + StaticStruct(vdata, ..) => { + process_variant(cx, &mut stmts, vdata); + } + StaticEnum(enum_def, ..) => { + for variant in &enum_def.variants { + process_variant(cx, &mut stmts, &variant.data); + } + } + _ => cx.span_bug(trait_span, "unexpected substructure in `derive(Eq)`"), + } + cx.expr_block(cx.block(trait_span, stmts)) +} diff --git a/src/librustc_builtin_macros/deriving/cmp/ord.rs b/src/librustc_builtin_macros/deriving/cmp/ord.rs new file mode 100644 index 00000000000..e009763da1b --- /dev/null +++ b/src/librustc_builtin_macros/deriving/cmp/ord.rs @@ -0,0 +1,113 @@ +use crate::deriving::generic::ty::*; +use crate::deriving::generic::*; +use crate::deriving::path_std; + +use syntax::ast::{self, Expr, MetaItem}; +use syntax::ptr::P; +use syntax::symbol::sym; +use syntax_expand::base::{Annotatable, ExtCtxt}; +use syntax_pos::Span; + +pub fn expand_deriving_ord( + cx: &mut ExtCtxt<'_>, + span: Span, + mitem: &MetaItem, + item: &Annotatable, + push: &mut dyn FnMut(Annotatable), +) { + let inline = cx.meta_word(span, sym::inline); + let attrs = vec![cx.attribute(inline)]; + let trait_def = TraitDef { + span, + attributes: Vec::new(), + path: path_std!(cx, cmp::Ord), + additional_bounds: Vec::new(), + generics: LifetimeBounds::empty(), + is_unsafe: false, + supports_unions: false, + methods: vec![MethodDef { + name: "cmp", + generics: LifetimeBounds::empty(), + explicit_self: borrowed_explicit_self(), + args: vec![(borrowed_self(), "other")], + ret_ty: Literal(path_std!(cx, cmp::Ordering)), + attributes: attrs, + is_unsafe: false, + unify_fieldless_variants: true, + combine_substructure: combine_substructure(Box::new(|a, b, c| cs_cmp(a, b, c))), + }], + associated_types: Vec::new(), + }; + + trait_def.expand(cx, mitem, item, push) +} + +pub fn ordering_collapsed( + cx: &mut ExtCtxt<'_>, + span: Span, + self_arg_tags: &[ast::Ident], +) -> P { + let lft = cx.expr_ident(span, self_arg_tags[0]); + let rgt = cx.expr_addr_of(span, cx.expr_ident(span, self_arg_tags[1])); + cx.expr_method_call(span, lft, ast::Ident::new(sym::cmp, span), vec![rgt]) +} + +pub fn cs_cmp(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> P { + let test_id = ast::Ident::new(sym::cmp, span); + let equals_path = cx.path_global(span, cx.std_path(&[sym::cmp, sym::Ordering, sym::Equal])); + + let cmp_path = cx.std_path(&[sym::cmp, sym::Ord, sym::cmp]); + + // Builds: + // + // match ::std::cmp::Ord::cmp(&self_field1, &other_field1) { + // ::std::cmp::Ordering::Equal => + // match ::std::cmp::Ord::cmp(&self_field2, &other_field2) { + // ::std::cmp::Ordering::Equal => { + // ... + // } + // cmp => cmp + // }, + // cmp => cmp + // } + // + cs_fold( + // foldr nests the if-elses correctly, leaving the first field + // as the outermost one, and the last as the innermost. + false, + |cx, span, old, self_f, other_fs| { + // match new { + // ::std::cmp::Ordering::Equal => old, + // cmp => cmp + // } + + let new = { + let other_f = match other_fs { + [o_f] => o_f, + _ => cx.span_bug(span, "not exactly 2 arguments in `derive(Ord)`"), + }; + + let args = + vec![cx.expr_addr_of(span, self_f), cx.expr_addr_of(span, other_f.clone())]; + + cx.expr_call_global(span, cmp_path.clone(), args) + }; + + let eq_arm = cx.arm(span, cx.pat_path(span, equals_path.clone()), old); + let neq_arm = cx.arm(span, cx.pat_ident(span, test_id), cx.expr_ident(span, test_id)); + + cx.expr_match(span, new, vec![eq_arm, neq_arm]) + }, + cx.expr_path(equals_path.clone()), + Box::new(|cx, span, (self_args, tag_tuple), _non_self_args| { + if self_args.len() != 2 { + cx.span_bug(span, "not exactly 2 arguments in `derive(Ord)`") + } else { + ordering_collapsed(cx, span, tag_tuple) + } + }), + cx, + span, + substr, + ) +} diff --git a/src/librustc_builtin_macros/deriving/cmp/partial_eq.rs b/src/librustc_builtin_macros/deriving/cmp/partial_eq.rs new file mode 100644 index 00000000000..91c13b76a00 --- /dev/null +++ b/src/librustc_builtin_macros/deriving/cmp/partial_eq.rs @@ -0,0 +1,112 @@ +use crate::deriving::generic::ty::*; +use crate::deriving::generic::*; +use crate::deriving::{path_local, path_std}; + +use syntax::ast::{BinOpKind, Expr, MetaItem}; +use syntax::ptr::P; +use syntax::symbol::sym; +use syntax_expand::base::{Annotatable, ExtCtxt}; +use syntax_pos::Span; + +pub fn expand_deriving_partial_eq( + cx: &mut ExtCtxt<'_>, + span: Span, + mitem: &MetaItem, + item: &Annotatable, + push: &mut dyn FnMut(Annotatable), +) { + // structures are equal if all fields are equal, and non equal, if + // any fields are not equal or if the enum variants are different + fn cs_op( + cx: &mut ExtCtxt<'_>, + span: Span, + substr: &Substructure<'_>, + op: BinOpKind, + combiner: BinOpKind, + base: bool, + ) -> P { + let op = |cx: &mut ExtCtxt<'_>, span: Span, self_f: P, other_fs: &[P]| { + let other_f = match other_fs { + [o_f] => o_f, + _ => cx.span_bug(span, "not exactly 2 arguments in `derive(PartialEq)`"), + }; + + cx.expr_binary(span, op, self_f, other_f.clone()) + }; + + cs_fold1( + true, // use foldl + |cx, span, subexpr, self_f, other_fs| { + let eq = op(cx, span, self_f, other_fs); + cx.expr_binary(span, combiner, subexpr, eq) + }, + |cx, args| { + match args { + Some((span, self_f, other_fs)) => { + // Special-case the base case to generate cleaner code. + op(cx, span, self_f, other_fs) + } + None => cx.expr_bool(span, base), + } + }, + Box::new(|cx, span, _, _| cx.expr_bool(span, !base)), + cx, + span, + substr, + ) + } + + fn cs_eq(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> P { + cs_op(cx, span, substr, BinOpKind::Eq, BinOpKind::And, true) + } + fn cs_ne(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> P { + cs_op(cx, span, substr, BinOpKind::Ne, BinOpKind::Or, false) + } + + macro_rules! md { + ($name:expr, $f:ident) => {{ + let inline = cx.meta_word(span, sym::inline); + let attrs = vec![cx.attribute(inline)]; + MethodDef { + name: $name, + generics: LifetimeBounds::empty(), + explicit_self: borrowed_explicit_self(), + args: vec![(borrowed_self(), "other")], + ret_ty: Literal(path_local!(bool)), + attributes: attrs, + is_unsafe: false, + unify_fieldless_variants: true, + combine_substructure: combine_substructure(Box::new(|a, b, c| $f(a, b, c))), + } + }}; + } + + super::inject_impl_of_structural_trait( + cx, + span, + item, + path_std!(cx, marker::StructuralPartialEq), + push, + ); + + // avoid defining `ne` if we can + // c-like enums, enums without any fields and structs without fields + // can safely define only `eq`. + let mut methods = vec![md!("eq", cs_eq)]; + if !is_type_without_fields(item) { + methods.push(md!("ne", cs_ne)); + } + + let trait_def = TraitDef { + span, + attributes: Vec::new(), + path: path_std!(cx, cmp::PartialEq), + additional_bounds: Vec::new(), + generics: LifetimeBounds::empty(), + is_unsafe: false, + supports_unions: false, + methods, + associated_types: Vec::new(), + }; + trait_def.expand(cx, mitem, item, push) +} diff --git a/src/librustc_builtin_macros/deriving/cmp/partial_ord.rs b/src/librustc_builtin_macros/deriving/cmp/partial_ord.rs new file mode 100644 index 00000000000..760ed325f36 --- /dev/null +++ b/src/librustc_builtin_macros/deriving/cmp/partial_ord.rs @@ -0,0 +1,302 @@ +pub use OrderingOp::*; + +use crate::deriving::generic::ty::*; +use crate::deriving::generic::*; +use crate::deriving::{path_local, path_std, pathvec_std}; + +use syntax::ast::{self, BinOpKind, Expr, MetaItem}; +use syntax::ptr::P; +use syntax::symbol::{sym, Symbol}; +use syntax_expand::base::{Annotatable, ExtCtxt}; +use syntax_pos::Span; + +pub fn expand_deriving_partial_ord( + cx: &mut ExtCtxt<'_>, + span: Span, + mitem: &MetaItem, + item: &Annotatable, + push: &mut dyn FnMut(Annotatable), +) { + macro_rules! md { + ($name:expr, $op:expr, $equal:expr) => {{ + let inline = cx.meta_word(span, sym::inline); + let attrs = vec![cx.attribute(inline)]; + MethodDef { + name: $name, + generics: LifetimeBounds::empty(), + explicit_self: borrowed_explicit_self(), + args: vec![(borrowed_self(), "other")], + ret_ty: Literal(path_local!(bool)), + attributes: attrs, + is_unsafe: false, + unify_fieldless_variants: true, + combine_substructure: combine_substructure(Box::new(|cx, span, substr| { + cs_op($op, $equal, cx, span, substr) + })), + } + }}; + } + + let ordering_ty = Literal(path_std!(cx, cmp::Ordering)); + let ret_ty = Literal(Path::new_( + pathvec_std!(cx, option::Option), + None, + vec![Box::new(ordering_ty)], + PathKind::Std, + )); + + let inline = cx.meta_word(span, sym::inline); + let attrs = vec![cx.attribute(inline)]; + + let partial_cmp_def = MethodDef { + name: "partial_cmp", + generics: LifetimeBounds::empty(), + explicit_self: borrowed_explicit_self(), + args: vec![(borrowed_self(), "other")], + ret_ty, + attributes: attrs, + is_unsafe: false, + unify_fieldless_variants: true, + combine_substructure: combine_substructure(Box::new(|cx, span, substr| { + cs_partial_cmp(cx, span, substr) + })), + }; + + // avoid defining extra methods if we can + // c-like enums, enums without any fields and structs without fields + // can safely define only `partial_cmp`. + let methods = if is_type_without_fields(item) { + vec![partial_cmp_def] + } else { + vec![ + partial_cmp_def, + md!("lt", true, false), + md!("le", true, true), + md!("gt", false, false), + md!("ge", false, true), + ] + }; + + let trait_def = TraitDef { + span, + attributes: vec![], + path: path_std!(cx, cmp::PartialOrd), + additional_bounds: vec![], + generics: LifetimeBounds::empty(), + is_unsafe: false, + supports_unions: false, + methods, + associated_types: Vec::new(), + }; + trait_def.expand(cx, mitem, item, push) +} + +#[derive(Copy, Clone)] +pub enum OrderingOp { + PartialCmpOp, + LtOp, + LeOp, + GtOp, + GeOp, +} + +pub fn some_ordering_collapsed( + cx: &mut ExtCtxt<'_>, + span: Span, + op: OrderingOp, + self_arg_tags: &[ast::Ident], +) -> P { + let lft = cx.expr_ident(span, self_arg_tags[0]); + let rgt = cx.expr_addr_of(span, cx.expr_ident(span, self_arg_tags[1])); + let op_str = match op { + PartialCmpOp => "partial_cmp", + LtOp => "lt", + LeOp => "le", + GtOp => "gt", + GeOp => "ge", + }; + cx.expr_method_call(span, lft, cx.ident_of(op_str, span), vec![rgt]) +} + +pub fn cs_partial_cmp(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> P { + let test_id = ast::Ident::new(sym::cmp, span); + let ordering = cx.path_global(span, cx.std_path(&[sym::cmp, sym::Ordering, sym::Equal])); + let ordering_expr = cx.expr_path(ordering.clone()); + let equals_expr = cx.expr_some(span, ordering_expr); + + let partial_cmp_path = cx.std_path(&[sym::cmp, sym::PartialOrd, sym::partial_cmp]); + + // Builds: + // + // match ::std::cmp::PartialOrd::partial_cmp(&self_field1, &other_field1) { + // ::std::option::Option::Some(::std::cmp::Ordering::Equal) => + // match ::std::cmp::PartialOrd::partial_cmp(&self_field2, &other_field2) { + // ::std::option::Option::Some(::std::cmp::Ordering::Equal) => { + // ... + // } + // cmp => cmp + // }, + // cmp => cmp + // } + // + cs_fold( + // foldr nests the if-elses correctly, leaving the first field + // as the outermost one, and the last as the innermost. + false, + |cx, span, old, self_f, other_fs| { + // match new { + // Some(::std::cmp::Ordering::Equal) => old, + // cmp => cmp + // } + + let new = { + let other_f = match other_fs { + [o_f] => o_f, + _ => cx.span_bug(span, "not exactly 2 arguments in `derive(PartialOrd)`"), + }; + + let args = + vec![cx.expr_addr_of(span, self_f), cx.expr_addr_of(span, other_f.clone())]; + + cx.expr_call_global(span, partial_cmp_path.clone(), args) + }; + + let eq_arm = cx.arm(span, cx.pat_some(span, cx.pat_path(span, ordering.clone())), old); + let neq_arm = cx.arm(span, cx.pat_ident(span, test_id), cx.expr_ident(span, test_id)); + + cx.expr_match(span, new, vec![eq_arm, neq_arm]) + }, + equals_expr, + Box::new(|cx, span, (self_args, tag_tuple), _non_self_args| { + if self_args.len() != 2 { + cx.span_bug(span, "not exactly 2 arguments in `derive(PartialOrd)`") + } else { + some_ordering_collapsed(cx, span, PartialCmpOp, tag_tuple) + } + }), + cx, + span, + substr, + ) +} + +/// Strict inequality. +fn cs_op( + less: bool, + inclusive: bool, + cx: &mut ExtCtxt<'_>, + span: Span, + substr: &Substructure<'_>, +) -> P { + let ordering_path = |cx: &mut ExtCtxt<'_>, name: &str| { + cx.expr_path( + cx.path_global(span, cx.std_path(&[sym::cmp, sym::Ordering, Symbol::intern(name)])), + ) + }; + + let par_cmp = |cx: &mut ExtCtxt<'_>, span, self_f: P, other_fs: &[P], default| { + let other_f = match other_fs { + [o_f] => o_f, + _ => cx.span_bug(span, "not exactly 2 arguments in `derive(PartialOrd)`"), + }; + + // `PartialOrd::partial_cmp(self.fi, other.fi)` + let cmp_path = cx.expr_path( + cx.path_global(span, cx.std_path(&[sym::cmp, sym::PartialOrd, sym::partial_cmp])), + ); + let cmp = cx.expr_call( + span, + cmp_path, + vec![cx.expr_addr_of(span, self_f), cx.expr_addr_of(span, other_f.clone())], + ); + + let default = ordering_path(cx, default); + // `Option::unwrap_or(_, Ordering::Equal)` + let unwrap_path = cx.expr_path( + cx.path_global(span, cx.std_path(&[sym::option, sym::Option, sym::unwrap_or])), + ); + cx.expr_call(span, unwrap_path, vec![cmp, default]) + }; + + let fold = cs_fold1( + false, // need foldr + |cx, span, subexpr, self_f, other_fs| { + // build up a series of `partial_cmp`s from the inside + // out (hence foldr) to get lexical ordering, i.e., for op == + // `ast::lt` + // + // ``` + // Ordering::then_with( + // Option::unwrap_or( + // PartialOrd::partial_cmp(self.f1, other.f1), Ordering::Equal) + // ), + // Option::unwrap_or( + // PartialOrd::partial_cmp(self.f2, other.f2), Ordering::Greater) + // ) + // ) + // == Ordering::Less + // ``` + // + // and for op == + // `ast::le` + // + // ``` + // Ordering::then_with( + // Option::unwrap_or( + // PartialOrd::partial_cmp(self.f1, other.f1), Ordering::Equal) + // ), + // Option::unwrap_or( + // PartialOrd::partial_cmp(self.f2, other.f2), Ordering::Greater) + // ) + // ) + // != Ordering::Greater + // ``` + // + // The optimiser should remove the redundancy. We explicitly + // get use the binops to avoid auto-deref dereferencing too many + // layers of pointers, if the type includes pointers. + + // `Option::unwrap_or(PartialOrd::partial_cmp(self.fi, other.fi), Ordering::Equal)` + let par_cmp = par_cmp(cx, span, self_f, other_fs, "Equal"); + + // `Ordering::then_with(Option::unwrap_or(..), ..)` + let then_with_path = cx.expr_path( + cx.path_global(span, cx.std_path(&[sym::cmp, sym::Ordering, sym::then_with])), + ); + cx.expr_call(span, then_with_path, vec![par_cmp, cx.lambda0(span, subexpr)]) + }, + |cx, args| match args { + Some((span, self_f, other_fs)) => { + let opposite = if less { "Greater" } else { "Less" }; + par_cmp(cx, span, self_f, other_fs, opposite) + } + None => cx.expr_bool(span, inclusive), + }, + Box::new(|cx, span, (self_args, tag_tuple), _non_self_args| { + if self_args.len() != 2 { + cx.span_bug(span, "not exactly 2 arguments in `derive(PartialOrd)`") + } else { + let op = match (less, inclusive) { + (false, false) => GtOp, + (false, true) => GeOp, + (true, false) => LtOp, + (true, true) => LeOp, + }; + some_ordering_collapsed(cx, span, op, tag_tuple) + } + }), + cx, + span, + substr, + ); + + match *substr.fields { + EnumMatching(.., ref all_fields) | Struct(.., ref all_fields) if !all_fields.is_empty() => { + let ordering = ordering_path(cx, if less ^ inclusive { "Less" } else { "Greater" }); + let comp_op = if inclusive { BinOpKind::Ne } else { BinOpKind::Eq }; + + cx.expr_binary(span, comp_op, fold, ordering) + } + _ => fold, + } +} diff --git a/src/librustc_builtin_macros/deriving/debug.rs b/src/librustc_builtin_macros/deriving/debug.rs new file mode 100644 index 00000000000..c145b63274e --- /dev/null +++ b/src/librustc_builtin_macros/deriving/debug.rs @@ -0,0 +1,137 @@ +use crate::deriving::generic::ty::*; +use crate::deriving::generic::*; +use crate::deriving::path_std; + +use syntax::ast::{self, Ident}; +use syntax::ast::{Expr, MetaItem}; +use syntax::ptr::P; +use syntax::symbol::sym; +use syntax_expand::base::{Annotatable, ExtCtxt}; +use syntax_pos::{Span, DUMMY_SP}; + +pub fn expand_deriving_debug( + cx: &mut ExtCtxt<'_>, + span: Span, + mitem: &MetaItem, + item: &Annotatable, + push: &mut dyn FnMut(Annotatable), +) { + // &mut ::std::fmt::Formatter + let fmtr = + Ptr(Box::new(Literal(path_std!(cx, fmt::Formatter))), Borrowed(None, ast::Mutability::Mut)); + + let trait_def = TraitDef { + span, + attributes: Vec::new(), + path: path_std!(cx, fmt::Debug), + additional_bounds: Vec::new(), + generics: LifetimeBounds::empty(), + is_unsafe: false, + supports_unions: false, + methods: vec![MethodDef { + name: "fmt", + generics: LifetimeBounds::empty(), + explicit_self: borrowed_explicit_self(), + args: vec![(fmtr, "f")], + ret_ty: Literal(path_std!(cx, fmt::Result)), + attributes: Vec::new(), + is_unsafe: false, + unify_fieldless_variants: false, + combine_substructure: combine_substructure(Box::new(|a, b, c| { + show_substructure(a, b, c) + })), + }], + associated_types: Vec::new(), + }; + trait_def.expand(cx, mitem, item, push) +} + +/// We use the debug builders to do the heavy lifting here +fn show_substructure(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> P { + // build fmt.debug_struct().field(, &)....build() + // or fmt.debug_tuple().field(&)....build() + // based on the "shape". + let (ident, vdata, fields) = match substr.fields { + Struct(vdata, fields) => (substr.type_ident, *vdata, fields), + EnumMatching(_, _, v, fields) => (v.ident, &v.data, fields), + EnumNonMatchingCollapsed(..) | StaticStruct(..) | StaticEnum(..) => { + cx.span_bug(span, "nonsensical .fields in `#[derive(Debug)]`") + } + }; + + // We want to make sure we have the ctxt set so that we can use unstable methods + let span = cx.with_def_site_ctxt(span); + let name = cx.expr_lit(span, ast::LitKind::Str(ident.name, ast::StrStyle::Cooked)); + let builder = cx.ident_of("debug_trait_builder", span); + let builder_expr = cx.expr_ident(span, builder.clone()); + + let fmt = substr.nonself_args[0].clone(); + + let mut stmts = vec![]; + match vdata { + ast::VariantData::Tuple(..) | ast::VariantData::Unit(..) => { + // tuple struct/"normal" variant + let expr = cx.expr_method_call(span, fmt, cx.ident_of("debug_tuple", span), vec![name]); + stmts.push(cx.stmt_let(span, true, builder, expr)); + + for field in fields { + // Use double indirection to make sure this works for unsized types + let field = cx.expr_addr_of(field.span, field.self_.clone()); + let field = cx.expr_addr_of(field.span, field); + + let expr = cx.expr_method_call( + span, + builder_expr.clone(), + Ident::new(sym::field, span), + vec![field], + ); + + // Use `let _ = expr;` to avoid triggering the + // unused_results lint. + stmts.push(stmt_let_undescore(cx, span, expr)); + } + } + ast::VariantData::Struct(..) => { + // normal struct/struct variant + let expr = + cx.expr_method_call(span, fmt, cx.ident_of("debug_struct", span), vec![name]); + stmts.push(cx.stmt_let(DUMMY_SP, true, builder, expr)); + + for field in fields { + let name = cx.expr_lit( + field.span, + ast::LitKind::Str(field.name.unwrap().name, ast::StrStyle::Cooked), + ); + + // Use double indirection to make sure this works for unsized types + let field = cx.expr_addr_of(field.span, field.self_.clone()); + let field = cx.expr_addr_of(field.span, field); + let expr = cx.expr_method_call( + span, + builder_expr.clone(), + Ident::new(sym::field, span), + vec![name, field], + ); + stmts.push(stmt_let_undescore(cx, span, expr)); + } + } + } + + let expr = cx.expr_method_call(span, builder_expr, cx.ident_of("finish", span), vec![]); + + stmts.push(cx.stmt_expr(expr)); + let block = cx.block(span, stmts); + cx.expr_block(block) +} + +fn stmt_let_undescore(cx: &mut ExtCtxt<'_>, sp: Span, expr: P) -> ast::Stmt { + let local = P(ast::Local { + pat: cx.pat_wild(sp), + ty: None, + init: Some(expr), + id: ast::DUMMY_NODE_ID, + span: sp, + attrs: ast::AttrVec::new(), + }); + ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp } +} diff --git a/src/librustc_builtin_macros/deriving/decodable.rs b/src/librustc_builtin_macros/deriving/decodable.rs new file mode 100644 index 00000000000..7f21440d49a --- /dev/null +++ b/src/librustc_builtin_macros/deriving/decodable.rs @@ -0,0 +1,225 @@ +//! The compiler code necessary for `#[derive(RustcDecodable)]`. See encodable.rs for more. + +use crate::deriving::generic::ty::*; +use crate::deriving::generic::*; +use crate::deriving::pathvec_std; + +use syntax::ast; +use syntax::ast::{Expr, MetaItem, Mutability}; +use syntax::ptr::P; +use syntax::symbol::Symbol; +use syntax_expand::base::{Annotatable, ExtCtxt}; +use syntax_pos::Span; + +pub fn expand_deriving_rustc_decodable( + cx: &mut ExtCtxt<'_>, + span: Span, + mitem: &MetaItem, + item: &Annotatable, + push: &mut dyn FnMut(Annotatable), +) { + let krate = "rustc_serialize"; + let typaram = "__D"; + + let trait_def = TraitDef { + span, + attributes: Vec::new(), + path: Path::new_(vec![krate, "Decodable"], None, vec![], PathKind::Global), + additional_bounds: Vec::new(), + generics: LifetimeBounds::empty(), + is_unsafe: false, + supports_unions: false, + methods: vec![MethodDef { + name: "decode", + generics: LifetimeBounds { + lifetimes: Vec::new(), + bounds: vec![( + typaram, + vec![Path::new_(vec![krate, "Decoder"], None, vec![], PathKind::Global)], + )], + }, + explicit_self: None, + args: vec![( + Ptr(Box::new(Literal(Path::new_local(typaram))), Borrowed(None, Mutability::Mut)), + "d", + )], + ret_ty: Literal(Path::new_( + pathvec_std!(cx, result::Result), + None, + vec![ + Box::new(Self_), + Box::new(Literal(Path::new_( + vec![typaram, "Error"], + None, + vec![], + PathKind::Local, + ))), + ], + PathKind::Std, + )), + attributes: Vec::new(), + is_unsafe: false, + unify_fieldless_variants: false, + combine_substructure: combine_substructure(Box::new(|a, b, c| { + decodable_substructure(a, b, c, krate) + })), + }], + associated_types: Vec::new(), + }; + + trait_def.expand(cx, mitem, item, push) +} + +fn decodable_substructure( + cx: &mut ExtCtxt<'_>, + trait_span: Span, + substr: &Substructure<'_>, + krate: &str, +) -> P { + let decoder = substr.nonself_args[0].clone(); + let recurse = vec![ + cx.ident_of(krate, trait_span), + cx.ident_of("Decodable", trait_span), + cx.ident_of("decode", trait_span), + ]; + let exprdecode = cx.expr_path(cx.path_global(trait_span, recurse)); + // throw an underscore in front to suppress unused variable warnings + let blkarg = cx.ident_of("_d", trait_span); + let blkdecoder = cx.expr_ident(trait_span, blkarg); + + return match *substr.fields { + StaticStruct(_, ref summary) => { + let nfields = match *summary { + Unnamed(ref fields, _) => fields.len(), + Named(ref fields) => fields.len(), + }; + let read_struct_field = cx.ident_of("read_struct_field", trait_span); + + let path = cx.path_ident(trait_span, substr.type_ident); + let result = + decode_static_fields(cx, trait_span, path, summary, |cx, span, name, field| { + cx.expr_try( + span, + cx.expr_method_call( + span, + blkdecoder.clone(), + read_struct_field, + vec![ + cx.expr_str(span, name), + cx.expr_usize(span, field), + exprdecode.clone(), + ], + ), + ) + }); + let result = cx.expr_ok(trait_span, result); + cx.expr_method_call( + trait_span, + decoder, + cx.ident_of("read_struct", trait_span), + vec![ + cx.expr_str(trait_span, substr.type_ident.name), + cx.expr_usize(trait_span, nfields), + cx.lambda1(trait_span, result, blkarg), + ], + ) + } + StaticEnum(_, ref fields) => { + let variant = cx.ident_of("i", trait_span); + + let mut arms = Vec::with_capacity(fields.len() + 1); + let mut variants = Vec::with_capacity(fields.len()); + let rvariant_arg = cx.ident_of("read_enum_variant_arg", trait_span); + + for (i, &(ident, v_span, ref parts)) in fields.iter().enumerate() { + variants.push(cx.expr_str(v_span, ident.name)); + + let path = cx.path(trait_span, vec![substr.type_ident, ident]); + let decoded = + decode_static_fields(cx, v_span, path, parts, |cx, span, _, field| { + let idx = cx.expr_usize(span, field); + cx.expr_try( + span, + cx.expr_method_call( + span, + blkdecoder.clone(), + rvariant_arg, + vec![idx, exprdecode.clone()], + ), + ) + }); + + arms.push(cx.arm(v_span, cx.pat_lit(v_span, cx.expr_usize(v_span, i)), decoded)); + } + + arms.push(cx.arm_unreachable(trait_span)); + + let result = cx.expr_ok( + trait_span, + cx.expr_match(trait_span, cx.expr_ident(trait_span, variant), arms), + ); + let lambda = cx.lambda(trait_span, vec![blkarg, variant], result); + let variant_vec = cx.expr_vec(trait_span, variants); + let variant_vec = cx.expr_addr_of(trait_span, variant_vec); + let result = cx.expr_method_call( + trait_span, + blkdecoder, + cx.ident_of("read_enum_variant", trait_span), + vec![variant_vec, lambda], + ); + cx.expr_method_call( + trait_span, + decoder, + cx.ident_of("read_enum", trait_span), + vec![ + cx.expr_str(trait_span, substr.type_ident.name), + cx.lambda1(trait_span, result, blkarg), + ], + ) + } + _ => cx.bug("expected StaticEnum or StaticStruct in derive(Decodable)"), + }; +} + +/// Creates a decoder for a single enum variant/struct: +/// - `outer_pat_path` is the path to this enum variant/struct +/// - `getarg` should retrieve the `usize`-th field with name `@str`. +fn decode_static_fields( + cx: &mut ExtCtxt<'_>, + trait_span: Span, + outer_pat_path: ast::Path, + fields: &StaticFields, + mut getarg: F, +) -> P +where + F: FnMut(&mut ExtCtxt<'_>, Span, Symbol, usize) -> P, +{ + match *fields { + Unnamed(ref fields, is_tuple) => { + let path_expr = cx.expr_path(outer_pat_path); + if !is_tuple { + path_expr + } else { + let fields = fields + .iter() + .enumerate() + .map(|(i, &span)| getarg(cx, span, Symbol::intern(&format!("_field{}", i)), i)) + .collect(); + + cx.expr_call(trait_span, path_expr, fields) + } + } + Named(ref fields) => { + // use the field's span to get nicer error messages. + let fields = fields + .iter() + .enumerate() + .map(|(i, &(ident, span))| { + let arg = getarg(cx, span, ident.name, i); + cx.field_imm(span, ident, arg) + }) + .collect(); + cx.expr_struct(trait_span, outer_pat_path, fields) + } + } +} diff --git a/src/librustc_builtin_macros/deriving/default.rs b/src/librustc_builtin_macros/deriving/default.rs new file mode 100644 index 00000000000..d623e1fa4cc --- /dev/null +++ b/src/librustc_builtin_macros/deriving/default.rs @@ -0,0 +1,83 @@ +use crate::deriving::generic::ty::*; +use crate::deriving::generic::*; +use crate::deriving::path_std; + +use syntax::ast::{Expr, MetaItem}; +use syntax::ptr::P; +use syntax::span_err; +use syntax::symbol::{kw, sym}; +use syntax_expand::base::{Annotatable, DummyResult, ExtCtxt}; +use syntax_pos::Span; + +use rustc_error_codes::*; + +pub fn expand_deriving_default( + cx: &mut ExtCtxt<'_>, + span: Span, + mitem: &MetaItem, + item: &Annotatable, + push: &mut dyn FnMut(Annotatable), +) { + let inline = cx.meta_word(span, sym::inline); + let attrs = vec![cx.attribute(inline)]; + let trait_def = TraitDef { + span, + attributes: Vec::new(), + path: path_std!(cx, default::Default), + additional_bounds: Vec::new(), + generics: LifetimeBounds::empty(), + is_unsafe: false, + supports_unions: false, + methods: vec![MethodDef { + name: "default", + generics: LifetimeBounds::empty(), + explicit_self: None, + args: Vec::new(), + ret_ty: Self_, + attributes: attrs, + is_unsafe: false, + unify_fieldless_variants: false, + combine_substructure: combine_substructure(Box::new(|a, b, c| { + default_substructure(a, b, c) + })), + }], + associated_types: Vec::new(), + }; + trait_def.expand(cx, mitem, item, push) +} + +fn default_substructure( + cx: &mut ExtCtxt<'_>, + trait_span: Span, + substr: &Substructure<'_>, +) -> P { + // Note that `kw::Default` is "default" and `sym::Default` is "Default"! + let default_ident = cx.std_path(&[kw::Default, sym::Default, kw::Default]); + let default_call = |span| cx.expr_call_global(span, default_ident.clone(), Vec::new()); + + return match *substr.fields { + StaticStruct(_, ref summary) => match *summary { + Unnamed(ref fields, is_tuple) => { + if !is_tuple { + cx.expr_ident(trait_span, substr.type_ident) + } else { + let exprs = fields.iter().map(|sp| default_call(*sp)).collect(); + cx.expr_call_ident(trait_span, substr.type_ident, exprs) + } + } + Named(ref fields) => { + let default_fields = fields + .iter() + .map(|&(ident, span)| cx.field_imm(span, ident, default_call(span))) + .collect(); + cx.expr_struct_ident(trait_span, substr.type_ident, default_fields) + } + }, + StaticEnum(..) => { + span_err!(cx, trait_span, E0665, "`Default` cannot be derived for enums, only structs"); + // let compilation continue + DummyResult::raw_expr(trait_span, true) + } + _ => cx.span_bug(trait_span, "method in `derive(Default)`"), + }; +} diff --git a/src/librustc_builtin_macros/deriving/encodable.rs b/src/librustc_builtin_macros/deriving/encodable.rs new file mode 100644 index 00000000000..98b0160d6e8 --- /dev/null +++ b/src/librustc_builtin_macros/deriving/encodable.rs @@ -0,0 +1,287 @@ +//! The compiler code necessary to implement the `#[derive(RustcEncodable)]` +//! (and `RustcDecodable`, in `decodable.rs`) extension. The idea here is that +//! type-defining items may be tagged with +//! `#[derive(RustcEncodable, RustcDecodable)]`. +//! +//! For example, a type like: +//! +//! ``` +//! #[derive(RustcEncodable, RustcDecodable)] +//! struct Node { id: usize } +//! ``` +//! +//! would generate two implementations like: +//! +//! ``` +//! # struct Node { id: usize } +//! impl, E> Encodable for Node { +//! fn encode(&self, s: &mut S) -> Result<(), E> { +//! s.emit_struct("Node", 1, |this| { +//! this.emit_struct_field("id", 0, |this| { +//! Encodable::encode(&self.id, this) +//! /* this.emit_usize(self.id) can also be used */ +//! }) +//! }) +//! } +//! } +//! +//! impl, E> Decodable for Node { +//! fn decode(d: &mut D) -> Result { +//! d.read_struct("Node", 1, |this| { +//! match this.read_struct_field("id", 0, |this| Decodable::decode(this)) { +//! Ok(id) => Ok(Node { id: id }), +//! Err(e) => Err(e), +//! } +//! }) +//! } +//! } +//! ``` +//! +//! Other interesting scenarios are when the item has type parameters or +//! references other non-built-in types. A type definition like: +//! +//! ``` +//! # #[derive(RustcEncodable, RustcDecodable)] +//! # struct Span; +//! #[derive(RustcEncodable, RustcDecodable)] +//! struct Spanned { node: T, span: Span } +//! ``` +//! +//! would yield functions like: +//! +//! ``` +//! # #[derive(RustcEncodable, RustcDecodable)] +//! # struct Span; +//! # struct Spanned { node: T, span: Span } +//! impl< +//! S: Encoder, +//! E, +//! T: Encodable +//! > Encodable for Spanned { +//! fn encode(&self, s: &mut S) -> Result<(), E> { +//! s.emit_struct("Spanned", 2, |this| { +//! this.emit_struct_field("node", 0, |this| self.node.encode(this)) +//! .unwrap(); +//! this.emit_struct_field("span", 1, |this| self.span.encode(this)) +//! }) +//! } +//! } +//! +//! impl< +//! D: Decoder, +//! E, +//! T: Decodable +//! > Decodable for Spanned { +//! fn decode(d: &mut D) -> Result, E> { +//! d.read_struct("Spanned", 2, |this| { +//! Ok(Spanned { +//! node: this.read_struct_field("node", 0, |this| Decodable::decode(this)) +//! .unwrap(), +//! span: this.read_struct_field("span", 1, |this| Decodable::decode(this)) +//! .unwrap(), +//! }) +//! }) +//! } +//! } +//! ``` + +use crate::deriving::generic::ty::*; +use crate::deriving::generic::*; +use crate::deriving::pathvec_std; + +use syntax::ast::{Expr, ExprKind, MetaItem, Mutability}; +use syntax::ptr::P; +use syntax::symbol::Symbol; +use syntax_expand::base::{Annotatable, ExtCtxt}; +use syntax_pos::Span; + +pub fn expand_deriving_rustc_encodable( + cx: &mut ExtCtxt<'_>, + span: Span, + mitem: &MetaItem, + item: &Annotatable, + push: &mut dyn FnMut(Annotatable), +) { + let krate = "rustc_serialize"; + let typaram = "__S"; + + let trait_def = TraitDef { + span, + attributes: Vec::new(), + path: Path::new_(vec![krate, "Encodable"], None, vec![], PathKind::Global), + additional_bounds: Vec::new(), + generics: LifetimeBounds::empty(), + is_unsafe: false, + supports_unions: false, + methods: vec![MethodDef { + name: "encode", + generics: LifetimeBounds { + lifetimes: Vec::new(), + bounds: vec![( + typaram, + vec![Path::new_(vec![krate, "Encoder"], None, vec![], PathKind::Global)], + )], + }, + explicit_self: borrowed_explicit_self(), + args: vec![( + Ptr(Box::new(Literal(Path::new_local(typaram))), Borrowed(None, Mutability::Mut)), + "s", + )], + ret_ty: Literal(Path::new_( + pathvec_std!(cx, result::Result), + None, + vec![ + Box::new(Tuple(Vec::new())), + Box::new(Literal(Path::new_( + vec![typaram, "Error"], + None, + vec![], + PathKind::Local, + ))), + ], + PathKind::Std, + )), + attributes: Vec::new(), + is_unsafe: false, + unify_fieldless_variants: false, + combine_substructure: combine_substructure(Box::new(|a, b, c| { + encodable_substructure(a, b, c, krate) + })), + }], + associated_types: Vec::new(), + }; + + trait_def.expand(cx, mitem, item, push) +} + +fn encodable_substructure( + cx: &mut ExtCtxt<'_>, + trait_span: Span, + substr: &Substructure<'_>, + krate: &'static str, +) -> P { + let encoder = substr.nonself_args[0].clone(); + // throw an underscore in front to suppress unused variable warnings + let blkarg = cx.ident_of("_e", trait_span); + let blkencoder = cx.expr_ident(trait_span, blkarg); + let fn_path = cx.expr_path(cx.path_global( + trait_span, + vec![ + cx.ident_of(krate, trait_span), + cx.ident_of("Encodable", trait_span), + cx.ident_of("encode", trait_span), + ], + )); + + return match *substr.fields { + Struct(_, ref fields) => { + let emit_struct_field = cx.ident_of("emit_struct_field", trait_span); + let mut stmts = Vec::new(); + for (i, &FieldInfo { name, ref self_, span, .. }) in fields.iter().enumerate() { + let name = match name { + Some(id) => id.name, + None => Symbol::intern(&format!("_field{}", i)), + }; + let self_ref = cx.expr_addr_of(span, self_.clone()); + let enc = cx.expr_call(span, fn_path.clone(), vec![self_ref, blkencoder.clone()]); + let lambda = cx.lambda1(span, enc, blkarg); + let call = cx.expr_method_call( + span, + blkencoder.clone(), + emit_struct_field, + vec![cx.expr_str(span, name), cx.expr_usize(span, i), lambda], + ); + + // last call doesn't need a try! + let last = fields.len() - 1; + let call = if i != last { + cx.expr_try(span, call) + } else { + cx.expr(span, ExprKind::Ret(Some(call))) + }; + + let stmt = cx.stmt_expr(call); + stmts.push(stmt); + } + + // unit structs have no fields and need to return Ok() + let blk = if stmts.is_empty() { + let ok = cx.expr_ok(trait_span, cx.expr_tuple(trait_span, vec![])); + cx.lambda1(trait_span, ok, blkarg) + } else { + cx.lambda_stmts_1(trait_span, stmts, blkarg) + }; + + cx.expr_method_call( + trait_span, + encoder, + cx.ident_of("emit_struct", trait_span), + vec![ + cx.expr_str(trait_span, substr.type_ident.name), + cx.expr_usize(trait_span, fields.len()), + blk, + ], + ) + } + + EnumMatching(idx, _, variant, ref fields) => { + // We're not generating an AST that the borrow checker is expecting, + // so we need to generate a unique local variable to take the + // mutable loan out on, otherwise we get conflicts which don't + // actually exist. + let me = cx.stmt_let(trait_span, false, blkarg, encoder); + let encoder = cx.expr_ident(trait_span, blkarg); + let emit_variant_arg = cx.ident_of("emit_enum_variant_arg", trait_span); + let mut stmts = Vec::new(); + if !fields.is_empty() { + let last = fields.len() - 1; + for (i, &FieldInfo { ref self_, span, .. }) in fields.iter().enumerate() { + let self_ref = cx.expr_addr_of(span, self_.clone()); + let enc = + cx.expr_call(span, fn_path.clone(), vec![self_ref, blkencoder.clone()]); + let lambda = cx.lambda1(span, enc, blkarg); + let call = cx.expr_method_call( + span, + blkencoder.clone(), + emit_variant_arg, + vec![cx.expr_usize(span, i), lambda], + ); + let call = if i != last { + cx.expr_try(span, call) + } else { + cx.expr(span, ExprKind::Ret(Some(call))) + }; + stmts.push(cx.stmt_expr(call)); + } + } else { + let ok = cx.expr_ok(trait_span, cx.expr_tuple(trait_span, vec![])); + let ret_ok = cx.expr(trait_span, ExprKind::Ret(Some(ok))); + stmts.push(cx.stmt_expr(ret_ok)); + } + + let blk = cx.lambda_stmts_1(trait_span, stmts, blkarg); + let name = cx.expr_str(trait_span, variant.ident.name); + let call = cx.expr_method_call( + trait_span, + blkencoder, + cx.ident_of("emit_enum_variant", trait_span), + vec![ + name, + cx.expr_usize(trait_span, idx), + cx.expr_usize(trait_span, fields.len()), + blk, + ], + ); + let blk = cx.lambda1(trait_span, call, blkarg); + let ret = cx.expr_method_call( + trait_span, + encoder, + cx.ident_of("emit_enum", trait_span), + vec![cx.expr_str(trait_span, substr.type_ident.name), blk], + ); + cx.expr_block(cx.block(trait_span, vec![me, cx.stmt_expr(ret)])) + } + + _ => cx.bug("expected Struct or EnumMatching in derive(Encodable)"), + }; +} diff --git a/src/librustc_builtin_macros/deriving/generic/mod.rs b/src/librustc_builtin_macros/deriving/generic/mod.rs new file mode 100644 index 00000000000..7d7b73ebb42 --- /dev/null +++ b/src/librustc_builtin_macros/deriving/generic/mod.rs @@ -0,0 +1,1812 @@ +//! Some code that abstracts away much of the boilerplate of writing +//! `derive` instances for traits. Among other things it manages getting +//! access to the fields of the 4 different sorts of structs and enum +//! variants, as well as creating the method and impl ast instances. +//! +//! Supported features (fairly exhaustive): +//! +//! - Methods taking any number of parameters of any type, and returning +//! any type, other than vectors, bottom and closures. +//! - Generating `impl`s for types with type parameters and lifetimes +//! (e.g., `Option`), the parameters are automatically given the +//! current trait as a bound. (This includes separate type parameters +//! and lifetimes for methods.) +//! - Additional bounds on the type parameters (`TraitDef.additional_bounds`) +//! +//! The most important thing for implementors is the `Substructure` and +//! `SubstructureFields` objects. The latter groups 5 possibilities of the +//! arguments: +//! +//! - `Struct`, when `Self` is a struct (including tuple structs, e.g +//! `struct T(i32, char)`). +//! - `EnumMatching`, when `Self` is an enum and all the arguments are the +//! same variant of the enum (e.g., `Some(1)`, `Some(3)` and `Some(4)`) +//! - `EnumNonMatchingCollapsed` when `Self` is an enum and the arguments +//! are not the same variant (e.g., `None`, `Some(1)` and `None`). +//! - `StaticEnum` and `StaticStruct` for static methods, where the type +//! being derived upon is either an enum or struct respectively. (Any +//! argument with type Self is just grouped among the non-self +//! arguments.) +//! +//! In the first two cases, the values from the corresponding fields in +//! all the arguments are grouped together. For `EnumNonMatchingCollapsed` +//! this isn't possible (different variants have different fields), so the +//! fields are inaccessible. (Previous versions of the deriving infrastructure +//! had a way to expand into code that could access them, at the cost of +//! generating exponential amounts of code; see issue #15375). There are no +//! fields with values in the static cases, so these are treated entirely +//! differently. +//! +//! The non-static cases have `Option` in several places associated +//! with field `expr`s. This represents the name of the field it is +//! associated with. It is only not `None` when the associated field has +//! an identifier in the source code. For example, the `x`s in the +//! following snippet +//! +//! ```rust +//! # #![allow(dead_code)] +//! struct A { x : i32 } +//! +//! struct B(i32); +//! +//! enum C { +//! C0(i32), +//! C1 { x: i32 } +//! } +//! ``` +//! +//! The `i32`s in `B` and `C0` don't have an identifier, so the +//! `Option`s would be `None` for them. +//! +//! In the static cases, the structure is summarized, either into the just +//! spans of the fields or a list of spans and the field idents (for tuple +//! structs and record structs, respectively), or a list of these, for +//! enums (one for each variant). For empty struct and empty enum +//! variants, it is represented as a count of 0. +//! +//! # "`cs`" functions +//! +//! The `cs_...` functions ("combine substructure) are designed to +//! make life easier by providing some pre-made recipes for common +//! threads; mostly calling the function being derived on all the +//! arguments and then combining them back together in some way (or +//! letting the user chose that). They are not meant to be the only +//! way to handle the structures that this code creates. +//! +//! # Examples +//! +//! The following simplified `PartialEq` is used for in-code examples: +//! +//! ```rust +//! trait PartialEq { +//! fn eq(&self, other: &Self) -> bool; +//! } +//! impl PartialEq for i32 { +//! fn eq(&self, other: &i32) -> bool { +//! *self == *other +//! } +//! } +//! ``` +//! +//! Some examples of the values of `SubstructureFields` follow, using the +//! above `PartialEq`, `A`, `B` and `C`. +//! +//! ## Structs +//! +//! When generating the `expr` for the `A` impl, the `SubstructureFields` is +//! +//! ```{.text} +//! Struct(vec![FieldInfo { +//! span: +//! name: Some(), +//! self_: , +//! other: vec![, +//! name: None, +//! self_: +//! other: vec![] +//! }]) +//! ``` +//! +//! ## Enums +//! +//! When generating the `expr` for a call with `self == C0(a)` and `other +//! == C0(b)`, the SubstructureFields is +//! +//! ```{.text} +//! EnumMatching(0, , +//! vec![FieldInfo { +//! span: +//! name: None, +//! self_: , +//! other: vec![] +//! }]) +//! ``` +//! +//! For `C1 {x}` and `C1 {x}`, +//! +//! ```{.text} +//! EnumMatching(1, , +//! vec![FieldInfo { +//! span: +//! name: Some(), +//! self_: , +//! other: vec![] +//! }]) +//! ``` +//! +//! For `C0(a)` and `C1 {x}` , +//! +//! ```{.text} +//! EnumNonMatchingCollapsed( +//! vec![, ], +//! &[, ], +//! &[, ]) +//! ``` +//! +//! It is the same for when the arguments are flipped to `C1 {x}` and +//! `C0(a)`; the only difference is what the values of the identifiers +//! and will +//! be in the generated code. +//! +//! `EnumNonMatchingCollapsed` deliberately provides far less information +//! than is generally available for a given pair of variants; see #15375 +//! for discussion. +//! +//! ## Static +//! +//! A static method on the types above would result in, +//! +//! ```{.text} +//! StaticStruct(, Named(vec![(, )])) +//! +//! StaticStruct(, Unnamed(vec![])) +//! +//! StaticEnum(, +//! vec![(, , Unnamed(vec![])), +//! (, , Named(vec![(, )]))]) +//! ``` + +pub use StaticFields::*; +pub use SubstructureFields::*; + +use std::cell::RefCell; +use std::iter; +use std::vec; + +use syntax::ast::{self, BinOpKind, EnumDef, Expr, Generics, Ident, PatKind}; +use syntax::ast::{GenericArg, GenericParamKind, VariantData}; +use syntax::attr; +use syntax::ptr::P; +use syntax::sess::ParseSess; +use syntax::source_map::respan; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::util::map_in_place::MapInPlace; +use syntax_expand::base::{Annotatable, ExtCtxt}; +use syntax_pos::Span; + +use ty::{LifetimeBounds, Path, Ptr, PtrTy, Self_, Ty}; + +use crate::deriving; + +pub mod ty; + +pub struct TraitDef<'a> { + /// The span for the current #[derive(Foo)] header. + pub span: Span, + + pub attributes: Vec, + + /// Path of the trait, including any type parameters + pub path: Path<'a>, + + /// Additional bounds required of any type parameters of the type, + /// other than the current trait + pub additional_bounds: Vec>, + + /// Any extra lifetimes and/or bounds, e.g., `D: serialize::Decoder` + pub generics: LifetimeBounds<'a>, + + /// Is it an `unsafe` trait? + pub is_unsafe: bool, + + /// Can this trait be derived for unions? + pub supports_unions: bool, + + pub methods: Vec>, + + pub associated_types: Vec<(ast::Ident, Ty<'a>)>, +} + +pub struct MethodDef<'a> { + /// name of the method + pub name: &'a str, + /// List of generics, e.g., `R: rand::Rng` + pub generics: LifetimeBounds<'a>, + + /// Whether there is a self argument (outer Option) i.e., whether + /// this is a static function, and whether it is a pointer (inner + /// Option) + pub explicit_self: Option>, + + /// Arguments other than the self argument + pub args: Vec<(Ty<'a>, &'a str)>, + + /// Returns type + pub ret_ty: Ty<'a>, + + pub attributes: Vec, + + // Is it an `unsafe fn`? + pub is_unsafe: bool, + + /// Can we combine fieldless variants for enums into a single match arm? + pub unify_fieldless_variants: bool, + + pub combine_substructure: RefCell>, +} + +/// All the data about the data structure/method being derived upon. +pub struct Substructure<'a> { + /// ident of self + pub type_ident: Ident, + /// ident of the method + pub method_ident: Ident, + /// dereferenced access to any `Self_` or `Ptr(Self_, _)` arguments + pub self_args: &'a [P], + /// verbatim access to any other arguments + pub nonself_args: &'a [P], + pub fields: &'a SubstructureFields<'a>, +} + +/// Summary of the relevant parts of a struct/enum field. +pub struct FieldInfo<'a> { + pub span: Span, + /// None for tuple structs/normal enum variants, Some for normal + /// structs/struct enum variants. + pub name: Option, + /// The expression corresponding to this field of `self` + /// (specifically, a reference to it). + pub self_: P, + /// The expressions corresponding to references to this field in + /// the other `Self` arguments. + pub other: Vec>, + /// The attributes on the field + pub attrs: &'a [ast::Attribute], +} + +/// Fields for a static method +pub enum StaticFields { + /// Tuple and unit structs/enum variants like this. + Unnamed(Vec, bool /*is tuple*/), + /// Normal structs/struct variants. + Named(Vec<(Ident, Span)>), +} + +/// A summary of the possible sets of fields. +pub enum SubstructureFields<'a> { + Struct(&'a ast::VariantData, Vec>), + /// Matching variants of the enum: variant index, variant count, ast::Variant, + /// fields: the field name is only non-`None` in the case of a struct + /// variant. + EnumMatching(usize, usize, &'a ast::Variant, Vec>), + + /// Non-matching variants of the enum, but with all state hidden from + /// the consequent code. The first component holds `Ident`s for all of + /// the `Self` arguments; the second component is a slice of all of the + /// variants for the enum itself, and the third component is a list of + /// `Ident`s bound to the variant index values for each of the actual + /// input `Self` arguments. + EnumNonMatchingCollapsed(Vec, &'a [ast::Variant], &'a [Ident]), + + /// A static method where `Self` is a struct. + StaticStruct(&'a ast::VariantData, StaticFields), + /// A static method where `Self` is an enum. + StaticEnum(&'a ast::EnumDef, Vec<(Ident, Span, StaticFields)>), +} + +/// Combine the values of all the fields together. The last argument is +/// all the fields of all the structures. +pub type CombineSubstructureFunc<'a> = + Box, Span, &Substructure<'_>) -> P + 'a>; + +/// Deal with non-matching enum variants. The tuple is a list of +/// identifiers (one for each `Self` argument, which could be any of the +/// variants since they have been collapsed together) and the identifiers +/// holding the variant index value for each of the `Self` arguments. The +/// last argument is all the non-`Self` args of the method being derived. +pub type EnumNonMatchCollapsedFunc<'a> = + Box, Span, (&[Ident], &[Ident]), &[P]) -> P + 'a>; + +pub fn combine_substructure( + f: CombineSubstructureFunc<'_>, +) -> RefCell> { + RefCell::new(f) +} + +/// This method helps to extract all the type parameters referenced from a +/// type. For a type parameter ``, it looks for either a `TyPath` that +/// is not global and starts with `T`, or a `TyQPath`. +fn find_type_parameters( + ty: &ast::Ty, + ty_param_names: &[ast::Name], + cx: &ExtCtxt<'_>, +) -> Vec> { + use syntax::visit; + + struct Visitor<'a, 'b> { + cx: &'a ExtCtxt<'b>, + ty_param_names: &'a [ast::Name], + types: Vec>, + } + + impl<'a, 'b> visit::Visitor<'a> for Visitor<'a, 'b> { + fn visit_ty(&mut self, ty: &'a ast::Ty) { + if let ast::TyKind::Path(_, ref path) = ty.kind { + if let Some(segment) = path.segments.first() { + if self.ty_param_names.contains(&segment.ident.name) { + self.types.push(P(ty.clone())); + } + } + } + + visit::walk_ty(self, ty) + } + + fn visit_mac(&mut self, mac: &ast::Mac) { + self.cx.span_err(mac.span(), "`derive` cannot be used on items with type macros"); + } + } + + let mut visitor = Visitor { cx, ty_param_names, types: Vec::new() }; + visit::Visitor::visit_ty(&mut visitor, ty); + + visitor.types +} + +impl<'a> TraitDef<'a> { + pub fn expand( + self, + cx: &mut ExtCtxt<'_>, + mitem: &ast::MetaItem, + item: &'a Annotatable, + push: &mut dyn FnMut(Annotatable), + ) { + self.expand_ext(cx, mitem, item, push, false); + } + + pub fn expand_ext( + self, + cx: &mut ExtCtxt<'_>, + mitem: &ast::MetaItem, + item: &'a Annotatable, + push: &mut dyn FnMut(Annotatable), + from_scratch: bool, + ) { + match *item { + Annotatable::Item(ref item) => { + let is_packed = item.attrs.iter().any(|attr| { + for r in attr::find_repr_attrs(&cx.parse_sess, attr) { + if let attr::ReprPacked(_) = r { + return true; + } + } + false + }); + let has_no_type_params = match item.kind { + ast::ItemKind::Struct(_, ref generics) + | ast::ItemKind::Enum(_, ref generics) + | ast::ItemKind::Union(_, ref generics) => { + !generics.params.iter().any(|param| match param.kind { + ast::GenericParamKind::Type { .. } => true, + _ => false, + }) + } + _ => { + // Non-ADT derive is an error, but it should have been + // set earlier; see + // libsyntax_expand/expand.rs:MacroExpander::fully_expand_fragment() + // libsyntax_expand/base.rs:Annotatable::derive_allowed() + return; + } + }; + let container_id = cx.current_expansion.id.expn_data().parent; + let always_copy = has_no_type_params && cx.resolver.has_derive_copy(container_id); + let use_temporaries = is_packed && always_copy; + + let newitem = match item.kind { + ast::ItemKind::Struct(ref struct_def, ref generics) => self.expand_struct_def( + cx, + &struct_def, + item.ident, + generics, + from_scratch, + use_temporaries, + ), + ast::ItemKind::Enum(ref enum_def, ref generics) => { + // We ignore `use_temporaries` here, because + // `repr(packed)` enums cause an error later on. + // + // This can only cause further compilation errors + // downstream in blatantly illegal code, so it + // is fine. + self.expand_enum_def( + cx, + enum_def, + &item.attrs, + item.ident, + generics, + from_scratch, + ) + } + ast::ItemKind::Union(ref struct_def, ref generics) => { + if self.supports_unions { + self.expand_struct_def( + cx, + &struct_def, + item.ident, + generics, + from_scratch, + use_temporaries, + ) + } else { + cx.span_err(mitem.span, "this trait cannot be derived for unions"); + return; + } + } + _ => unreachable!(), + }; + // Keep the lint attributes of the previous item to control how the + // generated implementations are linted + let mut attrs = newitem.attrs.clone(); + attrs.extend( + item.attrs + .iter() + .filter(|a| { + [ + sym::allow, + sym::warn, + sym::deny, + sym::forbid, + sym::stable, + sym::unstable, + ] + .contains(&a.name_or_empty()) + }) + .cloned(), + ); + push(Annotatable::Item(P(ast::Item { attrs: attrs, ..(*newitem).clone() }))) + } + _ => { + // Non-Item derive is an error, but it should have been + // set earlier; see + // libsyntax_expand/expand.rs:MacroExpander::fully_expand_fragment() + // libsyntax_expand/base.rs:Annotatable::derive_allowed() + return; + } + } + } + + /// Given that we are deriving a trait `DerivedTrait` for a type like: + /// + /// ```ignore (only-for-syntax-highlight) + /// struct Struct<'a, ..., 'z, A, B: DeclaredTrait, C, ..., Z> where C: WhereTrait { + /// a: A, + /// b: B::Item, + /// b1: ::Item, + /// c1: ::Item, + /// c2: Option<::Item>, + /// ... + /// } + /// ``` + /// + /// create an impl like: + /// + /// ```ignore (only-for-syntax-highlight) + /// impl<'a, ..., 'z, A, B: DeclaredTrait, C, ... Z> where + /// C: WhereTrait, + /// A: DerivedTrait + B1 + ... + BN, + /// B: DerivedTrait + B1 + ... + BN, + /// C: DerivedTrait + B1 + ... + BN, + /// B::Item: DerivedTrait + B1 + ... + BN, + /// ::Item: DerivedTrait + B1 + ... + BN, + /// ... + /// { + /// ... + /// } + /// ``` + /// + /// where B1, ..., BN are the bounds given by `bounds_paths`.'. Z is a phantom type, and + /// therefore does not get bound by the derived trait. + fn create_derived_impl( + &self, + cx: &mut ExtCtxt<'_>, + type_ident: Ident, + generics: &Generics, + field_tys: Vec>, + methods: Vec, + ) -> P { + let trait_path = self.path.to_path(cx, self.span, type_ident, generics); + + // Transform associated types from `deriving::ty::Ty` into `ast::AssocItem` + let associated_types = + self.associated_types.iter().map(|&(ident, ref type_def)| ast::AssocItem { + id: ast::DUMMY_NODE_ID, + span: self.span, + ident, + vis: respan(self.span.shrink_to_lo(), ast::VisibilityKind::Inherited), + defaultness: ast::Defaultness::Final, + attrs: Vec::new(), + generics: Generics::default(), + kind: ast::AssocItemKind::TyAlias( + Vec::new(), + Some(type_def.to_ty(cx, self.span, type_ident, generics)), + ), + tokens: None, + }); + + let Generics { mut params, mut where_clause, span } = + self.generics.to_generics(cx, self.span, type_ident, generics); + + // Create the generic parameters + params.extend(generics.params.iter().map(|param| match param.kind { + GenericParamKind::Lifetime { .. } => param.clone(), + GenericParamKind::Type { .. } => { + // I don't think this can be moved out of the loop, since + // a GenericBound requires an ast id + let bounds: Vec<_> = + // extra restrictions on the generics parameters to the + // type being derived upon + self.additional_bounds.iter().map(|p| { + cx.trait_bound(p.to_path(cx, self.span, type_ident, generics)) + }).chain( + // require the current trait + iter::once(cx.trait_bound(trait_path.clone())) + ).chain( + // also add in any bounds from the declaration + param.bounds.iter().cloned() + ).collect(); + + cx.typaram(self.span, param.ident, vec![], bounds, None) + } + GenericParamKind::Const { .. } => param.clone(), + })); + + // and similarly for where clauses + where_clause.predicates.extend(generics.where_clause.predicates.iter().map(|clause| { + match *clause { + ast::WherePredicate::BoundPredicate(ref wb) => { + ast::WherePredicate::BoundPredicate(ast::WhereBoundPredicate { + span: self.span, + bound_generic_params: wb.bound_generic_params.clone(), + bounded_ty: wb.bounded_ty.clone(), + bounds: wb.bounds.iter().cloned().collect(), + }) + } + ast::WherePredicate::RegionPredicate(ref rb) => { + ast::WherePredicate::RegionPredicate(ast::WhereRegionPredicate { + span: self.span, + lifetime: rb.lifetime, + bounds: rb.bounds.iter().cloned().collect(), + }) + } + ast::WherePredicate::EqPredicate(ref we) => { + ast::WherePredicate::EqPredicate(ast::WhereEqPredicate { + id: ast::DUMMY_NODE_ID, + span: self.span, + lhs_ty: we.lhs_ty.clone(), + rhs_ty: we.rhs_ty.clone(), + }) + } + } + })); + + { + // Extra scope required here so ty_params goes out of scope before params is moved + + let mut ty_params = params + .iter() + .filter_map(|param| match param.kind { + ast::GenericParamKind::Type { .. } => Some(param), + _ => None, + }) + .peekable(); + + if ty_params.peek().is_some() { + let ty_param_names: Vec = + ty_params.map(|ty_param| ty_param.ident.name).collect(); + + for field_ty in field_tys { + let tys = find_type_parameters(&field_ty, &ty_param_names, cx); + + for ty in tys { + // if we have already handled this type, skip it + if let ast::TyKind::Path(_, ref p) = ty.kind { + if p.segments.len() == 1 + && ty_param_names.contains(&p.segments[0].ident.name) + { + continue; + }; + } + let mut bounds: Vec<_> = self + .additional_bounds + .iter() + .map(|p| cx.trait_bound(p.to_path(cx, self.span, type_ident, generics))) + .collect(); + + // require the current trait + bounds.push(cx.trait_bound(trait_path.clone())); + + let predicate = ast::WhereBoundPredicate { + span: self.span, + bound_generic_params: Vec::new(), + bounded_ty: ty, + bounds, + }; + + let predicate = ast::WherePredicate::BoundPredicate(predicate); + where_clause.predicates.push(predicate); + } + } + } + } + + let trait_generics = Generics { params, where_clause, span }; + + // Create the reference to the trait. + let trait_ref = cx.trait_ref(trait_path); + + let self_params: Vec<_> = generics + .params + .iter() + .map(|param| match param.kind { + GenericParamKind::Lifetime { .. } => { + GenericArg::Lifetime(cx.lifetime(self.span, param.ident)) + } + GenericParamKind::Type { .. } => { + GenericArg::Type(cx.ty_ident(self.span, param.ident)) + } + GenericParamKind::Const { .. } => { + GenericArg::Const(cx.const_ident(self.span, param.ident)) + } + }) + .collect(); + + // Create the type of `self`. + let path = cx.path_all(self.span, false, vec![type_ident], self_params); + let self_type = cx.ty_path(path); + + let attr = cx.attribute(cx.meta_word(self.span, sym::automatically_derived)); + // Just mark it now since we know that it'll end up used downstream + attr::mark_used(&attr); + let opt_trait_ref = Some(trait_ref); + let unused_qual = { + let word = syntax::attr::mk_nested_word_item(Ident::new( + Symbol::intern("unused_qualifications"), + self.span, + )); + let list = syntax::attr::mk_list_item(Ident::new(sym::allow, self.span), vec![word]); + cx.attribute(list) + }; + + let mut a = vec![attr, unused_qual]; + a.extend(self.attributes.iter().cloned()); + + let unsafety = if self.is_unsafe { ast::Unsafety::Unsafe } else { ast::Unsafety::Normal }; + + cx.item( + self.span, + Ident::invalid(), + a, + ast::ItemKind::Impl( + unsafety, + ast::ImplPolarity::Positive, + ast::Defaultness::Final, + trait_generics, + opt_trait_ref, + self_type, + methods.into_iter().chain(associated_types).collect(), + ), + ) + } + + fn expand_struct_def( + &self, + cx: &mut ExtCtxt<'_>, + struct_def: &'a VariantData, + type_ident: Ident, + generics: &Generics, + from_scratch: bool, + use_temporaries: bool, + ) -> P { + let field_tys: Vec> = + struct_def.fields().iter().map(|field| field.ty.clone()).collect(); + + let methods = self + .methods + .iter() + .map(|method_def| { + let (explicit_self, self_args, nonself_args, tys) = + method_def.split_self_nonself_args(cx, self, type_ident, generics); + + let body = if from_scratch || method_def.is_static() { + method_def.expand_static_struct_method_body( + cx, + self, + struct_def, + type_ident, + &self_args[..], + &nonself_args[..], + ) + } else { + method_def.expand_struct_method_body( + cx, + self, + struct_def, + type_ident, + &self_args[..], + &nonself_args[..], + use_temporaries, + ) + }; + + method_def.create_method(cx, self, type_ident, generics, explicit_self, tys, body) + }) + .collect(); + + self.create_derived_impl(cx, type_ident, generics, field_tys, methods) + } + + fn expand_enum_def( + &self, + cx: &mut ExtCtxt<'_>, + enum_def: &'a EnumDef, + type_attrs: &[ast::Attribute], + type_ident: Ident, + generics: &Generics, + from_scratch: bool, + ) -> P { + let mut field_tys = Vec::new(); + + for variant in &enum_def.variants { + field_tys.extend(variant.data.fields().iter().map(|field| field.ty.clone())); + } + + let methods = self + .methods + .iter() + .map(|method_def| { + let (explicit_self, self_args, nonself_args, tys) = + method_def.split_self_nonself_args(cx, self, type_ident, generics); + + let body = if from_scratch || method_def.is_static() { + method_def.expand_static_enum_method_body( + cx, + self, + enum_def, + type_ident, + &self_args[..], + &nonself_args[..], + ) + } else { + method_def.expand_enum_method_body( + cx, + self, + enum_def, + type_attrs, + type_ident, + self_args, + &nonself_args[..], + ) + }; + + method_def.create_method(cx, self, type_ident, generics, explicit_self, tys, body) + }) + .collect(); + + self.create_derived_impl(cx, type_ident, generics, field_tys, methods) + } +} + +fn find_repr_type_name(sess: &ParseSess, type_attrs: &[ast::Attribute]) -> &'static str { + let mut repr_type_name = "isize"; + for a in type_attrs { + for r in &attr::find_repr_attrs(sess, a) { + repr_type_name = match *r { + attr::ReprPacked(_) + | attr::ReprSimd + | attr::ReprAlign(_) + | attr::ReprTransparent => continue, + + attr::ReprC => "i32", + + attr::ReprInt(attr::SignedInt(ast::IntTy::Isize)) => "isize", + attr::ReprInt(attr::SignedInt(ast::IntTy::I8)) => "i8", + attr::ReprInt(attr::SignedInt(ast::IntTy::I16)) => "i16", + attr::ReprInt(attr::SignedInt(ast::IntTy::I32)) => "i32", + attr::ReprInt(attr::SignedInt(ast::IntTy::I64)) => "i64", + attr::ReprInt(attr::SignedInt(ast::IntTy::I128)) => "i128", + + attr::ReprInt(attr::UnsignedInt(ast::UintTy::Usize)) => "usize", + attr::ReprInt(attr::UnsignedInt(ast::UintTy::U8)) => "u8", + attr::ReprInt(attr::UnsignedInt(ast::UintTy::U16)) => "u16", + attr::ReprInt(attr::UnsignedInt(ast::UintTy::U32)) => "u32", + attr::ReprInt(attr::UnsignedInt(ast::UintTy::U64)) => "u64", + attr::ReprInt(attr::UnsignedInt(ast::UintTy::U128)) => "u128", + } + } + } + repr_type_name +} + +impl<'a> MethodDef<'a> { + fn call_substructure_method( + &self, + cx: &mut ExtCtxt<'_>, + trait_: &TraitDef<'_>, + type_ident: Ident, + self_args: &[P], + nonself_args: &[P], + fields: &SubstructureFields<'_>, + ) -> P { + let substructure = Substructure { + type_ident, + method_ident: cx.ident_of(self.name, trait_.span), + self_args, + nonself_args, + fields, + }; + let mut f = self.combine_substructure.borrow_mut(); + let f: &mut CombineSubstructureFunc<'_> = &mut *f; + f(cx, trait_.span, &substructure) + } + + fn get_ret_ty( + &self, + cx: &mut ExtCtxt<'_>, + trait_: &TraitDef<'_>, + generics: &Generics, + type_ident: Ident, + ) -> P { + self.ret_ty.to_ty(cx, trait_.span, type_ident, generics) + } + + fn is_static(&self) -> bool { + self.explicit_self.is_none() + } + + fn split_self_nonself_args( + &self, + cx: &mut ExtCtxt<'_>, + trait_: &TraitDef<'_>, + type_ident: Ident, + generics: &Generics, + ) -> (Option, Vec>, Vec>, Vec<(Ident, P)>) { + let mut self_args = Vec::new(); + let mut nonself_args = Vec::new(); + let mut arg_tys = Vec::new(); + let mut nonstatic = false; + + let ast_explicit_self = self.explicit_self.as_ref().map(|self_ptr| { + let (self_expr, explicit_self) = ty::get_explicit_self(cx, trait_.span, self_ptr); + + self_args.push(self_expr); + nonstatic = true; + + explicit_self + }); + + for (ty, name) in self.args.iter() { + let ast_ty = ty.to_ty(cx, trait_.span, type_ident, generics); + let ident = cx.ident_of(name, trait_.span); + arg_tys.push((ident, ast_ty)); + + let arg_expr = cx.expr_ident(trait_.span, ident); + + match *ty { + // for static methods, just treat any Self + // arguments as a normal arg + Self_ if nonstatic => { + self_args.push(arg_expr); + } + Ptr(ref ty, _) if (if let Self_ = **ty { true } else { false }) && nonstatic => { + self_args.push(cx.expr_deref(trait_.span, arg_expr)) + } + _ => { + nonself_args.push(arg_expr); + } + } + } + + (ast_explicit_self, self_args, nonself_args, arg_tys) + } + + fn create_method( + &self, + cx: &mut ExtCtxt<'_>, + trait_: &TraitDef<'_>, + type_ident: Ident, + generics: &Generics, + explicit_self: Option, + arg_types: Vec<(Ident, P)>, + body: P, + ) -> ast::AssocItem { + // Create the generics that aren't for `Self`. + let fn_generics = self.generics.to_generics(cx, trait_.span, type_ident, generics); + + let args = { + let self_args = explicit_self.map(|explicit_self| { + let ident = Ident::with_dummy_span(kw::SelfLower).with_span_pos(trait_.span); + ast::Param::from_self(ast::AttrVec::default(), explicit_self, ident) + }); + let nonself_args = + arg_types.into_iter().map(|(name, ty)| cx.param(trait_.span, name, ty)); + self_args.into_iter().chain(nonself_args).collect() + }; + + let ret_type = self.get_ret_ty(cx, trait_, generics, type_ident); + + let method_ident = cx.ident_of(self.name, trait_.span); + let fn_decl = cx.fn_decl(args, ast::FunctionRetTy::Ty(ret_type)); + let body_block = cx.block_expr(body); + + let unsafety = if self.is_unsafe { ast::Unsafety::Unsafe } else { ast::Unsafety::Normal }; + + let trait_lo_sp = trait_.span.shrink_to_lo(); + + let sig = ast::FnSig { + header: ast::FnHeader { unsafety, ext: ast::Extern::None, ..ast::FnHeader::default() }, + decl: fn_decl, + }; + + // Create the method. + ast::AssocItem { + id: ast::DUMMY_NODE_ID, + attrs: self.attributes.clone(), + generics: fn_generics, + span: trait_.span, + vis: respan(trait_lo_sp, ast::VisibilityKind::Inherited), + defaultness: ast::Defaultness::Final, + ident: method_ident, + kind: ast::AssocItemKind::Fn(sig, Some(body_block)), + tokens: None, + } + } + + /// ``` + /// #[derive(PartialEq)] + /// # struct Dummy; + /// struct A { x: i32, y: i32 } + /// + /// // equivalent to: + /// impl PartialEq for A { + /// fn eq(&self, other: &A) -> bool { + /// match *self { + /// A {x: ref __self_0_0, y: ref __self_0_1} => { + /// match *other { + /// A {x: ref __self_1_0, y: ref __self_1_1} => { + /// __self_0_0.eq(__self_1_0) && __self_0_1.eq(__self_1_1) + /// } + /// } + /// } + /// } + /// } + /// } + /// + /// // or if A is repr(packed) - note fields are matched by-value + /// // instead of by-reference. + /// impl PartialEq for A { + /// fn eq(&self, other: &A) -> bool { + /// match *self { + /// A {x: __self_0_0, y: __self_0_1} => { + /// match other { + /// A {x: __self_1_0, y: __self_1_1} => { + /// __self_0_0.eq(&__self_1_0) && __self_0_1.eq(&__self_1_1) + /// } + /// } + /// } + /// } + /// } + /// } + /// ``` + fn expand_struct_method_body<'b>( + &self, + cx: &mut ExtCtxt<'_>, + trait_: &TraitDef<'b>, + struct_def: &'b VariantData, + type_ident: Ident, + self_args: &[P], + nonself_args: &[P], + use_temporaries: bool, + ) -> P { + let mut raw_fields = Vec::new(); // Vec<[fields of self], + // [fields of next Self arg], [etc]> + let mut patterns = Vec::new(); + for i in 0..self_args.len() { + let struct_path = cx.path(trait_.span, vec![type_ident]); + let (pat, ident_expr) = trait_.create_struct_pattern( + cx, + struct_path, + struct_def, + &format!("__self_{}", i), + ast::Mutability::Not, + use_temporaries, + ); + patterns.push(pat); + raw_fields.push(ident_expr); + } + + // transpose raw_fields + let fields = if !raw_fields.is_empty() { + let mut raw_fields = raw_fields.into_iter().map(|v| v.into_iter()); + let first_field = raw_fields.next().unwrap(); + let mut other_fields: Vec> = raw_fields.collect(); + first_field + .map(|(span, opt_id, field, attrs)| FieldInfo { + span, + name: opt_id, + self_: field, + other: other_fields + .iter_mut() + .map(|l| match l.next().unwrap() { + (.., ex, _) => ex, + }) + .collect(), + attrs, + }) + .collect() + } else { + cx.span_bug(trait_.span, "no `self` parameter for method in generic `derive`") + }; + + // body of the inner most destructuring match + let mut body = self.call_substructure_method( + cx, + trait_, + type_ident, + self_args, + nonself_args, + &Struct(struct_def, fields), + ); + + // make a series of nested matches, to destructure the + // structs. This is actually right-to-left, but it shouldn't + // matter. + for (arg_expr, pat) in self_args.iter().zip(patterns) { + body = cx.expr_match( + trait_.span, + arg_expr.clone(), + vec![cx.arm(trait_.span, pat.clone(), body)], + ) + } + + body + } + + fn expand_static_struct_method_body( + &self, + cx: &mut ExtCtxt<'_>, + trait_: &TraitDef<'_>, + struct_def: &VariantData, + type_ident: Ident, + self_args: &[P], + nonself_args: &[P], + ) -> P { + let summary = trait_.summarise_struct(cx, struct_def); + + self.call_substructure_method( + cx, + trait_, + type_ident, + self_args, + nonself_args, + &StaticStruct(struct_def, summary), + ) + } + + /// ``` + /// #[derive(PartialEq)] + /// # struct Dummy; + /// enum A { + /// A1, + /// A2(i32) + /// } + /// + /// // is equivalent to + /// + /// impl PartialEq for A { + /// fn eq(&self, other: &A) -> ::bool { + /// match (&*self, &*other) { + /// (&A1, &A1) => true, + /// (&A2(ref self_0), + /// &A2(ref __arg_1_0)) => (*self_0).eq(&(*__arg_1_0)), + /// _ => { + /// let __self_vi = match *self { A1(..) => 0, A2(..) => 1 }; + /// let __arg_1_vi = match *other { A1(..) => 0, A2(..) => 1 }; + /// false + /// } + /// } + /// } + /// } + /// ``` + /// + /// (Of course `__self_vi` and `__arg_1_vi` are unused for + /// `PartialEq`, and those subcomputations will hopefully be removed + /// as their results are unused. The point of `__self_vi` and + /// `__arg_1_vi` is for `PartialOrd`; see #15503.) + fn expand_enum_method_body<'b>( + &self, + cx: &mut ExtCtxt<'_>, + trait_: &TraitDef<'b>, + enum_def: &'b EnumDef, + type_attrs: &[ast::Attribute], + type_ident: Ident, + self_args: Vec>, + nonself_args: &[P], + ) -> P { + self.build_enum_match_tuple( + cx, + trait_, + enum_def, + type_attrs, + type_ident, + self_args, + nonself_args, + ) + } + + /// Creates a match for a tuple of all `self_args`, where either all + /// variants match, or it falls into a catch-all for when one variant + /// does not match. + + /// There are N + 1 cases because is a case for each of the N + /// variants where all of the variants match, and one catch-all for + /// when one does not match. + + /// As an optimization we generate code which checks whether all variants + /// match first which makes llvm see that C-like enums can be compiled into + /// a simple equality check (for PartialEq). + + /// The catch-all handler is provided access the variant index values + /// for each of the self-args, carried in precomputed variables. + + /// ```{.text} + /// let __self0_vi = unsafe { + /// std::intrinsics::discriminant_value(&self) } as i32; + /// let __self1_vi = unsafe { + /// std::intrinsics::discriminant_value(&arg1) } as i32; + /// let __self2_vi = unsafe { + /// std::intrinsics::discriminant_value(&arg2) } as i32; + /// + /// if __self0_vi == __self1_vi && __self0_vi == __self2_vi && ... { + /// match (...) { + /// (Variant1, Variant1, ...) => Body1 + /// (Variant2, Variant2, ...) => Body2, + /// ... + /// _ => ::core::intrinsics::unreachable() + /// } + /// } + /// else { + /// ... // catch-all remainder can inspect above variant index values. + /// } + /// ``` + fn build_enum_match_tuple<'b>( + &self, + cx: &mut ExtCtxt<'_>, + trait_: &TraitDef<'b>, + enum_def: &'b EnumDef, + type_attrs: &[ast::Attribute], + type_ident: Ident, + mut self_args: Vec>, + nonself_args: &[P], + ) -> P { + let sp = trait_.span; + let variants = &enum_def.variants; + + let self_arg_names = iter::once("__self".to_string()) + .chain( + self_args + .iter() + .enumerate() + .skip(1) + .map(|(arg_count, _self_arg)| format!("__arg_{}", arg_count)), + ) + .collect::>(); + + let self_arg_idents = + self_arg_names.iter().map(|name| cx.ident_of(name, sp)).collect::>(); + + // The `vi_idents` will be bound, solely in the catch-all, to + // a series of let statements mapping each self_arg to an int + // value corresponding to its discriminant. + let vi_idents = self_arg_names + .iter() + .map(|name| { + let vi_suffix = format!("{}_vi", &name[..]); + cx.ident_of(&vi_suffix[..], trait_.span) + }) + .collect::>(); + + // Builds, via callback to call_substructure_method, the + // delegated expression that handles the catch-all case, + // using `__variants_tuple` to drive logic if necessary. + let catch_all_substructure = + EnumNonMatchingCollapsed(self_arg_idents, &variants[..], &vi_idents[..]); + + let first_fieldless = variants.iter().find(|v| v.data.fields().is_empty()); + + // These arms are of the form: + // (Variant1, Variant1, ...) => Body1 + // (Variant2, Variant2, ...) => Body2 + // ... + // where each tuple has length = self_args.len() + let mut match_arms: Vec = variants + .iter() + .enumerate() + .filter(|&(_, v)| !(self.unify_fieldless_variants && v.data.fields().is_empty())) + .map(|(index, variant)| { + let mk_self_pat = |cx: &mut ExtCtxt<'_>, self_arg_name: &str| { + let (p, idents) = trait_.create_enum_variant_pattern( + cx, + type_ident, + variant, + self_arg_name, + ast::Mutability::Not, + ); + (cx.pat(sp, PatKind::Ref(p, ast::Mutability::Not)), idents) + }; + + // A single arm has form (&VariantK, &VariantK, ...) => BodyK + // (see "Final wrinkle" note below for why.) + let mut subpats = Vec::with_capacity(self_arg_names.len()); + let mut self_pats_idents = Vec::with_capacity(self_arg_names.len() - 1); + let first_self_pat_idents = { + let (p, idents) = mk_self_pat(cx, &self_arg_names[0]); + subpats.push(p); + idents + }; + for self_arg_name in &self_arg_names[1..] { + let (p, idents) = mk_self_pat(cx, &self_arg_name[..]); + subpats.push(p); + self_pats_idents.push(idents); + } + + // Here is the pat = `(&VariantK, &VariantK, ...)` + let single_pat = cx.pat_tuple(sp, subpats); + + // For the BodyK, we need to delegate to our caller, + // passing it an EnumMatching to indicate which case + // we are in. + + // All of the Self args have the same variant in these + // cases. So we transpose the info in self_pats_idents + // to gather the getter expressions together, in the + // form that EnumMatching expects. + + // The transposition is driven by walking across the + // arg fields of the variant for the first self pat. + let field_tuples = first_self_pat_idents + .into_iter() + .enumerate() + // For each arg field of self, pull out its getter expr ... + .map(|(field_index, (sp, opt_ident, self_getter_expr, attrs))| { + // ... but FieldInfo also wants getter expr + // for matching other arguments of Self type; + // so walk across the *other* self_pats_idents + // and pull out getter for same field in each + // of them (using `field_index` tracked above). + // That is the heart of the transposition. + let others = self_pats_idents + .iter() + .map(|fields| { + let (_, _opt_ident, ref other_getter_expr, _) = fields[field_index]; + + // All Self args have same variant, so + // opt_idents are the same. (Assert + // here to make it self-evident that + // it is okay to ignore `_opt_ident`.) + assert!(opt_ident == _opt_ident); + + other_getter_expr.clone() + }) + .collect::>>(); + + FieldInfo { + span: sp, + name: opt_ident, + self_: self_getter_expr, + other: others, + attrs, + } + }) + .collect::>>(); + + // Now, for some given VariantK, we have built up + // expressions for referencing every field of every + // Self arg, assuming all are instances of VariantK. + // Build up code associated with such a case. + let substructure = EnumMatching(index, variants.len(), variant, field_tuples); + let arm_expr = self.call_substructure_method( + cx, + trait_, + type_ident, + &self_args[..], + nonself_args, + &substructure, + ); + + cx.arm(sp, single_pat, arm_expr) + }) + .collect(); + + let default = match first_fieldless { + Some(v) if self.unify_fieldless_variants => { + // We need a default case that handles the fieldless variants. + // The index and actual variant aren't meaningful in this case, + // so just use whatever + let substructure = EnumMatching(0, variants.len(), v, Vec::new()); + Some(self.call_substructure_method( + cx, + trait_, + type_ident, + &self_args[..], + nonself_args, + &substructure, + )) + } + _ if variants.len() > 1 && self_args.len() > 1 => { + // Since we know that all the arguments will match if we reach + // the match expression we add the unreachable intrinsics as the + // result of the catch all which should help llvm in optimizing it + Some(deriving::call_intrinsic(cx, sp, "unreachable", vec![])) + } + _ => None, + }; + if let Some(arm) = default { + match_arms.push(cx.arm(sp, cx.pat_wild(sp), arm)); + } + + // We will usually need the catch-all after matching the + // tuples `(VariantK, VariantK, ...)` for each VariantK of the + // enum. But: + // + // * when there is only one Self arg, the arms above suffice + // (and the deriving we call back into may not be prepared to + // handle EnumNonMatchCollapsed), and, + // + // * when the enum has only one variant, the single arm that + // is already present always suffices. + // + // * In either of the two cases above, if we *did* add a + // catch-all `_` match, it would trigger the + // unreachable-pattern error. + // + if variants.len() > 1 && self_args.len() > 1 { + // Build a series of let statements mapping each self_arg + // to its discriminant value. If this is a C-style enum + // with a specific repr type, then casts the values to + // that type. Otherwise casts to `i32` (the default repr + // type). + // + // i.e., for `enum E { A, B(1), C(T, T) }`, and a deriving + // with three Self args, builds three statements: + // + // ``` + // let __self0_vi = unsafe { + // std::intrinsics::discriminant_value(&self) } as i32; + // let __self1_vi = unsafe { + // std::intrinsics::discriminant_value(&arg1) } as i32; + // let __self2_vi = unsafe { + // std::intrinsics::discriminant_value(&arg2) } as i32; + // ``` + let mut index_let_stmts: Vec = Vec::with_capacity(vi_idents.len() + 1); + + // We also build an expression which checks whether all discriminants are equal + // discriminant_test = __self0_vi == __self1_vi && __self0_vi == __self2_vi && ... + let mut discriminant_test = cx.expr_bool(sp, true); + + let target_type_name = find_repr_type_name(&cx.parse_sess, type_attrs); + + let mut first_ident = None; + for (&ident, self_arg) in vi_idents.iter().zip(&self_args) { + let self_addr = cx.expr_addr_of(sp, self_arg.clone()); + let variant_value = + deriving::call_intrinsic(cx, sp, "discriminant_value", vec![self_addr]); + + let target_ty = cx.ty_ident(sp, cx.ident_of(target_type_name, sp)); + let variant_disr = cx.expr_cast(sp, variant_value, target_ty); + let let_stmt = cx.stmt_let(sp, false, ident, variant_disr); + index_let_stmts.push(let_stmt); + + match first_ident { + Some(first) => { + let first_expr = cx.expr_ident(sp, first); + let id = cx.expr_ident(sp, ident); + let test = cx.expr_binary(sp, BinOpKind::Eq, first_expr, id); + discriminant_test = + cx.expr_binary(sp, BinOpKind::And, discriminant_test, test) + } + None => { + first_ident = Some(ident); + } + } + } + + let arm_expr = self.call_substructure_method( + cx, + trait_, + type_ident, + &self_args[..], + nonself_args, + &catch_all_substructure, + ); + + // Final wrinkle: the self_args are expressions that deref + // down to desired places, but we cannot actually deref + // them when they are fed as r-values into a tuple + // expression; here add a layer of borrowing, turning + // `(*self, *__arg_0, ...)` into `(&*self, &*__arg_0, ...)`. + self_args.map_in_place(|self_arg| cx.expr_addr_of(sp, self_arg)); + let match_arg = cx.expr(sp, ast::ExprKind::Tup(self_args)); + + // Lastly we create an expression which branches on all discriminants being equal + // if discriminant_test { + // match (...) { + // (Variant1, Variant1, ...) => Body1 + // (Variant2, Variant2, ...) => Body2, + // ... + // _ => ::core::intrinsics::unreachable() + // } + // } + // else { + // + // } + let all_match = cx.expr_match(sp, match_arg, match_arms); + let arm_expr = cx.expr_if(sp, discriminant_test, all_match, Some(arm_expr)); + index_let_stmts.push(cx.stmt_expr(arm_expr)); + cx.expr_block(cx.block(sp, index_let_stmts)) + } else if variants.is_empty() { + // As an additional wrinkle, For a zero-variant enum A, + // currently the compiler + // will accept `fn (a: &Self) { match *a { } }` + // but rejects `fn (a: &Self) { match (&*a,) { } }` + // as well as `fn (a: &Self) { match ( *a,) { } }` + // + // This means that the strategy of building up a tuple of + // all Self arguments fails when Self is a zero variant + // enum: rustc rejects the expanded program, even though + // the actual code tends to be impossible to execute (at + // least safely), according to the type system. + // + // The most expedient fix for this is to just let the + // code fall through to the catch-all. But even this is + // error-prone, since the catch-all as defined above would + // generate code like this: + // + // _ => { let __self0 = match *self { }; + // let __self1 = match *__arg_0 { }; + // } + // + // Which is yields bindings for variables which type + // inference cannot resolve to unique types. + // + // One option to the above might be to add explicit type + // annotations. But the *only* reason to go down that path + // would be to try to make the expanded output consistent + // with the case when the number of enum variants >= 1. + // + // That just isn't worth it. In fact, trying to generate + // sensible code for *any* deriving on a zero-variant enum + // does not make sense. But at the same time, for now, we + // do not want to cause a compile failure just because the + // user happened to attach a deriving to their + // zero-variant enum. + // + // Instead, just generate a failing expression for the + // zero variant case, skipping matches and also skipping + // delegating back to the end user code entirely. + // + // (See also #4499 and #12609; note that some of the + // discussions there influence what choice we make here; + // e.g., if we feature-gate `match x { ... }` when x refers + // to an uninhabited type (e.g., a zero-variant enum or a + // type holding such an enum), but do not feature-gate + // zero-variant enums themselves, then attempting to + // derive Debug on such a type could here generate code + // that needs the feature gate enabled.) + + deriving::call_intrinsic(cx, sp, "unreachable", vec![]) + } else { + // Final wrinkle: the self_args are expressions that deref + // down to desired places, but we cannot actually deref + // them when they are fed as r-values into a tuple + // expression; here add a layer of borrowing, turning + // `(*self, *__arg_0, ...)` into `(&*self, &*__arg_0, ...)`. + self_args.map_in_place(|self_arg| cx.expr_addr_of(sp, self_arg)); + let match_arg = cx.expr(sp, ast::ExprKind::Tup(self_args)); + cx.expr_match(sp, match_arg, match_arms) + } + } + + fn expand_static_enum_method_body( + &self, + cx: &mut ExtCtxt<'_>, + trait_: &TraitDef<'_>, + enum_def: &EnumDef, + type_ident: Ident, + self_args: &[P], + nonself_args: &[P], + ) -> P { + let summary = enum_def + .variants + .iter() + .map(|v| { + let sp = v.span.with_ctxt(trait_.span.ctxt()); + let summary = trait_.summarise_struct(cx, &v.data); + (v.ident, sp, summary) + }) + .collect(); + self.call_substructure_method( + cx, + trait_, + type_ident, + self_args, + nonself_args, + &StaticEnum(enum_def, summary), + ) + } +} + +// general helper methods. +impl<'a> TraitDef<'a> { + fn summarise_struct(&self, cx: &mut ExtCtxt<'_>, struct_def: &VariantData) -> StaticFields { + let mut named_idents = Vec::new(); + let mut just_spans = Vec::new(); + for field in struct_def.fields() { + let sp = field.span.with_ctxt(self.span.ctxt()); + match field.ident { + Some(ident) => named_idents.push((ident, sp)), + _ => just_spans.push(sp), + } + } + + let is_tuple = if let ast::VariantData::Tuple(..) = struct_def { true } else { false }; + match (just_spans.is_empty(), named_idents.is_empty()) { + (false, false) => cx.span_bug( + self.span, + "a struct with named and unnamed \ + fields in generic `derive`", + ), + // named fields + (_, false) => Named(named_idents), + // unnamed fields + (false, _) => Unnamed(just_spans, is_tuple), + // empty + _ => Named(Vec::new()), + } + } + + fn create_subpatterns( + &self, + cx: &mut ExtCtxt<'_>, + field_paths: Vec, + mutbl: ast::Mutability, + use_temporaries: bool, + ) -> Vec> { + field_paths + .iter() + .map(|path| { + let binding_mode = if use_temporaries { + ast::BindingMode::ByValue(ast::Mutability::Not) + } else { + ast::BindingMode::ByRef(mutbl) + }; + cx.pat(path.span, PatKind::Ident(binding_mode, (*path).clone(), None)) + }) + .collect() + } + + fn create_struct_pattern( + &self, + cx: &mut ExtCtxt<'_>, + struct_path: ast::Path, + struct_def: &'a VariantData, + prefix: &str, + mutbl: ast::Mutability, + use_temporaries: bool, + ) -> (P, Vec<(Span, Option, P, &'a [ast::Attribute])>) { + let mut paths = Vec::new(); + let mut ident_exprs = Vec::new(); + for (i, struct_field) in struct_def.fields().iter().enumerate() { + let sp = struct_field.span.with_ctxt(self.span.ctxt()); + let ident = cx.ident_of(&format!("{}_{}", prefix, i), self.span); + paths.push(ident.with_span_pos(sp)); + let val = cx.expr_path(cx.path_ident(sp, ident)); + let val = if use_temporaries { val } else { cx.expr_deref(sp, val) }; + let val = cx.expr(sp, ast::ExprKind::Paren(val)); + + ident_exprs.push((sp, struct_field.ident, val, &struct_field.attrs[..])); + } + + let subpats = self.create_subpatterns(cx, paths, mutbl, use_temporaries); + let pattern = match *struct_def { + VariantData::Struct(..) => { + let field_pats = subpats + .into_iter() + .zip(&ident_exprs) + .map(|(pat, &(sp, ident, ..))| { + if ident.is_none() { + cx.span_bug(sp, "a braced struct with unnamed fields in `derive`"); + } + ast::FieldPat { + ident: ident.unwrap(), + is_shorthand: false, + attrs: ast::AttrVec::new(), + id: ast::DUMMY_NODE_ID, + span: pat.span.with_ctxt(self.span.ctxt()), + pat, + is_placeholder: false, + } + }) + .collect(); + cx.pat_struct(self.span, struct_path, field_pats) + } + VariantData::Tuple(..) => cx.pat_tuple_struct(self.span, struct_path, subpats), + VariantData::Unit(..) => cx.pat_path(self.span, struct_path), + }; + + (pattern, ident_exprs) + } + + fn create_enum_variant_pattern( + &self, + cx: &mut ExtCtxt<'_>, + enum_ident: ast::Ident, + variant: &'a ast::Variant, + prefix: &str, + mutbl: ast::Mutability, + ) -> (P, Vec<(Span, Option, P, &'a [ast::Attribute])>) { + let sp = variant.span.with_ctxt(self.span.ctxt()); + let variant_path = cx.path(sp, vec![enum_ident, variant.ident]); + let use_temporaries = false; // enums can't be repr(packed) + self.create_struct_pattern(cx, variant_path, &variant.data, prefix, mutbl, use_temporaries) + } +} + +// helpful premade recipes + +pub fn cs_fold_fields<'a, F>( + use_foldl: bool, + mut f: F, + base: P, + cx: &mut ExtCtxt<'_>, + all_fields: &[FieldInfo<'a>], +) -> P +where + F: FnMut(&mut ExtCtxt<'_>, Span, P, P, &[P]) -> P, +{ + if use_foldl { + all_fields + .iter() + .fold(base, |old, field| f(cx, field.span, old, field.self_.clone(), &field.other)) + } else { + all_fields + .iter() + .rev() + .fold(base, |old, field| f(cx, field.span, old, field.self_.clone(), &field.other)) + } +} + +pub fn cs_fold_enumnonmatch( + mut enum_nonmatch_f: EnumNonMatchCollapsedFunc<'_>, + cx: &mut ExtCtxt<'_>, + trait_span: Span, + substructure: &Substructure<'_>, +) -> P { + match *substructure.fields { + EnumNonMatchingCollapsed(ref all_args, _, tuple) => { + enum_nonmatch_f(cx, trait_span, (&all_args[..], tuple), substructure.nonself_args) + } + _ => cx.span_bug(trait_span, "cs_fold_enumnonmatch expected an EnumNonMatchingCollapsed"), + } +} + +pub fn cs_fold_static(cx: &mut ExtCtxt<'_>, trait_span: Span) -> P { + cx.span_bug(trait_span, "static function in `derive`") +} + +/// Fold the fields. `use_foldl` controls whether this is done +/// left-to-right (`true`) or right-to-left (`false`). +pub fn cs_fold( + use_foldl: bool, + f: F, + base: P, + enum_nonmatch_f: EnumNonMatchCollapsedFunc<'_>, + cx: &mut ExtCtxt<'_>, + trait_span: Span, + substructure: &Substructure<'_>, +) -> P +where + F: FnMut(&mut ExtCtxt<'_>, Span, P, P, &[P]) -> P, +{ + match *substructure.fields { + EnumMatching(.., ref all_fields) | Struct(_, ref all_fields) => { + cs_fold_fields(use_foldl, f, base, cx, all_fields) + } + EnumNonMatchingCollapsed(..) => { + cs_fold_enumnonmatch(enum_nonmatch_f, cx, trait_span, substructure) + } + StaticEnum(..) | StaticStruct(..) => cs_fold_static(cx, trait_span), + } +} + +/// Function to fold over fields, with three cases, to generate more efficient and concise code. +/// When the `substructure` has grouped fields, there are two cases: +/// Zero fields: call the base case function with `None` (like the usual base case of `cs_fold`). +/// One or more fields: call the base case function on the first value (which depends on +/// `use_fold`), and use that as the base case. Then perform `cs_fold` on the remainder of the +/// fields. +/// When the `substructure` is a `EnumNonMatchingCollapsed`, the result of `enum_nonmatch_f` +/// is returned. Statics may not be folded over. +/// See `cs_op` in `partial_ord.rs` for a model example. +pub fn cs_fold1( + use_foldl: bool, + f: F, + mut b: B, + enum_nonmatch_f: EnumNonMatchCollapsedFunc<'_>, + cx: &mut ExtCtxt<'_>, + trait_span: Span, + substructure: &Substructure<'_>, +) -> P +where + F: FnMut(&mut ExtCtxt<'_>, Span, P, P, &[P]) -> P, + B: FnMut(&mut ExtCtxt<'_>, Option<(Span, P, &[P])>) -> P, +{ + match *substructure.fields { + EnumMatching(.., ref all_fields) | Struct(_, ref all_fields) => { + let (base, all_fields) = match (all_fields.is_empty(), use_foldl) { + (false, true) => { + let field = &all_fields[0]; + let args = (field.span, field.self_.clone(), &field.other[..]); + (b(cx, Some(args)), &all_fields[1..]) + } + (false, false) => { + let idx = all_fields.len() - 1; + let field = &all_fields[idx]; + let args = (field.span, field.self_.clone(), &field.other[..]); + (b(cx, Some(args)), &all_fields[..idx]) + } + (true, _) => (b(cx, None), &all_fields[..]), + }; + + cs_fold_fields(use_foldl, f, base, cx, all_fields) + } + EnumNonMatchingCollapsed(..) => { + cs_fold_enumnonmatch(enum_nonmatch_f, cx, trait_span, substructure) + } + StaticEnum(..) | StaticStruct(..) => cs_fold_static(cx, trait_span), + } +} + +/// Returns `true` if the type has no value fields +/// (for an enum, no variant has any fields) +pub fn is_type_without_fields(item: &Annotatable) -> bool { + if let Annotatable::Item(ref item) = *item { + match item.kind { + ast::ItemKind::Enum(ref enum_def, _) => { + enum_def.variants.iter().all(|v| v.data.fields().is_empty()) + } + ast::ItemKind::Struct(ref variant_data, _) => variant_data.fields().is_empty(), + _ => false, + } + } else { + false + } +} diff --git a/src/librustc_builtin_macros/deriving/generic/ty.rs b/src/librustc_builtin_macros/deriving/generic/ty.rs new file mode 100644 index 00000000000..7eab15aff77 --- /dev/null +++ b/src/librustc_builtin_macros/deriving/generic/ty.rs @@ -0,0 +1,283 @@ +//! A mini version of ast::Ty, which is easier to use, and features an explicit `Self` type to use +//! when specifying impls to be derived. + +pub use PtrTy::*; +pub use Ty::*; + +use syntax::ast::{self, Expr, GenericArg, GenericParamKind, Generics, Ident, SelfKind}; +use syntax::ptr::P; +use syntax::source_map::{respan, DUMMY_SP}; +use syntax_expand::base::ExtCtxt; +use syntax_pos::symbol::kw; +use syntax_pos::Span; + +/// The types of pointers +#[derive(Clone)] +pub enum PtrTy { + /// &'lifetime mut + Borrowed(Option, ast::Mutability), + /// *mut + #[allow(dead_code)] + Raw(ast::Mutability), +} + +/// A path, e.g., `::std::option::Option::` (global). Has support +/// for type parameters and a lifetime. +#[derive(Clone)] +pub struct Path<'a> { + path: Vec<&'a str>, + lifetime: Option, + params: Vec>>, + kind: PathKind, +} + +#[derive(Clone)] +pub enum PathKind { + Local, + Global, + Std, +} + +impl<'a> Path<'a> { + pub fn new(path: Vec<&str>) -> Path<'_> { + Path::new_(path, None, Vec::new(), PathKind::Std) + } + pub fn new_local(path: &str) -> Path<'_> { + Path::new_(vec![path], None, Vec::new(), PathKind::Local) + } + pub fn new_<'r>( + path: Vec<&'r str>, + lifetime: Option, + params: Vec>>, + kind: PathKind, + ) -> Path<'r> { + Path { path, lifetime, params, kind } + } + + pub fn to_ty( + &self, + cx: &ExtCtxt<'_>, + span: Span, + self_ty: Ident, + self_generics: &Generics, + ) -> P { + cx.ty_path(self.to_path(cx, span, self_ty, self_generics)) + } + pub fn to_path( + &self, + cx: &ExtCtxt<'_>, + span: Span, + self_ty: Ident, + self_generics: &Generics, + ) -> ast::Path { + let mut idents = self.path.iter().map(|s| cx.ident_of(*s, span)).collect(); + let lt = mk_lifetimes(cx, span, &self.lifetime); + let tys: Vec> = + self.params.iter().map(|t| t.to_ty(cx, span, self_ty, self_generics)).collect(); + let params = lt + .into_iter() + .map(|lt| GenericArg::Lifetime(lt)) + .chain(tys.into_iter().map(|ty| GenericArg::Type(ty))) + .collect(); + + match self.kind { + PathKind::Global => cx.path_all(span, true, idents, params), + PathKind::Local => cx.path_all(span, false, idents, params), + PathKind::Std => { + let def_site = cx.with_def_site_ctxt(DUMMY_SP); + idents.insert(0, Ident::new(kw::DollarCrate, def_site)); + cx.path_all(span, false, idents, params) + } + } + } +} + +/// A type. Supports pointers, Self, and literals. +#[derive(Clone)] +pub enum Ty<'a> { + Self_, + /// &/Box/ Ty + Ptr(Box>, PtrTy), + /// mod::mod::Type<[lifetime], [Params...]>, including a plain type + /// parameter, and things like `i32` + Literal(Path<'a>), + /// includes unit + Tuple(Vec>), +} + +pub fn borrowed_ptrty() -> PtrTy { + Borrowed(None, ast::Mutability::Not) +} +pub fn borrowed(ty: Box>) -> Ty<'_> { + Ptr(ty, borrowed_ptrty()) +} + +pub fn borrowed_explicit_self() -> Option> { + Some(Some(borrowed_ptrty())) +} + +pub fn borrowed_self<'r>() -> Ty<'r> { + borrowed(Box::new(Self_)) +} + +pub fn nil_ty<'r>() -> Ty<'r> { + Tuple(Vec::new()) +} + +fn mk_lifetime(cx: &ExtCtxt<'_>, span: Span, lt: &Option) -> Option { + lt.map(|ident| cx.lifetime(span, ident)) +} + +fn mk_lifetimes(cx: &ExtCtxt<'_>, span: Span, lt: &Option) -> Vec { + mk_lifetime(cx, span, lt).into_iter().collect() +} + +impl<'a> Ty<'a> { + pub fn to_ty( + &self, + cx: &ExtCtxt<'_>, + span: Span, + self_ty: Ident, + self_generics: &Generics, + ) -> P { + match *self { + Ptr(ref ty, ref ptr) => { + let raw_ty = ty.to_ty(cx, span, self_ty, self_generics); + match *ptr { + Borrowed(ref lt, mutbl) => { + let lt = mk_lifetime(cx, span, lt); + cx.ty_rptr(span, raw_ty, lt, mutbl) + } + Raw(mutbl) => cx.ty_ptr(span, raw_ty, mutbl), + } + } + Literal(ref p) => p.to_ty(cx, span, self_ty, self_generics), + Self_ => cx.ty_path(self.to_path(cx, span, self_ty, self_generics)), + Tuple(ref fields) => { + let ty = ast::TyKind::Tup( + fields.iter().map(|f| f.to_ty(cx, span, self_ty, self_generics)).collect(), + ); + cx.ty(span, ty) + } + } + } + + pub fn to_path( + &self, + cx: &ExtCtxt<'_>, + span: Span, + self_ty: Ident, + generics: &Generics, + ) -> ast::Path { + match *self { + Self_ => { + let params: Vec<_> = generics + .params + .iter() + .map(|param| match param.kind { + GenericParamKind::Lifetime { .. } => { + GenericArg::Lifetime(ast::Lifetime { id: param.id, ident: param.ident }) + } + GenericParamKind::Type { .. } => { + GenericArg::Type(cx.ty_ident(span, param.ident)) + } + GenericParamKind::Const { .. } => { + GenericArg::Const(cx.const_ident(span, param.ident)) + } + }) + .collect(); + + cx.path_all(span, false, vec![self_ty], params) + } + Literal(ref p) => p.to_path(cx, span, self_ty, generics), + Ptr(..) => cx.span_bug(span, "pointer in a path in generic `derive`"), + Tuple(..) => cx.span_bug(span, "tuple in a path in generic `derive`"), + } + } +} + +fn mk_ty_param( + cx: &ExtCtxt<'_>, + span: Span, + name: &str, + attrs: &[ast::Attribute], + bounds: &[Path<'_>], + self_ident: Ident, + self_generics: &Generics, +) -> ast::GenericParam { + let bounds = bounds + .iter() + .map(|b| { + let path = b.to_path(cx, span, self_ident, self_generics); + cx.trait_bound(path) + }) + .collect(); + cx.typaram(span, cx.ident_of(name, span), attrs.to_owned(), bounds, None) +} + +fn mk_generics(params: Vec, span: Span) -> Generics { + Generics { params, where_clause: ast::WhereClause { predicates: Vec::new(), span }, span } +} + +/// Lifetimes and bounds on type parameters +#[derive(Clone)] +pub struct LifetimeBounds<'a> { + pub lifetimes: Vec<(&'a str, Vec<&'a str>)>, + pub bounds: Vec<(&'a str, Vec>)>, +} + +impl<'a> LifetimeBounds<'a> { + pub fn empty() -> LifetimeBounds<'a> { + LifetimeBounds { lifetimes: Vec::new(), bounds: Vec::new() } + } + pub fn to_generics( + &self, + cx: &ExtCtxt<'_>, + span: Span, + self_ty: Ident, + self_generics: &Generics, + ) -> Generics { + let generic_params = self + .lifetimes + .iter() + .map(|&(lt, ref bounds)| { + let bounds = bounds + .iter() + .map(|b| ast::GenericBound::Outlives(cx.lifetime(span, Ident::from_str(b)))); + cx.lifetime_def(span, Ident::from_str(lt), vec![], bounds.collect()) + }) + .chain(self.bounds.iter().map(|t| { + let (name, ref bounds) = *t; + mk_ty_param(cx, span, name, &[], &bounds, self_ty, self_generics) + })) + .collect(); + + mk_generics(generic_params, span) + } +} + +pub fn get_explicit_self( + cx: &ExtCtxt<'_>, + span: Span, + self_ptr: &Option, +) -> (P, ast::ExplicitSelf) { + // this constructs a fresh `self` path + let self_path = cx.expr_self(span); + match *self_ptr { + None => (self_path, respan(span, SelfKind::Value(ast::Mutability::Not))), + Some(ref ptr) => { + let self_ty = respan( + span, + match *ptr { + Borrowed(ref lt, mutbl) => { + let lt = lt.map(|s| cx.lifetime(span, s)); + SelfKind::Region(lt, mutbl) + } + Raw(_) => cx.span_bug(span, "attempted to use *self in deriving definition"), + }, + ); + let self_expr = cx.expr_deref(span, self_path); + (self_expr, self_ty) + } + } +} diff --git a/src/librustc_builtin_macros/deriving/hash.rs b/src/librustc_builtin_macros/deriving/hash.rs new file mode 100644 index 00000000000..acf18ac70e6 --- /dev/null +++ b/src/librustc_builtin_macros/deriving/hash.rs @@ -0,0 +1,92 @@ +use crate::deriving::generic::ty::*; +use crate::deriving::generic::*; +use crate::deriving::{self, path_std, pathvec_std}; + +use syntax::ast::{Expr, MetaItem, Mutability}; +use syntax::ptr::P; +use syntax::symbol::sym; +use syntax_expand::base::{Annotatable, ExtCtxt}; +use syntax_pos::Span; + +pub fn expand_deriving_hash( + cx: &mut ExtCtxt<'_>, + span: Span, + mitem: &MetaItem, + item: &Annotatable, + push: &mut dyn FnMut(Annotatable), +) { + let path = Path::new_(pathvec_std!(cx, hash::Hash), None, vec![], PathKind::Std); + + let typaram = "__H"; + + let arg = Path::new_local(typaram); + let hash_trait_def = TraitDef { + span, + attributes: Vec::new(), + path, + additional_bounds: Vec::new(), + generics: LifetimeBounds::empty(), + is_unsafe: false, + supports_unions: false, + methods: vec![MethodDef { + name: "hash", + generics: LifetimeBounds { + lifetimes: Vec::new(), + bounds: vec![(typaram, vec![path_std!(cx, hash::Hasher)])], + }, + explicit_self: borrowed_explicit_self(), + args: vec![(Ptr(Box::new(Literal(arg)), Borrowed(None, Mutability::Mut)), "state")], + ret_ty: nil_ty(), + attributes: vec![], + is_unsafe: false, + unify_fieldless_variants: true, + combine_substructure: combine_substructure(Box::new(|a, b, c| { + hash_substructure(a, b, c) + })), + }], + associated_types: Vec::new(), + }; + + hash_trait_def.expand(cx, mitem, item, push); +} + +fn hash_substructure(cx: &mut ExtCtxt<'_>, trait_span: Span, substr: &Substructure<'_>) -> P { + let state_expr = match &substr.nonself_args { + &[o_f] => o_f, + _ => cx.span_bug(trait_span, "incorrect number of arguments in `derive(Hash)`"), + }; + let call_hash = |span, thing_expr| { + let hash_path = { + let strs = cx.std_path(&[sym::hash, sym::Hash, sym::hash]); + + cx.expr_path(cx.path_global(span, strs)) + }; + let ref_thing = cx.expr_addr_of(span, thing_expr); + let expr = cx.expr_call(span, hash_path, vec![ref_thing, state_expr.clone()]); + cx.stmt_expr(expr) + }; + let mut stmts = Vec::new(); + + let fields = match *substr.fields { + Struct(_, ref fs) | EnumMatching(_, 1, .., ref fs) => fs, + EnumMatching(.., ref fs) => { + let variant_value = deriving::call_intrinsic( + cx, + trait_span, + "discriminant_value", + vec![cx.expr_self(trait_span)], + ); + + stmts.push(call_hash(trait_span, variant_value)); + + fs + } + _ => cx.span_bug(trait_span, "impossible substructure in `derive(Hash)`"), + }; + + stmts.extend( + fields.iter().map(|FieldInfo { ref self_, span, .. }| call_hash(*span, self_.clone())), + ); + + cx.expr_block(cx.block(trait_span, stmts)) +} diff --git a/src/librustc_builtin_macros/deriving/mod.rs b/src/librustc_builtin_macros/deriving/mod.rs new file mode 100644 index 00000000000..ca4d4fbc5bd --- /dev/null +++ b/src/librustc_builtin_macros/deriving/mod.rs @@ -0,0 +1,171 @@ +//! The compiler code necessary to implement the `#[derive]` extensions. + +use syntax::ast::{self, ItemKind, MetaItem}; +use syntax::ptr::P; +use syntax::symbol::{sym, Symbol}; +use syntax_expand::base::{Annotatable, ExtCtxt, MultiItemModifier}; +use syntax_pos::Span; + +macro path_local($x:ident) { + generic::ty::Path::new_local(stringify!($x)) +} + +macro pathvec_std($cx:expr, $($rest:ident)::+) {{ + vec![ $( stringify!($rest) ),+ ] +}} + +macro path_std($($x:tt)*) { + generic::ty::Path::new( pathvec_std!( $($x)* ) ) +} + +pub mod bounds; +pub mod clone; +pub mod debug; +pub mod decodable; +pub mod default; +pub mod encodable; +pub mod hash; + +#[path = "cmp/eq.rs"] +pub mod eq; +#[path = "cmp/ord.rs"] +pub mod ord; +#[path = "cmp/partial_eq.rs"] +pub mod partial_eq; +#[path = "cmp/partial_ord.rs"] +pub mod partial_ord; + +pub mod generic; + +crate struct BuiltinDerive( + crate fn(&mut ExtCtxt<'_>, Span, &MetaItem, &Annotatable, &mut dyn FnMut(Annotatable)), +); + +impl MultiItemModifier for BuiltinDerive { + fn expand( + &self, + ecx: &mut ExtCtxt<'_>, + span: Span, + meta_item: &MetaItem, + item: Annotatable, + ) -> Vec { + // FIXME: Built-in derives often forget to give spans contexts, + // so we are doing it here in a centralized way. + let span = ecx.with_def_site_ctxt(span); + let mut items = Vec::new(); + (self.0)(ecx, span, meta_item, &item, &mut |a| items.push(a)); + items + } +} + +/// Constructs an expression that calls an intrinsic +fn call_intrinsic( + cx: &ExtCtxt<'_>, + span: Span, + intrinsic: &str, + args: Vec>, +) -> P { + let span = cx.with_def_site_ctxt(span); + let path = cx.std_path(&[sym::intrinsics, Symbol::intern(intrinsic)]); + let call = cx.expr_call_global(span, path, args); + + cx.expr_block(P(ast::Block { + stmts: vec![cx.stmt_expr(call)], + id: ast::DUMMY_NODE_ID, + rules: ast::BlockCheckMode::Unsafe(ast::CompilerGenerated), + span, + })) +} + +// Injects `impl<...> Structural for ItemType<...> { }`. In particular, +// does *not* add `where T: Structural` for parameters `T` in `...`. +// (That's the main reason we cannot use TraitDef here.) +fn inject_impl_of_structural_trait( + cx: &mut ExtCtxt<'_>, + span: Span, + item: &Annotatable, + structural_path: generic::ty::Path<'_>, + push: &mut dyn FnMut(Annotatable), +) { + let item = match *item { + Annotatable::Item(ref item) => item, + _ => { + // Non-Item derive is an error, but it should have been + // set earlier; see + // libsyntax_expand/expand.rs:MacroExpander::fully_expand_fragment() + // libsyntax_expand/base.rs:Annotatable::derive_allowed() + return; + } + }; + + let generics = match item.kind { + ItemKind::Struct(_, ref generics) | ItemKind::Enum(_, ref generics) => generics, + // Do not inject `impl Structural for Union`. (`PartialEq` does not + // support unions, so we will see error downstream.) + ItemKind::Union(..) => return, + _ => unreachable!(), + }; + + // Create generics param list for where clauses and impl headers + let mut generics = generics.clone(); + + // Create the type of `self`. + // + // in addition, remove defaults from type params (impls cannot have them). + let self_params: Vec<_> = generics + .params + .iter_mut() + .map(|param| match &mut param.kind { + ast::GenericParamKind::Lifetime => { + ast::GenericArg::Lifetime(cx.lifetime(span, param.ident)) + } + ast::GenericParamKind::Type { default } => { + *default = None; + ast::GenericArg::Type(cx.ty_ident(span, param.ident)) + } + ast::GenericParamKind::Const { ty: _ } => { + ast::GenericArg::Const(cx.const_ident(span, param.ident)) + } + }) + .collect(); + + let type_ident = item.ident; + + let trait_ref = cx.trait_ref(structural_path.to_path(cx, span, type_ident, &generics)); + let self_type = cx.ty_path(cx.path_all(span, false, vec![type_ident], self_params)); + + // It would be nice to also encode constraint `where Self: Eq` (by adding it + // onto `generics` cloned above). Unfortunately, that strategy runs afoul of + // rust-lang/rust#48214. So we perform that additional check in the compiler + // itself, instead of encoding it here. + + // Keep the lint and stability attributes of the original item, to control + // how the generated implementation is linted. + let mut attrs = Vec::new(); + attrs.extend( + item.attrs + .iter() + .filter(|a| { + [sym::allow, sym::warn, sym::deny, sym::forbid, sym::stable, sym::unstable] + .contains(&a.name_or_empty()) + }) + .cloned(), + ); + + let newitem = cx.item( + span, + ast::Ident::invalid(), + attrs, + ItemKind::Impl( + ast::Unsafety::Normal, + ast::ImplPolarity::Positive, + ast::Defaultness::Final, + generics, + Some(trait_ref), + self_type, + Vec::new(), + ), + ); + + push(Annotatable::Item(newitem)); +} diff --git a/src/librustc_builtin_macros/env.rs b/src/librustc_builtin_macros/env.rs new file mode 100644 index 00000000000..c9ecbabc8ff --- /dev/null +++ b/src/librustc_builtin_macros/env.rs @@ -0,0 +1,88 @@ +// The compiler code necessary to support the env! extension. Eventually this +// should all get sucked into either the compiler syntax extension plugin +// interface. +// + +use syntax::ast::{self, GenericArg, Ident}; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::tokenstream::TokenStream; +use syntax_expand::base::{self, *}; +use syntax_pos::Span; + +use std::env; + +pub fn expand_option_env<'cx>( + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let var = match get_single_str_from_tts(cx, sp, tts, "option_env!") { + None => return DummyResult::any(sp), + Some(v) => v, + }; + + let sp = cx.with_def_site_ctxt(sp); + let e = match env::var(&var.as_str()) { + Err(..) => { + let lt = cx.lifetime(sp, Ident::new(kw::StaticLifetime, sp)); + cx.expr_path(cx.path_all( + sp, + true, + cx.std_path(&[sym::option, sym::Option, sym::None]), + vec![GenericArg::Type(cx.ty_rptr( + sp, + cx.ty_ident(sp, Ident::new(sym::str, sp)), + Some(lt), + ast::Mutability::Not, + ))], + )) + } + Ok(s) => cx.expr_call_global( + sp, + cx.std_path(&[sym::option, sym::Option, sym::Some]), + vec![cx.expr_str(sp, Symbol::intern(&s))], + ), + }; + MacEager::expr(e) +} + +pub fn expand_env<'cx>( + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let mut exprs = match get_exprs_from_tts(cx, sp, tts) { + Some(ref exprs) if exprs.is_empty() => { + cx.span_err(sp, "env! takes 1 or 2 arguments"); + return DummyResult::any(sp); + } + None => return DummyResult::any(sp), + Some(exprs) => exprs.into_iter(), + }; + + let var = match expr_to_string(cx, exprs.next().unwrap(), "expected string literal") { + None => return DummyResult::any(sp), + Some((v, _style)) => v, + }; + let msg = match exprs.next() { + None => Symbol::intern(&format!("environment variable `{}` not defined", var)), + Some(second) => match expr_to_string(cx, second, "expected string literal") { + None => return DummyResult::any(sp), + Some((s, _style)) => s, + }, + }; + + if exprs.next().is_some() { + cx.span_err(sp, "env! takes 1 or 2 arguments"); + return DummyResult::any(sp); + } + + let e = match env::var(&*var.as_str()) { + Err(_) => { + cx.span_err(sp, &msg.as_str()); + return DummyResult::any(sp); + } + Ok(s) => cx.expr_str(sp, Symbol::intern(&s)), + }; + MacEager::expr(e) +} diff --git a/src/librustc_builtin_macros/format.rs b/src/librustc_builtin_macros/format.rs new file mode 100644 index 00000000000..1d1f68a4906 --- /dev/null +++ b/src/librustc_builtin_macros/format.rs @@ -0,0 +1,1233 @@ +use ArgumentType::*; +use Position::*; + +use fmt_macros as parse; + +use errors::pluralize; +use errors::Applicability; +use errors::DiagnosticBuilder; + +use syntax::ast; +use syntax::ptr::P; +use syntax::symbol::{sym, Symbol}; +use syntax::token; +use syntax::tokenstream::TokenStream; +use syntax_expand::base::{self, *}; +use syntax_pos::{MultiSpan, Span}; + +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use std::borrow::Cow; +use std::collections::hash_map::Entry; + +#[derive(PartialEq)] +enum ArgumentType { + Placeholder(&'static str), + Count, +} + +enum Position { + Exact(usize), + Named(Symbol), +} + +struct Context<'a, 'b> { + ecx: &'a mut ExtCtxt<'b>, + /// The macro's call site. References to unstable formatting internals must + /// use this span to pass the stability checker. + macsp: Span, + /// The span of the format string literal. + fmtsp: Span, + + /// List of parsed argument expressions. + /// Named expressions are resolved early, and are appended to the end of + /// argument expressions. + /// + /// Example showing the various data structures in motion: + /// + /// * Original: `"{foo:o} {:o} {foo:x} {0:x} {1:o} {:x} {1:x} {0:o}"` + /// * Implicit argument resolution: `"{foo:o} {0:o} {foo:x} {0:x} {1:o} {1:x} {1:x} {0:o}"` + /// * Name resolution: `"{2:o} {0:o} {2:x} {0:x} {1:o} {1:x} {1:x} {0:o}"` + /// * `arg_types` (in JSON): `[[0, 1, 0], [0, 1, 1], [0, 1]]` + /// * `arg_unique_types` (in simplified JSON): `[["o", "x"], ["o", "x"], ["o", "x"]]` + /// * `names` (in JSON): `{"foo": 2}` + args: Vec>, + /// Placeholder slot numbers indexed by argument. + arg_types: Vec>, + /// Unique format specs seen for each argument. + arg_unique_types: Vec>, + /// Map from named arguments to their resolved indices. + names: FxHashMap, + + /// The latest consecutive literal strings, or empty if there weren't any. + literal: String, + + /// Collection of the compiled `rt::Argument` structures + pieces: Vec>, + /// Collection of string literals + str_pieces: Vec>, + /// Stays `true` if all formatting parameters are default (as in "{}{}"). + all_pieces_simple: bool, + + /// Mapping between positional argument references and indices into the + /// final generated static argument array. We record the starting indices + /// corresponding to each positional argument, and number of references + /// consumed so far for each argument, to facilitate correct `Position` + /// mapping in `build_piece`. In effect this can be seen as a "flattened" + /// version of `arg_unique_types`. + /// + /// Again with the example described above in docstring for `args`: + /// + /// * `arg_index_map` (in JSON): `[[0, 1, 0], [2, 3, 3], [4, 5]]` + arg_index_map: Vec>, + + /// Starting offset of count argument slots. + count_args_index_offset: usize, + + /// Count argument slots and tracking data structures. + /// Count arguments are separately tracked for de-duplication in case + /// multiple references are made to one argument. For example, in this + /// format string: + /// + /// * Original: `"{:.*} {:.foo$} {1:.*} {:.0$}"` + /// * Implicit argument resolution: `"{1:.0$} {2:.foo$} {1:.3$} {4:.0$}"` + /// * Name resolution: `"{1:.0$} {2:.5$} {1:.3$} {4:.0$}"` + /// * `count_positions` (in JSON): `{0: 0, 5: 1, 3: 2}` + /// * `count_args`: `vec![Exact(0), Exact(5), Exact(3)]` + count_args: Vec, + /// Relative slot numbers for count arguments. + count_positions: FxHashMap, + /// Number of count slots assigned. + count_positions_count: usize, + + /// Current position of the implicit positional arg pointer, as if it + /// still existed in this phase of processing. + /// Used only for `all_pieces_simple` tracking in `build_piece`. + curarg: usize, + /// Current piece being evaluated, used for error reporting. + curpiece: usize, + /// Keep track of invalid references to positional arguments. + invalid_refs: Vec<(usize, usize)>, + /// Spans of all the formatting arguments, in order. + arg_spans: Vec, + /// All the formatting arguments that have formatting flags set, in order for diagnostics. + arg_with_formatting: Vec>, + /// Whether this formatting string is a literal or it comes from a macro. + is_literal: bool, +} + +/// Parses the arguments from the given list of tokens, returning the diagnostic +/// if there's a parse error so we can continue parsing other format! +/// expressions. +/// +/// If parsing succeeds, the return value is: +/// +/// ```text +/// Some((fmtstr, parsed arguments, index map for named arguments)) +/// ``` +fn parse_args<'a>( + ecx: &mut ExtCtxt<'a>, + sp: Span, + tts: TokenStream, +) -> Result<(P, Vec>, FxHashMap), DiagnosticBuilder<'a>> { + let mut args = Vec::>::new(); + let mut names = FxHashMap::::default(); + + let mut p = ecx.new_parser_from_tts(tts); + + if p.token == token::Eof { + return Err(ecx.struct_span_err(sp, "requires at least a format string argument")); + } + + let fmtstr = p.parse_expr()?; + let mut first = true; + let mut named = false; + + while p.token != token::Eof { + if !p.eat(&token::Comma) { + if first { + // After `format!(""` we always expect *only* a comma... + let mut err = ecx.struct_span_err(p.token.span, "expected token: `,`"); + err.span_label(p.token.span, "expected `,`"); + p.maybe_annotate_with_ascription(&mut err, false); + return Err(err); + } else { + // ...after that delegate to `expect` to also include the other expected tokens. + return Err(p.expect(&token::Comma).err().unwrap()); + } + } + first = false; + if p.token == token::Eof { + break; + } // accept trailing commas + if p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq) { + named = true; + let name = if let token::Ident(name, _) = p.token.kind { + p.bump(); + name + } else { + unreachable!(); + }; + + p.expect(&token::Eq)?; + let e = p.parse_expr()?; + if let Some(prev) = names.get(&name) { + ecx.struct_span_err(e.span, &format!("duplicate argument named `{}`", name)) + .span_label(args[*prev].span, "previously here") + .span_label(e.span, "duplicate argument") + .emit(); + continue; + } + + // Resolve names into slots early. + // Since all the positional args are already seen at this point + // if the input is valid, we can simply append to the positional + // args. And remember the names. + let slot = args.len(); + names.insert(name, slot); + args.push(e); + } else { + let e = p.parse_expr()?; + if named { + let mut err = ecx + .struct_span_err(e.span, "positional arguments cannot follow named arguments"); + err.span_label(e.span, "positional arguments must be before named arguments"); + for (_, pos) in &names { + err.span_label(args[*pos].span, "named argument"); + } + err.emit(); + } + args.push(e); + } + } + Ok((fmtstr, args, names)) +} + +impl<'a, 'b> Context<'a, 'b> { + fn resolve_name_inplace(&self, p: &mut parse::Piece<'_>) { + // NOTE: the `unwrap_or` branch is needed in case of invalid format + // arguments, e.g., `format_args!("{foo}")`. + let lookup = |s: Symbol| *self.names.get(&s).unwrap_or(&0); + + match *p { + parse::String(_) => {} + parse::NextArgument(ref mut arg) => { + if let parse::ArgumentNamed(s) = arg.position { + arg.position = parse::ArgumentIs(lookup(s)); + } + if let parse::CountIsName(s) = arg.format.width { + arg.format.width = parse::CountIsParam(lookup(s)); + } + if let parse::CountIsName(s) = arg.format.precision { + arg.format.precision = parse::CountIsParam(lookup(s)); + } + } + } + } + + /// Verifies one piece of a parse string, and remembers it if valid. + /// All errors are not emitted as fatal so we can continue giving errors + /// about this and possibly other format strings. + fn verify_piece(&mut self, p: &parse::Piece<'_>) { + match *p { + parse::String(..) => {} + parse::NextArgument(ref arg) => { + // width/precision first, if they have implicit positional + // parameters it makes more sense to consume them first. + self.verify_count(arg.format.width); + self.verify_count(arg.format.precision); + + // argument second, if it's an implicit positional parameter + // it's written second, so it should come after width/precision. + let pos = match arg.position { + parse::ArgumentIs(i) | parse::ArgumentImplicitlyIs(i) => Exact(i), + parse::ArgumentNamed(s) => Named(s), + }; + + let ty = Placeholder(match &arg.format.ty[..] { + "" => "Display", + "?" => "Debug", + "e" => "LowerExp", + "E" => "UpperExp", + "o" => "Octal", + "p" => "Pointer", + "b" => "Binary", + "x" => "LowerHex", + "X" => "UpperHex", + _ => { + let fmtsp = self.fmtsp; + let sp = arg.format.ty_span.map(|sp| fmtsp.from_inner(sp)); + let mut err = self.ecx.struct_span_err( + sp.unwrap_or(fmtsp), + &format!("unknown format trait `{}`", arg.format.ty), + ); + err.note( + "the only appropriate formatting traits are:\n\ + - ``, which uses the `Display` trait\n\ + - `?`, which uses the `Debug` trait\n\ + - `e`, which uses the `LowerExp` trait\n\ + - `E`, which uses the `UpperExp` trait\n\ + - `o`, which uses the `Octal` trait\n\ + - `p`, which uses the `Pointer` trait\n\ + - `b`, which uses the `Binary` trait\n\ + - `x`, which uses the `LowerHex` trait\n\ + - `X`, which uses the `UpperHex` trait", + ); + if let Some(sp) = sp { + for (fmt, name) in &[ + ("", "Display"), + ("?", "Debug"), + ("e", "LowerExp"), + ("E", "UpperExp"), + ("o", "Octal"), + ("p", "Pointer"), + ("b", "Binary"), + ("x", "LowerHex"), + ("X", "UpperHex"), + ] { + err.tool_only_span_suggestion( + sp, + &format!("use the `{}` trait", name), + fmt.to_string(), + Applicability::MaybeIncorrect, + ); + } + } + err.emit(); + "" + } + }); + self.verify_arg_type(pos, ty); + self.curpiece += 1; + } + } + } + + fn verify_count(&mut self, c: parse::Count) { + match c { + parse::CountImplied | parse::CountIs(..) => {} + parse::CountIsParam(i) => { + self.verify_arg_type(Exact(i), Count); + } + parse::CountIsName(s) => { + self.verify_arg_type(Named(s), Count); + } + } + } + + fn describe_num_args(&self) -> Cow<'_, str> { + match self.args.len() { + 0 => "no arguments were given".into(), + 1 => "there is 1 argument".into(), + x => format!("there are {} arguments", x).into(), + } + } + + /// Handle invalid references to positional arguments. Output different + /// errors for the case where all arguments are positional and for when + /// there are named arguments or numbered positional arguments in the + /// format string. + fn report_invalid_references(&self, numbered_position_args: bool) { + let mut e; + let sp = if self.is_literal { + // Point at the formatting arguments. + MultiSpan::from_spans(self.arg_spans.clone()) + } else { + MultiSpan::from_span(self.fmtsp) + }; + let refs = + self.invalid_refs.iter().map(|(r, pos)| (r.to_string(), self.arg_spans.get(*pos))); + + let mut zero_based_note = false; + + let count = self.pieces.len() + + self.arg_with_formatting.iter().filter(|fmt| fmt.precision_span.is_some()).count(); + if self.names.is_empty() && !numbered_position_args && count != self.args.len() { + e = self.ecx.struct_span_err( + sp, + &format!( + "{} positional argument{} in format string, but {}", + count, + pluralize!(count), + self.describe_num_args(), + ), + ); + for arg in &self.args { + // Point at the arguments that will be formatted. + e.span_label(arg.span, ""); + } + } else { + let (mut refs, spans): (Vec<_>, Vec<_>) = refs.unzip(); + // Avoid `invalid reference to positional arguments 7 and 7 (there is 1 argument)` + // for `println!("{7:7$}", 1);` + refs.sort(); + refs.dedup(); + let (arg_list, mut sp) = if refs.len() == 1 { + let spans: Vec<_> = spans.into_iter().filter_map(|sp| sp.map(|sp| *sp)).collect(); + ( + format!("argument {}", refs[0]), + if spans.is_empty() { + MultiSpan::from_span(self.fmtsp) + } else { + MultiSpan::from_spans(spans) + }, + ) + } else { + let pos = MultiSpan::from_spans(spans.into_iter().map(|s| *s.unwrap()).collect()); + let reg = refs.pop().unwrap(); + (format!("arguments {head} and {tail}", head = refs.join(", "), tail = reg,), pos) + }; + if !self.is_literal { + sp = MultiSpan::from_span(self.fmtsp); + } + + e = self.ecx.struct_span_err( + sp, + &format!( + "invalid reference to positional {} ({})", + arg_list, + self.describe_num_args() + ), + ); + zero_based_note = true; + }; + + for fmt in &self.arg_with_formatting { + if let Some(span) = fmt.precision_span { + let span = self.fmtsp.from_inner(span); + match fmt.precision { + parse::CountIsParam(pos) if pos > self.args.len() => { + e.span_label( + span, + &format!( + "this precision flag expects an `usize` argument at position {}, \ + but {}", + pos, + self.describe_num_args(), + ), + ); + zero_based_note = true; + } + parse::CountIsParam(pos) => { + let count = self.pieces.len() + + self + .arg_with_formatting + .iter() + .filter(|fmt| fmt.precision_span.is_some()) + .count(); + e.span_label(span, &format!( + "this precision flag adds an extra required argument at position {}, \ + which is why there {} expected", + pos, + if count == 1 { + "is 1 argument".to_string() + } else { + format!("are {} arguments", count) + }, + )); + if let Some(arg) = self.args.get(pos) { + e.span_label( + arg.span, + "this parameter corresponds to the precision flag", + ); + } + zero_based_note = true; + } + _ => {} + } + } + if let Some(span) = fmt.width_span { + let span = self.fmtsp.from_inner(span); + match fmt.width { + parse::CountIsParam(pos) if pos > self.args.len() => { + e.span_label( + span, + &format!( + "this width flag expects an `usize` argument at position {}, \ + but {}", + pos, + self.describe_num_args(), + ), + ); + zero_based_note = true; + } + _ => {} + } + } + } + if zero_based_note { + e.note("positional arguments are zero-based"); + } + if !self.arg_with_formatting.is_empty() { + e.note( + "for information about formatting flags, visit \ + https://doc.rust-lang.org/std/fmt/index.html", + ); + } + + e.emit(); + } + + /// Actually verifies and tracks a given format placeholder + /// (a.k.a. argument). + fn verify_arg_type(&mut self, arg: Position, ty: ArgumentType) { + match arg { + Exact(arg) => { + if self.args.len() <= arg { + self.invalid_refs.push((arg, self.curpiece)); + return; + } + match ty { + Placeholder(_) => { + // record every (position, type) combination only once + let ref mut seen_ty = self.arg_unique_types[arg]; + let i = seen_ty.iter().position(|x| *x == ty).unwrap_or_else(|| { + let i = seen_ty.len(); + seen_ty.push(ty); + i + }); + self.arg_types[arg].push(i); + } + Count => { + if let Entry::Vacant(e) = self.count_positions.entry(arg) { + let i = self.count_positions_count; + e.insert(i); + self.count_args.push(Exact(arg)); + self.count_positions_count += 1; + } + } + } + } + + Named(name) => { + match self.names.get(&name) { + Some(&idx) => { + // Treat as positional arg. + self.verify_arg_type(Exact(idx), ty) + } + None => { + let msg = format!("there is no argument named `{}`", name); + let sp = if self.is_literal { + *self.arg_spans.get(self.curpiece).unwrap_or(&self.fmtsp) + } else { + self.fmtsp + }; + let mut err = self.ecx.struct_span_err(sp, &msg[..]); + err.emit(); + } + } + } + } + } + + /// Builds the mapping between format placeholders and argument objects. + fn build_index_map(&mut self) { + // NOTE: Keep the ordering the same as `into_expr`'s expansion would do! + let args_len = self.args.len(); + self.arg_index_map.reserve(args_len); + + let mut sofar = 0usize; + + // Map the arguments + for i in 0..args_len { + let ref arg_types = self.arg_types[i]; + let arg_offsets = arg_types.iter().map(|offset| sofar + *offset).collect::>(); + self.arg_index_map.push(arg_offsets); + sofar += self.arg_unique_types[i].len(); + } + + // Record starting index for counts, which appear just after arguments + self.count_args_index_offset = sofar; + } + + fn rtpath(ecx: &ExtCtxt<'_>, s: &str) -> Vec { + ecx.std_path(&[sym::fmt, sym::rt, sym::v1, Symbol::intern(s)]) + } + + fn build_count(&self, c: parse::Count) -> P { + let sp = self.macsp; + let count = |c, arg| { + let mut path = Context::rtpath(self.ecx, "Count"); + path.push(self.ecx.ident_of(c, sp)); + match arg { + Some(arg) => self.ecx.expr_call_global(sp, path, vec![arg]), + None => self.ecx.expr_path(self.ecx.path_global(sp, path)), + } + }; + match c { + parse::CountIs(i) => count("Is", Some(self.ecx.expr_usize(sp, i))), + parse::CountIsParam(i) => { + // This needs mapping too, as `i` is referring to a macro + // argument. If `i` is not found in `count_positions` then + // the error had already been emitted elsewhere. + let i = self.count_positions.get(&i).cloned().unwrap_or(0) + + self.count_args_index_offset; + count("Param", Some(self.ecx.expr_usize(sp, i))) + } + parse::CountImplied => count("Implied", None), + // should never be the case, names are already resolved + parse::CountIsName(_) => panic!("should never happen"), + } + } + + /// Build a literal expression from the accumulated string literals + fn build_literal_string(&mut self) -> P { + let sp = self.fmtsp; + let s = Symbol::intern(&self.literal); + self.literal.clear(); + self.ecx.expr_str(sp, s) + } + + /// Builds a static `rt::Argument` from a `parse::Piece` or append + /// to the `literal` string. + fn build_piece( + &mut self, + piece: &parse::Piece<'a>, + arg_index_consumed: &mut Vec, + ) -> Option> { + let sp = self.macsp; + match *piece { + parse::String(s) => { + self.literal.push_str(s); + None + } + parse::NextArgument(ref arg) => { + // Build the position + let pos = { + let pos = |c, arg| { + let mut path = Context::rtpath(self.ecx, "Position"); + path.push(self.ecx.ident_of(c, sp)); + match arg { + Some(i) => { + let arg = self.ecx.expr_usize(sp, i); + self.ecx.expr_call_global(sp, path, vec![arg]) + } + None => self.ecx.expr_path(self.ecx.path_global(sp, path)), + } + }; + match arg.position { + parse::ArgumentIs(i) | parse::ArgumentImplicitlyIs(i) => { + // Map to index in final generated argument array + // in case of multiple types specified + let arg_idx = match arg_index_consumed.get_mut(i) { + None => 0, // error already emitted elsewhere + Some(offset) => { + let ref idx_map = self.arg_index_map[i]; + // unwrap_or branch: error already emitted elsewhere + let arg_idx = *idx_map.get(*offset).unwrap_or(&0); + *offset += 1; + arg_idx + } + }; + pos("At", Some(arg_idx)) + } + + // should never be the case, because names are already + // resolved. + parse::ArgumentNamed(_) => panic!("should never happen"), + } + }; + + let simple_arg = parse::Argument { + position: { + // We don't have ArgumentNext any more, so we have to + // track the current argument ourselves. + let i = self.curarg; + self.curarg += 1; + parse::ArgumentIs(i) + }, + format: parse::FormatSpec { + fill: arg.format.fill, + align: parse::AlignUnknown, + flags: 0, + precision: parse::CountImplied, + precision_span: None, + width: parse::CountImplied, + width_span: None, + ty: arg.format.ty, + ty_span: arg.format.ty_span, + }, + }; + + let fill = arg.format.fill.unwrap_or(' '); + + let pos_simple = arg.position.index() == simple_arg.position.index(); + + if arg.format.precision_span.is_some() || arg.format.width_span.is_some() { + self.arg_with_formatting.push(arg.format); + } + if !pos_simple || arg.format != simple_arg.format || fill != ' ' { + self.all_pieces_simple = false; + } + + // Build the format + let fill = self.ecx.expr_lit(sp, ast::LitKind::Char(fill)); + let align = |name| { + let mut p = Context::rtpath(self.ecx, "Alignment"); + p.push(self.ecx.ident_of(name, sp)); + self.ecx.path_global(sp, p) + }; + let align = match arg.format.align { + parse::AlignLeft => align("Left"), + parse::AlignRight => align("Right"), + parse::AlignCenter => align("Center"), + parse::AlignUnknown => align("Unknown"), + }; + let align = self.ecx.expr_path(align); + let flags = self.ecx.expr_u32(sp, arg.format.flags); + let prec = self.build_count(arg.format.precision); + let width = self.build_count(arg.format.width); + let path = self.ecx.path_global(sp, Context::rtpath(self.ecx, "FormatSpec")); + let fmt = self.ecx.expr_struct( + sp, + path, + vec![ + self.ecx.field_imm(sp, self.ecx.ident_of("fill", sp), fill), + self.ecx.field_imm(sp, self.ecx.ident_of("align", sp), align), + self.ecx.field_imm(sp, self.ecx.ident_of("flags", sp), flags), + self.ecx.field_imm(sp, self.ecx.ident_of("precision", sp), prec), + self.ecx.field_imm(sp, self.ecx.ident_of("width", sp), width), + ], + ); + + let path = self.ecx.path_global(sp, Context::rtpath(self.ecx, "Argument")); + Some(self.ecx.expr_struct( + sp, + path, + vec![ + self.ecx.field_imm(sp, self.ecx.ident_of("position", sp), pos), + self.ecx.field_imm(sp, self.ecx.ident_of("format", sp), fmt), + ], + )) + } + } + } + + /// Actually builds the expression which the format_args! block will be + /// expanded to. + fn into_expr(self) -> P { + let mut locals = + Vec::with_capacity((0..self.args.len()).map(|i| self.arg_unique_types[i].len()).sum()); + let mut counts = Vec::with_capacity(self.count_args.len()); + let mut pats = Vec::with_capacity(self.args.len()); + let mut heads = Vec::with_capacity(self.args.len()); + + let names_pos: Vec<_> = (0..self.args.len()) + .map(|i| self.ecx.ident_of(&format!("arg{}", i), self.macsp)) + .collect(); + + // First, build up the static array which will become our precompiled + // format "string" + let pieces = self.ecx.expr_vec_slice(self.fmtsp, self.str_pieces); + + // Before consuming the expressions, we have to remember spans for + // count arguments as they are now generated separate from other + // arguments, hence have no access to the `P`'s. + let spans_pos: Vec<_> = self.args.iter().map(|e| e.span.clone()).collect(); + + // Right now there is a bug such that for the expression: + // foo(bar(&1)) + // the lifetime of `1` doesn't outlast the call to `bar`, so it's not + // valid for the call to `foo`. To work around this all arguments to the + // format! string are shoved into locals. Furthermore, we shove the address + // of each variable because we don't want to move out of the arguments + // passed to this function. + for (i, e) in self.args.into_iter().enumerate() { + let name = names_pos[i]; + let span = self.ecx.with_def_site_ctxt(e.span); + pats.push(self.ecx.pat_ident(span, name)); + for ref arg_ty in self.arg_unique_types[i].iter() { + locals.push(Context::format_arg(self.ecx, self.macsp, e.span, arg_ty, name)); + } + heads.push(self.ecx.expr_addr_of(e.span, e)); + } + for pos in self.count_args { + let index = match pos { + Exact(i) => i, + _ => panic!("should never happen"), + }; + let name = names_pos[index]; + let span = spans_pos[index]; + counts.push(Context::format_arg(self.ecx, self.macsp, span, &Count, name)); + } + + // Now create a vector containing all the arguments + let args = locals.into_iter().chain(counts.into_iter()); + + let args_array = self.ecx.expr_vec(self.macsp, args.collect()); + + // Constructs an AST equivalent to: + // + // match (&arg0, &arg1) { + // (tmp0, tmp1) => args_array + // } + // + // It was: + // + // let tmp0 = &arg0; + // let tmp1 = &arg1; + // args_array + // + // Because of #11585 the new temporary lifetime rule, the enclosing + // statements for these temporaries become the let's themselves. + // If one or more of them are RefCell's, RefCell borrow() will also + // end there; they don't last long enough for args_array to use them. + // The match expression solves the scope problem. + // + // Note, it may also very well be transformed to: + // + // match arg0 { + // ref tmp0 => { + // match arg1 => { + // ref tmp1 => args_array } } } + // + // But the nested match expression is proved to perform not as well + // as series of let's; the first approach does. + let pat = self.ecx.pat_tuple(self.macsp, pats); + let arm = self.ecx.arm(self.macsp, pat, args_array); + let head = self.ecx.expr(self.macsp, ast::ExprKind::Tup(heads)); + let result = self.ecx.expr_match(self.macsp, head, vec![arm]); + + let args_slice = self.ecx.expr_addr_of(self.macsp, result); + + // Now create the fmt::Arguments struct with all our locals we created. + let (fn_name, fn_args) = if self.all_pieces_simple { + ("new_v1", vec![pieces, args_slice]) + } else { + // Build up the static array which will store our precompiled + // nonstandard placeholders, if there are any. + let fmt = self.ecx.expr_vec_slice(self.macsp, self.pieces); + + ("new_v1_formatted", vec![pieces, args_slice, fmt]) + }; + + let path = self.ecx.std_path(&[sym::fmt, sym::Arguments, Symbol::intern(fn_name)]); + self.ecx.expr_call_global(self.macsp, path, fn_args) + } + + fn format_arg( + ecx: &ExtCtxt<'_>, + macsp: Span, + mut sp: Span, + ty: &ArgumentType, + arg: ast::Ident, + ) -> P { + sp = ecx.with_def_site_ctxt(sp); + let arg = ecx.expr_ident(sp, arg); + let trait_ = match *ty { + Placeholder(trait_) if trait_ == "" => return DummyResult::raw_expr(sp, true), + Placeholder(trait_) => trait_, + Count => { + let path = ecx.std_path(&[sym::fmt, sym::ArgumentV1, sym::from_usize]); + return ecx.expr_call_global(macsp, path, vec![arg]); + } + }; + + let path = ecx.std_path(&[sym::fmt, Symbol::intern(trait_), sym::fmt]); + let format_fn = ecx.path_global(sp, path); + let path = ecx.std_path(&[sym::fmt, sym::ArgumentV1, sym::new]); + ecx.expr_call_global(macsp, path, vec![arg, ecx.expr_path(format_fn)]) + } +} + +fn expand_format_args_impl<'cx>( + ecx: &'cx mut ExtCtxt<'_>, + mut sp: Span, + tts: TokenStream, + nl: bool, +) -> Box { + sp = ecx.with_def_site_ctxt(sp); + match parse_args(ecx, sp, tts) { + Ok((efmt, args, names)) => { + MacEager::expr(expand_preparsed_format_args(ecx, sp, efmt, args, names, nl)) + } + Err(mut err) => { + err.emit(); + DummyResult::any(sp) + } + } +} + +pub fn expand_format_args<'cx>( + ecx: &'cx mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + expand_format_args_impl(ecx, sp, tts, false) +} + +pub fn expand_format_args_nl<'cx>( + ecx: &'cx mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + expand_format_args_impl(ecx, sp, tts, true) +} + +/// Take the various parts of `format_args!(efmt, args..., name=names...)` +/// and construct the appropriate formatting expression. +pub fn expand_preparsed_format_args( + ecx: &mut ExtCtxt<'_>, + sp: Span, + efmt: P, + args: Vec>, + names: FxHashMap, + append_newline: bool, +) -> P { + // NOTE: this verbose way of initializing `Vec>` is because + // `ArgumentType` does not derive `Clone`. + let arg_types: Vec<_> = (0..args.len()).map(|_| Vec::new()).collect(); + let arg_unique_types: Vec<_> = (0..args.len()).map(|_| Vec::new()).collect(); + + let mut macsp = ecx.call_site(); + macsp = ecx.with_def_site_ctxt(macsp); + + let msg = "format argument must be a string literal"; + let fmt_sp = efmt.span; + let (fmt_str, fmt_style, fmt_span) = match expr_to_spanned_string(ecx, efmt, msg) { + Ok(mut fmt) if append_newline => { + fmt.0 = Symbol::intern(&format!("{}\n", fmt.0)); + fmt + } + Ok(fmt) => fmt, + Err(err) => { + if let Some(mut err) = err { + let sugg_fmt = match args.len() { + 0 => "{}".to_string(), + _ => format!("{}{{}}", "{} ".repeat(args.len())), + }; + err.span_suggestion( + fmt_sp.shrink_to_lo(), + "you might be missing a string literal to format with", + format!("\"{}\", ", sugg_fmt), + Applicability::MaybeIncorrect, + ); + err.emit(); + } + return DummyResult::raw_expr(sp, true); + } + }; + + let (is_literal, fmt_snippet) = match ecx.source_map().span_to_snippet(fmt_sp) { + Ok(s) => (s.starts_with("\"") || s.starts_with("r#"), Some(s)), + _ => (false, None), + }; + + let str_style = match fmt_style { + ast::StrStyle::Cooked => None, + ast::StrStyle::Raw(raw) => Some(raw as usize), + }; + + /// Finds the indices of all characters that have been processed and differ between the actual + /// written code (code snippet) and the `InternedString` that get's processed in the `Parser` + /// in order to properly synthethise the intra-string `Span`s for error diagnostics. + fn find_skips(snippet: &str, is_raw: bool) -> Vec { + let mut eat_ws = false; + let mut s = snippet.chars().enumerate().peekable(); + let mut skips = vec![]; + while let Some((pos, c)) = s.next() { + match (c, s.peek()) { + // skip whitespace and empty lines ending in '\\' + ('\\', Some((next_pos, '\n'))) if !is_raw => { + eat_ws = true; + skips.push(pos); + skips.push(*next_pos); + let _ = s.next(); + } + ('\\', Some((next_pos, '\n'))) + | ('\\', Some((next_pos, 'n'))) + | ('\\', Some((next_pos, 't'))) + if eat_ws => + { + skips.push(pos); + skips.push(*next_pos); + let _ = s.next(); + } + (' ', _) | ('\n', _) | ('\t', _) if eat_ws => { + skips.push(pos); + } + ('\\', Some((next_pos, 'n'))) + | ('\\', Some((next_pos, 't'))) + | ('\\', Some((next_pos, '0'))) + | ('\\', Some((next_pos, '\\'))) + | ('\\', Some((next_pos, '\''))) + | ('\\', Some((next_pos, '\"'))) => { + skips.push(*next_pos); + let _ = s.next(); + } + ('\\', Some((_, 'x'))) if !is_raw => { + for _ in 0..3 { + // consume `\xAB` literal + if let Some((pos, _)) = s.next() { + skips.push(pos); + } else { + break; + } + } + } + ('\\', Some((_, 'u'))) if !is_raw => { + if let Some((pos, _)) = s.next() { + skips.push(pos); + } + if let Some((next_pos, next_c)) = s.next() { + if next_c == '{' { + skips.push(next_pos); + let mut i = 0; // consume up to 6 hexanumeric chars + closing `}` + while let (Some((next_pos, c)), true) = (s.next(), i < 7) { + if c.is_digit(16) { + skips.push(next_pos); + } else if c == '}' { + skips.push(next_pos); + break; + } else { + break; + } + i += 1; + } + } else if next_c.is_digit(16) { + skips.push(next_pos); + // We suggest adding `{` and `}` when appropriate, accept it here as if + // it were correct + let mut i = 0; // consume up to 6 hexanumeric chars + while let (Some((next_pos, c)), _) = (s.next(), i < 6) { + if c.is_digit(16) { + skips.push(next_pos); + } else { + break; + } + i += 1; + } + } + } + } + _ if eat_ws => { + // `take_while(|c| c.is_whitespace())` + eat_ws = false; + } + _ => {} + } + } + skips + } + + let skips = if let (true, Some(ref snippet)) = (is_literal, fmt_snippet.as_ref()) { + let r_start = str_style.map(|r| r + 1).unwrap_or(0); + let r_end = str_style.map(|r| r).unwrap_or(0); + let s = &snippet[r_start + 1..snippet.len() - r_end - 1]; + find_skips(s, str_style.is_some()) + } else { + vec![] + }; + + let fmt_str = &fmt_str.as_str(); // for the suggestions below + let mut parser = parse::Parser::new(fmt_str, str_style, skips, append_newline); + + let mut unverified_pieces = Vec::new(); + while let Some(piece) = parser.next() { + if !parser.errors.is_empty() { + break; + } else { + unverified_pieces.push(piece); + } + } + + if !parser.errors.is_empty() { + let err = parser.errors.remove(0); + let sp = fmt_span.from_inner(err.span); + let mut e = ecx.struct_span_err(sp, &format!("invalid format string: {}", err.description)); + e.span_label(sp, err.label + " in format string"); + if let Some(note) = err.note { + e.note(¬e); + } + if let Some((label, span)) = err.secondary_label { + let sp = fmt_span.from_inner(span); + e.span_label(sp, label); + } + e.emit(); + return DummyResult::raw_expr(sp, true); + } + + let arg_spans = parser.arg_places.iter().map(|span| fmt_span.from_inner(*span)).collect(); + + let named_pos: FxHashSet = names.values().cloned().collect(); + + let mut cx = Context { + ecx, + args, + arg_types, + arg_unique_types, + names, + curarg: 0, + curpiece: 0, + arg_index_map: Vec::new(), + count_args: Vec::new(), + count_positions: FxHashMap::default(), + count_positions_count: 0, + count_args_index_offset: 0, + literal: String::new(), + pieces: Vec::with_capacity(unverified_pieces.len()), + str_pieces: Vec::with_capacity(unverified_pieces.len()), + all_pieces_simple: true, + macsp, + fmtsp: fmt_span, + invalid_refs: Vec::new(), + arg_spans, + arg_with_formatting: Vec::new(), + is_literal, + }; + + // This needs to happen *after* the Parser has consumed all pieces to create all the spans + let pieces = unverified_pieces + .into_iter() + .map(|mut piece| { + cx.verify_piece(&piece); + cx.resolve_name_inplace(&mut piece); + piece + }) + .collect::>(); + + let numbered_position_args = pieces.iter().any(|arg: &parse::Piece<'_>| match *arg { + parse::String(_) => false, + parse::NextArgument(arg) => match arg.position { + parse::Position::ArgumentIs(_) => true, + _ => false, + }, + }); + + cx.build_index_map(); + + let mut arg_index_consumed = vec![0usize; cx.arg_index_map.len()]; + + for piece in pieces { + if let Some(piece) = cx.build_piece(&piece, &mut arg_index_consumed) { + let s = cx.build_literal_string(); + cx.str_pieces.push(s); + cx.pieces.push(piece); + } + } + + if !cx.literal.is_empty() { + let s = cx.build_literal_string(); + cx.str_pieces.push(s); + } + + if cx.invalid_refs.len() >= 1 { + cx.report_invalid_references(numbered_position_args); + } + + // Make sure that all arguments were used and all arguments have types. + let errs = cx + .arg_types + .iter() + .enumerate() + .filter(|(i, ty)| ty.is_empty() && !cx.count_positions.contains_key(&i)) + .map(|(i, _)| { + let msg = if named_pos.contains(&i) { + // named argument + "named argument never used" + } else { + // positional argument + "argument never used" + }; + (cx.args[i].span, msg) + }) + .collect::>(); + + let errs_len = errs.len(); + if !errs.is_empty() { + let args_used = cx.arg_types.len() - errs_len; + let args_unused = errs_len; + + let mut diag = { + if errs_len == 1 { + let (sp, msg) = errs.into_iter().next().unwrap(); + let mut diag = cx.ecx.struct_span_err(sp, msg); + diag.span_label(sp, msg); + diag + } else { + let mut diag = cx.ecx.struct_span_err( + errs.iter().map(|&(sp, _)| sp).collect::>(), + "multiple unused formatting arguments", + ); + diag.span_label(cx.fmtsp, "multiple missing formatting specifiers"); + for (sp, msg) in errs { + diag.span_label(sp, msg); + } + diag + } + }; + + // Used to ensure we only report translations for *one* kind of foreign format. + let mut found_foreign = false; + // Decide if we want to look for foreign formatting directives. + if args_used < args_unused { + use super::format_foreign as foreign; + + // The set of foreign substitutions we've explained. This prevents spamming the user + // with `%d should be written as {}` over and over again. + let mut explained = FxHashSet::default(); + + macro_rules! check_foreign { + ($kind:ident) => {{ + let mut show_doc_note = false; + + let mut suggestions = vec![]; + // account for `"` and account for raw strings `r#` + let padding = str_style.map(|i| i + 2).unwrap_or(1); + for sub in foreign::$kind::iter_subs(fmt_str, padding) { + let trn = match sub.translate() { + Some(trn) => trn, + + // If it has no translation, don't call it out specifically. + None => continue, + }; + + let pos = sub.position(); + let sub = String::from(sub.as_str()); + if explained.contains(&sub) { + continue; + } + explained.insert(sub.clone()); + + if !found_foreign { + found_foreign = true; + show_doc_note = true; + } + + if let Some(inner_sp) = pos { + let sp = fmt_sp.from_inner(inner_sp); + suggestions.push((sp, trn)); + } else { + diag.help(&format!("`{}` should be written as `{}`", sub, trn)); + } + } + + if show_doc_note { + diag.note(concat!( + stringify!($kind), + " formatting not supported; see the documentation for `std::fmt`", + )); + } + if suggestions.len() > 0 { + diag.multipart_suggestion( + "format specifiers use curly braces", + suggestions, + Applicability::MachineApplicable, + ); + } + }}; + } + + check_foreign!(printf); + if !found_foreign { + check_foreign!(shell); + } + } + if !found_foreign && errs_len == 1 { + diag.span_label(cx.fmtsp, "formatting specifier missing"); + } + + diag.emit(); + } + + cx.into_expr() +} diff --git a/src/librustc_builtin_macros/format_foreign.rs b/src/librustc_builtin_macros/format_foreign.rs new file mode 100644 index 00000000000..9c151cf94b4 --- /dev/null +++ b/src/librustc_builtin_macros/format_foreign.rs @@ -0,0 +1,827 @@ +pub mod printf { + use super::strcursor::StrCursor as Cur; + use syntax_pos::InnerSpan; + + /// Represents a single `printf`-style substitution. + #[derive(Clone, PartialEq, Debug)] + pub enum Substitution<'a> { + /// A formatted output substitution with its internal byte offset. + Format(Format<'a>), + /// A literal `%%` escape. + Escape, + } + + impl<'a> Substitution<'a> { + pub fn as_str(&self) -> &str { + match *self { + Substitution::Format(ref fmt) => fmt.span, + Substitution::Escape => "%%", + } + } + + pub fn position(&self) -> Option { + match *self { + Substitution::Format(ref fmt) => Some(fmt.position), + _ => None, + } + } + + pub fn set_position(&mut self, start: usize, end: usize) { + match self { + Substitution::Format(ref mut fmt) => { + fmt.position = InnerSpan::new(start, end); + } + _ => {} + } + } + + /// Translate this substitution into an equivalent Rust formatting directive. + /// + /// This ignores cases where the substitution does not have an exact equivalent, or where + /// the substitution would be unnecessary. + pub fn translate(&self) -> Option { + match *self { + Substitution::Format(ref fmt) => fmt.translate(), + Substitution::Escape => None, + } + } + } + + #[derive(Clone, PartialEq, Debug)] + /// A single `printf`-style formatting directive. + pub struct Format<'a> { + /// The entire original formatting directive. + pub span: &'a str, + /// The (1-based) parameter to be converted. + pub parameter: Option, + /// Formatting flags. + pub flags: &'a str, + /// Minimum width of the output. + pub width: Option, + /// Precision of the conversion. + pub precision: Option, + /// Length modifier for the conversion. + pub length: Option<&'a str>, + /// Type of parameter being converted. + pub type_: &'a str, + /// Byte offset for the start and end of this formatting directive. + pub position: InnerSpan, + } + + impl Format<'_> { + /// Translate this directive into an equivalent Rust formatting directive. + /// + /// Returns `None` in cases where the `printf` directive does not have an exact Rust + /// equivalent, rather than guessing. + pub fn translate(&self) -> Option { + use std::fmt::Write; + + let (c_alt, c_zero, c_left, c_plus) = { + let mut c_alt = false; + let mut c_zero = false; + let mut c_left = false; + let mut c_plus = false; + for c in self.flags.chars() { + match c { + '#' => c_alt = true, + '0' => c_zero = true, + '-' => c_left = true, + '+' => c_plus = true, + _ => return None, + } + } + (c_alt, c_zero, c_left, c_plus) + }; + + // Has a special form in Rust for numbers. + let fill = c_zero.then_some("0"); + + let align = c_left.then_some("<"); + + // Rust doesn't have an equivalent to the `' '` flag. + let sign = c_plus.then_some("+"); + + // Not *quite* the same, depending on the type... + let alt = c_alt; + + let width = match self.width { + Some(Num::Next) => { + // NOTE: Rust doesn't support this. + return None; + } + w @ Some(Num::Arg(_)) => w, + w @ Some(Num::Num(_)) => w, + None => None, + }; + + let precision = self.precision; + + // NOTE: although length *can* have an effect, we can't duplicate the effect in Rust, so + // we just ignore it. + + let (type_, use_zero_fill, is_int) = match self.type_ { + "d" | "i" | "u" => (None, true, true), + "f" | "F" => (None, false, false), + "s" | "c" => (None, false, false), + "e" | "E" => (Some(self.type_), true, false), + "x" | "X" | "o" => (Some(self.type_), true, true), + "p" => (Some(self.type_), false, true), + "g" => (Some("e"), true, false), + "G" => (Some("E"), true, false), + _ => return None, + }; + + let (fill, width, precision) = match (is_int, width, precision) { + (true, Some(_), Some(_)) => { + // Rust can't duplicate this insanity. + return None; + } + (true, None, Some(p)) => (Some("0"), Some(p), None), + (true, w, None) => (fill, w, None), + (false, w, p) => (fill, w, p), + }; + + let align = match (self.type_, width.is_some(), align.is_some()) { + ("s", true, false) => Some(">"), + _ => align, + }; + + let (fill, zero_fill) = match (fill, use_zero_fill) { + (Some("0"), true) => (None, true), + (fill, _) => (fill, false), + }; + + let alt = match type_ { + Some("x") | Some("X") => alt, + _ => false, + }; + + let has_options = fill.is_some() + || align.is_some() + || sign.is_some() + || alt + || zero_fill + || width.is_some() + || precision.is_some() + || type_.is_some(); + + // Initialise with a rough guess. + let cap = self.span.len() + if has_options { 2 } else { 0 }; + let mut s = String::with_capacity(cap); + + s.push_str("{"); + + if let Some(arg) = self.parameter { + write!(s, "{}", arg.checked_sub(1)?).ok()?; + } + + if has_options { + s.push_str(":"); + + let align = if let Some(fill) = fill { + s.push_str(fill); + align.or(Some(">")) + } else { + align + }; + + if let Some(align) = align { + s.push_str(align); + } + + if let Some(sign) = sign { + s.push_str(sign); + } + + if alt { + s.push_str("#"); + } + + if zero_fill { + s.push_str("0"); + } + + if let Some(width) = width { + width.translate(&mut s).ok()?; + } + + if let Some(precision) = precision { + s.push_str("."); + precision.translate(&mut s).ok()?; + } + + if let Some(type_) = type_ { + s.push_str(type_); + } + } + + s.push_str("}"); + Some(s) + } + } + + /// A general number used in a `printf` formatting directive. + #[derive(Copy, Clone, PartialEq, Debug)] + pub enum Num { + // The range of these values is technically bounded by `NL_ARGMAX`... but, at least for GNU + // libc, it apparently has no real fixed limit. A `u16` is used here on the basis that it + // is *vanishingly* unlikely that *anyone* is going to try formatting something wider, or + // with more precision, than 32 thousand positions which is so wide it couldn't possibly fit + // on a screen. + /// A specific, fixed value. + Num(u16), + /// The value is derived from a positional argument. + Arg(u16), + /// The value is derived from the "next" unconverted argument. + Next, + } + + impl Num { + fn from_str(s: &str, arg: Option<&str>) -> Self { + if let Some(arg) = arg { + Num::Arg(arg.parse().unwrap_or_else(|_| panic!("invalid format arg `{:?}`", arg))) + } else if s == "*" { + Num::Next + } else { + Num::Num(s.parse().unwrap_or_else(|_| panic!("invalid format num `{:?}`", s))) + } + } + + fn translate(&self, s: &mut String) -> std::fmt::Result { + use std::fmt::Write; + match *self { + Num::Num(n) => write!(s, "{}", n), + Num::Arg(n) => { + let n = n.checked_sub(1).ok_or(std::fmt::Error)?; + write!(s, "{}$", n) + } + Num::Next => write!(s, "*"), + } + } + } + + /// Returns an iterator over all substitutions in a given string. + pub fn iter_subs(s: &str, start_pos: usize) -> Substitutions<'_> { + Substitutions { s, pos: start_pos } + } + + /// Iterator over substitutions in a string. + pub struct Substitutions<'a> { + s: &'a str, + pos: usize, + } + + impl<'a> Iterator for Substitutions<'a> { + type Item = Substitution<'a>; + fn next(&mut self) -> Option { + let (mut sub, tail) = parse_next_substitution(self.s)?; + self.s = tail; + match sub { + Substitution::Format(_) => { + if let Some(inner_span) = sub.position() { + sub.set_position(inner_span.start + self.pos, inner_span.end + self.pos); + self.pos += inner_span.end; + } + } + Substitution::Escape => self.pos += 2, + } + Some(sub) + } + + fn size_hint(&self) -> (usize, Option) { + // Substitutions are at least 2 characters long. + (0, Some(self.s.len() / 2)) + } + } + + enum State { + Start, + Flags, + Width, + WidthArg, + Prec, + PrecInner, + Length, + Type, + } + + /// Parse the next substitution from the input string. + pub fn parse_next_substitution(s: &str) -> Option<(Substitution<'_>, &str)> { + use self::State::*; + + let at = { + let start = s.find('%')?; + match s[start + 1..].chars().next()? { + '%' => return Some((Substitution::Escape, &s[start + 2..])), + _ => { /* fall-through */ } + } + + Cur::new_at(&s[..], start) + }; + + // This is meant to be a translation of the following regex: + // + // ```regex + // (?x) + // ^ % + // (?: (?P \d+) \$ )? + // (?P [-+ 0\#']* ) + // (?P \d+ | \* (?: (?P \d+) \$ )? )? + // (?: \. (?P \d+ | \* (?: (?P \d+) \$ )? ) )? + // (?P + // # Standard + // hh | h | ll | l | L | z | j | t + // + // # Other + // | I32 | I64 | I | q + // )? + // (?P . ) + // ``` + + // Used to establish the full span at the end. + let start = at; + // The current position within the string. + let mut at = at.at_next_cp()?; + // `c` is the next codepoint, `next` is a cursor after it. + let (mut c, mut next) = at.next_cp()?; + + // Update `at`, `c`, and `next`, exiting if we're out of input. + macro_rules! move_to { + ($cur:expr) => {{ + at = $cur; + let (c_, next_) = at.next_cp()?; + c = c_; + next = next_; + }}; + } + + // Constructs a result when parsing fails. + // + // Note: `move` used to capture copies of the cursors as they are *now*. + let fallback = move || { + return Some(( + Substitution::Format(Format { + span: start.slice_between(next).unwrap(), + parameter: None, + flags: "", + width: None, + precision: None, + length: None, + type_: at.slice_between(next).unwrap(), + position: InnerSpan::new(start.at, next.at), + }), + next.slice_after(), + )); + }; + + // Next parsing state. + let mut state = Start; + + // Sadly, Rust isn't *quite* smart enough to know these *must* be initialised by the end. + let mut parameter: Option = None; + let mut flags: &str = ""; + let mut width: Option = None; + let mut precision: Option = None; + let mut length: Option<&str> = None; + let mut type_: &str = ""; + let end: Cur<'_>; + + if let Start = state { + match c { + '1'..='9' => { + let end = at_next_cp_while(next, is_digit); + match end.next_cp() { + // Yes, this *is* the parameter. + Some(('$', end2)) => { + state = Flags; + parameter = Some(at.slice_between(end).unwrap().parse().unwrap()); + move_to!(end2); + } + // Wait, no, actually, it's the width. + Some(_) => { + state = Prec; + parameter = None; + flags = ""; + width = Some(Num::from_str(at.slice_between(end).unwrap(), None)); + move_to!(end); + } + // It's invalid, is what it is. + None => return fallback(), + } + } + _ => { + state = Flags; + parameter = None; + move_to!(at); + } + } + } + + if let Flags = state { + let end = at_next_cp_while(at, is_flag); + state = Width; + flags = at.slice_between(end).unwrap(); + move_to!(end); + } + + if let Width = state { + match c { + '*' => { + state = WidthArg; + move_to!(next); + } + '1'..='9' => { + let end = at_next_cp_while(next, is_digit); + state = Prec; + width = Some(Num::from_str(at.slice_between(end).unwrap(), None)); + move_to!(end); + } + _ => { + state = Prec; + width = None; + move_to!(at); + } + } + } + + if let WidthArg = state { + let end = at_next_cp_while(at, is_digit); + match end.next_cp() { + Some(('$', end2)) => { + state = Prec; + width = Some(Num::from_str("", Some(at.slice_between(end).unwrap()))); + move_to!(end2); + } + _ => { + state = Prec; + width = Some(Num::Next); + move_to!(end); + } + } + } + + if let Prec = state { + match c { + '.' => { + state = PrecInner; + move_to!(next); + } + _ => { + state = Length; + precision = None; + move_to!(at); + } + } + } + + if let PrecInner = state { + match c { + '*' => { + let end = at_next_cp_while(next, is_digit); + match end.next_cp() { + Some(('$', end2)) => { + state = Length; + precision = Some(Num::from_str("*", next.slice_between(end))); + move_to!(end2); + } + _ => { + state = Length; + precision = Some(Num::Next); + move_to!(end); + } + } + } + '0'..='9' => { + let end = at_next_cp_while(next, is_digit); + state = Length; + precision = Some(Num::from_str(at.slice_between(end).unwrap(), None)); + move_to!(end); + } + _ => return fallback(), + } + } + + if let Length = state { + let c1_next1 = next.next_cp(); + match (c, c1_next1) { + ('h', Some(('h', next1))) | ('l', Some(('l', next1))) => { + state = Type; + length = Some(at.slice_between(next1).unwrap()); + move_to!(next1); + } + + ('h', _) | ('l', _) | ('L', _) | ('z', _) | ('j', _) | ('t', _) | ('q', _) => { + state = Type; + length = Some(at.slice_between(next).unwrap()); + move_to!(next); + } + + ('I', _) => { + let end = next + .at_next_cp() + .and_then(|end| end.at_next_cp()) + .map(|end| (next.slice_between(end).unwrap(), end)); + let end = match end { + Some(("32", end)) => end, + Some(("64", end)) => end, + _ => next, + }; + state = Type; + length = Some(at.slice_between(end).unwrap()); + move_to!(end); + } + + _ => { + state = Type; + length = None; + move_to!(at); + } + } + } + + if let Type = state { + drop(c); + type_ = at.slice_between(next).unwrap(); + + // Don't use `move_to!` here, as we *can* be at the end of the input. + at = next; + } + + drop(c); + drop(next); + + end = at; + let position = InnerSpan::new(start.at, end.at); + + let f = Format { + span: start.slice_between(end).unwrap(), + parameter, + flags, + width, + precision, + length, + type_, + position, + }; + Some((Substitution::Format(f), end.slice_after())) + } + + fn at_next_cp_while(mut cur: Cur<'_>, mut pred: F) -> Cur<'_> + where + F: FnMut(char) -> bool, + { + loop { + match cur.next_cp() { + Some((c, next)) => { + if pred(c) { + cur = next; + } else { + return cur; + } + } + None => return cur, + } + } + } + + fn is_digit(c: char) -> bool { + match c { + '0'..='9' => true, + _ => false, + } + } + + fn is_flag(c: char) -> bool { + match c { + '0' | '-' | '+' | ' ' | '#' | '\'' => true, + _ => false, + } + } + + #[cfg(test)] + mod tests; +} + +pub mod shell { + use super::strcursor::StrCursor as Cur; + use syntax_pos::InnerSpan; + + #[derive(Clone, PartialEq, Debug)] + pub enum Substitution<'a> { + Ordinal(u8, (usize, usize)), + Name(&'a str, (usize, usize)), + Escape((usize, usize)), + } + + impl Substitution<'_> { + pub fn as_str(&self) -> String { + match self { + Substitution::Ordinal(n, _) => format!("${}", n), + Substitution::Name(n, _) => format!("${}", n), + Substitution::Escape(_) => "$$".into(), + } + } + + pub fn position(&self) -> Option { + match self { + Substitution::Ordinal(_, pos) + | Substitution::Name(_, pos) + | Substitution::Escape(pos) => Some(InnerSpan::new(pos.0, pos.1)), + } + } + + pub fn set_position(&mut self, start: usize, end: usize) { + match self { + Substitution::Ordinal(_, ref mut pos) + | Substitution::Name(_, ref mut pos) + | Substitution::Escape(ref mut pos) => *pos = (start, end), + } + } + + pub fn translate(&self) -> Option { + match *self { + Substitution::Ordinal(n, _) => Some(format!("{{{}}}", n)), + Substitution::Name(n, _) => Some(format!("{{{}}}", n)), + Substitution::Escape(_) => None, + } + } + } + + /// Returns an iterator over all substitutions in a given string. + pub fn iter_subs(s: &str, start_pos: usize) -> Substitutions<'_> { + Substitutions { s, pos: start_pos } + } + + /// Iterator over substitutions in a string. + pub struct Substitutions<'a> { + s: &'a str, + pos: usize, + } + + impl<'a> Iterator for Substitutions<'a> { + type Item = Substitution<'a>; + fn next(&mut self) -> Option { + match parse_next_substitution(self.s) { + Some((mut sub, tail)) => { + self.s = tail; + if let Some(InnerSpan { start, end }) = sub.position() { + sub.set_position(start + self.pos, end + self.pos); + self.pos += end; + } + Some(sub) + } + None => None, + } + } + + fn size_hint(&self) -> (usize, Option) { + (0, Some(self.s.len())) + } + } + + /// Parse the next substitution from the input string. + pub fn parse_next_substitution(s: &str) -> Option<(Substitution<'_>, &str)> { + let at = { + let start = s.find('$')?; + match s[start + 1..].chars().next()? { + '$' => return Some((Substitution::Escape((start, start + 2)), &s[start + 2..])), + c @ '0'..='9' => { + let n = (c as u8) - b'0'; + return Some((Substitution::Ordinal(n, (start, start + 2)), &s[start + 2..])); + } + _ => { /* fall-through */ } + } + + Cur::new_at(&s[..], start) + }; + + let at = at.at_next_cp()?; + let (c, inner) = at.next_cp()?; + + if !is_ident_head(c) { + None + } else { + let end = at_next_cp_while(inner, is_ident_tail); + let slice = at.slice_between(end).unwrap(); + let start = at.at - 1; + let end_pos = at.at + slice.len(); + Some((Substitution::Name(slice, (start, end_pos)), end.slice_after())) + } + } + + fn at_next_cp_while(mut cur: Cur<'_>, mut pred: F) -> Cur<'_> + where + F: FnMut(char) -> bool, + { + loop { + match cur.next_cp() { + Some((c, next)) => { + if pred(c) { + cur = next; + } else { + return cur; + } + } + None => return cur, + } + } + } + + fn is_ident_head(c: char) -> bool { + match c { + 'a'..='z' | 'A'..='Z' | '_' => true, + _ => false, + } + } + + fn is_ident_tail(c: char) -> bool { + match c { + '0'..='9' => true, + c => is_ident_head(c), + } + } + + #[cfg(test)] + mod tests; +} + +mod strcursor { + pub struct StrCursor<'a> { + s: &'a str, + pub at: usize, + } + + impl<'a> StrCursor<'a> { + pub fn new_at(s: &'a str, at: usize) -> StrCursor<'a> { + StrCursor { s, at } + } + + pub fn at_next_cp(mut self) -> Option> { + match self.try_seek_right_cp() { + true => Some(self), + false => None, + } + } + + pub fn next_cp(mut self) -> Option<(char, StrCursor<'a>)> { + let cp = self.cp_after()?; + self.seek_right(cp.len_utf8()); + Some((cp, self)) + } + + fn slice_before(&self) -> &'a str { + &self.s[0..self.at] + } + + pub fn slice_after(&self) -> &'a str { + &self.s[self.at..] + } + + pub fn slice_between(&self, until: StrCursor<'a>) -> Option<&'a str> { + if !str_eq_literal(self.s, until.s) { + None + } else { + use std::cmp::{max, min}; + let beg = min(self.at, until.at); + let end = max(self.at, until.at); + Some(&self.s[beg..end]) + } + } + + fn cp_after(&self) -> Option { + self.slice_after().chars().next() + } + + fn try_seek_right_cp(&mut self) -> bool { + match self.slice_after().chars().next() { + Some(c) => { + self.at += c.len_utf8(); + true + } + None => false, + } + } + + fn seek_right(&mut self, bytes: usize) { + self.at += bytes; + } + } + + impl Copy for StrCursor<'_> {} + + impl<'a> Clone for StrCursor<'a> { + fn clone(&self) -> StrCursor<'a> { + *self + } + } + + impl std::fmt::Debug for StrCursor<'_> { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(fmt, "StrCursor({:?} | {:?})", self.slice_before(), self.slice_after()) + } + } + + fn str_eq_literal(a: &str, b: &str) -> bool { + a.as_bytes().as_ptr() == b.as_bytes().as_ptr() && a.len() == b.len() + } +} diff --git a/src/librustc_builtin_macros/format_foreign/printf/tests.rs b/src/librustc_builtin_macros/format_foreign/printf/tests.rs new file mode 100644 index 00000000000..b9a85a84d6c --- /dev/null +++ b/src/librustc_builtin_macros/format_foreign/printf/tests.rs @@ -0,0 +1,145 @@ +use super::{iter_subs, parse_next_substitution as pns, Format as F, Num as N, Substitution as S}; + +macro_rules! assert_eq_pnsat { + ($lhs:expr, $rhs:expr) => { + assert_eq!( + pns($lhs).and_then(|(s, _)| s.translate()), + $rhs.map(>::from) + ) + }; +} + +#[test] +fn test_escape() { + assert_eq!(pns("has no escapes"), None); + assert_eq!(pns("has no escapes, either %"), None); + assert_eq!(pns("*so* has a %% escape"), Some((S::Escape, " escape"))); + assert_eq!(pns("%% leading escape"), Some((S::Escape, " leading escape"))); + assert_eq!(pns("trailing escape %%"), Some((S::Escape, ""))); +} + +#[test] +fn test_parse() { + macro_rules! assert_pns_eq_sub { + ($in_:expr, { + $param:expr, $flags:expr, + $width:expr, $prec:expr, $len:expr, $type_:expr, + $pos:expr, + }) => { + assert_eq!( + pns(concat!($in_, "!")), + Some(( + S::Format(F { + span: $in_, + parameter: $param, + flags: $flags, + width: $width, + precision: $prec, + length: $len, + type_: $type_, + position: syntax_pos::InnerSpan::new($pos.0, $pos.1), + }), + "!" + )) + ) + }; + } + + assert_pns_eq_sub!("%!", + { None, "", None, None, None, "!", (0, 2), }); + assert_pns_eq_sub!("%c", + { None, "", None, None, None, "c", (0, 2), }); + assert_pns_eq_sub!("%s", + { None, "", None, None, None, "s", (0, 2), }); + assert_pns_eq_sub!("%06d", + { None, "0", Some(N::Num(6)), None, None, "d", (0, 4), }); + assert_pns_eq_sub!("%4.2f", + { None, "", Some(N::Num(4)), Some(N::Num(2)), None, "f", (0, 5), }); + assert_pns_eq_sub!("%#x", + { None, "#", None, None, None, "x", (0, 3), }); + assert_pns_eq_sub!("%-10s", + { None, "-", Some(N::Num(10)), None, None, "s", (0, 5), }); + assert_pns_eq_sub!("%*s", + { None, "", Some(N::Next), None, None, "s", (0, 3), }); + assert_pns_eq_sub!("%-10.*s", + { None, "-", Some(N::Num(10)), Some(N::Next), None, "s", (0, 7), }); + assert_pns_eq_sub!("%-*.*s", + { None, "-", Some(N::Next), Some(N::Next), None, "s", (0, 6), }); + assert_pns_eq_sub!("%.6i", + { None, "", None, Some(N::Num(6)), None, "i", (0, 4), }); + assert_pns_eq_sub!("%+i", + { None, "+", None, None, None, "i", (0, 3), }); + assert_pns_eq_sub!("%08X", + { None, "0", Some(N::Num(8)), None, None, "X", (0, 4), }); + assert_pns_eq_sub!("%lu", + { None, "", None, None, Some("l"), "u", (0, 3), }); + assert_pns_eq_sub!("%Iu", + { None, "", None, None, Some("I"), "u", (0, 3), }); + assert_pns_eq_sub!("%I32u", + { None, "", None, None, Some("I32"), "u", (0, 5), }); + assert_pns_eq_sub!("%I64u", + { None, "", None, None, Some("I64"), "u", (0, 5), }); + assert_pns_eq_sub!("%'d", + { None, "'", None, None, None, "d", (0, 3), }); + assert_pns_eq_sub!("%10s", + { None, "", Some(N::Num(10)), None, None, "s", (0, 4), }); + assert_pns_eq_sub!("%-10.10s", + { None, "-", Some(N::Num(10)), Some(N::Num(10)), None, "s", (0, 8), }); + assert_pns_eq_sub!("%1$d", + { Some(1), "", None, None, None, "d", (0, 4), }); + assert_pns_eq_sub!("%2$.*3$d", + { Some(2), "", None, Some(N::Arg(3)), None, "d", (0, 8), }); + assert_pns_eq_sub!("%1$*2$.*3$d", + { Some(1), "", Some(N::Arg(2)), Some(N::Arg(3)), None, "d", (0, 11), }); + assert_pns_eq_sub!("%-8ld", + { None, "-", Some(N::Num(8)), None, Some("l"), "d", (0, 5), }); +} + +#[test] +fn test_iter() { + let s = "The %d'th word %% is: `%.*s` %!\n"; + let subs: Vec<_> = iter_subs(s, 0).map(|sub| sub.translate()).collect(); + assert_eq!( + subs.iter().map(|ms| ms.as_ref().map(|s| &s[..])).collect::>(), + vec![Some("{}"), None, Some("{:.*}"), None] + ); +} + +/// Checks that the translations are what we expect. +#[test] +fn test_translation() { + assert_eq_pnsat!("%c", Some("{}")); + assert_eq_pnsat!("%d", Some("{}")); + assert_eq_pnsat!("%u", Some("{}")); + assert_eq_pnsat!("%x", Some("{:x}")); + assert_eq_pnsat!("%X", Some("{:X}")); + assert_eq_pnsat!("%e", Some("{:e}")); + assert_eq_pnsat!("%E", Some("{:E}")); + assert_eq_pnsat!("%f", Some("{}")); + assert_eq_pnsat!("%g", Some("{:e}")); + assert_eq_pnsat!("%G", Some("{:E}")); + assert_eq_pnsat!("%s", Some("{}")); + assert_eq_pnsat!("%p", Some("{:p}")); + + assert_eq_pnsat!("%06d", Some("{:06}")); + assert_eq_pnsat!("%4.2f", Some("{:4.2}")); + assert_eq_pnsat!("%#x", Some("{:#x}")); + assert_eq_pnsat!("%-10s", Some("{:<10}")); + assert_eq_pnsat!("%*s", None); + assert_eq_pnsat!("%-10.*s", Some("{:<10.*}")); + assert_eq_pnsat!("%-*.*s", None); + assert_eq_pnsat!("%.6i", Some("{:06}")); + assert_eq_pnsat!("%+i", Some("{:+}")); + assert_eq_pnsat!("%08X", Some("{:08X}")); + assert_eq_pnsat!("%lu", Some("{}")); + assert_eq_pnsat!("%Iu", Some("{}")); + assert_eq_pnsat!("%I32u", Some("{}")); + assert_eq_pnsat!("%I64u", Some("{}")); + assert_eq_pnsat!("%'d", None); + assert_eq_pnsat!("%10s", Some("{:>10}")); + assert_eq_pnsat!("%-10.10s", Some("{:<10.10}")); + assert_eq_pnsat!("%1$d", Some("{0}")); + assert_eq_pnsat!("%2$.*3$d", Some("{1:02$}")); + assert_eq_pnsat!("%1$*2$.*3$s", Some("{0:>1$.2$}")); + assert_eq_pnsat!("%-8ld", Some("{:<8}")); +} diff --git a/src/librustc_builtin_macros/format_foreign/shell/tests.rs b/src/librustc_builtin_macros/format_foreign/shell/tests.rs new file mode 100644 index 00000000000..ed8fe81dfcd --- /dev/null +++ b/src/librustc_builtin_macros/format_foreign/shell/tests.rs @@ -0,0 +1,56 @@ +use super::{parse_next_substitution as pns, Substitution as S}; + +macro_rules! assert_eq_pnsat { + ($lhs:expr, $rhs:expr) => { + assert_eq!( + pns($lhs).and_then(|(f, _)| f.translate()), + $rhs.map(>::from) + ) + }; +} + +#[test] +fn test_escape() { + assert_eq!(pns("has no escapes"), None); + assert_eq!(pns("has no escapes, either $"), None); + assert_eq!(pns("*so* has a $$ escape"), Some((S::Escape((11, 13)), " escape"))); + assert_eq!(pns("$$ leading escape"), Some((S::Escape((0, 2)), " leading escape"))); + assert_eq!(pns("trailing escape $$"), Some((S::Escape((16, 18)), ""))); +} + +#[test] +fn test_parse() { + macro_rules! assert_pns_eq_sub { + ($in_:expr, $kind:ident($arg:expr, $pos:expr)) => { + assert_eq!(pns(concat!($in_, "!")), Some((S::$kind($arg.into(), $pos), "!"))) + }; + } + + assert_pns_eq_sub!("$0", Ordinal(0, (0, 2))); + assert_pns_eq_sub!("$1", Ordinal(1, (0, 2))); + assert_pns_eq_sub!("$9", Ordinal(9, (0, 2))); + assert_pns_eq_sub!("$N", Name("N", (0, 2))); + assert_pns_eq_sub!("$NAME", Name("NAME", (0, 5))); +} + +#[test] +fn test_iter() { + use super::iter_subs; + let s = "The $0'th word $$ is: `$WORD` $!\n"; + let subs: Vec<_> = iter_subs(s, 0).map(|sub| sub.translate()).collect(); + assert_eq!( + subs.iter().map(|ms| ms.as_ref().map(|s| &s[..])).collect::>(), + vec![Some("{0}"), None, Some("{WORD}")] + ); +} + +#[test] +fn test_translation() { + assert_eq_pnsat!("$0", Some("{0}")); + assert_eq_pnsat!("$9", Some("{9}")); + assert_eq_pnsat!("$1", Some("{1}")); + assert_eq_pnsat!("$10", Some("{1}")); + assert_eq_pnsat!("$stuff", Some("{stuff}")); + assert_eq_pnsat!("$NAME", Some("{NAME}")); + assert_eq_pnsat!("$PREFIX/bin", Some("{PREFIX}")); +} diff --git a/src/librustc_builtin_macros/global_allocator.rs b/src/librustc_builtin_macros/global_allocator.rs new file mode 100644 index 00000000000..edfdda4703c --- /dev/null +++ b/src/librustc_builtin_macros/global_allocator.rs @@ -0,0 +1,175 @@ +use crate::util::check_builtin_macro_attribute; + +use syntax::ast::{self, Attribute, Expr, FnHeader, FnSig, Generics, Ident, Param}; +use syntax::ast::{ItemKind, Mutability, Stmt, Ty, TyKind, Unsafety}; +use syntax::expand::allocator::{AllocatorKind, AllocatorMethod, AllocatorTy, ALLOCATOR_METHODS}; +use syntax::ptr::P; +use syntax::symbol::{kw, sym, Symbol}; +use syntax_expand::base::{Annotatable, ExtCtxt}; +use syntax_pos::Span; + +pub fn expand( + ecx: &mut ExtCtxt<'_>, + _span: Span, + meta_item: &ast::MetaItem, + item: Annotatable, +) -> Vec { + check_builtin_macro_attribute(ecx, meta_item, sym::global_allocator); + + let not_static = |item: Annotatable| { + ecx.parse_sess.span_diagnostic.span_err(item.span(), "allocators must be statics"); + vec![item] + }; + let item = match item { + Annotatable::Item(item) => match item.kind { + ItemKind::Static(..) => item, + _ => return not_static(Annotatable::Item(item)), + }, + _ => return not_static(item), + }; + + // Generate a bunch of new items using the AllocFnFactory + let span = ecx.with_def_site_ctxt(item.span); + let f = AllocFnFactory { span, kind: AllocatorKind::Global, global: item.ident, cx: ecx }; + + // Generate item statements for the allocator methods. + let stmts = ALLOCATOR_METHODS.iter().map(|method| f.allocator_fn(method)).collect(); + + // Generate anonymous constant serving as container for the allocator methods. + let const_ty = ecx.ty(span, TyKind::Tup(Vec::new())); + let const_body = ecx.expr_block(ecx.block(span, stmts)); + let const_item = ecx.item_const(span, Ident::new(kw::Underscore, span), const_ty, const_body); + + // Return the original item and the new methods. + vec![Annotatable::Item(item), Annotatable::Item(const_item)] +} + +struct AllocFnFactory<'a, 'b> { + span: Span, + kind: AllocatorKind, + global: Ident, + cx: &'b ExtCtxt<'a>, +} + +impl AllocFnFactory<'_, '_> { + fn allocator_fn(&self, method: &AllocatorMethod) -> Stmt { + let mut abi_args = Vec::new(); + let mut i = 0; + let ref mut mk = || { + let name = self.cx.ident_of(&format!("arg{}", i), self.span); + i += 1; + name + }; + let args = method.inputs.iter().map(|ty| self.arg_ty(ty, &mut abi_args, mk)).collect(); + let result = self.call_allocator(method.name, args); + let (output_ty, output_expr) = self.ret_ty(&method.output, result); + let decl = self.cx.fn_decl(abi_args, ast::FunctionRetTy::Ty(output_ty)); + let header = FnHeader { unsafety: Unsafety::Unsafe, ..FnHeader::default() }; + let sig = FnSig { decl, header }; + let kind = ItemKind::Fn(sig, Generics::default(), self.cx.block_expr(output_expr)); + let item = self.cx.item( + self.span, + self.cx.ident_of(&self.kind.fn_name(method.name), self.span), + self.attrs(), + kind, + ); + self.cx.stmt_item(self.span, item) + } + + fn call_allocator(&self, method: &str, mut args: Vec>) -> P { + let method = self.cx.std_path(&[ + Symbol::intern("alloc"), + Symbol::intern("GlobalAlloc"), + Symbol::intern(method), + ]); + let method = self.cx.expr_path(self.cx.path(self.span, method)); + let allocator = self.cx.path_ident(self.span, self.global); + let allocator = self.cx.expr_path(allocator); + let allocator = self.cx.expr_addr_of(self.span, allocator); + args.insert(0, allocator); + + self.cx.expr_call(self.span, method, args) + } + + fn attrs(&self) -> Vec { + let special = sym::rustc_std_internal_symbol; + let special = self.cx.meta_word(self.span, special); + vec![self.cx.attribute(special)] + } + + fn arg_ty( + &self, + ty: &AllocatorTy, + args: &mut Vec, + ident: &mut dyn FnMut() -> Ident, + ) -> P { + match *ty { + AllocatorTy::Layout => { + let usize = self.cx.path_ident(self.span, Ident::new(sym::usize, self.span)); + let ty_usize = self.cx.ty_path(usize); + let size = ident(); + let align = ident(); + args.push(self.cx.param(self.span, size, ty_usize.clone())); + args.push(self.cx.param(self.span, align, ty_usize)); + + let layout_new = self.cx.std_path(&[ + Symbol::intern("alloc"), + Symbol::intern("Layout"), + Symbol::intern("from_size_align_unchecked"), + ]); + let layout_new = self.cx.expr_path(self.cx.path(self.span, layout_new)); + let size = self.cx.expr_ident(self.span, size); + let align = self.cx.expr_ident(self.span, align); + let layout = self.cx.expr_call(self.span, layout_new, vec![size, align]); + layout + } + + AllocatorTy::Ptr => { + let ident = ident(); + args.push(self.cx.param(self.span, ident, self.ptr_u8())); + let arg = self.cx.expr_ident(self.span, ident); + self.cx.expr_cast(self.span, arg, self.ptr_u8()) + } + + AllocatorTy::Usize => { + let ident = ident(); + args.push(self.cx.param(self.span, ident, self.usize())); + self.cx.expr_ident(self.span, ident) + } + + AllocatorTy::ResultPtr | AllocatorTy::Unit => { + panic!("can't convert AllocatorTy to an argument") + } + } + } + + fn ret_ty(&self, ty: &AllocatorTy, expr: P) -> (P, P) { + match *ty { + AllocatorTy::ResultPtr => { + // We're creating: + // + // #expr as *mut u8 + + let expr = self.cx.expr_cast(self.span, expr, self.ptr_u8()); + (self.ptr_u8(), expr) + } + + AllocatorTy::Unit => (self.cx.ty(self.span, TyKind::Tup(Vec::new())), expr), + + AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + panic!("can't convert `AllocatorTy` to an output") + } + } + } + + fn usize(&self) -> P { + let usize = self.cx.path_ident(self.span, Ident::new(sym::usize, self.span)); + self.cx.ty_path(usize) + } + + fn ptr_u8(&self) -> P { + let u8 = self.cx.path_ident(self.span, Ident::new(sym::u8, self.span)); + let ty_u8 = self.cx.ty_path(u8); + self.cx.ty_ptr(self.span, ty_u8, Mutability::Mut) + } +} diff --git a/src/librustc_builtin_macros/global_asm.rs b/src/librustc_builtin_macros/global_asm.rs new file mode 100644 index 00000000000..fc933e4673a --- /dev/null +++ b/src/librustc_builtin_macros/global_asm.rs @@ -0,0 +1,64 @@ +/// Module-level assembly support. +/// +/// The macro defined here allows you to specify "top-level", +/// "file-scoped", or "module-level" assembly. These synonyms +/// all correspond to LLVM's module-level inline assembly instruction. +/// +/// For example, `global_asm!("some assembly here")` codegens to +/// LLVM's `module asm "some assembly here"`. All of LLVM's caveats +/// therefore apply. +use errors::DiagnosticBuilder; + +use smallvec::smallvec; +use syntax::ast; +use syntax::ptr::P; +use syntax::source_map::respan; +use syntax::token; +use syntax::tokenstream::TokenStream; +use syntax_expand::base::{self, *}; +use syntax_pos::Span; + +pub fn expand_global_asm<'cx>( + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + match parse_global_asm(cx, sp, tts) { + Ok(Some(global_asm)) => MacEager::items(smallvec![P(ast::Item { + ident: ast::Ident::invalid(), + attrs: Vec::new(), + id: ast::DUMMY_NODE_ID, + kind: ast::ItemKind::GlobalAsm(P(global_asm)), + vis: respan(sp.shrink_to_lo(), ast::VisibilityKind::Inherited), + span: cx.with_def_site_ctxt(sp), + tokens: None, + })]), + Ok(None) => DummyResult::any(sp), + Err(mut err) => { + err.emit(); + DummyResult::any(sp) + } + } +} + +fn parse_global_asm<'a>( + cx: &mut ExtCtxt<'a>, + sp: Span, + tts: TokenStream, +) -> Result, DiagnosticBuilder<'a>> { + let mut p = cx.new_parser_from_tts(tts); + + if p.token == token::Eof { + let mut err = cx.struct_span_err(sp, "macro requires a string literal as an argument"); + err.span_label(sp, "string literal required"); + return Err(err); + } + + let expr = p.parse_expr()?; + let (asm, _) = match expr_to_string(cx, expr, "inline assembly must be a string literal") { + Some((s, st)) => (s, st), + None => return Ok(None), + }; + + Ok(Some(ast::GlobalAsm { asm })) +} diff --git a/src/librustc_builtin_macros/lib.rs b/src/librustc_builtin_macros/lib.rs new file mode 100644 index 00000000000..40aafece8c6 --- /dev/null +++ b/src/librustc_builtin_macros/lib.rs @@ -0,0 +1,109 @@ +//! This crate contains implementations of built-in macros and other code generating facilities +//! injecting code into the crate before it is lowered to HIR. + +#![doc(html_root_url = "https://doc.rust-lang.org/nightly/")] +#![feature(bool_to_option)] +#![feature(crate_visibility_modifier)] +#![feature(decl_macro)] +#![feature(nll)] +#![feature(proc_macro_internals)] +#![feature(proc_macro_quote)] + +extern crate proc_macro; + +use crate::deriving::*; + +use syntax::ast::Ident; +use syntax::edition::Edition; +use syntax::symbol::sym; +use syntax_expand::base::{MacroExpanderFn, Resolver, SyntaxExtension, SyntaxExtensionKind}; +use syntax_expand::proc_macro::BangProcMacro; + +mod asm; +mod assert; +mod cfg; +mod compile_error; +mod concat; +mod concat_idents; +mod deriving; +mod env; +mod format; +mod format_foreign; +mod global_allocator; +mod global_asm; +mod log_syntax; +mod source_util; +mod test; +mod trace_macros; +mod util; + +pub mod cmdline_attrs; +pub mod proc_macro_harness; +pub mod standard_library_imports; +pub mod test_harness; + +pub fn register_builtin_macros(resolver: &mut dyn Resolver, edition: Edition) { + let mut register = |name, kind| { + resolver.register_builtin_macro( + Ident::with_dummy_span(name), + SyntaxExtension { is_builtin: true, ..SyntaxExtension::default(kind, edition) }, + ) + }; + macro register_bang($($name:ident: $f:expr,)*) { + $(register(sym::$name, SyntaxExtensionKind::LegacyBang(Box::new($f as MacroExpanderFn)));)* + } + macro register_attr($($name:ident: $f:expr,)*) { + $(register(sym::$name, SyntaxExtensionKind::LegacyAttr(Box::new($f)));)* + } + macro register_derive($($name:ident: $f:expr,)*) { + $(register(sym::$name, SyntaxExtensionKind::LegacyDerive(Box::new(BuiltinDerive($f))));)* + } + + register_bang! { + asm: asm::expand_asm, + assert: assert::expand_assert, + cfg: cfg::expand_cfg, + column: source_util::expand_column, + compile_error: compile_error::expand_compile_error, + concat_idents: concat_idents::expand_concat_idents, + concat: concat::expand_concat, + env: env::expand_env, + file: source_util::expand_file, + format_args_nl: format::expand_format_args_nl, + format_args: format::expand_format_args, + global_asm: global_asm::expand_global_asm, + include_bytes: source_util::expand_include_bytes, + include_str: source_util::expand_include_str, + include: source_util::expand_include, + line: source_util::expand_line, + log_syntax: log_syntax::expand_log_syntax, + module_path: source_util::expand_mod, + option_env: env::expand_option_env, + stringify: source_util::expand_stringify, + trace_macros: trace_macros::expand_trace_macros, + } + + register_attr! { + bench: test::expand_bench, + global_allocator: global_allocator::expand, + test: test::expand_test, + test_case: test::expand_test_case, + } + + register_derive! { + Clone: clone::expand_deriving_clone, + Copy: bounds::expand_deriving_copy, + Debug: debug::expand_deriving_debug, + Default: default::expand_deriving_default, + Eq: eq::expand_deriving_eq, + Hash: hash::expand_deriving_hash, + Ord: ord::expand_deriving_ord, + PartialEq: partial_eq::expand_deriving_partial_eq, + PartialOrd: partial_ord::expand_deriving_partial_ord, + RustcDecodable: decodable::expand_deriving_rustc_decodable, + RustcEncodable: encodable::expand_deriving_rustc_encodable, + } + + let client = proc_macro::bridge::client::Client::expand1(proc_macro::quote); + register(sym::quote, SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client }))); +} diff --git a/src/librustc_builtin_macros/log_syntax.rs b/src/librustc_builtin_macros/log_syntax.rs new file mode 100644 index 00000000000..111226be877 --- /dev/null +++ b/src/librustc_builtin_macros/log_syntax.rs @@ -0,0 +1,15 @@ +use syntax::print; +use syntax::tokenstream::TokenStream; +use syntax_expand::base; +use syntax_pos; + +pub fn expand_log_syntax<'cx>( + _cx: &'cx mut base::ExtCtxt<'_>, + sp: syntax_pos::Span, + tts: TokenStream, +) -> Box { + println!("{}", print::pprust::tts_to_string(tts)); + + // any so that `log_syntax` can be invoked as an expression and item. + base::DummyResult::any_valid(sp) +} diff --git a/src/librustc_builtin_macros/proc_macro_harness.rs b/src/librustc_builtin_macros/proc_macro_harness.rs new file mode 100644 index 00000000000..b6436cc1646 --- /dev/null +++ b/src/librustc_builtin_macros/proc_macro_harness.rs @@ -0,0 +1,463 @@ +use std::mem; + +use smallvec::smallvec; +use syntax::ast::{self, Ident}; +use syntax::attr; +use syntax::expand::is_proc_macro_attr; +use syntax::print::pprust; +use syntax::ptr::P; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym}; +use syntax::visit::{self, Visitor}; +use syntax_expand::base::{ExtCtxt, Resolver}; +use syntax_expand::expand::{AstFragment, ExpansionConfig}; +use syntax_pos::hygiene::AstPass; +use syntax_pos::{Span, DUMMY_SP}; + +struct ProcMacroDerive { + trait_name: ast::Name, + function_name: Ident, + span: Span, + attrs: Vec, +} + +enum ProcMacroDefType { + Attr, + Bang, +} + +struct ProcMacroDef { + function_name: Ident, + span: Span, + def_type: ProcMacroDefType, +} + +enum ProcMacro { + Derive(ProcMacroDerive), + Def(ProcMacroDef), +} + +struct CollectProcMacros<'a> { + macros: Vec, + in_root: bool, + handler: &'a errors::Handler, + is_proc_macro_crate: bool, + is_test_crate: bool, +} + +pub fn inject( + sess: &ParseSess, + resolver: &mut dyn Resolver, + mut krate: ast::Crate, + is_proc_macro_crate: bool, + has_proc_macro_decls: bool, + is_test_crate: bool, + num_crate_types: usize, + handler: &errors::Handler, +) -> ast::Crate { + let ecfg = ExpansionConfig::default("proc_macro".to_string()); + let mut cx = ExtCtxt::new(sess, ecfg, resolver); + + let mut collect = CollectProcMacros { + macros: Vec::new(), + in_root: true, + handler, + is_proc_macro_crate, + is_test_crate, + }; + + if has_proc_macro_decls || is_proc_macro_crate { + visit::walk_crate(&mut collect, &krate); + } + // NOTE: If you change the order of macros in this vec + // for any reason, you must also update 'raw_proc_macro' + // in src/librustc_metadata/decoder.rs + let macros = collect.macros; + + if !is_proc_macro_crate { + return krate; + } + + if num_crate_types > 1 { + handler.err("cannot mix `proc-macro` crate type with others"); + } + + if is_test_crate { + return krate; + } + + krate.module.items.push(mk_decls(&mut cx, ¯os)); + + krate +} + +impl<'a> CollectProcMacros<'a> { + fn check_not_pub_in_root(&self, vis: &ast::Visibility, sp: Span) { + if self.is_proc_macro_crate && self.in_root && vis.node.is_pub() { + self.handler.span_err( + sp, + "`proc-macro` crate types currently cannot export any items other \ + than functions tagged with `#[proc_macro]`, `#[proc_macro_derive]`, \ + or `#[proc_macro_attribute]`", + ); + } + } + + fn collect_custom_derive(&mut self, item: &'a ast::Item, attr: &'a ast::Attribute) { + // Once we've located the `#[proc_macro_derive]` attribute, verify + // that it's of the form `#[proc_macro_derive(Foo)]` or + // `#[proc_macro_derive(Foo, attributes(A, ..))]` + let list = match attr.meta_item_list() { + Some(list) => list, + None => return, + }; + if list.len() != 1 && list.len() != 2 { + self.handler.span_err(attr.span, "attribute must have either one or two arguments"); + return; + } + let trait_attr = match list[0].meta_item() { + Some(meta_item) => meta_item, + _ => { + self.handler.span_err(list[0].span(), "not a meta item"); + return; + } + }; + let trait_ident = match trait_attr.ident() { + Some(trait_ident) if trait_attr.is_word() => trait_ident, + _ => { + self.handler.span_err(trait_attr.span, "must only be one word"); + return; + } + }; + + if !trait_ident.name.can_be_raw() { + self.handler.span_err( + trait_attr.span, + &format!("`{}` cannot be a name of derive macro", trait_ident), + ); + } + + let attributes_attr = list.get(1); + let proc_attrs: Vec<_> = if let Some(attr) = attributes_attr { + if !attr.check_name(sym::attributes) { + self.handler.span_err(attr.span(), "second argument must be `attributes`") + } + attr.meta_item_list() + .unwrap_or_else(|| { + self.handler + .span_err(attr.span(), "attribute must be of form: `attributes(foo, bar)`"); + &[] + }) + .into_iter() + .filter_map(|attr| { + let attr = match attr.meta_item() { + Some(meta_item) => meta_item, + _ => { + self.handler.span_err(attr.span(), "not a meta item"); + return None; + } + }; + + let ident = match attr.ident() { + Some(ident) if attr.is_word() => ident, + _ => { + self.handler.span_err(attr.span, "must only be one word"); + return None; + } + }; + if !ident.name.can_be_raw() { + self.handler.span_err( + attr.span, + &format!("`{}` cannot be a name of derive helper attribute", ident), + ); + } + + Some(ident.name) + }) + .collect() + } else { + Vec::new() + }; + + if self.in_root && item.vis.node.is_pub() { + self.macros.push(ProcMacro::Derive(ProcMacroDerive { + span: item.span, + trait_name: trait_ident.name, + function_name: item.ident, + attrs: proc_attrs, + })); + } else { + let msg = if !self.in_root { + "functions tagged with `#[proc_macro_derive]` must \ + currently reside in the root of the crate" + } else { + "functions tagged with `#[proc_macro_derive]` must be `pub`" + }; + self.handler.span_err(item.span, msg); + } + } + + fn collect_attr_proc_macro(&mut self, item: &'a ast::Item) { + if self.in_root && item.vis.node.is_pub() { + self.macros.push(ProcMacro::Def(ProcMacroDef { + span: item.span, + function_name: item.ident, + def_type: ProcMacroDefType::Attr, + })); + } else { + let msg = if !self.in_root { + "functions tagged with `#[proc_macro_attribute]` must \ + currently reside in the root of the crate" + } else { + "functions tagged with `#[proc_macro_attribute]` must be `pub`" + }; + self.handler.span_err(item.span, msg); + } + } + + fn collect_bang_proc_macro(&mut self, item: &'a ast::Item) { + if self.in_root && item.vis.node.is_pub() { + self.macros.push(ProcMacro::Def(ProcMacroDef { + span: item.span, + function_name: item.ident, + def_type: ProcMacroDefType::Bang, + })); + } else { + let msg = if !self.in_root { + "functions tagged with `#[proc_macro]` must \ + currently reside in the root of the crate" + } else { + "functions tagged with `#[proc_macro]` must be `pub`" + }; + self.handler.span_err(item.span, msg); + } + } +} + +impl<'a> Visitor<'a> for CollectProcMacros<'a> { + fn visit_item(&mut self, item: &'a ast::Item) { + if let ast::ItemKind::MacroDef(..) = item.kind { + if self.is_proc_macro_crate && attr::contains_name(&item.attrs, sym::macro_export) { + let msg = + "cannot export macro_rules! macros from a `proc-macro` crate type currently"; + self.handler.span_err(item.span, msg); + } + } + + // First up, make sure we're checking a bare function. If we're not then + // we're just not interested in this item. + // + // If we find one, try to locate a `#[proc_macro_derive]` attribute on it. + let is_fn = match item.kind { + ast::ItemKind::Fn(..) => true, + _ => false, + }; + + let mut found_attr: Option<&'a ast::Attribute> = None; + + for attr in &item.attrs { + if is_proc_macro_attr(&attr) { + if let Some(prev_attr) = found_attr { + let prev_item = prev_attr.get_normal_item(); + let item = attr.get_normal_item(); + let path_str = pprust::path_to_string(&item.path); + let msg = if item.path.segments[0].ident.name + == prev_item.path.segments[0].ident.name + { + format!( + "only one `#[{}]` attribute is allowed on any given function", + path_str, + ) + } else { + format!( + "`#[{}]` and `#[{}]` attributes cannot both be applied + to the same function", + path_str, + pprust::path_to_string(&prev_item.path), + ) + }; + + self.handler + .struct_span_err(attr.span, &msg) + .span_label(prev_attr.span, "previous attribute here") + .emit(); + + return; + } + + found_attr = Some(attr); + } + } + + let attr = match found_attr { + None => { + self.check_not_pub_in_root(&item.vis, item.span); + let prev_in_root = mem::replace(&mut self.in_root, false); + visit::walk_item(self, item); + self.in_root = prev_in_root; + return; + } + Some(attr) => attr, + }; + + if !is_fn { + let msg = format!( + "the `#[{}]` attribute may only be used on bare functions", + pprust::path_to_string(&attr.get_normal_item().path), + ); + + self.handler.span_err(attr.span, &msg); + return; + } + + if self.is_test_crate { + return; + } + + if !self.is_proc_macro_crate { + let msg = format!( + "the `#[{}]` attribute is only usable with crates of the `proc-macro` crate type", + pprust::path_to_string(&attr.get_normal_item().path), + ); + + self.handler.span_err(attr.span, &msg); + return; + } + + if attr.check_name(sym::proc_macro_derive) { + self.collect_custom_derive(item, attr); + } else if attr.check_name(sym::proc_macro_attribute) { + self.collect_attr_proc_macro(item); + } else if attr.check_name(sym::proc_macro) { + self.collect_bang_proc_macro(item); + }; + + let prev_in_root = mem::replace(&mut self.in_root, false); + visit::walk_item(self, item); + self.in_root = prev_in_root; + } + + fn visit_mac(&mut self, mac: &'a ast::Mac) { + visit::walk_mac(self, mac) + } +} + +// Creates a new module which looks like: +// +// const _: () = { +// extern crate proc_macro; +// +// use proc_macro::bridge::client::ProcMacro; +// +// #[rustc_proc_macro_decls] +// #[allow(deprecated)] +// static DECLS: &[ProcMacro] = &[ +// ProcMacro::custom_derive($name_trait1, &[], ::$name1); +// ProcMacro::custom_derive($name_trait2, &["attribute_name"], ::$name2); +// // ... +// ]; +// } +fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> P { + let expn_id = cx.resolver.expansion_for_ast_pass( + DUMMY_SP, + AstPass::ProcMacroHarness, + &[sym::rustc_attrs, sym::proc_macro_internals], + None, + ); + let span = DUMMY_SP.with_def_site_ctxt(expn_id); + + let proc_macro = Ident::new(sym::proc_macro, span); + let krate = cx.item(span, proc_macro, Vec::new(), ast::ItemKind::ExternCrate(None)); + + let bridge = cx.ident_of("bridge", span); + let client = cx.ident_of("client", span); + let proc_macro_ty = cx.ident_of("ProcMacro", span); + let custom_derive = cx.ident_of("custom_derive", span); + let attr = cx.ident_of("attr", span); + let bang = cx.ident_of("bang", span); + + let decls = { + let local_path = + |sp: Span, name| cx.expr_path(cx.path(sp.with_ctxt(span.ctxt()), vec![name])); + let proc_macro_ty_method_path = |method| { + cx.expr_path(cx.path(span, vec![proc_macro, bridge, client, proc_macro_ty, method])) + }; + macros + .iter() + .map(|m| match m { + ProcMacro::Derive(cd) => cx.expr_call( + span, + proc_macro_ty_method_path(custom_derive), + vec![ + cx.expr_str(cd.span, cd.trait_name), + cx.expr_vec_slice( + span, + cd.attrs.iter().map(|&s| cx.expr_str(cd.span, s)).collect::>(), + ), + local_path(cd.span, cd.function_name), + ], + ), + ProcMacro::Def(ca) => { + let ident = match ca.def_type { + ProcMacroDefType::Attr => attr, + ProcMacroDefType::Bang => bang, + }; + + cx.expr_call( + span, + proc_macro_ty_method_path(ident), + vec![ + cx.expr_str(ca.span, ca.function_name.name), + local_path(ca.span, ca.function_name), + ], + ) + } + }) + .collect() + }; + + let decls_static = cx + .item_static( + span, + cx.ident_of("_DECLS", span), + cx.ty_rptr( + span, + cx.ty( + span, + ast::TyKind::Slice( + cx.ty_path(cx.path(span, vec![proc_macro, bridge, client, proc_macro_ty])), + ), + ), + None, + ast::Mutability::Not, + ), + ast::Mutability::Not, + cx.expr_vec_slice(span, decls), + ) + .map(|mut i| { + let attr = cx.meta_word(span, sym::rustc_proc_macro_decls); + i.attrs.push(cx.attribute(attr)); + + let deprecated_attr = attr::mk_nested_word_item(Ident::new(sym::deprecated, span)); + let allow_deprecated_attr = + attr::mk_list_item(Ident::new(sym::allow, span), vec![deprecated_attr]); + i.attrs.push(cx.attribute(allow_deprecated_attr)); + + i + }); + + let block = cx.expr_block( + cx.block(span, vec![cx.stmt_item(span, krate), cx.stmt_item(span, decls_static)]), + ); + + let anon_constant = cx.item_const( + span, + ast::Ident::new(kw::Underscore, span), + cx.ty(span, ast::TyKind::Tup(Vec::new())), + block, + ); + + // Integrate the new item into existing module structures. + let items = AstFragment::Items(smallvec![anon_constant]); + cx.monotonic_expander().fully_expand_fragment(items).make_items().pop().unwrap() +} diff --git a/src/librustc_builtin_macros/source_util.rs b/src/librustc_builtin_macros/source_util.rs new file mode 100644 index 00000000000..fccc36e2ea8 --- /dev/null +++ b/src/librustc_builtin_macros/source_util.rs @@ -0,0 +1,216 @@ +use rustc_parse::{self, new_sub_parser_from_file, parser::Parser, DirectoryOwnership}; +use syntax::ast; +use syntax::early_buffered_lints::INCOMPLETE_INCLUDE; +use syntax::print::pprust; +use syntax::ptr::P; +use syntax::symbol::Symbol; +use syntax::token; +use syntax::tokenstream::TokenStream; +use syntax_expand::base::{self, *}; +use syntax_expand::panictry; + +use smallvec::SmallVec; +use syntax_pos::{self, Pos, Span}; + +use rustc_data_structures::sync::Lrc; + +// These macros all relate to the file system; they either return +// the column/row/filename of the expression, or they include +// a given file into the current one. + +/// line!(): expands to the current line number +pub fn expand_line( + cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let sp = cx.with_def_site_ctxt(sp); + base::check_zero_tts(cx, sp, tts, "line!"); + + let topmost = cx.expansion_cause().unwrap_or(sp); + let loc = cx.source_map().lookup_char_pos(topmost.lo()); + + base::MacEager::expr(cx.expr_u32(topmost, loc.line as u32)) +} + +/* column!(): expands to the current column number */ +pub fn expand_column( + cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let sp = cx.with_def_site_ctxt(sp); + base::check_zero_tts(cx, sp, tts, "column!"); + + let topmost = cx.expansion_cause().unwrap_or(sp); + let loc = cx.source_map().lookup_char_pos(topmost.lo()); + + base::MacEager::expr(cx.expr_u32(topmost, loc.col.to_usize() as u32 + 1)) +} + +/// file!(): expands to the current filename */ +/// The source_file (`loc.file`) contains a bunch more information we could spit +/// out if we wanted. +pub fn expand_file( + cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let sp = cx.with_def_site_ctxt(sp); + base::check_zero_tts(cx, sp, tts, "file!"); + + let topmost = cx.expansion_cause().unwrap_or(sp); + let loc = cx.source_map().lookup_char_pos(topmost.lo()); + base::MacEager::expr(cx.expr_str(topmost, Symbol::intern(&loc.file.name.to_string()))) +} + +pub fn expand_stringify( + cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let sp = cx.with_def_site_ctxt(sp); + let s = pprust::tts_to_string(tts); + base::MacEager::expr(cx.expr_str(sp, Symbol::intern(&s))) +} + +pub fn expand_mod( + cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let sp = cx.with_def_site_ctxt(sp); + base::check_zero_tts(cx, sp, tts, "module_path!"); + let mod_path = &cx.current_expansion.module.mod_path; + let string = mod_path.iter().map(|x| x.to_string()).collect::>().join("::"); + + base::MacEager::expr(cx.expr_str(sp, Symbol::intern(&string))) +} + +/// include! : parse the given file as an expr +/// This is generally a bad idea because it's going to behave +/// unhygienically. +pub fn expand_include<'cx>( + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let sp = cx.with_def_site_ctxt(sp); + let file = match get_single_str_from_tts(cx, sp, tts, "include!") { + Some(f) => f, + None => return DummyResult::any(sp), + }; + // The file will be added to the code map by the parser + let file = match cx.resolve_path(file, sp) { + Ok(f) => f, + Err(mut err) => { + err.emit(); + return DummyResult::any(sp); + } + }; + let directory_ownership = DirectoryOwnership::Owned { relative: None }; + let p = new_sub_parser_from_file(cx.parse_sess(), &file, directory_ownership, None, sp); + + struct ExpandResult<'a> { + p: Parser<'a>, + } + impl<'a> base::MacResult for ExpandResult<'a> { + fn make_expr(mut self: Box>) -> Option> { + let r = panictry!(self.p.parse_expr()); + if self.p.token != token::Eof { + self.p.sess.buffer_lint( + &INCOMPLETE_INCLUDE, + self.p.token.span, + ast::CRATE_NODE_ID, + "include macro expected single expression in source", + ); + } + Some(r) + } + + fn make_items(mut self: Box>) -> Option; 1]>> { + let mut ret = SmallVec::new(); + while self.p.token != token::Eof { + match panictry!(self.p.parse_item()) { + Some(item) => ret.push(item), + None => { + let token = pprust::token_to_string(&self.p.token); + self.p + .sess + .span_diagnostic + .span_fatal( + self.p.token.span, + &format!("expected item, found `{}`", token), + ) + .raise(); + } + } + } + Some(ret) + } + } + + Box::new(ExpandResult { p }) +} + +// include_str! : read the given file, insert it as a literal string expr +pub fn expand_include_str( + cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let sp = cx.with_def_site_ctxt(sp); + let file = match get_single_str_from_tts(cx, sp, tts, "include_str!") { + Some(f) => f, + None => return DummyResult::any(sp), + }; + let file = match cx.resolve_path(file, sp) { + Ok(f) => f, + Err(mut err) => { + err.emit(); + return DummyResult::any(sp); + } + }; + match cx.source_map().load_binary_file(&file) { + Ok(bytes) => match std::str::from_utf8(&bytes) { + Ok(src) => { + let interned_src = Symbol::intern(&src); + base::MacEager::expr(cx.expr_str(sp, interned_src)) + } + Err(_) => { + cx.span_err(sp, &format!("{} wasn't a utf-8 file", file.display())); + DummyResult::any(sp) + } + }, + Err(e) => { + cx.span_err(sp, &format!("couldn't read {}: {}", file.display(), e)); + DummyResult::any(sp) + } + } +} + +pub fn expand_include_bytes( + cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + let sp = cx.with_def_site_ctxt(sp); + let file = match get_single_str_from_tts(cx, sp, tts, "include_bytes!") { + Some(f) => f, + None => return DummyResult::any(sp), + }; + let file = match cx.resolve_path(file, sp) { + Ok(f) => f, + Err(mut err) => { + err.emit(); + return DummyResult::any(sp); + } + }; + match cx.source_map().load_binary_file(&file) { + Ok(bytes) => base::MacEager::expr(cx.expr_lit(sp, ast::LitKind::ByteStr(Lrc::new(bytes)))), + Err(e) => { + cx.span_err(sp, &format!("couldn't read {}: {}", file.display(), e)); + DummyResult::any(sp) + } + } +} diff --git a/src/librustc_builtin_macros/standard_library_imports.rs b/src/librustc_builtin_macros/standard_library_imports.rs new file mode 100644 index 00000000000..50f86a0f3ec --- /dev/null +++ b/src/librustc_builtin_macros/standard_library_imports.rs @@ -0,0 +1,85 @@ +use syntax::edition::Edition; +use syntax::ptr::P; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym, Ident, Symbol}; +use syntax::{ast, attr}; +use syntax_expand::base::{ExtCtxt, Resolver}; +use syntax_expand::expand::ExpansionConfig; +use syntax_pos::hygiene::AstPass; +use syntax_pos::DUMMY_SP; + +pub fn inject( + mut krate: ast::Crate, + resolver: &mut dyn Resolver, + sess: &ParseSess, + alt_std_name: Option, +) -> (ast::Crate, Option) { + let rust_2018 = sess.edition >= Edition::Edition2018; + + // the first name in this list is the crate name of the crate with the prelude + let names: &[Symbol] = if attr::contains_name(&krate.attrs, sym::no_core) { + return (krate, None); + } else if attr::contains_name(&krate.attrs, sym::no_std) { + if attr::contains_name(&krate.attrs, sym::compiler_builtins) { + &[sym::core] + } else { + &[sym::core, sym::compiler_builtins] + } + } else { + &[sym::std] + }; + + let expn_id = resolver.expansion_for_ast_pass( + DUMMY_SP, + AstPass::StdImports, + &[sym::prelude_import], + None, + ); + let span = DUMMY_SP.with_def_site_ctxt(expn_id); + let call_site = DUMMY_SP.with_call_site_ctxt(expn_id); + + let ecfg = ExpansionConfig::default("std_lib_injection".to_string()); + let cx = ExtCtxt::new(sess, ecfg, resolver); + + // .rev() to preserve ordering above in combination with insert(0, ...) + for &name in names.iter().rev() { + let ident = if rust_2018 { Ident::new(name, span) } else { Ident::new(name, call_site) }; + krate.module.items.insert( + 0, + cx.item( + span, + ident, + vec![cx.attribute(cx.meta_word(span, sym::macro_use))], + ast::ItemKind::ExternCrate(alt_std_name), + ), + ); + } + + // The crates have been injected, the assumption is that the first one is + // the one with the prelude. + let name = names[0]; + + let import_path = if rust_2018 { + [name, sym::prelude, sym::v1].iter().map(|symbol| ast::Ident::new(*symbol, span)).collect() + } else { + [kw::PathRoot, name, sym::prelude, sym::v1] + .iter() + .map(|symbol| ast::Ident::new(*symbol, span)) + .collect() + }; + + let use_item = cx.item( + span, + ast::Ident::invalid(), + vec![cx.attribute(cx.meta_word(span, sym::prelude_import))], + ast::ItemKind::Use(P(ast::UseTree { + prefix: cx.path(span, import_path), + kind: ast::UseTreeKind::Glob, + span, + })), + ); + + krate.module.items.insert(0, use_item); + + (krate, Some(name)) +} diff --git a/src/librustc_builtin_macros/test.rs b/src/librustc_builtin_macros/test.rs new file mode 100644 index 00000000000..edf427edaae --- /dev/null +++ b/src/librustc_builtin_macros/test.rs @@ -0,0 +1,439 @@ +/// The expansion from a test function to the appropriate test struct for libtest +/// Ideally, this code would be in libtest but for efficiency and error messages it lives here. +use crate::util::check_builtin_macro_attribute; + +use syntax::ast; +use syntax::attr; +use syntax::print::pprust; +use syntax::source_map::respan; +use syntax::symbol::{sym, Symbol}; +use syntax_expand::base::*; +use syntax_pos::Span; + +use std::iter; + +// #[test_case] is used by custom test authors to mark tests +// When building for test, it needs to make the item public and gensym the name +// Otherwise, we'll omit the item. This behavior means that any item annotated +// with #[test_case] is never addressable. +// +// We mark item with an inert attribute "rustc_test_marker" which the test generation +// logic will pick up on. +pub fn expand_test_case( + ecx: &mut ExtCtxt<'_>, + attr_sp: Span, + meta_item: &ast::MetaItem, + anno_item: Annotatable, +) -> Vec { + check_builtin_macro_attribute(ecx, meta_item, sym::test_case); + + if !ecx.ecfg.should_test { + return vec![]; + } + + let sp = ecx.with_def_site_ctxt(attr_sp); + let mut item = anno_item.expect_item(); + item = item.map(|mut item| { + item.vis = respan(item.vis.span, ast::VisibilityKind::Public); + item.ident.span = item.ident.span.with_ctxt(sp.ctxt()); + item.attrs.push(ecx.attribute(ecx.meta_word(sp, sym::rustc_test_marker))); + item + }); + + return vec![Annotatable::Item(item)]; +} + +pub fn expand_test( + cx: &mut ExtCtxt<'_>, + attr_sp: Span, + meta_item: &ast::MetaItem, + item: Annotatable, +) -> Vec { + check_builtin_macro_attribute(cx, meta_item, sym::test); + expand_test_or_bench(cx, attr_sp, item, false) +} + +pub fn expand_bench( + cx: &mut ExtCtxt<'_>, + attr_sp: Span, + meta_item: &ast::MetaItem, + item: Annotatable, +) -> Vec { + check_builtin_macro_attribute(cx, meta_item, sym::bench); + expand_test_or_bench(cx, attr_sp, item, true) +} + +pub fn expand_test_or_bench( + cx: &mut ExtCtxt<'_>, + attr_sp: Span, + item: Annotatable, + is_bench: bool, +) -> Vec { + // If we're not in test configuration, remove the annotated item + if !cx.ecfg.should_test { + return vec![]; + } + + let item = if let Annotatable::Item(i) = item { + i + } else { + cx.parse_sess + .span_diagnostic + .span_fatal( + item.span(), + "`#[test]` attribute is only allowed on non associated functions", + ) + .raise(); + }; + + if let ast::ItemKind::Mac(_) = item.kind { + cx.parse_sess.span_diagnostic.span_warn( + item.span, + "`#[test]` attribute should not be used on macros. Use `#[cfg(test)]` instead.", + ); + return vec![Annotatable::Item(item)]; + } + + // has_*_signature will report any errors in the type so compilation + // will fail. We shouldn't try to expand in this case because the errors + // would be spurious. + if (!is_bench && !has_test_signature(cx, &item)) + || (is_bench && !has_bench_signature(cx, &item)) + { + return vec![Annotatable::Item(item)]; + } + + let (sp, attr_sp) = (cx.with_def_site_ctxt(item.span), cx.with_def_site_ctxt(attr_sp)); + + let test_id = ast::Ident::new(sym::test, attr_sp); + + // creates test::$name + let test_path = |name| cx.path(sp, vec![test_id, cx.ident_of(name, sp)]); + + // creates test::ShouldPanic::$name + let should_panic_path = + |name| cx.path(sp, vec![test_id, cx.ident_of("ShouldPanic", sp), cx.ident_of(name, sp)]); + + // creates test::TestType::$name + let test_type_path = + |name| cx.path(sp, vec![test_id, cx.ident_of("TestType", sp), cx.ident_of(name, sp)]); + + // creates $name: $expr + let field = |name, expr| cx.field_imm(sp, cx.ident_of(name, sp), expr); + + let test_fn = if is_bench { + // A simple ident for a lambda + let b = cx.ident_of("b", attr_sp); + + cx.expr_call( + sp, + cx.expr_path(test_path("StaticBenchFn")), + vec![ + // |b| self::test::assert_test_result( + cx.lambda1( + sp, + cx.expr_call( + sp, + cx.expr_path(test_path("assert_test_result")), + vec![ + // super::$test_fn(b) + cx.expr_call( + sp, + cx.expr_path(cx.path(sp, vec![item.ident])), + vec![cx.expr_ident(sp, b)], + ), + ], + ), + b, + ), // ) + ], + ) + } else { + cx.expr_call( + sp, + cx.expr_path(test_path("StaticTestFn")), + vec![ + // || { + cx.lambda0( + sp, + // test::assert_test_result( + cx.expr_call( + sp, + cx.expr_path(test_path("assert_test_result")), + vec![ + // $test_fn() + cx.expr_call(sp, cx.expr_path(cx.path(sp, vec![item.ident])), vec![]), // ) + ], + ), // } + ), // ) + ], + ) + }; + + let mut test_const = cx.item( + sp, + ast::Ident::new(item.ident.name, sp), + vec![ + // #[cfg(test)] + cx.attribute(attr::mk_list_item( + ast::Ident::new(sym::cfg, attr_sp), + vec![attr::mk_nested_word_item(ast::Ident::new(sym::test, attr_sp))], + )), + // #[rustc_test_marker] + cx.attribute(cx.meta_word(attr_sp, sym::rustc_test_marker)), + ], + // const $ident: test::TestDescAndFn = + ast::ItemKind::Const( + cx.ty(sp, ast::TyKind::Path(None, test_path("TestDescAndFn"))), + // test::TestDescAndFn { + cx.expr_struct( + sp, + test_path("TestDescAndFn"), + vec![ + // desc: test::TestDesc { + field( + "desc", + cx.expr_struct( + sp, + test_path("TestDesc"), + vec![ + // name: "path::to::test" + field( + "name", + cx.expr_call( + sp, + cx.expr_path(test_path("StaticTestName")), + vec![cx.expr_str( + sp, + Symbol::intern(&item_path( + // skip the name of the root module + &cx.current_expansion.module.mod_path[1..], + &item.ident, + )), + )], + ), + ), + // ignore: true | false + field("ignore", cx.expr_bool(sp, should_ignore(&item))), + // allow_fail: true | false + field("allow_fail", cx.expr_bool(sp, should_fail(&item))), + // should_panic: ... + field( + "should_panic", + match should_panic(cx, &item) { + // test::ShouldPanic::No + ShouldPanic::No => cx.expr_path(should_panic_path("No")), + // test::ShouldPanic::Yes + ShouldPanic::Yes(None) => { + cx.expr_path(should_panic_path("Yes")) + } + // test::ShouldPanic::YesWithMessage("...") + ShouldPanic::Yes(Some(sym)) => cx.expr_call( + sp, + cx.expr_path(should_panic_path("YesWithMessage")), + vec![cx.expr_str(sp, sym)], + ), + }, + ), + // test_type: ... + field( + "test_type", + match test_type(cx) { + // test::TestType::UnitTest + TestType::UnitTest => { + cx.expr_path(test_type_path("UnitTest")) + } + // test::TestType::IntegrationTest + TestType::IntegrationTest => { + cx.expr_path(test_type_path("IntegrationTest")) + } + // test::TestPath::Unknown + TestType::Unknown => { + cx.expr_path(test_type_path("Unknown")) + } + }, + ), + // }, + ], + ), + ), + // testfn: test::StaticTestFn(...) | test::StaticBenchFn(...) + field("testfn", test_fn), // } + ], + ), // } + ), + ); + test_const = test_const.map(|mut tc| { + tc.vis.node = ast::VisibilityKind::Public; + tc + }); + + // extern crate test + let test_extern = cx.item(sp, test_id, vec![], ast::ItemKind::ExternCrate(None)); + + log::debug!("synthetic test item:\n{}\n", pprust::item_to_string(&test_const)); + + vec![ + // Access to libtest under a hygienic name + Annotatable::Item(test_extern), + // The generated test case + Annotatable::Item(test_const), + // The original item + Annotatable::Item(item), + ] +} + +fn item_path(mod_path: &[ast::Ident], item_ident: &ast::Ident) -> String { + mod_path + .iter() + .chain(iter::once(item_ident)) + .map(|x| x.to_string()) + .collect::>() + .join("::") +} + +enum ShouldPanic { + No, + Yes(Option), +} + +fn should_ignore(i: &ast::Item) -> bool { + attr::contains_name(&i.attrs, sym::ignore) +} + +fn should_fail(i: &ast::Item) -> bool { + attr::contains_name(&i.attrs, sym::allow_fail) +} + +fn should_panic(cx: &ExtCtxt<'_>, i: &ast::Item) -> ShouldPanic { + match attr::find_by_name(&i.attrs, sym::should_panic) { + Some(attr) => { + let ref sd = cx.parse_sess.span_diagnostic; + + match attr.meta_item_list() { + // Handle #[should_panic(expected = "foo")] + Some(list) => { + let msg = list + .iter() + .find(|mi| mi.check_name(sym::expected)) + .and_then(|mi| mi.meta_item()) + .and_then(|mi| mi.value_str()); + if list.len() != 1 || msg.is_none() { + sd.struct_span_warn( + attr.span, + "argument must be of the form: \ + `expected = \"error message\"`", + ) + .note( + "Errors in this attribute were erroneously \ + allowed and will become a hard error in a \ + future release.", + ) + .emit(); + ShouldPanic::Yes(None) + } else { + ShouldPanic::Yes(msg) + } + } + // Handle #[should_panic] and #[should_panic = "expected"] + None => ShouldPanic::Yes(attr.value_str()), + } + } + None => ShouldPanic::No, + } +} + +enum TestType { + UnitTest, + IntegrationTest, + Unknown, +} + +/// Attempts to determine the type of test. +/// Since doctests are created without macro expanding, only possible variants here +/// are `UnitTest`, `IntegrationTest` or `Unknown`. +fn test_type(cx: &ExtCtxt<'_>) -> TestType { + // Root path from context contains the topmost sources directory of the crate. + // I.e., for `project` with sources in `src` and tests in `tests` folders + // (no matter how many nested folders lie inside), + // there will be two different root paths: `/project/src` and `/project/tests`. + let crate_path = cx.root_path.as_path(); + + if crate_path.ends_with("src") { + // `/src` folder contains unit-tests. + TestType::UnitTest + } else if crate_path.ends_with("tests") { + // `/tests` folder contains integration tests. + TestType::IntegrationTest + } else { + // Crate layout doesn't match expected one, test type is unknown. + TestType::Unknown + } +} + +fn has_test_signature(cx: &ExtCtxt<'_>, i: &ast::Item) -> bool { + let has_should_panic_attr = attr::contains_name(&i.attrs, sym::should_panic); + let ref sd = cx.parse_sess.span_diagnostic; + if let ast::ItemKind::Fn(ref sig, ref generics, _) = i.kind { + if sig.header.unsafety == ast::Unsafety::Unsafe { + sd.span_err(i.span, "unsafe functions cannot be used for tests"); + return false; + } + if sig.header.asyncness.node.is_async() { + sd.span_err(i.span, "async functions cannot be used for tests"); + return false; + } + + // If the termination trait is active, the compiler will check that the output + // type implements the `Termination` trait as `libtest` enforces that. + let has_output = match sig.decl.output { + ast::FunctionRetTy::Default(..) => false, + ast::FunctionRetTy::Ty(ref t) if t.kind.is_unit() => false, + _ => true, + }; + + if !sig.decl.inputs.is_empty() { + sd.span_err(i.span, "functions used as tests can not have any arguments"); + return false; + } + + match (has_output, has_should_panic_attr) { + (true, true) => { + sd.span_err(i.span, "functions using `#[should_panic]` must return `()`"); + false + } + (true, false) => { + if !generics.params.is_empty() { + sd.span_err(i.span, "functions used as tests must have signature fn() -> ()"); + false + } else { + true + } + } + (false, _) => true, + } + } else { + sd.span_err(i.span, "only functions may be used as tests"); + false + } +} + +fn has_bench_signature(cx: &ExtCtxt<'_>, i: &ast::Item) -> bool { + let has_sig = if let ast::ItemKind::Fn(ref sig, _, _) = i.kind { + // N.B., inadequate check, but we're running + // well before resolve, can't get too deep. + sig.decl.inputs.len() == 1 + } else { + false + }; + + if !has_sig { + cx.parse_sess.span_diagnostic.span_err( + i.span, + "functions used as benches must have \ + signature `fn(&mut Bencher) -> impl Termination`", + ); + } + + has_sig +} diff --git a/src/librustc_builtin_macros/test_harness.rs b/src/librustc_builtin_macros/test_harness.rs new file mode 100644 index 00000000000..b00fc3d26c1 --- /dev/null +++ b/src/librustc_builtin_macros/test_harness.rs @@ -0,0 +1,366 @@ +// Code that generates a test runner to run all the tests in a crate + +use log::debug; +use rustc_feature::Features; +use rustc_target::spec::PanicStrategy; +use smallvec::{smallvec, SmallVec}; +use syntax::ast::{self, Ident}; +use syntax::attr; +use syntax::entry::{self, EntryPointType}; +use syntax::mut_visit::{ExpectOne, *}; +use syntax::ptr::P; +use syntax::sess::ParseSess; +use syntax::source_map::respan; +use syntax::symbol::{sym, Symbol}; +use syntax_expand::base::{ExtCtxt, Resolver}; +use syntax_expand::expand::{AstFragment, ExpansionConfig}; +use syntax_pos::hygiene::{AstPass, SyntaxContext, Transparency}; +use syntax_pos::{Span, DUMMY_SP}; + +use std::{iter, mem}; + +struct Test { + span: Span, + ident: Ident, +} + +struct TestCtxt<'a> { + ext_cx: ExtCtxt<'a>, + panic_strategy: PanicStrategy, + def_site: Span, + test_cases: Vec, + reexport_test_harness_main: Option, + test_runner: Option, +} + +// Traverse the crate, collecting all the test functions, eliding any +// existing main functions, and synthesizing a main test harness +pub fn inject( + sess: &ParseSess, + resolver: &mut dyn Resolver, + should_test: bool, + krate: &mut ast::Crate, + span_diagnostic: &errors::Handler, + features: &Features, + panic_strategy: PanicStrategy, + platform_panic_strategy: PanicStrategy, + enable_panic_abort_tests: bool, +) { + // Check for #![reexport_test_harness_main = "some_name"] which gives the + // main test function the name `some_name` without hygiene. This needs to be + // unconditional, so that the attribute is still marked as used in + // non-test builds. + let reexport_test_harness_main = + attr::first_attr_value_str_by_name(&krate.attrs, sym::reexport_test_harness_main); + + // Do this here so that the test_runner crate attribute gets marked as used + // even in non-test builds + let test_runner = get_test_runner(span_diagnostic, &krate); + + if should_test { + let panic_strategy = match (panic_strategy, enable_panic_abort_tests) { + (PanicStrategy::Abort, true) => PanicStrategy::Abort, + (PanicStrategy::Abort, false) if panic_strategy == platform_panic_strategy => { + // Silently allow compiling with panic=abort on these platforms, + // but with old behavior (abort if a test fails). + PanicStrategy::Unwind + } + (PanicStrategy::Abort, false) => { + span_diagnostic.err( + "building tests with panic=abort is not supported \ + without `-Zpanic_abort_tests`", + ); + PanicStrategy::Unwind + } + (PanicStrategy::Unwind, _) => PanicStrategy::Unwind, + }; + generate_test_harness( + sess, + resolver, + reexport_test_harness_main, + krate, + features, + panic_strategy, + test_runner, + ) + } +} + +struct TestHarnessGenerator<'a> { + cx: TestCtxt<'a>, + tests: Vec, +} + +impl<'a> MutVisitor for TestHarnessGenerator<'a> { + fn visit_crate(&mut self, c: &mut ast::Crate) { + noop_visit_crate(c, self); + + // Create a main function to run our tests + c.module.items.push(mk_main(&mut self.cx)); + } + + fn flat_map_item(&mut self, i: P) -> SmallVec<[P; 1]> { + let mut item = i.into_inner(); + if is_test_case(&item) { + debug!("this is a test item"); + + let test = Test { span: item.span, ident: item.ident }; + self.tests.push(test); + } + + // We don't want to recurse into anything other than mods, since + // mods or tests inside of functions will break things + if let ast::ItemKind::Mod(mut module) = item.kind { + let tests = mem::take(&mut self.tests); + noop_visit_mod(&mut module, self); + let mut tests = mem::replace(&mut self.tests, tests); + + if !tests.is_empty() { + let parent = + if item.id == ast::DUMMY_NODE_ID { ast::CRATE_NODE_ID } else { item.id }; + // Create an identifier that will hygienically resolve the test + // case name, even in another module. + let expn_id = self.cx.ext_cx.resolver.expansion_for_ast_pass( + module.inner, + AstPass::TestHarness, + &[], + Some(parent), + ); + for test in &mut tests { + // See the comment on `mk_main` for why we're using + // `apply_mark` directly. + test.ident.span = test.ident.span.apply_mark(expn_id, Transparency::Opaque); + } + self.cx.test_cases.extend(tests); + } + item.kind = ast::ItemKind::Mod(module); + } + smallvec![P(item)] + } + + fn visit_mac(&mut self, _mac: &mut ast::Mac) { + // Do nothing. + } +} + +/// A folder used to remove any entry points (like fn main) because the harness +/// generator will provide its own +struct EntryPointCleaner { + // Current depth in the ast + depth: usize, + def_site: Span, +} + +impl MutVisitor for EntryPointCleaner { + fn flat_map_item(&mut self, i: P) -> SmallVec<[P; 1]> { + self.depth += 1; + let item = noop_flat_map_item(i, self).expect_one("noop did something"); + self.depth -= 1; + + // Remove any #[main] or #[start] from the AST so it doesn't + // clash with the one we're going to add, but mark it as + // #[allow(dead_code)] to avoid printing warnings. + let item = match entry::entry_point_type(&item, self.depth) { + EntryPointType::MainNamed | EntryPointType::MainAttr | EntryPointType::Start => item + .map(|ast::Item { id, ident, attrs, kind, vis, span, tokens }| { + let allow_ident = Ident::new(sym::allow, self.def_site); + let dc_nested = attr::mk_nested_word_item(Ident::from_str_and_span( + "dead_code", + self.def_site, + )); + let allow_dead_code_item = attr::mk_list_item(allow_ident, vec![dc_nested]); + let allow_dead_code = attr::mk_attr_outer(allow_dead_code_item); + + ast::Item { + id, + ident, + attrs: attrs + .into_iter() + .filter(|attr| { + !attr.check_name(sym::main) && !attr.check_name(sym::start) + }) + .chain(iter::once(allow_dead_code)) + .collect(), + kind, + vis, + span, + tokens, + } + }), + EntryPointType::None | EntryPointType::OtherMain => item, + }; + + smallvec![item] + } + + fn visit_mac(&mut self, _mac: &mut ast::Mac) { + // Do nothing. + } +} + +/// Crawl over the crate, inserting test reexports and the test main function +fn generate_test_harness( + sess: &ParseSess, + resolver: &mut dyn Resolver, + reexport_test_harness_main: Option, + krate: &mut ast::Crate, + features: &Features, + panic_strategy: PanicStrategy, + test_runner: Option, +) { + let mut econfig = ExpansionConfig::default("test".to_string()); + econfig.features = Some(features); + + let ext_cx = ExtCtxt::new(sess, econfig, resolver); + + let expn_id = ext_cx.resolver.expansion_for_ast_pass( + DUMMY_SP, + AstPass::TestHarness, + &[sym::main, sym::test, sym::rustc_attrs], + None, + ); + let def_site = DUMMY_SP.with_def_site_ctxt(expn_id); + + // Remove the entry points + let mut cleaner = EntryPointCleaner { depth: 0, def_site }; + cleaner.visit_crate(krate); + + let cx = TestCtxt { + ext_cx, + panic_strategy, + def_site, + test_cases: Vec::new(), + reexport_test_harness_main, + test_runner, + }; + + TestHarnessGenerator { cx, tests: Vec::new() }.visit_crate(krate); +} + +/// Creates a function item for use as the main function of a test build. +/// This function will call the `test_runner` as specified by the crate attribute +/// +/// By default this expands to +/// +/// #[main] +/// pub fn main() { +/// extern crate test; +/// test::test_main_static(&[ +/// &test_const1, +/// &test_const2, +/// &test_const3, +/// ]); +/// } +/// +/// Most of the Ident have the usual def-site hygiene for the AST pass. The +/// exception is the `test_const`s. These have a syntax context that has two +/// opaque marks: one from the expansion of `test` or `test_case`, and one +/// generated in `TestHarnessGenerator::flat_map_item`. When resolving this +/// identifier after failing to find a matching identifier in the root module +/// we remove the outer mark, and try resolving at its def-site, which will +/// then resolve to `test_const`. +/// +/// The expansion here can be controlled by two attributes: +/// +/// `reexport_test_harness_main` provides a different name for the `main` +/// function and `test_runner` provides a path that replaces +/// `test::test_main_static`. +fn mk_main(cx: &mut TestCtxt<'_>) -> P { + let sp = cx.def_site; + let ecx = &cx.ext_cx; + let test_id = Ident::new(sym::test, sp); + + let runner_name = match cx.panic_strategy { + PanicStrategy::Unwind => "test_main_static", + PanicStrategy::Abort => "test_main_static_abort", + }; + + // test::test_main_static(...) + let mut test_runner = cx + .test_runner + .clone() + .unwrap_or(ecx.path(sp, vec![test_id, ecx.ident_of(runner_name, sp)])); + + test_runner.span = sp; + + let test_main_path_expr = ecx.expr_path(test_runner); + let call_test_main = ecx.expr_call(sp, test_main_path_expr, vec![mk_tests_slice(cx, sp)]); + let call_test_main = ecx.stmt_expr(call_test_main); + + // extern crate test + let test_extern_stmt = + ecx.stmt_item(sp, ecx.item(sp, test_id, vec![], ast::ItemKind::ExternCrate(None))); + + // #[main] + let main_meta = ecx.meta_word(sp, sym::main); + let main_attr = ecx.attribute(main_meta); + + // pub fn main() { ... } + let main_ret_ty = ecx.ty(sp, ast::TyKind::Tup(vec![])); + + // If no test runner is provided we need to import the test crate + let main_body = if cx.test_runner.is_none() { + ecx.block(sp, vec![test_extern_stmt, call_test_main]) + } else { + ecx.block(sp, vec![call_test_main]) + }; + + let decl = ecx.fn_decl(vec![], ast::FunctionRetTy::Ty(main_ret_ty)); + let sig = ast::FnSig { decl, header: ast::FnHeader::default() }; + let main = ast::ItemKind::Fn(sig, ast::Generics::default(), main_body); + + // Honor the reexport_test_harness_main attribute + let main_id = match cx.reexport_test_harness_main { + Some(sym) => Ident::new(sym, sp.with_ctxt(SyntaxContext::root())), + None => Ident::new(sym::main, sp), + }; + + let main = P(ast::Item { + ident: main_id, + attrs: vec![main_attr], + id: ast::DUMMY_NODE_ID, + kind: main, + vis: respan(sp, ast::VisibilityKind::Public), + span: sp, + tokens: None, + }); + + // Integrate the new item into existing module structures. + let main = AstFragment::Items(smallvec![main]); + cx.ext_cx.monotonic_expander().fully_expand_fragment(main).make_items().pop().unwrap() +} + +/// Creates a slice containing every test like so: +/// &[&test1, &test2] +fn mk_tests_slice(cx: &TestCtxt<'_>, sp: Span) -> P { + debug!("building test vector from {} tests", cx.test_cases.len()); + let ref ecx = cx.ext_cx; + + ecx.expr_vec_slice( + sp, + cx.test_cases + .iter() + .map(|test| { + ecx.expr_addr_of(test.span, ecx.expr_path(ecx.path(test.span, vec![test.ident]))) + }) + .collect(), + ) +} + +fn is_test_case(i: &ast::Item) -> bool { + attr::contains_name(&i.attrs, sym::rustc_test_marker) +} + +fn get_test_runner(sd: &errors::Handler, krate: &ast::Crate) -> Option { + let test_attr = attr::find_by_name(&krate.attrs, sym::test_runner)?; + test_attr.meta_item_list().map(|meta_list| { + if meta_list.len() != 1 { + sd.span_fatal(test_attr.span, "`#![test_runner(..)]` accepts exactly 1 argument") + .raise() + } + match meta_list[0].meta_item() { + Some(meta_item) if meta_item.is_word() => meta_item.path.clone(), + _ => sd.span_fatal(test_attr.span, "`test_runner` argument must be a path").raise(), + } + }) +} diff --git a/src/librustc_builtin_macros/trace_macros.rs b/src/librustc_builtin_macros/trace_macros.rs new file mode 100644 index 00000000000..96ae5bf5b4e --- /dev/null +++ b/src/librustc_builtin_macros/trace_macros.rs @@ -0,0 +1,29 @@ +use syntax::symbol::kw; +use syntax::tokenstream::{TokenStream, TokenTree}; +use syntax_expand::base::{self, ExtCtxt}; +use syntax_pos::Span; + +pub fn expand_trace_macros( + cx: &mut ExtCtxt<'_>, + sp: Span, + tt: TokenStream, +) -> Box { + let mut cursor = tt.into_trees(); + let mut err = false; + let value = match &cursor.next() { + Some(TokenTree::Token(token)) if token.is_keyword(kw::True) => true, + Some(TokenTree::Token(token)) if token.is_keyword(kw::False) => false, + _ => { + err = true; + false + } + }; + err |= cursor.next().is_some(); + if err { + cx.span_err(sp, "trace_macros! accepts only `true` or `false`") + } else { + cx.set_trace_macros(value); + } + + base::DummyResult::any_valid(sp) +} diff --git a/src/librustc_builtin_macros/util.rs b/src/librustc_builtin_macros/util.rs new file mode 100644 index 00000000000..aedd5aac1a9 --- /dev/null +++ b/src/librustc_builtin_macros/util.rs @@ -0,0 +1,12 @@ +use rustc_feature::AttributeTemplate; +use rustc_parse::validate_attr; +use syntax::ast::MetaItem; +use syntax_expand::base::ExtCtxt; +use syntax_pos::Symbol; + +pub fn check_builtin_macro_attribute(ecx: &ExtCtxt<'_>, meta_item: &MetaItem, name: Symbol) { + // All the built-in macro attributes are "words" at the moment. + let template = AttributeTemplate::only_word(); + let attr = ecx.attribute(meta_item.clone()); + validate_attr::check_builtin_attribute(ecx.parse_sess, &attr, name, template); +} diff --git a/src/librustc_expand/Cargo.toml b/src/librustc_expand/Cargo.toml new file mode 100644 index 00000000000..897d5a65ba3 --- /dev/null +++ b/src/librustc_expand/Cargo.toml @@ -0,0 +1,23 @@ +[package] +authors = ["The Rust Project Developers"] +name = "syntax_expand" +version = "0.0.0" +edition = "2018" +build = false + +[lib] +name = "syntax_expand" +path = "lib.rs" +doctest = false + +[dependencies] +rustc_serialize = { path = "../libserialize", package = "serialize" } +log = "0.4" +syntax_pos = { path = "../libsyntax_pos" } +errors = { path = "../librustc_errors", package = "rustc_errors" } +rustc_data_structures = { path = "../librustc_data_structures" } +rustc_feature = { path = "../librustc_feature" } +rustc_lexer = { path = "../librustc_lexer" } +rustc_parse = { path = "../librustc_parse" } +smallvec = { version = "1.0", features = ["union", "may_dangle"] } +syntax = { path = "../libsyntax" } diff --git a/src/librustc_expand/base.rs b/src/librustc_expand/base.rs new file mode 100644 index 00000000000..60bc591c095 --- /dev/null +++ b/src/librustc_expand/base.rs @@ -0,0 +1,1181 @@ +use crate::expand::{self, AstFragment, Invocation}; + +use rustc_parse::{self, parser, DirectoryOwnership, MACRO_ARGUMENTS}; +use syntax::ast::{self, Attribute, Name, NodeId, PatKind}; +use syntax::attr::{self, Deprecation, HasAttrs, Stability}; +use syntax::edition::Edition; +use syntax::mut_visit::{self, MutVisitor}; +use syntax::ptr::P; +use syntax::sess::ParseSess; +use syntax::source_map::SourceMap; +use syntax::symbol::{kw, sym, Ident, Symbol}; +use syntax::token; +use syntax::tokenstream::{self, TokenStream}; +use syntax::visit::Visitor; + +use errors::{DiagnosticBuilder, DiagnosticId}; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::{self, Lrc}; +use smallvec::{smallvec, SmallVec}; +use syntax_pos::hygiene::{AstPass, ExpnData, ExpnId, ExpnKind}; +use syntax_pos::{FileName, MultiSpan, Span, DUMMY_SP}; + +use std::default::Default; +use std::iter; +use std::path::PathBuf; +use std::rc::Rc; + +crate use syntax_pos::hygiene::MacroKind; + +#[derive(Debug, Clone)] +pub enum Annotatable { + Item(P), + TraitItem(P), + ImplItem(P), + ForeignItem(P), + Stmt(P), + Expr(P), + Arm(ast::Arm), + Field(ast::Field), + FieldPat(ast::FieldPat), + GenericParam(ast::GenericParam), + Param(ast::Param), + StructField(ast::StructField), + Variant(ast::Variant), +} + +impl HasAttrs for Annotatable { + fn attrs(&self) -> &[Attribute] { + match *self { + Annotatable::Item(ref item) => &item.attrs, + Annotatable::TraitItem(ref trait_item) => &trait_item.attrs, + Annotatable::ImplItem(ref impl_item) => &impl_item.attrs, + Annotatable::ForeignItem(ref foreign_item) => &foreign_item.attrs, + Annotatable::Stmt(ref stmt) => stmt.attrs(), + Annotatable::Expr(ref expr) => &expr.attrs, + Annotatable::Arm(ref arm) => &arm.attrs, + Annotatable::Field(ref field) => &field.attrs, + Annotatable::FieldPat(ref fp) => &fp.attrs, + Annotatable::GenericParam(ref gp) => &gp.attrs, + Annotatable::Param(ref p) => &p.attrs, + Annotatable::StructField(ref sf) => &sf.attrs, + Annotatable::Variant(ref v) => &v.attrs(), + } + } + + fn visit_attrs)>(&mut self, f: F) { + match self { + Annotatable::Item(item) => item.visit_attrs(f), + Annotatable::TraitItem(trait_item) => trait_item.visit_attrs(f), + Annotatable::ImplItem(impl_item) => impl_item.visit_attrs(f), + Annotatable::ForeignItem(foreign_item) => foreign_item.visit_attrs(f), + Annotatable::Stmt(stmt) => stmt.visit_attrs(f), + Annotatable::Expr(expr) => expr.visit_attrs(f), + Annotatable::Arm(arm) => arm.visit_attrs(f), + Annotatable::Field(field) => field.visit_attrs(f), + Annotatable::FieldPat(fp) => fp.visit_attrs(f), + Annotatable::GenericParam(gp) => gp.visit_attrs(f), + Annotatable::Param(p) => p.visit_attrs(f), + Annotatable::StructField(sf) => sf.visit_attrs(f), + Annotatable::Variant(v) => v.visit_attrs(f), + } + } +} + +impl Annotatable { + pub fn span(&self) -> Span { + match *self { + Annotatable::Item(ref item) => item.span, + Annotatable::TraitItem(ref trait_item) => trait_item.span, + Annotatable::ImplItem(ref impl_item) => impl_item.span, + Annotatable::ForeignItem(ref foreign_item) => foreign_item.span, + Annotatable::Stmt(ref stmt) => stmt.span, + Annotatable::Expr(ref expr) => expr.span, + Annotatable::Arm(ref arm) => arm.span, + Annotatable::Field(ref field) => field.span, + Annotatable::FieldPat(ref fp) => fp.pat.span, + Annotatable::GenericParam(ref gp) => gp.ident.span, + Annotatable::Param(ref p) => p.span, + Annotatable::StructField(ref sf) => sf.span, + Annotatable::Variant(ref v) => v.span, + } + } + + pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { + match self { + Annotatable::Item(item) => visitor.visit_item(item), + Annotatable::TraitItem(trait_item) => visitor.visit_trait_item(trait_item), + Annotatable::ImplItem(impl_item) => visitor.visit_impl_item(impl_item), + Annotatable::ForeignItem(foreign_item) => visitor.visit_foreign_item(foreign_item), + Annotatable::Stmt(stmt) => visitor.visit_stmt(stmt), + Annotatable::Expr(expr) => visitor.visit_expr(expr), + Annotatable::Arm(arm) => visitor.visit_arm(arm), + Annotatable::Field(field) => visitor.visit_field(field), + Annotatable::FieldPat(fp) => visitor.visit_field_pattern(fp), + Annotatable::GenericParam(gp) => visitor.visit_generic_param(gp), + Annotatable::Param(p) => visitor.visit_param(p), + Annotatable::StructField(sf) => visitor.visit_struct_field(sf), + Annotatable::Variant(v) => visitor.visit_variant(v), + } + } + + pub fn expect_item(self) -> P { + match self { + Annotatable::Item(i) => i, + _ => panic!("expected Item"), + } + } + + pub fn map_item_or(self, mut f: F, mut or: G) -> Annotatable + where + F: FnMut(P) -> P, + G: FnMut(Annotatable) -> Annotatable, + { + match self { + Annotatable::Item(i) => Annotatable::Item(f(i)), + _ => or(self), + } + } + + pub fn expect_trait_item(self) -> ast::AssocItem { + match self { + Annotatable::TraitItem(i) => i.into_inner(), + _ => panic!("expected Item"), + } + } + + pub fn expect_impl_item(self) -> ast::AssocItem { + match self { + Annotatable::ImplItem(i) => i.into_inner(), + _ => panic!("expected Item"), + } + } + + pub fn expect_foreign_item(self) -> ast::ForeignItem { + match self { + Annotatable::ForeignItem(i) => i.into_inner(), + _ => panic!("expected foreign item"), + } + } + + pub fn expect_stmt(self) -> ast::Stmt { + match self { + Annotatable::Stmt(stmt) => stmt.into_inner(), + _ => panic!("expected statement"), + } + } + + pub fn expect_expr(self) -> P { + match self { + Annotatable::Expr(expr) => expr, + _ => panic!("expected expression"), + } + } + + pub fn expect_arm(self) -> ast::Arm { + match self { + Annotatable::Arm(arm) => arm, + _ => panic!("expected match arm"), + } + } + + pub fn expect_field(self) -> ast::Field { + match self { + Annotatable::Field(field) => field, + _ => panic!("expected field"), + } + } + + pub fn expect_field_pattern(self) -> ast::FieldPat { + match self { + Annotatable::FieldPat(fp) => fp, + _ => panic!("expected field pattern"), + } + } + + pub fn expect_generic_param(self) -> ast::GenericParam { + match self { + Annotatable::GenericParam(gp) => gp, + _ => panic!("expected generic parameter"), + } + } + + pub fn expect_param(self) -> ast::Param { + match self { + Annotatable::Param(param) => param, + _ => panic!("expected parameter"), + } + } + + pub fn expect_struct_field(self) -> ast::StructField { + match self { + Annotatable::StructField(sf) => sf, + _ => panic!("expected struct field"), + } + } + + pub fn expect_variant(self) -> ast::Variant { + match self { + Annotatable::Variant(v) => v, + _ => panic!("expected variant"), + } + } + + pub fn derive_allowed(&self) -> bool { + match *self { + Annotatable::Item(ref item) => match item.kind { + ast::ItemKind::Struct(..) | ast::ItemKind::Enum(..) | ast::ItemKind::Union(..) => { + true + } + _ => false, + }, + _ => false, + } + } +} + +// `meta_item` is the annotation, and `item` is the item being modified. +// FIXME Decorators should follow the same pattern too. +pub trait MultiItemModifier { + fn expand( + &self, + ecx: &mut ExtCtxt<'_>, + span: Span, + meta_item: &ast::MetaItem, + item: Annotatable, + ) -> Vec; +} + +impl MultiItemModifier for F +where + F: Fn(&mut ExtCtxt<'_>, Span, &ast::MetaItem, Annotatable) -> T, + T: Into>, +{ + fn expand( + &self, + ecx: &mut ExtCtxt<'_>, + span: Span, + meta_item: &ast::MetaItem, + item: Annotatable, + ) -> Vec { + (*self)(ecx, span, meta_item, item).into() + } +} + +impl Into> for Annotatable { + fn into(self) -> Vec { + vec![self] + } +} + +pub trait ProcMacro { + fn expand<'cx>(&self, ecx: &'cx mut ExtCtxt<'_>, span: Span, ts: TokenStream) -> TokenStream; +} + +impl ProcMacro for F +where + F: Fn(TokenStream) -> TokenStream, +{ + fn expand<'cx>(&self, _ecx: &'cx mut ExtCtxt<'_>, _span: Span, ts: TokenStream) -> TokenStream { + // FIXME setup implicit context in TLS before calling self. + (*self)(ts) + } +} + +pub trait AttrProcMacro { + fn expand<'cx>( + &self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + annotation: TokenStream, + annotated: TokenStream, + ) -> TokenStream; +} + +impl AttrProcMacro for F +where + F: Fn(TokenStream, TokenStream) -> TokenStream, +{ + fn expand<'cx>( + &self, + _ecx: &'cx mut ExtCtxt<'_>, + _span: Span, + annotation: TokenStream, + annotated: TokenStream, + ) -> TokenStream { + // FIXME setup implicit context in TLS before calling self. + (*self)(annotation, annotated) + } +} + +/// Represents a thing that maps token trees to Macro Results +pub trait TTMacroExpander { + fn expand<'cx>( + &self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + input: TokenStream, + ) -> Box; +} + +pub type MacroExpanderFn = + for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) -> Box; + +impl TTMacroExpander for F +where + F: for<'cx> Fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) -> Box, +{ + fn expand<'cx>( + &self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + mut input: TokenStream, + ) -> Box { + struct AvoidInterpolatedIdents; + + impl MutVisitor for AvoidInterpolatedIdents { + fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) { + if let tokenstream::TokenTree::Token(token) = tt { + if let token::Interpolated(nt) = &token.kind { + if let token::NtIdent(ident, is_raw) = **nt { + *tt = tokenstream::TokenTree::token( + token::Ident(ident.name, is_raw), + ident.span, + ); + } + } + } + mut_visit::noop_visit_tt(tt, self) + } + + fn visit_mac(&mut self, mac: &mut ast::Mac) { + mut_visit::noop_visit_mac(mac, self) + } + } + AvoidInterpolatedIdents.visit_tts(&mut input); + (*self)(ecx, span, input) + } +} + +// Use a macro because forwarding to a simple function has type system issues +macro_rules! make_stmts_default { + ($me:expr) => { + $me.make_expr().map(|e| { + smallvec![ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: e.span, + kind: ast::StmtKind::Expr(e), + }] + }) + }; +} + +/// The result of a macro expansion. The return values of the various +/// methods are spliced into the AST at the callsite of the macro. +pub trait MacResult { + /// Creates an expression. + fn make_expr(self: Box) -> Option> { + None + } + /// Creates zero or more items. + fn make_items(self: Box) -> Option; 1]>> { + None + } + + /// Creates zero or more impl items. + fn make_impl_items(self: Box) -> Option> { + None + } + + /// Creates zero or more trait items. + fn make_trait_items(self: Box) -> Option> { + None + } + + /// Creates zero or more items in an `extern {}` block + fn make_foreign_items(self: Box) -> Option> { + None + } + + /// Creates a pattern. + fn make_pat(self: Box) -> Option> { + None + } + + /// Creates zero or more statements. + /// + /// By default this attempts to create an expression statement, + /// returning None if that fails. + fn make_stmts(self: Box) -> Option> { + make_stmts_default!(self) + } + + fn make_ty(self: Box) -> Option> { + None + } + + fn make_arms(self: Box) -> Option> { + None + } + + fn make_fields(self: Box) -> Option> { + None + } + + fn make_field_patterns(self: Box) -> Option> { + None + } + + fn make_generic_params(self: Box) -> Option> { + None + } + + fn make_params(self: Box) -> Option> { + None + } + + fn make_struct_fields(self: Box) -> Option> { + None + } + + fn make_variants(self: Box) -> Option> { + None + } +} + +macro_rules! make_MacEager { + ( $( $fld:ident: $t:ty, )* ) => { + /// `MacResult` implementation for the common case where you've already + /// built each form of AST that you might return. + #[derive(Default)] + pub struct MacEager { + $( + pub $fld: Option<$t>, + )* + } + + impl MacEager { + $( + pub fn $fld(v: $t) -> Box { + Box::new(MacEager { + $fld: Some(v), + ..Default::default() + }) + } + )* + } + } +} + +make_MacEager! { + expr: P, + pat: P, + items: SmallVec<[P; 1]>, + impl_items: SmallVec<[ast::AssocItem; 1]>, + trait_items: SmallVec<[ast::AssocItem; 1]>, + foreign_items: SmallVec<[ast::ForeignItem; 1]>, + stmts: SmallVec<[ast::Stmt; 1]>, + ty: P, +} + +impl MacResult for MacEager { + fn make_expr(self: Box) -> Option> { + self.expr + } + + fn make_items(self: Box) -> Option; 1]>> { + self.items + } + + fn make_impl_items(self: Box) -> Option> { + self.impl_items + } + + fn make_trait_items(self: Box) -> Option> { + self.trait_items + } + + fn make_foreign_items(self: Box) -> Option> { + self.foreign_items + } + + fn make_stmts(self: Box) -> Option> { + match self.stmts.as_ref().map_or(0, |s| s.len()) { + 0 => make_stmts_default!(self), + _ => self.stmts, + } + } + + fn make_pat(self: Box) -> Option> { + if let Some(p) = self.pat { + return Some(p); + } + if let Some(e) = self.expr { + if let ast::ExprKind::Lit(_) = e.kind { + return Some(P(ast::Pat { + id: ast::DUMMY_NODE_ID, + span: e.span, + kind: PatKind::Lit(e), + })); + } + } + None + } + + fn make_ty(self: Box) -> Option> { + self.ty + } +} + +/// Fill-in macro expansion result, to allow compilation to continue +/// after hitting errors. +#[derive(Copy, Clone)] +pub struct DummyResult { + is_error: bool, + span: Span, +} + +impl DummyResult { + /// Creates a default MacResult that can be anything. + /// + /// Use this as a return value after hitting any errors and + /// calling `span_err`. + pub fn any(span: Span) -> Box { + Box::new(DummyResult { is_error: true, span }) + } + + /// Same as `any`, but must be a valid fragment, not error. + pub fn any_valid(span: Span) -> Box { + Box::new(DummyResult { is_error: false, span }) + } + + /// A plain dummy expression. + pub fn raw_expr(sp: Span, is_error: bool) -> P { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) }, + span: sp, + attrs: ast::AttrVec::new(), + }) + } + + /// A plain dummy pattern. + pub fn raw_pat(sp: Span) -> ast::Pat { + ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp } + } + + /// A plain dummy type. + pub fn raw_ty(sp: Span, is_error: bool) -> P { + P(ast::Ty { + id: ast::DUMMY_NODE_ID, + kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) }, + span: sp, + }) + } +} + +impl MacResult for DummyResult { + fn make_expr(self: Box) -> Option> { + Some(DummyResult::raw_expr(self.span, self.is_error)) + } + + fn make_pat(self: Box) -> Option> { + Some(P(DummyResult::raw_pat(self.span))) + } + + fn make_items(self: Box) -> Option; 1]>> { + Some(SmallVec::new()) + } + + fn make_impl_items(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_trait_items(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_foreign_items(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_stmts(self: Box) -> Option> { + Some(smallvec![ast::Stmt { + id: ast::DUMMY_NODE_ID, + kind: ast::StmtKind::Expr(DummyResult::raw_expr(self.span, self.is_error)), + span: self.span, + }]) + } + + fn make_ty(self: Box) -> Option> { + Some(DummyResult::raw_ty(self.span, self.is_error)) + } + + fn make_arms(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_fields(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_field_patterns(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_generic_params(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_params(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_struct_fields(self: Box) -> Option> { + Some(SmallVec::new()) + } + + fn make_variants(self: Box) -> Option> { + Some(SmallVec::new()) + } +} + +/// A syntax extension kind. +pub enum SyntaxExtensionKind { + /// A token-based function-like macro. + Bang( + /// An expander with signature TokenStream -> TokenStream. + Box, + ), + + /// An AST-based function-like macro. + LegacyBang( + /// An expander with signature TokenStream -> AST. + Box, + ), + + /// A token-based attribute macro. + Attr( + /// An expander with signature (TokenStream, TokenStream) -> TokenStream. + /// The first TokenSteam is the attribute itself, the second is the annotated item. + /// The produced TokenSteam replaces the input TokenSteam. + Box, + ), + + /// An AST-based attribute macro. + LegacyAttr( + /// An expander with signature (AST, AST) -> AST. + /// The first AST fragment is the attribute itself, the second is the annotated item. + /// The produced AST fragment replaces the input AST fragment. + Box, + ), + + /// A trivial attribute "macro" that does nothing, + /// only keeps the attribute and marks it as inert, + /// thus making it ineligible for further expansion. + NonMacroAttr { + /// Suppresses the `unused_attributes` lint for this attribute. + mark_used: bool, + }, + + /// A token-based derive macro. + Derive( + /// An expander with signature TokenStream -> TokenStream (not yet). + /// The produced TokenSteam is appended to the input TokenSteam. + Box, + ), + + /// An AST-based derive macro. + LegacyDerive( + /// An expander with signature AST -> AST. + /// The produced AST fragment is appended to the input AST fragment. + Box, + ), +} + +/// A struct representing a macro definition in "lowered" form ready for expansion. +pub struct SyntaxExtension { + /// A syntax extension kind. + pub kind: SyntaxExtensionKind, + /// Span of the macro definition. + pub span: Span, + /// Whitelist of unstable features that are treated as stable inside this macro. + pub allow_internal_unstable: Option>, + /// Suppresses the `unsafe_code` lint for code produced by this macro. + pub allow_internal_unsafe: bool, + /// Enables the macro helper hack (`ident!(...)` -> `$crate::ident!(...)`) for this macro. + pub local_inner_macros: bool, + /// The macro's stability info. + pub stability: Option, + /// The macro's deprecation info. + pub deprecation: Option, + /// Names of helper attributes registered by this macro. + pub helper_attrs: Vec, + /// Edition of the crate in which this macro is defined. + pub edition: Edition, + /// Built-in macros have a couple of special properties like availability + /// in `#[no_implicit_prelude]` modules, so we have to keep this flag. + pub is_builtin: bool, + /// We have to identify macros providing a `Copy` impl early for compatibility reasons. + pub is_derive_copy: bool, +} + +impl SyntaxExtension { + /// Returns which kind of macro calls this syntax extension. + pub fn macro_kind(&self) -> MacroKind { + match self.kind { + SyntaxExtensionKind::Bang(..) | SyntaxExtensionKind::LegacyBang(..) => MacroKind::Bang, + SyntaxExtensionKind::Attr(..) + | SyntaxExtensionKind::LegacyAttr(..) + | SyntaxExtensionKind::NonMacroAttr { .. } => MacroKind::Attr, + SyntaxExtensionKind::Derive(..) | SyntaxExtensionKind::LegacyDerive(..) => { + MacroKind::Derive + } + } + } + + /// Constructs a syntax extension with default properties. + pub fn default(kind: SyntaxExtensionKind, edition: Edition) -> SyntaxExtension { + SyntaxExtension { + span: DUMMY_SP, + allow_internal_unstable: None, + allow_internal_unsafe: false, + local_inner_macros: false, + stability: None, + deprecation: None, + helper_attrs: Vec::new(), + edition, + is_builtin: false, + is_derive_copy: false, + kind, + } + } + + /// Constructs a syntax extension with the given properties + /// and other properties converted from attributes. + pub fn new( + sess: &ParseSess, + kind: SyntaxExtensionKind, + span: Span, + helper_attrs: Vec, + edition: Edition, + name: Name, + attrs: &[ast::Attribute], + ) -> SyntaxExtension { + let allow_internal_unstable = attr::allow_internal_unstable(&attrs, &sess.span_diagnostic) + .map(|features| features.collect::>().into()); + + let mut local_inner_macros = false; + if let Some(macro_export) = attr::find_by_name(attrs, sym::macro_export) { + if let Some(l) = macro_export.meta_item_list() { + local_inner_macros = attr::list_contains_name(&l, sym::local_inner_macros); + } + } + + let is_builtin = attr::contains_name(attrs, sym::rustc_builtin_macro); + let (stability, const_stability) = attr::find_stability(&sess, attrs, span); + if const_stability.is_some() { + sess.span_diagnostic.span_err(span, "macros cannot have const stability attributes"); + } + + SyntaxExtension { + kind, + span, + allow_internal_unstable, + allow_internal_unsafe: attr::contains_name(attrs, sym::allow_internal_unsafe), + local_inner_macros, + stability, + deprecation: attr::find_deprecation(&sess, attrs, span), + helper_attrs, + edition, + is_builtin, + is_derive_copy: is_builtin && name == sym::Copy, + } + } + + pub fn dummy_bang(edition: Edition) -> SyntaxExtension { + fn expander<'cx>( + _: &'cx mut ExtCtxt<'_>, + span: Span, + _: TokenStream, + ) -> Box { + DummyResult::any(span) + } + SyntaxExtension::default(SyntaxExtensionKind::LegacyBang(Box::new(expander)), edition) + } + + pub fn dummy_derive(edition: Edition) -> SyntaxExtension { + fn expander( + _: &mut ExtCtxt<'_>, + _: Span, + _: &ast::MetaItem, + _: Annotatable, + ) -> Vec { + Vec::new() + } + SyntaxExtension::default(SyntaxExtensionKind::Derive(Box::new(expander)), edition) + } + + pub fn non_macro_attr(mark_used: bool, edition: Edition) -> SyntaxExtension { + SyntaxExtension::default(SyntaxExtensionKind::NonMacroAttr { mark_used }, edition) + } + + pub fn expn_data(&self, parent: ExpnId, call_site: Span, descr: Symbol) -> ExpnData { + ExpnData { + kind: ExpnKind::Macro(self.macro_kind(), descr), + parent, + call_site, + def_site: self.span, + allow_internal_unstable: self.allow_internal_unstable.clone(), + allow_internal_unsafe: self.allow_internal_unsafe, + local_inner_macros: self.local_inner_macros, + edition: self.edition, + } + } +} + +/// Result of resolving a macro invocation. +pub enum InvocationRes { + Single(Lrc), + DeriveContainer(Vec>), +} + +/// Error type that denotes indeterminacy. +pub struct Indeterminate; + +pub trait Resolver { + fn next_node_id(&mut self) -> NodeId; + + fn resolve_dollar_crates(&mut self); + fn visit_ast_fragment_with_placeholders(&mut self, expn_id: ExpnId, fragment: &AstFragment); + fn register_builtin_macro(&mut self, ident: ast::Ident, ext: SyntaxExtension); + + fn expansion_for_ast_pass( + &mut self, + call_site: Span, + pass: AstPass, + features: &[Symbol], + parent_module_id: Option, + ) -> ExpnId; + + fn resolve_imports(&mut self); + + fn resolve_macro_invocation( + &mut self, + invoc: &Invocation, + eager_expansion_root: ExpnId, + force: bool, + ) -> Result; + + fn check_unused_macros(&mut self); + + fn has_derive_copy(&self, expn_id: ExpnId) -> bool; + fn add_derive_copy(&mut self, expn_id: ExpnId); +} + +#[derive(Clone)] +pub struct ModuleData { + pub mod_path: Vec, + pub directory: PathBuf, +} + +#[derive(Clone)] +pub struct ExpansionData { + pub id: ExpnId, + pub depth: usize, + pub module: Rc, + pub directory_ownership: DirectoryOwnership, + pub prior_type_ascription: Option<(Span, bool)>, +} + +/// One of these is made during expansion and incrementally updated as we go; +/// when a macro expansion occurs, the resulting nodes have the `backtrace() +/// -> expn_data` of their expansion context stored into their span. +pub struct ExtCtxt<'a> { + pub parse_sess: &'a ParseSess, + pub ecfg: expand::ExpansionConfig<'a>, + pub root_path: PathBuf, + pub resolver: &'a mut dyn Resolver, + pub current_expansion: ExpansionData, + pub expansions: FxHashMap>, +} + +impl<'a> ExtCtxt<'a> { + pub fn new( + parse_sess: &'a ParseSess, + ecfg: expand::ExpansionConfig<'a>, + resolver: &'a mut dyn Resolver, + ) -> ExtCtxt<'a> { + ExtCtxt { + parse_sess, + ecfg, + root_path: PathBuf::new(), + resolver, + current_expansion: ExpansionData { + id: ExpnId::root(), + depth: 0, + module: Rc::new(ModuleData { mod_path: Vec::new(), directory: PathBuf::new() }), + directory_ownership: DirectoryOwnership::Owned { relative: None }, + prior_type_ascription: None, + }, + expansions: FxHashMap::default(), + } + } + + /// Returns a `Folder` for deeply expanding all macros in an AST node. + pub fn expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { + expand::MacroExpander::new(self, false) + } + + /// Returns a `Folder` that deeply expands all macros and assigns all `NodeId`s in an AST node. + /// Once `NodeId`s are assigned, the node may not be expanded, removed, or otherwise modified. + pub fn monotonic_expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { + expand::MacroExpander::new(self, true) + } + pub fn new_parser_from_tts(&self, stream: TokenStream) -> parser::Parser<'a> { + rustc_parse::stream_to_parser(self.parse_sess, stream, MACRO_ARGUMENTS) + } + pub fn source_map(&self) -> &'a SourceMap { + self.parse_sess.source_map() + } + pub fn parse_sess(&self) -> &'a ParseSess { + self.parse_sess + } + pub fn call_site(&self) -> Span { + self.current_expansion.id.expn_data().call_site + } + + /// Equivalent of `Span::def_site` from the proc macro API, + /// except that the location is taken from the span passed as an argument. + pub fn with_def_site_ctxt(&self, span: Span) -> Span { + span.with_def_site_ctxt(self.current_expansion.id) + } + + /// Equivalent of `Span::call_site` from the proc macro API, + /// except that the location is taken from the span passed as an argument. + pub fn with_call_site_ctxt(&self, span: Span) -> Span { + span.with_call_site_ctxt(self.current_expansion.id) + } + + /// Equivalent of `Span::mixed_site` from the proc macro API, + /// except that the location is taken from the span passed as an argument. + pub fn with_mixed_site_ctxt(&self, span: Span) -> Span { + span.with_mixed_site_ctxt(self.current_expansion.id) + } + + /// Returns span for the macro which originally caused the current expansion to happen. + /// + /// Stops backtracing at include! boundary. + pub fn expansion_cause(&self) -> Option { + self.current_expansion.id.expansion_cause() + } + + pub fn struct_span_warn>(&self, sp: S, msg: &str) -> DiagnosticBuilder<'a> { + self.parse_sess.span_diagnostic.struct_span_warn(sp, msg) + } + pub fn struct_span_err>(&self, sp: S, msg: &str) -> DiagnosticBuilder<'a> { + self.parse_sess.span_diagnostic.struct_span_err(sp, msg) + } + pub fn struct_span_fatal>(&self, sp: S, msg: &str) -> DiagnosticBuilder<'a> { + self.parse_sess.span_diagnostic.struct_span_fatal(sp, msg) + } + + /// Emit `msg` attached to `sp`, and stop compilation immediately. + /// + /// `span_err` should be strongly preferred where-ever possible: + /// this should *only* be used when: + /// + /// - continuing has a high risk of flow-on errors (e.g., errors in + /// declaring a macro would cause all uses of that macro to + /// complain about "undefined macro"), or + /// - there is literally nothing else that can be done (however, + /// in most cases one can construct a dummy expression/item to + /// substitute; we never hit resolve/type-checking so the dummy + /// value doesn't have to match anything) + pub fn span_fatal>(&self, sp: S, msg: &str) -> ! { + self.parse_sess.span_diagnostic.span_fatal(sp, msg).raise(); + } + + /// Emit `msg` attached to `sp`, without immediately stopping + /// compilation. + /// + /// Compilation will be stopped in the near future (at the end of + /// the macro expansion phase). + pub fn span_err>(&self, sp: S, msg: &str) { + self.parse_sess.span_diagnostic.span_err(sp, msg); + } + pub fn span_err_with_code>(&self, sp: S, msg: &str, code: DiagnosticId) { + self.parse_sess.span_diagnostic.span_err_with_code(sp, msg, code); + } + pub fn span_warn>(&self, sp: S, msg: &str) { + self.parse_sess.span_diagnostic.span_warn(sp, msg); + } + pub fn span_bug>(&self, sp: S, msg: &str) -> ! { + self.parse_sess.span_diagnostic.span_bug(sp, msg); + } + pub fn trace_macros_diag(&mut self) { + for (sp, notes) in self.expansions.iter() { + let mut db = self.parse_sess.span_diagnostic.span_note_diag(*sp, "trace_macro"); + for note in notes { + db.note(note); + } + db.emit(); + } + // Fixme: does this result in errors? + self.expansions.clear(); + } + pub fn bug(&self, msg: &str) -> ! { + self.parse_sess.span_diagnostic.bug(msg); + } + pub fn trace_macros(&self) -> bool { + self.ecfg.trace_mac + } + pub fn set_trace_macros(&mut self, x: bool) { + self.ecfg.trace_mac = x + } + pub fn ident_of(&self, st: &str, sp: Span) -> ast::Ident { + ast::Ident::from_str_and_span(st, sp) + } + pub fn std_path(&self, components: &[Symbol]) -> Vec { + let def_site = self.with_def_site_ctxt(DUMMY_SP); + iter::once(Ident::new(kw::DollarCrate, def_site)) + .chain(components.iter().map(|&s| Ident::with_dummy_span(s))) + .collect() + } + pub fn name_of(&self, st: &str) -> ast::Name { + Symbol::intern(st) + } + + pub fn check_unused_macros(&mut self) { + self.resolver.check_unused_macros(); + } + + /// Resolves a path mentioned inside Rust code. + /// + /// This unifies the logic used for resolving `include_X!`, and `#[doc(include)]` file paths. + /// + /// Returns an absolute path to the file that `path` refers to. + pub fn resolve_path( + &self, + path: impl Into, + span: Span, + ) -> Result> { + let path = path.into(); + + // Relative paths are resolved relative to the file in which they are found + // after macro expansion (that is, they are unhygienic). + if !path.is_absolute() { + let callsite = span.source_callsite(); + let mut result = match self.source_map().span_to_unmapped_path(callsite) { + FileName::Real(path) => path, + FileName::DocTest(path, _) => path, + other => { + return Err(self.struct_span_err( + span, + &format!("cannot resolve relative path in non-file source `{}`", other), + )); + } + }; + result.pop(); + result.push(path); + Ok(result) + } else { + Ok(path) + } + } +} + +/// Extracts a string literal from the macro expanded version of `expr`, +/// emitting `err_msg` if `expr` is not a string literal. This does not stop +/// compilation on error, merely emits a non-fatal error and returns `None`. +pub fn expr_to_spanned_string<'a>( + cx: &'a mut ExtCtxt<'_>, + expr: P, + err_msg: &str, +) -> Result<(Symbol, ast::StrStyle, Span), Option>> { + // Perform eager expansion on the expression. + // We want to be able to handle e.g., `concat!("foo", "bar")`. + let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); + + Err(match expr.kind { + ast::ExprKind::Lit(ref l) => match l.kind { + ast::LitKind::Str(s, style) => return Ok((s, style, expr.span)), + ast::LitKind::Err(_) => None, + _ => Some(cx.struct_span_err(l.span, err_msg)), + }, + ast::ExprKind::Err => None, + _ => Some(cx.struct_span_err(expr.span, err_msg)), + }) +} + +pub fn expr_to_string( + cx: &mut ExtCtxt<'_>, + expr: P, + err_msg: &str, +) -> Option<(Symbol, ast::StrStyle)> { + expr_to_spanned_string(cx, expr, err_msg) + .map_err(|err| err.map(|mut err| err.emit())) + .ok() + .map(|(symbol, style, _)| (symbol, style)) +} + +/// Non-fatally assert that `tts` is empty. Note that this function +/// returns even when `tts` is non-empty, macros that *need* to stop +/// compilation should call +/// `cx.parse_sess.span_diagnostic.abort_if_errors()` (this should be +/// done as rarely as possible). +pub fn check_zero_tts(cx: &ExtCtxt<'_>, sp: Span, tts: TokenStream, name: &str) { + if !tts.is_empty() { + cx.span_err(sp, &format!("{} takes no arguments", name)); + } +} + +/// Interpreting `tts` as a comma-separated sequence of expressions, +/// expect exactly one string literal, or emit an error and return `None`. +pub fn get_single_str_from_tts( + cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, + name: &str, +) -> Option { + let mut p = cx.new_parser_from_tts(tts); + if p.token == token::Eof { + cx.span_err(sp, &format!("{} takes 1 argument", name)); + return None; + } + let ret = panictry!(p.parse_expr()); + let _ = p.eat(&token::Comma); + + if p.token != token::Eof { + cx.span_err(sp, &format!("{} takes 1 argument", name)); + } + expr_to_string(cx, ret, "argument must be a string literal").map(|(s, _)| s.to_string()) +} + +/// Extracts comma-separated expressions from `tts`. If there is a +/// parsing error, emit a non-fatal error and return `None`. +pub fn get_exprs_from_tts( + cx: &mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Option>> { + let mut p = cx.new_parser_from_tts(tts); + let mut es = Vec::new(); + while p.token != token::Eof { + let expr = panictry!(p.parse_expr()); + + // Perform eager expansion on the expression. + // We want to be able to handle e.g., `concat!("foo", "bar")`. + let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); + + es.push(expr); + if p.eat(&token::Comma) { + continue; + } + if p.token != token::Eof { + cx.span_err(sp, "expected token: `,`"); + return None; + } + } + Some(es) +} diff --git a/src/librustc_expand/build.rs b/src/librustc_expand/build.rs new file mode 100644 index 00000000000..96020acb3b4 --- /dev/null +++ b/src/librustc_expand/build.rs @@ -0,0 +1,663 @@ +use crate::base::ExtCtxt; + +use syntax::ast::{self, AttrVec, BlockCheckMode, Expr, Ident, PatKind, UnOp}; +use syntax::attr; +use syntax::ptr::P; +use syntax::source_map::{respan, Spanned}; +use syntax::symbol::{kw, sym, Symbol}; + +use syntax_pos::{Pos, Span}; + +impl<'a> ExtCtxt<'a> { + pub fn path(&self, span: Span, strs: Vec) -> ast::Path { + self.path_all(span, false, strs, vec![]) + } + pub fn path_ident(&self, span: Span, id: ast::Ident) -> ast::Path { + self.path(span, vec![id]) + } + pub fn path_global(&self, span: Span, strs: Vec) -> ast::Path { + self.path_all(span, true, strs, vec![]) + } + pub fn path_all( + &self, + span: Span, + global: bool, + mut idents: Vec, + args: Vec, + ) -> ast::Path { + assert!(!idents.is_empty()); + let add_root = global && !idents[0].is_path_segment_keyword(); + let mut segments = Vec::with_capacity(idents.len() + add_root as usize); + if add_root { + segments.push(ast::PathSegment::path_root(span)); + } + let last_ident = idents.pop().unwrap(); + segments.extend( + idents.into_iter().map(|ident| ast::PathSegment::from_ident(ident.with_span_pos(span))), + ); + let args = if !args.is_empty() { + ast::AngleBracketedArgs { args, constraints: Vec::new(), span }.into() + } else { + None + }; + segments.push(ast::PathSegment { + ident: last_ident.with_span_pos(span), + id: ast::DUMMY_NODE_ID, + args, + }); + ast::Path { span, segments } + } + + pub fn ty_mt(&self, ty: P, mutbl: ast::Mutability) -> ast::MutTy { + ast::MutTy { ty, mutbl } + } + + pub fn ty(&self, span: Span, kind: ast::TyKind) -> P { + P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind }) + } + + pub fn ty_path(&self, path: ast::Path) -> P { + self.ty(path.span, ast::TyKind::Path(None, path)) + } + + // Might need to take bounds as an argument in the future, if you ever want + // to generate a bounded existential trait type. + pub fn ty_ident(&self, span: Span, ident: ast::Ident) -> P { + self.ty_path(self.path_ident(span, ident)) + } + + pub fn anon_const(&self, span: Span, kind: ast::ExprKind) -> ast::AnonConst { + ast::AnonConst { + id: ast::DUMMY_NODE_ID, + value: P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() }), + } + } + + pub fn const_ident(&self, span: Span, ident: ast::Ident) -> ast::AnonConst { + self.anon_const(span, ast::ExprKind::Path(None, self.path_ident(span, ident))) + } + + pub fn ty_rptr( + &self, + span: Span, + ty: P, + lifetime: Option, + mutbl: ast::Mutability, + ) -> P { + self.ty(span, ast::TyKind::Rptr(lifetime, self.ty_mt(ty, mutbl))) + } + + pub fn ty_ptr(&self, span: Span, ty: P, mutbl: ast::Mutability) -> P { + self.ty(span, ast::TyKind::Ptr(self.ty_mt(ty, mutbl))) + } + + pub fn typaram( + &self, + span: Span, + ident: ast::Ident, + attrs: Vec, + bounds: ast::GenericBounds, + default: Option>, + ) -> ast::GenericParam { + ast::GenericParam { + ident: ident.with_span_pos(span), + id: ast::DUMMY_NODE_ID, + attrs: attrs.into(), + bounds, + kind: ast::GenericParamKind::Type { default }, + is_placeholder: false, + } + } + + pub fn trait_ref(&self, path: ast::Path) -> ast::TraitRef { + ast::TraitRef { path, ref_id: ast::DUMMY_NODE_ID } + } + + pub fn poly_trait_ref(&self, span: Span, path: ast::Path) -> ast::PolyTraitRef { + ast::PolyTraitRef { + bound_generic_params: Vec::new(), + trait_ref: self.trait_ref(path), + span, + } + } + + pub fn trait_bound(&self, path: ast::Path) -> ast::GenericBound { + ast::GenericBound::Trait( + self.poly_trait_ref(path.span, path), + ast::TraitBoundModifier::None, + ) + } + + pub fn lifetime(&self, span: Span, ident: ast::Ident) -> ast::Lifetime { + ast::Lifetime { id: ast::DUMMY_NODE_ID, ident: ident.with_span_pos(span) } + } + + pub fn lifetime_def( + &self, + span: Span, + ident: ast::Ident, + attrs: Vec, + bounds: ast::GenericBounds, + ) -> ast::GenericParam { + let lifetime = self.lifetime(span, ident); + ast::GenericParam { + ident: lifetime.ident, + id: lifetime.id, + attrs: attrs.into(), + bounds, + kind: ast::GenericParamKind::Lifetime, + is_placeholder: false, + } + } + + pub fn stmt_expr(&self, expr: P) -> ast::Stmt { + ast::Stmt { id: ast::DUMMY_NODE_ID, span: expr.span, kind: ast::StmtKind::Expr(expr) } + } + + pub fn stmt_let( + &self, + sp: Span, + mutbl: bool, + ident: ast::Ident, + ex: P, + ) -> ast::Stmt { + let pat = if mutbl { + let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Mut); + self.pat_ident_binding_mode(sp, ident, binding_mode) + } else { + self.pat_ident(sp, ident) + }; + let local = P(ast::Local { + pat, + ty: None, + init: Some(ex), + id: ast::DUMMY_NODE_ID, + span: sp, + attrs: AttrVec::new(), + }); + ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp } + } + + // Generates `let _: Type;`, which is usually used for type assertions. + pub fn stmt_let_type_only(&self, span: Span, ty: P) -> ast::Stmt { + let local = P(ast::Local { + pat: self.pat_wild(span), + ty: Some(ty), + init: None, + id: ast::DUMMY_NODE_ID, + span, + attrs: AttrVec::new(), + }); + ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span } + } + + pub fn stmt_item(&self, sp: Span, item: P) -> ast::Stmt { + ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Item(item), span: sp } + } + + pub fn block_expr(&self, expr: P) -> P { + self.block( + expr.span, + vec![ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: expr.span, + kind: ast::StmtKind::Expr(expr), + }], + ) + } + pub fn block(&self, span: Span, stmts: Vec) -> P { + P(ast::Block { stmts, id: ast::DUMMY_NODE_ID, rules: BlockCheckMode::Default, span }) + } + + pub fn expr(&self, span: Span, kind: ast::ExprKind) -> P { + P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() }) + } + + pub fn expr_path(&self, path: ast::Path) -> P { + self.expr(path.span, ast::ExprKind::Path(None, path)) + } + + pub fn expr_ident(&self, span: Span, id: ast::Ident) -> P { + self.expr_path(self.path_ident(span, id)) + } + pub fn expr_self(&self, span: Span) -> P { + self.expr_ident(span, Ident::with_dummy_span(kw::SelfLower)) + } + + pub fn expr_binary( + &self, + sp: Span, + op: ast::BinOpKind, + lhs: P, + rhs: P, + ) -> P { + self.expr(sp, ast::ExprKind::Binary(Spanned { node: op, span: sp }, lhs, rhs)) + } + + pub fn expr_deref(&self, sp: Span, e: P) -> P { + self.expr(sp, ast::ExprKind::Unary(UnOp::Deref, e)) + } + + pub fn expr_addr_of(&self, sp: Span, e: P) -> P { + self.expr(sp, ast::ExprKind::AddrOf(ast::BorrowKind::Ref, ast::Mutability::Not, e)) + } + + pub fn expr_call( + &self, + span: Span, + expr: P, + args: Vec>, + ) -> P { + self.expr(span, ast::ExprKind::Call(expr, args)) + } + pub fn expr_call_ident( + &self, + span: Span, + id: ast::Ident, + args: Vec>, + ) -> P { + self.expr(span, ast::ExprKind::Call(self.expr_ident(span, id), args)) + } + pub fn expr_call_global( + &self, + sp: Span, + fn_path: Vec, + args: Vec>, + ) -> P { + let pathexpr = self.expr_path(self.path_global(sp, fn_path)); + self.expr_call(sp, pathexpr, args) + } + pub fn expr_method_call( + &self, + span: Span, + expr: P, + ident: ast::Ident, + mut args: Vec>, + ) -> P { + args.insert(0, expr); + let segment = ast::PathSegment::from_ident(ident.with_span_pos(span)); + self.expr(span, ast::ExprKind::MethodCall(segment, args)) + } + pub fn expr_block(&self, b: P) -> P { + self.expr(b.span, ast::ExprKind::Block(b, None)) + } + pub fn field_imm(&self, span: Span, ident: Ident, e: P) -> ast::Field { + ast::Field { + ident: ident.with_span_pos(span), + expr: e, + span, + is_shorthand: false, + attrs: AttrVec::new(), + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + } + } + pub fn expr_struct( + &self, + span: Span, + path: ast::Path, + fields: Vec, + ) -> P { + self.expr(span, ast::ExprKind::Struct(path, fields, None)) + } + pub fn expr_struct_ident( + &self, + span: Span, + id: ast::Ident, + fields: Vec, + ) -> P { + self.expr_struct(span, self.path_ident(span, id), fields) + } + + pub fn expr_lit(&self, span: Span, lit_kind: ast::LitKind) -> P { + let lit = ast::Lit::from_lit_kind(lit_kind, span); + self.expr(span, ast::ExprKind::Lit(lit)) + } + pub fn expr_usize(&self, span: Span, i: usize) -> P { + self.expr_lit( + span, + ast::LitKind::Int(i as u128, ast::LitIntType::Unsigned(ast::UintTy::Usize)), + ) + } + pub fn expr_u32(&self, sp: Span, u: u32) -> P { + self.expr_lit(sp, ast::LitKind::Int(u as u128, ast::LitIntType::Unsigned(ast::UintTy::U32))) + } + pub fn expr_bool(&self, sp: Span, value: bool) -> P { + self.expr_lit(sp, ast::LitKind::Bool(value)) + } + + pub fn expr_vec(&self, sp: Span, exprs: Vec>) -> P { + self.expr(sp, ast::ExprKind::Array(exprs)) + } + pub fn expr_vec_slice(&self, sp: Span, exprs: Vec>) -> P { + self.expr_addr_of(sp, self.expr_vec(sp, exprs)) + } + pub fn expr_str(&self, sp: Span, s: Symbol) -> P { + self.expr_lit(sp, ast::LitKind::Str(s, ast::StrStyle::Cooked)) + } + + pub fn expr_cast(&self, sp: Span, expr: P, ty: P) -> P { + self.expr(sp, ast::ExprKind::Cast(expr, ty)) + } + + pub fn expr_some(&self, sp: Span, expr: P) -> P { + let some = self.std_path(&[sym::option, sym::Option, sym::Some]); + self.expr_call_global(sp, some, vec![expr]) + } + + pub fn expr_tuple(&self, sp: Span, exprs: Vec>) -> P { + self.expr(sp, ast::ExprKind::Tup(exprs)) + } + + pub fn expr_fail(&self, span: Span, msg: Symbol) -> P { + let loc = self.source_map().lookup_char_pos(span.lo()); + let expr_file = self.expr_str(span, Symbol::intern(&loc.file.name.to_string())); + let expr_line = self.expr_u32(span, loc.line as u32); + let expr_col = self.expr_u32(span, loc.col.to_usize() as u32 + 1); + let expr_loc_tuple = self.expr_tuple(span, vec![expr_file, expr_line, expr_col]); + let expr_loc_ptr = self.expr_addr_of(span, expr_loc_tuple); + self.expr_call_global( + span, + [sym::std, sym::rt, sym::begin_panic].iter().map(|s| Ident::new(*s, span)).collect(), + vec![self.expr_str(span, msg), expr_loc_ptr], + ) + } + + pub fn expr_unreachable(&self, span: Span) -> P { + self.expr_fail(span, Symbol::intern("internal error: entered unreachable code")) + } + + pub fn expr_ok(&self, sp: Span, expr: P) -> P { + let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); + self.expr_call_global(sp, ok, vec![expr]) + } + + pub fn expr_try(&self, sp: Span, head: P) -> P { + let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); + let ok_path = self.path_global(sp, ok); + let err = self.std_path(&[sym::result, sym::Result, sym::Err]); + let err_path = self.path_global(sp, err); + + let binding_variable = self.ident_of("__try_var", sp); + let binding_pat = self.pat_ident(sp, binding_variable); + let binding_expr = self.expr_ident(sp, binding_variable); + + // `Ok(__try_var)` pattern + let ok_pat = self.pat_tuple_struct(sp, ok_path, vec![binding_pat.clone()]); + + // `Err(__try_var)` (pattern and expression respectively) + let err_pat = self.pat_tuple_struct(sp, err_path.clone(), vec![binding_pat]); + let err_inner_expr = + self.expr_call(sp, self.expr_path(err_path), vec![binding_expr.clone()]); + // `return Err(__try_var)` + let err_expr = self.expr(sp, ast::ExprKind::Ret(Some(err_inner_expr))); + + // `Ok(__try_var) => __try_var` + let ok_arm = self.arm(sp, ok_pat, binding_expr); + // `Err(__try_var) => return Err(__try_var)` + let err_arm = self.arm(sp, err_pat, err_expr); + + // `match head { Ok() => ..., Err() => ... }` + self.expr_match(sp, head, vec![ok_arm, err_arm]) + } + + pub fn pat(&self, span: Span, kind: PatKind) -> P { + P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span }) + } + pub fn pat_wild(&self, span: Span) -> P { + self.pat(span, PatKind::Wild) + } + pub fn pat_lit(&self, span: Span, expr: P) -> P { + self.pat(span, PatKind::Lit(expr)) + } + pub fn pat_ident(&self, span: Span, ident: ast::Ident) -> P { + let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Not); + self.pat_ident_binding_mode(span, ident, binding_mode) + } + + pub fn pat_ident_binding_mode( + &self, + span: Span, + ident: ast::Ident, + bm: ast::BindingMode, + ) -> P { + let pat = PatKind::Ident(bm, ident.with_span_pos(span), None); + self.pat(span, pat) + } + pub fn pat_path(&self, span: Span, path: ast::Path) -> P { + self.pat(span, PatKind::Path(None, path)) + } + pub fn pat_tuple_struct( + &self, + span: Span, + path: ast::Path, + subpats: Vec>, + ) -> P { + self.pat(span, PatKind::TupleStruct(path, subpats)) + } + pub fn pat_struct( + &self, + span: Span, + path: ast::Path, + field_pats: Vec, + ) -> P { + self.pat(span, PatKind::Struct(path, field_pats, false)) + } + pub fn pat_tuple(&self, span: Span, pats: Vec>) -> P { + self.pat(span, PatKind::Tuple(pats)) + } + + pub fn pat_some(&self, span: Span, pat: P) -> P { + let some = self.std_path(&[sym::option, sym::Option, sym::Some]); + let path = self.path_global(span, some); + self.pat_tuple_struct(span, path, vec![pat]) + } + + pub fn pat_none(&self, span: Span) -> P { + let some = self.std_path(&[sym::option, sym::Option, sym::None]); + let path = self.path_global(span, some); + self.pat_path(span, path) + } + + pub fn pat_ok(&self, span: Span, pat: P) -> P { + let some = self.std_path(&[sym::result, sym::Result, sym::Ok]); + let path = self.path_global(span, some); + self.pat_tuple_struct(span, path, vec![pat]) + } + + pub fn pat_err(&self, span: Span, pat: P) -> P { + let some = self.std_path(&[sym::result, sym::Result, sym::Err]); + let path = self.path_global(span, some); + self.pat_tuple_struct(span, path, vec![pat]) + } + + pub fn arm(&self, span: Span, pat: P, expr: P) -> ast::Arm { + ast::Arm { + attrs: vec![], + pat, + guard: None, + body: expr, + span, + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + } + } + + pub fn arm_unreachable(&self, span: Span) -> ast::Arm { + self.arm(span, self.pat_wild(span), self.expr_unreachable(span)) + } + + pub fn expr_match(&self, span: Span, arg: P, arms: Vec) -> P { + self.expr(span, ast::ExprKind::Match(arg, arms)) + } + + pub fn expr_if( + &self, + span: Span, + cond: P, + then: P, + els: Option>, + ) -> P { + let els = els.map(|x| self.expr_block(self.block_expr(x))); + self.expr(span, ast::ExprKind::If(cond, self.block_expr(then), els)) + } + + pub fn lambda_fn_decl( + &self, + span: Span, + fn_decl: P, + body: P, + fn_decl_span: Span, + ) -> P { + self.expr( + span, + ast::ExprKind::Closure( + ast::CaptureBy::Ref, + ast::IsAsync::NotAsync, + ast::Movability::Movable, + fn_decl, + body, + fn_decl_span, + ), + ) + } + + pub fn lambda(&self, span: Span, ids: Vec, body: P) -> P { + let fn_decl = self.fn_decl( + ids.iter().map(|id| self.param(span, *id, self.ty(span, ast::TyKind::Infer))).collect(), + ast::FunctionRetTy::Default(span), + ); + + // FIXME -- We are using `span` as the span of the `|...|` + // part of the lambda, but it probably (maybe?) corresponds to + // the entire lambda body. Probably we should extend the API + // here, but that's not entirely clear. + self.expr( + span, + ast::ExprKind::Closure( + ast::CaptureBy::Ref, + ast::IsAsync::NotAsync, + ast::Movability::Movable, + fn_decl, + body, + span, + ), + ) + } + + pub fn lambda0(&self, span: Span, body: P) -> P { + self.lambda(span, Vec::new(), body) + } + + pub fn lambda1(&self, span: Span, body: P, ident: ast::Ident) -> P { + self.lambda(span, vec![ident], body) + } + + pub fn lambda_stmts_1( + &self, + span: Span, + stmts: Vec, + ident: ast::Ident, + ) -> P { + self.lambda1(span, self.expr_block(self.block(span, stmts)), ident) + } + + pub fn param(&self, span: Span, ident: ast::Ident, ty: P) -> ast::Param { + let arg_pat = self.pat_ident(span, ident); + ast::Param { + attrs: AttrVec::default(), + id: ast::DUMMY_NODE_ID, + pat: arg_pat, + span, + ty, + is_placeholder: false, + } + } + + // FIXME: unused `self` + pub fn fn_decl(&self, inputs: Vec, output: ast::FunctionRetTy) -> P { + P(ast::FnDecl { inputs, output }) + } + + pub fn item( + &self, + span: Span, + name: Ident, + attrs: Vec, + kind: ast::ItemKind, + ) -> P { + // FIXME: Would be nice if our generated code didn't violate + // Rust coding conventions + P(ast::Item { + ident: name, + attrs, + id: ast::DUMMY_NODE_ID, + kind, + vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), + span, + tokens: None, + }) + } + + pub fn variant(&self, span: Span, ident: Ident, tys: Vec>) -> ast::Variant { + let vis_span = span.shrink_to_lo(); + let fields: Vec<_> = tys + .into_iter() + .map(|ty| ast::StructField { + span: ty.span, + ty, + ident: None, + vis: respan(vis_span, ast::VisibilityKind::Inherited), + attrs: Vec::new(), + id: ast::DUMMY_NODE_ID, + is_placeholder: false, + }) + .collect(); + + let vdata = if fields.is_empty() { + ast::VariantData::Unit(ast::DUMMY_NODE_ID) + } else { + ast::VariantData::Tuple(fields, ast::DUMMY_NODE_ID) + }; + + ast::Variant { + attrs: Vec::new(), + data: vdata, + disr_expr: None, + id: ast::DUMMY_NODE_ID, + ident, + vis: respan(vis_span, ast::VisibilityKind::Inherited), + span, + is_placeholder: false, + } + } + + pub fn item_static( + &self, + span: Span, + name: Ident, + ty: P, + mutbl: ast::Mutability, + expr: P, + ) -> P { + self.item(span, name, Vec::new(), ast::ItemKind::Static(ty, mutbl, expr)) + } + + pub fn item_const( + &self, + span: Span, + name: Ident, + ty: P, + expr: P, + ) -> P { + self.item(span, name, Vec::new(), ast::ItemKind::Const(ty, expr)) + } + + pub fn attribute(&self, mi: ast::MetaItem) -> ast::Attribute { + attr::mk_attr_outer(mi) + } + + pub fn meta_word(&self, sp: Span, w: ast::Name) -> ast::MetaItem { + attr::mk_word_item(Ident::new(w, sp)) + } +} diff --git a/src/librustc_expand/expand.rs b/src/librustc_expand/expand.rs new file mode 100644 index 00000000000..089dd471f3b --- /dev/null +++ b/src/librustc_expand/expand.rs @@ -0,0 +1,1725 @@ +use crate::base::*; +use crate::config::StripUnconfigured; +use crate::hygiene::{ExpnData, ExpnId, ExpnKind, SyntaxContext}; +use crate::mbe::macro_rules::annotate_err_with_kind; +use crate::placeholders::{placeholder, PlaceholderExpander}; +use crate::proc_macro::collect_derives; + +use rustc_feature::Features; +use rustc_parse::configure; +use rustc_parse::parser::Parser; +use rustc_parse::validate_attr; +use rustc_parse::DirectoryOwnership; +use syntax::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path}; +use syntax::ast::{ItemKind, MacArgs, MacStmtStyle, StmtKind}; +use syntax::attr::{self, is_builtin_attr, HasAttrs}; +use syntax::feature_gate::{self, feature_err}; +use syntax::mut_visit::*; +use syntax::print::pprust; +use syntax::ptr::P; +use syntax::sess::ParseSess; +use syntax::source_map::respan; +use syntax::symbol::{sym, Symbol}; +use syntax::token; +use syntax::tokenstream::{TokenStream, TokenTree}; +use syntax::util::map_in_place::MapInPlace; +use syntax::visit::{self, Visitor}; + +use errors::{Applicability, FatalError, PResult}; +use smallvec::{smallvec, SmallVec}; +use syntax_pos::{FileName, Span, DUMMY_SP}; + +use rustc_data_structures::sync::Lrc; +use std::io::ErrorKind; +use std::ops::DerefMut; +use std::path::PathBuf; +use std::rc::Rc; +use std::{iter, mem, slice}; + +macro_rules! ast_fragments { + ( + $($Kind:ident($AstTy:ty) { + $kind_name:expr; + $(one fn $mut_visit_ast:ident; fn $visit_ast:ident;)? + $(many fn $flat_map_ast_elt:ident; fn $visit_ast_elt:ident;)? + fn $make_ast:ident; + })* + ) => { + /// A fragment of AST that can be produced by a single macro expansion. + /// Can also serve as an input and intermediate result for macro expansion operations. + pub enum AstFragment { + OptExpr(Option>), + $($Kind($AstTy),)* + } + + /// "Discriminant" of an AST fragment. + #[derive(Copy, Clone, PartialEq, Eq)] + pub enum AstFragmentKind { + OptExpr, + $($Kind,)* + } + + impl AstFragmentKind { + pub fn name(self) -> &'static str { + match self { + AstFragmentKind::OptExpr => "expression", + $(AstFragmentKind::$Kind => $kind_name,)* + } + } + + fn make_from<'a>(self, result: Box) -> Option { + match self { + AstFragmentKind::OptExpr => + result.make_expr().map(Some).map(AstFragment::OptExpr), + $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)* + } + } + } + + impl AstFragment { + pub fn add_placeholders(&mut self, placeholders: &[NodeId]) { + if placeholders.is_empty() { + return; + } + match self { + $($(AstFragment::$Kind(ast) => ast.extend(placeholders.iter().flat_map(|id| { + // We are repeating through arguments with `many`, to do that we have to + // mention some macro variable from those arguments even if it's not used. + macro _repeating($flat_map_ast_elt) {} + placeholder(AstFragmentKind::$Kind, *id, None).$make_ast() + })),)?)* + _ => panic!("unexpected AST fragment kind") + } + } + + pub fn make_opt_expr(self) -> Option> { + match self { + AstFragment::OptExpr(expr) => expr, + _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), + } + } + + $(pub fn $make_ast(self) -> $AstTy { + match self { + AstFragment::$Kind(ast) => ast, + _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), + } + })* + + pub fn mut_visit_with(&mut self, vis: &mut F) { + match self { + AstFragment::OptExpr(opt_expr) => { + visit_clobber(opt_expr, |opt_expr| { + if let Some(expr) = opt_expr { + vis.filter_map_expr(expr) + } else { + None + } + }); + } + $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)* + $($(AstFragment::$Kind(ast) => + ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)* + } + } + + pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { + match *self { + AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr), + AstFragment::OptExpr(None) => {} + $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)* + $($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] { + visitor.$visit_ast_elt(ast_elt); + })?)* + } + } + } + + impl<'a> MacResult for crate::mbe::macro_rules::ParserAnyMacro<'a> { + $(fn $make_ast(self: Box>) + -> Option<$AstTy> { + Some(self.make(AstFragmentKind::$Kind).$make_ast()) + })* + } + } +} + +ast_fragments! { + Expr(P) { "expression"; one fn visit_expr; fn visit_expr; fn make_expr; } + Pat(P) { "pattern"; one fn visit_pat; fn visit_pat; fn make_pat; } + Ty(P) { "type"; one fn visit_ty; fn visit_ty; fn make_ty; } + Stmts(SmallVec<[ast::Stmt; 1]>) { + "statement"; many fn flat_map_stmt; fn visit_stmt; fn make_stmts; + } + Items(SmallVec<[P; 1]>) { + "item"; many fn flat_map_item; fn visit_item; fn make_items; + } + TraitItems(SmallVec<[ast::AssocItem; 1]>) { + "trait item"; many fn flat_map_trait_item; fn visit_trait_item; fn make_trait_items; + } + ImplItems(SmallVec<[ast::AssocItem; 1]>) { + "impl item"; many fn flat_map_impl_item; fn visit_impl_item; fn make_impl_items; + } + ForeignItems(SmallVec<[ast::ForeignItem; 1]>) { + "foreign item"; + many fn flat_map_foreign_item; + fn visit_foreign_item; + fn make_foreign_items; + } + Arms(SmallVec<[ast::Arm; 1]>) { + "match arm"; many fn flat_map_arm; fn visit_arm; fn make_arms; + } + Fields(SmallVec<[ast::Field; 1]>) { + "field expression"; many fn flat_map_field; fn visit_field; fn make_fields; + } + FieldPats(SmallVec<[ast::FieldPat; 1]>) { + "field pattern"; + many fn flat_map_field_pattern; + fn visit_field_pattern; + fn make_field_patterns; + } + GenericParams(SmallVec<[ast::GenericParam; 1]>) { + "generic parameter"; + many fn flat_map_generic_param; + fn visit_generic_param; + fn make_generic_params; + } + Params(SmallVec<[ast::Param; 1]>) { + "function parameter"; many fn flat_map_param; fn visit_param; fn make_params; + } + StructFields(SmallVec<[ast::StructField; 1]>) { + "field"; + many fn flat_map_struct_field; + fn visit_struct_field; + fn make_struct_fields; + } + Variants(SmallVec<[ast::Variant; 1]>) { + "variant"; many fn flat_map_variant; fn visit_variant; fn make_variants; + } +} + +impl AstFragmentKind { + fn dummy(self, span: Span) -> AstFragment { + self.make_from(DummyResult::any(span)).expect("couldn't create a dummy AST fragment") + } + + fn expect_from_annotatables>( + self, + items: I, + ) -> AstFragment { + let mut items = items.into_iter(); + match self { + AstFragmentKind::Arms => { + AstFragment::Arms(items.map(Annotatable::expect_arm).collect()) + } + AstFragmentKind::Fields => { + AstFragment::Fields(items.map(Annotatable::expect_field).collect()) + } + AstFragmentKind::FieldPats => { + AstFragment::FieldPats(items.map(Annotatable::expect_field_pattern).collect()) + } + AstFragmentKind::GenericParams => { + AstFragment::GenericParams(items.map(Annotatable::expect_generic_param).collect()) + } + AstFragmentKind::Params => { + AstFragment::Params(items.map(Annotatable::expect_param).collect()) + } + AstFragmentKind::StructFields => { + AstFragment::StructFields(items.map(Annotatable::expect_struct_field).collect()) + } + AstFragmentKind::Variants => { + AstFragment::Variants(items.map(Annotatable::expect_variant).collect()) + } + AstFragmentKind::Items => { + AstFragment::Items(items.map(Annotatable::expect_item).collect()) + } + AstFragmentKind::ImplItems => { + AstFragment::ImplItems(items.map(Annotatable::expect_impl_item).collect()) + } + AstFragmentKind::TraitItems => { + AstFragment::TraitItems(items.map(Annotatable::expect_trait_item).collect()) + } + AstFragmentKind::ForeignItems => { + AstFragment::ForeignItems(items.map(Annotatable::expect_foreign_item).collect()) + } + AstFragmentKind::Stmts => { + AstFragment::Stmts(items.map(Annotatable::expect_stmt).collect()) + } + AstFragmentKind::Expr => AstFragment::Expr( + items.next().expect("expected exactly one expression").expect_expr(), + ), + AstFragmentKind::OptExpr => { + AstFragment::OptExpr(items.next().map(Annotatable::expect_expr)) + } + AstFragmentKind::Pat | AstFragmentKind::Ty => { + panic!("patterns and types aren't annotatable") + } + } + } +} + +pub struct Invocation { + pub kind: InvocationKind, + pub fragment_kind: AstFragmentKind, + pub expansion_data: ExpansionData, +} + +pub enum InvocationKind { + Bang { + mac: ast::Mac, + span: Span, + }, + Attr { + attr: ast::Attribute, + item: Annotatable, + // Required for resolving derive helper attributes. + derives: Vec, + // We temporarily report errors for attribute macros placed after derives + after_derive: bool, + }, + Derive { + path: Path, + item: Annotatable, + }, + /// "Invocation" that contains all derives from an item, + /// broken into multiple `Derive` invocations when expanded. + /// FIXME: Find a way to remove it. + DeriveContainer { + derives: Vec, + item: Annotatable, + }, +} + +impl InvocationKind { + fn placeholder_visibility(&self) -> Option { + // HACK: For unnamed fields placeholders should have the same visibility as the actual + // fields because for tuple structs/variants resolve determines visibilities of their + // constructor using these field visibilities before attributes on them are are expanded. + // The assumption is that the attribute expansion cannot change field visibilities, + // and it holds because only inert attributes are supported in this position. + match self { + InvocationKind::Attr { item: Annotatable::StructField(field), .. } + | InvocationKind::Derive { item: Annotatable::StructField(field), .. } + | InvocationKind::DeriveContainer { item: Annotatable::StructField(field), .. } + if field.ident.is_none() => + { + Some(field.vis.clone()) + } + _ => None, + } + } +} + +impl Invocation { + pub fn span(&self) -> Span { + match &self.kind { + InvocationKind::Bang { span, .. } => *span, + InvocationKind::Attr { attr, .. } => attr.span, + InvocationKind::Derive { path, .. } => path.span, + InvocationKind::DeriveContainer { item, .. } => item.span(), + } + } +} + +pub struct MacroExpander<'a, 'b> { + pub cx: &'a mut ExtCtxt<'b>, + monotonic: bool, // cf. `cx.monotonic_expander()` +} + +impl<'a, 'b> MacroExpander<'a, 'b> { + pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { + MacroExpander { cx, monotonic } + } + + pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate { + let mut module = ModuleData { + mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)], + directory: match self.cx.source_map().span_to_unmapped_path(krate.span) { + FileName::Real(path) => path, + other => PathBuf::from(other.to_string()), + }, + }; + module.directory.pop(); + self.cx.root_path = module.directory.clone(); + self.cx.current_expansion.module = Rc::new(module); + + let orig_mod_span = krate.module.inner; + + let krate_item = AstFragment::Items(smallvec![P(ast::Item { + attrs: krate.attrs, + span: krate.span, + kind: ast::ItemKind::Mod(krate.module), + ident: Ident::invalid(), + id: ast::DUMMY_NODE_ID, + vis: respan(krate.span.shrink_to_lo(), ast::VisibilityKind::Public), + tokens: None, + })]); + + match self.fully_expand_fragment(krate_item).make_items().pop().map(P::into_inner) { + Some(ast::Item { attrs, kind: ast::ItemKind::Mod(module), .. }) => { + krate.attrs = attrs; + krate.module = module; + } + None => { + // Resolution failed so we return an empty expansion + krate.attrs = vec![]; + krate.module = ast::Mod { inner: orig_mod_span, items: vec![], inline: true }; + } + _ => unreachable!(), + }; + self.cx.trace_macros_diag(); + krate + } + + // Recursively expand all macro invocations in this AST fragment. + pub fn fully_expand_fragment(&mut self, input_fragment: AstFragment) -> AstFragment { + let orig_expansion_data = self.cx.current_expansion.clone(); + self.cx.current_expansion.depth = 0; + + // Collect all macro invocations and replace them with placeholders. + let (mut fragment_with_placeholders, mut invocations) = + self.collect_invocations(input_fragment, &[]); + + // Optimization: if we resolve all imports now, + // we'll be able to immediately resolve most of imported macros. + self.resolve_imports(); + + // Resolve paths in all invocations and produce output expanded fragments for them, but + // do not insert them into our input AST fragment yet, only store in `expanded_fragments`. + // The output fragments also go through expansion recursively until no invocations are left. + // Unresolved macros produce dummy outputs as a recovery measure. + invocations.reverse(); + let mut expanded_fragments = Vec::new(); + let mut undetermined_invocations = Vec::new(); + let (mut progress, mut force) = (false, !self.monotonic); + loop { + let invoc = if let Some(invoc) = invocations.pop() { + invoc + } else { + self.resolve_imports(); + if undetermined_invocations.is_empty() { + break; + } + invocations = mem::take(&mut undetermined_invocations); + force = !mem::replace(&mut progress, false); + continue; + }; + + let eager_expansion_root = + if self.monotonic { invoc.expansion_data.id } else { orig_expansion_data.id }; + let res = match self.cx.resolver.resolve_macro_invocation( + &invoc, + eager_expansion_root, + force, + ) { + Ok(res) => res, + Err(Indeterminate) => { + undetermined_invocations.push(invoc); + continue; + } + }; + + progress = true; + let ExpansionData { depth, id: expn_id, .. } = invoc.expansion_data; + self.cx.current_expansion = invoc.expansion_data.clone(); + + // FIXME(jseyfried): Refactor out the following logic + let (expanded_fragment, new_invocations) = match res { + InvocationRes::Single(ext) => { + let fragment = self.expand_invoc(invoc, &ext.kind); + self.collect_invocations(fragment, &[]) + } + InvocationRes::DeriveContainer(_exts) => { + // FIXME: Consider using the derive resolutions (`_exts`) immediately, + // instead of enqueuing the derives to be resolved again later. + let (derives, item) = match invoc.kind { + InvocationKind::DeriveContainer { derives, item } => (derives, item), + _ => unreachable!(), + }; + if !item.derive_allowed() { + let attr = attr::find_by_name(item.attrs(), sym::derive) + .expect("`derive` attribute should exist"); + let span = attr.span; + let mut err = self.cx.struct_span_err( + span, + "`derive` may only be applied to structs, enums and unions", + ); + if let ast::AttrStyle::Inner = attr.style { + let trait_list = derives + .iter() + .map(|t| pprust::path_to_string(t)) + .collect::>(); + let suggestion = format!("#[derive({})]", trait_list.join(", ")); + err.span_suggestion( + span, + "try an outer attribute", + suggestion, + // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT + Applicability::MaybeIncorrect, + ); + } + err.emit(); + } + + let mut item = self.fully_configure(item); + item.visit_attrs(|attrs| attrs.retain(|a| !a.has_name(sym::derive))); + + let mut derive_placeholders = Vec::with_capacity(derives.len()); + invocations.reserve(derives.len()); + for path in derives { + let expn_id = ExpnId::fresh(None); + derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id)); + invocations.push(Invocation { + kind: InvocationKind::Derive { path, item: item.clone() }, + fragment_kind: invoc.fragment_kind, + expansion_data: ExpansionData { + id: expn_id, + ..invoc.expansion_data.clone() + }, + }); + } + let fragment = + invoc.fragment_kind.expect_from_annotatables(::std::iter::once(item)); + self.collect_invocations(fragment, &derive_placeholders) + } + }; + + if expanded_fragments.len() < depth { + expanded_fragments.push(Vec::new()); + } + expanded_fragments[depth - 1].push((expn_id, expanded_fragment)); + if !self.cx.ecfg.single_step { + invocations.extend(new_invocations.into_iter().rev()); + } + } + + self.cx.current_expansion = orig_expansion_data; + + // Finally incorporate all the expanded macros into the input AST fragment. + let mut placeholder_expander = PlaceholderExpander::new(self.cx, self.monotonic); + while let Some(expanded_fragments) = expanded_fragments.pop() { + for (expn_id, expanded_fragment) in expanded_fragments.into_iter().rev() { + placeholder_expander + .add(NodeId::placeholder_from_expn_id(expn_id), expanded_fragment); + } + } + fragment_with_placeholders.mut_visit_with(&mut placeholder_expander); + fragment_with_placeholders + } + + fn resolve_imports(&mut self) { + if self.monotonic { + self.cx.resolver.resolve_imports(); + } + } + + /// Collects all macro invocations reachable at this time in this AST fragment, and replace + /// them with "placeholders" - dummy macro invocations with specially crafted `NodeId`s. + /// Then call into resolver that builds a skeleton ("reduced graph") of the fragment and + /// prepares data for resolving paths of macro invocations. + fn collect_invocations( + &mut self, + mut fragment: AstFragment, + extra_placeholders: &[NodeId], + ) -> (AstFragment, Vec) { + // Resolve `$crate`s in the fragment for pretty-printing. + self.cx.resolver.resolve_dollar_crates(); + + let invocations = { + let mut collector = InvocationCollector { + cfg: StripUnconfigured { + sess: self.cx.parse_sess, + features: self.cx.ecfg.features, + }, + cx: self.cx, + invocations: Vec::new(), + monotonic: self.monotonic, + }; + fragment.mut_visit_with(&mut collector); + fragment.add_placeholders(extra_placeholders); + collector.invocations + }; + + if self.monotonic { + self.cx + .resolver + .visit_ast_fragment_with_placeholders(self.cx.current_expansion.id, &fragment); + } + + (fragment, invocations) + } + + fn fully_configure(&mut self, item: Annotatable) -> Annotatable { + let mut cfg = + StripUnconfigured { sess: self.cx.parse_sess, features: self.cx.ecfg.features }; + // Since the item itself has already been configured by the InvocationCollector, + // we know that fold result vector will contain exactly one element + match item { + Annotatable::Item(item) => Annotatable::Item(cfg.flat_map_item(item).pop().unwrap()), + Annotatable::TraitItem(item) => Annotatable::TraitItem( + item.map(|item| cfg.flat_map_trait_item(item).pop().unwrap()), + ), + Annotatable::ImplItem(item) => { + Annotatable::ImplItem(item.map(|item| cfg.flat_map_impl_item(item).pop().unwrap())) + } + Annotatable::ForeignItem(item) => Annotatable::ForeignItem( + item.map(|item| cfg.flat_map_foreign_item(item).pop().unwrap()), + ), + Annotatable::Stmt(stmt) => { + Annotatable::Stmt(stmt.map(|stmt| cfg.flat_map_stmt(stmt).pop().unwrap())) + } + Annotatable::Expr(mut expr) => Annotatable::Expr({ + cfg.visit_expr(&mut expr); + expr + }), + Annotatable::Arm(arm) => Annotatable::Arm(cfg.flat_map_arm(arm).pop().unwrap()), + Annotatable::Field(field) => { + Annotatable::Field(cfg.flat_map_field(field).pop().unwrap()) + } + Annotatable::FieldPat(fp) => { + Annotatable::FieldPat(cfg.flat_map_field_pattern(fp).pop().unwrap()) + } + Annotatable::GenericParam(param) => { + Annotatable::GenericParam(cfg.flat_map_generic_param(param).pop().unwrap()) + } + Annotatable::Param(param) => { + Annotatable::Param(cfg.flat_map_param(param).pop().unwrap()) + } + Annotatable::StructField(sf) => { + Annotatable::StructField(cfg.flat_map_struct_field(sf).pop().unwrap()) + } + Annotatable::Variant(v) => Annotatable::Variant(cfg.flat_map_variant(v).pop().unwrap()), + } + } + + fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtensionKind) -> AstFragment { + if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit { + let expn_data = self.cx.current_expansion.id.expn_data(); + let suggested_limit = self.cx.ecfg.recursion_limit * 2; + let mut err = self.cx.struct_span_err( + expn_data.call_site, + &format!( + "recursion limit reached while expanding the macro `{}`", + expn_data.kind.descr() + ), + ); + err.help(&format!( + "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate", + suggested_limit + )); + err.emit(); + self.cx.trace_macros_diag(); + FatalError.raise(); + } + + let (fragment_kind, span) = (invoc.fragment_kind, invoc.span()); + match invoc.kind { + InvocationKind::Bang { mac, .. } => match ext { + SyntaxExtensionKind::Bang(expander) => { + self.gate_proc_macro_expansion_kind(span, fragment_kind); + let tok_result = expander.expand(self.cx, span, mac.args.inner_tokens()); + self.parse_ast_fragment(tok_result, fragment_kind, &mac.path, span) + } + SyntaxExtensionKind::LegacyBang(expander) => { + let prev = self.cx.current_expansion.prior_type_ascription; + self.cx.current_expansion.prior_type_ascription = mac.prior_type_ascription; + let tok_result = expander.expand(self.cx, span, mac.args.inner_tokens()); + let result = if let Some(result) = fragment_kind.make_from(tok_result) { + result + } else { + let msg = format!( + "non-{kind} macro in {kind} position: {path}", + kind = fragment_kind.name(), + path = pprust::path_to_string(&mac.path), + ); + self.cx.span_err(span, &msg); + self.cx.trace_macros_diag(); + fragment_kind.dummy(span) + }; + self.cx.current_expansion.prior_type_ascription = prev; + result + } + _ => unreachable!(), + }, + InvocationKind::Attr { attr, mut item, .. } => match ext { + SyntaxExtensionKind::Attr(expander) => { + self.gate_proc_macro_input(&item); + self.gate_proc_macro_attr_item(span, &item); + let item_tok = TokenTree::token( + token::Interpolated(Lrc::new(match item { + Annotatable::Item(item) => token::NtItem(item), + Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()), + Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()), + Annotatable::ForeignItem(item) => { + token::NtForeignItem(item.into_inner()) + } + Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()), + Annotatable::Expr(expr) => token::NtExpr(expr), + Annotatable::Arm(..) + | Annotatable::Field(..) + | Annotatable::FieldPat(..) + | Annotatable::GenericParam(..) + | Annotatable::Param(..) + | Annotatable::StructField(..) + | Annotatable::Variant(..) => panic!("unexpected annotatable"), + })), + DUMMY_SP, + ) + .into(); + let item = attr.unwrap_normal_item(); + if let MacArgs::Eq(..) = item.args { + self.cx.span_err(span, "key-value macro attributes are not supported"); + } + let tok_result = + expander.expand(self.cx, span, item.args.inner_tokens(), item_tok); + self.parse_ast_fragment(tok_result, fragment_kind, &item.path, span) + } + SyntaxExtensionKind::LegacyAttr(expander) => { + match validate_attr::parse_meta(self.cx.parse_sess, &attr) { + Ok(meta) => { + let item = expander.expand(self.cx, span, &meta, item); + fragment_kind.expect_from_annotatables(item) + } + Err(mut err) => { + err.emit(); + fragment_kind.dummy(span) + } + } + } + SyntaxExtensionKind::NonMacroAttr { mark_used } => { + attr::mark_known(&attr); + if *mark_used { + attr::mark_used(&attr); + } + item.visit_attrs(|attrs| attrs.push(attr)); + fragment_kind.expect_from_annotatables(iter::once(item)) + } + _ => unreachable!(), + }, + InvocationKind::Derive { path, item } => match ext { + SyntaxExtensionKind::Derive(expander) + | SyntaxExtensionKind::LegacyDerive(expander) => { + if !item.derive_allowed() { + return fragment_kind.dummy(span); + } + if let SyntaxExtensionKind::Derive(..) = ext { + self.gate_proc_macro_input(&item); + } + let meta = ast::MetaItem { kind: ast::MetaItemKind::Word, span, path }; + let items = expander.expand(self.cx, span, &meta, item); + fragment_kind.expect_from_annotatables(items) + } + _ => unreachable!(), + }, + InvocationKind::DeriveContainer { .. } => unreachable!(), + } + } + + fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) { + let kind = match item { + Annotatable::Item(_) + | Annotatable::TraitItem(_) + | Annotatable::ImplItem(_) + | Annotatable::ForeignItem(_) => return, + Annotatable::Stmt(_) => "statements", + Annotatable::Expr(_) => "expressions", + Annotatable::Arm(..) + | Annotatable::Field(..) + | Annotatable::FieldPat(..) + | Annotatable::GenericParam(..) + | Annotatable::Param(..) + | Annotatable::StructField(..) + | Annotatable::Variant(..) => panic!("unexpected annotatable"), + }; + if self.cx.ecfg.proc_macro_hygiene() { + return; + } + feature_err( + self.cx.parse_sess, + sym::proc_macro_hygiene, + span, + &format!("custom attributes cannot be applied to {}", kind), + ) + .emit(); + } + + fn gate_proc_macro_input(&self, annotatable: &Annotatable) { + struct GateProcMacroInput<'a> { + parse_sess: &'a ParseSess, + } + + impl<'ast, 'a> Visitor<'ast> for GateProcMacroInput<'a> { + fn visit_item(&mut self, item: &'ast ast::Item) { + match &item.kind { + ast::ItemKind::Mod(module) if !module.inline => { + feature_err( + self.parse_sess, + sym::proc_macro_hygiene, + item.span, + "non-inline modules in proc macro input are unstable", + ) + .emit(); + } + _ => {} + } + + visit::walk_item(self, item); + } + + fn visit_mac(&mut self, _: &'ast ast::Mac) {} + } + + if !self.cx.ecfg.proc_macro_hygiene() { + annotatable.visit_with(&mut GateProcMacroInput { parse_sess: self.cx.parse_sess }); + } + } + + fn gate_proc_macro_expansion_kind(&self, span: Span, kind: AstFragmentKind) { + let kind = match kind { + AstFragmentKind::Expr | AstFragmentKind::OptExpr => "expressions", + AstFragmentKind::Pat => "patterns", + AstFragmentKind::Stmts => "statements", + AstFragmentKind::Ty + | AstFragmentKind::Items + | AstFragmentKind::TraitItems + | AstFragmentKind::ImplItems + | AstFragmentKind::ForeignItems => return, + AstFragmentKind::Arms + | AstFragmentKind::Fields + | AstFragmentKind::FieldPats + | AstFragmentKind::GenericParams + | AstFragmentKind::Params + | AstFragmentKind::StructFields + | AstFragmentKind::Variants => panic!("unexpected AST fragment kind"), + }; + if self.cx.ecfg.proc_macro_hygiene() { + return; + } + feature_err( + self.cx.parse_sess, + sym::proc_macro_hygiene, + span, + &format!("procedural macros cannot be expanded to {}", kind), + ) + .emit(); + } + + fn parse_ast_fragment( + &mut self, + toks: TokenStream, + kind: AstFragmentKind, + path: &Path, + span: Span, + ) -> AstFragment { + let mut parser = self.cx.new_parser_from_tts(toks); + match parse_ast_fragment(&mut parser, kind, false) { + Ok(fragment) => { + ensure_complete_parse(&mut parser, path, kind.name(), span); + fragment + } + Err(mut err) => { + err.set_span(span); + annotate_err_with_kind(&mut err, kind, span); + err.emit(); + self.cx.trace_macros_diag(); + kind.dummy(span) + } + } + } +} + +pub fn parse_ast_fragment<'a>( + this: &mut Parser<'a>, + kind: AstFragmentKind, + macro_legacy_warnings: bool, +) -> PResult<'a, AstFragment> { + Ok(match kind { + AstFragmentKind::Items => { + let mut items = SmallVec::new(); + while let Some(item) = this.parse_item()? { + items.push(item); + } + AstFragment::Items(items) + } + AstFragmentKind::TraitItems => { + let mut items = SmallVec::new(); + while this.token != token::Eof { + items.push(this.parse_trait_item(&mut false)?); + } + AstFragment::TraitItems(items) + } + AstFragmentKind::ImplItems => { + let mut items = SmallVec::new(); + while this.token != token::Eof { + items.push(this.parse_impl_item(&mut false)?); + } + AstFragment::ImplItems(items) + } + AstFragmentKind::ForeignItems => { + let mut items = SmallVec::new(); + while this.token != token::Eof { + items.push(this.parse_foreign_item(DUMMY_SP)?); + } + AstFragment::ForeignItems(items) + } + AstFragmentKind::Stmts => { + let mut stmts = SmallVec::new(); + while this.token != token::Eof && + // won't make progress on a `}` + this.token != token::CloseDelim(token::Brace) + { + if let Some(stmt) = this.parse_full_stmt(macro_legacy_warnings)? { + stmts.push(stmt); + } + } + AstFragment::Stmts(stmts) + } + AstFragmentKind::Expr => AstFragment::Expr(this.parse_expr()?), + AstFragmentKind::OptExpr => { + if this.token != token::Eof { + AstFragment::OptExpr(Some(this.parse_expr()?)) + } else { + AstFragment::OptExpr(None) + } + } + AstFragmentKind::Ty => AstFragment::Ty(this.parse_ty()?), + AstFragmentKind::Pat => AstFragment::Pat(this.parse_pat(None)?), + AstFragmentKind::Arms + | AstFragmentKind::Fields + | AstFragmentKind::FieldPats + | AstFragmentKind::GenericParams + | AstFragmentKind::Params + | AstFragmentKind::StructFields + | AstFragmentKind::Variants => panic!("unexpected AST fragment kind"), + }) +} + +pub fn ensure_complete_parse<'a>( + this: &mut Parser<'a>, + macro_path: &Path, + kind_name: &str, + span: Span, +) { + if this.token != token::Eof { + let token = pprust::token_to_string(&this.token); + let msg = format!("macro expansion ignores token `{}` and any following", token); + // Avoid emitting backtrace info twice. + let def_site_span = this.token.span.with_ctxt(SyntaxContext::root()); + let mut err = this.struct_span_err(def_site_span, &msg); + err.span_label(span, "caused by the macro expansion here"); + let msg = format!( + "the usage of `{}!` is likely invalid in {} context", + pprust::path_to_string(macro_path), + kind_name, + ); + err.note(&msg); + let semi_span = this.sess.source_map().next_point(span); + + let semi_full_span = semi_span.to(this.sess.source_map().next_point(semi_span)); + match this.sess.source_map().span_to_snippet(semi_full_span) { + Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => { + err.span_suggestion( + semi_span, + "you might be missing a semicolon here", + ";".to_owned(), + Applicability::MaybeIncorrect, + ); + } + _ => {} + } + err.emit(); + } +} + +struct InvocationCollector<'a, 'b> { + cx: &'a mut ExtCtxt<'b>, + cfg: StripUnconfigured<'a>, + invocations: Vec, + monotonic: bool, +} + +impl<'a, 'b> InvocationCollector<'a, 'b> { + fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment { + // Expansion data for all the collected invocations is set upon their resolution, + // with exception of the derive container case which is not resolved and can get + // its expansion data immediately. + let expn_data = match &kind { + InvocationKind::DeriveContainer { item, .. } => Some(ExpnData { + parent: self.cx.current_expansion.id, + ..ExpnData::default( + ExpnKind::Macro(MacroKind::Attr, sym::derive), + item.span(), + self.cx.parse_sess.edition, + ) + }), + _ => None, + }; + let expn_id = ExpnId::fresh(expn_data); + let vis = kind.placeholder_visibility(); + self.invocations.push(Invocation { + kind, + fragment_kind, + expansion_data: ExpansionData { + id: expn_id, + depth: self.cx.current_expansion.depth + 1, + ..self.cx.current_expansion.clone() + }, + }); + placeholder(fragment_kind, NodeId::placeholder_from_expn_id(expn_id), vis) + } + + fn collect_bang(&mut self, mac: ast::Mac, span: Span, kind: AstFragmentKind) -> AstFragment { + self.collect(kind, InvocationKind::Bang { mac, span }) + } + + fn collect_attr( + &mut self, + attr: Option, + derives: Vec, + item: Annotatable, + kind: AstFragmentKind, + after_derive: bool, + ) -> AstFragment { + self.collect( + kind, + match attr { + Some(attr) => InvocationKind::Attr { attr, item, derives, after_derive }, + None => InvocationKind::DeriveContainer { derives, item }, + }, + ) + } + + fn find_attr_invoc( + &self, + attrs: &mut Vec, + after_derive: &mut bool, + ) -> Option { + let attr = attrs + .iter() + .position(|a| { + if a.has_name(sym::derive) { + *after_derive = true; + } + !attr::is_known(a) && !is_builtin_attr(a) + }) + .map(|i| attrs.remove(i)); + if let Some(attr) = &attr { + if !self.cx.ecfg.custom_inner_attributes() + && attr.style == ast::AttrStyle::Inner + && !attr.has_name(sym::test) + { + feature_err( + &self.cx.parse_sess, + sym::custom_inner_attributes, + attr.span, + "non-builtin inner attributes are unstable", + ) + .emit(); + } + } + attr + } + + /// If `item` is an attr invocation, remove and return the macro attribute and derive traits. + fn classify_item( + &mut self, + item: &mut T, + ) -> (Option, Vec, /* after_derive */ bool) + where + T: HasAttrs, + { + let (mut attr, mut traits, mut after_derive) = (None, Vec::new(), false); + + item.visit_attrs(|mut attrs| { + attr = self.find_attr_invoc(&mut attrs, &mut after_derive); + traits = collect_derives(&mut self.cx, &mut attrs); + }); + + (attr, traits, after_derive) + } + + /// Alternative to `classify_item()` that ignores `#[derive]` so invocations fallthrough + /// to the unused-attributes lint (making it an error on statements and expressions + /// is a breaking change) + fn classify_nonitem( + &mut self, + nonitem: &mut T, + ) -> (Option, /* after_derive */ bool) { + let (mut attr, mut after_derive) = (None, false); + + nonitem.visit_attrs(|mut attrs| { + attr = self.find_attr_invoc(&mut attrs, &mut after_derive); + }); + + (attr, after_derive) + } + + fn configure(&mut self, node: T) -> Option { + self.cfg.configure(node) + } + + // Detect use of feature-gated or invalid attributes on macro invocations + // since they will not be detected after macro expansion. + fn check_attributes(&mut self, attrs: &[ast::Attribute]) { + let features = self.cx.ecfg.features.unwrap(); + for attr in attrs.iter() { + feature_gate::check_attribute(attr, self.cx.parse_sess, features); + validate_attr::check_meta(self.cx.parse_sess, attr); + + // macros are expanded before any lint passes so this warning has to be hardcoded + if attr.has_name(sym::derive) { + self.cx + .struct_span_warn(attr.span, "`#[derive]` does nothing on macro invocations") + .note("this may become a hard error in a future release") + .emit(); + } + } + } +} + +impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { + fn visit_expr(&mut self, expr: &mut P) { + self.cfg.configure_expr(expr); + visit_clobber(expr.deref_mut(), |mut expr| { + self.cfg.configure_expr_kind(&mut expr.kind); + + // ignore derives so they remain unused + let (attr, after_derive) = self.classify_nonitem(&mut expr); + + if attr.is_some() { + // Collect the invoc regardless of whether or not attributes are permitted here + // expansion will eat the attribute so it won't error later. + attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); + + // AstFragmentKind::Expr requires the macro to emit an expression. + return self + .collect_attr( + attr, + vec![], + Annotatable::Expr(P(expr)), + AstFragmentKind::Expr, + after_derive, + ) + .make_expr() + .into_inner(); + } + + if let ast::ExprKind::Mac(mac) = expr.kind { + self.check_attributes(&expr.attrs); + self.collect_bang(mac, expr.span, AstFragmentKind::Expr).make_expr().into_inner() + } else { + noop_visit_expr(&mut expr, self); + expr + } + }); + } + + fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { + let mut arm = configure!(self, arm); + + let (attr, traits, after_derive) = self.classify_item(&mut arm); + if attr.is_some() || !traits.is_empty() { + return self + .collect_attr( + attr, + traits, + Annotatable::Arm(arm), + AstFragmentKind::Arms, + after_derive, + ) + .make_arms(); + } + + noop_flat_map_arm(arm, self) + } + + fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { + let mut field = configure!(self, field); + + let (attr, traits, after_derive) = self.classify_item(&mut field); + if attr.is_some() || !traits.is_empty() { + return self + .collect_attr( + attr, + traits, + Annotatable::Field(field), + AstFragmentKind::Fields, + after_derive, + ) + .make_fields(); + } + + noop_flat_map_field(field, self) + } + + fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { + let mut fp = configure!(self, fp); + + let (attr, traits, after_derive) = self.classify_item(&mut fp); + if attr.is_some() || !traits.is_empty() { + return self + .collect_attr( + attr, + traits, + Annotatable::FieldPat(fp), + AstFragmentKind::FieldPats, + after_derive, + ) + .make_field_patterns(); + } + + noop_flat_map_field_pattern(fp, self) + } + + fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { + let mut p = configure!(self, p); + + let (attr, traits, after_derive) = self.classify_item(&mut p); + if attr.is_some() || !traits.is_empty() { + return self + .collect_attr( + attr, + traits, + Annotatable::Param(p), + AstFragmentKind::Params, + after_derive, + ) + .make_params(); + } + + noop_flat_map_param(p, self) + } + + fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { + let mut sf = configure!(self, sf); + + let (attr, traits, after_derive) = self.classify_item(&mut sf); + if attr.is_some() || !traits.is_empty() { + return self + .collect_attr( + attr, + traits, + Annotatable::StructField(sf), + AstFragmentKind::StructFields, + after_derive, + ) + .make_struct_fields(); + } + + noop_flat_map_struct_field(sf, self) + } + + fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { + let mut variant = configure!(self, variant); + + let (attr, traits, after_derive) = self.classify_item(&mut variant); + if attr.is_some() || !traits.is_empty() { + return self + .collect_attr( + attr, + traits, + Annotatable::Variant(variant), + AstFragmentKind::Variants, + after_derive, + ) + .make_variants(); + } + + noop_flat_map_variant(variant, self) + } + + fn filter_map_expr(&mut self, expr: P) -> Option> { + let expr = configure!(self, expr); + expr.filter_map(|mut expr| { + self.cfg.configure_expr_kind(&mut expr.kind); + + // Ignore derives so they remain unused. + let (attr, after_derive) = self.classify_nonitem(&mut expr); + + if attr.is_some() { + attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); + + return self + .collect_attr( + attr, + vec![], + Annotatable::Expr(P(expr)), + AstFragmentKind::OptExpr, + after_derive, + ) + .make_opt_expr() + .map(|expr| expr.into_inner()); + } + + if let ast::ExprKind::Mac(mac) = expr.kind { + self.check_attributes(&expr.attrs); + self.collect_bang(mac, expr.span, AstFragmentKind::OptExpr) + .make_opt_expr() + .map(|expr| expr.into_inner()) + } else { + Some({ + noop_visit_expr(&mut expr, self); + expr + }) + } + }) + } + + fn visit_pat(&mut self, pat: &mut P) { + self.cfg.configure_pat(pat); + match pat.kind { + PatKind::Mac(_) => {} + _ => return noop_visit_pat(pat, self), + } + + visit_clobber(pat, |mut pat| match mem::replace(&mut pat.kind, PatKind::Wild) { + PatKind::Mac(mac) => self.collect_bang(mac, pat.span, AstFragmentKind::Pat).make_pat(), + _ => unreachable!(), + }); + } + + fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { + let mut stmt = configure!(self, stmt); + + // we'll expand attributes on expressions separately + if !stmt.is_expr() { + let (attr, derives, after_derive) = if stmt.is_item() { + self.classify_item(&mut stmt) + } else { + // ignore derives on non-item statements so it falls through + // to the unused-attributes lint + let (attr, after_derive) = self.classify_nonitem(&mut stmt); + (attr, vec![], after_derive) + }; + + if attr.is_some() || !derives.is_empty() { + return self + .collect_attr( + attr, + derives, + Annotatable::Stmt(P(stmt)), + AstFragmentKind::Stmts, + after_derive, + ) + .make_stmts(); + } + } + + if let StmtKind::Mac(mac) = stmt.kind { + let (mac, style, attrs) = mac.into_inner(); + self.check_attributes(&attrs); + let mut placeholder = + self.collect_bang(mac, stmt.span, AstFragmentKind::Stmts).make_stmts(); + + // If this is a macro invocation with a semicolon, then apply that + // semicolon to the final statement produced by expansion. + if style == MacStmtStyle::Semicolon { + if let Some(stmt) = placeholder.pop() { + placeholder.push(stmt.add_trailing_semicolon()); + } + } + + return placeholder; + } + + // The placeholder expander gives ids to statements, so we avoid folding the id here. + let ast::Stmt { id, kind, span } = stmt; + noop_flat_map_stmt_kind(kind, self) + .into_iter() + .map(|kind| ast::Stmt { id, kind, span }) + .collect() + } + + fn visit_block(&mut self, block: &mut P) { + let old_directory_ownership = self.cx.current_expansion.directory_ownership; + self.cx.current_expansion.directory_ownership = DirectoryOwnership::UnownedViaBlock; + noop_visit_block(block, self); + self.cx.current_expansion.directory_ownership = old_directory_ownership; + } + + fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { + let mut item = configure!(self, item); + + let (attr, traits, after_derive) = self.classify_item(&mut item); + if attr.is_some() || !traits.is_empty() { + return self + .collect_attr( + attr, + traits, + Annotatable::Item(item), + AstFragmentKind::Items, + after_derive, + ) + .make_items(); + } + + match item.kind { + ast::ItemKind::Mac(..) => { + self.check_attributes(&item.attrs); + item.and_then(|item| match item.kind { + ItemKind::Mac(mac) => self + .collect( + AstFragmentKind::Items, + InvocationKind::Bang { mac, span: item.span }, + ) + .make_items(), + _ => unreachable!(), + }) + } + ast::ItemKind::Mod(ast::Mod { inner, .. }) => { + if item.ident == Ident::invalid() { + return noop_flat_map_item(item, self); + } + + let orig_directory_ownership = self.cx.current_expansion.directory_ownership; + let mut module = (*self.cx.current_expansion.module).clone(); + module.mod_path.push(item.ident); + + // Detect if this is an inline module (`mod m { ... }` as opposed to `mod m;`). + // In the non-inline case, `inner` is never the dummy span (cf. `parse_item_mod`). + // Thus, if `inner` is the dummy span, we know the module is inline. + let inline_module = item.span.contains(inner) || inner.is_dummy(); + + if inline_module { + if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, sym::path) { + self.cx.current_expansion.directory_ownership = + DirectoryOwnership::Owned { relative: None }; + module.directory.push(&*path.as_str()); + } else { + module.directory.push(&*item.ident.as_str()); + } + } else { + let path = self.cx.parse_sess.source_map().span_to_unmapped_path(inner); + let mut path = match path { + FileName::Real(path) => path, + other => PathBuf::from(other.to_string()), + }; + let directory_ownership = match path.file_name().unwrap().to_str() { + Some("mod.rs") => DirectoryOwnership::Owned { relative: None }, + Some(_) => DirectoryOwnership::Owned { relative: Some(item.ident) }, + None => DirectoryOwnership::UnownedViaMod, + }; + path.pop(); + module.directory = path; + self.cx.current_expansion.directory_ownership = directory_ownership; + } + + let orig_module = + mem::replace(&mut self.cx.current_expansion.module, Rc::new(module)); + let result = noop_flat_map_item(item, self); + self.cx.current_expansion.module = orig_module; + self.cx.current_expansion.directory_ownership = orig_directory_ownership; + result + } + + _ => noop_flat_map_item(item, self), + } + } + + fn flat_map_trait_item(&mut self, item: ast::AssocItem) -> SmallVec<[ast::AssocItem; 1]> { + let mut item = configure!(self, item); + + let (attr, traits, after_derive) = self.classify_item(&mut item); + if attr.is_some() || !traits.is_empty() { + return self + .collect_attr( + attr, + traits, + Annotatable::TraitItem(P(item)), + AstFragmentKind::TraitItems, + after_derive, + ) + .make_trait_items(); + } + + match item.kind { + ast::AssocItemKind::Macro(mac) => { + let ast::AssocItem { attrs, span, .. } = item; + self.check_attributes(&attrs); + self.collect_bang(mac, span, AstFragmentKind::TraitItems).make_trait_items() + } + _ => noop_flat_map_assoc_item(item, self), + } + } + + fn flat_map_impl_item(&mut self, item: ast::AssocItem) -> SmallVec<[ast::AssocItem; 1]> { + let mut item = configure!(self, item); + + let (attr, traits, after_derive) = self.classify_item(&mut item); + if attr.is_some() || !traits.is_empty() { + return self + .collect_attr( + attr, + traits, + Annotatable::ImplItem(P(item)), + AstFragmentKind::ImplItems, + after_derive, + ) + .make_impl_items(); + } + + match item.kind { + ast::AssocItemKind::Macro(mac) => { + let ast::AssocItem { attrs, span, .. } = item; + self.check_attributes(&attrs); + self.collect_bang(mac, span, AstFragmentKind::ImplItems).make_impl_items() + } + _ => noop_flat_map_assoc_item(item, self), + } + } + + fn visit_ty(&mut self, ty: &mut P) { + match ty.kind { + ast::TyKind::Mac(_) => {} + _ => return noop_visit_ty(ty, self), + }; + + visit_clobber(ty, |mut ty| match mem::replace(&mut ty.kind, ast::TyKind::Err) { + ast::TyKind::Mac(mac) => self.collect_bang(mac, ty.span, AstFragmentKind::Ty).make_ty(), + _ => unreachable!(), + }); + } + + fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) { + self.cfg.configure_foreign_mod(foreign_mod); + noop_visit_foreign_mod(foreign_mod, self); + } + + fn flat_map_foreign_item( + &mut self, + mut foreign_item: ast::ForeignItem, + ) -> SmallVec<[ast::ForeignItem; 1]> { + let (attr, traits, after_derive) = self.classify_item(&mut foreign_item); + + if attr.is_some() || !traits.is_empty() { + return self + .collect_attr( + attr, + traits, + Annotatable::ForeignItem(P(foreign_item)), + AstFragmentKind::ForeignItems, + after_derive, + ) + .make_foreign_items(); + } + + if let ast::ForeignItemKind::Macro(mac) = foreign_item.kind { + self.check_attributes(&foreign_item.attrs); + return self + .collect_bang(mac, foreign_item.span, AstFragmentKind::ForeignItems) + .make_foreign_items(); + } + + noop_flat_map_foreign_item(foreign_item, self) + } + + fn visit_item_kind(&mut self, item: &mut ast::ItemKind) { + match item { + ast::ItemKind::MacroDef(..) => {} + _ => { + self.cfg.configure_item_kind(item); + noop_visit_item_kind(item, self); + } + } + } + + fn flat_map_generic_param( + &mut self, + param: ast::GenericParam, + ) -> SmallVec<[ast::GenericParam; 1]> { + let mut param = configure!(self, param); + + let (attr, traits, after_derive) = self.classify_item(&mut param); + if attr.is_some() || !traits.is_empty() { + return self + .collect_attr( + attr, + traits, + Annotatable::GenericParam(param), + AstFragmentKind::GenericParams, + after_derive, + ) + .make_generic_params(); + } + + noop_flat_map_generic_param(param, self) + } + + fn visit_attribute(&mut self, at: &mut ast::Attribute) { + // turn `#[doc(include="filename")]` attributes into `#[doc(include(file="filename", + // contents="file contents")]` attributes + if !at.check_name(sym::doc) { + return noop_visit_attribute(at, self); + } + + if let Some(list) = at.meta_item_list() { + if !list.iter().any(|it| it.check_name(sym::include)) { + return noop_visit_attribute(at, self); + } + + let mut items = vec![]; + + for mut it in list { + if !it.check_name(sym::include) { + items.push({ + noop_visit_meta_list_item(&mut it, self); + it + }); + continue; + } + + if let Some(file) = it.value_str() { + let err_count = self.cx.parse_sess.span_diagnostic.err_count(); + self.check_attributes(slice::from_ref(at)); + if self.cx.parse_sess.span_diagnostic.err_count() > err_count { + // avoid loading the file if they haven't enabled the feature + return noop_visit_attribute(at, self); + } + + let filename = match self.cx.resolve_path(&*file.as_str(), it.span()) { + Ok(filename) => filename, + Err(mut err) => { + err.emit(); + continue; + } + }; + + match self.cx.source_map().load_file(&filename) { + Ok(source_file) => { + let src = source_file + .src + .as_ref() + .expect("freshly loaded file should have a source"); + let src_interned = Symbol::intern(src.as_str()); + + let include_info = vec![ + ast::NestedMetaItem::MetaItem(attr::mk_name_value_item_str( + Ident::with_dummy_span(sym::file), + file, + DUMMY_SP, + )), + ast::NestedMetaItem::MetaItem(attr::mk_name_value_item_str( + Ident::with_dummy_span(sym::contents), + src_interned, + DUMMY_SP, + )), + ]; + + let include_ident = Ident::with_dummy_span(sym::include); + let item = attr::mk_list_item(include_ident, include_info); + items.push(ast::NestedMetaItem::MetaItem(item)); + } + Err(e) => { + let lit = + it.meta_item().and_then(|item| item.name_value_literal()).unwrap(); + + if e.kind() == ErrorKind::InvalidData { + self.cx + .struct_span_err( + lit.span, + &format!("{} wasn't a utf-8 file", filename.display()), + ) + .span_label(lit.span, "contains invalid utf-8") + .emit(); + } else { + let mut err = self.cx.struct_span_err( + lit.span, + &format!("couldn't read {}: {}", filename.display(), e), + ); + err.span_label(lit.span, "couldn't read file"); + + err.emit(); + } + } + } + } else { + let mut err = self.cx.struct_span_err( + it.span(), + &format!("expected path to external documentation"), + ); + + // Check if the user erroneously used `doc(include(...))` syntax. + let literal = it.meta_item_list().and_then(|list| { + if list.len() == 1 { + list[0].literal().map(|literal| &literal.kind) + } else { + None + } + }); + + let (path, applicability) = match &literal { + Some(LitKind::Str(path, ..)) => { + (path.to_string(), Applicability::MachineApplicable) + } + _ => (String::from(""), Applicability::HasPlaceholders), + }; + + err.span_suggestion( + it.span(), + "provide a file path with `=`", + format!("include = \"{}\"", path), + applicability, + ); + + err.emit(); + } + } + + let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items); + *at = attr::Attribute { + kind: ast::AttrKind::Normal(AttrItem { + path: meta.path, + args: meta.kind.mac_args(meta.span), + }), + span: at.span, + id: at.id, + style: at.style, + }; + } else { + noop_visit_attribute(at, self) + } + } + + fn visit_id(&mut self, id: &mut ast::NodeId) { + if self.monotonic { + debug_assert_eq!(*id, ast::DUMMY_NODE_ID); + *id = self.cx.resolver.next_node_id() + } + } + + fn visit_fn_decl(&mut self, mut fn_decl: &mut P) { + self.cfg.configure_fn_decl(&mut fn_decl); + noop_visit_fn_decl(fn_decl, self); + } +} + +pub struct ExpansionConfig<'feat> { + pub crate_name: String, + pub features: Option<&'feat Features>, + pub recursion_limit: usize, + pub trace_mac: bool, + pub should_test: bool, // If false, strip `#[test]` nodes + pub single_step: bool, + pub keep_macs: bool, +} + +impl<'feat> ExpansionConfig<'feat> { + pub fn default(crate_name: String) -> ExpansionConfig<'static> { + ExpansionConfig { + crate_name, + features: None, + recursion_limit: 1024, + trace_mac: false, + should_test: false, + single_step: false, + keep_macs: false, + } + } + + fn proc_macro_hygiene(&self) -> bool { + self.features.map_or(false, |features| features.proc_macro_hygiene) + } + fn custom_inner_attributes(&self) -> bool { + self.features.map_or(false, |features| features.custom_inner_attributes) + } +} diff --git a/src/librustc_expand/lib.rs b/src/librustc_expand/lib.rs new file mode 100644 index 00000000000..258a7478329 --- /dev/null +++ b/src/librustc_expand/lib.rs @@ -0,0 +1,67 @@ +#![feature(crate_visibility_modifier)] +#![feature(decl_macro)] +#![feature(proc_macro_diagnostic)] +#![feature(proc_macro_internals)] +#![feature(proc_macro_span)] + +extern crate proc_macro as pm; + +// A variant of 'try!' that panics on an Err. This is used as a crutch on the +// way towards a non-panic!-prone parser. It should be used for fatal parsing +// errors; eventually we plan to convert all code using panictry to just use +// normal try. +#[macro_export] +macro_rules! panictry { + ($e:expr) => {{ + use errors::FatalError; + use std::result::Result::{Err, Ok}; + match $e { + Ok(e) => e, + Err(mut e) => { + e.emit(); + FatalError.raise() + } + } + }}; +} + +mod placeholders; +mod proc_macro_server; + +pub use mbe::macro_rules::compile_declarative_macro; +crate use syntax_pos::hygiene; +pub mod base; +pub mod build; +pub mod expand; +pub use rustc_parse::config; +pub mod proc_macro; + +crate mod mbe; + +// HACK(Centril, #64197): These shouldn't really be here. +// Rather, they should be with their respective modules which are defined in other crates. +// However, since for now constructing a `ParseSess` sorta requires `config` from this crate, +// these tests will need to live here in the iterim. + +#[cfg(test)] +mod tests; +#[cfg(test)] +mod parse { + #[cfg(test)] + mod tests; + #[cfg(test)] + mod lexer { + #[cfg(test)] + mod tests; + } +} +#[cfg(test)] +mod tokenstream { + #[cfg(test)] + mod tests; +} +#[cfg(test)] +mod mut_visit { + #[cfg(test)] + mod tests; +} diff --git a/src/librustc_expand/mbe.rs b/src/librustc_expand/mbe.rs new file mode 100644 index 00000000000..0473b653424 --- /dev/null +++ b/src/librustc_expand/mbe.rs @@ -0,0 +1,156 @@ +//! This module implements declarative macros: old `macro_rules` and the newer +//! `macro`. Declarative macros are also known as "macro by example", and that's +//! why we call this module `mbe`. For external documentation, prefer the +//! official terminology: "declarative macros". + +crate mod macro_check; +crate mod macro_parser; +crate mod macro_rules; +crate mod quoted; +crate mod transcribe; + +use syntax::ast; +use syntax::token::{self, Token, TokenKind}; +use syntax::tokenstream::DelimSpan; + +use syntax_pos::Span; + +use rustc_data_structures::sync::Lrc; + +/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note +/// that the delimiter itself might be `NoDelim`. +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +struct Delimited { + delim: token::DelimToken, + tts: Vec, +} + +impl Delimited { + /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter. + fn open_tt(&self, span: DelimSpan) -> TokenTree { + TokenTree::token(token::OpenDelim(self.delim), span.open) + } + + /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter. + fn close_tt(&self, span: DelimSpan) -> TokenTree { + TokenTree::token(token::CloseDelim(self.delim), span.close) + } +} + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +struct SequenceRepetition { + /// The sequence of token trees + tts: Vec, + /// The optional separator + separator: Option, + /// Whether the sequence can be repeated zero (*), or one or more times (+) + kleene: KleeneToken, + /// The number of `Match`s that appear in the sequence (and subsequences) + num_captures: usize, +} + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)] +struct KleeneToken { + span: Span, + op: KleeneOp, +} + +impl KleeneToken { + fn new(op: KleeneOp, span: Span) -> KleeneToken { + KleeneToken { span, op } + } +} + +/// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) +/// for token sequences. +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)] +enum KleeneOp { + /// Kleene star (`*`) for zero or more repetitions + ZeroOrMore, + /// Kleene plus (`+`) for one or more repetitions + OneOrMore, + /// Kleene optional (`?`) for zero or one reptitions + ZeroOrOne, +} + +/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)` +/// are "first-class" token trees. Useful for parsing macros. +#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)] +enum TokenTree { + Token(Token), + Delimited(DelimSpan, Lrc), + /// A kleene-style repetition sequence + Sequence(DelimSpan, Lrc), + /// e.g., `$var` + MetaVar(Span, ast::Ident), + /// e.g., `$var:expr`. This is only used in the left hand side of MBE macros. + MetaVarDecl( + Span, + ast::Ident, /* name to bind */ + ast::Ident, /* kind of nonterminal */ + ), +} + +impl TokenTree { + /// Return the number of tokens in the tree. + fn len(&self) -> usize { + match *self { + TokenTree::Delimited(_, ref delimed) => match delimed.delim { + token::NoDelim => delimed.tts.len(), + _ => delimed.tts.len() + 2, + }, + TokenTree::Sequence(_, ref seq) => seq.tts.len(), + _ => 0, + } + } + + /// Returns `true` if the given token tree is delimited. + fn is_delimited(&self) -> bool { + match *self { + TokenTree::Delimited(..) => true, + _ => false, + } + } + + /// Returns `true` if the given token tree is a token of the given kind. + fn is_token(&self, expected_kind: &TokenKind) -> bool { + match self { + TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind, + _ => false, + } + } + + /// Gets the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences. + fn get_tt(&self, index: usize) -> TokenTree { + match (self, index) { + (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => { + delimed.tts[index].clone() + } + (&TokenTree::Delimited(span, ref delimed), _) => { + if index == 0 { + return delimed.open_tt(span); + } + if index == delimed.tts.len() + 1 { + return delimed.close_tt(span); + } + delimed.tts[index - 1].clone() + } + (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(), + _ => panic!("Cannot expand a token tree"), + } + } + + /// Retrieves the `TokenTree`'s span. + fn span(&self) -> Span { + match *self { + TokenTree::Token(Token { span, .. }) + | TokenTree::MetaVar(span, _) + | TokenTree::MetaVarDecl(span, _, _) => span, + TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(), + } + } + + fn token(kind: TokenKind, span: Span) -> TokenTree { + TokenTree::Token(Token::new(kind, span)) + } +} diff --git a/src/librustc_expand/mbe/macro_check.rs b/src/librustc_expand/mbe/macro_check.rs new file mode 100644 index 00000000000..616fddd3c1c --- /dev/null +++ b/src/librustc_expand/mbe/macro_check.rs @@ -0,0 +1,627 @@ +//! Checks that meta-variables in macro definition are correctly declared and used. +//! +//! # What is checked +//! +//! ## Meta-variables must not be bound twice +//! +//! ``` +//! macro_rules! foo { ($x:tt $x:tt) => { $x }; } +//! ``` +//! +//! This check is sound (no false-negative) and complete (no false-positive). +//! +//! ## Meta-variables must not be free +//! +//! ``` +//! macro_rules! foo { () => { $x }; } +//! ``` +//! +//! This check is also done at macro instantiation but only if the branch is taken. +//! +//! ## Meta-variables must repeat at least as many times as their binder +//! +//! ``` +//! macro_rules! foo { ($($x:tt)*) => { $x }; } +//! ``` +//! +//! This check is also done at macro instantiation but only if the branch is taken. +//! +//! ## Meta-variables must repeat with the same Kleene operators as their binder +//! +//! ``` +//! macro_rules! foo { ($($x:tt)+) => { $($x)* }; } +//! ``` +//! +//! This check is not done at macro instantiation. +//! +//! # Disclaimer +//! +//! In the presence of nested macros (a macro defined in a macro), those checks may have false +//! positives and false negatives. We try to detect those cases by recognizing potential macro +//! definitions in RHSes, but nested macros may be hidden through the use of particular values of +//! meta-variables. +//! +//! ## Examples of false positive +//! +//! False positives can come from cases where we don't recognize a nested macro, because it depends +//! on particular values of meta-variables. In the following example, we think both instances of +//! `$x` are free, which is a correct statement if `$name` is anything but `macro_rules`. But when +//! `$name` is `macro_rules`, like in the instantiation below, then `$x:tt` is actually a binder of +//! the nested macro and `$x` is bound to it. +//! +//! ``` +//! macro_rules! foo { ($name:ident) => { $name! bar { ($x:tt) => { $x }; } }; } +//! foo!(macro_rules); +//! ``` +//! +//! False positives can also come from cases where we think there is a nested macro while there +//! isn't. In the following example, we think `$x` is free, which is incorrect because `bar` is not +//! a nested macro since it is not evaluated as code by `stringify!`. +//! +//! ``` +//! macro_rules! foo { () => { stringify!(macro_rules! bar { () => { $x }; }) }; } +//! ``` +//! +//! ## Examples of false negative +//! +//! False negatives can come from cases where we don't recognize a meta-variable, because it depends +//! on particular values of meta-variables. In the following examples, we don't see that if `$d` is +//! instantiated with `$` then `$d z` becomes `$z` in the nested macro definition and is thus a free +//! meta-variable. Note however, that if `foo` is instantiated, then we would check the definition +//! of `bar` and would see the issue. +//! +//! ``` +//! macro_rules! foo { ($d:tt) => { macro_rules! bar { ($y:tt) => { $d z }; } }; } +//! ``` +//! +//! # How it is checked +//! +//! There are 3 main functions: `check_binders`, `check_occurrences`, and `check_nested_macro`. They +//! all need some kind of environment. +//! +//! ## Environments +//! +//! Environments are used to pass information. +//! +//! ### From LHS to RHS +//! +//! When checking a LHS with `check_binders`, we produce (and use) an environment for binders, +//! namely `Binders`. This is a mapping from binder name to information about that binder: the span +//! of the binder for error messages and the stack of Kleene operators under which it was bound in +//! the LHS. +//! +//! This environment is used by both the LHS and RHS. The LHS uses it to detect duplicate binders. +//! The RHS uses it to detect the other errors. +//! +//! ### From outer macro to inner macro +//! +//! When checking the RHS of an outer macro and we detect a nested macro definition, we push the +//! current state, namely `MacroState`, to an environment of nested macro definitions. Each state +//! stores the LHS binders when entering the macro definition as well as the stack of Kleene +//! operators under which the inner macro is defined in the RHS. +//! +//! This environment is a stack representing the nesting of macro definitions. As such, the stack of +//! Kleene operators under which a meta-variable is repeating is the concatenation of the stacks +//! stored when entering a macro definition starting from the state in which the meta-variable is +//! bound. +use crate::mbe::{KleeneToken, TokenTree}; + +use syntax::ast::NodeId; +use syntax::early_buffered_lints::META_VARIABLE_MISUSE; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym}; +use syntax::token::{DelimToken, Token, TokenKind}; + +use rustc_data_structures::fx::FxHashMap; +use smallvec::SmallVec; +use syntax_pos::{symbol::Ident, MultiSpan, Span}; + +/// Stack represented as linked list. +/// +/// Those are used for environments because they grow incrementally and are not mutable. +enum Stack<'a, T> { + /// Empty stack. + Empty, + /// A non-empty stack. + Push { + /// The top element. + top: T, + /// The previous elements. + prev: &'a Stack<'a, T>, + }, +} + +impl<'a, T> Stack<'a, T> { + /// Returns whether a stack is empty. + fn is_empty(&self) -> bool { + match *self { + Stack::Empty => true, + _ => false, + } + } + + /// Returns a new stack with an element of top. + fn push(&'a self, top: T) -> Stack<'a, T> { + Stack::Push { top, prev: self } + } +} + +impl<'a, T> Iterator for &'a Stack<'a, T> { + type Item = &'a T; + + // Iterates from top to bottom of the stack. + fn next(&mut self) -> Option<&'a T> { + match *self { + Stack::Empty => None, + Stack::Push { ref top, ref prev } => { + *self = prev; + Some(top) + } + } + } +} + +impl From<&Stack<'_, KleeneToken>> for SmallVec<[KleeneToken; 1]> { + fn from(ops: &Stack<'_, KleeneToken>) -> SmallVec<[KleeneToken; 1]> { + let mut ops: SmallVec<[KleeneToken; 1]> = ops.cloned().collect(); + // The stack is innermost on top. We want outermost first. + ops.reverse(); + ops + } +} + +/// Information attached to a meta-variable binder in LHS. +struct BinderInfo { + /// The span of the meta-variable in LHS. + span: Span, + /// The stack of Kleene operators (outermost first). + ops: SmallVec<[KleeneToken; 1]>, +} + +/// An environment of meta-variables to their binder information. +type Binders = FxHashMap; + +/// The state at which we entered a macro definition in the RHS of another macro definition. +struct MacroState<'a> { + /// The binders of the branch where we entered the macro definition. + binders: &'a Binders, + /// The stack of Kleene operators (outermost first) where we entered the macro definition. + ops: SmallVec<[KleeneToken; 1]>, +} + +/// Checks that meta-variables are used correctly in a macro definition. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `span` is used when no spans are available +/// - `lhses` and `rhses` should have the same length and represent the macro definition +pub(super) fn check_meta_variables( + sess: &ParseSess, + node_id: NodeId, + span: Span, + lhses: &[TokenTree], + rhses: &[TokenTree], +) -> bool { + if lhses.len() != rhses.len() { + sess.span_diagnostic.span_bug(span, "length mismatch between LHSes and RHSes") + } + let mut valid = true; + for (lhs, rhs) in lhses.iter().zip(rhses.iter()) { + let mut binders = Binders::default(); + check_binders(sess, node_id, lhs, &Stack::Empty, &mut binders, &Stack::Empty, &mut valid); + check_occurrences(sess, node_id, rhs, &Stack::Empty, &binders, &Stack::Empty, &mut valid); + } + valid +} + +/// Checks `lhs` as part of the LHS of a macro definition, extends `binders` with new binders, and +/// sets `valid` to false in case of errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `lhs` is checked as part of a LHS +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the LHS +/// - `ops` is the stack of Kleene operators from the LHS +/// - `valid` is set in case of errors +fn check_binders( + sess: &ParseSess, + node_id: NodeId, + lhs: &TokenTree, + macros: &Stack<'_, MacroState<'_>>, + binders: &mut Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + match *lhs { + TokenTree::Token(..) => {} + // This can only happen when checking a nested macro because this LHS is then in the RHS of + // the outer macro. See ui/macros/macro-of-higher-order.rs where $y:$fragment in the + // LHS of the nested macro (and RHS of the outer macro) is parsed as MetaVar(y) Colon + // MetaVar(fragment) and not as MetaVarDecl(y, fragment). + TokenTree::MetaVar(span, name) => { + if macros.is_empty() { + sess.span_diagnostic.span_bug(span, "unexpected MetaVar in lhs"); + } + // There are 3 possibilities: + if let Some(prev_info) = binders.get(&name) { + // 1. The meta-variable is already bound in the current LHS: This is an error. + let mut span = MultiSpan::from_span(span); + span.push_span_label(prev_info.span, "previous declaration".into()); + buffer_lint(sess, span, node_id, "duplicate matcher binding"); + } else if get_binder_info(macros, binders, name).is_none() { + // 2. The meta-variable is free: This is a binder. + binders.insert(name, BinderInfo { span, ops: ops.into() }); + } else { + // 3. The meta-variable is bound: This is an occurrence. + check_occurrences(sess, node_id, lhs, macros, binders, ops, valid); + } + } + // Similarly, this can only happen when checking a toplevel macro. + TokenTree::MetaVarDecl(span, name, _kind) => { + if !macros.is_empty() { + sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in nested lhs"); + } + if let Some(prev_info) = get_binder_info(macros, binders, name) { + // Duplicate binders at the top-level macro definition are errors. The lint is only + // for nested macro definitions. + sess.span_diagnostic + .struct_span_err(span, "duplicate matcher binding") + .span_label(span, "duplicate binding") + .span_label(prev_info.span, "previous binding") + .emit(); + *valid = false; + } else { + binders.insert(name, BinderInfo { span, ops: ops.into() }); + } + } + TokenTree::Delimited(_, ref del) => { + for tt in &del.tts { + check_binders(sess, node_id, tt, macros, binders, ops, valid); + } + } + TokenTree::Sequence(_, ref seq) => { + let ops = ops.push(seq.kleene); + for tt in &seq.tts { + check_binders(sess, node_id, tt, macros, binders, &ops, valid); + } + } + } +} + +/// Returns the binder information of a meta-variable. +/// +/// Arguments: +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the current binders +/// - `name` is the name of the meta-variable we are looking for +fn get_binder_info<'a>( + mut macros: &'a Stack<'a, MacroState<'a>>, + binders: &'a Binders, + name: Ident, +) -> Option<&'a BinderInfo> { + binders.get(&name).or_else(|| macros.find_map(|state| state.binders.get(&name))) +} + +/// Checks `rhs` as part of the RHS of a macro definition and sets `valid` to false in case of +/// errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `rhs` is checked as part of a RHS +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `valid` is set in case of errors +fn check_occurrences( + sess: &ParseSess, + node_id: NodeId, + rhs: &TokenTree, + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + match *rhs { + TokenTree::Token(..) => {} + TokenTree::MetaVarDecl(span, _name, _kind) => { + sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in rhs") + } + TokenTree::MetaVar(span, name) => { + check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name); + } + TokenTree::Delimited(_, ref del) => { + check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid); + } + TokenTree::Sequence(_, ref seq) => { + let ops = ops.push(seq.kleene); + check_nested_occurrences(sess, node_id, &seq.tts, macros, binders, &ops, valid); + } + } +} + +/// Represents the processed prefix of a nested macro. +#[derive(Clone, Copy, PartialEq, Eq)] +enum NestedMacroState { + /// Nothing that matches a nested macro definition was processed yet. + Empty, + /// The token `macro_rules` was processed. + MacroRules, + /// The tokens `macro_rules!` were processed. + MacroRulesNot, + /// The tokens `macro_rules!` followed by a name were processed. The name may be either directly + /// an identifier or a meta-variable (that hopefully would be instantiated by an identifier). + MacroRulesNotName, + /// The keyword `macro` was processed. + Macro, + /// The keyword `macro` followed by a name was processed. + MacroName, + /// The keyword `macro` followed by a name and a token delimited by parentheses was processed. + MacroNameParen, +} + +/// Checks `tts` as part of the RHS of a macro definition, tries to recognize nested macro +/// definitions, and sets `valid` to false in case of errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `tts` is checked as part of a RHS and may contain macro definitions +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `valid` is set in case of errors +fn check_nested_occurrences( + sess: &ParseSess, + node_id: NodeId, + tts: &[TokenTree], + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + let mut state = NestedMacroState::Empty; + let nested_macros = macros.push(MacroState { binders, ops: ops.into() }); + let mut nested_binders = Binders::default(); + for tt in tts { + match (state, tt) { + ( + NestedMacroState::Empty, + &TokenTree::Token(Token { kind: TokenKind::Ident(name, false), .. }), + ) => { + if name == sym::macro_rules { + state = NestedMacroState::MacroRules; + } else if name == kw::Macro { + state = NestedMacroState::Macro; + } + } + ( + NestedMacroState::MacroRules, + &TokenTree::Token(Token { kind: TokenKind::Not, .. }), + ) => { + state = NestedMacroState::MacroRulesNot; + } + ( + NestedMacroState::MacroRulesNot, + &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), + ) => { + state = NestedMacroState::MacroRulesNotName; + } + (NestedMacroState::MacroRulesNot, &TokenTree::MetaVar(..)) => { + state = NestedMacroState::MacroRulesNotName; + // We check that the meta-variable is correctly used. + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + (NestedMacroState::MacroRulesNotName, &TokenTree::Delimited(_, ref del)) + | (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Brace => + { + let legacy = state == NestedMacroState::MacroRulesNotName; + state = NestedMacroState::Empty; + let rest = + check_nested_macro(sess, node_id, legacy, &del.tts, &nested_macros, valid); + // If we did not check the whole macro definition, then check the rest as if outside + // the macro definition. + check_nested_occurrences( + sess, + node_id, + &del.tts[rest..], + macros, + binders, + ops, + valid, + ); + } + ( + NestedMacroState::Macro, + &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), + ) => { + state = NestedMacroState::MacroName; + } + (NestedMacroState::Macro, &TokenTree::MetaVar(..)) => { + state = NestedMacroState::MacroName; + // We check that the meta-variable is correctly used. + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Paren => + { + state = NestedMacroState::MacroNameParen; + nested_binders = Binders::default(); + check_binders( + sess, + node_id, + tt, + &nested_macros, + &mut nested_binders, + &Stack::Empty, + valid, + ); + } + (NestedMacroState::MacroNameParen, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Brace => + { + state = NestedMacroState::Empty; + check_occurrences( + sess, + node_id, + tt, + &nested_macros, + &nested_binders, + &Stack::Empty, + valid, + ); + } + (_, ref tt) => { + state = NestedMacroState::Empty; + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + } + } +} + +/// Checks the body of nested macro, returns where the check stopped, and sets `valid` to false in +/// case of errors. +/// +/// The token trees are checked as long as they look like a list of (LHS) => {RHS} token trees. This +/// check is a best-effort to detect a macro definition. It returns the position in `tts` where we +/// stopped checking because we detected we were not in a macro definition anymore. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `legacy` specifies whether the macro is legacy +/// - `tts` is checked as a list of (LHS) => {RHS} +/// - `macros` is the stack of outer macros +/// - `valid` is set in case of errors +fn check_nested_macro( + sess: &ParseSess, + node_id: NodeId, + legacy: bool, + tts: &[TokenTree], + macros: &Stack<'_, MacroState<'_>>, + valid: &mut bool, +) -> usize { + let n = tts.len(); + let mut i = 0; + let separator = if legacy { TokenKind::Semi } else { TokenKind::Comma }; + loop { + // We expect 3 token trees: `(LHS) => {RHS}`. The separator is checked after. + if i + 2 >= n + || !tts[i].is_delimited() + || !tts[i + 1].is_token(&TokenKind::FatArrow) + || !tts[i + 2].is_delimited() + { + break; + } + let lhs = &tts[i]; + let rhs = &tts[i + 2]; + let mut binders = Binders::default(); + check_binders(sess, node_id, lhs, macros, &mut binders, &Stack::Empty, valid); + check_occurrences(sess, node_id, rhs, macros, &binders, &Stack::Empty, valid); + // Since the last semicolon is optional for legacy macros and decl_macro are not terminated, + // we increment our checked position by how many token trees we already checked (the 3 + // above) before checking for the separator. + i += 3; + if i == n || !tts[i].is_token(&separator) { + break; + } + // We increment our checked position for the semicolon. + i += 1; + } + i +} + +/// Checks that a meta-variable occurrence is valid. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `span` is the span of the meta-variable to check +/// - `name` is the name of the meta-variable to check +fn check_ops_is_prefix( + sess: &ParseSess, + node_id: NodeId, + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + span: Span, + name: Ident, +) { + let macros = macros.push(MacroState { binders, ops: ops.into() }); + // Accumulates the stacks the operators of each state until (and including when) the + // meta-variable is found. The innermost stack is first. + let mut acc: SmallVec<[&SmallVec<[KleeneToken; 1]>; 1]> = SmallVec::new(); + for state in ¯os { + acc.push(&state.ops); + if let Some(binder) = state.binders.get(&name) { + // This variable concatenates the stack of operators from the RHS of the LHS where the + // meta-variable was defined to where it is used (in possibly nested macros). The + // outermost operator is first. + let mut occurrence_ops: SmallVec<[KleeneToken; 2]> = SmallVec::new(); + // We need to iterate from the end to start with outermost stack. + for ops in acc.iter().rev() { + occurrence_ops.extend_from_slice(ops); + } + ops_is_prefix(sess, node_id, span, name, &binder.ops, &occurrence_ops); + return; + } + } + buffer_lint(sess, span.into(), node_id, &format!("unknown macro variable `{}`", name)); +} + +/// Returns whether `binder_ops` is a prefix of `occurrence_ops`. +/// +/// The stack of Kleene operators of a meta-variable occurrence just needs to have the stack of +/// Kleene operators of its binder as a prefix. +/// +/// Consider $i in the following example: +/// +/// ( $( $i:ident = $($j:ident),+ );* ) => { $($( $i += $j; )+)* } +/// +/// It occurs under the Kleene stack ["*", "+"] and is bound under ["*"] only. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `span` is the span of the meta-variable being check +/// - `name` is the name of the meta-variable being check +/// - `binder_ops` is the stack of Kleene operators for the binder +/// - `occurrence_ops` is the stack of Kleene operators for the occurrence +fn ops_is_prefix( + sess: &ParseSess, + node_id: NodeId, + span: Span, + name: Ident, + binder_ops: &[KleeneToken], + occurrence_ops: &[KleeneToken], +) { + for (i, binder) in binder_ops.iter().enumerate() { + if i >= occurrence_ops.len() { + let mut span = MultiSpan::from_span(span); + span.push_span_label(binder.span, "expected repetition".into()); + let message = &format!("variable '{}' is still repeating at this depth", name); + buffer_lint(sess, span, node_id, message); + return; + } + let occurrence = &occurrence_ops[i]; + if occurrence.op != binder.op { + let mut span = MultiSpan::from_span(span); + span.push_span_label(binder.span, "expected repetition".into()); + span.push_span_label(occurrence.span, "conflicting repetition".into()); + let message = "meta-variable repeats with different Kleene operator"; + buffer_lint(sess, span, node_id, message); + return; + } + } +} + +fn buffer_lint(sess: &ParseSess, span: MultiSpan, node_id: NodeId, message: &str) { + sess.buffer_lint(&META_VARIABLE_MISUSE, span, node_id, message); +} diff --git a/src/librustc_expand/mbe/macro_parser.rs b/src/librustc_expand/mbe/macro_parser.rs new file mode 100644 index 00000000000..24253e1bdc2 --- /dev/null +++ b/src/librustc_expand/mbe/macro_parser.rs @@ -0,0 +1,930 @@ +//! This is an NFA-based parser, which calls out to the main rust parser for named non-terminals +//! (which it commits to fully when it hits one in a grammar). There's a set of current NFA threads +//! and a set of next ones. Instead of NTs, we have a special case for Kleene star. The big-O, in +//! pathological cases, is worse than traditional use of NFA or Earley parsing, but it's an easier +//! fit for Macro-by-Example-style rules. +//! +//! (In order to prevent the pathological case, we'd need to lazily construct the resulting +//! `NamedMatch`es at the very end. It'd be a pain, and require more memory to keep around old +//! items, but it would also save overhead) +//! +//! We don't say this parser uses the Earley algorithm, because it's unnecessarily inaccurate. +//! The macro parser restricts itself to the features of finite state automata. Earley parsers +//! can be described as an extension of NFAs with completion rules, prediction rules, and recursion. +//! +//! Quick intro to how the parser works: +//! +//! A 'position' is a dot in the middle of a matcher, usually represented as a +//! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`. +//! +//! The parser walks through the input a character at a time, maintaining a list +//! of threads consistent with the current position in the input string: `cur_items`. +//! +//! As it processes them, it fills up `eof_items` with threads that would be valid if +//! the macro invocation is now over, `bb_items` with threads that are waiting on +//! a Rust non-terminal like `$e:expr`, and `next_items` with threads that are waiting +//! on a particular token. Most of the logic concerns moving the · through the +//! repetitions indicated by Kleene stars. The rules for moving the · without +//! consuming any input are called epsilon transitions. It only advances or calls +//! out to the real Rust parser when no `cur_items` threads remain. +//! +//! Example: +//! +//! ```text, ignore +//! Start parsing a a a a b against [· a $( a )* a b]. +//! +//! Remaining input: a a a a b +//! next: [· a $( a )* a b] +//! +//! - - - Advance over an a. - - - +//! +//! Remaining input: a a a b +//! cur: [a · $( a )* a b] +//! Descend/Skip (first item). +//! next: [a $( · a )* a b] [a $( a )* · a b]. +//! +//! - - - Advance over an a. - - - +//! +//! Remaining input: a a b +//! cur: [a $( a · )* a b] [a $( a )* a · b] +//! Follow epsilon transition: Finish/Repeat (first item) +//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] +//! +//! - - - Advance over an a. - - - (this looks exactly like the last step) +//! +//! Remaining input: a b +//! cur: [a $( a · )* a b] [a $( a )* a · b] +//! Follow epsilon transition: Finish/Repeat (first item) +//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] +//! +//! - - - Advance over an a. - - - (this looks exactly like the last step) +//! +//! Remaining input: b +//! cur: [a $( a · )* a b] [a $( a )* a · b] +//! Follow epsilon transition: Finish/Repeat (first item) +//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] +//! +//! - - - Advance over a b. - - - +//! +//! Remaining input: '' +//! eof: [a $( a )* a b ·] +//! ``` + +crate use NamedMatch::*; +crate use ParseResult::*; +use TokenTreeOrTokenTreeSlice::*; + +use crate::mbe::{self, TokenTree}; + +use rustc_parse::parser::{FollowedByType, Parser, PathStyle}; +use rustc_parse::Directory; +use syntax::ast::{Ident, Name}; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::token::{self, DocComment, Nonterminal, Token}; +use syntax::tokenstream::TokenStream; + +use errors::{FatalError, PResult}; +use smallvec::{smallvec, SmallVec}; +use syntax_pos::Span; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use std::collections::hash_map::Entry::{Occupied, Vacant}; +use std::mem; +use std::ops::{Deref, DerefMut}; + +// To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body. + +/// Either a sequence of token trees or a single one. This is used as the representation of the +/// sequence of tokens that make up a matcher. +#[derive(Clone)] +enum TokenTreeOrTokenTreeSlice<'tt> { + Tt(TokenTree), + TtSeq(&'tt [TokenTree]), +} + +impl<'tt> TokenTreeOrTokenTreeSlice<'tt> { + /// Returns the number of constituent top-level token trees of `self` (top-level in that it + /// will not recursively descend into subtrees). + fn len(&self) -> usize { + match *self { + TtSeq(ref v) => v.len(), + Tt(ref tt) => tt.len(), + } + } + + /// The `index`-th token tree of `self`. + fn get_tt(&self, index: usize) -> TokenTree { + match *self { + TtSeq(ref v) => v[index].clone(), + Tt(ref tt) => tt.get_tt(index), + } + } +} + +/// An unzipping of `TokenTree`s... see the `stack` field of `MatcherPos`. +/// +/// This is used by `inner_parse_loop` to keep track of delimited submatchers that we have +/// descended into. +#[derive(Clone)] +struct MatcherTtFrame<'tt> { + /// The "parent" matcher that we are descending into. + elts: TokenTreeOrTokenTreeSlice<'tt>, + /// The position of the "dot" in `elts` at the time we descended. + idx: usize, +} + +type NamedMatchVec = SmallVec<[NamedMatch; 4]>; + +/// Represents a single "position" (aka "matcher position", aka "item"), as +/// described in the module documentation. +/// +/// Here: +/// +/// - `'root` represents the lifetime of the stack slot that holds the root +/// `MatcherPos`. As described in `MatcherPosHandle`, the root `MatcherPos` +/// structure is stored on the stack, but subsequent instances are put into +/// the heap. +/// - `'tt` represents the lifetime of the token trees that this matcher +/// position refers to. +/// +/// It is important to distinguish these two lifetimes because we have a +/// `SmallVec>` below, and the destructor of +/// that is considered to possibly access the data from its elements (it lacks +/// a `#[may_dangle]` attribute). As a result, the compiler needs to know that +/// all the elements in that `SmallVec` strictly outlive the root stack slot +/// lifetime. By separating `'tt` from `'root`, we can show that. +#[derive(Clone)] +struct MatcherPos<'root, 'tt> { + /// The token or sequence of tokens that make up the matcher + top_elts: TokenTreeOrTokenTreeSlice<'tt>, + + /// The position of the "dot" in this matcher + idx: usize, + + /// For each named metavar in the matcher, we keep track of token trees matched against the + /// metavar by the black box parser. In particular, there may be more than one match per + /// metavar if we are in a repetition (each repetition matches each of the variables). + /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the + /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of + /// the current position of the `self` matcher position in the shared `matches` list. + /// + /// Also, note that while we are descending into a sequence, matchers are given their own + /// `matches` vector. Only once we reach the end of a full repetition of the sequence do we add + /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep` + /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one + /// wants the shared `matches`, one should use `up.matches`. + matches: Box<[Lrc]>, + /// The position in `matches` corresponding to the first metavar in this matcher's sequence of + /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds + /// to `matches[match_lo]`. + match_lo: usize, + /// The position in `matches` corresponding to the metavar we are currently trying to match + /// against the source token stream. `match_lo <= match_cur <= match_hi`. + match_cur: usize, + /// Similar to `match_lo` except `match_hi` is the position in `matches` of the _last_ metavar + /// in this matcher. + match_hi: usize, + + // The following fields are used if we are matching a repetition. If we aren't, they should be + // `None`. + /// The KleeneOp of this sequence if we are in a repetition. + seq_op: Option, + + /// The separator if we are in a repetition. + sep: Option, + + /// The "parent" matcher position if we are in a repetition. That is, the matcher position just + /// before we enter the sequence. + up: Option>, + + /// Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from + /// a delimited token tree (e.g., something wrapped in `(` `)`) or to get the contents of a doc + /// comment... + /// + /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. ) + /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does + /// that where the bottom of the stack is the outermost matcher. + /// Also, throughout the comments, this "descent" is often referred to as "unzipping"... + stack: SmallVec<[MatcherTtFrame<'tt>; 1]>, +} + +impl<'root, 'tt> MatcherPos<'root, 'tt> { + /// Adds `m` as a named match for the `idx`-th metavar. + fn push_match(&mut self, idx: usize, m: NamedMatch) { + let matches = Lrc::make_mut(&mut self.matches[idx]); + matches.push(m); + } +} + +// Lots of MatcherPos instances are created at runtime. Allocating them on the +// heap is slow. Furthermore, using SmallVec to allocate them all +// on the stack is also slow, because MatcherPos is quite a large type and +// instances get moved around a lot between vectors, which requires lots of +// slow memcpy calls. +// +// Therefore, the initial MatcherPos is always allocated on the stack, +// subsequent ones (of which there aren't that many) are allocated on the heap, +// and this type is used to encapsulate both cases. +enum MatcherPosHandle<'root, 'tt> { + Ref(&'root mut MatcherPos<'root, 'tt>), + Box(Box>), +} + +impl<'root, 'tt> Clone for MatcherPosHandle<'root, 'tt> { + // This always produces a new Box. + fn clone(&self) -> Self { + MatcherPosHandle::Box(match *self { + MatcherPosHandle::Ref(ref r) => Box::new((**r).clone()), + MatcherPosHandle::Box(ref b) => b.clone(), + }) + } +} + +impl<'root, 'tt> Deref for MatcherPosHandle<'root, 'tt> { + type Target = MatcherPos<'root, 'tt>; + fn deref(&self) -> &Self::Target { + match *self { + MatcherPosHandle::Ref(ref r) => r, + MatcherPosHandle::Box(ref b) => b, + } + } +} + +impl<'root, 'tt> DerefMut for MatcherPosHandle<'root, 'tt> { + fn deref_mut(&mut self) -> &mut MatcherPos<'root, 'tt> { + match *self { + MatcherPosHandle::Ref(ref mut r) => r, + MatcherPosHandle::Box(ref mut b) => b, + } + } +} + +/// Represents the possible results of an attempted parse. +crate enum ParseResult { + /// Parsed successfully. + Success(T), + /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected + /// end of macro invocation. Otherwise, it indicates that no rules expected the given token. + Failure(Token, &'static str), + /// Fatal error (malformed macro?). Abort compilation. + Error(syntax_pos::Span, String), +} + +/// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es. +/// This represents the mapping of metavars to the token trees they bind to. +crate type NamedParseResult = ParseResult>; + +/// Count how many metavars are named in the given matcher `ms`. +pub(super) fn count_names(ms: &[TokenTree]) -> usize { + ms.iter().fold(0, |count, elt| { + count + + match *elt { + TokenTree::Sequence(_, ref seq) => seq.num_captures, + TokenTree::Delimited(_, ref delim) => count_names(&delim.tts), + TokenTree::MetaVar(..) => 0, + TokenTree::MetaVarDecl(..) => 1, + TokenTree::Token(..) => 0, + } + }) +} + +/// `len` `Vec`s (initially shared and empty) that will store matches of metavars. +fn create_matches(len: usize) -> Box<[Lrc]> { + if len == 0 { + vec![] + } else { + let empty_matches = Lrc::new(SmallVec::new()); + vec![empty_matches; len] + } + .into_boxed_slice() +} + +/// Generates the top-level matcher position in which the "dot" is before the first token of the +/// matcher `ms`. +fn initial_matcher_pos<'root, 'tt>(ms: &'tt [TokenTree]) -> MatcherPos<'root, 'tt> { + let match_idx_hi = count_names(ms); + let matches = create_matches(match_idx_hi); + MatcherPos { + // Start with the top level matcher given to us + top_elts: TtSeq(ms), // "elts" is an abbr. for "elements" + // The "dot" is before the first token of the matcher + idx: 0, + + // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`. + // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since + // we haven't actually matched anything yet. + matches, + match_lo: 0, + match_cur: 0, + match_hi: match_idx_hi, + + // Haven't descended into any delimiters, so empty stack + stack: smallvec![], + + // Haven't descended into any sequences, so both of these are `None`. + seq_op: None, + sep: None, + up: None, + } +} + +/// `NamedMatch` is a pattern-match result for a single `token::MATCH_NONTERMINAL`: +/// so it is associated with a single ident in a parse, and all +/// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type +/// (expr, item, etc). Each leaf in a single `NamedMatch` corresponds to a +/// single `token::MATCH_NONTERMINAL` in the `TokenTree` that produced it. +/// +/// The in-memory structure of a particular `NamedMatch` represents the match +/// that occurred when a particular subset of a matcher was applied to a +/// particular token tree. +/// +/// The width of each `MatchedSeq` in the `NamedMatch`, and the identity of +/// the `MatchedNonterminal`s, will depend on the token tree it was applied +/// to: each `MatchedSeq` corresponds to a single `TTSeq` in the originating +/// token tree. The depth of the `NamedMatch` structure will therefore depend +/// only on the nesting depth of `ast::TTSeq`s in the originating +/// token tree it was derived from. +#[derive(Debug, Clone)] +crate enum NamedMatch { + MatchedSeq(Lrc), + MatchedNonterminal(Lrc), +} + +/// Takes a sequence of token trees `ms` representing a matcher which successfully matched input +/// and an iterator of items that matched input and produces a `NamedParseResult`. +fn nameize>( + sess: &ParseSess, + ms: &[TokenTree], + mut res: I, +) -> NamedParseResult { + // Recursively descend into each type of matcher (e.g., sequences, delimited, metavars) and make + // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one + // binding, then there is an error. If it does, then we insert the binding into the + // `NamedParseResult`. + fn n_rec>( + sess: &ParseSess, + m: &TokenTree, + res: &mut I, + ret_val: &mut FxHashMap, + ) -> Result<(), (syntax_pos::Span, String)> { + match *m { + TokenTree::Sequence(_, ref seq) => { + for next_m in &seq.tts { + n_rec(sess, next_m, res.by_ref(), ret_val)? + } + } + TokenTree::Delimited(_, ref delim) => { + for next_m in &delim.tts { + n_rec(sess, next_m, res.by_ref(), ret_val)?; + } + } + TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { + if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { + return Err((span, "missing fragment specifier".to_string())); + } + } + TokenTree::MetaVarDecl(sp, bind_name, _) => match ret_val.entry(bind_name) { + Vacant(spot) => { + spot.insert(res.next().unwrap()); + } + Occupied(..) => return Err((sp, format!("duplicated bind name: {}", bind_name))), + }, + TokenTree::MetaVar(..) | TokenTree::Token(..) => (), + } + + Ok(()) + } + + let mut ret_val = FxHashMap::default(); + for m in ms { + match n_rec(sess, m, res.by_ref(), &mut ret_val) { + Ok(_) => {} + Err((sp, msg)) => return Error(sp, msg), + } + } + + Success(ret_val) +} + +/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) +fn token_name_eq(t1: &Token, t2: &Token) -> bool { + if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) { + ident1.name == ident2.name && is_raw1 == is_raw2 + } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) { + ident1.name == ident2.name + } else { + t1.kind == t2.kind + } +} + +/// Process the matcher positions of `cur_items` until it is empty. In the process, this will +/// produce more items in `next_items`, `eof_items`, and `bb_items`. +/// +/// For more info about the how this happens, see the module-level doc comments and the inline +/// comments of this function. +/// +/// # Parameters +/// +/// - `sess`: the parsing session into which errors are emitted. +/// - `cur_items`: the set of current items to be processed. This should be empty by the end of a +/// successful execution of this function. +/// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in +/// the function `parse`. +/// - `eof_items`: the set of items that would be valid if this was the EOF. +/// - `bb_items`: the set of items that are waiting for the black-box parser. +/// - `token`: the current token of the parser. +/// - `span`: the `Span` in the source code corresponding to the token trees we are trying to match +/// against the matcher positions in `cur_items`. +/// +/// # Returns +/// +/// A `ParseResult`. Note that matches are kept track of through the items generated. +fn inner_parse_loop<'root, 'tt>( + sess: &ParseSess, + cur_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, + next_items: &mut Vec>, + eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, + bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, + token: &Token, +) -> ParseResult<()> { + // Pop items from `cur_items` until it is empty. + while let Some(mut item) = cur_items.pop() { + // When unzipped trees end, remove them. This corresponds to backtracking out of a + // delimited submatcher into which we already descended. In backtracking out again, we need + // to advance the "dot" past the delimiters in the outer matcher. + while item.idx >= item.top_elts.len() { + match item.stack.pop() { + Some(MatcherTtFrame { elts, idx }) => { + item.top_elts = elts; + item.idx = idx + 1; + } + None => break, + } + } + + // Get the current position of the "dot" (`idx`) in `item` and the number of token trees in + // the matcher (`len`). + let idx = item.idx; + let len = item.top_elts.len(); + + // If `idx >= len`, then we are at or past the end of the matcher of `item`. + if idx >= len { + // We are repeating iff there is a parent. If the matcher is inside of a repetition, + // then we could be at the end of a sequence or at the beginning of the next + // repetition. + if item.up.is_some() { + // At this point, regardless of whether there is a separator, we should add all + // matches from the complete repetition of the sequence to the shared, top-level + // `matches` list (actually, `up.matches`, which could itself not be the top-level, + // but anyway...). Moreover, we add another item to `cur_items` in which the "dot" + // is at the end of the `up` matcher. This ensures that the "dot" in the `up` + // matcher is also advanced sufficiently. + // + // NOTE: removing the condition `idx == len` allows trailing separators. + if idx == len { + // Get the `up` matcher + let mut new_pos = item.up.clone().unwrap(); + + // Add matches from this repetition to the `matches` of `up` + for idx in item.match_lo..item.match_hi { + let sub = item.matches[idx].clone(); + new_pos.push_match(idx, MatchedSeq(sub)); + } + + // Move the "dot" past the repetition in `up` + new_pos.match_cur = item.match_hi; + new_pos.idx += 1; + cur_items.push(new_pos); + } + + // Check if we need a separator. + if idx == len && item.sep.is_some() { + // We have a separator, and it is the current token. We can advance past the + // separator token. + if item.sep.as_ref().map(|sep| token_name_eq(token, sep)).unwrap_or(false) { + item.idx += 1; + next_items.push(item); + } + } + // We don't need a separator. Move the "dot" back to the beginning of the matcher + // and try to match again UNLESS we are only allowed to have _one_ repetition. + else if item.seq_op != Some(mbe::KleeneOp::ZeroOrOne) { + item.match_cur = item.match_lo; + item.idx = 0; + cur_items.push(item); + } + } + // If we are not in a repetition, then being at the end of a matcher means that we have + // reached the potential end of the input. + else { + eof_items.push(item); + } + } + // We are in the middle of a matcher. + else { + // Look at what token in the matcher we are trying to match the current token (`token`) + // against. Depending on that, we may generate new items. + match item.top_elts.get_tt(idx) { + // Need to descend into a sequence + TokenTree::Sequence(sp, seq) => { + // Examine the case where there are 0 matches of this sequence. We are + // implicitly disallowing OneOrMore from having 0 matches here. Thus, that will + // result in a "no rules expected token" error by virtue of this matcher not + // working. + if seq.kleene.op == mbe::KleeneOp::ZeroOrMore + || seq.kleene.op == mbe::KleeneOp::ZeroOrOne + { + let mut new_item = item.clone(); + new_item.match_cur += seq.num_captures; + new_item.idx += 1; + for idx in item.match_cur..item.match_cur + seq.num_captures { + new_item.push_match(idx, MatchedSeq(Lrc::new(smallvec![]))); + } + cur_items.push(new_item); + } + + let matches = create_matches(item.matches.len()); + cur_items.push(MatcherPosHandle::Box(Box::new(MatcherPos { + stack: smallvec![], + sep: seq.separator.clone(), + seq_op: Some(seq.kleene.op), + idx: 0, + matches, + match_lo: item.match_cur, + match_cur: item.match_cur, + match_hi: item.match_cur + seq.num_captures, + up: Some(item), + top_elts: Tt(TokenTree::Sequence(sp, seq)), + }))); + } + + // We need to match a metavar (but the identifier is invalid)... this is an error + TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { + if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { + return Error(span, "missing fragment specifier".to_string()); + } + } + + // We need to match a metavar with a valid ident... call out to the black-box + // parser by adding an item to `bb_items`. + TokenTree::MetaVarDecl(_, _, id) => { + // Built-in nonterminals never start with these tokens, + // so we can eliminate them from consideration. + if may_begin_with(token, id.name) { + bb_items.push(item); + } + } + + // We need to descend into a delimited submatcher or a doc comment. To do this, we + // push the current matcher onto a stack and push a new item containing the + // submatcher onto `cur_items`. + // + // At the beginning of the loop, if we reach the end of the delimited submatcher, + // we pop the stack to backtrack out of the descent. + seq @ TokenTree::Delimited(..) + | seq @ TokenTree::Token(Token { kind: DocComment(..), .. }) => { + let lower_elts = mem::replace(&mut item.top_elts, Tt(seq)); + let idx = item.idx; + item.stack.push(MatcherTtFrame { elts: lower_elts, idx }); + item.idx = 0; + cur_items.push(item); + } + + // We just matched a normal token. We can just advance the parser. + TokenTree::Token(t) if token_name_eq(&t, token) => { + item.idx += 1; + next_items.push(item); + } + + // There was another token that was not `token`... This means we can't add any + // rules. NOTE that this is not necessarily an error unless _all_ items in + // `cur_items` end up doing this. There may still be some other matchers that do + // end up working out. + TokenTree::Token(..) | TokenTree::MetaVar(..) => {} + } + } + } + + // Yay a successful parse (so far)! + Success(()) +} + +/// Use the given sequence of token trees (`ms`) as a matcher. Match the given token stream `tts` +/// against it and return the match. +/// +/// # Parameters +/// +/// - `sess`: The session into which errors are emitted +/// - `tts`: The tokenstream we are matching against the pattern `ms` +/// - `ms`: A sequence of token trees representing a pattern against which we are matching +/// - `directory`: Information about the file locations (needed for the black-box parser) +/// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box +/// parser) +pub(super) fn parse( + sess: &ParseSess, + tts: TokenStream, + ms: &[TokenTree], + directory: Option>, + recurse_into_modules: bool, +) -> NamedParseResult { + // Create a parser that can be used for the "black box" parts. + let mut parser = + Parser::new(sess, tts, directory, recurse_into_modules, true, rustc_parse::MACRO_ARGUMENTS); + + // A queue of possible matcher positions. We initialize it with the matcher position in which + // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then + // processes all of these possible matcher positions and produces possible next positions into + // `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items` + // and we start over again. + // + // This MatcherPos instance is allocated on the stack. All others -- and + // there are frequently *no* others! -- are allocated on the heap. + let mut initial = initial_matcher_pos(ms); + let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)]; + let mut next_items = Vec::new(); + + loop { + // Matcher positions black-box parsed by parser.rs (`parser`) + let mut bb_items = SmallVec::new(); + + // Matcher positions that would be valid if the macro invocation was over now + let mut eof_items = SmallVec::new(); + assert!(next_items.is_empty()); + + // Process `cur_items` until either we have finished the input or we need to get some + // parsing from the black-box parser done. The result is that `next_items` will contain a + // bunch of possible next matcher positions in `next_items`. + match inner_parse_loop( + sess, + &mut cur_items, + &mut next_items, + &mut eof_items, + &mut bb_items, + &parser.token, + ) { + Success(_) => {} + Failure(token, msg) => return Failure(token, msg), + Error(sp, msg) => return Error(sp, msg), + } + + // inner parse loop handled all cur_items, so it's empty + assert!(cur_items.is_empty()); + + // We need to do some post processing after the `inner_parser_loop`. + // + // Error messages here could be improved with links to original rules. + + // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise, + // either the parse is ambiguous (which should never happen) or there is a syntax error. + if parser.token == token::Eof { + if eof_items.len() == 1 { + let matches = + eof_items[0].matches.iter_mut().map(|dv| Lrc::make_mut(dv).pop().unwrap()); + return nameize(sess, ms, matches); + } else if eof_items.len() > 1 { + return Error( + parser.token.span, + "ambiguity: multiple successful parses".to_string(), + ); + } else { + return Failure( + Token::new( + token::Eof, + if parser.token.span.is_dummy() { + parser.token.span + } else { + sess.source_map().next_point(parser.token.span) + }, + ), + "missing tokens in macro arguments", + ); + } + } + // Performance hack: eof_items may share matchers via Rc with other things that we want + // to modify. Dropping eof_items now may drop these refcounts to 1, preventing an + // unnecessary implicit clone later in Rc::make_mut. + drop(eof_items); + + // Another possibility is that we need to call out to parse some rust nonterminal + // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong. + if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 { + let nts = bb_items + .iter() + .map(|item| match item.top_elts.get_tt(item.idx) { + TokenTree::MetaVarDecl(_, bind, name) => format!("{} ('{}')", name, bind), + _ => panic!(), + }) + .collect::>() + .join(" or "); + + return Error( + parser.token.span, + format!( + "local ambiguity: multiple parsing options: {}", + match next_items.len() { + 0 => format!("built-in NTs {}.", nts), + 1 => format!("built-in NTs {} or 1 other option.", nts), + n => format!("built-in NTs {} or {} other options.", nts, n), + } + ), + ); + } + // If there are no possible next positions AND we aren't waiting for the black-box parser, + // then there is a syntax error. + else if bb_items.is_empty() && next_items.is_empty() { + return Failure(parser.token.take(), "no rules expected this token in macro call"); + } + // Dump all possible `next_items` into `cur_items` for the next iteration. + else if !next_items.is_empty() { + // Now process the next token + cur_items.extend(next_items.drain(..)); + parser.bump(); + } + // Finally, we have the case where we need to call the black-box parser to get some + // nonterminal. + else { + assert_eq!(bb_items.len(), 1); + + let mut item = bb_items.pop().unwrap(); + if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) { + let match_cur = item.match_cur; + item.push_match( + match_cur, + MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, ident.name))), + ); + item.idx += 1; + item.match_cur += 1; + } else { + unreachable!() + } + cur_items.push(item); + } + + assert!(!cur_items.is_empty()); + } +} + +/// The token is an identifier, but not `_`. +/// We prohibit passing `_` to macros expecting `ident` for now. +fn get_macro_name(token: &Token) -> Option<(Name, bool)> { + match token.kind { + token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)), + _ => None, + } +} + +/// Checks whether a non-terminal may begin with a particular token. +/// +/// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that +/// token. Be conservative (return true) if not sure. +fn may_begin_with(token: &Token, name: Name) -> bool { + /// Checks whether the non-terminal may contain a single (non-keyword) identifier. + fn may_be_ident(nt: &token::Nonterminal) -> bool { + match *nt { + token::NtItem(_) | token::NtBlock(_) | token::NtVis(_) => false, + _ => true, + } + } + + match name { + sym::expr => { + token.can_begin_expr() + // This exception is here for backwards compatibility. + && !token.is_keyword(kw::Let) + } + sym::ty => token.can_begin_type(), + sym::ident => get_macro_name(token).is_some(), + sym::literal => token.can_begin_literal_or_bool(), + sym::vis => match token.kind { + // The follow-set of :vis + "priv" keyword + interpolated + token::Comma | token::Ident(..) | token::Interpolated(_) => true, + _ => token.can_begin_type(), + }, + sym::block => match token.kind { + token::OpenDelim(token::Brace) => true, + token::Interpolated(ref nt) => match **nt { + token::NtItem(_) + | token::NtPat(_) + | token::NtTy(_) + | token::NtIdent(..) + | token::NtMeta(_) + | token::NtPath(_) + | token::NtVis(_) => false, // none of these may start with '{'. + _ => true, + }, + _ => false, + }, + sym::path | sym::meta => match token.kind { + token::ModSep | token::Ident(..) => true, + token::Interpolated(ref nt) => match **nt { + token::NtPath(_) | token::NtMeta(_) => true, + _ => may_be_ident(&nt), + }, + _ => false, + }, + sym::pat => match token.kind { + token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) + token::OpenDelim(token::Paren) | // tuple pattern + token::OpenDelim(token::Bracket) | // slice pattern + token::BinOp(token::And) | // reference + token::BinOp(token::Minus) | // negative literal + token::AndAnd | // double reference + token::Literal(..) | // literal + token::DotDot | // range pattern (future compat) + token::DotDotDot | // range pattern (future compat) + token::ModSep | // path + token::Lt | // path (UFCS constant) + token::BinOp(token::Shl) => true, // path (double UFCS) + token::Interpolated(ref nt) => may_be_ident(nt), + _ => false, + }, + sym::lifetime => match token.kind { + token::Lifetime(_) => true, + token::Interpolated(ref nt) => match **nt { + token::NtLifetime(_) | token::NtTT(_) => true, + _ => false, + }, + _ => false, + }, + _ => match token.kind { + token::CloseDelim(_) => false, + _ => true, + }, + } +} + +/// A call to the "black-box" parser to parse some Rust non-terminal. +/// +/// # Parameters +/// +/// - `p`: the "black-box" parser to use +/// - `sp`: the `Span` we want to parse +/// - `name`: the name of the metavar _matcher_ we want to match (e.g., `tt`, `ident`, `block`, +/// etc...) +/// +/// # Returns +/// +/// The parsed non-terminal. +fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal { + // FIXME(Centril): Consider moving this to `parser.rs` to make + // the visibilities of the methods used below `pub(super)` at most. + + if name == sym::tt { + return token::NtTT(p.parse_token_tree()); + } + // check at the beginning and the parser checks after each bump + p.process_potential_macro_variable(); + match parse_nt_inner(p, sp, name) { + Ok(nt) => nt, + Err(mut err) => { + err.emit(); + FatalError.raise(); + } + } +} + +fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a, Nonterminal> { + Ok(match name { + sym::item => match p.parse_item()? { + Some(i) => token::NtItem(i), + None => return Err(p.fatal("expected an item keyword")), + }, + sym::block => token::NtBlock(p.parse_block()?), + sym::stmt => match p.parse_stmt()? { + Some(s) => token::NtStmt(s), + None => return Err(p.fatal("expected a statement")), + }, + sym::pat => token::NtPat(p.parse_pat(None)?), + sym::expr => token::NtExpr(p.parse_expr()?), + sym::literal => token::NtLiteral(p.parse_literal_maybe_minus()?), + sym::ty => token::NtTy(p.parse_ty()?), + // this could be handled like a token, since it is one + sym::ident => { + if let Some((name, is_raw)) = get_macro_name(&p.token) { + let span = p.token.span; + p.bump(); + token::NtIdent(Ident::new(name, span), is_raw) + } else { + let token_str = pprust::token_to_string(&p.token); + return Err(p.fatal(&format!("expected ident, found {}", &token_str))); + } + } + sym::path => token::NtPath(p.parse_path(PathStyle::Type)?), + sym::meta => token::NtMeta(p.parse_attr_item()?), + sym::vis => token::NtVis(p.parse_visibility(FollowedByType::Yes)?), + sym::lifetime => { + if p.check_lifetime() { + token::NtLifetime(p.expect_lifetime().ident) + } else { + let token_str = pprust::token_to_string(&p.token); + return Err(p.fatal(&format!("expected a lifetime, found `{}`", &token_str))); + } + } + // this is not supposed to happen, since it has been checked + // when compiling the macro. + _ => p.span_bug(sp, "invalid fragment specifier"), + }) +} diff --git a/src/librustc_expand/mbe/macro_rules.rs b/src/librustc_expand/mbe/macro_rules.rs new file mode 100644 index 00000000000..2b2ed8c9248 --- /dev/null +++ b/src/librustc_expand/mbe/macro_rules.rs @@ -0,0 +1,1207 @@ +use crate::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander}; +use crate::base::{SyntaxExtension, SyntaxExtensionKind}; +use crate::expand::{ensure_complete_parse, parse_ast_fragment, AstFragment, AstFragmentKind}; +use crate::mbe; +use crate::mbe::macro_check; +use crate::mbe::macro_parser::parse; +use crate::mbe::macro_parser::{Error, Failure, Success}; +use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedParseResult}; +use crate::mbe::transcribe::transcribe; + +use rustc_feature::Features; +use rustc_parse::parser::Parser; +use rustc_parse::Directory; +use syntax::ast; +use syntax::attr::{self, TransparencyError}; +use syntax::edition::Edition; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::token::{self, NtTT, Token, TokenKind::*}; +use syntax::tokenstream::{DelimSpan, TokenStream}; +use syntax_pos::hygiene::Transparency; +use syntax_pos::Span; + +use errors::{DiagnosticBuilder, FatalError}; +use log::debug; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use std::borrow::Cow; +use std::collections::hash_map::Entry; +use std::{mem, slice}; + +use errors::Applicability; + +const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \ + `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, \ + `literal`, `path`, `meta`, `tt`, `item` and `vis`"; + +crate struct ParserAnyMacro<'a> { + parser: Parser<'a>, + + /// Span of the expansion site of the macro this parser is for + site_span: Span, + /// The ident of the macro we're parsing + macro_ident: ast::Ident, + arm_span: Span, +} + +crate fn annotate_err_with_kind( + err: &mut DiagnosticBuilder<'_>, + kind: AstFragmentKind, + span: Span, +) { + match kind { + AstFragmentKind::Ty => { + err.span_label(span, "this macro call doesn't expand to a type"); + } + AstFragmentKind::Pat => { + err.span_label(span, "this macro call doesn't expand to a pattern"); + } + _ => {} + }; +} + +/// Instead of e.g. `vec![a, b, c]` in a pattern context, suggest `[a, b, c]`. +fn suggest_slice_pat(e: &mut DiagnosticBuilder<'_>, site_span: Span, parser: &Parser<'_>) { + let mut suggestion = None; + if let Ok(code) = parser.sess.source_map().span_to_snippet(site_span) { + if let Some(bang) = code.find('!') { + suggestion = Some(code[bang + 1..].to_string()); + } + } + if let Some(suggestion) = suggestion { + e.span_suggestion( + site_span, + "use a slice pattern here instead", + suggestion, + Applicability::MachineApplicable, + ); + } else { + e.span_label(site_span, "use a slice pattern here instead"); + } + e.help( + "for more information, see https://doc.rust-lang.org/edition-guide/\ + rust-2018/slice-patterns.html", + ); +} + +impl<'a> ParserAnyMacro<'a> { + crate fn make(mut self: Box>, kind: AstFragmentKind) -> AstFragment { + let ParserAnyMacro { site_span, macro_ident, ref mut parser, arm_span } = *self; + let fragment = panictry!(parse_ast_fragment(parser, kind, true).map_err(|mut e| { + if parser.token == token::Eof && e.message().ends_with(", found ``") { + if !e.span.is_dummy() { + // early end of macro arm (#52866) + e.replace_span_with(parser.sess.source_map().next_point(parser.token.span)); + } + let msg = &e.message[0]; + e.message[0] = ( + format!( + "macro expansion ends with an incomplete expression: {}", + msg.0.replace(", found ``", ""), + ), + msg.1, + ); + } + if e.span.is_dummy() { + // Get around lack of span in error (#30128) + e.replace_span_with(site_span); + if parser.sess.source_map().span_to_filename(arm_span).is_real() { + e.span_label(arm_span, "in this macro arm"); + } + } else if !parser.sess.source_map().span_to_filename(parser.token.span).is_real() { + e.span_label(site_span, "in this macro invocation"); + } + match kind { + AstFragmentKind::Pat if macro_ident.name == sym::vec => { + suggest_slice_pat(&mut e, site_span, parser); + } + _ => annotate_err_with_kind(&mut e, kind, site_span), + }; + e + })); + + // We allow semicolons at the end of expressions -- e.g., the semicolon in + // `macro_rules! m { () => { panic!(); } }` isn't parsed by `.parse_expr()`, + // but `m!()` is allowed in expression positions (cf. issue #34706). + if kind == AstFragmentKind::Expr && parser.token == token::Semi { + parser.bump(); + } + + // Make sure we don't have any tokens left to parse so we don't silently drop anything. + let path = ast::Path::from_ident(macro_ident.with_span_pos(site_span)); + ensure_complete_parse(parser, &path, kind.name(), site_span); + fragment + } +} + +struct MacroRulesMacroExpander { + name: ast::Ident, + span: Span, + transparency: Transparency, + lhses: Vec, + rhses: Vec, + valid: bool, +} + +impl TTMacroExpander for MacroRulesMacroExpander { + fn expand<'cx>( + &self, + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + input: TokenStream, + ) -> Box { + if !self.valid { + return DummyResult::any(sp); + } + generic_extension( + cx, + sp, + self.span, + self.name, + self.transparency, + input, + &self.lhses, + &self.rhses, + ) + } +} + +fn trace_macros_note(cx: &mut ExtCtxt<'_>, sp: Span, message: String) { + let sp = sp.macro_backtrace().last().map(|trace| trace.call_site).unwrap_or(sp); + cx.expansions.entry(sp).or_default().push(message); +} + +/// Given `lhses` and `rhses`, this is the new macro we create +fn generic_extension<'cx>( + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + def_span: Span, + name: ast::Ident, + transparency: Transparency, + arg: TokenStream, + lhses: &[mbe::TokenTree], + rhses: &[mbe::TokenTree], +) -> Box { + if cx.trace_macros() { + let msg = format!("expanding `{}! {{ {} }}`", name, pprust::tts_to_string(arg.clone())); + trace_macros_note(cx, sp, msg); + } + + // Which arm's failure should we report? (the one furthest along) + let mut best_failure: Option<(Token, &str)> = None; + for (i, lhs) in lhses.iter().enumerate() { + // try each arm's matchers + let lhs_tt = match *lhs { + mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], + _ => cx.span_bug(sp, "malformed macro lhs"), + }; + + // Take a snapshot of the state of pre-expansion gating at this point. + // This is used so that if a matcher is not `Success(..)`ful, + // then the spans which became gated when parsing the unsuccessful matcher + // are not recorded. On the first `Success(..)`ful matcher, the spans are merged. + let mut gated_spans_snaphot = mem::take(&mut *cx.parse_sess.gated_spans.spans.borrow_mut()); + + match parse_tt(cx, lhs_tt, arg.clone()) { + Success(named_matches) => { + // The matcher was `Success(..)`ful. + // Merge the gated spans from parsing the matcher with the pre-existing ones. + cx.parse_sess.gated_spans.merge(gated_spans_snaphot); + + let rhs = match rhses[i] { + // ignore delimiters + mbe::TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(), + _ => cx.span_bug(sp, "malformed macro rhs"), + }; + let arm_span = rhses[i].span(); + + let rhs_spans = rhs.iter().map(|t| t.span()).collect::>(); + // rhs has holes ( `$id` and `$(...)` that need filled) + let mut tts = transcribe(cx, &named_matches, rhs, transparency); + + // Replace all the tokens for the corresponding positions in the macro, to maintain + // proper positions in error reporting, while maintaining the macro_backtrace. + if rhs_spans.len() == tts.len() { + tts = tts.map_enumerated(|i, mut tt| { + let mut sp = rhs_spans[i]; + sp = sp.with_ctxt(tt.span().ctxt()); + tt.set_span(sp); + tt + }); + } + + if cx.trace_macros() { + let msg = format!("to `{}`", pprust::tts_to_string(tts.clone())); + trace_macros_note(cx, sp, msg); + } + + let directory = Directory { + path: Cow::from(cx.current_expansion.module.directory.as_path()), + ownership: cx.current_expansion.directory_ownership, + }; + let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None); + p.root_module_name = + cx.current_expansion.module.mod_path.last().map(|id| id.to_string()); + p.last_type_ascription = cx.current_expansion.prior_type_ascription; + + p.process_potential_macro_variable(); + // Let the context choose how to interpret the result. + // Weird, but useful for X-macros. + return Box::new(ParserAnyMacro { + parser: p, + + // Pass along the original expansion site and the name of the macro + // so we can print a useful error message if the parse of the expanded + // macro leaves unparsed tokens. + site_span: sp, + macro_ident: name, + arm_span, + }); + } + Failure(token, msg) => match best_failure { + Some((ref best_token, _)) if best_token.span.lo() >= token.span.lo() => {} + _ => best_failure = Some((token, msg)), + }, + Error(err_sp, ref msg) => cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]), + } + + // The matcher was not `Success(..)`ful. + // Restore to the state before snapshotting and maybe try again. + mem::swap(&mut gated_spans_snaphot, &mut cx.parse_sess.gated_spans.spans.borrow_mut()); + } + + let (token, label) = best_failure.expect("ran no matchers"); + let span = token.span.substitute_dummy(sp); + let mut err = cx.struct_span_err(span, &parse_failure_msg(&token)); + err.span_label(span, label); + if !def_span.is_dummy() && cx.source_map().span_to_filename(def_span).is_real() { + err.span_label(cx.source_map().def_span(def_span), "when calling this macro"); + } + + // Check whether there's a missing comma in this macro call, like `println!("{}" a);` + if let Some((arg, comma_span)) = arg.add_comma() { + for lhs in lhses { + // try each arm's matchers + let lhs_tt = match *lhs { + mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], + _ => continue, + }; + match parse_tt(cx, lhs_tt, arg.clone()) { + Success(_) => { + if comma_span.is_dummy() { + err.note("you might be missing a comma"); + } else { + err.span_suggestion_short( + comma_span, + "missing comma here", + ", ".to_string(), + Applicability::MachineApplicable, + ); + } + } + _ => {} + } + } + } + err.emit(); + cx.trace_macros_diag(); + DummyResult::any(sp) +} + +// Note that macro-by-example's input is also matched against a token tree: +// $( $lhs:tt => $rhs:tt );+ +// +// Holy self-referential! + +/// Converts a macro item into a syntax extension. +pub fn compile_declarative_macro( + sess: &ParseSess, + features: &Features, + def: &ast::Item, + edition: Edition, +) -> SyntaxExtension { + let diag = &sess.span_diagnostic; + let lhs_nm = ast::Ident::new(sym::lhs, def.span); + let rhs_nm = ast::Ident::new(sym::rhs, def.span); + let tt_spec = ast::Ident::new(sym::tt, def.span); + + // Parse the macro_rules! invocation + let (is_legacy, body) = match &def.kind { + ast::ItemKind::MacroDef(macro_def) => (macro_def.legacy, macro_def.body.inner_tokens()), + _ => unreachable!(), + }; + + // The pattern that macro_rules matches. + // The grammar for macro_rules! is: + // $( $lhs:tt => $rhs:tt );+ + // ...quasiquoting this would be nice. + // These spans won't matter, anyways + let argument_gram = vec![ + mbe::TokenTree::Sequence( + DelimSpan::dummy(), + Lrc::new(mbe::SequenceRepetition { + tts: vec![ + mbe::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec), + mbe::TokenTree::token(token::FatArrow, def.span), + mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec), + ], + separator: Some(Token::new( + if is_legacy { token::Semi } else { token::Comma }, + def.span, + )), + kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, def.span), + num_captures: 2, + }), + ), + // to phase into semicolon-termination instead of semicolon-separation + mbe::TokenTree::Sequence( + DelimSpan::dummy(), + Lrc::new(mbe::SequenceRepetition { + tts: vec![mbe::TokenTree::token( + if is_legacy { token::Semi } else { token::Comma }, + def.span, + )], + separator: None, + kleene: mbe::KleeneToken::new(mbe::KleeneOp::ZeroOrMore, def.span), + num_captures: 0, + }), + ), + ]; + + let argument_map = match parse(sess, body, &argument_gram, None, true) { + Success(m) => m, + Failure(token, msg) => { + let s = parse_failure_msg(&token); + let sp = token.span.substitute_dummy(def.span); + let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s); + err.span_label(sp, msg); + err.emit(); + FatalError.raise(); + } + Error(sp, s) => { + sess.span_diagnostic.span_fatal(sp.substitute_dummy(def.span), &s).raise(); + } + }; + + let mut valid = true; + + // Extract the arguments: + let lhses = match argument_map[&lhs_nm] { + MatchedSeq(ref s) => s + .iter() + .map(|m| { + if let MatchedNonterminal(ref nt) = *m { + if let NtTT(ref tt) = **nt { + let tt = mbe::quoted::parse(tt.clone().into(), true, sess).pop().unwrap(); + valid &= check_lhs_nt_follows(sess, features, &def.attrs, &tt); + return tt; + } + } + sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") + }) + .collect::>(), + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs"), + }; + + let rhses = match argument_map[&rhs_nm] { + MatchedSeq(ref s) => s + .iter() + .map(|m| { + if let MatchedNonterminal(ref nt) = *m { + if let NtTT(ref tt) = **nt { + return mbe::quoted::parse(tt.clone().into(), false, sess).pop().unwrap(); + } + } + sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") + }) + .collect::>(), + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs"), + }; + + for rhs in &rhses { + valid &= check_rhs(sess, rhs); + } + + // don't abort iteration early, so that errors for multiple lhses can be reported + for lhs in &lhses { + valid &= check_lhs_no_empty_seq(sess, slice::from_ref(lhs)); + } + + // We use CRATE_NODE_ID instead of `def.id` otherwise we may emit buffered lints for a node id + // that is not lint-checked and trigger the "failed to process buffered lint here" bug. + valid &= macro_check::check_meta_variables(sess, ast::CRATE_NODE_ID, def.span, &lhses, &rhses); + + let (transparency, transparency_error) = attr::find_transparency(&def.attrs, is_legacy); + match transparency_error { + Some(TransparencyError::UnknownTransparency(value, span)) => { + diag.span_err(span, &format!("unknown macro transparency: `{}`", value)) + } + Some(TransparencyError::MultipleTransparencyAttrs(old_span, new_span)) => { + diag.span_err(vec![old_span, new_span], "multiple macro transparency attributes") + } + None => {} + } + + let expander: Box<_> = Box::new(MacroRulesMacroExpander { + name: def.ident, + span: def.span, + transparency, + lhses, + rhses, + valid, + }); + + SyntaxExtension::new( + sess, + SyntaxExtensionKind::LegacyBang(expander), + def.span, + Vec::new(), + edition, + def.ident.name, + &def.attrs, + ) +} + +fn check_lhs_nt_follows( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + lhs: &mbe::TokenTree, +) -> bool { + // lhs is going to be like TokenTree::Delimited(...), where the + // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens. + if let mbe::TokenTree::Delimited(_, ref tts) = *lhs { + check_matcher(sess, features, attrs, &tts.tts) + } else { + let msg = "invalid macro matcher; matchers must be contained in balanced delimiters"; + sess.span_diagnostic.span_err(lhs.span(), msg); + false + } + // we don't abort on errors on rejection, the driver will do that for us + // after parsing/expansion. we can report every error in every macro this way. +} + +/// Checks that the lhs contains no repetition which could match an empty token +/// tree, because then the matcher would hang indefinitely. +fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool { + use mbe::TokenTree; + for tt in tts { + match *tt { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (), + TokenTree::Delimited(_, ref del) => { + if !check_lhs_no_empty_seq(sess, &del.tts) { + return false; + } + } + TokenTree::Sequence(span, ref seq) => { + if seq.separator.is_none() + && seq.tts.iter().all(|seq_tt| match *seq_tt { + TokenTree::MetaVarDecl(_, _, id) => id.name == sym::vis, + TokenTree::Sequence(_, ref sub_seq) => { + sub_seq.kleene.op == mbe::KleeneOp::ZeroOrMore + || sub_seq.kleene.op == mbe::KleeneOp::ZeroOrOne + } + _ => false, + }) + { + let sp = span.entire(); + sess.span_diagnostic.span_err(sp, "repetition matches empty token tree"); + return false; + } + if !check_lhs_no_empty_seq(sess, &seq.tts) { + return false; + } + } + } + } + + true +} + +fn check_rhs(sess: &ParseSess, rhs: &mbe::TokenTree) -> bool { + match *rhs { + mbe::TokenTree::Delimited(..) => return true, + _ => sess.span_diagnostic.span_err(rhs.span(), "macro rhs must be delimited"), + } + false +} + +fn check_matcher( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + matcher: &[mbe::TokenTree], +) -> bool { + let first_sets = FirstSets::new(matcher); + let empty_suffix = TokenSet::empty(); + let err = sess.span_diagnostic.err_count(); + check_matcher_core(sess, features, attrs, &first_sets, matcher, &empty_suffix); + err == sess.span_diagnostic.err_count() +} + +// `The FirstSets` for a matcher is a mapping from subsequences in the +// matcher to the FIRST set for that subsequence. +// +// This mapping is partially precomputed via a backwards scan over the +// token trees of the matcher, which provides a mapping from each +// repetition sequence to its *first* set. +// +// (Hypothetically, sequences should be uniquely identifiable via their +// spans, though perhaps that is false, e.g., for macro-generated macros +// that do not try to inject artificial span information. My plan is +// to try to catch such cases ahead of time and not include them in +// the precomputed mapping.) +struct FirstSets { + // this maps each TokenTree::Sequence `$(tt ...) SEP OP` that is uniquely identified by its + // span in the original matcher to the First set for the inner sequence `tt ...`. + // + // If two sequences have the same span in a matcher, then map that + // span to None (invalidating the mapping here and forcing the code to + // use a slow path). + first: FxHashMap>, +} + +impl FirstSets { + fn new(tts: &[mbe::TokenTree]) -> FirstSets { + use mbe::TokenTree; + + let mut sets = FirstSets { first: FxHashMap::default() }; + build_recur(&mut sets, tts); + return sets; + + // walks backward over `tts`, returning the FIRST for `tts` + // and updating `sets` at the same time for all sequence + // substructure we find within `tts`. + fn build_recur(sets: &mut FirstSets, tts: &[TokenTree]) -> TokenSet { + let mut first = TokenSet::empty(); + for tt in tts.iter().rev() { + match *tt { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { + first.replace_with(tt.clone()); + } + TokenTree::Delimited(span, ref delimited) => { + build_recur(sets, &delimited.tts[..]); + first.replace_with(delimited.open_tt(span)); + } + TokenTree::Sequence(sp, ref seq_rep) => { + let subfirst = build_recur(sets, &seq_rep.tts[..]); + + match sets.first.entry(sp.entire()) { + Entry::Vacant(vac) => { + vac.insert(Some(subfirst.clone())); + } + Entry::Occupied(mut occ) => { + // if there is already an entry, then a span must have collided. + // This should not happen with typical macro_rules macros, + // but syntax extensions need not maintain distinct spans, + // so distinct syntax trees can be assigned the same span. + // In such a case, the map cannot be trusted; so mark this + // entry as unusable. + occ.insert(None); + } + } + + // If the sequence contents can be empty, then the first + // token could be the separator token itself. + + if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { + first.add_one_maybe(TokenTree::Token(sep.clone())); + } + + // Reverse scan: Sequence comes before `first`. + if subfirst.maybe_empty + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne + { + // If sequence is potentially empty, then + // union them (preserving first emptiness). + first.add_all(&TokenSet { maybe_empty: true, ..subfirst }); + } else { + // Otherwise, sequence guaranteed + // non-empty; replace first. + first = subfirst; + } + } + } + } + + first + } + } + + // walks forward over `tts` until all potential FIRST tokens are + // identified. + fn first(&self, tts: &[mbe::TokenTree]) -> TokenSet { + use mbe::TokenTree; + + let mut first = TokenSet::empty(); + for tt in tts.iter() { + assert!(first.maybe_empty); + match *tt { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { + first.add_one(tt.clone()); + return first; + } + TokenTree::Delimited(span, ref delimited) => { + first.add_one(delimited.open_tt(span)); + return first; + } + TokenTree::Sequence(sp, ref seq_rep) => { + let subfirst_owned; + let subfirst = match self.first.get(&sp.entire()) { + Some(&Some(ref subfirst)) => subfirst, + Some(&None) => { + subfirst_owned = self.first(&seq_rep.tts[..]); + &subfirst_owned + } + None => { + panic!("We missed a sequence during FirstSets construction"); + } + }; + + // If the sequence contents can be empty, then the first + // token could be the separator token itself. + if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { + first.add_one_maybe(TokenTree::Token(sep.clone())); + } + + assert!(first.maybe_empty); + first.add_all(subfirst); + if subfirst.maybe_empty + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore + || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne + { + // Continue scanning for more first + // tokens, but also make sure we + // restore empty-tracking state. + first.maybe_empty = true; + continue; + } else { + return first; + } + } + } + } + + // we only exit the loop if `tts` was empty or if every + // element of `tts` matches the empty sequence. + assert!(first.maybe_empty); + first + } +} + +// A set of `mbe::TokenTree`s, which may include `TokenTree::Match`s +// (for macro-by-example syntactic variables). It also carries the +// `maybe_empty` flag; that is true if and only if the matcher can +// match an empty token sequence. +// +// The First set is computed on submatchers like `$($a:expr b),* $(c)* d`, +// which has corresponding FIRST = {$a:expr, c, d}. +// Likewise, `$($a:expr b),* $(c)+ d` has FIRST = {$a:expr, c}. +// +// (Notably, we must allow for *-op to occur zero times.) +#[derive(Clone, Debug)] +struct TokenSet { + tokens: Vec, + maybe_empty: bool, +} + +impl TokenSet { + // Returns a set for the empty sequence. + fn empty() -> Self { + TokenSet { tokens: Vec::new(), maybe_empty: true } + } + + // Returns the set `{ tok }` for the single-token (and thus + // non-empty) sequence [tok]. + fn singleton(tok: mbe::TokenTree) -> Self { + TokenSet { tokens: vec![tok], maybe_empty: false } + } + + // Changes self to be the set `{ tok }`. + // Since `tok` is always present, marks self as non-empty. + fn replace_with(&mut self, tok: mbe::TokenTree) { + self.tokens.clear(); + self.tokens.push(tok); + self.maybe_empty = false; + } + + // Changes self to be the empty set `{}`; meant for use when + // the particular token does not matter, but we want to + // record that it occurs. + fn replace_with_irrelevant(&mut self) { + self.tokens.clear(); + self.maybe_empty = false; + } + + // Adds `tok` to the set for `self`, marking sequence as non-empy. + fn add_one(&mut self, tok: mbe::TokenTree) { + if !self.tokens.contains(&tok) { + self.tokens.push(tok); + } + self.maybe_empty = false; + } + + // Adds `tok` to the set for `self`. (Leaves `maybe_empty` flag alone.) + fn add_one_maybe(&mut self, tok: mbe::TokenTree) { + if !self.tokens.contains(&tok) { + self.tokens.push(tok); + } + } + + // Adds all elements of `other` to this. + // + // (Since this is a set, we filter out duplicates.) + // + // If `other` is potentially empty, then preserves the previous + // setting of the empty flag of `self`. If `other` is guaranteed + // non-empty, then `self` is marked non-empty. + fn add_all(&mut self, other: &Self) { + for tok in &other.tokens { + if !self.tokens.contains(tok) { + self.tokens.push(tok.clone()); + } + } + if !other.maybe_empty { + self.maybe_empty = false; + } + } +} + +// Checks that `matcher` is internally consistent and that it +// can legally be followed by a token `N`, for all `N` in `follow`. +// (If `follow` is empty, then it imposes no constraint on +// the `matcher`.) +// +// Returns the set of NT tokens that could possibly come last in +// `matcher`. (If `matcher` matches the empty sequence, then +// `maybe_empty` will be set to true.) +// +// Requires that `first_sets` is pre-computed for `matcher`; +// see `FirstSets::new`. +fn check_matcher_core( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + first_sets: &FirstSets, + matcher: &[mbe::TokenTree], + follow: &TokenSet, +) -> TokenSet { + use mbe::TokenTree; + + let mut last = TokenSet::empty(); + + // 2. For each token and suffix [T, SUFFIX] in M: + // ensure that T can be followed by SUFFIX, and if SUFFIX may be empty, + // then ensure T can also be followed by any element of FOLLOW. + 'each_token: for i in 0..matcher.len() { + let token = &matcher[i]; + let suffix = &matcher[i + 1..]; + + let build_suffix_first = || { + let mut s = first_sets.first(suffix); + if s.maybe_empty { + s.add_all(follow); + } + s + }; + + // (we build `suffix_first` on demand below; you can tell + // which cases are supposed to fall through by looking for the + // initialization of this variable.) + let suffix_first; + + // First, update `last` so that it corresponds to the set + // of NT tokens that might end the sequence `... token`. + match *token { + TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { + let can_be_followed_by_any; + if let Err(bad_frag) = has_legal_fragment_specifier(sess, features, attrs, token) { + let msg = format!("invalid fragment specifier `{}`", bad_frag); + sess.span_diagnostic + .struct_span_err(token.span(), &msg) + .help(VALID_FRAGMENT_NAMES_MSG) + .emit(); + // (This eliminates false positives and duplicates + // from error messages.) + can_be_followed_by_any = true; + } else { + can_be_followed_by_any = token_can_be_followed_by_any(token); + } + + if can_be_followed_by_any { + // don't need to track tokens that work with any, + last.replace_with_irrelevant(); + // ... and don't need to check tokens that can be + // followed by anything against SUFFIX. + continue 'each_token; + } else { + last.replace_with(token.clone()); + suffix_first = build_suffix_first(); + } + } + TokenTree::Delimited(span, ref d) => { + let my_suffix = TokenSet::singleton(d.close_tt(span)); + check_matcher_core(sess, features, attrs, first_sets, &d.tts, &my_suffix); + // don't track non NT tokens + last.replace_with_irrelevant(); + + // also, we don't need to check delimited sequences + // against SUFFIX + continue 'each_token; + } + TokenTree::Sequence(_, ref seq_rep) => { + suffix_first = build_suffix_first(); + // The trick here: when we check the interior, we want + // to include the separator (if any) as a potential + // (but not guaranteed) element of FOLLOW. So in that + // case, we make a temp copy of suffix and stuff + // delimiter in there. + // + // FIXME: Should I first scan suffix_first to see if + // delimiter is already in it before I go through the + // work of cloning it? But then again, this way I may + // get a "tighter" span? + let mut new; + let my_suffix = if let Some(sep) = &seq_rep.separator { + new = suffix_first.clone(); + new.add_one_maybe(TokenTree::Token(sep.clone())); + &new + } else { + &suffix_first + }; + + // At this point, `suffix_first` is built, and + // `my_suffix` is some TokenSet that we can use + // for checking the interior of `seq_rep`. + let next = + check_matcher_core(sess, features, attrs, first_sets, &seq_rep.tts, my_suffix); + if next.maybe_empty { + last.add_all(&next); + } else { + last = next; + } + + // the recursive call to check_matcher_core already ran the 'each_last + // check below, so we can just keep going forward here. + continue 'each_token; + } + } + + // (`suffix_first` guaranteed initialized once reaching here.) + + // Now `last` holds the complete set of NT tokens that could + // end the sequence before SUFFIX. Check that every one works with `suffix`. + 'each_last: for token in &last.tokens { + if let TokenTree::MetaVarDecl(_, name, frag_spec) = *token { + for next_token in &suffix_first.tokens { + match is_in_follow(next_token, frag_spec.name) { + IsInFollow::Invalid(msg, help) => { + sess.span_diagnostic + .struct_span_err(next_token.span(), &msg) + .help(help) + .emit(); + // don't bother reporting every source of + // conflict for a particular element of `last`. + continue 'each_last; + } + IsInFollow::Yes => {} + IsInFollow::No(possible) => { + let may_be = if last.tokens.len() == 1 && suffix_first.tokens.len() == 1 + { + "is" + } else { + "may be" + }; + + let sp = next_token.span(); + let mut err = sess.span_diagnostic.struct_span_err( + sp, + &format!( + "`${name}:{frag}` {may_be} followed by `{next}`, which \ + is not allowed for `{frag}` fragments", + name = name, + frag = frag_spec, + next = quoted_tt_to_string(next_token), + may_be = may_be + ), + ); + err.span_label( + sp, + format!("not allowed after `{}` fragments", frag_spec), + ); + let msg = "allowed there are: "; + match possible { + &[] => {} + &[t] => { + err.note(&format!( + "only {} is allowed after `{}` fragments", + t, frag_spec, + )); + } + ts => { + err.note(&format!( + "{}{} or {}", + msg, + ts[..ts.len() - 1] + .iter() + .map(|s| *s) + .collect::>() + .join(", "), + ts[ts.len() - 1], + )); + } + } + err.emit(); + } + } + } + } + } + } + last +} + +fn token_can_be_followed_by_any(tok: &mbe::TokenTree) -> bool { + if let mbe::TokenTree::MetaVarDecl(_, _, frag_spec) = *tok { + frag_can_be_followed_by_any(frag_spec.name) + } else { + // (Non NT's can always be followed by anthing in matchers.) + true + } +} + +/// Returns `true` if a fragment of type `frag` can be followed by any sort of +/// token. We use this (among other things) as a useful approximation +/// for when `frag` can be followed by a repetition like `$(...)*` or +/// `$(...)+`. In general, these can be a bit tricky to reason about, +/// so we adopt a conservative position that says that any fragment +/// specifier which consumes at most one token tree can be followed by +/// a fragment specifier (indeed, these fragments can be followed by +/// ANYTHING without fear of future compatibility hazards). +fn frag_can_be_followed_by_any(frag: Symbol) -> bool { + match frag { + sym::item | // always terminated by `}` or `;` + sym::block | // exactly one token tree + sym::ident | // exactly one token tree + sym::literal | // exactly one token tree + sym::meta | // exactly one token tree + sym::lifetime | // exactly one token tree + sym::tt => // exactly one token tree + true, + + _ => + false, + } +} + +enum IsInFollow { + Yes, + No(&'static [&'static str]), + Invalid(String, &'static str), +} + +/// Returns `true` if `frag` can legally be followed by the token `tok`. For +/// fragments that can consume an unbounded number of tokens, `tok` +/// must be within a well-defined follow set. This is intended to +/// guarantee future compatibility: for example, without this rule, if +/// we expanded `expr` to include a new binary operator, we might +/// break macros that were relying on that binary operator as a +/// separator. +// when changing this do not forget to update doc/book/macros.md! +fn is_in_follow(tok: &mbe::TokenTree, frag: Symbol) -> IsInFollow { + use mbe::TokenTree; + + if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok { + // closing a token tree can never be matched by any fragment; + // iow, we always require that `(` and `)` match, etc. + IsInFollow::Yes + } else { + match frag { + sym::item => { + // since items *must* be followed by either a `;` or a `}`, we can + // accept anything after them + IsInFollow::Yes + } + sym::block => { + // anything can follow block, the braces provide an easy boundary to + // maintain + IsInFollow::Yes + } + sym::stmt | sym::expr => { + const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"]; + match tok { + TokenTree::Token(token) => match token.kind { + FatArrow | Comma | Semi => IsInFollow::Yes, + _ => IsInFollow::No(TOKENS), + }, + _ => IsInFollow::No(TOKENS), + } + } + sym::pat => { + const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"]; + match tok { + TokenTree::Token(token) => match token.kind { + FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes, + Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes, + _ => IsInFollow::No(TOKENS), + }, + _ => IsInFollow::No(TOKENS), + } + } + sym::path | sym::ty => { + const TOKENS: &[&str] = &[ + "`{`", "`[`", "`=>`", "`,`", "`>`", "`=`", "`:`", "`;`", "`|`", "`as`", + "`where`", + ]; + match tok { + TokenTree::Token(token) => match token.kind { + OpenDelim(token::DelimToken::Brace) + | OpenDelim(token::DelimToken::Bracket) + | Comma + | FatArrow + | Colon + | Eq + | Gt + | BinOp(token::Shr) + | Semi + | BinOp(token::Or) => IsInFollow::Yes, + Ident(name, false) if name == kw::As || name == kw::Where => { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), + }, + TokenTree::MetaVarDecl(_, _, frag) if frag.name == sym::block => { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), + } + } + sym::ident | sym::lifetime => { + // being a single token, idents and lifetimes are harmless + IsInFollow::Yes + } + sym::literal => { + // literals may be of a single token, or two tokens (negative numbers) + IsInFollow::Yes + } + sym::meta | sym::tt => { + // being either a single token or a delimited sequence, tt is + // harmless + IsInFollow::Yes + } + sym::vis => { + // Explicitly disallow `priv`, on the off chance it comes back. + const TOKENS: &[&str] = &["`,`", "an ident", "a type"]; + match tok { + TokenTree::Token(token) => match token.kind { + Comma => IsInFollow::Yes, + Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes, + _ => { + if token.can_begin_type() { + IsInFollow::Yes + } else { + IsInFollow::No(TOKENS) + } + } + }, + TokenTree::MetaVarDecl(_, _, frag) + if frag.name == sym::ident + || frag.name == sym::ty + || frag.name == sym::path => + { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), + } + } + kw::Invalid => IsInFollow::Yes, + _ => IsInFollow::Invalid( + format!("invalid fragment specifier `{}`", frag), + VALID_FRAGMENT_NAMES_MSG, + ), + } + } +} + +fn has_legal_fragment_specifier( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + tok: &mbe::TokenTree, +) -> Result<(), String> { + debug!("has_legal_fragment_specifier({:?})", tok); + if let mbe::TokenTree::MetaVarDecl(_, _, ref frag_spec) = *tok { + let frag_span = tok.span(); + if !is_legal_fragment_specifier(sess, features, attrs, frag_spec.name, frag_span) { + return Err(frag_spec.to_string()); + } + } + Ok(()) +} + +fn is_legal_fragment_specifier( + _sess: &ParseSess, + _features: &Features, + _attrs: &[ast::Attribute], + frag_name: Symbol, + _frag_span: Span, +) -> bool { + /* + * If new fragment specifiers are invented in nightly, `_sess`, + * `_features`, `_attrs`, and `_frag_span` will be useful here + * for checking against feature gates. See past versions of + * this function. + */ + match frag_name { + sym::item + | sym::block + | sym::stmt + | sym::expr + | sym::pat + | sym::lifetime + | sym::path + | sym::ty + | sym::ident + | sym::meta + | sym::tt + | sym::vis + | sym::literal + | kw::Invalid => true, + _ => false, + } +} + +fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String { + match *tt { + mbe::TokenTree::Token(ref token) => pprust::token_to_string(&token), + mbe::TokenTree::MetaVar(_, name) => format!("${}", name), + mbe::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind), + _ => panic!( + "unexpected mbe::TokenTree::{{Sequence or Delimited}} \ + in follow set checker" + ), + } +} + +/// Use this token tree as a matcher to parse given tts. +fn parse_tt(cx: &ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream) -> NamedParseResult { + // `None` is because we're not interpolating + let directory = Directory { + path: Cow::from(cx.current_expansion.module.directory.as_path()), + ownership: cx.current_expansion.directory_ownership, + }; + parse(cx.parse_sess(), tts, mtch, Some(directory), true) +} + +/// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For +/// other tokens, this is "unexpected token...". +fn parse_failure_msg(tok: &Token) -> String { + match tok.kind { + token::Eof => "unexpected end of macro invocation".to_string(), + _ => format!("no rules expected the token `{}`", pprust::token_to_string(tok),), + } +} diff --git a/src/librustc_expand/mbe/quoted.rs b/src/librustc_expand/mbe/quoted.rs new file mode 100644 index 00000000000..56b97cbb7c6 --- /dev/null +++ b/src/librustc_expand/mbe/quoted.rs @@ -0,0 +1,248 @@ +use crate::mbe::macro_parser; +use crate::mbe::{Delimited, KleeneOp, KleeneToken, SequenceRepetition, TokenTree}; + +use syntax::ast; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::symbol::kw; +use syntax::token::{self, Token}; +use syntax::tokenstream; + +use syntax_pos::Span; + +use rustc_data_structures::sync::Lrc; + +/// Takes a `tokenstream::TokenStream` and returns a `Vec`. Specifically, this +/// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a +/// collection of `TokenTree` for use in parsing a macro. +/// +/// # Parameters +/// +/// - `input`: a token stream to read from, the contents of which we are parsing. +/// - `expect_matchers`: `parse` can be used to parse either the "patterns" or the "body" of a +/// macro. Both take roughly the same form _except_ that in a pattern, metavars are declared with +/// their "matcher" type. For example `$var:expr` or `$id:ident`. In this example, `expr` and +/// `ident` are "matchers". They are not present in the body of a macro rule -- just in the +/// pattern, so we pass a parameter to indicate whether to expect them or not. +/// - `sess`: the parsing session. Any errors will be emitted to this session. +/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use +/// unstable features or not. +/// - `edition`: which edition are we in. +/// - `macro_node_id`: the NodeId of the macro we are parsing. +/// +/// # Returns +/// +/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`. +pub(super) fn parse( + input: tokenstream::TokenStream, + expect_matchers: bool, + sess: &ParseSess, +) -> Vec { + // Will contain the final collection of `self::TokenTree` + let mut result = Vec::new(); + + // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming + // additional trees if need be. + let mut trees = input.trees(); + while let Some(tree) = trees.next() { + // Given the parsed tree, if there is a metavar and we are expecting matchers, actually + // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). + let tree = parse_tree(tree, &mut trees, expect_matchers, sess); + match tree { + TokenTree::MetaVar(start_sp, ident) if expect_matchers => { + let span = match trees.next() { + Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { + match trees.next() { + Some(tokenstream::TokenTree::Token(token)) => match token.ident() { + Some((kind, _)) => { + let span = token.span.with_lo(start_sp.lo()); + result.push(TokenTree::MetaVarDecl(span, ident, kind)); + continue; + } + _ => token.span, + }, + tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), + } + } + tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp), + }; + sess.missing_fragment_specifiers.borrow_mut().insert(span); + result.push(TokenTree::MetaVarDecl(span, ident, ast::Ident::invalid())); + } + + // Not a metavar or no matchers allowed, so just return the tree + _ => result.push(tree), + } + } + result +} + +/// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a +/// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree` +/// for use in parsing a macro. +/// +/// Converting the given tree may involve reading more tokens. +/// +/// # Parameters +/// +/// - `tree`: the tree we wish to convert. +/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish +/// converting `tree` +/// - `expect_matchers`: same as for `parse` (see above). +/// - `sess`: the parsing session. Any errors will be emitted to this session. +/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use +/// unstable features or not. +fn parse_tree( + tree: tokenstream::TokenTree, + trees: &mut impl Iterator, + expect_matchers: bool, + sess: &ParseSess, +) -> TokenTree { + // Depending on what `tree` is, we could be parsing different parts of a macro + match tree { + // `tree` is a `$` token. Look at the next token in `trees` + tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() { + // `tree` is followed by a delimited set of token trees. This indicates the beginning + // of a repetition sequence in the macro (e.g. `$(pat)*`). + Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { + // Must have `(` not `{` or `[` + if delim != token::Paren { + let tok = pprust::token_kind_to_string(&token::OpenDelim(delim)); + let msg = format!("expected `(`, found `{}`", tok); + sess.span_diagnostic.span_err(span.entire(), &msg); + } + // Parse the contents of the sequence itself + let sequence = parse(tts.into(), expect_matchers, sess); + // Get the Kleene operator and optional separator + let (separator, kleene) = parse_sep_and_kleene_op(trees, span.entire(), sess); + // Count the number of captured "names" (i.e., named metavars) + let name_captures = macro_parser::count_names(&sequence); + TokenTree::Sequence( + span, + Lrc::new(SequenceRepetition { + tts: sequence, + separator, + kleene, + num_captures: name_captures, + }), + ) + } + + // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special + // metavariable that names the crate of the invocation. + Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { + let (ident, is_raw) = token.ident().unwrap(); + let span = ident.span.with_lo(span.lo()); + if ident.name == kw::Crate && !is_raw { + TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span) + } else { + TokenTree::MetaVar(span, ident) + } + } + + // `tree` is followed by a random token. This is an error. + Some(tokenstream::TokenTree::Token(token)) => { + let msg = + format!("expected identifier, found `{}`", pprust::token_to_string(&token),); + sess.span_diagnostic.span_err(token.span, &msg); + TokenTree::MetaVar(token.span, ast::Ident::invalid()) + } + + // There are no more tokens. Just return the `$` we already have. + None => TokenTree::token(token::Dollar, span), + }, + + // `tree` is an arbitrary token. Keep it. + tokenstream::TokenTree::Token(token) => TokenTree::Token(token), + + // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to + // descend into the delimited set and further parse it. + tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( + span, + Lrc::new(Delimited { delim, tts: parse(tts.into(), expect_matchers, sess) }), + ), + } +} + +/// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return +/// `None`. +fn kleene_op(token: &Token) -> Option { + match token.kind { + token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), + token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), + token::Question => Some(KleeneOp::ZeroOrOne), + _ => None, + } +} + +/// Parse the next token tree of the input looking for a KleeneOp. Returns +/// +/// - Ok(Ok((op, span))) if the next token tree is a KleeneOp +/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp +/// - Err(span) if the next token tree is not a token +fn parse_kleene_op( + input: &mut impl Iterator, + span: Span, +) -> Result, Span> { + match input.next() { + Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) { + Some(op) => Ok(Ok((op, token.span))), + None => Ok(Err(token)), + }, + tree => Err(tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span)), + } +} + +/// Attempt to parse a single Kleene star, possibly with a separator. +/// +/// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the +/// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing +/// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator +/// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some +/// stream of tokens in an invocation of a macro. +/// +/// This function will take some input iterator `input` corresponding to `span` and a parsing +/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene +/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an +/// error with the appropriate span is emitted to `sess` and a dummy value is returned. +fn parse_sep_and_kleene_op( + input: &mut impl Iterator, + span: Span, + sess: &ParseSess, +) -> (Option, KleeneToken) { + // We basically look at two token trees here, denoted as #1 and #2 below + let span = match parse_kleene_op(input, span) { + // #1 is a `?`, `+`, or `*` KleeneOp + Ok(Ok((op, span))) => return (None, KleeneToken::new(op, span)), + + // #1 is a separator followed by #2, a KleeneOp + Ok(Err(token)) => match parse_kleene_op(input, token.span) { + // #2 is the `?` Kleene op, which does not take a separator (error) + Ok(Ok((KleeneOp::ZeroOrOne, span))) => { + // Error! + sess.span_diagnostic.span_err( + token.span, + "the `?` macro repetition operator does not take a separator", + ); + + // Return a dummy + return (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)); + } + + // #2 is a KleeneOp :D + Ok(Ok((op, span))) => return (Some(token), KleeneToken::new(op, span)), + + // #2 is a random token or not a token at all :( + Ok(Err(Token { span, .. })) | Err(span) => span, + }, + + // #1 is not a token + Err(span) => span, + }; + + // If we ever get to this point, we have experienced an "unexpected token" error + sess.span_diagnostic.span_err(span, "expected one of: `*`, `+`, or `?`"); + + // Return a dummy + (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)) +} diff --git a/src/librustc_expand/mbe/transcribe.rs b/src/librustc_expand/mbe/transcribe.rs new file mode 100644 index 00000000000..0605f7ff36d --- /dev/null +++ b/src/librustc_expand/mbe/transcribe.rs @@ -0,0 +1,392 @@ +use crate::base::ExtCtxt; +use crate::mbe; +use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; + +use syntax::ast::{Ident, Mac}; +use syntax::mut_visit::{self, MutVisitor}; +use syntax::token::{self, NtTT, Token}; +use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; + +use smallvec::{smallvec, SmallVec}; + +use errors::pluralize; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use syntax_pos::hygiene::{ExpnId, Transparency}; +use syntax_pos::Span; + +use std::mem; + +// A Marker adds the given mark to the syntax context. +struct Marker(ExpnId, Transparency); + +impl MutVisitor for Marker { + fn visit_span(&mut self, span: &mut Span) { + *span = span.apply_mark(self.0, self.1) + } + + fn visit_mac(&mut self, mac: &mut Mac) { + mut_visit::noop_visit_mac(mac, self) + } +} + +/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). +enum Frame { + Delimited { forest: Lrc, idx: usize, span: DelimSpan }, + Sequence { forest: Lrc, idx: usize, sep: Option }, +} + +impl Frame { + /// Construct a new frame around the delimited set of tokens. + fn new(tts: Vec) -> Frame { + let forest = Lrc::new(mbe::Delimited { delim: token::NoDelim, tts }); + Frame::Delimited { forest, idx: 0, span: DelimSpan::dummy() } + } +} + +impl Iterator for Frame { + type Item = mbe::TokenTree; + + fn next(&mut self) -> Option { + match *self { + Frame::Delimited { ref forest, ref mut idx, .. } => { + *idx += 1; + forest.tts.get(*idx - 1).cloned() + } + Frame::Sequence { ref forest, ref mut idx, .. } => { + *idx += 1; + forest.tts.get(*idx - 1).cloned() + } + } + } +} + +/// This can do Macro-By-Example transcription. +/// - `interp` is a map of meta-variables to the tokens (non-terminals) they matched in the +/// invocation. We are assuming we already know there is a match. +/// - `src` is the RHS of the MBE, that is, the "example" we are filling in. +/// +/// For example, +/// +/// ```rust +/// macro_rules! foo { +/// ($id:ident) => { println!("{}", stringify!($id)); } +/// } +/// +/// foo!(bar); +/// ``` +/// +/// `interp` would contain `$id => bar` and `src` would contain `println!("{}", stringify!($id));`. +/// +/// `transcribe` would return a `TokenStream` containing `println!("{}", stringify!(bar));`. +/// +/// Along the way, we do some additional error checking. +pub(super) fn transcribe( + cx: &ExtCtxt<'_>, + interp: &FxHashMap, + src: Vec, + transparency: Transparency, +) -> TokenStream { + // Nothing for us to transcribe... + if src.is_empty() { + return TokenStream::default(); + } + + // We descend into the RHS (`src`), expanding things as we go. This stack contains the things + // we have yet to expand/are still expanding. We start the stack off with the whole RHS. + let mut stack: SmallVec<[Frame; 1]> = smallvec![Frame::new(src)]; + + // As we descend in the RHS, we will need to be able to match nested sequences of matchers. + // `repeats` keeps track of where we are in matching at each level, with the last element being + // the most deeply nested sequence. This is used as a stack. + let mut repeats = Vec::new(); + + // `result` contains resulting token stream from the TokenTree we just finished processing. At + // the end, this will contain the full result of transcription, but at arbitrary points during + // `transcribe`, `result` will contain subsets of the final result. + // + // Specifically, as we descend into each TokenTree, we will push the existing results onto the + // `result_stack` and clear `results`. We will then produce the results of transcribing the + // TokenTree into `results`. Then, as we unwind back out of the `TokenTree`, we will pop the + // `result_stack` and append `results` too it to produce the new `results` up to that point. + // + // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level + // again, and we are done transcribing. + let mut result: Vec = Vec::new(); + let mut result_stack = Vec::new(); + let mut marker = Marker(cx.current_expansion.id, transparency); + + loop { + // Look at the last frame on the stack. + let tree = if let Some(tree) = stack.last_mut().unwrap().next() { + // If it still has a TokenTree we have not looked at yet, use that tree. + tree + } + // The else-case never produces a value for `tree` (it `continue`s or `return`s). + else { + // Otherwise, if we have just reached the end of a sequence and we can keep repeating, + // go back to the beginning of the sequence. + if let Frame::Sequence { idx, sep, .. } = stack.last_mut().unwrap() { + let (repeat_idx, repeat_len) = repeats.last_mut().unwrap(); + *repeat_idx += 1; + if repeat_idx < repeat_len { + *idx = 0; + if let Some(sep) = sep { + result.push(TokenTree::Token(sep.clone()).into()); + } + continue; + } + } + + // We are done with the top of the stack. Pop it. Depending on what it was, we do + // different things. Note that the outermost item must be the delimited, wrapped RHS + // that was passed in originally to `transcribe`. + match stack.pop().unwrap() { + // Done with a sequence. Pop from repeats. + Frame::Sequence { .. } => { + repeats.pop(); + } + + // We are done processing a Delimited. If this is the top-level delimited, we are + // done. Otherwise, we unwind the result_stack to append what we have produced to + // any previous results. + Frame::Delimited { forest, span, .. } => { + if result_stack.is_empty() { + // No results left to compute! We are back at the top-level. + return TokenStream::new(result); + } + + // Step back into the parent Delimited. + let tree = + TokenTree::Delimited(span, forest.delim, TokenStream::new(result).into()); + result = result_stack.pop().unwrap(); + result.push(tree.into()); + } + } + continue; + }; + + // At this point, we know we are in the middle of a TokenTree (the last one on `stack`). + // `tree` contains the next `TokenTree` to be processed. + match tree { + // We are descending into a sequence. We first make sure that the matchers in the RHS + // and the matches in `interp` have the same shape. Otherwise, either the caller or the + // macro writer has made a mistake. + seq @ mbe::TokenTree::Sequence(..) => { + match lockstep_iter_size(&seq, interp, &repeats) { + LockstepIterSize::Unconstrained => { + cx.span_fatal( + seq.span(), /* blame macro writer */ + "attempted to repeat an expression containing no syntax variables \ + matched as repeating at this depth", + ); + } + + LockstepIterSize::Contradiction(ref msg) => { + // FIXME: this really ought to be caught at macro definition time... It + // happens when two meta-variables are used in the same repetition in a + // sequence, but they come from different sequence matchers and repeat + // different amounts. + cx.span_fatal(seq.span(), &msg[..]); + } + + LockstepIterSize::Constraint(len, _) => { + // We do this to avoid an extra clone above. We know that this is a + // sequence already. + let (sp, seq) = if let mbe::TokenTree::Sequence(sp, seq) = seq { + (sp, seq) + } else { + unreachable!() + }; + + // Is the repetition empty? + if len == 0 { + if seq.kleene.op == mbe::KleeneOp::OneOrMore { + // FIXME: this really ought to be caught at macro definition + // time... It happens when the Kleene operator in the matcher and + // the body for the same meta-variable do not match. + cx.span_fatal(sp.entire(), "this must repeat at least once"); + } + } else { + // 0 is the initial counter (we have done 0 repretitions so far). `len` + // is the total number of reptitions we should generate. + repeats.push((0, len)); + + // The first time we encounter the sequence we push it to the stack. It + // then gets reused (see the beginning of the loop) until we are done + // repeating. + stack.push(Frame::Sequence { + idx: 0, + sep: seq.separator.clone(), + forest: seq, + }); + } + } + } + } + + // Replace the meta-var with the matched token tree from the invocation. + mbe::TokenTree::MetaVar(mut sp, mut ident) => { + // Find the matched nonterminal from the macro invocation, and use it to replace + // the meta-var. + if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { + if let MatchedNonterminal(ref nt) = cur_matched { + // FIXME #2887: why do we apply a mark when matching a token tree meta-var + // (e.g. `$x:tt`), but not when we are matching any other type of token + // tree? + if let NtTT(ref tt) = **nt { + result.push(tt.clone().into()); + } else { + marker.visit_span(&mut sp); + let token = TokenTree::token(token::Interpolated(nt.clone()), sp); + result.push(token.into()); + } + } else { + // We were unable to descend far enough. This is an error. + cx.span_fatal( + sp, /* blame the macro writer */ + &format!("variable '{}' is still repeating at this depth", ident), + ); + } + } else { + // If we aren't able to match the meta-var, we push it back into the result but + // with modified syntax context. (I believe this supports nested macros). + marker.visit_span(&mut sp); + marker.visit_ident(&mut ident); + result.push(TokenTree::token(token::Dollar, sp).into()); + result.push(TokenTree::Token(Token::from_ast_ident(ident)).into()); + } + } + + // If we are entering a new delimiter, we push its contents to the `stack` to be + // processed, and we push all of the currently produced results to the `result_stack`. + // We will produce all of the results of the inside of the `Delimited` and then we will + // jump back out of the Delimited, pop the result_stack and add the new results back to + // the previous results (from outside the Delimited). + mbe::TokenTree::Delimited(mut span, delimited) => { + mut_visit::visit_delim_span(&mut span, &mut marker); + stack.push(Frame::Delimited { forest: delimited, idx: 0, span }); + result_stack.push(mem::take(&mut result)); + } + + // Nothing much to do here. Just push the token to the result, being careful to + // preserve syntax context. + mbe::TokenTree::Token(token) => { + let mut tt = TokenTree::Token(token); + marker.visit_tt(&mut tt); + result.push(tt.into()); + } + + // There should be no meta-var declarations in the invocation of a macro. + mbe::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"), + } + } +} + +/// Lookup the meta-var named `ident` and return the matched token tree from the invocation using +/// the set of matches `interpolations`. +/// +/// See the definition of `repeats` in the `transcribe` function. `repeats` is used to descend +/// into the right place in nested matchers. If we attempt to descend too far, the macro writer has +/// made a mistake, and we return `None`. +fn lookup_cur_matched<'a>( + ident: Ident, + interpolations: &'a FxHashMap, + repeats: &[(usize, usize)], +) -> Option<&'a NamedMatch> { + interpolations.get(&ident).map(|matched| { + let mut matched = matched; + for &(idx, _) in repeats { + match matched { + MatchedNonterminal(_) => break, + MatchedSeq(ref ads) => matched = ads.get(idx).unwrap(), + } + } + + matched + }) +} + +/// An accumulator over a TokenTree to be used with `fold`. During transcription, we need to make +/// sure that the size of each sequence and all of its nested sequences are the same as the sizes +/// of all the matched (nested) sequences in the macro invocation. If they don't match, somebody +/// has made a mistake (either the macro writer or caller). +#[derive(Clone)] +enum LockstepIterSize { + /// No constraints on length of matcher. This is true for any TokenTree variants except a + /// `MetaVar` with an actual `MatchedSeq` (as opposed to a `MatchedNonterminal`). + Unconstrained, + + /// A `MetaVar` with an actual `MatchedSeq`. The length of the match and the name of the + /// meta-var are returned. + Constraint(usize, Ident), + + /// Two `Constraint`s on the same sequence had different lengths. This is an error. + Contradiction(String), +} + +impl LockstepIterSize { + /// Find incompatibilities in matcher/invocation sizes. + /// - `Unconstrained` is compatible with everything. + /// - `Contradiction` is incompatible with everything. + /// - `Constraint(len)` is only compatible with other constraints of the same length. + fn with(self, other: LockstepIterSize) -> LockstepIterSize { + match self { + LockstepIterSize::Unconstrained => other, + LockstepIterSize::Contradiction(_) => self, + LockstepIterSize::Constraint(l_len, ref l_id) => match other { + LockstepIterSize::Unconstrained => self, + LockstepIterSize::Contradiction(_) => other, + LockstepIterSize::Constraint(r_len, _) if l_len == r_len => self, + LockstepIterSize::Constraint(r_len, r_id) => { + let msg = format!( + "meta-variable `{}` repeats {} time{}, but `{}` repeats {} time{}", + l_id, + l_len, + pluralize!(l_len), + r_id, + r_len, + pluralize!(r_len), + ); + LockstepIterSize::Contradiction(msg) + } + }, + } + } +} + +/// Given a `tree`, make sure that all sequences have the same length as the matches for the +/// appropriate meta-vars in `interpolations`. +/// +/// Note that if `repeats` does not match the exact correct depth of a meta-var, +/// `lookup_cur_matched` will return `None`, which is why this still works even in the presnece of +/// multiple nested matcher sequences. +fn lockstep_iter_size( + tree: &mbe::TokenTree, + interpolations: &FxHashMap, + repeats: &[(usize, usize)], +) -> LockstepIterSize { + use mbe::TokenTree; + match *tree { + TokenTree::Delimited(_, ref delimed) => { + delimed.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { + size.with(lockstep_iter_size(tt, interpolations, repeats)) + }) + } + TokenTree::Sequence(_, ref seq) => { + seq.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { + size.with(lockstep_iter_size(tt, interpolations, repeats)) + }) + } + TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => { + match lookup_cur_matched(name, interpolations, repeats) { + Some(matched) => match matched { + MatchedNonterminal(_) => LockstepIterSize::Unconstrained, + MatchedSeq(ref ads) => LockstepIterSize::Constraint(ads.len(), name), + }, + _ => LockstepIterSize::Unconstrained, + } + } + TokenTree::Token(..) => LockstepIterSize::Unconstrained, + } +} diff --git a/src/librustc_expand/mut_visit/tests.rs b/src/librustc_expand/mut_visit/tests.rs new file mode 100644 index 00000000000..003ce0fcb1f --- /dev/null +++ b/src/librustc_expand/mut_visit/tests.rs @@ -0,0 +1,72 @@ +use crate::tests::{matches_codepattern, string_to_crate}; + +use syntax::ast::{self, Ident}; +use syntax::mut_visit::{self, MutVisitor}; +use syntax::print::pprust; +use syntax::with_default_globals; + +// This version doesn't care about getting comments or doc-strings in. +fn fake_print_crate(s: &mut pprust::State<'_>, krate: &ast::Crate) { + s.print_mod(&krate.module, &krate.attrs) +} + +// Change every identifier to "zz". +struct ToZzIdentMutVisitor; + +impl MutVisitor for ToZzIdentMutVisitor { + fn visit_ident(&mut self, ident: &mut ast::Ident) { + *ident = Ident::from_str("zz"); + } + fn visit_mac(&mut self, mac: &mut ast::Mac) { + mut_visit::noop_visit_mac(mac, self) + } +} + +// Maybe add to `expand.rs`. +macro_rules! assert_pred { + ($pred:expr, $predname:expr, $a:expr , $b:expr) => {{ + let pred_val = $pred; + let a_val = $a; + let b_val = $b; + if !(pred_val(&a_val, &b_val)) { + panic!("expected args satisfying {}, got {} and {}", $predname, a_val, b_val); + } + }}; +} + +// Make sure idents get transformed everywhere. +#[test] +fn ident_transformation() { + with_default_globals(|| { + let mut zz_visitor = ToZzIdentMutVisitor; + let mut krate = + string_to_crate("#[a] mod b {fn c (d : e, f : g) {h!(i,j,k);l;m}}".to_string()); + zz_visitor.visit_crate(&mut krate); + assert_pred!( + matches_codepattern, + "matches_codepattern", + pprust::to_string(|s| fake_print_crate(s, &krate)), + "#[zz]mod zz{fn zz(zz:zz,zz:zz){zz!(zz,zz,zz);zz;zz}}".to_string() + ); + }) +} + +// Make sure idents get transformed even inside macro defs. +#[test] +fn ident_transformation_in_defs() { + with_default_globals(|| { + let mut zz_visitor = ToZzIdentMutVisitor; + let mut krate = string_to_crate( + "macro_rules! a {(b $c:expr $(d $e:token)f+ => \ + (g $(d $d $e)+))} " + .to_string(), + ); + zz_visitor.visit_crate(&mut krate); + assert_pred!( + matches_codepattern, + "matches_codepattern", + pprust::to_string(|s| fake_print_crate(s, &krate)), + "macro_rules! zz{(zz$zz:zz$(zz $zz:zz)zz+=>(zz$(zz$zz$zz)+))}".to_string() + ); + }) +} diff --git a/src/librustc_expand/parse/lexer/tests.rs b/src/librustc_expand/parse/lexer/tests.rs new file mode 100644 index 00000000000..2ca0224812b --- /dev/null +++ b/src/librustc_expand/parse/lexer/tests.rs @@ -0,0 +1,256 @@ +use rustc_data_structures::sync::Lrc; +use rustc_parse::lexer::StringReader; +use syntax::sess::ParseSess; +use syntax::source_map::{FilePathMapping, SourceMap}; +use syntax::token::{self, Token, TokenKind}; +use syntax::util::comments::is_doc_comment; +use syntax::with_default_globals; +use syntax_pos::symbol::Symbol; +use syntax_pos::{BytePos, Span}; + +use errors::{emitter::EmitterWriter, Handler}; +use std::io; +use std::path::PathBuf; + +fn mk_sess(sm: Lrc) -> ParseSess { + let emitter = EmitterWriter::new( + Box::new(io::sink()), + Some(sm.clone()), + false, + false, + false, + None, + false, + ); + ParseSess::with_span_handler(Handler::with_emitter(true, None, Box::new(emitter)), sm) +} + +// Creates a string reader for the given string. +fn setup<'a>(sm: &SourceMap, sess: &'a ParseSess, teststr: String) -> StringReader<'a> { + let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr); + StringReader::new(sess, sf, None) +} + +#[test] +fn t1() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + let mut string_reader = setup( + &sm, + &sh, + "/* my source file */ fn main() { println!(\"zebra\"); }\n".to_string(), + ); + assert_eq!(string_reader.next_token(), token::Comment); + assert_eq!(string_reader.next_token(), token::Whitespace); + let tok1 = string_reader.next_token(); + let tok2 = Token::new(mk_ident("fn"), Span::with_root_ctxt(BytePos(21), BytePos(23))); + assert_eq!(tok1.kind, tok2.kind); + assert_eq!(tok1.span, tok2.span); + assert_eq!(string_reader.next_token(), token::Whitespace); + // Read another token. + let tok3 = string_reader.next_token(); + assert_eq!(string_reader.pos.clone(), BytePos(28)); + let tok4 = Token::new(mk_ident("main"), Span::with_root_ctxt(BytePos(24), BytePos(28))); + assert_eq!(tok3.kind, tok4.kind); + assert_eq!(tok3.span, tok4.span); + + assert_eq!(string_reader.next_token(), token::OpenDelim(token::Paren)); + assert_eq!(string_reader.pos.clone(), BytePos(29)) + }) +} + +// Checks that the given reader produces the desired stream +// of tokens (stop checking after exhausting `expected`). +fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec) { + for expected_tok in &expected { + assert_eq!(&string_reader.next_token(), expected_tok); + } +} + +// Makes the identifier by looking up the string in the interner. +fn mk_ident(id: &str) -> TokenKind { + token::Ident(Symbol::intern(id), false) +} + +fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind { + TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern)) +} + +#[test] +fn doublecolon_parsing() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization( + setup(&sm, &sh, "a b".to_string()), + vec![mk_ident("a"), token::Whitespace, mk_ident("b")], + ); + }) +} + +#[test] +fn doublecolon_parsing_2() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization( + setup(&sm, &sh, "a::b".to_string()), + vec![mk_ident("a"), token::Colon, token::Colon, mk_ident("b")], + ); + }) +} + +#[test] +fn doublecolon_parsing_3() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization( + setup(&sm, &sh, "a ::b".to_string()), + vec![mk_ident("a"), token::Whitespace, token::Colon, token::Colon, mk_ident("b")], + ); + }) +} + +#[test] +fn doublecolon_parsing_4() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization( + setup(&sm, &sh, "a:: b".to_string()), + vec![mk_ident("a"), token::Colon, token::Colon, token::Whitespace, mk_ident("b")], + ); + }) +} + +#[test] +fn character_a() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!(setup(&sm, &sh, "'a'".to_string()).next_token(), mk_lit(token::Char, "a", None),); + }) +} + +#[test] +fn character_space() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!(setup(&sm, &sh, "' '".to_string()).next_token(), mk_lit(token::Char, " ", None),); + }) +} + +#[test] +fn character_escaped() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!( + setup(&sm, &sh, "'\\n'".to_string()).next_token(), + mk_lit(token::Char, "\\n", None), + ); + }) +} + +#[test] +fn lifetime_name() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!( + setup(&sm, &sh, "'abc".to_string()).next_token(), + token::Lifetime(Symbol::intern("'abc")), + ); + }) +} + +#[test] +fn raw_string() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!( + setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token(), + mk_lit(token::StrRaw(3), "\"#a\\b\x00c\"", None), + ); + }) +} + +#[test] +fn literal_suffixes() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + macro_rules! test { + ($input: expr, $tok_type: ident, $tok_contents: expr) => {{ + assert_eq!( + setup(&sm, &sh, format!("{}suffix", $input)).next_token(), + mk_lit(token::$tok_type, $tok_contents, Some("suffix")), + ); + // with a whitespace separator + assert_eq!( + setup(&sm, &sh, format!("{} suffix", $input)).next_token(), + mk_lit(token::$tok_type, $tok_contents, None), + ); + }}; + } + + test!("'a'", Char, "a"); + test!("b'a'", Byte, "a"); + test!("\"a\"", Str, "a"); + test!("b\"a\"", ByteStr, "a"); + test!("1234", Integer, "1234"); + test!("0b101", Integer, "0b101"); + test!("0xABC", Integer, "0xABC"); + test!("1.0", Float, "1.0"); + test!("1.0e10", Float, "1.0e10"); + + assert_eq!( + setup(&sm, &sh, "2us".to_string()).next_token(), + mk_lit(token::Integer, "2", Some("us")), + ); + assert_eq!( + setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token(), + mk_lit(token::StrRaw(3), "raw", Some("suffix")), + ); + assert_eq!( + setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token(), + mk_lit(token::ByteStrRaw(3), "raw", Some("suffix")), + ); + }) +} + +#[test] +fn line_doc_comments() { + assert!(is_doc_comment("///")); + assert!(is_doc_comment("/// blah")); + assert!(!is_doc_comment("////")); +} + +#[test] +fn nested_block_comments() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + let mut lexer = setup(&sm, &sh, "/* /* */ */'a'".to_string()); + assert_eq!(lexer.next_token(), token::Comment); + assert_eq!(lexer.next_token(), mk_lit(token::Char, "a", None)); + }) +} + +#[test] +fn crlf_comments() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + let mut lexer = setup(&sm, &sh, "// test\r\n/// test\r\n".to_string()); + let comment = lexer.next_token(); + assert_eq!(comment.kind, token::Comment); + assert_eq!((comment.span.lo(), comment.span.hi()), (BytePos(0), BytePos(7))); + assert_eq!(lexer.next_token(), token::Whitespace); + assert_eq!(lexer.next_token(), token::DocComment(Symbol::intern("/// test"))); + }) +} diff --git a/src/librustc_expand/parse/tests.rs b/src/librustc_expand/parse/tests.rs new file mode 100644 index 00000000000..833fda6a2eb --- /dev/null +++ b/src/librustc_expand/parse/tests.rs @@ -0,0 +1,348 @@ +use crate::tests::{matches_codepattern, string_to_stream, with_error_checking_parse}; + +use errors::PResult; +use rustc_parse::new_parser_from_source_str; +use syntax::ast::{self, Name, PatKind}; +use syntax::print::pprust::item_to_string; +use syntax::ptr::P; +use syntax::sess::ParseSess; +use syntax::source_map::FilePathMapping; +use syntax::symbol::{kw, sym, Symbol}; +use syntax::token::{self, Token}; +use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree}; +use syntax::visit; +use syntax::with_default_globals; +use syntax_pos::{BytePos, FileName, Pos, Span}; + +use std::path::PathBuf; + +fn sess() -> ParseSess { + ParseSess::new(FilePathMapping::empty()) +} + +/// Parses an item. +/// +/// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and `Err` +/// when a syntax error occurred. +fn parse_item_from_source_str( + name: FileName, + source: String, + sess: &ParseSess, +) -> PResult<'_, Option>> { + new_parser_from_source_str(sess, name, source).parse_item() +} + +// Produces a `syntax_pos::span`. +fn sp(a: u32, b: u32) -> Span { + Span::with_root_ctxt(BytePos(a), BytePos(b)) +} + +/// Parses a string, return an expression. +fn string_to_expr(source_str: String) -> P { + with_error_checking_parse(source_str, &sess(), |p| p.parse_expr()) +} + +/// Parses a string, returns an item. +fn string_to_item(source_str: String) -> Option> { + with_error_checking_parse(source_str, &sess(), |p| p.parse_item()) +} + +#[should_panic] +#[test] +fn bad_path_expr_1() { + with_default_globals(|| { + string_to_expr("::abc::def::return".to_string()); + }) +} + +// Checks the token-tree-ization of macros. +#[test] +fn string_to_tts_macro() { + with_default_globals(|| { + let tts: Vec<_> = + string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect(); + let tts: &[TokenTree] = &tts[..]; + + match tts { + [TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }), TokenTree::Token(Token { kind: token::Not, .. }), TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }), TokenTree::Delimited(_, macro_delim, macro_tts)] + if name_macro_rules == &sym::macro_rules && name_zip.as_str() == "zip" => + { + let tts = ¯o_tts.trees().collect::>(); + match &tts[..] { + [TokenTree::Delimited(_, first_delim, first_tts), TokenTree::Token(Token { kind: token::FatArrow, .. }), TokenTree::Delimited(_, second_delim, second_tts)] + if macro_delim == &token::Paren => + { + let tts = &first_tts.trees().collect::>(); + match &tts[..] { + [TokenTree::Token(Token { kind: token::Dollar, .. }), TokenTree::Token(Token { kind: token::Ident(name, false), .. })] + if first_delim == &token::Paren && name.as_str() == "a" => {} + _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), + } + let tts = &second_tts.trees().collect::>(); + match &tts[..] { + [TokenTree::Token(Token { kind: token::Dollar, .. }), TokenTree::Token(Token { kind: token::Ident(name, false), .. })] + if second_delim == &token::Paren && name.as_str() == "a" => {} + _ => panic!("value 4: {:?} {:?}", second_delim, second_tts), + } + } + _ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts), + } + } + _ => panic!("value: {:?}", tts), + } + }) +} + +#[test] +fn string_to_tts_1() { + with_default_globals(|| { + let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); + + let expected = TokenStream::new(vec![ + TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(), + TokenTree::token(token::Ident(Name::intern("a"), false), sp(3, 4)).into(), + TokenTree::Delimited( + DelimSpan::from_pair(sp(5, 6), sp(13, 14)), + token::DelimToken::Paren, + TokenStream::new(vec![ + TokenTree::token(token::Ident(Name::intern("b"), false), sp(6, 7)).into(), + TokenTree::token(token::Colon, sp(8, 9)).into(), + TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(), + ]) + .into(), + ) + .into(), + TokenTree::Delimited( + DelimSpan::from_pair(sp(15, 16), sp(20, 21)), + token::DelimToken::Brace, + TokenStream::new(vec![ + TokenTree::token(token::Ident(Name::intern("b"), false), sp(17, 18)).into(), + TokenTree::token(token::Semi, sp(18, 19)).into(), + ]) + .into(), + ) + .into(), + ]); + + assert_eq!(tts, expected); + }) +} + +#[test] +fn parse_use() { + with_default_globals(|| { + let use_s = "use foo::bar::baz;"; + let vitem = string_to_item(use_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], use_s); + + let use_s = "use foo::bar as baz;"; + let vitem = string_to_item(use_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], use_s); + }) +} + +#[test] +fn parse_extern_crate() { + with_default_globals(|| { + let ex_s = "extern crate foo;"; + let vitem = string_to_item(ex_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], ex_s); + + let ex_s = "extern crate foo as bar;"; + let vitem = string_to_item(ex_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], ex_s); + }) +} + +fn get_spans_of_pat_idents(src: &str) -> Vec { + let item = string_to_item(src.to_string()).unwrap(); + + struct PatIdentVisitor { + spans: Vec, + } + impl<'a> visit::Visitor<'a> for PatIdentVisitor { + fn visit_pat(&mut self, p: &'a ast::Pat) { + match p.kind { + PatKind::Ident(_, ref ident, _) => { + self.spans.push(ident.span.clone()); + } + _ => { + visit::walk_pat(self, p); + } + } + } + } + let mut v = PatIdentVisitor { spans: Vec::new() }; + visit::walk_item(&mut v, &item); + return v.spans; +} + +#[test] +fn span_of_self_arg_pat_idents_are_correct() { + with_default_globals(|| { + let srcs = [ + "impl z { fn a (&self, &myarg: i32) {} }", + "impl z { fn a (&mut self, &myarg: i32) {} }", + "impl z { fn a (&'a self, &myarg: i32) {} }", + "impl z { fn a (self, &myarg: i32) {} }", + "impl z { fn a (self: Foo, &myarg: i32) {} }", + ]; + + for &src in &srcs { + let spans = get_spans_of_pat_idents(src); + let (lo, hi) = (spans[0].lo(), spans[0].hi()); + assert!( + "self" == &src[lo.to_usize()..hi.to_usize()], + "\"{}\" != \"self\". src=\"{}\"", + &src[lo.to_usize()..hi.to_usize()], + src + ) + } + }) +} + +#[test] +fn parse_exprs() { + with_default_globals(|| { + // just make sure that they parse.... + string_to_expr("3 + 4".to_string()); + string_to_expr("a::z.froob(b,&(987+3))".to_string()); + }) +} + +#[test] +fn attrs_fix_bug() { + with_default_globals(|| { + string_to_item( + "pub fn mk_file_writer(path: &Path, flags: &[FileFlag]) + -> Result, String> { +#[cfg(windows)] +fn wb() -> c_int { + (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int +} + +#[cfg(unix)] +fn wb() -> c_int { O_WRONLY as c_int } + +let mut fflags: c_int = wb(); +}" + .to_string(), + ); + }) +} + +#[test] +fn crlf_doc_comments() { + with_default_globals(|| { + let sess = sess(); + + let name_1 = FileName::Custom("crlf_source_1".to_string()); + let source = "/// doc comment\r\nfn foo() {}".to_string(); + let item = parse_item_from_source_str(name_1, source, &sess).unwrap().unwrap(); + let doc = item.attrs.iter().filter_map(|at| at.doc_str()).next().unwrap(); + assert_eq!(doc.as_str(), "/// doc comment"); + + let name_2 = FileName::Custom("crlf_source_2".to_string()); + let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string(); + let item = parse_item_from_source_str(name_2, source, &sess).unwrap().unwrap(); + let docs = item.attrs.iter().filter_map(|at| at.doc_str()).collect::>(); + let b: &[_] = &[Symbol::intern("/// doc comment"), Symbol::intern("/// line 2")]; + assert_eq!(&docs[..], b); + + let name_3 = FileName::Custom("clrf_source_3".to_string()); + let source = "/** doc comment\r\n * with CRLF */\r\nfn foo() {}".to_string(); + let item = parse_item_from_source_str(name_3, source, &sess).unwrap().unwrap(); + let doc = item.attrs.iter().filter_map(|at| at.doc_str()).next().unwrap(); + assert_eq!(doc.as_str(), "/** doc comment\n * with CRLF */"); + }); +} + +#[test] +fn ttdelim_span() { + fn parse_expr_from_source_str( + name: FileName, + source: String, + sess: &ParseSess, + ) -> PResult<'_, P> { + new_parser_from_source_str(sess, name, source).parse_expr() + } + + with_default_globals(|| { + let sess = sess(); + let expr = parse_expr_from_source_str( + PathBuf::from("foo").into(), + "foo!( fn main() { body } )".to_string(), + &sess, + ) + .unwrap(); + + let tts: Vec<_> = match expr.kind { + ast::ExprKind::Mac(ref mac) => mac.args.inner_tokens().trees().collect(), + _ => panic!("not a macro"), + }; + + let span = tts.iter().rev().next().unwrap().span(); + + match sess.source_map().span_to_snippet(span) { + Ok(s) => assert_eq!(&s[..], "{ body }"), + Err(_) => panic!("could not get snippet"), + } + }); +} + +// This tests that when parsing a string (rather than a file) we don't try +// and read in a file for a module declaration and just parse a stub. +// See `recurse_into_file_modules` in the parser. +#[test] +fn out_of_line_mod() { + with_default_globals(|| { + let item = parse_item_from_source_str( + PathBuf::from("foo").into(), + "mod foo { struct S; mod this_does_not_exist; }".to_owned(), + &sess(), + ) + .unwrap() + .unwrap(); + + if let ast::ItemKind::Mod(ref m) = item.kind { + assert!(m.items.len() == 2); + } else { + panic!(); + } + }); +} + +#[test] +fn eqmodws() { + assert_eq!(matches_codepattern("", ""), true); + assert_eq!(matches_codepattern("", "a"), false); + assert_eq!(matches_codepattern("a", ""), false); + assert_eq!(matches_codepattern("a", "a"), true); + assert_eq!(matches_codepattern("a b", "a \n\t\r b"), true); + assert_eq!(matches_codepattern("a b ", "a \n\t\r b"), true); + assert_eq!(matches_codepattern("a b", "a \n\t\r b "), false); + assert_eq!(matches_codepattern("a b", "a b"), true); + assert_eq!(matches_codepattern("ab", "a b"), false); + assert_eq!(matches_codepattern("a b", "ab"), true); + assert_eq!(matches_codepattern(" a b", "ab"), true); +} + +#[test] +fn pattern_whitespace() { + assert_eq!(matches_codepattern("", "\x0C"), false); + assert_eq!(matches_codepattern("a b ", "a \u{0085}\n\t\r b"), true); + assert_eq!(matches_codepattern("a b", "a \u{0085}\n\t\r b "), false); +} + +#[test] +fn non_pattern_whitespace() { + // These have the property 'White_Space' but not 'Pattern_White_Space' + assert_eq!(matches_codepattern("a b", "a\u{2002}b"), false); + assert_eq!(matches_codepattern("a b", "a\u{2002}b"), false); + assert_eq!(matches_codepattern("\u{205F}a b", "ab"), false); + assert_eq!(matches_codepattern("a \u{3000}b", "ab"), false); +} diff --git a/src/librustc_expand/placeholders.rs b/src/librustc_expand/placeholders.rs new file mode 100644 index 00000000000..231a5a19cb6 --- /dev/null +++ b/src/librustc_expand/placeholders.rs @@ -0,0 +1,339 @@ +use crate::base::ExtCtxt; +use crate::expand::{AstFragment, AstFragmentKind}; + +use syntax::ast; +use syntax::mut_visit::*; +use syntax::ptr::P; +use syntax::source_map::{dummy_spanned, DUMMY_SP}; + +use smallvec::{smallvec, SmallVec}; + +use rustc_data_structures::fx::FxHashMap; + +pub fn placeholder( + kind: AstFragmentKind, + id: ast::NodeId, + vis: Option, +) -> AstFragment { + fn mac_placeholder() -> ast::Mac { + ast::Mac { + path: ast::Path { span: DUMMY_SP, segments: Vec::new() }, + args: P(ast::MacArgs::Empty), + prior_type_ascription: None, + } + } + + let ident = ast::Ident::invalid(); + let attrs = Vec::new(); + let generics = ast::Generics::default(); + let vis = vis.unwrap_or_else(|| dummy_spanned(ast::VisibilityKind::Inherited)); + let span = DUMMY_SP; + let expr_placeholder = || { + P(ast::Expr { + id, + span, + attrs: ast::AttrVec::new(), + kind: ast::ExprKind::Mac(mac_placeholder()), + }) + }; + let ty = || P(ast::Ty { id, kind: ast::TyKind::Mac(mac_placeholder()), span }); + let pat = || P(ast::Pat { id, kind: ast::PatKind::Mac(mac_placeholder()), span }); + + match kind { + AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()), + AstFragmentKind::OptExpr => AstFragment::OptExpr(Some(expr_placeholder())), + AstFragmentKind::Items => AstFragment::Items(smallvec![P(ast::Item { + id, + span, + ident, + vis, + attrs, + kind: ast::ItemKind::Mac(mac_placeholder()), + tokens: None, + })]), + AstFragmentKind::TraitItems => AstFragment::TraitItems(smallvec![ast::AssocItem { + id, + span, + ident, + vis, + attrs, + generics, + kind: ast::AssocItemKind::Macro(mac_placeholder()), + defaultness: ast::Defaultness::Final, + tokens: None, + }]), + AstFragmentKind::ImplItems => AstFragment::ImplItems(smallvec![ast::AssocItem { + id, + span, + ident, + vis, + attrs, + generics, + kind: ast::AssocItemKind::Macro(mac_placeholder()), + defaultness: ast::Defaultness::Final, + tokens: None, + }]), + AstFragmentKind::ForeignItems => AstFragment::ForeignItems(smallvec![ast::ForeignItem { + id, + span, + ident, + vis, + attrs, + kind: ast::ForeignItemKind::Macro(mac_placeholder()), + tokens: None, + }]), + AstFragmentKind::Pat => { + AstFragment::Pat(P(ast::Pat { id, span, kind: ast::PatKind::Mac(mac_placeholder()) })) + } + AstFragmentKind::Ty => { + AstFragment::Ty(P(ast::Ty { id, span, kind: ast::TyKind::Mac(mac_placeholder()) })) + } + AstFragmentKind::Stmts => AstFragment::Stmts(smallvec![{ + let mac = P((mac_placeholder(), ast::MacStmtStyle::Braces, ast::AttrVec::new())); + ast::Stmt { id, span, kind: ast::StmtKind::Mac(mac) } + }]), + AstFragmentKind::Arms => AstFragment::Arms(smallvec![ast::Arm { + attrs: Default::default(), + body: expr_placeholder(), + guard: None, + id, + pat: pat(), + span, + is_placeholder: true, + }]), + AstFragmentKind::Fields => AstFragment::Fields(smallvec![ast::Field { + attrs: Default::default(), + expr: expr_placeholder(), + id, + ident, + is_shorthand: false, + span, + is_placeholder: true, + }]), + AstFragmentKind::FieldPats => AstFragment::FieldPats(smallvec![ast::FieldPat { + attrs: Default::default(), + id, + ident, + is_shorthand: false, + pat: pat(), + span, + is_placeholder: true, + }]), + AstFragmentKind::GenericParams => AstFragment::GenericParams(smallvec![{ + ast::GenericParam { + attrs: Default::default(), + bounds: Default::default(), + id, + ident, + is_placeholder: true, + kind: ast::GenericParamKind::Lifetime, + } + }]), + AstFragmentKind::Params => AstFragment::Params(smallvec![ast::Param { + attrs: Default::default(), + id, + pat: pat(), + span, + ty: ty(), + is_placeholder: true, + }]), + AstFragmentKind::StructFields => AstFragment::StructFields(smallvec![ast::StructField { + attrs: Default::default(), + id, + ident: None, + span, + ty: ty(), + vis, + is_placeholder: true, + }]), + AstFragmentKind::Variants => AstFragment::Variants(smallvec![ast::Variant { + attrs: Default::default(), + data: ast::VariantData::Struct(Default::default(), false), + disr_expr: None, + id, + ident, + span, + vis, + is_placeholder: true, + }]), + } +} + +pub struct PlaceholderExpander<'a, 'b> { + expanded_fragments: FxHashMap, + cx: &'a mut ExtCtxt<'b>, + monotonic: bool, +} + +impl<'a, 'b> PlaceholderExpander<'a, 'b> { + pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { + PlaceholderExpander { cx, expanded_fragments: FxHashMap::default(), monotonic } + } + + pub fn add(&mut self, id: ast::NodeId, mut fragment: AstFragment) { + fragment.mut_visit_with(self); + self.expanded_fragments.insert(id, fragment); + } + + fn remove(&mut self, id: ast::NodeId) -> AstFragment { + self.expanded_fragments.remove(&id).unwrap() + } +} + +impl<'a, 'b> MutVisitor for PlaceholderExpander<'a, 'b> { + fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { + if arm.is_placeholder { + self.remove(arm.id).make_arms() + } else { + noop_flat_map_arm(arm, self) + } + } + + fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { + if field.is_placeholder { + self.remove(field.id).make_fields() + } else { + noop_flat_map_field(field, self) + } + } + + fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { + if fp.is_placeholder { + self.remove(fp.id).make_field_patterns() + } else { + noop_flat_map_field_pattern(fp, self) + } + } + + fn flat_map_generic_param( + &mut self, + param: ast::GenericParam, + ) -> SmallVec<[ast::GenericParam; 1]> { + if param.is_placeholder { + self.remove(param.id).make_generic_params() + } else { + noop_flat_map_generic_param(param, self) + } + } + + fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { + if p.is_placeholder { + self.remove(p.id).make_params() + } else { + noop_flat_map_param(p, self) + } + } + + fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { + if sf.is_placeholder { + self.remove(sf.id).make_struct_fields() + } else { + noop_flat_map_struct_field(sf, self) + } + } + + fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { + if variant.is_placeholder { + self.remove(variant.id).make_variants() + } else { + noop_flat_map_variant(variant, self) + } + } + + fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { + match item.kind { + ast::ItemKind::Mac(_) => return self.remove(item.id).make_items(), + ast::ItemKind::MacroDef(_) => return smallvec![item], + _ => {} + } + + noop_flat_map_item(item, self) + } + + fn flat_map_trait_item(&mut self, item: ast::AssocItem) -> SmallVec<[ast::AssocItem; 1]> { + match item.kind { + ast::AssocItemKind::Macro(_) => self.remove(item.id).make_trait_items(), + _ => noop_flat_map_assoc_item(item, self), + } + } + + fn flat_map_impl_item(&mut self, item: ast::AssocItem) -> SmallVec<[ast::AssocItem; 1]> { + match item.kind { + ast::AssocItemKind::Macro(_) => self.remove(item.id).make_impl_items(), + _ => noop_flat_map_assoc_item(item, self), + } + } + + fn flat_map_foreign_item(&mut self, item: ast::ForeignItem) -> SmallVec<[ast::ForeignItem; 1]> { + match item.kind { + ast::ForeignItemKind::Macro(_) => self.remove(item.id).make_foreign_items(), + _ => noop_flat_map_foreign_item(item, self), + } + } + + fn visit_expr(&mut self, expr: &mut P) { + match expr.kind { + ast::ExprKind::Mac(_) => *expr = self.remove(expr.id).make_expr(), + _ => noop_visit_expr(expr, self), + } + } + + fn filter_map_expr(&mut self, expr: P) -> Option> { + match expr.kind { + ast::ExprKind::Mac(_) => self.remove(expr.id).make_opt_expr(), + _ => noop_filter_map_expr(expr, self), + } + } + + fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { + let (style, mut stmts) = match stmt.kind { + ast::StmtKind::Mac(mac) => (mac.1, self.remove(stmt.id).make_stmts()), + _ => return noop_flat_map_stmt(stmt, self), + }; + + if style == ast::MacStmtStyle::Semicolon { + if let Some(stmt) = stmts.pop() { + stmts.push(stmt.add_trailing_semicolon()); + } + } + + stmts + } + + fn visit_pat(&mut self, pat: &mut P) { + match pat.kind { + ast::PatKind::Mac(_) => *pat = self.remove(pat.id).make_pat(), + _ => noop_visit_pat(pat, self), + } + } + + fn visit_ty(&mut self, ty: &mut P) { + match ty.kind { + ast::TyKind::Mac(_) => *ty = self.remove(ty.id).make_ty(), + _ => noop_visit_ty(ty, self), + } + } + + fn visit_block(&mut self, block: &mut P) { + noop_visit_block(block, self); + + for stmt in block.stmts.iter_mut() { + if self.monotonic { + assert_eq!(stmt.id, ast::DUMMY_NODE_ID); + stmt.id = self.cx.resolver.next_node_id(); + } + } + } + + fn visit_mod(&mut self, module: &mut ast::Mod) { + noop_visit_mod(module, self); + module.items.retain(|item| match item.kind { + ast::ItemKind::Mac(_) if !self.cx.ecfg.keep_macs => false, // remove macro definitions + _ => true, + }); + } + + fn visit_mac(&mut self, _mac: &mut ast::Mac) { + // Do nothing. + } +} diff --git a/src/librustc_expand/proc_macro.rs b/src/librustc_expand/proc_macro.rs new file mode 100644 index 00000000000..9f42ec13b56 --- /dev/null +++ b/src/librustc_expand/proc_macro.rs @@ -0,0 +1,239 @@ +use crate::base::{self, *}; +use crate::proc_macro_server; + +use syntax::ast::{self, ItemKind, MetaItemKind, NestedMetaItem}; +use syntax::errors::{Applicability, FatalError}; +use syntax::symbol::sym; +use syntax::token; +use syntax::tokenstream::{self, TokenStream}; + +use rustc_data_structures::sync::Lrc; +use syntax_pos::{Span, DUMMY_SP}; + +const EXEC_STRATEGY: pm::bridge::server::SameThread = pm::bridge::server::SameThread; + +pub struct BangProcMacro { + pub client: pm::bridge::client::Client pm::TokenStream>, +} + +impl base::ProcMacro for BangProcMacro { + fn expand<'cx>( + &self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + input: TokenStream, + ) -> TokenStream { + let server = proc_macro_server::Rustc::new(ecx); + match self.client.run(&EXEC_STRATEGY, server, input) { + Ok(stream) => stream, + Err(e) => { + let msg = "proc macro panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + } + } +} + +pub struct AttrProcMacro { + pub client: pm::bridge::client::Client pm::TokenStream>, +} + +impl base::AttrProcMacro for AttrProcMacro { + fn expand<'cx>( + &self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + annotation: TokenStream, + annotated: TokenStream, + ) -> TokenStream { + let server = proc_macro_server::Rustc::new(ecx); + match self.client.run(&EXEC_STRATEGY, server, annotation, annotated) { + Ok(stream) => stream, + Err(e) => { + let msg = "custom attribute panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + } + } +} + +pub struct ProcMacroDerive { + pub client: pm::bridge::client::Client pm::TokenStream>, +} + +impl MultiItemModifier for ProcMacroDerive { + fn expand( + &self, + ecx: &mut ExtCtxt<'_>, + span: Span, + _meta_item: &ast::MetaItem, + item: Annotatable, + ) -> Vec { + let item = match item { + Annotatable::Arm(..) + | Annotatable::Field(..) + | Annotatable::FieldPat(..) + | Annotatable::GenericParam(..) + | Annotatable::Param(..) + | Annotatable::StructField(..) + | Annotatable::Variant(..) => panic!("unexpected annotatable"), + Annotatable::Item(item) => item, + Annotatable::ImplItem(_) + | Annotatable::TraitItem(_) + | Annotatable::ForeignItem(_) + | Annotatable::Stmt(_) + | Annotatable::Expr(_) => { + ecx.span_err( + span, + "proc-macro derives may only be \ + applied to a struct, enum, or union", + ); + return Vec::new(); + } + }; + match item.kind { + ItemKind::Struct(..) | ItemKind::Enum(..) | ItemKind::Union(..) => {} + _ => { + ecx.span_err( + span, + "proc-macro derives may only be \ + applied to a struct, enum, or union", + ); + return Vec::new(); + } + } + + let token = token::Interpolated(Lrc::new(token::NtItem(item))); + let input = tokenstream::TokenTree::token(token, DUMMY_SP).into(); + + let server = proc_macro_server::Rustc::new(ecx); + let stream = match self.client.run(&EXEC_STRATEGY, server, input) { + Ok(stream) => stream, + Err(e) => { + let msg = "proc-macro derive panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + }; + + let error_count_before = ecx.parse_sess.span_diagnostic.err_count(); + let msg = "proc-macro derive produced unparseable tokens"; + + let mut parser = + rustc_parse::stream_to_parser(ecx.parse_sess, stream, Some("proc-macro derive")); + let mut items = vec![]; + + loop { + match parser.parse_item() { + Ok(None) => break, + Ok(Some(item)) => items.push(Annotatable::Item(item)), + Err(mut err) => { + // FIXME: handle this better + err.cancel(); + ecx.struct_span_fatal(span, msg).emit(); + FatalError.raise(); + } + } + } + + // fail if there have been errors emitted + if ecx.parse_sess.span_diagnostic.err_count() > error_count_before { + ecx.struct_span_fatal(span, msg).emit(); + FatalError.raise(); + } + + items + } +} + +crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec) -> Vec { + let mut result = Vec::new(); + attrs.retain(|attr| { + if !attr.has_name(sym::derive) { + return true; + } + + // 1) First let's ensure that it's a meta item. + let nmis = match attr.meta_item_list() { + None => { + cx.struct_span_err(attr.span, "malformed `derive` attribute input") + .span_suggestion( + attr.span, + "missing traits to be derived", + "#[derive(Trait1, Trait2, ...)]".to_owned(), + Applicability::HasPlaceholders, + ) + .emit(); + return false; + } + Some(x) => x, + }; + + let mut error_reported_filter_map = false; + let mut error_reported_map = false; + let traits = nmis + .into_iter() + // 2) Moreover, let's ensure we have a path and not `#[derive("foo")]`. + .filter_map(|nmi| match nmi { + NestedMetaItem::Literal(lit) => { + error_reported_filter_map = true; + cx.struct_span_err(lit.span, "expected path to a trait, found literal") + .help("for example, write `#[derive(Debug)]` for `Debug`") + .emit(); + None + } + NestedMetaItem::MetaItem(mi) => Some(mi), + }) + // 3) Finally, we only accept `#[derive($path_0, $path_1, ..)]` + // but not e.g. `#[derive($path_0 = "value", $path_1(abc))]`. + // In this case we can still at least determine that the user + // wanted this trait to be derived, so let's keep it. + .map(|mi| { + let mut traits_dont_accept = |title, action| { + error_reported_map = true; + let sp = mi.span.with_lo(mi.path.span.hi()); + cx.struct_span_err(sp, title) + .span_suggestion( + sp, + action, + String::new(), + Applicability::MachineApplicable, + ) + .emit(); + }; + match &mi.kind { + MetaItemKind::List(..) => traits_dont_accept( + "traits in `#[derive(...)]` don't accept arguments", + "remove the arguments", + ), + MetaItemKind::NameValue(..) => traits_dont_accept( + "traits in `#[derive(...)]` don't accept values", + "remove the value", + ), + MetaItemKind::Word => {} + } + mi.path + }); + + result.extend(traits); + !error_reported_filter_map && !error_reported_map + }); + result +} diff --git a/src/librustc_expand/proc_macro_server.rs b/src/librustc_expand/proc_macro_server.rs new file mode 100644 index 00000000000..790e1f0edc0 --- /dev/null +++ b/src/librustc_expand/proc_macro_server.rs @@ -0,0 +1,689 @@ +use crate::base::ExtCtxt; + +use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str}; +use syntax::ast; +use syntax::print::pprust; +use syntax::sess::ParseSess; +use syntax::token; +use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; +use syntax::util::comments; +use syntax_pos::symbol::{kw, sym, Symbol}; +use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span}; + +use errors::Diagnostic; +use rustc_data_structures::sync::Lrc; + +use pm::bridge::{server, TokenTree}; +use pm::{Delimiter, Level, LineColumn, Spacing}; +use std::ops::Bound; +use std::{ascii, panic}; + +trait FromInternal { + fn from_internal(x: T) -> Self; +} + +trait ToInternal { + fn to_internal(self) -> T; +} + +impl FromInternal for Delimiter { + fn from_internal(delim: token::DelimToken) -> Delimiter { + match delim { + token::Paren => Delimiter::Parenthesis, + token::Brace => Delimiter::Brace, + token::Bracket => Delimiter::Bracket, + token::NoDelim => Delimiter::None, + } + } +} + +impl ToInternal for Delimiter { + fn to_internal(self) -> token::DelimToken { + match self { + Delimiter::Parenthesis => token::Paren, + Delimiter::Brace => token::Brace, + Delimiter::Bracket => token::Bracket, + Delimiter::None => token::NoDelim, + } + } +} + +impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> + for TokenTree +{ + fn from_internal( + ((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec), + ) -> Self { + use syntax::token::*; + + let joint = is_joint == Joint; + let Token { kind, span } = match tree { + tokenstream::TokenTree::Delimited(span, delim, tts) => { + let delimiter = Delimiter::from_internal(delim); + return TokenTree::Group(Group { delimiter, stream: tts.into(), span }); + } + tokenstream::TokenTree::Token(token) => token, + }; + + macro_rules! tt { + ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( + TokenTree::$ty(self::$ty { + $($field $(: $value)*,)+ + span, + }) + ); + ($ty:ident::$method:ident($($value:expr),*)) => ( + TokenTree::$ty(self::$ty::$method($($value,)* span)) + ); + } + macro_rules! op { + ($a:expr) => { + tt!(Punct::new($a, joint)) + }; + ($a:expr, $b:expr) => {{ + stack.push(tt!(Punct::new($b, joint))); + tt!(Punct::new($a, true)) + }}; + ($a:expr, $b:expr, $c:expr) => {{ + stack.push(tt!(Punct::new($c, joint))); + stack.push(tt!(Punct::new($b, true))); + tt!(Punct::new($a, true)) + }}; + } + + match kind { + Eq => op!('='), + Lt => op!('<'), + Le => op!('<', '='), + EqEq => op!('=', '='), + Ne => op!('!', '='), + Ge => op!('>', '='), + Gt => op!('>'), + AndAnd => op!('&', '&'), + OrOr => op!('|', '|'), + Not => op!('!'), + Tilde => op!('~'), + BinOp(Plus) => op!('+'), + BinOp(Minus) => op!('-'), + BinOp(Star) => op!('*'), + BinOp(Slash) => op!('/'), + BinOp(Percent) => op!('%'), + BinOp(Caret) => op!('^'), + BinOp(And) => op!('&'), + BinOp(Or) => op!('|'), + BinOp(Shl) => op!('<', '<'), + BinOp(Shr) => op!('>', '>'), + BinOpEq(Plus) => op!('+', '='), + BinOpEq(Minus) => op!('-', '='), + BinOpEq(Star) => op!('*', '='), + BinOpEq(Slash) => op!('/', '='), + BinOpEq(Percent) => op!('%', '='), + BinOpEq(Caret) => op!('^', '='), + BinOpEq(And) => op!('&', '='), + BinOpEq(Or) => op!('|', '='), + BinOpEq(Shl) => op!('<', '<', '='), + BinOpEq(Shr) => op!('>', '>', '='), + At => op!('@'), + Dot => op!('.'), + DotDot => op!('.', '.'), + DotDotDot => op!('.', '.', '.'), + DotDotEq => op!('.', '.', '='), + Comma => op!(','), + Semi => op!(';'), + Colon => op!(':'), + ModSep => op!(':', ':'), + RArrow => op!('-', '>'), + LArrow => op!('<', '-'), + FatArrow => op!('=', '>'), + Pound => op!('#'), + Dollar => op!('$'), + Question => op!('?'), + SingleQuote => op!('\''), + + Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), + Ident(name, is_raw) => tt!(Ident::new(name, is_raw)), + Lifetime(name) => { + let ident = ast::Ident::new(name, span).without_first_quote(); + stack.push(tt!(Ident::new(ident.name, false))); + tt!(Punct::new('\'', true)) + } + Literal(lit) => tt!(Literal { lit }), + DocComment(c) => { + let style = comments::doc_comment_style(&c.as_str()); + let stripped = comments::strip_doc_comment_decoration(&c.as_str()); + let mut escaped = String::new(); + for ch in stripped.chars() { + escaped.extend(ch.escape_debug()); + } + let stream = vec![ + Ident(sym::doc, false), + Eq, + TokenKind::lit(token::Str, Symbol::intern(&escaped), None), + ] + .into_iter() + .map(|kind| tokenstream::TokenTree::token(kind, span)) + .collect(); + stack.push(TokenTree::Group(Group { + delimiter: Delimiter::Bracket, + stream, + span: DelimSpan::from_single(span), + })); + if style == ast::AttrStyle::Inner { + stack.push(tt!(Punct::new('!', false))); + } + tt!(Punct::new('#', false)) + } + + Interpolated(nt) => { + let stream = nt_to_tokenstream(&nt, sess, span); + TokenTree::Group(Group { + delimiter: Delimiter::None, + stream, + span: DelimSpan::from_single(span), + }) + } + + OpenDelim(..) | CloseDelim(..) => unreachable!(), + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(), + } + } +} + +impl ToInternal for TokenTree { + fn to_internal(self) -> TokenStream { + use syntax::token::*; + + let (ch, joint, span) = match self { + TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), + TokenTree::Group(Group { delimiter, stream, span }) => { + return tokenstream::TokenTree::Delimited( + span, + delimiter.to_internal(), + stream.into(), + ) + .into(); + } + TokenTree::Ident(self::Ident { sym, is_raw, span }) => { + return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); + } + TokenTree::Literal(self::Literal { + lit: token::Lit { kind: token::Integer, symbol, suffix }, + span, + }) if symbol.as_str().starts_with("-") => { + let minus = BinOp(BinOpToken::Minus); + let symbol = Symbol::intern(&symbol.as_str()[1..]); + let integer = TokenKind::lit(token::Integer, symbol, suffix); + let a = tokenstream::TokenTree::token(minus, span); + let b = tokenstream::TokenTree::token(integer, span); + return vec![a, b].into_iter().collect(); + } + TokenTree::Literal(self::Literal { + lit: token::Lit { kind: token::Float, symbol, suffix }, + span, + }) if symbol.as_str().starts_with("-") => { + let minus = BinOp(BinOpToken::Minus); + let symbol = Symbol::intern(&symbol.as_str()[1..]); + let float = TokenKind::lit(token::Float, symbol, suffix); + let a = tokenstream::TokenTree::token(minus, span); + let b = tokenstream::TokenTree::token(float, span); + return vec![a, b].into_iter().collect(); + } + TokenTree::Literal(self::Literal { lit, span }) => { + return tokenstream::TokenTree::token(Literal(lit), span).into(); + } + }; + + let kind = match ch { + '=' => Eq, + '<' => Lt, + '>' => Gt, + '!' => Not, + '~' => Tilde, + '+' => BinOp(Plus), + '-' => BinOp(Minus), + '*' => BinOp(Star), + '/' => BinOp(Slash), + '%' => BinOp(Percent), + '^' => BinOp(Caret), + '&' => BinOp(And), + '|' => BinOp(Or), + '@' => At, + '.' => Dot, + ',' => Comma, + ';' => Semi, + ':' => Colon, + '#' => Pound, + '$' => Dollar, + '?' => Question, + '\'' => SingleQuote, + _ => unreachable!(), + }; + + let tree = tokenstream::TokenTree::token(kind, span); + TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })]) + } +} + +impl ToInternal for Level { + fn to_internal(self) -> errors::Level { + match self { + Level::Error => errors::Level::Error, + Level::Warning => errors::Level::Warning, + Level::Note => errors::Level::Note, + Level::Help => errors::Level::Help, + _ => unreachable!("unknown proc_macro::Level variant: {:?}", self), + } + } +} + +#[derive(Clone)] +pub struct TokenStreamIter { + cursor: tokenstream::Cursor, + stack: Vec>, +} + +#[derive(Clone)] +pub struct Group { + delimiter: Delimiter, + stream: TokenStream, + span: DelimSpan, +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Punct { + ch: char, + // NB. not using `Spacing` here because it doesn't implement `Hash`. + joint: bool, + span: Span, +} + +impl Punct { + fn new(ch: char, joint: bool, span: Span) -> Punct { + const LEGAL_CHARS: &[char] = &[ + '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', + ':', '#', '$', '?', '\'', + ]; + if !LEGAL_CHARS.contains(&ch) { + panic!("unsupported character `{:?}`", ch) + } + Punct { ch, joint, span } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Ident { + sym: Symbol, + is_raw: bool, + span: Span, +} + +impl Ident { + fn is_valid(string: &str) -> bool { + let mut chars = string.chars(); + if let Some(start) = chars.next() { + rustc_lexer::is_id_start(start) && chars.all(rustc_lexer::is_id_continue) + } else { + false + } + } + fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident { + let string = sym.as_str(); + if !Self::is_valid(&string) { + panic!("`{:?}` is not a valid identifier", string) + } + if is_raw && !sym.can_be_raw() { + panic!("`{}` cannot be a raw identifier", string); + } + Ident { sym, is_raw, span } + } + fn dollar_crate(span: Span) -> Ident { + // `$crate` is accepted as an ident only if it comes from the compiler. + Ident { sym: kw::DollarCrate, is_raw: false, span } + } +} + +// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. +#[derive(Clone, Debug)] +pub struct Literal { + lit: token::Lit, + span: Span, +} + +pub(crate) struct Rustc<'a> { + sess: &'a ParseSess, + def_site: Span, + call_site: Span, + mixed_site: Span, +} + +impl<'a> Rustc<'a> { + pub fn new(cx: &'a ExtCtxt<'_>) -> Self { + let expn_data = cx.current_expansion.id.expn_data(); + Rustc { + sess: cx.parse_sess, + def_site: cx.with_def_site_ctxt(expn_data.def_site), + call_site: cx.with_call_site_ctxt(expn_data.call_site), + mixed_site: cx.with_mixed_site_ctxt(expn_data.call_site), + } + } + + fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option) -> Literal { + Literal { lit: token::Lit::new(kind, symbol, suffix), span: server::Span::call_site(self) } + } +} + +impl server::Types for Rustc<'_> { + type TokenStream = TokenStream; + type TokenStreamBuilder = tokenstream::TokenStreamBuilder; + type TokenStreamIter = TokenStreamIter; + type Group = Group; + type Punct = Punct; + type Ident = Ident; + type Literal = Literal; + type SourceFile = Lrc; + type MultiSpan = Vec; + type Diagnostic = Diagnostic; + type Span = Span; +} + +impl server::TokenStream for Rustc<'_> { + fn new(&mut self) -> Self::TokenStream { + TokenStream::default() + } + fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { + stream.is_empty() + } + fn from_str(&mut self, src: &str) -> Self::TokenStream { + parse_stream_from_source_str( + FileName::proc_macro_source_code(src), + src.to_string(), + self.sess, + Some(self.call_site), + ) + } + fn to_string(&mut self, stream: &Self::TokenStream) -> String { + pprust::tts_to_string(stream.clone()) + } + fn from_token_tree( + &mut self, + tree: TokenTree, + ) -> Self::TokenStream { + tree.to_internal() + } + fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { + TokenStreamIter { cursor: stream.trees(), stack: vec![] } + } +} + +impl server::TokenStreamBuilder for Rustc<'_> { + fn new(&mut self) -> Self::TokenStreamBuilder { + tokenstream::TokenStreamBuilder::new() + } + fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) { + builder.push(stream); + } + fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream { + builder.build() + } +} + +impl server::TokenStreamIter for Rustc<'_> { + fn next( + &mut self, + iter: &mut Self::TokenStreamIter, + ) -> Option> { + loop { + let tree = iter.stack.pop().or_else(|| { + let next = iter.cursor.next_with_joint()?; + Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) + })?; + // HACK: The condition "dummy span + group with empty delimiter" represents an AST + // fragment approximately converted into a token stream. This may happen, for + // example, with inputs to proc macro attributes, including derives. Such "groups" + // need to flattened during iteration over stream's token trees. + // Eventually this needs to be removed in favor of keeping original token trees + // and not doing the roundtrip through AST. + if let TokenTree::Group(ref group) = tree { + if group.delimiter == Delimiter::None && group.span.entire().is_dummy() { + iter.cursor.append(group.stream.clone()); + continue; + } + } + return Some(tree); + } + } +} + +impl server::Group for Rustc<'_> { + fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group { + Group { delimiter, stream, span: DelimSpan::from_single(server::Span::call_site(self)) } + } + fn delimiter(&mut self, group: &Self::Group) -> Delimiter { + group.delimiter + } + fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { + group.stream.clone() + } + fn span(&mut self, group: &Self::Group) -> Self::Span { + group.span.entire() + } + fn span_open(&mut self, group: &Self::Group) -> Self::Span { + group.span.open + } + fn span_close(&mut self, group: &Self::Group) -> Self::Span { + group.span.close + } + fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { + group.span = DelimSpan::from_single(span); + } +} + +impl server::Punct for Rustc<'_> { + fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { + Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) + } + fn as_char(&mut self, punct: Self::Punct) -> char { + punct.ch + } + fn spacing(&mut self, punct: Self::Punct) -> Spacing { + if punct.joint { Spacing::Joint } else { Spacing::Alone } + } + fn span(&mut self, punct: Self::Punct) -> Self::Span { + punct.span + } + fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { + Punct { span, ..punct } + } +} + +impl server::Ident for Rustc<'_> { + fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { + Ident::new(Symbol::intern(string), is_raw, span) + } + fn span(&mut self, ident: Self::Ident) -> Self::Span { + ident.span + } + fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { + Ident { span, ..ident } + } +} + +impl server::Literal for Rustc<'_> { + // FIXME(eddyb) `Literal` should not expose internal `Debug` impls. + fn debug(&mut self, literal: &Self::Literal) -> String { + format!("{:?}", literal) + } + fn integer(&mut self, n: &str) -> Self::Literal { + self.lit(token::Integer, Symbol::intern(n), None) + } + fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { + self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) + } + fn float(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), None) + } + fn f32(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) + } + fn f64(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) + } + fn string(&mut self, string: &str) -> Self::Literal { + let mut escaped = String::new(); + for ch in string.chars() { + escaped.extend(ch.escape_debug()); + } + self.lit(token::Str, Symbol::intern(&escaped), None) + } + fn character(&mut self, ch: char) -> Self::Literal { + let mut escaped = String::new(); + escaped.extend(ch.escape_unicode()); + self.lit(token::Char, Symbol::intern(&escaped), None) + } + fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { + let string = bytes + .iter() + .cloned() + .flat_map(ascii::escape_default) + .map(Into::::into) + .collect::(); + self.lit(token::ByteStr, Symbol::intern(&string), None) + } + fn span(&mut self, literal: &Self::Literal) -> Self::Span { + literal.span + } + fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { + literal.span = span; + } + fn subspan( + &mut self, + literal: &Self::Literal, + start: Bound, + end: Bound, + ) -> Option { + let span = literal.span; + let length = span.hi().to_usize() - span.lo().to_usize(); + + let start = match start { + Bound::Included(lo) => lo, + Bound::Excluded(lo) => lo + 1, + Bound::Unbounded => 0, + }; + + let end = match end { + Bound::Included(hi) => hi + 1, + Bound::Excluded(hi) => hi, + Bound::Unbounded => length, + }; + + // Bounds check the values, preventing addition overflow and OOB spans. + if start > u32::max_value() as usize + || end > u32::max_value() as usize + || (u32::max_value() - start as u32) < span.lo().to_u32() + || (u32::max_value() - end as u32) < span.lo().to_u32() + || start >= end + || end > length + { + return None; + } + + let new_lo = span.lo() + BytePos::from_usize(start); + let new_hi = span.lo() + BytePos::from_usize(end); + Some(span.with_lo(new_lo).with_hi(new_hi)) + } +} + +impl server::SourceFile for Rustc<'_> { + fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { + Lrc::ptr_eq(file1, file2) + } + fn path(&mut self, file: &Self::SourceFile) -> String { + match file.name { + FileName::Real(ref path) => path + .to_str() + .expect("non-UTF8 file path in `proc_macro::SourceFile::path`") + .to_string(), + _ => file.name.to_string(), + } + } + fn is_real(&mut self, file: &Self::SourceFile) -> bool { + file.is_real_file() + } +} + +impl server::MultiSpan for Rustc<'_> { + fn new(&mut self) -> Self::MultiSpan { + vec![] + } + fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) { + spans.push(span) + } +} + +impl server::Diagnostic for Rustc<'_> { + fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic { + let mut diag = Diagnostic::new(level.to_internal(), msg); + diag.set_span(MultiSpan::from_spans(spans)); + diag + } + fn sub( + &mut self, + diag: &mut Self::Diagnostic, + level: Level, + msg: &str, + spans: Self::MultiSpan, + ) { + diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None); + } + fn emit(&mut self, diag: Self::Diagnostic) { + self.sess.span_diagnostic.emit_diagnostic(&diag); + } +} + +impl server::Span for Rustc<'_> { + fn debug(&mut self, span: Self::Span) -> String { + format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) + } + fn def_site(&mut self) -> Self::Span { + self.def_site + } + fn call_site(&mut self) -> Self::Span { + self.call_site + } + fn mixed_site(&mut self) -> Self::Span { + self.mixed_site + } + fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { + self.sess.source_map().lookup_char_pos(span.lo()).file + } + fn parent(&mut self, span: Self::Span) -> Option { + span.parent() + } + fn source(&mut self, span: Self::Span) -> Self::Span { + span.source_callsite() + } + fn start(&mut self, span: Self::Span) -> LineColumn { + let loc = self.sess.source_map().lookup_char_pos(span.lo()); + LineColumn { line: loc.line, column: loc.col.to_usize() } + } + fn end(&mut self, span: Self::Span) -> LineColumn { + let loc = self.sess.source_map().lookup_char_pos(span.hi()); + LineColumn { line: loc.line, column: loc.col.to_usize() } + } + fn join(&mut self, first: Self::Span, second: Self::Span) -> Option { + let self_loc = self.sess.source_map().lookup_char_pos(first.lo()); + let other_loc = self.sess.source_map().lookup_char_pos(second.lo()); + + if self_loc.file.name != other_loc.file.name { + return None; + } + + Some(first.to(second)) + } + fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { + span.with_ctxt(at.ctxt()) + } + fn source_text(&mut self, span: Self::Span) -> Option { + self.sess.source_map().span_to_snippet(span).ok() + } +} diff --git a/src/librustc_expand/tests.rs b/src/librustc_expand/tests.rs new file mode 100644 index 00000000000..4f5ff97e48d --- /dev/null +++ b/src/librustc_expand/tests.rs @@ -0,0 +1,1012 @@ +use rustc_parse::{new_parser_from_source_str, parser::Parser, source_file_to_stream}; +use syntax::ast; +use syntax::sess::ParseSess; +use syntax::source_map::{FilePathMapping, SourceMap}; +use syntax::tokenstream::TokenStream; +use syntax::with_default_globals; +use syntax_pos::{BytePos, MultiSpan, Span}; + +use errors::emitter::EmitterWriter; +use errors::{Handler, PResult}; +use rustc_data_structures::sync::Lrc; + +use std::io; +use std::io::prelude::*; +use std::iter::Peekable; +use std::path::{Path, PathBuf}; +use std::str; +use std::sync::{Arc, Mutex}; + +/// Map string to parser (via tts). +fn string_to_parser(ps: &ParseSess, source_str: String) -> Parser<'_> { + new_parser_from_source_str(ps, PathBuf::from("bogofile").into(), source_str) +} + +crate fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T +where + F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>, +{ + let mut p = string_to_parser(&ps, s); + let x = f(&mut p).unwrap(); + p.sess.span_diagnostic.abort_if_errors(); + x +} + +/// Maps a string to tts, using a made-up filename. +crate fn string_to_stream(source_str: String) -> TokenStream { + let ps = ParseSess::new(FilePathMapping::empty()); + source_file_to_stream( + &ps, + ps.source_map().new_source_file(PathBuf::from("bogofile").into(), source_str), + None, + ) + .0 +} + +/// Parses a string, returns a crate. +crate fn string_to_crate(source_str: String) -> ast::Crate { + let ps = ParseSess::new(FilePathMapping::empty()); + with_error_checking_parse(source_str, &ps, |p| p.parse_crate_mod()) +} + +/// Does the given string match the pattern? whitespace in the first string +/// may be deleted or replaced with other whitespace to match the pattern. +/// This function is relatively Unicode-ignorant; fortunately, the careful design +/// of UTF-8 mitigates this ignorance. It doesn't do NKF-normalization(?). +crate fn matches_codepattern(a: &str, b: &str) -> bool { + let mut a_iter = a.chars().peekable(); + let mut b_iter = b.chars().peekable(); + + loop { + let (a, b) = match (a_iter.peek(), b_iter.peek()) { + (None, None) => return true, + (None, _) => return false, + (Some(&a), None) => { + if rustc_lexer::is_whitespace(a) { + break; // Trailing whitespace check is out of loop for borrowck. + } else { + return false; + } + } + (Some(&a), Some(&b)) => (a, b), + }; + + if rustc_lexer::is_whitespace(a) && rustc_lexer::is_whitespace(b) { + // Skip whitespace for `a` and `b`. + scan_for_non_ws_or_end(&mut a_iter); + scan_for_non_ws_or_end(&mut b_iter); + } else if rustc_lexer::is_whitespace(a) { + // Skip whitespace for `a`. + scan_for_non_ws_or_end(&mut a_iter); + } else if a == b { + a_iter.next(); + b_iter.next(); + } else { + return false; + } + } + + // Check if a has *only* trailing whitespace. + a_iter.all(rustc_lexer::is_whitespace) +} + +/// Advances the given peekable `Iterator` until it reaches a non-whitespace character. +fn scan_for_non_ws_or_end>(iter: &mut Peekable) { + while iter.peek().copied().map(|c| rustc_lexer::is_whitespace(c)) == Some(true) { + iter.next(); + } +} + +/// Identifies a position in the text by the n'th occurrence of a string. +struct Position { + string: &'static str, + count: usize, +} + +struct SpanLabel { + start: Position, + end: Position, + label: &'static str, +} + +crate struct Shared { + pub data: Arc>, +} + +impl Write for Shared { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.data.lock().unwrap().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.data.lock().unwrap().flush() + } +} + +fn test_harness(file_text: &str, span_labels: Vec, expected_output: &str) { + with_default_globals(|| { + let output = Arc::new(Mutex::new(Vec::new())); + + let source_map = Lrc::new(SourceMap::new(FilePathMapping::empty())); + source_map.new_source_file(Path::new("test.rs").to_owned().into(), file_text.to_owned()); + + let primary_span = make_span(&file_text, &span_labels[0].start, &span_labels[0].end); + let mut msp = MultiSpan::from_span(primary_span); + for span_label in span_labels { + let span = make_span(&file_text, &span_label.start, &span_label.end); + msp.push_span_label(span, span_label.label.to_string()); + println!("span: {:?} label: {:?}", span, span_label.label); + println!("text: {:?}", source_map.span_to_snippet(span)); + } + + let emitter = EmitterWriter::new( + Box::new(Shared { data: output.clone() }), + Some(source_map.clone()), + false, + false, + false, + None, + false, + ); + let handler = Handler::with_emitter(true, None, Box::new(emitter)); + handler.span_err(msp, "foo"); + + assert!( + expected_output.chars().next() == Some('\n'), + "expected output should begin with newline" + ); + let expected_output = &expected_output[1..]; + + let bytes = output.lock().unwrap(); + let actual_output = str::from_utf8(&bytes).unwrap(); + println!("expected output:\n------\n{}------", expected_output); + println!("actual output:\n------\n{}------", actual_output); + + assert!(expected_output == actual_output) + }) +} + +fn make_span(file_text: &str, start: &Position, end: &Position) -> Span { + let start = make_pos(file_text, start); + let end = make_pos(file_text, end) + end.string.len(); // just after matching thing ends + assert!(start <= end); + Span::with_root_ctxt(BytePos(start as u32), BytePos(end as u32)) +} + +fn make_pos(file_text: &str, pos: &Position) -> usize { + let mut remainder = file_text; + let mut offset = 0; + for _ in 0..pos.count { + if let Some(n) = remainder.find(&pos.string) { + offset += n; + remainder = &remainder[n + 1..]; + } else { + panic!("failed to find {} instances of {:?} in {:?}", pos.count, pos.string, file_text); + } + } + offset +} + +#[test] +fn ends_on_col0() { + test_harness( + r#" +fn foo() { +} +"#, + vec![SpanLabel { + start: Position { string: "{", count: 1 }, + end: Position { string: "}", count: 1 }, + label: "test", + }], + r#" +error: foo + --> test.rs:2:10 + | +2 | fn foo() { + | __________^ +3 | | } + | |_^ test + +"#, + ); +} + +#[test] +fn ends_on_col2() { + test_harness( + r#" +fn foo() { + + + } +"#, + vec![SpanLabel { + start: Position { string: "{", count: 1 }, + end: Position { string: "}", count: 1 }, + label: "test", + }], + r#" +error: foo + --> test.rs:2:10 + | +2 | fn foo() { + | __________^ +3 | | +4 | | +5 | | } + | |___^ test + +"#, + ); +} +#[test] +fn non_nested() { + test_harness( + r#" +fn foo() { + X0 Y0 + X1 Y1 + X2 Y2 +} +"#, + vec![ + SpanLabel { + start: Position { string: "X0", count: 1 }, + end: Position { string: "X2", count: 1 }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { string: "Y0", count: 1 }, + end: Position { string: "Y2", count: 1 }, + label: "`Y` is a good letter too", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | X0 Y0 + | ____^__- + | | ___| + | || +4 | || X1 Y1 +5 | || X2 Y2 + | ||____^__- `Y` is a good letter too + | |____| + | `X` is a good letter + +"#, + ); +} + +#[test] +fn nested() { + test_harness( + r#" +fn foo() { + X0 Y0 + Y1 X1 +} +"#, + vec![ + SpanLabel { + start: Position { string: "X0", count: 1 }, + end: Position { string: "X1", count: 1 }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { string: "Y0", count: 1 }, + end: Position { string: "Y1", count: 1 }, + label: "`Y` is a good letter too", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | X0 Y0 + | ____^__- + | | ___| + | || +4 | || Y1 X1 + | ||____-__^ `X` is a good letter + | |_____| + | `Y` is a good letter too + +"#, + ); +} + +#[test] +fn different_overlap() { + test_harness( + r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 + X2 Y2 Z2 + X3 Y3 Z3 +} +"#, + vec![ + SpanLabel { + start: Position { string: "Y0", count: 1 }, + end: Position { string: "X2", count: 1 }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { string: "Z1", count: 1 }, + end: Position { string: "X3", count: 1 }, + label: "`Y` is a good letter too", + }, + ], + r#" +error: foo + --> test.rs:3:6 + | +3 | X0 Y0 Z0 + | ______^ +4 | | X1 Y1 Z1 + | |_________- +5 | || X2 Y2 Z2 + | ||____^ `X` is a good letter +6 | | X3 Y3 Z3 + | |_____- `Y` is a good letter too + +"#, + ); +} + +#[test] +fn triple_overlap() { + test_harness( + r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 + X2 Y2 Z2 +} +"#, + vec![ + SpanLabel { + start: Position { string: "X0", count: 1 }, + end: Position { string: "X2", count: 1 }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { string: "Y0", count: 1 }, + end: Position { string: "Y2", count: 1 }, + label: "`Y` is a good letter too", + }, + SpanLabel { + start: Position { string: "Z0", count: 1 }, + end: Position { string: "Z2", count: 1 }, + label: "`Z` label", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | X0 Y0 Z0 + | _____^__-__- + | | ____|__| + | || ___| + | ||| +4 | ||| X1 Y1 Z1 +5 | ||| X2 Y2 Z2 + | |||____^__-__- `Z` label + | ||____|__| + | |____| `Y` is a good letter too + | `X` is a good letter + +"#, + ); +} + +#[test] +fn triple_exact_overlap() { + test_harness( + r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 + X2 Y2 Z2 +} +"#, + vec![ + SpanLabel { + start: Position { string: "X0", count: 1 }, + end: Position { string: "X2", count: 1 }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { string: "X0", count: 1 }, + end: Position { string: "X2", count: 1 }, + label: "`Y` is a good letter too", + }, + SpanLabel { + start: Position { string: "X0", count: 1 }, + end: Position { string: "X2", count: 1 }, + label: "`Z` label", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | / X0 Y0 Z0 +4 | | X1 Y1 Z1 +5 | | X2 Y2 Z2 + | | ^ + | | | + | | `X` is a good letter + | |____`Y` is a good letter too + | `Z` label + +"#, + ); +} + +#[test] +fn minimum_depth() { + test_harness( + r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 + X2 Y2 Z2 + X3 Y3 Z3 +} +"#, + vec![ + SpanLabel { + start: Position { string: "Y0", count: 1 }, + end: Position { string: "X1", count: 1 }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { string: "Y1", count: 1 }, + end: Position { string: "Z2", count: 1 }, + label: "`Y` is a good letter too", + }, + SpanLabel { + start: Position { string: "X2", count: 1 }, + end: Position { string: "Y3", count: 1 }, + label: "`Z`", + }, + ], + r#" +error: foo + --> test.rs:3:6 + | +3 | X0 Y0 Z0 + | ______^ +4 | | X1 Y1 Z1 + | |____^_- + | ||____| + | | `X` is a good letter +5 | | X2 Y2 Z2 + | |____-______- `Y` is a good letter too + | ____| + | | +6 | | X3 Y3 Z3 + | |________- `Z` + +"#, + ); +} + +#[test] +fn non_overlaping() { + test_harness( + r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 + X2 Y2 Z2 + X3 Y3 Z3 +} +"#, + vec![ + SpanLabel { + start: Position { string: "X0", count: 1 }, + end: Position { string: "X1", count: 1 }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { string: "Y2", count: 1 }, + end: Position { string: "Z3", count: 1 }, + label: "`Y` is a good letter too", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | / X0 Y0 Z0 +4 | | X1 Y1 Z1 + | |____^ `X` is a good letter +5 | X2 Y2 Z2 + | ______- +6 | | X3 Y3 Z3 + | |__________- `Y` is a good letter too + +"#, + ); +} + +#[test] +fn overlaping_start_and_end() { + test_harness( + r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 + X2 Y2 Z2 + X3 Y3 Z3 +} +"#, + vec![ + SpanLabel { + start: Position { string: "Y0", count: 1 }, + end: Position { string: "X1", count: 1 }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { string: "Z1", count: 1 }, + end: Position { string: "Z3", count: 1 }, + label: "`Y` is a good letter too", + }, + ], + r#" +error: foo + --> test.rs:3:6 + | +3 | X0 Y0 Z0 + | ______^ +4 | | X1 Y1 Z1 + | |____^____- + | ||____| + | | `X` is a good letter +5 | | X2 Y2 Z2 +6 | | X3 Y3 Z3 + | |___________- `Y` is a good letter too + +"#, + ); +} + +#[test] +fn multiple_labels_primary_without_message() { + test_harness( + r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { string: "b", count: 1 }, + end: Position { string: "}", count: 1 }, + label: "", + }, + SpanLabel { + start: Position { string: "a", count: 1 }, + end: Position { string: "d", count: 1 }, + label: "`a` is a good letter", + }, + SpanLabel { + start: Position { string: "c", count: 1 }, + end: Position { string: "c", count: 1 }, + label: "", + }, + ], + r#" +error: foo + --> test.rs:3:7 + | +3 | a { b { c } d } + | ----^^^^-^^-- `a` is a good letter + +"#, + ); +} + +#[test] +fn multiple_labels_secondary_without_message() { + test_harness( + r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { string: "a", count: 1 }, + end: Position { string: "d", count: 1 }, + label: "`a` is a good letter", + }, + SpanLabel { + start: Position { string: "b", count: 1 }, + end: Position { string: "}", count: 1 }, + label: "", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | a { b { c } d } + | ^^^^-------^^ `a` is a good letter + +"#, + ); +} + +#[test] +fn multiple_labels_primary_without_message_2() { + test_harness( + r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { string: "b", count: 1 }, + end: Position { string: "}", count: 1 }, + label: "`b` is a good letter", + }, + SpanLabel { + start: Position { string: "a", count: 1 }, + end: Position { string: "d", count: 1 }, + label: "", + }, + SpanLabel { + start: Position { string: "c", count: 1 }, + end: Position { string: "c", count: 1 }, + label: "", + }, + ], + r#" +error: foo + --> test.rs:3:7 + | +3 | a { b { c } d } + | ----^^^^-^^-- + | | + | `b` is a good letter + +"#, + ); +} + +#[test] +fn multiple_labels_secondary_without_message_2() { + test_harness( + r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { string: "a", count: 1 }, + end: Position { string: "d", count: 1 }, + label: "", + }, + SpanLabel { + start: Position { string: "b", count: 1 }, + end: Position { string: "}", count: 1 }, + label: "`b` is a good letter", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | a { b { c } d } + | ^^^^-------^^ + | | + | `b` is a good letter + +"#, + ); +} + +#[test] +fn multiple_labels_secondary_without_message_3() { + test_harness( + r#" +fn foo() { + a bc d +} +"#, + vec![ + SpanLabel { + start: Position { string: "a", count: 1 }, + end: Position { string: "b", count: 1 }, + label: "`a` is a good letter", + }, + SpanLabel { + start: Position { string: "c", count: 1 }, + end: Position { string: "d", count: 1 }, + label: "", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | a bc d + | ^^^^---- + | | + | `a` is a good letter + +"#, + ); +} + +#[test] +fn multiple_labels_without_message() { + test_harness( + r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { string: "a", count: 1 }, + end: Position { string: "d", count: 1 }, + label: "", + }, + SpanLabel { + start: Position { string: "b", count: 1 }, + end: Position { string: "}", count: 1 }, + label: "", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | a { b { c } d } + | ^^^^-------^^ + +"#, + ); +} + +#[test] +fn multiple_labels_without_message_2() { + test_harness( + r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { string: "b", count: 1 }, + end: Position { string: "}", count: 1 }, + label: "", + }, + SpanLabel { + start: Position { string: "a", count: 1 }, + end: Position { string: "d", count: 1 }, + label: "", + }, + SpanLabel { + start: Position { string: "c", count: 1 }, + end: Position { string: "c", count: 1 }, + label: "", + }, + ], + r#" +error: foo + --> test.rs:3:7 + | +3 | a { b { c } d } + | ----^^^^-^^-- + +"#, + ); +} + +#[test] +fn multiple_labels_with_message() { + test_harness( + r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { string: "a", count: 1 }, + end: Position { string: "d", count: 1 }, + label: "`a` is a good letter", + }, + SpanLabel { + start: Position { string: "b", count: 1 }, + end: Position { string: "}", count: 1 }, + label: "`b` is a good letter", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | a { b { c } d } + | ^^^^-------^^ + | | | + | | `b` is a good letter + | `a` is a good letter + +"#, + ); +} + +#[test] +fn single_label_with_message() { + test_harness( + r#" +fn foo() { + a { b { c } d } +} +"#, + vec![SpanLabel { + start: Position { string: "a", count: 1 }, + end: Position { string: "d", count: 1 }, + label: "`a` is a good letter", + }], + r#" +error: foo + --> test.rs:3:3 + | +3 | a { b { c } d } + | ^^^^^^^^^^^^^ `a` is a good letter + +"#, + ); +} + +#[test] +fn single_label_without_message() { + test_harness( + r#" +fn foo() { + a { b { c } d } +} +"#, + vec![SpanLabel { + start: Position { string: "a", count: 1 }, + end: Position { string: "d", count: 1 }, + label: "", + }], + r#" +error: foo + --> test.rs:3:3 + | +3 | a { b { c } d } + | ^^^^^^^^^^^^^ + +"#, + ); +} + +#[test] +fn long_snippet() { + test_harness( + r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 + X2 Y2 Z2 + X3 Y3 Z3 +} +"#, + vec![ + SpanLabel { + start: Position { string: "Y0", count: 1 }, + end: Position { string: "X1", count: 1 }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { string: "Z1", count: 1 }, + end: Position { string: "Z3", count: 1 }, + label: "`Y` is a good letter too", + }, + ], + r#" +error: foo + --> test.rs:3:6 + | +3 | X0 Y0 Z0 + | ______^ +4 | | X1 Y1 Z1 + | |____^____- + | ||____| + | | `X` is a good letter +5 | | 1 +6 | | 2 +7 | | 3 +... | +15 | | X2 Y2 Z2 +16 | | X3 Y3 Z3 + | |___________- `Y` is a good letter too + +"#, + ); +} + +#[test] +fn long_snippet_multiple_spans() { + test_harness( + r#" +fn foo() { + X0 Y0 Z0 +1 +2 +3 + X1 Y1 Z1 +4 +5 +6 + X2 Y2 Z2 +7 +8 +9 +10 + X3 Y3 Z3 +} +"#, + vec![ + SpanLabel { + start: Position { string: "Y0", count: 1 }, + end: Position { string: "Y3", count: 1 }, + label: "`Y` is a good letter", + }, + SpanLabel { + start: Position { string: "Z1", count: 1 }, + end: Position { string: "Z2", count: 1 }, + label: "`Z` is a good letter too", + }, + ], + r#" +error: foo + --> test.rs:3:6 + | +3 | X0 Y0 Z0 + | ______^ +4 | | 1 +5 | | 2 +6 | | 3 +7 | | X1 Y1 Z1 + | |_________- +8 | || 4 +9 | || 5 +10 | || 6 +11 | || X2 Y2 Z2 + | ||__________- `Z` is a good letter too +... | +15 | | 10 +16 | | X3 Y3 Z3 + | |_______^ `Y` is a good letter + +"#, + ); +} diff --git a/src/librustc_expand/tokenstream/tests.rs b/src/librustc_expand/tokenstream/tests.rs new file mode 100644 index 00000000000..e13999320df --- /dev/null +++ b/src/librustc_expand/tokenstream/tests.rs @@ -0,0 +1,110 @@ +use crate::tests::string_to_stream; + +use smallvec::smallvec; +use syntax::ast::Name; +use syntax::token; +use syntax::tokenstream::{TokenStream, TokenStreamBuilder, TokenTree}; +use syntax::with_default_globals; +use syntax_pos::{BytePos, Span}; + +fn string_to_ts(string: &str) -> TokenStream { + string_to_stream(string.to_owned()) +} + +fn sp(a: u32, b: u32) -> Span { + Span::with_root_ctxt(BytePos(a), BytePos(b)) +} + +#[test] +fn test_concat() { + with_default_globals(|| { + let test_res = string_to_ts("foo::bar::baz"); + let test_fst = string_to_ts("foo::bar"); + let test_snd = string_to_ts("::baz"); + let eq_res = TokenStream::from_streams(smallvec![test_fst, test_snd]); + assert_eq!(test_res.trees().count(), 5); + assert_eq!(eq_res.trees().count(), 5); + assert_eq!(test_res.eq_unspanned(&eq_res), true); + }) +} + +#[test] +fn test_to_from_bijection() { + with_default_globals(|| { + let test_start = string_to_ts("foo::bar(baz)"); + let test_end = test_start.trees().collect(); + assert_eq!(test_start, test_end) + }) +} + +#[test] +fn test_eq_0() { + with_default_globals(|| { + let test_res = string_to_ts("foo"); + let test_eqs = string_to_ts("foo"); + assert_eq!(test_res, test_eqs) + }) +} + +#[test] +fn test_eq_1() { + with_default_globals(|| { + let test_res = string_to_ts("::bar::baz"); + let test_eqs = string_to_ts("::bar::baz"); + assert_eq!(test_res, test_eqs) + }) +} + +#[test] +fn test_eq_3() { + with_default_globals(|| { + let test_res = string_to_ts(""); + let test_eqs = string_to_ts(""); + assert_eq!(test_res, test_eqs) + }) +} + +#[test] +fn test_diseq_0() { + with_default_globals(|| { + let test_res = string_to_ts("::bar::baz"); + let test_eqs = string_to_ts("bar::baz"); + assert_eq!(test_res == test_eqs, false) + }) +} + +#[test] +fn test_diseq_1() { + with_default_globals(|| { + let test_res = string_to_ts("(bar,baz)"); + let test_eqs = string_to_ts("bar,baz"); + assert_eq!(test_res == test_eqs, false) + }) +} + +#[test] +fn test_is_empty() { + with_default_globals(|| { + let test0: TokenStream = Vec::::new().into_iter().collect(); + let test1: TokenStream = + TokenTree::token(token::Ident(Name::intern("a"), false), sp(0, 1)).into(); + let test2 = string_to_ts("foo(bar::baz)"); + + assert_eq!(test0.is_empty(), true); + assert_eq!(test1.is_empty(), false); + assert_eq!(test2.is_empty(), false); + }) +} + +#[test] +fn test_dotdotdot() { + with_default_globals(|| { + let mut builder = TokenStreamBuilder::new(); + builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint()); + builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint()); + builder.push(TokenTree::token(token::Dot, sp(2, 3))); + let stream = builder.build(); + assert!(stream.eq_unspanned(&string_to_ts("..."))); + assert_eq!(stream.trees().count(), 1); + }) +} diff --git a/src/librustc_span/Cargo.toml b/src/librustc_span/Cargo.toml new file mode 100644 index 00000000000..2cac76085d2 --- /dev/null +++ b/src/librustc_span/Cargo.toml @@ -0,0 +1,21 @@ +[package] +authors = ["The Rust Project Developers"] +name = "syntax_pos" +version = "0.0.0" +edition = "2018" + +[lib] +name = "syntax_pos" +path = "lib.rs" +doctest = false + +[dependencies] +rustc_serialize = { path = "../libserialize", package = "serialize" } +rustc_macros = { path = "../librustc_macros" } +rustc_data_structures = { path = "../librustc_data_structures" } +rustc_index = { path = "../librustc_index" } +arena = { path = "../libarena" } +scoped-tls = "1.0" +unicode-width = "0.1.4" +cfg-if = "0.1.2" +log = "0.4" diff --git a/src/librustc_span/analyze_source_file.rs b/src/librustc_span/analyze_source_file.rs new file mode 100644 index 00000000000..b4beb3dc376 --- /dev/null +++ b/src/librustc_span/analyze_source_file.rs @@ -0,0 +1,274 @@ +use super::*; +use unicode_width::UnicodeWidthChar; + +#[cfg(test)] +mod tests; + +/// Finds all newlines, multi-byte characters, and non-narrow characters in a +/// SourceFile. +/// +/// This function will use an SSE2 enhanced implementation if hardware support +/// is detected at runtime. +pub fn analyze_source_file( + src: &str, + source_file_start_pos: BytePos, +) -> (Vec, Vec, Vec) { + let mut lines = vec![source_file_start_pos]; + let mut multi_byte_chars = vec![]; + let mut non_narrow_chars = vec![]; + + // Calls the right implementation, depending on hardware support available. + analyze_source_file_dispatch( + src, + source_file_start_pos, + &mut lines, + &mut multi_byte_chars, + &mut non_narrow_chars, + ); + + // The code above optimistically registers a new line *after* each \n + // it encounters. If that point is already outside the source_file, remove + // it again. + if let Some(&last_line_start) = lines.last() { + let source_file_end = source_file_start_pos + BytePos::from_usize(src.len()); + assert!(source_file_end >= last_line_start); + if last_line_start == source_file_end { + lines.pop(); + } + } + + (lines, multi_byte_chars, non_narrow_chars) +} + +cfg_if::cfg_if! { + if #[cfg(all(any(target_arch = "x86", target_arch = "x86_64")))] { + fn analyze_source_file_dispatch(src: &str, + source_file_start_pos: BytePos, + lines: &mut Vec, + multi_byte_chars: &mut Vec, + non_narrow_chars: &mut Vec) { + if is_x86_feature_detected!("sse2") { + unsafe { + analyze_source_file_sse2(src, + source_file_start_pos, + lines, + multi_byte_chars, + non_narrow_chars); + } + } else { + analyze_source_file_generic(src, + src.len(), + source_file_start_pos, + lines, + multi_byte_chars, + non_narrow_chars); + + } + } + + /// Checks 16 byte chunks of text at a time. If the chunk contains + /// something other than printable ASCII characters and newlines, the + /// function falls back to the generic implementation. Otherwise it uses + /// SSE2 intrinsics to quickly find all newlines. + #[target_feature(enable = "sse2")] + unsafe fn analyze_source_file_sse2(src: &str, + output_offset: BytePos, + lines: &mut Vec, + multi_byte_chars: &mut Vec, + non_narrow_chars: &mut Vec) { + #[cfg(target_arch = "x86")] + use std::arch::x86::*; + #[cfg(target_arch = "x86_64")] + use std::arch::x86_64::*; + + const CHUNK_SIZE: usize = 16; + + let src_bytes = src.as_bytes(); + + let chunk_count = src.len() / CHUNK_SIZE; + + // This variable keeps track of where we should start decoding a + // chunk. If a multi-byte character spans across chunk boundaries, + // we need to skip that part in the next chunk because we already + // handled it. + let mut intra_chunk_offset = 0; + + for chunk_index in 0 .. chunk_count { + let ptr = src_bytes.as_ptr() as *const __m128i; + // We don't know if the pointer is aligned to 16 bytes, so we + // use `loadu`, which supports unaligned loading. + let chunk = _mm_loadu_si128(ptr.offset(chunk_index as isize)); + + // For character in the chunk, see if its byte value is < 0, which + // indicates that it's part of a UTF-8 char. + let multibyte_test = _mm_cmplt_epi8(chunk, _mm_set1_epi8(0)); + // Create a bit mask from the comparison results. + let multibyte_mask = _mm_movemask_epi8(multibyte_test); + + // If the bit mask is all zero, we only have ASCII chars here: + if multibyte_mask == 0 { + assert!(intra_chunk_offset == 0); + + // Check if there are any control characters in the chunk. All + // control characters that we can encounter at this point have a + // byte value less than 32 or ... + let control_char_test0 = _mm_cmplt_epi8(chunk, _mm_set1_epi8(32)); + let control_char_mask0 = _mm_movemask_epi8(control_char_test0); + + // ... it's the ASCII 'DEL' character with a value of 127. + let control_char_test1 = _mm_cmpeq_epi8(chunk, _mm_set1_epi8(127)); + let control_char_mask1 = _mm_movemask_epi8(control_char_test1); + + let control_char_mask = control_char_mask0 | control_char_mask1; + + if control_char_mask != 0 { + // Check for newlines in the chunk + let newlines_test = _mm_cmpeq_epi8(chunk, _mm_set1_epi8(b'\n' as i8)); + let newlines_mask = _mm_movemask_epi8(newlines_test); + + if control_char_mask == newlines_mask { + // All control characters are newlines, record them + let mut newlines_mask = 0xFFFF0000 | newlines_mask as u32; + let output_offset = output_offset + + BytePos::from_usize(chunk_index * CHUNK_SIZE + 1); + + loop { + let index = newlines_mask.trailing_zeros(); + + if index >= CHUNK_SIZE as u32 { + // We have arrived at the end of the chunk. + break + } + + lines.push(BytePos(index) + output_offset); + + // Clear the bit, so we can find the next one. + newlines_mask &= (!1) << index; + } + + // We are done for this chunk. All control characters were + // newlines and we took care of those. + continue + } else { + // Some of the control characters are not newlines, + // fall through to the slow path below. + } + } else { + // No control characters, nothing to record for this chunk + continue + } + } + + // The slow path. + // There are control chars in here, fallback to generic decoding. + let scan_start = chunk_index * CHUNK_SIZE + intra_chunk_offset; + intra_chunk_offset = analyze_source_file_generic( + &src[scan_start .. ], + CHUNK_SIZE - intra_chunk_offset, + BytePos::from_usize(scan_start) + output_offset, + lines, + multi_byte_chars, + non_narrow_chars + ); + } + + // There might still be a tail left to analyze + let tail_start = chunk_count * CHUNK_SIZE + intra_chunk_offset; + if tail_start < src.len() { + analyze_source_file_generic(&src[tail_start as usize ..], + src.len() - tail_start, + output_offset + BytePos::from_usize(tail_start), + lines, + multi_byte_chars, + non_narrow_chars); + } + } + } else { + + // The target (or compiler version) does not support SSE2 ... + fn analyze_source_file_dispatch(src: &str, + source_file_start_pos: BytePos, + lines: &mut Vec, + multi_byte_chars: &mut Vec, + non_narrow_chars: &mut Vec) { + analyze_source_file_generic(src, + src.len(), + source_file_start_pos, + lines, + multi_byte_chars, + non_narrow_chars); + } + } +} + +// `scan_len` determines the number of bytes in `src` to scan. Note that the +// function can read past `scan_len` if a multi-byte character start within the +// range but extends past it. The overflow is returned by the function. +fn analyze_source_file_generic( + src: &str, + scan_len: usize, + output_offset: BytePos, + lines: &mut Vec, + multi_byte_chars: &mut Vec, + non_narrow_chars: &mut Vec, +) -> usize { + assert!(src.len() >= scan_len); + let mut i = 0; + let src_bytes = src.as_bytes(); + + while i < scan_len { + let byte = unsafe { + // We verified that i < scan_len <= src.len() + *src_bytes.get_unchecked(i as usize) + }; + + // How much to advance in order to get to the next UTF-8 char in the + // string. + let mut char_len = 1; + + if byte < 32 { + // This is an ASCII control character, it could be one of the cases + // that are interesting to us. + + let pos = BytePos::from_usize(i) + output_offset; + + match byte { + b'\n' => { + lines.push(pos + BytePos(1)); + } + b'\t' => { + non_narrow_chars.push(NonNarrowChar::Tab(pos)); + } + _ => { + non_narrow_chars.push(NonNarrowChar::ZeroWidth(pos)); + } + } + } else if byte >= 127 { + // The slow path: + // This is either ASCII control character "DEL" or the beginning of + // a multibyte char. Just decode to `char`. + let c = (&src[i..]).chars().next().unwrap(); + char_len = c.len_utf8(); + + let pos = BytePos::from_usize(i) + output_offset; + + if char_len > 1 { + assert!(char_len >= 2 && char_len <= 4); + let mbc = MultiByteChar { pos, bytes: char_len as u8 }; + multi_byte_chars.push(mbc); + } + + // Assume control characters are zero width. + // FIXME: How can we decide between `width` and `width_cjk`? + let char_width = UnicodeWidthChar::width(c).unwrap_or(0); + + if char_width != 1 { + non_narrow_chars.push(NonNarrowChar::new(pos, char_width)); + } + } + + i += char_len; + } + + i - scan_len +} diff --git a/src/librustc_span/analyze_source_file/tests.rs b/src/librustc_span/analyze_source_file/tests.rs new file mode 100644 index 00000000000..cb418a4bdaf --- /dev/null +++ b/src/librustc_span/analyze_source_file/tests.rs @@ -0,0 +1,142 @@ +use super::*; + +macro_rules! test { + (case: $test_name:ident, + text: $text:expr, + source_file_start_pos: $source_file_start_pos:expr, + lines: $lines:expr, + multi_byte_chars: $multi_byte_chars:expr, + non_narrow_chars: $non_narrow_chars:expr,) => { + #[test] + fn $test_name() { + let (lines, multi_byte_chars, non_narrow_chars) = + analyze_source_file($text, BytePos($source_file_start_pos)); + + let expected_lines: Vec = $lines.into_iter().map(|pos| BytePos(pos)).collect(); + + assert_eq!(lines, expected_lines); + + let expected_mbcs: Vec = $multi_byte_chars + .into_iter() + .map(|(pos, bytes)| MultiByteChar { pos: BytePos(pos), bytes }) + .collect(); + + assert_eq!(multi_byte_chars, expected_mbcs); + + let expected_nncs: Vec = $non_narrow_chars + .into_iter() + .map(|(pos, width)| NonNarrowChar::new(BytePos(pos), width)) + .collect(); + + assert_eq!(non_narrow_chars, expected_nncs); + } + }; +} + +test!( + case: empty_text, + text: "", + source_file_start_pos: 0, + lines: vec![], + multi_byte_chars: vec![], + non_narrow_chars: vec![], +); + +test!( + case: newlines_short, + text: "a\nc", + source_file_start_pos: 0, + lines: vec![0, 2], + multi_byte_chars: vec![], + non_narrow_chars: vec![], +); + +test!( + case: newlines_long, + text: "012345678\nabcdef012345678\na", + source_file_start_pos: 0, + lines: vec![0, 10, 26], + multi_byte_chars: vec![], + non_narrow_chars: vec![], +); + +test!( + case: newline_and_multi_byte_char_in_same_chunk, + text: "01234β789\nbcdef0123456789abcdef", + source_file_start_pos: 0, + lines: vec![0, 11], + multi_byte_chars: vec![(5, 2)], + non_narrow_chars: vec![], +); + +test!( + case: newline_and_control_char_in_same_chunk, + text: "01234\u{07}6789\nbcdef0123456789abcdef", + source_file_start_pos: 0, + lines: vec![0, 11], + multi_byte_chars: vec![], + non_narrow_chars: vec![(5, 0)], +); + +test!( + case: multi_byte_char_short, + text: "aβc", + source_file_start_pos: 0, + lines: vec![0], + multi_byte_chars: vec![(1, 2)], + non_narrow_chars: vec![], +); + +test!( + case: multi_byte_char_long, + text: "0123456789abcΔf012345β", + source_file_start_pos: 0, + lines: vec![0], + multi_byte_chars: vec![(13, 2), (22, 2)], + non_narrow_chars: vec![], +); + +test!( + case: multi_byte_char_across_chunk_boundary, + text: "0123456789abcdeΔ123456789abcdef01234", + source_file_start_pos: 0, + lines: vec![0], + multi_byte_chars: vec![(15, 2)], + non_narrow_chars: vec![], +); + +test!( + case: multi_byte_char_across_chunk_boundary_tail, + text: "0123456789abcdeΔ....", + source_file_start_pos: 0, + lines: vec![0], + multi_byte_chars: vec![(15, 2)], + non_narrow_chars: vec![], +); + +test!( + case: non_narrow_short, + text: "0\t2", + source_file_start_pos: 0, + lines: vec![0], + multi_byte_chars: vec![], + non_narrow_chars: vec![(1, 4)], +); + +test!( + case: non_narrow_long, + text: "01\t3456789abcdef01234567\u{07}9", + source_file_start_pos: 0, + lines: vec![0], + multi_byte_chars: vec![], + non_narrow_chars: vec![(2, 4), (24, 0)], +); + +test!( + case: output_offset_all, + text: "01\t345\n789abcΔf01234567\u{07}9\nbcΔf", + source_file_start_pos: 1000, + lines: vec![0 + 1000, 7 + 1000, 27 + 1000], + multi_byte_chars: vec![(13 + 1000, 2), (29 + 1000, 2)], + non_narrow_chars: vec![(2 + 1000, 4), (24 + 1000, 0)], +); diff --git a/src/librustc_span/caching_source_map_view.rs b/src/librustc_span/caching_source_map_view.rs new file mode 100644 index 00000000000..c329f2225b0 --- /dev/null +++ b/src/librustc_span/caching_source_map_view.rs @@ -0,0 +1,108 @@ +use crate::source_map::SourceMap; +use crate::{BytePos, SourceFile}; +use rustc_data_structures::sync::Lrc; + +#[derive(Clone)] +struct CacheEntry { + time_stamp: usize, + line_number: usize, + line_start: BytePos, + line_end: BytePos, + file: Lrc, + file_index: usize, +} + +#[derive(Clone)] +pub struct CachingSourceMapView<'cm> { + source_map: &'cm SourceMap, + line_cache: [CacheEntry; 3], + time_stamp: usize, +} + +impl<'cm> CachingSourceMapView<'cm> { + pub fn new(source_map: &'cm SourceMap) -> CachingSourceMapView<'cm> { + let files = source_map.files(); + let first_file = files[0].clone(); + let entry = CacheEntry { + time_stamp: 0, + line_number: 0, + line_start: BytePos(0), + line_end: BytePos(0), + file: first_file, + file_index: 0, + }; + + CachingSourceMapView { + source_map, + line_cache: [entry.clone(), entry.clone(), entry], + time_stamp: 0, + } + } + + pub fn byte_pos_to_line_and_col( + &mut self, + pos: BytePos, + ) -> Option<(Lrc, usize, BytePos)> { + self.time_stamp += 1; + + // Check if the position is in one of the cached lines + for cache_entry in self.line_cache.iter_mut() { + if pos >= cache_entry.line_start && pos < cache_entry.line_end { + cache_entry.time_stamp = self.time_stamp; + + return Some(( + cache_entry.file.clone(), + cache_entry.line_number, + pos - cache_entry.line_start, + )); + } + } + + // No cache hit ... + let mut oldest = 0; + for index in 1..self.line_cache.len() { + if self.line_cache[index].time_stamp < self.line_cache[oldest].time_stamp { + oldest = index; + } + } + + let cache_entry = &mut self.line_cache[oldest]; + + // If the entry doesn't point to the correct file, fix it up + if pos < cache_entry.file.start_pos || pos >= cache_entry.file.end_pos { + let file_valid; + if self.source_map.files().len() > 0 { + let file_index = self.source_map.lookup_source_file_idx(pos); + let file = self.source_map.files()[file_index].clone(); + + if pos >= file.start_pos && pos < file.end_pos { + cache_entry.file = file; + cache_entry.file_index = file_index; + file_valid = true; + } else { + file_valid = false; + } + } else { + file_valid = false; + } + + if !file_valid { + return None; + } + } + + let line_index = cache_entry.file.lookup_line(pos).unwrap(); + let line_bounds = cache_entry.file.line_bounds(line_index); + + cache_entry.line_number = line_index + 1; + cache_entry.line_start = line_bounds.0; + cache_entry.line_end = line_bounds.1; + cache_entry.time_stamp = self.time_stamp; + + return Some(( + cache_entry.file.clone(), + cache_entry.line_number, + pos - cache_entry.line_start, + )); + } +} diff --git a/src/librustc_span/edition.rs b/src/librustc_span/edition.rs new file mode 100644 index 00000000000..3017191563b --- /dev/null +++ b/src/librustc_span/edition.rs @@ -0,0 +1,83 @@ +use crate::symbol::{sym, Symbol}; +use std::fmt; +use std::str::FromStr; + +use rustc_macros::HashStable_Generic; + +/// The edition of the compiler (RFC 2052) +#[derive( + Clone, + Copy, + Hash, + PartialEq, + PartialOrd, + Debug, + RustcEncodable, + RustcDecodable, + Eq, + HashStable_Generic +)] +pub enum Edition { + // editions must be kept in order, oldest to newest + /// The 2015 edition + Edition2015, + /// The 2018 edition + Edition2018, + // when adding new editions, be sure to update: + // + // - Update the `ALL_EDITIONS` const + // - Update the EDITION_NAME_LIST const + // - add a `rust_####()` function to the session + // - update the enum in Cargo's sources as well +} + +// must be in order from oldest to newest +pub const ALL_EDITIONS: &[Edition] = &[Edition::Edition2015, Edition::Edition2018]; + +pub const EDITION_NAME_LIST: &str = "2015|2018"; + +pub const DEFAULT_EDITION: Edition = Edition::Edition2015; + +impl fmt::Display for Edition { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = match *self { + Edition::Edition2015 => "2015", + Edition::Edition2018 => "2018", + }; + write!(f, "{}", s) + } +} + +impl Edition { + pub fn lint_name(&self) -> &'static str { + match *self { + Edition::Edition2015 => "rust_2015_compatibility", + Edition::Edition2018 => "rust_2018_compatibility", + } + } + + pub fn feature_name(&self) -> Symbol { + match *self { + Edition::Edition2015 => sym::rust_2015_preview, + Edition::Edition2018 => sym::rust_2018_preview, + } + } + + pub fn is_stable(&self) -> bool { + match *self { + Edition::Edition2015 => true, + Edition::Edition2018 => true, + } + } +} + +impl FromStr for Edition { + type Err = (); + fn from_str(s: &str) -> Result { + match s { + "2015" => Ok(Edition::Edition2015), + "2018" => Ok(Edition::Edition2018), + _ => Err(()), + } + } +} diff --git a/src/librustc_span/fatal_error.rs b/src/librustc_span/fatal_error.rs new file mode 100644 index 00000000000..718c0ddbc63 --- /dev/null +++ b/src/librustc_span/fatal_error.rs @@ -0,0 +1,26 @@ +/// Used as a return value to signify a fatal error occurred. (It is also +/// used as the argument to panic at the moment, but that will eventually +/// not be true.) +#[derive(Copy, Clone, Debug)] +#[must_use] +pub struct FatalError; + +pub struct FatalErrorMarker; + +// Don't implement Send on FatalError. This makes it impossible to panic!(FatalError). +// We don't want to invoke the panic handler and print a backtrace for fatal errors. +impl !Send for FatalError {} + +impl FatalError { + pub fn raise(self) -> ! { + std::panic::resume_unwind(Box::new(FatalErrorMarker)) + } +} + +impl std::fmt::Display for FatalError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "parser fatal error") + } +} + +impl std::error::Error for FatalError {} diff --git a/src/librustc_span/hygiene.rs b/src/librustc_span/hygiene.rs new file mode 100644 index 00000000000..fd1f07c743b --- /dev/null +++ b/src/librustc_span/hygiene.rs @@ -0,0 +1,850 @@ +//! Machinery for hygienic macros, inspired by the `MTWT[1]` paper. +//! +//! `[1]` Matthew Flatt, Ryan Culpepper, David Darais, and Robert Bruce Findler. 2012. +//! *Macros that work together: Compile-time bindings, partial expansion, +//! and definition contexts*. J. Funct. Program. 22, 2 (March 2012), 181-216. +//! DOI=10.1017/S0956796812000093 + +// Hygiene data is stored in a global variable and accessed via TLS, which +// means that accesses are somewhat expensive. (`HygieneData::with` +// encapsulates a single access.) Therefore, on hot code paths it is worth +// ensuring that multiple HygieneData accesses are combined into a single +// `HygieneData::with`. +// +// This explains why `HygieneData`, `SyntaxContext` and `ExpnId` have interfaces +// with a certain amount of redundancy in them. For example, +// `SyntaxContext::outer_expn_data` combines `SyntaxContext::outer` and +// `ExpnId::expn_data` so that two `HygieneData` accesses can be performed within +// a single `HygieneData::with` call. +// +// It also explains why many functions appear in `HygieneData` and again in +// `SyntaxContext` or `ExpnId`. For example, `HygieneData::outer` and +// `SyntaxContext::outer` do the same thing, but the former is for use within a +// `HygieneData::with` call while the latter is for use outside such a call. +// When modifying this file it is important to understand this distinction, +// because getting it wrong can lead to nested `HygieneData::with` calls that +// trigger runtime aborts. (Fortunately these are obvious and easy to fix.) + +use crate::edition::Edition; +use crate::symbol::{kw, sym, Symbol}; +use crate::GLOBALS; +use crate::{Span, DUMMY_SP}; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use rustc_macros::HashStable_Generic; +use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; +use std::fmt; + +/// A `SyntaxContext` represents a chain of pairs `(ExpnId, Transparency)` named "marks". +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SyntaxContext(u32); + +#[derive(Debug)] +struct SyntaxContextData { + outer_expn: ExpnId, + outer_transparency: Transparency, + parent: SyntaxContext, + /// This context, but with all transparent and semi-transparent expansions filtered away. + opaque: SyntaxContext, + /// This context, but with all transparent expansions filtered away. + opaque_and_semitransparent: SyntaxContext, + /// Name of the crate to which `$crate` with this context would resolve. + dollar_crate_name: Symbol, +} + +/// A unique ID associated with a macro invocation and expansion. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub struct ExpnId(u32); + +/// A property of a macro expansion that determines how identifiers +/// produced by that expansion are resolved. +#[derive( + Copy, + Clone, + PartialEq, + Eq, + PartialOrd, + Hash, + Debug, + RustcEncodable, + RustcDecodable, + HashStable_Generic +)] +pub enum Transparency { + /// Identifier produced by a transparent expansion is always resolved at call-site. + /// Call-site spans in procedural macros, hygiene opt-out in `macro` should use this. + Transparent, + /// Identifier produced by a semi-transparent expansion may be resolved + /// either at call-site or at definition-site. + /// If it's a local variable, label or `$crate` then it's resolved at def-site. + /// Otherwise it's resolved at call-site. + /// `macro_rules` macros behave like this, built-in macros currently behave like this too, + /// but that's an implementation detail. + SemiTransparent, + /// Identifier produced by an opaque expansion is always resolved at definition-site. + /// Def-site spans in procedural macros, identifiers from `macro` by default use this. + Opaque, +} + +impl ExpnId { + pub fn fresh(expn_data: Option) -> Self { + HygieneData::with(|data| data.fresh_expn(expn_data)) + } + + /// The ID of the theoretical expansion that generates freshly parsed, unexpanded AST. + #[inline] + pub fn root() -> Self { + ExpnId(0) + } + + #[inline] + pub fn as_u32(self) -> u32 { + self.0 + } + + #[inline] + pub fn from_u32(raw: u32) -> ExpnId { + ExpnId(raw) + } + + #[inline] + pub fn expn_data(self) -> ExpnData { + HygieneData::with(|data| data.expn_data(self).clone()) + } + + #[inline] + pub fn set_expn_data(self, expn_data: ExpnData) { + HygieneData::with(|data| { + let old_expn_data = &mut data.expn_data[self.0 as usize]; + assert!(old_expn_data.is_none(), "expansion data is reset for an expansion ID"); + *old_expn_data = Some(expn_data); + }) + } + + pub fn is_descendant_of(self, ancestor: ExpnId) -> bool { + HygieneData::with(|data| data.is_descendant_of(self, ancestor)) + } + + /// `expn_id.outer_expn_is_descendant_of(ctxt)` is equivalent to but faster than + /// `expn_id.is_descendant_of(ctxt.outer_expn())`. + pub fn outer_expn_is_descendant_of(self, ctxt: SyntaxContext) -> bool { + HygieneData::with(|data| data.is_descendant_of(self, data.outer_expn(ctxt))) + } + + /// Returns span for the macro which originally caused this expansion to happen. + /// + /// Stops backtracing at include! boundary. + pub fn expansion_cause(mut self) -> Option { + let mut last_macro = None; + loop { + let expn_data = self.expn_data(); + // Stop going up the backtrace once include! is encountered + if expn_data.is_root() || expn_data.kind.descr() == sym::include { + break; + } + self = expn_data.call_site.ctxt().outer_expn(); + last_macro = Some(expn_data.call_site); + } + last_macro + } +} + +#[derive(Debug)] +crate struct HygieneData { + /// Each expansion should have an associated expansion data, but sometimes there's a delay + /// between creation of an expansion ID and obtaining its data (e.g. macros are collected + /// first and then resolved later), so we use an `Option` here. + expn_data: Vec>, + syntax_context_data: Vec, + syntax_context_map: FxHashMap<(SyntaxContext, ExpnId, Transparency), SyntaxContext>, +} + +impl HygieneData { + crate fn new(edition: Edition) -> Self { + HygieneData { + expn_data: vec![Some(ExpnData::default(ExpnKind::Root, DUMMY_SP, edition))], + syntax_context_data: vec![SyntaxContextData { + outer_expn: ExpnId::root(), + outer_transparency: Transparency::Opaque, + parent: SyntaxContext(0), + opaque: SyntaxContext(0), + opaque_and_semitransparent: SyntaxContext(0), + dollar_crate_name: kw::DollarCrate, + }], + syntax_context_map: FxHashMap::default(), + } + } + + fn with T>(f: F) -> T { + GLOBALS.with(|globals| f(&mut *globals.hygiene_data.borrow_mut())) + } + + fn fresh_expn(&mut self, expn_data: Option) -> ExpnId { + self.expn_data.push(expn_data); + ExpnId(self.expn_data.len() as u32 - 1) + } + + fn expn_data(&self, expn_id: ExpnId) -> &ExpnData { + self.expn_data[expn_id.0 as usize].as_ref().expect("no expansion data for an expansion ID") + } + + fn is_descendant_of(&self, mut expn_id: ExpnId, ancestor: ExpnId) -> bool { + while expn_id != ancestor { + if expn_id == ExpnId::root() { + return false; + } + expn_id = self.expn_data(expn_id).parent; + } + true + } + + fn modern(&self, ctxt: SyntaxContext) -> SyntaxContext { + self.syntax_context_data[ctxt.0 as usize].opaque + } + + fn modern_and_legacy(&self, ctxt: SyntaxContext) -> SyntaxContext { + self.syntax_context_data[ctxt.0 as usize].opaque_and_semitransparent + } + + fn outer_expn(&self, ctxt: SyntaxContext) -> ExpnId { + self.syntax_context_data[ctxt.0 as usize].outer_expn + } + + fn outer_mark(&self, ctxt: SyntaxContext) -> (ExpnId, Transparency) { + let data = &self.syntax_context_data[ctxt.0 as usize]; + (data.outer_expn, data.outer_transparency) + } + + fn parent_ctxt(&self, ctxt: SyntaxContext) -> SyntaxContext { + self.syntax_context_data[ctxt.0 as usize].parent + } + + fn remove_mark(&self, ctxt: &mut SyntaxContext) -> (ExpnId, Transparency) { + let outer_mark = self.outer_mark(*ctxt); + *ctxt = self.parent_ctxt(*ctxt); + outer_mark + } + + fn marks(&self, mut ctxt: SyntaxContext) -> Vec<(ExpnId, Transparency)> { + let mut marks = Vec::new(); + while ctxt != SyntaxContext::root() { + marks.push(self.outer_mark(ctxt)); + ctxt = self.parent_ctxt(ctxt); + } + marks.reverse(); + marks + } + + fn walk_chain(&self, mut span: Span, to: SyntaxContext) -> Span { + while span.from_expansion() && span.ctxt() != to { + span = self.expn_data(self.outer_expn(span.ctxt())).call_site; + } + span + } + + fn adjust(&self, ctxt: &mut SyntaxContext, expn_id: ExpnId) -> Option { + let mut scope = None; + while !self.is_descendant_of(expn_id, self.outer_expn(*ctxt)) { + scope = Some(self.remove_mark(ctxt).0); + } + scope + } + + fn apply_mark( + &mut self, + ctxt: SyntaxContext, + expn_id: ExpnId, + transparency: Transparency, + ) -> SyntaxContext { + assert_ne!(expn_id, ExpnId::root()); + if transparency == Transparency::Opaque { + return self.apply_mark_internal(ctxt, expn_id, transparency); + } + + let call_site_ctxt = self.expn_data(expn_id).call_site.ctxt(); + let mut call_site_ctxt = if transparency == Transparency::SemiTransparent { + self.modern(call_site_ctxt) + } else { + self.modern_and_legacy(call_site_ctxt) + }; + + if call_site_ctxt == SyntaxContext::root() { + return self.apply_mark_internal(ctxt, expn_id, transparency); + } + + // Otherwise, `expn_id` is a macros 1.0 definition and the call site is in a + // macros 2.0 expansion, i.e., a macros 1.0 invocation is in a macros 2.0 definition. + // + // In this case, the tokens from the macros 1.0 definition inherit the hygiene + // at their invocation. That is, we pretend that the macros 1.0 definition + // was defined at its invocation (i.e., inside the macros 2.0 definition) + // so that the macros 2.0 definition remains hygienic. + // + // See the example at `test/ui/hygiene/legacy_interaction.rs`. + for (expn_id, transparency) in self.marks(ctxt) { + call_site_ctxt = self.apply_mark_internal(call_site_ctxt, expn_id, transparency); + } + self.apply_mark_internal(call_site_ctxt, expn_id, transparency) + } + + fn apply_mark_internal( + &mut self, + ctxt: SyntaxContext, + expn_id: ExpnId, + transparency: Transparency, + ) -> SyntaxContext { + let syntax_context_data = &mut self.syntax_context_data; + let mut opaque = syntax_context_data[ctxt.0 as usize].opaque; + let mut opaque_and_semitransparent = + syntax_context_data[ctxt.0 as usize].opaque_and_semitransparent; + + if transparency >= Transparency::Opaque { + let parent = opaque; + opaque = *self + .syntax_context_map + .entry((parent, expn_id, transparency)) + .or_insert_with(|| { + let new_opaque = SyntaxContext(syntax_context_data.len() as u32); + syntax_context_data.push(SyntaxContextData { + outer_expn: expn_id, + outer_transparency: transparency, + parent, + opaque: new_opaque, + opaque_and_semitransparent: new_opaque, + dollar_crate_name: kw::DollarCrate, + }); + new_opaque + }); + } + + if transparency >= Transparency::SemiTransparent { + let parent = opaque_and_semitransparent; + opaque_and_semitransparent = *self + .syntax_context_map + .entry((parent, expn_id, transparency)) + .or_insert_with(|| { + let new_opaque_and_semitransparent = + SyntaxContext(syntax_context_data.len() as u32); + syntax_context_data.push(SyntaxContextData { + outer_expn: expn_id, + outer_transparency: transparency, + parent, + opaque, + opaque_and_semitransparent: new_opaque_and_semitransparent, + dollar_crate_name: kw::DollarCrate, + }); + new_opaque_and_semitransparent + }); + } + + let parent = ctxt; + *self.syntax_context_map.entry((parent, expn_id, transparency)).or_insert_with(|| { + let new_opaque_and_semitransparent_and_transparent = + SyntaxContext(syntax_context_data.len() as u32); + syntax_context_data.push(SyntaxContextData { + outer_expn: expn_id, + outer_transparency: transparency, + parent, + opaque, + opaque_and_semitransparent, + dollar_crate_name: kw::DollarCrate, + }); + new_opaque_and_semitransparent_and_transparent + }) + } +} + +pub fn clear_syntax_context_map() { + HygieneData::with(|data| data.syntax_context_map = FxHashMap::default()); +} + +pub fn walk_chain(span: Span, to: SyntaxContext) -> Span { + HygieneData::with(|data| data.walk_chain(span, to)) +} + +pub fn update_dollar_crate_names(mut get_name: impl FnMut(SyntaxContext) -> Symbol) { + // The new contexts that need updating are at the end of the list and have `$crate` as a name. + let (len, to_update) = HygieneData::with(|data| { + ( + data.syntax_context_data.len(), + data.syntax_context_data + .iter() + .rev() + .take_while(|scdata| scdata.dollar_crate_name == kw::DollarCrate) + .count(), + ) + }); + // The callback must be called from outside of the `HygieneData` lock, + // since it will try to acquire it too. + let range_to_update = len - to_update..len; + let names: Vec<_> = + range_to_update.clone().map(|idx| get_name(SyntaxContext::from_u32(idx as u32))).collect(); + HygieneData::with(|data| { + range_to_update.zip(names.into_iter()).for_each(|(idx, name)| { + data.syntax_context_data[idx].dollar_crate_name = name; + }) + }) +} + +pub fn debug_hygiene_data(verbose: bool) -> String { + HygieneData::with(|data| { + if verbose { + format!("{:#?}", data) + } else { + let mut s = String::from(""); + s.push_str("Expansions:"); + data.expn_data.iter().enumerate().for_each(|(id, expn_info)| { + let expn_info = expn_info.as_ref().expect("no expansion data for an expansion ID"); + s.push_str(&format!( + "\n{}: parent: {:?}, call_site_ctxt: {:?}, kind: {:?}", + id, + expn_info.parent, + expn_info.call_site.ctxt(), + expn_info.kind, + )); + }); + s.push_str("\n\nSyntaxContexts:"); + data.syntax_context_data.iter().enumerate().for_each(|(id, ctxt)| { + s.push_str(&format!( + "\n#{}: parent: {:?}, outer_mark: ({:?}, {:?})", + id, ctxt.parent, ctxt.outer_expn, ctxt.outer_transparency, + )); + }); + s + } + }) +} + +impl SyntaxContext { + #[inline] + pub const fn root() -> Self { + SyntaxContext(0) + } + + #[inline] + crate fn as_u32(self) -> u32 { + self.0 + } + + #[inline] + crate fn from_u32(raw: u32) -> SyntaxContext { + SyntaxContext(raw) + } + + /// Extend a syntax context with a given expansion and transparency. + crate fn apply_mark(self, expn_id: ExpnId, transparency: Transparency) -> SyntaxContext { + HygieneData::with(|data| data.apply_mark(self, expn_id, transparency)) + } + + /// Pulls a single mark off of the syntax context. This effectively moves the + /// context up one macro definition level. That is, if we have a nested macro + /// definition as follows: + /// + /// ```rust + /// macro_rules! f { + /// macro_rules! g { + /// ... + /// } + /// } + /// ``` + /// + /// and we have a SyntaxContext that is referring to something declared by an invocation + /// of g (call it g1), calling remove_mark will result in the SyntaxContext for the + /// invocation of f that created g1. + /// Returns the mark that was removed. + pub fn remove_mark(&mut self) -> ExpnId { + HygieneData::with(|data| data.remove_mark(self).0) + } + + pub fn marks(self) -> Vec<(ExpnId, Transparency)> { + HygieneData::with(|data| data.marks(self)) + } + + /// Adjust this context for resolution in a scope created by the given expansion. + /// For example, consider the following three resolutions of `f`: + /// + /// ```rust + /// mod foo { pub fn f() {} } // `f`'s `SyntaxContext` is empty. + /// m!(f); + /// macro m($f:ident) { + /// mod bar { + /// pub fn f() {} // `f`'s `SyntaxContext` has a single `ExpnId` from `m`. + /// pub fn $f() {} // `$f`'s `SyntaxContext` is empty. + /// } + /// foo::f(); // `f`'s `SyntaxContext` has a single `ExpnId` from `m` + /// //^ Since `mod foo` is outside this expansion, `adjust` removes the mark from `f`, + /// //| and it resolves to `::foo::f`. + /// bar::f(); // `f`'s `SyntaxContext` has a single `ExpnId` from `m` + /// //^ Since `mod bar` not outside this expansion, `adjust` does not change `f`, + /// //| and it resolves to `::bar::f`. + /// bar::$f(); // `f`'s `SyntaxContext` is empty. + /// //^ Since `mod bar` is not outside this expansion, `adjust` does not change `$f`, + /// //| and it resolves to `::bar::$f`. + /// } + /// ``` + /// This returns the expansion whose definition scope we use to privacy check the resolution, + /// or `None` if we privacy check as usual (i.e., not w.r.t. a macro definition scope). + pub fn adjust(&mut self, expn_id: ExpnId) -> Option { + HygieneData::with(|data| data.adjust(self, expn_id)) + } + + /// Like `SyntaxContext::adjust`, but also modernizes `self`. + pub fn modernize_and_adjust(&mut self, expn_id: ExpnId) -> Option { + HygieneData::with(|data| { + *self = data.modern(*self); + data.adjust(self, expn_id) + }) + } + + /// Adjust this context for resolution in a scope created by the given expansion + /// via a glob import with the given `SyntaxContext`. + /// For example: + /// + /// ```rust + /// m!(f); + /// macro m($i:ident) { + /// mod foo { + /// pub fn f() {} // `f`'s `SyntaxContext` has a single `ExpnId` from `m`. + /// pub fn $i() {} // `$i`'s `SyntaxContext` is empty. + /// } + /// n(f); + /// macro n($j:ident) { + /// use foo::*; + /// f(); // `f`'s `SyntaxContext` has a mark from `m` and a mark from `n` + /// //^ `glob_adjust` removes the mark from `n`, so this resolves to `foo::f`. + /// $i(); // `$i`'s `SyntaxContext` has a mark from `n` + /// //^ `glob_adjust` removes the mark from `n`, so this resolves to `foo::$i`. + /// $j(); // `$j`'s `SyntaxContext` has a mark from `m` + /// //^ This cannot be glob-adjusted, so this is a resolution error. + /// } + /// } + /// ``` + /// This returns `None` if the context cannot be glob-adjusted. + /// Otherwise, it returns the scope to use when privacy checking (see `adjust` for details). + pub fn glob_adjust(&mut self, expn_id: ExpnId, glob_span: Span) -> Option> { + HygieneData::with(|data| { + let mut scope = None; + let mut glob_ctxt = data.modern(glob_span.ctxt()); + while !data.is_descendant_of(expn_id, data.outer_expn(glob_ctxt)) { + scope = Some(data.remove_mark(&mut glob_ctxt).0); + if data.remove_mark(self).0 != scope.unwrap() { + return None; + } + } + if data.adjust(self, expn_id).is_some() { + return None; + } + Some(scope) + }) + } + + /// Undo `glob_adjust` if possible: + /// + /// ```rust + /// if let Some(privacy_checking_scope) = self.reverse_glob_adjust(expansion, glob_ctxt) { + /// assert!(self.glob_adjust(expansion, glob_ctxt) == Some(privacy_checking_scope)); + /// } + /// ``` + pub fn reverse_glob_adjust( + &mut self, + expn_id: ExpnId, + glob_span: Span, + ) -> Option> { + HygieneData::with(|data| { + if data.adjust(self, expn_id).is_some() { + return None; + } + + let mut glob_ctxt = data.modern(glob_span.ctxt()); + let mut marks = Vec::new(); + while !data.is_descendant_of(expn_id, data.outer_expn(glob_ctxt)) { + marks.push(data.remove_mark(&mut glob_ctxt)); + } + + let scope = marks.last().map(|mark| mark.0); + while let Some((expn_id, transparency)) = marks.pop() { + *self = data.apply_mark(*self, expn_id, transparency); + } + Some(scope) + }) + } + + pub fn hygienic_eq(self, other: SyntaxContext, expn_id: ExpnId) -> bool { + HygieneData::with(|data| { + let mut self_modern = data.modern(self); + data.adjust(&mut self_modern, expn_id); + self_modern == data.modern(other) + }) + } + + #[inline] + pub fn modern(self) -> SyntaxContext { + HygieneData::with(|data| data.modern(self)) + } + + #[inline] + pub fn modern_and_legacy(self) -> SyntaxContext { + HygieneData::with(|data| data.modern_and_legacy(self)) + } + + #[inline] + pub fn outer_expn(self) -> ExpnId { + HygieneData::with(|data| data.outer_expn(self)) + } + + /// `ctxt.outer_expn_data()` is equivalent to but faster than + /// `ctxt.outer_expn().expn_data()`. + #[inline] + pub fn outer_expn_data(self) -> ExpnData { + HygieneData::with(|data| data.expn_data(data.outer_expn(self)).clone()) + } + + #[inline] + pub fn outer_mark_with_data(self) -> (ExpnId, Transparency, ExpnData) { + HygieneData::with(|data| { + let (expn_id, transparency) = data.outer_mark(self); + (expn_id, transparency, data.expn_data(expn_id).clone()) + }) + } + + pub fn dollar_crate_name(self) -> Symbol { + HygieneData::with(|data| data.syntax_context_data[self.0 as usize].dollar_crate_name) + } +} + +impl fmt::Debug for SyntaxContext { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "#{}", self.0) + } +} + +impl Span { + /// Creates a fresh expansion with given properties. + /// Expansions are normally created by macros, but in some cases expansions are created for + /// other compiler-generated code to set per-span properties like allowed unstable features. + /// The returned span belongs to the created expansion and has the new properties, + /// but its location is inherited from the current span. + pub fn fresh_expansion(self, expn_data: ExpnData) -> Span { + self.fresh_expansion_with_transparency(expn_data, Transparency::Transparent) + } + + pub fn fresh_expansion_with_transparency( + self, + expn_data: ExpnData, + transparency: Transparency, + ) -> Span { + HygieneData::with(|data| { + let expn_id = data.fresh_expn(Some(expn_data)); + self.with_ctxt(data.apply_mark(SyntaxContext::root(), expn_id, transparency)) + }) + } +} + +/// A subset of properties from both macro definition and macro call available through global data. +/// Avoid using this if you have access to the original definition or call structures. +#[derive(Clone, Debug, RustcEncodable, RustcDecodable, HashStable_Generic)] +pub struct ExpnData { + // --- The part unique to each expansion. + /// The kind of this expansion - macro or compiler desugaring. + pub kind: ExpnKind, + /// The expansion that produced this expansion. + #[stable_hasher(ignore)] + pub parent: ExpnId, + /// The location of the actual macro invocation or syntax sugar , e.g. + /// `let x = foo!();` or `if let Some(y) = x {}` + /// + /// This may recursively refer to other macro invocations, e.g., if + /// `foo!()` invoked `bar!()` internally, and there was an + /// expression inside `bar!`; the call_site of the expression in + /// the expansion would point to the `bar!` invocation; that + /// call_site span would have its own ExpnData, with the call_site + /// pointing to the `foo!` invocation. + pub call_site: Span, + + // --- The part specific to the macro/desugaring definition. + // --- It may be reasonable to share this part between expansions with the same definition, + // --- but such sharing is known to bring some minor inconveniences without also bringing + // --- noticeable perf improvements (PR #62898). + /// The span of the macro definition (possibly dummy). + /// This span serves only informational purpose and is not used for resolution. + pub def_site: Span, + /// List of #[unstable]/feature-gated features that the macro is allowed to use + /// internally without forcing the whole crate to opt-in + /// to them. + pub allow_internal_unstable: Option>, + /// Whether the macro is allowed to use `unsafe` internally + /// even if the user crate has `#![forbid(unsafe_code)]`. + pub allow_internal_unsafe: bool, + /// Enables the macro helper hack (`ident!(...)` -> `$crate::ident!(...)`) + /// for a given macro. + pub local_inner_macros: bool, + /// Edition of the crate in which the macro is defined. + pub edition: Edition, +} + +impl ExpnData { + /// Constructs expansion data with default properties. + pub fn default(kind: ExpnKind, call_site: Span, edition: Edition) -> ExpnData { + ExpnData { + kind, + parent: ExpnId::root(), + call_site, + def_site: DUMMY_SP, + allow_internal_unstable: None, + allow_internal_unsafe: false, + local_inner_macros: false, + edition, + } + } + + pub fn allow_unstable( + kind: ExpnKind, + call_site: Span, + edition: Edition, + allow_internal_unstable: Lrc<[Symbol]>, + ) -> ExpnData { + ExpnData { + allow_internal_unstable: Some(allow_internal_unstable), + ..ExpnData::default(kind, call_site, edition) + } + } + + #[inline] + pub fn is_root(&self) -> bool { + if let ExpnKind::Root = self.kind { true } else { false } + } +} + +/// Expansion kind. +#[derive(Clone, Debug, RustcEncodable, RustcDecodable, HashStable_Generic)] +pub enum ExpnKind { + /// No expansion, aka root expansion. Only `ExpnId::root()` has this kind. + Root, + /// Expansion produced by a macro. + Macro(MacroKind, Symbol), + /// Transform done by the compiler on the AST. + AstPass(AstPass), + /// Desugaring done by the compiler during HIR lowering. + Desugaring(DesugaringKind), +} + +impl ExpnKind { + pub fn descr(&self) -> Symbol { + match *self { + ExpnKind::Root => kw::PathRoot, + ExpnKind::Macro(_, descr) => descr, + ExpnKind::AstPass(kind) => Symbol::intern(kind.descr()), + ExpnKind::Desugaring(kind) => Symbol::intern(kind.descr()), + } + } +} + +/// The kind of macro invocation or definition. +#[derive( + Clone, + Copy, + PartialEq, + Eq, + RustcEncodable, + RustcDecodable, + Hash, + Debug, + HashStable_Generic +)] +pub enum MacroKind { + /// A bang macro `foo!()`. + Bang, + /// An attribute macro `#[foo]`. + Attr, + /// A derive macro `#[derive(Foo)]` + Derive, +} + +impl MacroKind { + pub fn descr(self) -> &'static str { + match self { + MacroKind::Bang => "macro", + MacroKind::Attr => "attribute macro", + MacroKind::Derive => "derive macro", + } + } + + pub fn descr_expected(self) -> &'static str { + match self { + MacroKind::Attr => "attribute", + _ => self.descr(), + } + } + + pub fn article(self) -> &'static str { + match self { + MacroKind::Attr => "an", + _ => "a", + } + } +} + +/// The kind of AST transform. +#[derive(Clone, Copy, PartialEq, Debug, RustcEncodable, RustcDecodable, HashStable_Generic)] +pub enum AstPass { + StdImports, + TestHarness, + ProcMacroHarness, +} + +impl AstPass { + fn descr(self) -> &'static str { + match self { + AstPass::StdImports => "standard library imports", + AstPass::TestHarness => "test harness", + AstPass::ProcMacroHarness => "proc macro harness", + } + } +} + +/// The kind of compiler desugaring. +#[derive(Clone, Copy, PartialEq, Debug, RustcEncodable, RustcDecodable, HashStable_Generic)] +pub enum DesugaringKind { + /// We desugar `if c { i } else { e }` to `match $ExprKind::Use(c) { true => i, _ => e }`. + /// However, we do not want to blame `c` for unreachability but rather say that `i` + /// is unreachable. This desugaring kind allows us to avoid blaming `c`. + /// This also applies to `while` loops. + CondTemporary, + QuestionMark, + TryBlock, + /// Desugaring of an `impl Trait` in return type position + /// to an `type Foo = impl Trait;` and replacing the + /// `impl Trait` with `Foo`. + OpaqueTy, + Async, + Await, + ForLoop, +} + +impl DesugaringKind { + /// The description wording should combine well with "desugaring of {}". + fn descr(self) -> &'static str { + match self { + DesugaringKind::CondTemporary => "`if` or `while` condition", + DesugaringKind::Async => "`async` block or function", + DesugaringKind::Await => "`await` expression", + DesugaringKind::QuestionMark => "operator `?`", + DesugaringKind::TryBlock => "`try` block", + DesugaringKind::OpaqueTy => "`impl Trait`", + DesugaringKind::ForLoop => "`for` loop", + } + } +} + +impl Encodable for ExpnId { + fn encode(&self, _: &mut E) -> Result<(), E::Error> { + Ok(()) // FIXME(jseyfried) intercrate hygiene + } +} + +impl Decodable for ExpnId { + fn decode(_: &mut D) -> Result { + Ok(ExpnId::root()) // FIXME(jseyfried) intercrate hygiene + } +} diff --git a/src/librustc_span/lib.rs b/src/librustc_span/lib.rs new file mode 100644 index 00000000000..a58c12f2350 --- /dev/null +++ b/src/librustc_span/lib.rs @@ -0,0 +1,1672 @@ +//! The source positions and related helper functions. +//! +//! ## Note +//! +//! This API is completely unstable and subject to change. + +#![doc(html_root_url = "https://doc.rust-lang.org/nightly/")] +#![feature(const_fn)] +#![feature(crate_visibility_modifier)] +#![feature(nll)] +#![feature(optin_builtin_traits)] +#![feature(rustc_attrs)] +#![feature(specialization)] +#![feature(step_trait)] + +use rustc_data_structures::AtomicRef; +use rustc_macros::HashStable_Generic; +use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; + +mod caching_source_map_view; +pub mod source_map; +pub use self::caching_source_map_view::CachingSourceMapView; + +pub mod edition; +use edition::Edition; +pub mod hygiene; +use hygiene::Transparency; +pub use hygiene::{DesugaringKind, ExpnData, ExpnId, ExpnKind, MacroKind, SyntaxContext}; + +mod span_encoding; +pub use span_encoding::{Span, DUMMY_SP}; + +pub mod symbol; +pub use symbol::{sym, Symbol}; + +mod analyze_source_file; +pub mod fatal_error; + +use rustc_data_structures::fingerprint::Fingerprint; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_data_structures::sync::{Lock, Lrc}; + +use std::borrow::Cow; +use std::cell::RefCell; +use std::cmp::{self, Ordering}; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::ops::{Add, Sub}; +use std::path::PathBuf; + +#[cfg(test)] +mod tests; + +pub struct Globals { + symbol_interner: Lock, + span_interner: Lock, + hygiene_data: Lock, +} + +impl Globals { + pub fn new(edition: Edition) -> Globals { + Globals { + symbol_interner: Lock::new(symbol::Interner::fresh()), + span_interner: Lock::new(span_encoding::SpanInterner::default()), + hygiene_data: Lock::new(hygiene::HygieneData::new(edition)), + } + } +} + +scoped_tls::scoped_thread_local!(pub static GLOBALS: Globals); + +/// Differentiates between real files and common virtual files. +#[derive( + Debug, + Eq, + PartialEq, + Clone, + Ord, + PartialOrd, + Hash, + RustcDecodable, + RustcEncodable, + HashStable_Generic +)] +pub enum FileName { + Real(PathBuf), + /// A macro. This includes the full name of the macro, so that there are no clashes. + Macros(String), + /// Call to `quote!`. + QuoteExpansion(u64), + /// Command line. + Anon(u64), + /// Hack in `src/libsyntax/parse.rs`. + // FIXME(jseyfried) + MacroExpansion(u64), + ProcMacroSourceCode(u64), + /// Strings provided as `--cfg [cfgspec]` stored in a `crate_cfg`. + CfgSpec(u64), + /// Strings provided as crate attributes in the CLI. + CliCrateAttr(u64), + /// Custom sources for explicit parser calls from plugins and drivers. + Custom(String), + DocTest(PathBuf, isize), +} + +impl std::fmt::Display for FileName { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use FileName::*; + match *self { + Real(ref path) => write!(fmt, "{}", path.display()), + Macros(ref name) => write!(fmt, "<{} macros>", name), + QuoteExpansion(_) => write!(fmt, ""), + MacroExpansion(_) => write!(fmt, ""), + Anon(_) => write!(fmt, ""), + ProcMacroSourceCode(_) => write!(fmt, ""), + CfgSpec(_) => write!(fmt, ""), + CliCrateAttr(_) => write!(fmt, ""), + Custom(ref s) => write!(fmt, "<{}>", s), + DocTest(ref path, _) => write!(fmt, "{}", path.display()), + } + } +} + +impl From for FileName { + fn from(p: PathBuf) -> Self { + assert!(!p.to_string_lossy().ends_with('>')); + FileName::Real(p) + } +} + +impl FileName { + pub fn is_real(&self) -> bool { + use FileName::*; + match *self { + Real(_) => true, + Macros(_) + | Anon(_) + | MacroExpansion(_) + | ProcMacroSourceCode(_) + | CfgSpec(_) + | CliCrateAttr(_) + | Custom(_) + | QuoteExpansion(_) + | DocTest(_, _) => false, + } + } + + pub fn is_macros(&self) -> bool { + use FileName::*; + match *self { + Real(_) + | Anon(_) + | MacroExpansion(_) + | ProcMacroSourceCode(_) + | CfgSpec(_) + | CliCrateAttr(_) + | Custom(_) + | QuoteExpansion(_) + | DocTest(_, _) => false, + Macros(_) => true, + } + } + + pub fn quote_expansion_source_code(src: &str) -> FileName { + let mut hasher = StableHasher::new(); + src.hash(&mut hasher); + FileName::QuoteExpansion(hasher.finish()) + } + + pub fn macro_expansion_source_code(src: &str) -> FileName { + let mut hasher = StableHasher::new(); + src.hash(&mut hasher); + FileName::MacroExpansion(hasher.finish()) + } + + pub fn anon_source_code(src: &str) -> FileName { + let mut hasher = StableHasher::new(); + src.hash(&mut hasher); + FileName::Anon(hasher.finish()) + } + + pub fn proc_macro_source_code(src: &str) -> FileName { + let mut hasher = StableHasher::new(); + src.hash(&mut hasher); + FileName::ProcMacroSourceCode(hasher.finish()) + } + + pub fn cfg_spec_source_code(src: &str) -> FileName { + let mut hasher = StableHasher::new(); + src.hash(&mut hasher); + FileName::QuoteExpansion(hasher.finish()) + } + + pub fn cli_crate_attr_source_code(src: &str) -> FileName { + let mut hasher = StableHasher::new(); + src.hash(&mut hasher); + FileName::CliCrateAttr(hasher.finish()) + } + + pub fn doc_test_source_code(path: PathBuf, line: isize) -> FileName { + FileName::DocTest(path, line) + } +} + +/// Spans represent a region of code, used for error reporting. Positions in spans +/// are *absolute* positions from the beginning of the source_map, not positions +/// relative to `SourceFile`s. Methods on the `SourceMap` can be used to relate spans back +/// to the original source. +/// You must be careful if the span crosses more than one file - you will not be +/// able to use many of the functions on spans in source_map and you cannot assume +/// that the length of the `span = hi - lo`; there may be space in the `BytePos` +/// range between files. +/// +/// `SpanData` is public because `Span` uses a thread-local interner and can't be +/// sent to other threads, but some pieces of performance infra run in a separate thread. +/// Using `Span` is generally preferred. +#[derive(Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd)] +pub struct SpanData { + pub lo: BytePos, + pub hi: BytePos, + /// Information about where the macro came from, if this piece of + /// code was created by a macro expansion. + pub ctxt: SyntaxContext, +} + +impl SpanData { + #[inline] + pub fn with_lo(&self, lo: BytePos) -> Span { + Span::new(lo, self.hi, self.ctxt) + } + #[inline] + pub fn with_hi(&self, hi: BytePos) -> Span { + Span::new(self.lo, hi, self.ctxt) + } + #[inline] + pub fn with_ctxt(&self, ctxt: SyntaxContext) -> Span { + Span::new(self.lo, self.hi, ctxt) + } +} + +// The interner is pointed to by a thread local value which is only set on the main thread +// with parallelization is disabled. So we don't allow `Span` to transfer between threads +// to avoid panics and other errors, even though it would be memory safe to do so. +#[cfg(not(parallel_compiler))] +impl !Send for Span {} +#[cfg(not(parallel_compiler))] +impl !Sync for Span {} + +impl PartialOrd for Span { + fn partial_cmp(&self, rhs: &Self) -> Option { + PartialOrd::partial_cmp(&self.data(), &rhs.data()) + } +} +impl Ord for Span { + fn cmp(&self, rhs: &Self) -> Ordering { + Ord::cmp(&self.data(), &rhs.data()) + } +} + +/// A collection of spans. Spans have two orthogonal attributes: +/// +/// - They can be *primary spans*. In this case they are the locus of +/// the error, and would be rendered with `^^^`. +/// - They can have a *label*. In this case, the label is written next +/// to the mark in the snippet when we render. +#[derive(Clone, Debug, Hash, PartialEq, Eq, RustcEncodable, RustcDecodable)] +pub struct MultiSpan { + primary_spans: Vec, + span_labels: Vec<(Span, String)>, +} + +impl Span { + #[inline] + pub fn lo(self) -> BytePos { + self.data().lo + } + #[inline] + pub fn with_lo(self, lo: BytePos) -> Span { + self.data().with_lo(lo) + } + #[inline] + pub fn hi(self) -> BytePos { + self.data().hi + } + #[inline] + pub fn with_hi(self, hi: BytePos) -> Span { + self.data().with_hi(hi) + } + #[inline] + pub fn ctxt(self) -> SyntaxContext { + self.data().ctxt + } + #[inline] + pub fn with_ctxt(self, ctxt: SyntaxContext) -> Span { + self.data().with_ctxt(ctxt) + } + + /// Returns `true` if this is a dummy span with any hygienic context. + #[inline] + pub fn is_dummy(self) -> bool { + let span = self.data(); + span.lo.0 == 0 && span.hi.0 == 0 + } + + /// Returns `true` if this span comes from a macro or desugaring. + #[inline] + pub fn from_expansion(self) -> bool { + self.ctxt() != SyntaxContext::root() + } + + #[inline] + pub fn with_root_ctxt(lo: BytePos, hi: BytePos) -> Span { + Span::new(lo, hi, SyntaxContext::root()) + } + + /// Returns a new span representing an empty span at the beginning of this span + #[inline] + pub fn shrink_to_lo(self) -> Span { + let span = self.data(); + span.with_hi(span.lo) + } + /// Returns a new span representing an empty span at the end of this span. + #[inline] + pub fn shrink_to_hi(self) -> Span { + let span = self.data(); + span.with_lo(span.hi) + } + + /// Returns `self` if `self` is not the dummy span, and `other` otherwise. + pub fn substitute_dummy(self, other: Span) -> Span { + if self.is_dummy() { other } else { self } + } + + /// Returns `true` if `self` fully encloses `other`. + pub fn contains(self, other: Span) -> bool { + let span = self.data(); + let other = other.data(); + span.lo <= other.lo && other.hi <= span.hi + } + + /// Returns `true` if `self` touches `other`. + pub fn overlaps(self, other: Span) -> bool { + let span = self.data(); + let other = other.data(); + span.lo < other.hi && other.lo < span.hi + } + + /// Returns `true` if the spans are equal with regards to the source text. + /// + /// Use this instead of `==` when either span could be generated code, + /// and you only care that they point to the same bytes of source text. + pub fn source_equal(&self, other: &Span) -> bool { + let span = self.data(); + let other = other.data(); + span.lo == other.lo && span.hi == other.hi + } + + /// Returns `Some(span)`, where the start is trimmed by the end of `other`. + pub fn trim_start(self, other: Span) -> Option { + let span = self.data(); + let other = other.data(); + if span.hi > other.hi { Some(span.with_lo(cmp::max(span.lo, other.hi))) } else { None } + } + + /// Returns the source span -- this is either the supplied span, or the span for + /// the macro callsite that expanded to it. + pub fn source_callsite(self) -> Span { + let expn_data = self.ctxt().outer_expn_data(); + if !expn_data.is_root() { expn_data.call_site.source_callsite() } else { self } + } + + /// The `Span` for the tokens in the previous macro expansion from which `self` was generated, + /// if any. + pub fn parent(self) -> Option { + let expn_data = self.ctxt().outer_expn_data(); + if !expn_data.is_root() { Some(expn_data.call_site) } else { None } + } + + /// Edition of the crate from which this span came. + pub fn edition(self) -> edition::Edition { + self.ctxt().outer_expn_data().edition + } + + #[inline] + pub fn rust_2015(&self) -> bool { + self.edition() == edition::Edition::Edition2015 + } + + #[inline] + pub fn rust_2018(&self) -> bool { + self.edition() >= edition::Edition::Edition2018 + } + + /// Returns the source callee. + /// + /// Returns `None` if the supplied span has no expansion trace, + /// else returns the `ExpnData` for the macro definition + /// corresponding to the source callsite. + pub fn source_callee(self) -> Option { + fn source_callee(expn_data: ExpnData) -> ExpnData { + let next_expn_data = expn_data.call_site.ctxt().outer_expn_data(); + if !next_expn_data.is_root() { source_callee(next_expn_data) } else { expn_data } + } + let expn_data = self.ctxt().outer_expn_data(); + if !expn_data.is_root() { Some(source_callee(expn_data)) } else { None } + } + + /// Checks if a span is "internal" to a macro in which `#[unstable]` + /// items can be used (that is, a macro marked with + /// `#[allow_internal_unstable]`). + pub fn allows_unstable(&self, feature: Symbol) -> bool { + self.ctxt().outer_expn_data().allow_internal_unstable.map_or(false, |features| { + features + .iter() + .any(|&f| f == feature || f == sym::allow_internal_unstable_backcompat_hack) + }) + } + + /// Checks if this span arises from a compiler desugaring of kind `kind`. + pub fn is_desugaring(&self, kind: DesugaringKind) -> bool { + match self.ctxt().outer_expn_data().kind { + ExpnKind::Desugaring(k) => k == kind, + _ => false, + } + } + + /// Returns the compiler desugaring that created this span, or `None` + /// if this span is not from a desugaring. + pub fn desugaring_kind(&self) -> Option { + match self.ctxt().outer_expn_data().kind { + ExpnKind::Desugaring(k) => Some(k), + _ => None, + } + } + + /// Checks if a span is "internal" to a macro in which `unsafe` + /// can be used without triggering the `unsafe_code` lint + // (that is, a macro marked with `#[allow_internal_unsafe]`). + pub fn allows_unsafe(&self) -> bool { + self.ctxt().outer_expn_data().allow_internal_unsafe + } + + pub fn macro_backtrace(mut self) -> Vec { + let mut prev_span = DUMMY_SP; + let mut result = vec![]; + loop { + let expn_data = self.ctxt().outer_expn_data(); + if expn_data.is_root() { + break; + } + // Don't print recursive invocations. + if !expn_data.call_site.source_equal(&prev_span) { + let (pre, post) = match expn_data.kind { + ExpnKind::Root => break, + ExpnKind::Desugaring(..) => ("desugaring of ", ""), + ExpnKind::AstPass(..) => ("", ""), + ExpnKind::Macro(macro_kind, _) => match macro_kind { + MacroKind::Bang => ("", "!"), + MacroKind::Attr => ("#[", "]"), + MacroKind::Derive => ("#[derive(", ")]"), + }, + }; + result.push(MacroBacktrace { + call_site: expn_data.call_site, + macro_decl_name: format!("{}{}{}", pre, expn_data.kind.descr(), post), + def_site_span: expn_data.def_site, + }); + } + + prev_span = self; + self = expn_data.call_site; + } + result + } + + /// Returns a `Span` that would enclose both `self` and `end`. + pub fn to(self, end: Span) -> Span { + let span_data = self.data(); + let end_data = end.data(); + // FIXME(jseyfried): `self.ctxt` should always equal `end.ctxt` here (cf. issue #23480). + // Return the macro span on its own to avoid weird diagnostic output. It is preferable to + // have an incomplete span than a completely nonsensical one. + if span_data.ctxt != end_data.ctxt { + if span_data.ctxt == SyntaxContext::root() { + return end; + } else if end_data.ctxt == SyntaxContext::root() { + return self; + } + // Both spans fall within a macro. + // FIXME(estebank): check if it is the *same* macro. + } + Span::new( + cmp::min(span_data.lo, end_data.lo), + cmp::max(span_data.hi, end_data.hi), + if span_data.ctxt == SyntaxContext::root() { end_data.ctxt } else { span_data.ctxt }, + ) + } + + /// Returns a `Span` between the end of `self` to the beginning of `end`. + pub fn between(self, end: Span) -> Span { + let span = self.data(); + let end = end.data(); + Span::new( + span.hi, + end.lo, + if end.ctxt == SyntaxContext::root() { end.ctxt } else { span.ctxt }, + ) + } + + /// Returns a `Span` between the beginning of `self` to the beginning of `end`. + pub fn until(self, end: Span) -> Span { + let span = self.data(); + let end = end.data(); + Span::new( + span.lo, + end.lo, + if end.ctxt == SyntaxContext::root() { end.ctxt } else { span.ctxt }, + ) + } + + pub fn from_inner(self, inner: InnerSpan) -> Span { + let span = self.data(); + Span::new( + span.lo + BytePos::from_usize(inner.start), + span.lo + BytePos::from_usize(inner.end), + span.ctxt, + ) + } + + /// Equivalent of `Span::def_site` from the proc macro API, + /// except that the location is taken from the `self` span. + pub fn with_def_site_ctxt(self, expn_id: ExpnId) -> Span { + self.with_ctxt_from_mark(expn_id, Transparency::Opaque) + } + + /// Equivalent of `Span::call_site` from the proc macro API, + /// except that the location is taken from the `self` span. + pub fn with_call_site_ctxt(&self, expn_id: ExpnId) -> Span { + self.with_ctxt_from_mark(expn_id, Transparency::Transparent) + } + + /// Equivalent of `Span::mixed_site` from the proc macro API, + /// except that the location is taken from the `self` span. + pub fn with_mixed_site_ctxt(&self, expn_id: ExpnId) -> Span { + self.with_ctxt_from_mark(expn_id, Transparency::SemiTransparent) + } + + /// Produces a span with the same location as `self` and context produced by a macro with the + /// given ID and transparency, assuming that macro was defined directly and not produced by + /// some other macro (which is the case for built-in and procedural macros). + pub fn with_ctxt_from_mark(self, expn_id: ExpnId, transparency: Transparency) -> Span { + self.with_ctxt(SyntaxContext::root().apply_mark(expn_id, transparency)) + } + + #[inline] + pub fn apply_mark(self, expn_id: ExpnId, transparency: Transparency) -> Span { + let span = self.data(); + span.with_ctxt(span.ctxt.apply_mark(expn_id, transparency)) + } + + #[inline] + pub fn remove_mark(&mut self) -> ExpnId { + let mut span = self.data(); + let mark = span.ctxt.remove_mark(); + *self = Span::new(span.lo, span.hi, span.ctxt); + mark + } + + #[inline] + pub fn adjust(&mut self, expn_id: ExpnId) -> Option { + let mut span = self.data(); + let mark = span.ctxt.adjust(expn_id); + *self = Span::new(span.lo, span.hi, span.ctxt); + mark + } + + #[inline] + pub fn modernize_and_adjust(&mut self, expn_id: ExpnId) -> Option { + let mut span = self.data(); + let mark = span.ctxt.modernize_and_adjust(expn_id); + *self = Span::new(span.lo, span.hi, span.ctxt); + mark + } + + #[inline] + pub fn glob_adjust(&mut self, expn_id: ExpnId, glob_span: Span) -> Option> { + let mut span = self.data(); + let mark = span.ctxt.glob_adjust(expn_id, glob_span); + *self = Span::new(span.lo, span.hi, span.ctxt); + mark + } + + #[inline] + pub fn reverse_glob_adjust( + &mut self, + expn_id: ExpnId, + glob_span: Span, + ) -> Option> { + let mut span = self.data(); + let mark = span.ctxt.reverse_glob_adjust(expn_id, glob_span); + *self = Span::new(span.lo, span.hi, span.ctxt); + mark + } + + #[inline] + pub fn modern(self) -> Span { + let span = self.data(); + span.with_ctxt(span.ctxt.modern()) + } + + #[inline] + pub fn modern_and_legacy(self) -> Span { + let span = self.data(); + span.with_ctxt(span.ctxt.modern_and_legacy()) + } +} + +#[derive(Clone, Debug)] +pub struct SpanLabel { + /// The span we are going to include in the final snippet. + pub span: Span, + + /// Is this a primary span? This is the "locus" of the message, + /// and is indicated with a `^^^^` underline, versus `----`. + pub is_primary: bool, + + /// What label should we attach to this span (if any)? + pub label: Option, +} + +impl Default for Span { + fn default() -> Self { + DUMMY_SP + } +} + +impl rustc_serialize::UseSpecializedEncodable for Span { + fn default_encode(&self, s: &mut S) -> Result<(), S::Error> { + let span = self.data(); + s.emit_struct("Span", 2, |s| { + s.emit_struct_field("lo", 0, |s| span.lo.encode(s))?; + + s.emit_struct_field("hi", 1, |s| span.hi.encode(s)) + }) + } +} + +impl rustc_serialize::UseSpecializedDecodable for Span { + fn default_decode(d: &mut D) -> Result { + d.read_struct("Span", 2, |d| { + let lo = d.read_struct_field("lo", 0, Decodable::decode)?; + let hi = d.read_struct_field("hi", 1, Decodable::decode)?; + Ok(Span::with_root_ctxt(lo, hi)) + }) + } +} + +pub fn default_span_debug(span: Span, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Span") + .field("lo", &span.lo()) + .field("hi", &span.hi()) + .field("ctxt", &span.ctxt()) + .finish() +} + +impl fmt::Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (*SPAN_DEBUG)(*self, f) + } +} + +impl fmt::Debug for SpanData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (*SPAN_DEBUG)(Span::new(self.lo, self.hi, self.ctxt), f) + } +} + +impl MultiSpan { + #[inline] + pub fn new() -> MultiSpan { + MultiSpan { primary_spans: vec![], span_labels: vec![] } + } + + pub fn from_span(primary_span: Span) -> MultiSpan { + MultiSpan { primary_spans: vec![primary_span], span_labels: vec![] } + } + + pub fn from_spans(vec: Vec) -> MultiSpan { + MultiSpan { primary_spans: vec, span_labels: vec![] } + } + + pub fn push_span_label(&mut self, span: Span, label: String) { + self.span_labels.push((span, label)); + } + + /// Selects the first primary span (if any). + pub fn primary_span(&self) -> Option { + self.primary_spans.first().cloned() + } + + /// Returns all primary spans. + pub fn primary_spans(&self) -> &[Span] { + &self.primary_spans + } + + /// Returns `true` if any of the primary spans are displayable. + pub fn has_primary_spans(&self) -> bool { + self.primary_spans.iter().any(|sp| !sp.is_dummy()) + } + + /// Returns `true` if this contains only a dummy primary span with any hygienic context. + pub fn is_dummy(&self) -> bool { + let mut is_dummy = true; + for span in &self.primary_spans { + if !span.is_dummy() { + is_dummy = false; + } + } + is_dummy + } + + /// Replaces all occurrences of one Span with another. Used to move `Span`s in areas that don't + /// display well (like std macros). Returns whether replacements occurred. + pub fn replace(&mut self, before: Span, after: Span) -> bool { + let mut replacements_occurred = false; + for primary_span in &mut self.primary_spans { + if *primary_span == before { + *primary_span = after; + replacements_occurred = true; + } + } + for span_label in &mut self.span_labels { + if span_label.0 == before { + span_label.0 = after; + replacements_occurred = true; + } + } + replacements_occurred + } + + /// Returns the strings to highlight. We always ensure that there + /// is an entry for each of the primary spans -- for each primary + /// span `P`, if there is at least one label with span `P`, we return + /// those labels (marked as primary). But otherwise we return + /// `SpanLabel` instances with empty labels. + pub fn span_labels(&self) -> Vec { + let is_primary = |span| self.primary_spans.contains(&span); + + let mut span_labels = self + .span_labels + .iter() + .map(|&(span, ref label)| SpanLabel { + span, + is_primary: is_primary(span), + label: Some(label.clone()), + }) + .collect::>(); + + for &span in &self.primary_spans { + if !span_labels.iter().any(|sl| sl.span == span) { + span_labels.push(SpanLabel { span, is_primary: true, label: None }); + } + } + + span_labels + } + + /// Returns `true` if any of the span labels is displayable. + pub fn has_span_labels(&self) -> bool { + self.span_labels.iter().any(|(sp, _)| !sp.is_dummy()) + } +} + +impl From for MultiSpan { + fn from(span: Span) -> MultiSpan { + MultiSpan::from_span(span) + } +} + +impl From> for MultiSpan { + fn from(spans: Vec) -> MultiSpan { + MultiSpan::from_spans(spans) + } +} + +/// Identifies an offset of a multi-byte character in a `SourceFile`. +#[derive(Copy, Clone, RustcEncodable, RustcDecodable, Eq, PartialEq, Debug)] +pub struct MultiByteChar { + /// The absolute offset of the character in the `SourceMap`. + pub pos: BytePos, + /// The number of bytes, `>= 2`. + pub bytes: u8, +} + +/// Identifies an offset of a non-narrow character in a `SourceFile`. +#[derive(Copy, Clone, RustcEncodable, RustcDecodable, Eq, PartialEq, Debug)] +pub enum NonNarrowChar { + /// Represents a zero-width character. + ZeroWidth(BytePos), + /// Represents a wide (full-width) character. + Wide(BytePos), + /// Represents a tab character, represented visually with a width of 4 characters. + Tab(BytePos), +} + +impl NonNarrowChar { + fn new(pos: BytePos, width: usize) -> Self { + match width { + 0 => NonNarrowChar::ZeroWidth(pos), + 2 => NonNarrowChar::Wide(pos), + 4 => NonNarrowChar::Tab(pos), + _ => panic!("width {} given for non-narrow character", width), + } + } + + /// Returns the absolute offset of the character in the `SourceMap`. + pub fn pos(&self) -> BytePos { + match *self { + NonNarrowChar::ZeroWidth(p) | NonNarrowChar::Wide(p) | NonNarrowChar::Tab(p) => p, + } + } + + /// Returns the width of the character, 0 (zero-width) or 2 (wide). + pub fn width(&self) -> usize { + match *self { + NonNarrowChar::ZeroWidth(_) => 0, + NonNarrowChar::Wide(_) => 2, + NonNarrowChar::Tab(_) => 4, + } + } +} + +impl Add for NonNarrowChar { + type Output = Self; + + fn add(self, rhs: BytePos) -> Self { + match self { + NonNarrowChar::ZeroWidth(pos) => NonNarrowChar::ZeroWidth(pos + rhs), + NonNarrowChar::Wide(pos) => NonNarrowChar::Wide(pos + rhs), + NonNarrowChar::Tab(pos) => NonNarrowChar::Tab(pos + rhs), + } + } +} + +impl Sub for NonNarrowChar { + type Output = Self; + + fn sub(self, rhs: BytePos) -> Self { + match self { + NonNarrowChar::ZeroWidth(pos) => NonNarrowChar::ZeroWidth(pos - rhs), + NonNarrowChar::Wide(pos) => NonNarrowChar::Wide(pos - rhs), + NonNarrowChar::Tab(pos) => NonNarrowChar::Tab(pos - rhs), + } + } +} + +/// Identifies an offset of a character that was normalized away from `SourceFile`. +#[derive(Copy, Clone, RustcEncodable, RustcDecodable, Eq, PartialEq, Debug)] +pub struct NormalizedPos { + /// The absolute offset of the character in the `SourceMap`. + pub pos: BytePos, + /// The difference between original and normalized string at position. + pub diff: u32, +} + +/// The state of the lazy external source loading mechanism of a `SourceFile`. +#[derive(PartialEq, Eq, Clone)] +pub enum ExternalSource { + /// The external source has been loaded already. + Present(String), + /// No attempt has been made to load the external source. + AbsentOk, + /// A failed attempt has been made to load the external source. + AbsentErr, + /// No external source has to be loaded, since the `SourceFile` represents a local crate. + Unneeded, +} + +impl ExternalSource { + pub fn is_absent(&self) -> bool { + match *self { + ExternalSource::Present(_) => false, + _ => true, + } + } + + pub fn get_source(&self) -> Option<&str> { + match *self { + ExternalSource::Present(ref src) => Some(src), + _ => None, + } + } +} + +#[derive(Debug)] +pub struct OffsetOverflowError; + +/// A single source in the `SourceMap`. +#[derive(Clone)] +pub struct SourceFile { + /// The name of the file that the source came from. Source that doesn't + /// originate from files has names between angle brackets by convention + /// (e.g., ``). + pub name: FileName, + /// `true` if the `name` field above has been modified by `--remap-path-prefix`. + pub name_was_remapped: bool, + /// The unmapped path of the file that the source came from. + /// Set to `None` if the `SourceFile` was imported from an external crate. + pub unmapped_path: Option, + /// Indicates which crate this `SourceFile` was imported from. + pub crate_of_origin: u32, + /// The complete source code. + pub src: Option>, + /// The source code's hash. + pub src_hash: u128, + /// The external source code (used for external crates, which will have a `None` + /// value as `self.src`. + pub external_src: Lock, + /// The start position of this source in the `SourceMap`. + pub start_pos: BytePos, + /// The end position of this source in the `SourceMap`. + pub end_pos: BytePos, + /// Locations of lines beginnings in the source code. + pub lines: Vec, + /// Locations of multi-byte characters in the source code. + pub multibyte_chars: Vec, + /// Width of characters that are not narrow in the source code. + pub non_narrow_chars: Vec, + /// Locations of characters removed during normalization. + pub normalized_pos: Vec, + /// A hash of the filename, used for speeding up hashing in incremental compilation. + pub name_hash: u128, +} + +impl Encodable for SourceFile { + fn encode(&self, s: &mut S) -> Result<(), S::Error> { + s.emit_struct("SourceFile", 8, |s| { + s.emit_struct_field("name", 0, |s| self.name.encode(s))?; + s.emit_struct_field("name_was_remapped", 1, |s| self.name_was_remapped.encode(s))?; + s.emit_struct_field("src_hash", 2, |s| self.src_hash.encode(s))?; + s.emit_struct_field("start_pos", 3, |s| self.start_pos.encode(s))?; + s.emit_struct_field("end_pos", 4, |s| self.end_pos.encode(s))?; + s.emit_struct_field("lines", 5, |s| { + let lines = &self.lines[..]; + // Store the length. + s.emit_u32(lines.len() as u32)?; + + if !lines.is_empty() { + // In order to preserve some space, we exploit the fact that + // the lines list is sorted and individual lines are + // probably not that long. Because of that we can store lines + // as a difference list, using as little space as possible + // for the differences. + let max_line_length = if lines.len() == 1 { + 0 + } else { + lines.windows(2).map(|w| w[1] - w[0]).map(|bp| bp.to_usize()).max().unwrap() + }; + + let bytes_per_diff: u8 = match max_line_length { + 0..=0xFF => 1, + 0x100..=0xFFFF => 2, + _ => 4, + }; + + // Encode the number of bytes used per diff. + bytes_per_diff.encode(s)?; + + // Encode the first element. + lines[0].encode(s)?; + + let diff_iter = (&lines[..]).windows(2).map(|w| (w[1] - w[0])); + + match bytes_per_diff { + 1 => { + for diff in diff_iter { + (diff.0 as u8).encode(s)? + } + } + 2 => { + for diff in diff_iter { + (diff.0 as u16).encode(s)? + } + } + 4 => { + for diff in diff_iter { + diff.0.encode(s)? + } + } + _ => unreachable!(), + } + } + + Ok(()) + })?; + s.emit_struct_field("multibyte_chars", 6, |s| self.multibyte_chars.encode(s))?; + s.emit_struct_field("non_narrow_chars", 7, |s| self.non_narrow_chars.encode(s))?; + s.emit_struct_field("name_hash", 8, |s| self.name_hash.encode(s))?; + s.emit_struct_field("normalized_pos", 9, |s| self.normalized_pos.encode(s)) + }) + } +} + +impl Decodable for SourceFile { + fn decode(d: &mut D) -> Result { + d.read_struct("SourceFile", 8, |d| { + let name: FileName = d.read_struct_field("name", 0, |d| Decodable::decode(d))?; + let name_was_remapped: bool = + d.read_struct_field("name_was_remapped", 1, |d| Decodable::decode(d))?; + let src_hash: u128 = d.read_struct_field("src_hash", 2, |d| Decodable::decode(d))?; + let start_pos: BytePos = + d.read_struct_field("start_pos", 3, |d| Decodable::decode(d))?; + let end_pos: BytePos = d.read_struct_field("end_pos", 4, |d| Decodable::decode(d))?; + let lines: Vec = d.read_struct_field("lines", 5, |d| { + let num_lines: u32 = Decodable::decode(d)?; + let mut lines = Vec::with_capacity(num_lines as usize); + + if num_lines > 0 { + // Read the number of bytes used per diff. + let bytes_per_diff: u8 = Decodable::decode(d)?; + + // Read the first element. + let mut line_start: BytePos = Decodable::decode(d)?; + lines.push(line_start); + + for _ in 1..num_lines { + let diff = match bytes_per_diff { + 1 => d.read_u8()? as u32, + 2 => d.read_u16()? as u32, + 4 => d.read_u32()?, + _ => unreachable!(), + }; + + line_start = line_start + BytePos(diff); + + lines.push(line_start); + } + } + + Ok(lines) + })?; + let multibyte_chars: Vec = + d.read_struct_field("multibyte_chars", 6, |d| Decodable::decode(d))?; + let non_narrow_chars: Vec = + d.read_struct_field("non_narrow_chars", 7, |d| Decodable::decode(d))?; + let name_hash: u128 = d.read_struct_field("name_hash", 8, |d| Decodable::decode(d))?; + let normalized_pos: Vec = + d.read_struct_field("normalized_pos", 9, |d| Decodable::decode(d))?; + Ok(SourceFile { + name, + name_was_remapped, + unmapped_path: None, + // `crate_of_origin` has to be set by the importer. + // This value matches up with `rustc::hir::def_id::INVALID_CRATE`. + // That constant is not available here, unfortunately. + crate_of_origin: std::u32::MAX - 1, + start_pos, + end_pos, + src: None, + src_hash, + external_src: Lock::new(ExternalSource::AbsentOk), + lines, + multibyte_chars, + non_narrow_chars, + normalized_pos, + name_hash, + }) + }) + } +} + +impl fmt::Debug for SourceFile { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(fmt, "SourceFile({})", self.name) + } +} + +impl SourceFile { + pub fn new( + name: FileName, + name_was_remapped: bool, + unmapped_path: FileName, + mut src: String, + start_pos: BytePos, + ) -> Result { + let normalized_pos = normalize_src(&mut src, start_pos); + + let src_hash = { + let mut hasher: StableHasher = StableHasher::new(); + hasher.write(src.as_bytes()); + hasher.finish::() + }; + let name_hash = { + let mut hasher: StableHasher = StableHasher::new(); + name.hash(&mut hasher); + hasher.finish::() + }; + let end_pos = start_pos.to_usize() + src.len(); + if end_pos > u32::max_value() as usize { + return Err(OffsetOverflowError); + } + + let (lines, multibyte_chars, non_narrow_chars) = + analyze_source_file::analyze_source_file(&src[..], start_pos); + + Ok(SourceFile { + name, + name_was_remapped, + unmapped_path: Some(unmapped_path), + crate_of_origin: 0, + src: Some(Lrc::new(src)), + src_hash, + external_src: Lock::new(ExternalSource::Unneeded), + start_pos, + end_pos: Pos::from_usize(end_pos), + lines, + multibyte_chars, + non_narrow_chars, + normalized_pos, + name_hash, + }) + } + + /// Returns the `BytePos` of the beginning of the current line. + pub fn line_begin_pos(&self, pos: BytePos) -> BytePos { + let line_index = self.lookup_line(pos).unwrap(); + self.lines[line_index] + } + + /// Add externally loaded source. + /// If the hash of the input doesn't match or no input is supplied via None, + /// it is interpreted as an error and the corresponding enum variant is set. + /// The return value signifies whether some kind of source is present. + pub fn add_external_src(&self, get_src: F) -> bool + where + F: FnOnce() -> Option, + { + if *self.external_src.borrow() == ExternalSource::AbsentOk { + let src = get_src(); + let mut external_src = self.external_src.borrow_mut(); + // Check that no-one else have provided the source while we were getting it + if *external_src == ExternalSource::AbsentOk { + if let Some(src) = src { + let mut hasher: StableHasher = StableHasher::new(); + hasher.write(src.as_bytes()); + + if hasher.finish::() == self.src_hash { + *external_src = ExternalSource::Present(src); + return true; + } + } else { + *external_src = ExternalSource::AbsentErr; + } + + false + } else { + self.src.is_some() || external_src.get_source().is_some() + } + } else { + self.src.is_some() || self.external_src.borrow().get_source().is_some() + } + } + + /// Gets a line from the list of pre-computed line-beginnings. + /// The line number here is 0-based. + pub fn get_line(&self, line_number: usize) -> Option> { + fn get_until_newline(src: &str, begin: usize) -> &str { + // We can't use `lines.get(line_number+1)` because we might + // be parsing when we call this function and thus the current + // line is the last one we have line info for. + let slice = &src[begin..]; + match slice.find('\n') { + Some(e) => &slice[..e], + None => slice, + } + } + + let begin = { + let line = if let Some(line) = self.lines.get(line_number) { + line + } else { + return None; + }; + let begin: BytePos = *line - self.start_pos; + begin.to_usize() + }; + + if let Some(ref src) = self.src { + Some(Cow::from(get_until_newline(src, begin))) + } else if let Some(src) = self.external_src.borrow().get_source() { + Some(Cow::Owned(String::from(get_until_newline(src, begin)))) + } else { + None + } + } + + pub fn is_real_file(&self) -> bool { + self.name.is_real() + } + + pub fn is_imported(&self) -> bool { + self.src.is_none() + } + + pub fn byte_length(&self) -> u32 { + self.end_pos.0 - self.start_pos.0 + } + pub fn count_lines(&self) -> usize { + self.lines.len() + } + + /// Finds the line containing the given position. The return value is the + /// index into the `lines` array of this `SourceFile`, not the 1-based line + /// number. If the source_file is empty or the position is located before the + /// first line, `None` is returned. + pub fn lookup_line(&self, pos: BytePos) -> Option { + if self.lines.len() == 0 { + return None; + } + + let line_index = lookup_line(&self.lines[..], pos); + assert!(line_index < self.lines.len() as isize); + if line_index >= 0 { Some(line_index as usize) } else { None } + } + + pub fn line_bounds(&self, line_index: usize) -> (BytePos, BytePos) { + if self.start_pos == self.end_pos { + return (self.start_pos, self.end_pos); + } + + assert!(line_index < self.lines.len()); + if line_index == (self.lines.len() - 1) { + (self.lines[line_index], self.end_pos) + } else { + (self.lines[line_index], self.lines[line_index + 1]) + } + } + + #[inline] + pub fn contains(&self, byte_pos: BytePos) -> bool { + byte_pos >= self.start_pos && byte_pos <= self.end_pos + } + + /// Calculates the original byte position relative to the start of the file + /// based on the given byte position. + pub fn original_relative_byte_pos(&self, pos: BytePos) -> BytePos { + // Diff before any records is 0. Otherwise use the previously recorded + // diff as that applies to the following characters until a new diff + // is recorded. + let diff = match self.normalized_pos.binary_search_by(|np| np.pos.cmp(&pos)) { + Ok(i) => self.normalized_pos[i].diff, + Err(i) if i == 0 => 0, + Err(i) => self.normalized_pos[i - 1].diff, + }; + + BytePos::from_u32(pos.0 - self.start_pos.0 + diff) + } +} + +/// Normalizes the source code and records the normalizations. +fn normalize_src(src: &mut String, start_pos: BytePos) -> Vec { + let mut normalized_pos = vec![]; + remove_bom(src, &mut normalized_pos); + normalize_newlines(src, &mut normalized_pos); + + // Offset all the positions by start_pos to match the final file positions. + for np in &mut normalized_pos { + np.pos.0 += start_pos.0; + } + + normalized_pos +} + +/// Removes UTF-8 BOM, if any. +fn remove_bom(src: &mut String, normalized_pos: &mut Vec) { + if src.starts_with("\u{feff}") { + src.drain(..3); + normalized_pos.push(NormalizedPos { pos: BytePos(0), diff: 3 }); + } +} + +/// Replaces `\r\n` with `\n` in-place in `src`. +/// +/// Returns error if there's a lone `\r` in the string +fn normalize_newlines(src: &mut String, normalized_pos: &mut Vec) { + if !src.as_bytes().contains(&b'\r') { + return; + } + + // We replace `\r\n` with `\n` in-place, which doesn't break utf-8 encoding. + // While we *can* call `as_mut_vec` and do surgery on the live string + // directly, let's rather steal the contents of `src`. This makes the code + // safe even if a panic occurs. + + let mut buf = std::mem::replace(src, String::new()).into_bytes(); + let mut gap_len = 0; + let mut tail = buf.as_mut_slice(); + let mut cursor = 0; + let original_gap = normalized_pos.last().map_or(0, |l| l.diff); + loop { + let idx = match find_crlf(&tail[gap_len..]) { + None => tail.len(), + Some(idx) => idx + gap_len, + }; + tail.copy_within(gap_len..idx, 0); + tail = &mut tail[idx - gap_len..]; + if tail.len() == gap_len { + break; + } + cursor += idx - gap_len; + gap_len += 1; + normalized_pos.push(NormalizedPos { + pos: BytePos::from_usize(cursor + 1), + diff: original_gap + gap_len as u32, + }); + } + + // Account for removed `\r`. + // After `set_len`, `buf` is guaranteed to contain utf-8 again. + let new_len = buf.len() - gap_len; + unsafe { + buf.set_len(new_len); + *src = String::from_utf8_unchecked(buf); + } + + fn find_crlf(src: &[u8]) -> Option { + let mut search_idx = 0; + while let Some(idx) = find_cr(&src[search_idx..]) { + if src[search_idx..].get(idx + 1) != Some(&b'\n') { + search_idx += idx + 1; + continue; + } + return Some(search_idx + idx); + } + None + } + + fn find_cr(src: &[u8]) -> Option { + src.iter().position(|&b| b == b'\r') + } +} + +// _____________________________________________________________________________ +// Pos, BytePos, CharPos +// + +pub trait Pos { + fn from_usize(n: usize) -> Self; + fn to_usize(&self) -> usize; + fn from_u32(n: u32) -> Self; + fn to_u32(&self) -> u32; +} + +/// A byte offset. Keep this small (currently 32-bits), as AST contains +/// a lot of them. +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] +pub struct BytePos(pub u32); + +/// A character offset. Because of multibyte UTF-8 characters, a byte offset +/// is not equivalent to a character offset. The `SourceMap` will convert `BytePos` +/// values to `CharPos` values as necessary. +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub struct CharPos(pub usize); + +// FIXME: lots of boilerplate in these impls, but so far my attempts to fix +// have been unsuccessful. + +impl Pos for BytePos { + #[inline(always)] + fn from_usize(n: usize) -> BytePos { + BytePos(n as u32) + } + + #[inline(always)] + fn to_usize(&self) -> usize { + self.0 as usize + } + + #[inline(always)] + fn from_u32(n: u32) -> BytePos { + BytePos(n) + } + + #[inline(always)] + fn to_u32(&self) -> u32 { + self.0 + } +} + +impl Add for BytePos { + type Output = BytePos; + + #[inline(always)] + fn add(self, rhs: BytePos) -> BytePos { + BytePos((self.to_usize() + rhs.to_usize()) as u32) + } +} + +impl Sub for BytePos { + type Output = BytePos; + + #[inline(always)] + fn sub(self, rhs: BytePos) -> BytePos { + BytePos((self.to_usize() - rhs.to_usize()) as u32) + } +} + +impl Encodable for BytePos { + fn encode(&self, s: &mut S) -> Result<(), S::Error> { + s.emit_u32(self.0) + } +} + +impl Decodable for BytePos { + fn decode(d: &mut D) -> Result { + Ok(BytePos(d.read_u32()?)) + } +} + +impl Pos for CharPos { + #[inline(always)] + fn from_usize(n: usize) -> CharPos { + CharPos(n) + } + + #[inline(always)] + fn to_usize(&self) -> usize { + self.0 + } + + #[inline(always)] + fn from_u32(n: u32) -> CharPos { + CharPos(n as usize) + } + + #[inline(always)] + fn to_u32(&self) -> u32 { + self.0 as u32 + } +} + +impl Add for CharPos { + type Output = CharPos; + + #[inline(always)] + fn add(self, rhs: CharPos) -> CharPos { + CharPos(self.to_usize() + rhs.to_usize()) + } +} + +impl Sub for CharPos { + type Output = CharPos; + + #[inline(always)] + fn sub(self, rhs: CharPos) -> CharPos { + CharPos(self.to_usize() - rhs.to_usize()) + } +} + +// _____________________________________________________________________________ +// Loc, SourceFileAndLine, SourceFileAndBytePos +// + +/// A source code location used for error reporting. +#[derive(Debug, Clone)] +pub struct Loc { + /// Information about the original source. + pub file: Lrc, + /// The (1-based) line number. + pub line: usize, + /// The (0-based) column offset. + pub col: CharPos, + /// The (0-based) column offset when displayed. + pub col_display: usize, +} + +// Used to be structural records. +#[derive(Debug)] +pub struct SourceFileAndLine { + pub sf: Lrc, + pub line: usize, +} +#[derive(Debug)] +pub struct SourceFileAndBytePos { + pub sf: Lrc, + pub pos: BytePos, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct LineInfo { + /// Index of line, starting from 0. + pub line_index: usize, + + /// Column in line where span begins, starting from 0. + pub start_col: CharPos, + + /// Column in line where span ends, starting from 0, exclusive. + pub end_col: CharPos, +} + +pub struct FileLines { + pub file: Lrc, + pub lines: Vec, +} + +pub static SPAN_DEBUG: AtomicRef) -> fmt::Result> = + AtomicRef::new(&(default_span_debug as fn(_, &mut fmt::Formatter<'_>) -> _)); + +#[derive(Debug)] +pub struct MacroBacktrace { + /// span where macro was applied to generate this code + pub call_site: Span, + + /// name of macro that was applied (e.g., "foo!" or "#[derive(Eq)]") + pub macro_decl_name: String, + + /// span where macro was defined (possibly dummy) + pub def_site_span: Span, +} + +// _____________________________________________________________________________ +// SpanLinesError, SpanSnippetError, DistinctSources, MalformedSourceMapPositions +// + +pub type FileLinesResult = Result; + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum SpanLinesError { + DistinctSources(DistinctSources), +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum SpanSnippetError { + IllFormedSpan(Span), + DistinctSources(DistinctSources), + MalformedForSourcemap(MalformedSourceMapPositions), + SourceNotAvailable { filename: FileName }, +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct DistinctSources { + pub begin: (FileName, BytePos), + pub end: (FileName, BytePos), +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct MalformedSourceMapPositions { + pub name: FileName, + pub source_len: usize, + pub begin_pos: BytePos, + pub end_pos: BytePos, +} + +/// Range inside of a `Span` used for diagnostics when we only have access to relative positions. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct InnerSpan { + pub start: usize, + pub end: usize, +} + +impl InnerSpan { + pub fn new(start: usize, end: usize) -> InnerSpan { + InnerSpan { start, end } + } +} + +// Given a slice of line start positions and a position, returns the index of +// the line the position is on. Returns -1 if the position is located before +// the first line. +fn lookup_line(lines: &[BytePos], pos: BytePos) -> isize { + match lines.binary_search(&pos) { + Ok(line) => line as isize, + Err(line) => line as isize - 1, + } +} + +/// Requirements for a `StableHashingContext` to be used in this crate. +/// This is a hack to allow using the `HashStable_Generic` derive macro +/// instead of implementing everything in librustc. +pub trait HashStableContext { + fn hash_spans(&self) -> bool; + fn byte_pos_to_line_and_col( + &mut self, + byte: BytePos, + ) -> Option<(Lrc, usize, BytePos)>; +} + +impl HashStable for Span +where + CTX: HashStableContext, +{ + /// Hashes a span in a stable way. We can't directly hash the span's `BytePos` + /// fields (that would be similar to hashing pointers, since those are just + /// offsets into the `SourceMap`). Instead, we hash the (file name, line, column) + /// triple, which stays the same even if the containing `SourceFile` has moved + /// within the `SourceMap`. + /// Also note that we are hashing byte offsets for the column, not unicode + /// codepoint offsets. For the purpose of the hash that's sufficient. + /// Also, hashing filenames is expensive so we avoid doing it twice when the + /// span starts and ends in the same file, which is almost always the case. + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + const TAG_VALID_SPAN: u8 = 0; + const TAG_INVALID_SPAN: u8 = 1; + const TAG_EXPANSION: u8 = 0; + const TAG_NO_EXPANSION: u8 = 1; + + if !ctx.hash_spans() { + return; + } + + if *self == DUMMY_SP { + return std::hash::Hash::hash(&TAG_INVALID_SPAN, hasher); + } + + // If this is not an empty or invalid span, we want to hash the last + // position that belongs to it, as opposed to hashing the first + // position past it. + let span = self.data(); + let (file_lo, line_lo, col_lo) = match ctx.byte_pos_to_line_and_col(span.lo) { + Some(pos) => pos, + None => { + return std::hash::Hash::hash(&TAG_INVALID_SPAN, hasher); + } + }; + + if !file_lo.contains(span.hi) { + return std::hash::Hash::hash(&TAG_INVALID_SPAN, hasher); + } + + std::hash::Hash::hash(&TAG_VALID_SPAN, hasher); + // We truncate the stable ID hash and line and column numbers. The chances + // of causing a collision this way should be minimal. + std::hash::Hash::hash(&(file_lo.name_hash as u64), hasher); + + let col = (col_lo.0 as u64) & 0xFF; + let line = ((line_lo as u64) & 0xFF_FF_FF) << 8; + let len = ((span.hi - span.lo).0 as u64) << 32; + let line_col_len = col | line | len; + std::hash::Hash::hash(&line_col_len, hasher); + + if span.ctxt == SyntaxContext::root() { + TAG_NO_EXPANSION.hash_stable(ctx, hasher); + } else { + TAG_EXPANSION.hash_stable(ctx, hasher); + + // Since the same expansion context is usually referenced many + // times, we cache a stable hash of it and hash that instead of + // recursing every time. + thread_local! { + static CACHE: RefCell> = Default::default(); + } + + let sub_hash: u64 = CACHE.with(|cache| { + let expn_id = span.ctxt.outer_expn(); + + if let Some(&sub_hash) = cache.borrow().get(&expn_id) { + return sub_hash; + } + + let mut hasher = StableHasher::new(); + expn_id.expn_data().hash_stable(ctx, &mut hasher); + let sub_hash: Fingerprint = hasher.finish(); + let sub_hash = sub_hash.to_smaller_hash(); + cache.borrow_mut().insert(expn_id, sub_hash); + sub_hash + }); + + sub_hash.hash_stable(ctx, hasher); + } + } +} diff --git a/src/librustc_span/source_map.rs b/src/librustc_span/source_map.rs new file mode 100644 index 00000000000..0b9b9fe7887 --- /dev/null +++ b/src/librustc_span/source_map.rs @@ -0,0 +1,984 @@ +//! The `SourceMap` tracks all the source code used within a single crate, mapping +//! from integer byte positions to the original source code location. Each bit +//! of source parsed during crate parsing (typically files, in-memory strings, +//! or various bits of macro expansion) cover a continuous range of bytes in the +//! `SourceMap` and are represented by `SourceFile`s. Byte positions are stored in +//! `Span` and used pervasively in the compiler. They are absolute positions +//! within the `SourceMap`, which upon request can be converted to line and column +//! information, source code snippets, etc. + +pub use crate::hygiene::{ExpnData, ExpnKind}; +pub use crate::*; + +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::stable_hasher::StableHasher; +use rustc_data_structures::sync::{Lock, LockGuard, Lrc, MappedLockGuard}; +use std::cmp; +use std::hash::Hash; +use std::path::{Path, PathBuf}; + +use log::debug; +use std::env; +use std::fs; +use std::io; + +#[cfg(test)] +mod tests; + +/// Returns the span itself if it doesn't come from a macro expansion, +/// otherwise return the call site span up to the `enclosing_sp` by +/// following the `expn_data` chain. +pub fn original_sp(sp: Span, enclosing_sp: Span) -> Span { + let expn_data1 = sp.ctxt().outer_expn_data(); + let expn_data2 = enclosing_sp.ctxt().outer_expn_data(); + if expn_data1.is_root() || !expn_data2.is_root() && expn_data1.call_site == expn_data2.call_site + { + sp + } else { + original_sp(expn_data1.call_site, enclosing_sp) + } +} + +#[derive(Clone, RustcEncodable, RustcDecodable, Debug, Copy, HashStable_Generic)] +pub struct Spanned { + pub node: T, + pub span: Span, +} + +pub fn respan(sp: Span, t: T) -> Spanned { + Spanned { node: t, span: sp } +} + +pub fn dummy_spanned(t: T) -> Spanned { + respan(DUMMY_SP, t) +} + +// _____________________________________________________________________________ +// SourceFile, MultiByteChar, FileName, FileLines +// + +/// An abstraction over the fs operations used by the Parser. +pub trait FileLoader { + /// Query the existence of a file. + fn file_exists(&self, path: &Path) -> bool; + + /// Returns an absolute path to a file, if possible. + fn abs_path(&self, path: &Path) -> Option; + + /// Read the contents of an UTF-8 file into memory. + fn read_file(&self, path: &Path) -> io::Result; +} + +/// A FileLoader that uses std::fs to load real files. +pub struct RealFileLoader; + +impl FileLoader for RealFileLoader { + fn file_exists(&self, path: &Path) -> bool { + fs::metadata(path).is_ok() + } + + fn abs_path(&self, path: &Path) -> Option { + if path.is_absolute() { + Some(path.to_path_buf()) + } else { + env::current_dir().ok().map(|cwd| cwd.join(path)) + } + } + + fn read_file(&self, path: &Path) -> io::Result { + fs::read_to_string(path) + } +} + +// This is a `SourceFile` identifier that is used to correlate `SourceFile`s between +// subsequent compilation sessions (which is something we need to do during +// incremental compilation). +#[derive(Copy, Clone, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable, Debug)] +pub struct StableSourceFileId(u128); + +impl StableSourceFileId { + pub fn new(source_file: &SourceFile) -> StableSourceFileId { + StableSourceFileId::new_from_pieces( + &source_file.name, + source_file.name_was_remapped, + source_file.unmapped_path.as_ref(), + ) + } + + pub fn new_from_pieces( + name: &FileName, + name_was_remapped: bool, + unmapped_path: Option<&FileName>, + ) -> StableSourceFileId { + let mut hasher = StableHasher::new(); + + name.hash(&mut hasher); + name_was_remapped.hash(&mut hasher); + unmapped_path.hash(&mut hasher); + + StableSourceFileId(hasher.finish()) + } +} + +// _____________________________________________________________________________ +// SourceMap +// + +#[derive(Default)] +pub(super) struct SourceMapFiles { + source_files: Vec>, + stable_id_to_source_file: FxHashMap>, +} + +pub struct SourceMap { + files: Lock, + file_loader: Box, + // This is used to apply the file path remapping as specified via + // `--remap-path-prefix` to all `SourceFile`s allocated within this `SourceMap`. + path_mapping: FilePathMapping, +} + +impl SourceMap { + pub fn new(path_mapping: FilePathMapping) -> SourceMap { + SourceMap { files: Default::default(), file_loader: Box::new(RealFileLoader), path_mapping } + } + + pub fn with_file_loader( + file_loader: Box, + path_mapping: FilePathMapping, + ) -> SourceMap { + SourceMap { files: Default::default(), file_loader, path_mapping } + } + + pub fn path_mapping(&self) -> &FilePathMapping { + &self.path_mapping + } + + pub fn file_exists(&self, path: &Path) -> bool { + self.file_loader.file_exists(path) + } + + pub fn load_file(&self, path: &Path) -> io::Result> { + let src = self.file_loader.read_file(path)?; + let filename = path.to_owned().into(); + Ok(self.new_source_file(filename, src)) + } + + /// Loads source file as a binary blob. + /// + /// Unlike `load_file`, guarantees that no normalization like BOM-removal + /// takes place. + pub fn load_binary_file(&self, path: &Path) -> io::Result> { + // Ideally, this should use `self.file_loader`, but it can't + // deal with binary files yet. + let bytes = fs::read(path)?; + + // We need to add file to the `SourceMap`, so that it is present + // in dep-info. There's also an edge case that file might be both + // loaded as a binary via `include_bytes!` and as proper `SourceFile` + // via `mod`, so we try to use real file contents and not just an + // empty string. + let text = std::str::from_utf8(&bytes).unwrap_or("").to_string(); + self.new_source_file(path.to_owned().into(), text); + Ok(bytes) + } + + pub fn files(&self) -> MappedLockGuard<'_, Vec>> { + LockGuard::map(self.files.borrow(), |files| &mut files.source_files) + } + + pub fn source_file_by_stable_id( + &self, + stable_id: StableSourceFileId, + ) -> Option> { + self.files.borrow().stable_id_to_source_file.get(&stable_id).map(|sf| sf.clone()) + } + + fn next_start_pos(&self) -> usize { + match self.files.borrow().source_files.last() { + None => 0, + // Add one so there is some space between files. This lets us distinguish + // positions in the `SourceMap`, even in the presence of zero-length files. + Some(last) => last.end_pos.to_usize() + 1, + } + } + + /// Creates a new `SourceFile`. + /// If a file already exists in the `SourceMap` with the same ID, that file is returned + /// unmodified. + pub fn new_source_file(&self, filename: FileName, src: String) -> Lrc { + self.try_new_source_file(filename, src).unwrap_or_else(|OffsetOverflowError| { + eprintln!("fatal error: rustc does not support files larger than 4GB"); + crate::fatal_error::FatalError.raise() + }) + } + + fn try_new_source_file( + &self, + filename: FileName, + src: String, + ) -> Result, OffsetOverflowError> { + let start_pos = self.next_start_pos(); + + // The path is used to determine the directory for loading submodules and + // include files, so it must be before remapping. + // Note that filename may not be a valid path, eg it may be `` etc, + // but this is okay because the directory determined by `path.pop()` will + // be empty, so the working directory will be used. + let unmapped_path = filename.clone(); + + let (filename, was_remapped) = match filename { + FileName::Real(filename) => { + let (filename, was_remapped) = self.path_mapping.map_prefix(filename); + (FileName::Real(filename), was_remapped) + } + other => (other, false), + }; + + let file_id = + StableSourceFileId::new_from_pieces(&filename, was_remapped, Some(&unmapped_path)); + + let lrc_sf = match self.source_file_by_stable_id(file_id) { + Some(lrc_sf) => lrc_sf, + None => { + let source_file = Lrc::new(SourceFile::new( + filename, + was_remapped, + unmapped_path, + src, + Pos::from_usize(start_pos), + )?); + + let mut files = self.files.borrow_mut(); + + files.source_files.push(source_file.clone()); + files.stable_id_to_source_file.insert(file_id, source_file.clone()); + + source_file + } + }; + Ok(lrc_sf) + } + + /// Allocates a new `SourceFile` representing a source file from an external + /// crate. The source code of such an "imported `SourceFile`" is not available, + /// but we still know enough to generate accurate debuginfo location + /// information for things inlined from other crates. + pub fn new_imported_source_file( + &self, + filename: FileName, + name_was_remapped: bool, + crate_of_origin: u32, + src_hash: u128, + name_hash: u128, + source_len: usize, + mut file_local_lines: Vec, + mut file_local_multibyte_chars: Vec, + mut file_local_non_narrow_chars: Vec, + mut file_local_normalized_pos: Vec, + ) -> Lrc { + let start_pos = self.next_start_pos(); + + let end_pos = Pos::from_usize(start_pos + source_len); + let start_pos = Pos::from_usize(start_pos); + + for pos in &mut file_local_lines { + *pos = *pos + start_pos; + } + + for mbc in &mut file_local_multibyte_chars { + mbc.pos = mbc.pos + start_pos; + } + + for swc in &mut file_local_non_narrow_chars { + *swc = *swc + start_pos; + } + + for nc in &mut file_local_normalized_pos { + nc.pos = nc.pos + start_pos; + } + + let source_file = Lrc::new(SourceFile { + name: filename, + name_was_remapped, + unmapped_path: None, + crate_of_origin, + src: None, + src_hash, + external_src: Lock::new(ExternalSource::AbsentOk), + start_pos, + end_pos, + lines: file_local_lines, + multibyte_chars: file_local_multibyte_chars, + non_narrow_chars: file_local_non_narrow_chars, + normalized_pos: file_local_normalized_pos, + name_hash, + }); + + let mut files = self.files.borrow_mut(); + + files.source_files.push(source_file.clone()); + files + .stable_id_to_source_file + .insert(StableSourceFileId::new(&source_file), source_file.clone()); + + source_file + } + + pub fn mk_substr_filename(&self, sp: Span) -> String { + let pos = self.lookup_char_pos(sp.lo()); + format!("<{}:{}:{}>", pos.file.name, pos.line, pos.col.to_usize() + 1) + } + + // If there is a doctest offset, applies it to the line. + pub fn doctest_offset_line(&self, file: &FileName, orig: usize) -> usize { + return match file { + FileName::DocTest(_, offset) => { + return if *offset >= 0 { + orig + *offset as usize + } else { + orig - (-(*offset)) as usize + }; + } + _ => orig, + }; + } + + /// Looks up source information about a `BytePos`. + pub fn lookup_char_pos(&self, pos: BytePos) -> Loc { + let chpos = self.bytepos_to_file_charpos(pos); + match self.lookup_line(pos) { + Ok(SourceFileAndLine { sf: f, line: a }) => { + let line = a + 1; // Line numbers start at 1 + let linebpos = f.lines[a]; + let linechpos = self.bytepos_to_file_charpos(linebpos); + let col = chpos - linechpos; + + let col_display = { + let start_width_idx = f + .non_narrow_chars + .binary_search_by_key(&linebpos, |x| x.pos()) + .unwrap_or_else(|x| x); + let end_width_idx = f + .non_narrow_chars + .binary_search_by_key(&pos, |x| x.pos()) + .unwrap_or_else(|x| x); + let special_chars = end_width_idx - start_width_idx; + let non_narrow: usize = f.non_narrow_chars[start_width_idx..end_width_idx] + .into_iter() + .map(|x| x.width()) + .sum(); + col.0 - special_chars + non_narrow + }; + debug!("byte pos {:?} is on the line at byte pos {:?}", pos, linebpos); + debug!("char pos {:?} is on the line at char pos {:?}", chpos, linechpos); + debug!("byte is on line: {}", line); + assert!(chpos >= linechpos); + Loc { file: f, line, col, col_display } + } + Err(f) => { + let col_display = { + let end_width_idx = f + .non_narrow_chars + .binary_search_by_key(&pos, |x| x.pos()) + .unwrap_or_else(|x| x); + let non_narrow: usize = + f.non_narrow_chars[0..end_width_idx].into_iter().map(|x| x.width()).sum(); + chpos.0 - end_width_idx + non_narrow + }; + Loc { file: f, line: 0, col: chpos, col_display } + } + } + } + + // If the corresponding `SourceFile` is empty, does not return a line number. + pub fn lookup_line(&self, pos: BytePos) -> Result> { + let idx = self.lookup_source_file_idx(pos); + + let f = (*self.files.borrow().source_files)[idx].clone(); + + match f.lookup_line(pos) { + Some(line) => Ok(SourceFileAndLine { sf: f, line }), + None => Err(f), + } + } + + /// Returns `Some(span)`, a union of the LHS and RHS span. The LHS must precede the RHS. If + /// there are gaps between LHS and RHS, the resulting union will cross these gaps. + /// For this to work, + /// + /// * the syntax contexts of both spans much match, + /// * the LHS span needs to end on the same line the RHS span begins, + /// * the LHS span must start at or before the RHS span. + pub fn merge_spans(&self, sp_lhs: Span, sp_rhs: Span) -> Option { + // Ensure we're at the same expansion ID. + if sp_lhs.ctxt() != sp_rhs.ctxt() { + return None; + } + + let lhs_end = match self.lookup_line(sp_lhs.hi()) { + Ok(x) => x, + Err(_) => return None, + }; + let rhs_begin = match self.lookup_line(sp_rhs.lo()) { + Ok(x) => x, + Err(_) => return None, + }; + + // If we must cross lines to merge, don't merge. + if lhs_end.line != rhs_begin.line { + return None; + } + + // Ensure these follow the expected order and that we don't overlap. + if (sp_lhs.lo() <= sp_rhs.lo()) && (sp_lhs.hi() <= sp_rhs.lo()) { + Some(sp_lhs.to(sp_rhs)) + } else { + None + } + } + + pub fn span_to_string(&self, sp: Span) -> String { + if self.files.borrow().source_files.is_empty() && sp.is_dummy() { + return "no-location".to_string(); + } + + let lo = self.lookup_char_pos(sp.lo()); + let hi = self.lookup_char_pos(sp.hi()); + format!( + "{}:{}:{}: {}:{}", + lo.file.name, + lo.line, + lo.col.to_usize() + 1, + hi.line, + hi.col.to_usize() + 1, + ) + } + + pub fn span_to_filename(&self, sp: Span) -> FileName { + self.lookup_char_pos(sp.lo()).file.name.clone() + } + + pub fn span_to_unmapped_path(&self, sp: Span) -> FileName { + self.lookup_char_pos(sp.lo()) + .file + .unmapped_path + .clone() + .expect("`SourceMap::span_to_unmapped_path` called for imported `SourceFile`?") + } + + pub fn is_multiline(&self, sp: Span) -> bool { + let lo = self.lookup_char_pos(sp.lo()); + let hi = self.lookup_char_pos(sp.hi()); + lo.line != hi.line + } + + pub fn span_to_lines(&self, sp: Span) -> FileLinesResult { + debug!("span_to_lines(sp={:?})", sp); + + let lo = self.lookup_char_pos(sp.lo()); + debug!("span_to_lines: lo={:?}", lo); + let hi = self.lookup_char_pos(sp.hi()); + debug!("span_to_lines: hi={:?}", hi); + + if lo.file.start_pos != hi.file.start_pos { + return Err(SpanLinesError::DistinctSources(DistinctSources { + begin: (lo.file.name.clone(), lo.file.start_pos), + end: (hi.file.name.clone(), hi.file.start_pos), + })); + } + assert!(hi.line >= lo.line); + + let mut lines = Vec::with_capacity(hi.line - lo.line + 1); + + // The span starts partway through the first line, + // but after that it starts from offset 0. + let mut start_col = lo.col; + + // For every line but the last, it extends from `start_col` + // and to the end of the line. Be careful because the line + // numbers in Loc are 1-based, so we subtract 1 to get 0-based + // lines. + for line_index in lo.line - 1..hi.line - 1 { + let line_len = lo.file.get_line(line_index).map(|s| s.chars().count()).unwrap_or(0); + lines.push(LineInfo { line_index, start_col, end_col: CharPos::from_usize(line_len) }); + start_col = CharPos::from_usize(0); + } + + // For the last line, it extends from `start_col` to `hi.col`: + lines.push(LineInfo { line_index: hi.line - 1, start_col, end_col: hi.col }); + + Ok(FileLines { file: lo.file, lines }) + } + + /// Extracts the source surrounding the given `Span` using the `extract_source` function. The + /// extract function takes three arguments: a string slice containing the source, an index in + /// the slice for the beginning of the span and an index in the slice for the end of the span. + fn span_to_source(&self, sp: Span, extract_source: F) -> Result + where + F: Fn(&str, usize, usize) -> Result, + { + let local_begin = self.lookup_byte_offset(sp.lo()); + let local_end = self.lookup_byte_offset(sp.hi()); + + if local_begin.sf.start_pos != local_end.sf.start_pos { + return Err(SpanSnippetError::DistinctSources(DistinctSources { + begin: (local_begin.sf.name.clone(), local_begin.sf.start_pos), + end: (local_end.sf.name.clone(), local_end.sf.start_pos), + })); + } else { + self.ensure_source_file_source_present(local_begin.sf.clone()); + + let start_index = local_begin.pos.to_usize(); + let end_index = local_end.pos.to_usize(); + let source_len = (local_begin.sf.end_pos - local_begin.sf.start_pos).to_usize(); + + if start_index > end_index || end_index > source_len { + return Err(SpanSnippetError::MalformedForSourcemap(MalformedSourceMapPositions { + name: local_begin.sf.name.clone(), + source_len, + begin_pos: local_begin.pos, + end_pos: local_end.pos, + })); + } + + if let Some(ref src) = local_begin.sf.src { + return extract_source(src, start_index, end_index); + } else if let Some(src) = local_begin.sf.external_src.borrow().get_source() { + return extract_source(src, start_index, end_index); + } else { + return Err(SpanSnippetError::SourceNotAvailable { + filename: local_begin.sf.name.clone(), + }); + } + } + } + + /// Returns the source snippet as `String` corresponding to the given `Span`. + pub fn span_to_snippet(&self, sp: Span) -> Result { + self.span_to_source(sp, |src, start_index, end_index| { + src.get(start_index..end_index) + .map(|s| s.to_string()) + .ok_or_else(|| SpanSnippetError::IllFormedSpan(sp)) + }) + } + + pub fn span_to_margin(&self, sp: Span) -> Option { + match self.span_to_prev_source(sp) { + Err(_) => None, + Ok(source) => source + .split('\n') + .last() + .map(|last_line| last_line.len() - last_line.trim_start().len()), + } + } + + /// Returns the source snippet as `String` before the given `Span`. + pub fn span_to_prev_source(&self, sp: Span) -> Result { + self.span_to_source(sp, |src, start_index, _| { + src.get(..start_index) + .map(|s| s.to_string()) + .ok_or_else(|| SpanSnippetError::IllFormedSpan(sp)) + }) + } + + /// Extends the given `Span` to just after the previous occurrence of `c`. Return the same span + /// if no character could be found or if an error occurred while retrieving the code snippet. + pub fn span_extend_to_prev_char(&self, sp: Span, c: char) -> Span { + if let Ok(prev_source) = self.span_to_prev_source(sp) { + let prev_source = prev_source.rsplit(c).nth(0).unwrap_or("").trim_start(); + if !prev_source.is_empty() && !prev_source.contains('\n') { + return sp.with_lo(BytePos(sp.lo().0 - prev_source.len() as u32)); + } + } + + sp + } + + /// Extends the given `Span` to just after the previous occurrence of `pat` when surrounded by + /// whitespace. Returns the same span if no character could be found or if an error occurred + /// while retrieving the code snippet. + pub fn span_extend_to_prev_str(&self, sp: Span, pat: &str, accept_newlines: bool) -> Span { + // assure that the pattern is delimited, to avoid the following + // fn my_fn() + // ^^^^ returned span without the check + // ---------- correct span + for ws in &[" ", "\t", "\n"] { + let pat = pat.to_owned() + ws; + if let Ok(prev_source) = self.span_to_prev_source(sp) { + let prev_source = prev_source.rsplit(&pat).nth(0).unwrap_or("").trim_start(); + if !prev_source.is_empty() && (!prev_source.contains('\n') || accept_newlines) { + return sp.with_lo(BytePos(sp.lo().0 - prev_source.len() as u32)); + } + } + } + + sp + } + + /// Given a `Span`, tries to get a shorter span ending before the first occurrence of `char` + /// `c`. + pub fn span_until_char(&self, sp: Span, c: char) -> Span { + match self.span_to_snippet(sp) { + Ok(snippet) => { + let snippet = snippet.split(c).nth(0).unwrap_or("").trim_end(); + if !snippet.is_empty() && !snippet.contains('\n') { + sp.with_hi(BytePos(sp.lo().0 + snippet.len() as u32)) + } else { + sp + } + } + _ => sp, + } + } + + /// Given a `Span`, tries to get a shorter span ending just after the first occurrence of `char` + /// `c`. + pub fn span_through_char(&self, sp: Span, c: char) -> Span { + if let Ok(snippet) = self.span_to_snippet(sp) { + if let Some(offset) = snippet.find(c) { + return sp.with_hi(BytePos(sp.lo().0 + (offset + c.len_utf8()) as u32)); + } + } + sp + } + + /// Given a `Span`, gets a new `Span` covering the first token and all its trailing whitespace + /// or the original `Span`. + /// + /// If `sp` points to `"let mut x"`, then a span pointing at `"let "` will be returned. + pub fn span_until_non_whitespace(&self, sp: Span) -> Span { + let mut whitespace_found = false; + + self.span_take_while(sp, |c| { + if !whitespace_found && c.is_whitespace() { + whitespace_found = true; + } + + if whitespace_found && !c.is_whitespace() { false } else { true } + }) + } + + /// Given a `Span`, gets a new `Span` covering the first token without its trailing whitespace + /// or the original `Span` in case of error. + /// + /// If `sp` points to `"let mut x"`, then a span pointing at `"let"` will be returned. + pub fn span_until_whitespace(&self, sp: Span) -> Span { + self.span_take_while(sp, |c| !c.is_whitespace()) + } + + /// Given a `Span`, gets a shorter one until `predicate` yields `false`. + pub fn span_take_while

(&self, sp: Span, predicate: P) -> Span + where + P: for<'r> FnMut(&'r char) -> bool, + { + if let Ok(snippet) = self.span_to_snippet(sp) { + let offset = snippet.chars().take_while(predicate).map(|c| c.len_utf8()).sum::(); + + sp.with_hi(BytePos(sp.lo().0 + (offset as u32))) + } else { + sp + } + } + + pub fn def_span(&self, sp: Span) -> Span { + self.span_until_char(sp, '{') + } + + /// Returns a new span representing just the start point of this span. + pub fn start_point(&self, sp: Span) -> Span { + let pos = sp.lo().0; + let width = self.find_width_of_character_at_span(sp, false); + let corrected_start_position = pos.checked_add(width).unwrap_or(pos); + let end_point = BytePos(cmp::max(corrected_start_position, sp.lo().0)); + sp.with_hi(end_point) + } + + /// Returns a new span representing just the end point of this span. + pub fn end_point(&self, sp: Span) -> Span { + let pos = sp.hi().0; + + let width = self.find_width_of_character_at_span(sp, false); + let corrected_end_position = pos.checked_sub(width).unwrap_or(pos); + + let end_point = BytePos(cmp::max(corrected_end_position, sp.lo().0)); + sp.with_lo(end_point) + } + + /// Returns a new span representing the next character after the end-point of this span. + pub fn next_point(&self, sp: Span) -> Span { + let start_of_next_point = sp.hi().0; + + let width = self.find_width_of_character_at_span(sp, true); + // If the width is 1, then the next span should point to the same `lo` and `hi`. However, + // in the case of a multibyte character, where the width != 1, the next span should + // span multiple bytes to include the whole character. + let end_of_next_point = + start_of_next_point.checked_add(width - 1).unwrap_or(start_of_next_point); + + let end_of_next_point = BytePos(cmp::max(sp.lo().0 + 1, end_of_next_point)); + Span::new(BytePos(start_of_next_point), end_of_next_point, sp.ctxt()) + } + + /// Finds the width of a character, either before or after the provided span. + fn find_width_of_character_at_span(&self, sp: Span, forwards: bool) -> u32 { + let sp = sp.data(); + if sp.lo == sp.hi { + debug!("find_width_of_character_at_span: early return empty span"); + return 1; + } + + let local_begin = self.lookup_byte_offset(sp.lo); + let local_end = self.lookup_byte_offset(sp.hi); + debug!( + "find_width_of_character_at_span: local_begin=`{:?}`, local_end=`{:?}`", + local_begin, local_end + ); + + if local_begin.sf.start_pos != local_end.sf.start_pos { + debug!("find_width_of_character_at_span: begin and end are in different files"); + return 1; + } + + let start_index = local_begin.pos.to_usize(); + let end_index = local_end.pos.to_usize(); + debug!( + "find_width_of_character_at_span: start_index=`{:?}`, end_index=`{:?}`", + start_index, end_index + ); + + // Disregard indexes that are at the start or end of their spans, they can't fit bigger + // characters. + if (!forwards && end_index == usize::min_value()) + || (forwards && start_index == usize::max_value()) + { + debug!("find_width_of_character_at_span: start or end of span, cannot be multibyte"); + return 1; + } + + let source_len = (local_begin.sf.end_pos - local_begin.sf.start_pos).to_usize(); + debug!("find_width_of_character_at_span: source_len=`{:?}`", source_len); + // Ensure indexes are also not malformed. + if start_index > end_index || end_index > source_len { + debug!("find_width_of_character_at_span: source indexes are malformed"); + return 1; + } + + let src = local_begin.sf.external_src.borrow(); + + // We need to extend the snippet to the end of the src rather than to end_index so when + // searching forwards for boundaries we've got somewhere to search. + let snippet = if let Some(ref src) = local_begin.sf.src { + let len = src.len(); + (&src[start_index..len]) + } else if let Some(src) = src.get_source() { + let len = src.len(); + (&src[start_index..len]) + } else { + return 1; + }; + debug!("find_width_of_character_at_span: snippet=`{:?}`", snippet); + + let mut target = if forwards { end_index + 1 } else { end_index - 1 }; + debug!("find_width_of_character_at_span: initial target=`{:?}`", target); + + while !snippet.is_char_boundary(target - start_index) && target < source_len { + target = if forwards { + target + 1 + } else { + match target.checked_sub(1) { + Some(target) => target, + None => { + break; + } + } + }; + debug!("find_width_of_character_at_span: target=`{:?}`", target); + } + debug!("find_width_of_character_at_span: final target=`{:?}`", target); + + if forwards { (target - end_index) as u32 } else { (end_index - target) as u32 } + } + + pub fn get_source_file(&self, filename: &FileName) -> Option> { + for sf in self.files.borrow().source_files.iter() { + if *filename == sf.name { + return Some(sf.clone()); + } + } + None + } + + /// For a global `BytePos`, computes the local offset within the containing `SourceFile`. + pub fn lookup_byte_offset(&self, bpos: BytePos) -> SourceFileAndBytePos { + let idx = self.lookup_source_file_idx(bpos); + let sf = (*self.files.borrow().source_files)[idx].clone(); + let offset = bpos - sf.start_pos; + SourceFileAndBytePos { sf, pos: offset } + } + + /// Converts an absolute `BytePos` to a `CharPos` relative to the `SourceFile`. + pub fn bytepos_to_file_charpos(&self, bpos: BytePos) -> CharPos { + let idx = self.lookup_source_file_idx(bpos); + let map = &(*self.files.borrow().source_files)[idx]; + + // The number of extra bytes due to multibyte chars in the `SourceFile`. + let mut total_extra_bytes = 0; + + for mbc in map.multibyte_chars.iter() { + debug!("{}-byte char at {:?}", mbc.bytes, mbc.pos); + if mbc.pos < bpos { + // Every character is at least one byte, so we only + // count the actual extra bytes. + total_extra_bytes += mbc.bytes as u32 - 1; + // We should never see a byte position in the middle of a + // character. + assert!(bpos.to_u32() >= mbc.pos.to_u32() + mbc.bytes as u32); + } else { + break; + } + } + + assert!(map.start_pos.to_u32() + total_extra_bytes <= bpos.to_u32()); + CharPos(bpos.to_usize() - map.start_pos.to_usize() - total_extra_bytes as usize) + } + + // Returns the index of the `SourceFile` (in `self.files`) that contains `pos`. + pub fn lookup_source_file_idx(&self, pos: BytePos) -> usize { + self.files + .borrow() + .source_files + .binary_search_by_key(&pos, |key| key.start_pos) + .unwrap_or_else(|p| p - 1) + } + + pub fn count_lines(&self) -> usize { + self.files().iter().fold(0, |a, f| a + f.count_lines()) + } + + pub fn generate_fn_name_span(&self, span: Span) -> Option { + let prev_span = self.span_extend_to_prev_str(span, "fn", true); + self.span_to_snippet(prev_span) + .map(|snippet| { + let len = snippet + .find(|c: char| !c.is_alphanumeric() && c != '_') + .expect("no label after fn"); + prev_span.with_hi(BytePos(prev_span.lo().0 + len as u32)) + }) + .ok() + } + + /// Takes the span of a type parameter in a function signature and try to generate a span for + /// the function name (with generics) and a new snippet for this span with the pointed type + /// parameter as a new local type parameter. + /// + /// For instance: + /// ```rust,ignore (pseudo-Rust) + /// // Given span + /// fn my_function(param: T) + /// // ^ Original span + /// + /// // Result + /// fn my_function(param: T) + /// // ^^^^^^^^^^^ Generated span with snippet `my_function` + /// ``` + /// + /// Attention: The method used is very fragile since it essentially duplicates the work of the + /// parser. If you need to use this function or something similar, please consider updating the + /// `SourceMap` functions and this function to something more robust. + pub fn generate_local_type_param_snippet(&self, span: Span) -> Option<(Span, String)> { + // Try to extend the span to the previous "fn" keyword to retrieve the function + // signature. + let sugg_span = self.span_extend_to_prev_str(span, "fn", false); + if sugg_span != span { + if let Ok(snippet) = self.span_to_snippet(sugg_span) { + // Consume the function name. + let mut offset = snippet + .find(|c: char| !c.is_alphanumeric() && c != '_') + .expect("no label after fn"); + + // Consume the generics part of the function signature. + let mut bracket_counter = 0; + let mut last_char = None; + for c in snippet[offset..].chars() { + match c { + '<' => bracket_counter += 1, + '>' => bracket_counter -= 1, + '(' => { + if bracket_counter == 0 { + break; + } + } + _ => {} + } + offset += c.len_utf8(); + last_char = Some(c); + } + + // Adjust the suggestion span to encompass the function name with its generics. + let sugg_span = sugg_span.with_hi(BytePos(sugg_span.lo().0 + offset as u32)); + + // Prepare the new suggested snippet to append the type parameter that triggered + // the error in the generics of the function signature. + let mut new_snippet = if last_char == Some('>') { + format!("{}, ", &snippet[..(offset - '>'.len_utf8())]) + } else { + format!("{}<", &snippet[..offset]) + }; + new_snippet + .push_str(&self.span_to_snippet(span).unwrap_or_else(|_| "T".to_string())); + new_snippet.push('>'); + + return Some((sugg_span, new_snippet)); + } + } + + None + } + pub fn ensure_source_file_source_present(&self, source_file: Lrc) -> bool { + source_file.add_external_src(|| match source_file.name { + FileName::Real(ref name) => self.file_loader.read_file(name).ok(), + _ => None, + }) + } + pub fn call_span_if_macro(&self, sp: Span) -> Span { + if self.span_to_filename(sp.clone()).is_macros() { + let v = sp.macro_backtrace(); + if let Some(use_site) = v.last() { + return use_site.call_site; + } + } + sp + } +} + +#[derive(Clone)] +pub struct FilePathMapping { + mapping: Vec<(PathBuf, PathBuf)>, +} + +impl FilePathMapping { + pub fn empty() -> FilePathMapping { + FilePathMapping { mapping: vec![] } + } + + pub fn new(mapping: Vec<(PathBuf, PathBuf)>) -> FilePathMapping { + FilePathMapping { mapping } + } + + /// Applies any path prefix substitution as defined by the mapping. + /// The return value is the remapped path and a boolean indicating whether + /// the path was affected by the mapping. + pub fn map_prefix(&self, path: PathBuf) -> (PathBuf, bool) { + // NOTE: We are iterating over the mapping entries from last to first + // because entries specified later on the command line should + // take precedence. + for &(ref from, ref to) in self.mapping.iter().rev() { + if let Ok(rest) = path.strip_prefix(from) { + return (to.join(rest), true); + } + } + + (path, false) + } +} diff --git a/src/librustc_span/source_map/tests.rs b/src/librustc_span/source_map/tests.rs new file mode 100644 index 00000000000..79df1884f0d --- /dev/null +++ b/src/librustc_span/source_map/tests.rs @@ -0,0 +1,216 @@ +use super::*; + +use rustc_data_structures::sync::Lrc; + +fn init_source_map() -> SourceMap { + let sm = SourceMap::new(FilePathMapping::empty()); + sm.new_source_file(PathBuf::from("blork.rs").into(), "first line.\nsecond line".to_string()); + sm.new_source_file(PathBuf::from("empty.rs").into(), String::new()); + sm.new_source_file(PathBuf::from("blork2.rs").into(), "first line.\nsecond line".to_string()); + sm +} + +/// Tests `lookup_byte_offset`. +#[test] +fn t3() { + let sm = init_source_map(); + + let srcfbp1 = sm.lookup_byte_offset(BytePos(23)); + assert_eq!(srcfbp1.sf.name, PathBuf::from("blork.rs").into()); + assert_eq!(srcfbp1.pos, BytePos(23)); + + let srcfbp1 = sm.lookup_byte_offset(BytePos(24)); + assert_eq!(srcfbp1.sf.name, PathBuf::from("empty.rs").into()); + assert_eq!(srcfbp1.pos, BytePos(0)); + + let srcfbp2 = sm.lookup_byte_offset(BytePos(25)); + assert_eq!(srcfbp2.sf.name, PathBuf::from("blork2.rs").into()); + assert_eq!(srcfbp2.pos, BytePos(0)); +} + +/// Tests `bytepos_to_file_charpos`. +#[test] +fn t4() { + let sm = init_source_map(); + + let cp1 = sm.bytepos_to_file_charpos(BytePos(22)); + assert_eq!(cp1, CharPos(22)); + + let cp2 = sm.bytepos_to_file_charpos(BytePos(25)); + assert_eq!(cp2, CharPos(0)); +} + +/// Tests zero-length `SourceFile`s. +#[test] +fn t5() { + let sm = init_source_map(); + + let loc1 = sm.lookup_char_pos(BytePos(22)); + assert_eq!(loc1.file.name, PathBuf::from("blork.rs").into()); + assert_eq!(loc1.line, 2); + assert_eq!(loc1.col, CharPos(10)); + + let loc2 = sm.lookup_char_pos(BytePos(25)); + assert_eq!(loc2.file.name, PathBuf::from("blork2.rs").into()); + assert_eq!(loc2.line, 1); + assert_eq!(loc2.col, CharPos(0)); +} + +fn init_source_map_mbc() -> SourceMap { + let sm = SourceMap::new(FilePathMapping::empty()); + // "€" is a three-byte UTF8 char. + sm.new_source_file( + PathBuf::from("blork.rs").into(), + "fir€st €€€€ line.\nsecond line".to_string(), + ); + sm.new_source_file( + PathBuf::from("blork2.rs").into(), + "first line€€.\n€ second line".to_string(), + ); + sm +} + +/// Tests `bytepos_to_file_charpos` in the presence of multi-byte chars. +#[test] +fn t6() { + let sm = init_source_map_mbc(); + + let cp1 = sm.bytepos_to_file_charpos(BytePos(3)); + assert_eq!(cp1, CharPos(3)); + + let cp2 = sm.bytepos_to_file_charpos(BytePos(6)); + assert_eq!(cp2, CharPos(4)); + + let cp3 = sm.bytepos_to_file_charpos(BytePos(56)); + assert_eq!(cp3, CharPos(12)); + + let cp4 = sm.bytepos_to_file_charpos(BytePos(61)); + assert_eq!(cp4, CharPos(15)); +} + +/// Test `span_to_lines` for a span ending at the end of a `SourceFile`. +#[test] +fn t7() { + let sm = init_source_map(); + let span = Span::with_root_ctxt(BytePos(12), BytePos(23)); + let file_lines = sm.span_to_lines(span).unwrap(); + + assert_eq!(file_lines.file.name, PathBuf::from("blork.rs").into()); + assert_eq!(file_lines.lines.len(), 1); + assert_eq!(file_lines.lines[0].line_index, 1); +} + +/// Given a string like " ~~~~~~~~~~~~ ", produces a span +/// converting that range. The idea is that the string has the same +/// length as the input, and we uncover the byte positions. Note +/// that this can span lines and so on. +fn span_from_selection(input: &str, selection: &str) -> Span { + assert_eq!(input.len(), selection.len()); + let left_index = selection.find('~').unwrap() as u32; + let right_index = selection.rfind('~').map(|x| x as u32).unwrap_or(left_index); + Span::with_root_ctxt(BytePos(left_index), BytePos(right_index + 1)) +} + +/// Tests `span_to_snippet` and `span_to_lines` for a span converting 3 +/// lines in the middle of a file. +#[test] +fn span_to_snippet_and_lines_spanning_multiple_lines() { + let sm = SourceMap::new(FilePathMapping::empty()); + let inputtext = "aaaaa\nbbbbBB\nCCC\nDDDDDddddd\neee\n"; + let selection = " \n ~~\n~~~\n~~~~~ \n \n"; + sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_string()); + let span = span_from_selection(inputtext, selection); + + // Check that we are extracting the text we thought we were extracting. + assert_eq!(&sm.span_to_snippet(span).unwrap(), "BB\nCCC\nDDDDD"); + + // Check that span_to_lines gives us the complete result with the lines/cols we expected. + let lines = sm.span_to_lines(span).unwrap(); + let expected = vec![ + LineInfo { line_index: 1, start_col: CharPos(4), end_col: CharPos(6) }, + LineInfo { line_index: 2, start_col: CharPos(0), end_col: CharPos(3) }, + LineInfo { line_index: 3, start_col: CharPos(0), end_col: CharPos(5) }, + ]; + assert_eq!(lines.lines, expected); +} + +/// Test span_to_snippet for a span ending at the end of a `SourceFile`. +#[test] +fn t8() { + let sm = init_source_map(); + let span = Span::with_root_ctxt(BytePos(12), BytePos(23)); + let snippet = sm.span_to_snippet(span); + + assert_eq!(snippet, Ok("second line".to_string())); +} + +/// Test `span_to_str` for a span ending at the end of a `SourceFile`. +#[test] +fn t9() { + let sm = init_source_map(); + let span = Span::with_root_ctxt(BytePos(12), BytePos(23)); + let sstr = sm.span_to_string(span); + + assert_eq!(sstr, "blork.rs:2:1: 2:12"); +} + +/// Tests failing to merge two spans on different lines. +#[test] +fn span_merging_fail() { + let sm = SourceMap::new(FilePathMapping::empty()); + let inputtext = "bbbb BB\ncc CCC\n"; + let selection1 = " ~~\n \n"; + let selection2 = " \n ~~~\n"; + sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_owned()); + let span1 = span_from_selection(inputtext, selection1); + let span2 = span_from_selection(inputtext, selection2); + + assert!(sm.merge_spans(span1, span2).is_none()); +} + +/// Returns the span corresponding to the `n`th occurrence of `substring` in `source_text`. +trait SourceMapExtension { + fn span_substr( + &self, + file: &Lrc, + source_text: &str, + substring: &str, + n: usize, + ) -> Span; +} + +impl SourceMapExtension for SourceMap { + fn span_substr( + &self, + file: &Lrc, + source_text: &str, + substring: &str, + n: usize, + ) -> Span { + println!( + "span_substr(file={:?}/{:?}, substring={:?}, n={})", + file.name, file.start_pos, substring, n + ); + let mut i = 0; + let mut hi = 0; + loop { + let offset = source_text[hi..].find(substring).unwrap_or_else(|| { + panic!( + "source_text `{}` does not have {} occurrences of `{}`, only {}", + source_text, n, substring, i + ); + }); + let lo = hi + offset; + hi = lo + substring.len(); + if i == n { + let span = Span::with_root_ctxt( + BytePos(lo as u32 + file.start_pos.0), + BytePos(hi as u32 + file.start_pos.0), + ); + assert_eq!(&self.span_to_snippet(span).unwrap()[..], substring); + return span; + } + i += 1; + } + } +} diff --git a/src/librustc_span/span_encoding.rs b/src/librustc_span/span_encoding.rs new file mode 100644 index 00000000000..d769cf83a03 --- /dev/null +++ b/src/librustc_span/span_encoding.rs @@ -0,0 +1,140 @@ +// Spans are encoded using 1-bit tag and 2 different encoding formats (one for each tag value). +// One format is used for keeping span data inline, +// another contains index into an out-of-line span interner. +// The encoding format for inline spans were obtained by optimizing over crates in rustc/libstd. +// See https://internals.rust-lang.org/t/rfc-compiler-refactoring-spans/1357/28 + +use crate::hygiene::SyntaxContext; +use crate::GLOBALS; +use crate::{BytePos, SpanData}; + +use rustc_data_structures::fx::FxHashMap; + +/// A compressed span. +/// +/// `SpanData` is 12 bytes, which is a bit too big to stick everywhere. `Span` +/// is a form that only takes up 8 bytes, with less space for the length and +/// context. The vast majority (99.9%+) of `SpanData` instances will fit within +/// those 8 bytes; any `SpanData` whose fields don't fit into a `Span` are +/// stored in a separate interner table, and the `Span` will index into that +/// table. Interning is rare enough that the cost is low, but common enough +/// that the code is exercised regularly. +/// +/// An earlier version of this code used only 4 bytes for `Span`, but that was +/// slower because only 80--90% of spans could be stored inline (even less in +/// very large crates) and so the interner was used a lot more. +/// +/// Inline (compressed) format: +/// - `span.base_or_index == span_data.lo` +/// - `span.len_or_tag == len == span_data.hi - span_data.lo` (must be `<= MAX_LEN`) +/// - `span.ctxt == span_data.ctxt` (must be `<= MAX_CTXT`) +/// +/// Interned format: +/// - `span.base_or_index == index` (indexes into the interner table) +/// - `span.len_or_tag == LEN_TAG` (high bit set, all other bits are zero) +/// - `span.ctxt == 0` +/// +/// The inline form uses 0 for the tag value (rather than 1) so that we don't +/// need to mask out the tag bit when getting the length, and so that the +/// dummy span can be all zeroes. +/// +/// Notes about the choice of field sizes: +/// - `base` is 32 bits in both `Span` and `SpanData`, which means that `base` +/// values never cause interning. The number of bits needed for `base` +/// depends on the crate size. 32 bits allows up to 4 GiB of code in a crate. +/// `script-servo` is the largest crate in `rustc-perf`, requiring 26 bits +/// for some spans. +/// - `len` is 15 bits in `Span` (a u16, minus 1 bit for the tag) and 32 bits +/// in `SpanData`, which means that large `len` values will cause interning. +/// The number of bits needed for `len` does not depend on the crate size. +/// The most common number of bits for `len` are 0--7, with a peak usually at +/// 3 or 4, and then it drops off quickly from 8 onwards. 15 bits is enough +/// for 99.99%+ of cases, but larger values (sometimes 20+ bits) might occur +/// dozens of times in a typical crate. +/// - `ctxt` is 16 bits in `Span` and 32 bits in `SpanData`, which means that +/// large `ctxt` values will cause interning. The number of bits needed for +/// `ctxt` values depend partly on the crate size and partly on the form of +/// the code. No crates in `rustc-perf` need more than 15 bits for `ctxt`, +/// but larger crates might need more than 16 bits. +/// +#[derive(Clone, Copy, Eq, PartialEq, Hash)] +pub struct Span { + base_or_index: u32, + len_or_tag: u16, + ctxt_or_zero: u16, +} + +const LEN_TAG: u16 = 0b1000_0000_0000_0000; +const MAX_LEN: u32 = 0b0111_1111_1111_1111; +const MAX_CTXT: u32 = 0b1111_1111_1111_1111; + +/// Dummy span, both position and length are zero, syntax context is zero as well. +pub const DUMMY_SP: Span = Span { base_or_index: 0, len_or_tag: 0, ctxt_or_zero: 0 }; + +impl Span { + #[inline] + pub fn new(mut lo: BytePos, mut hi: BytePos, ctxt: SyntaxContext) -> Self { + if lo > hi { + std::mem::swap(&mut lo, &mut hi); + } + + let (base, len, ctxt2) = (lo.0, hi.0 - lo.0, ctxt.as_u32()); + + if len <= MAX_LEN && ctxt2 <= MAX_CTXT { + // Inline format. + Span { base_or_index: base, len_or_tag: len as u16, ctxt_or_zero: ctxt2 as u16 } + } else { + // Interned format. + let index = with_span_interner(|interner| interner.intern(&SpanData { lo, hi, ctxt })); + Span { base_or_index: index, len_or_tag: LEN_TAG, ctxt_or_zero: 0 } + } + } + + #[inline] + pub fn data(self) -> SpanData { + if self.len_or_tag != LEN_TAG { + // Inline format. + debug_assert!(self.len_or_tag as u32 <= MAX_LEN); + SpanData { + lo: BytePos(self.base_or_index), + hi: BytePos(self.base_or_index + self.len_or_tag as u32), + ctxt: SyntaxContext::from_u32(self.ctxt_or_zero as u32), + } + } else { + // Interned format. + debug_assert!(self.ctxt_or_zero == 0); + let index = self.base_or_index; + with_span_interner(|interner| *interner.get(index)) + } + } +} + +#[derive(Default)] +pub struct SpanInterner { + spans: FxHashMap, + span_data: Vec, +} + +impl SpanInterner { + fn intern(&mut self, span_data: &SpanData) -> u32 { + if let Some(index) = self.spans.get(span_data) { + return *index; + } + + let index = self.spans.len() as u32; + self.span_data.push(*span_data); + self.spans.insert(*span_data, index); + index + } + + #[inline] + fn get(&self, index: u32) -> &SpanData { + &self.span_data[index as usize] + } +} + +// If an interner exists, return it. Otherwise, prepare a fresh one. +#[inline] +fn with_span_interner T>(f: F) -> T { + GLOBALS.with(|globals| f(&mut *globals.span_interner.lock())) +} diff --git a/src/librustc_span/symbol.rs b/src/librustc_span/symbol.rs new file mode 100644 index 00000000000..7ae037faf15 --- /dev/null +++ b/src/librustc_span/symbol.rs @@ -0,0 +1,1213 @@ +//! An "interner" is a data structure that associates values with usize tags and +//! allows bidirectional lookup; i.e., given a value, one can easily find the +//! type, and vice versa. + +use arena::DroplessArena; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHashKey}; +use rustc_index::vec::Idx; +use rustc_macros::{symbols, HashStable_Generic}; +use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; +use rustc_serialize::{UseSpecializedDecodable, UseSpecializedEncodable}; + +use std::cmp::{Ord, PartialEq, PartialOrd}; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::str; + +use crate::{Span, DUMMY_SP, GLOBALS}; + +#[cfg(test)] +mod tests; + +symbols! { + // After modifying this list adjust `is_special`, `is_used_keyword`/`is_unused_keyword`, + // this should be rarely necessary though if the keywords are kept in alphabetic order. + Keywords { + // Special reserved identifiers used internally for elided lifetimes, + // unnamed method parameters, crate root module, error recovery etc. + Invalid: "", + PathRoot: "{{root}}", + DollarCrate: "$crate", + Underscore: "_", + + // Keywords that are used in stable Rust. + As: "as", + Break: "break", + Const: "const", + Continue: "continue", + Crate: "crate", + Else: "else", + Enum: "enum", + Extern: "extern", + False: "false", + Fn: "fn", + For: "for", + If: "if", + Impl: "impl", + In: "in", + Let: "let", + Loop: "loop", + Match: "match", + Mod: "mod", + Move: "move", + Mut: "mut", + Pub: "pub", + Ref: "ref", + Return: "return", + SelfLower: "self", + SelfUpper: "Self", + Static: "static", + Struct: "struct", + Super: "super", + Trait: "trait", + True: "true", + Type: "type", + Unsafe: "unsafe", + Use: "use", + Where: "where", + While: "while", + + // Keywords that are used in unstable Rust or reserved for future use. + Abstract: "abstract", + Become: "become", + Box: "box", + Do: "do", + Final: "final", + Macro: "macro", + Override: "override", + Priv: "priv", + Typeof: "typeof", + Unsized: "unsized", + Virtual: "virtual", + Yield: "yield", + + // Edition-specific keywords that are used in stable Rust. + Async: "async", // >= 2018 Edition only + Await: "await", // >= 2018 Edition only + Dyn: "dyn", // >= 2018 Edition only + + // Edition-specific keywords that are used in unstable Rust or reserved for future use. + Try: "try", // >= 2018 Edition only + + // Special lifetime names + UnderscoreLifetime: "'_", + StaticLifetime: "'static", + + // Weak keywords, have special meaning only in specific contexts. + Auto: "auto", + Catch: "catch", + Default: "default", + Raw: "raw", + Union: "union", + } + + // Symbols that can be referred to with syntax_pos::sym::*. The symbol is + // the stringified identifier unless otherwise specified (e.g. + // `proc_dash_macro` represents "proc-macro"). + // + // As well as the symbols listed, there are symbols for the the strings + // "0", "1", ..., "9", which are accessible via `sym::integer`. + Symbols { + aarch64_target_feature, + abi, + abi_amdgpu_kernel, + abi_efiapi, + abi_msp430_interrupt, + abi_ptx, + abi_sysv64, + abi_thiscall, + abi_unadjusted, + abi_vectorcall, + abi_x86_interrupt, + aborts, + add_with_overflow, + advanced_slice_patterns, + adx_target_feature, + alias, + align, + alignstack, + all, + allocator, + allocator_internals, + alloc_error_handler, + allow, + allowed, + allow_fail, + allow_internal_unsafe, + allow_internal_unstable, + allow_internal_unstable_backcompat_hack, + always, + and, + any, + arbitrary_enum_discriminant, + arbitrary_self_types, + Arguments, + ArgumentV1, + arm_target_feature, + asm, + assert, + associated_consts, + associated_type_bounds, + associated_type_defaults, + associated_types, + assume_init, + async_await, + async_closure, + attr, + attributes, + attr_literals, + augmented_assignments, + automatically_derived, + avx512_target_feature, + await_macro, + begin_panic, + bench, + bin, + bind_by_move_pattern_guards, + bindings_after_at, + block, + bool, + borrowck_graphviz_postflow, + borrowck_graphviz_preflow, + box_patterns, + box_syntax, + braced_empty_structs, + bswap, + bitreverse, + C, + caller_location, + cdylib, + cfg, + cfg_attr, + cfg_attr_multi, + cfg_doctest, + cfg_sanitize, + cfg_target_feature, + cfg_target_has_atomic, + cfg_target_thread_local, + cfg_target_vendor, + char, + clippy, + clone, + Clone, + clone_closures, + clone_from, + closure_to_fn_coercion, + cmp, + cmpxchg16b_target_feature, + cold, + column, + compile_error, + compiler_builtins, + concat, + concat_idents, + conservative_impl_trait, + console, + const_compare_raw_pointers, + const_constructor, + const_extern_fn, + const_fn, + const_fn_union, + const_generics, + const_if_match, + const_indexing, + const_in_array_repeat_expressions, + const_let, + const_loop, + const_mut_refs, + const_panic, + const_raw_ptr_deref, + const_raw_ptr_to_usize_cast, + const_transmute, + contents, + context, + convert, + Copy, + copy_closures, + core, + core_intrinsics, + crate_id, + crate_in_paths, + crate_local, + crate_name, + crate_type, + crate_visibility_modifier, + ctpop, + cttz, + cttz_nonzero, + ctlz, + ctlz_nonzero, + custom_attribute, + custom_derive, + custom_inner_attributes, + custom_test_frameworks, + c_variadic, + debug_trait, + declare_lint_pass, + decl_macro, + Debug, + Decodable, + Default, + default_lib_allocator, + default_type_parameter_fallback, + default_type_params, + delay_span_bug_from_inside_query, + deny, + deprecated, + deref, + deref_mut, + derive, + diagnostic, + direct, + doc, + doc_alias, + doc_cfg, + doc_keyword, + doc_masked, + doc_spotlight, + doctest, + document_private_items, + dotdoteq_in_patterns, + dotdot_in_tuple_patterns, + double_braced_crate: "{{crate}}", + double_braced_impl: "{{impl}}", + double_braced_misc: "{{misc}}", + double_braced_closure: "{{closure}}", + double_braced_constructor: "{{constructor}}", + double_braced_constant: "{{constant}}", + double_braced_opaque: "{{opaque}}", + dropck_eyepatch, + dropck_parametricity, + drop_types_in_const, + dylib, + dyn_trait, + eh_personality, + eh_unwind_resume, + enable, + Encodable, + env, + eq, + err, + Err, + Eq, + Equal, + enclosing_scope, + except, + exclusive_range_pattern, + exhaustive_integer_patterns, + exhaustive_patterns, + existential_type, + expected, + export_name, + expr, + extern_absolute_paths, + external_doc, + extern_crate_item_prelude, + extern_crate_self, + extern_in_paths, + extern_prelude, + extern_types, + f16c_target_feature, + f32, + f64, + feature, + ffi_returns_twice, + field, + field_init_shorthand, + file, + fmt, + fmt_internals, + fn_must_use, + forbid, + format_args, + format_args_nl, + from, + From, + from_desugaring, + from_error, + from_generator, + from_method, + from_ok, + from_usize, + fundamental, + future, + Future, + FxHashSet, + FxHashMap, + gen_future, + generators, + generic_associated_types, + generic_param_attrs, + global_allocator, + global_asm, + globs, + hash, + Hash, + HashSet, + HashMap, + hexagon_target_feature, + hidden, + homogeneous_aggregate, + html_favicon_url, + html_logo_url, + html_no_source, + html_playground_url, + html_root_url, + i128, + i128_type, + i16, + i32, + i64, + i8, + ident, + if_let, + if_while_or_patterns, + ignore, + impl_header_lifetime_elision, + impl_lint_pass, + impl_trait_in_bindings, + import_shadowing, + index, + index_mut, + in_band_lifetimes, + include, + include_bytes, + include_str, + inclusive_range_syntax, + infer_outlives_requirements, + infer_static_outlives_requirements, + inline, + intel, + into_future, + IntoFuture, + into_iter, + IntoIterator, + into_result, + intrinsics, + irrefutable_let_patterns, + isize, + issue, + issue_5723_bootstrap, + issue_tracker_base_url, + item, + item_context: "ItemContext", + item_like_imports, + iter, + Iterator, + keyword, + kind, + label, + label_break_value, + lang, + lang_items, + let_chains, + lhs, + lib, + lifetime, + line, + link, + linkage, + link_args, + link_cfg, + link_llvm_intrinsics, + link_name, + link_ordinal, + link_section, + LintPass, + lint_reasons, + literal, + local_inner_macros, + log_syntax, + loop_break_value, + macro_at_most_once_rep, + macro_escape, + macro_export, + macro_lifetime_matcher, + macro_literal_matcher, + macro_reexport, + macro_rules, + macros_in_extern, + macro_use, + macro_vis_matcher, + main, + managed_boxes, + marker, + marker_trait_attr, + masked, + match_beginning_vert, + match_default_bindings, + may_dangle, + maybe_uninit_uninit, + maybe_uninit_zeroed, + mem_uninitialized, + mem_zeroed, + member_constraints, + message, + meta, + min_align_of, + min_const_fn, + min_const_unsafe_fn, + mips_target_feature, + mmx_target_feature, + module, + module_path, + more_struct_aliases, + move_val_init, + movbe_target_feature, + mul_with_overflow, + must_use, + naked, + naked_functions, + name, + needs_allocator, + needs_drop, + needs_panic_runtime, + negate_unsigned, + never, + never_type, + never_type_fallback, + new, + next, + __next, + nll, + no_builtins, + no_core, + no_crate_inject, + no_debug, + no_default_passes, + no_implicit_prelude, + no_inline, + no_link, + no_main, + no_mangle, + non_ascii_idents, + None, + non_exhaustive, + non_modrs_mods, + no_stack_check, + no_start, + no_std, + not, + note, + object_safe_for_dispatch, + Ok, + omit_gdb_pretty_printer_section, + on, + on_unimplemented, + oom, + ops, + optimize, + optimize_attribute, + optin_builtin_traits, + option, + Option, + option_env, + opt_out_copy, + or, + or_patterns, + Ord, + Ordering, + Output, + overlapping_marker_traits, + packed, + panic, + panic_handler, + panic_impl, + panic_implementation, + panic_runtime, + parent_trait, + partial_cmp, + param_attrs, + PartialEq, + PartialOrd, + passes, + pat, + path, + pattern_parentheses, + Pending, + pin, + Pin, + pinned, + platform_intrinsics, + plugin, + plugin_registrar, + plugins, + Poll, + poll_with_tls_context, + powerpc_target_feature, + precise_pointer_size_matching, + pref_align_of, + prelude, + prelude_import, + primitive, + proc_dash_macro: "proc-macro", + proc_macro, + proc_macro_attribute, + proc_macro_def_site, + proc_macro_derive, + proc_macro_expr, + proc_macro_gen, + proc_macro_hygiene, + proc_macro_internals, + proc_macro_mod, + proc_macro_non_items, + proc_macro_path_invoc, + profiler_runtime, + ptr_offset_from, + pub_restricted, + pushpop_unsafe, + quad_precision_float, + question_mark, + quote, + Range, + RangeFrom, + RangeFull, + RangeInclusive, + RangeTo, + RangeToInclusive, + raw_dylib, + raw_identifiers, + raw_ref_op, + Ready, + reason, + recursion_limit, + reexport_test_harness_main, + reflect, + register_attr, + register_tool, + relaxed_adts, + repr, + repr128, + repr_align, + repr_align_enum, + repr_packed, + repr_simd, + repr_transparent, + re_rebalance_coherence, + result, + Result, + Return, + rhs, + rlib, + rotate_left, + rotate_right, + rt, + rtm_target_feature, + rust, + rust_2015_preview, + rust_2018_preview, + rust_begin_unwind, + rustc, + RustcDecodable, + RustcEncodable, + rustc_allocator, + rustc_allocator_nounwind, + rustc_allow_const_fn_ptr, + rustc_args_required_const, + rustc_attrs, + rustc_builtin_macro, + rustc_clean, + rustc_const_unstable, + rustc_const_stable, + rustc_conversion_suggestion, + rustc_def_path, + rustc_deprecated, + rustc_diagnostic_item, + rustc_diagnostic_macros, + rustc_dirty, + rustc_dummy, + rustc_dump_env_program_clauses, + rustc_dump_program_clauses, + rustc_dump_user_substs, + rustc_error, + rustc_expected_cgu_reuse, + rustc_if_this_changed, + rustc_inherit_overflow_checks, + rustc_layout, + rustc_layout_scalar_valid_range_end, + rustc_layout_scalar_valid_range_start, + rustc_macro_transparency, + rustc_mir, + rustc_nonnull_optimization_guaranteed, + rustc_object_lifetime_default, + rustc_on_unimplemented, + rustc_outlives, + rustc_paren_sugar, + rustc_partition_codegened, + rustc_partition_reused, + rustc_peek, + rustc_peek_definite_init, + rustc_peek_maybe_init, + rustc_peek_maybe_uninit, + rustc_peek_indirectly_mutable, + rustc_private, + rustc_proc_macro_decls, + rustc_promotable, + rustc_regions, + rustc_stable, + rustc_std_internal_symbol, + rustc_symbol_name, + rustc_synthetic, + rustc_reservation_impl, + rustc_test_marker, + rustc_then_this_would_need, + rustc_variance, + rustfmt, + rust_eh_personality, + rust_eh_unwind_resume, + rust_oom, + rvalue_static_promotion, + sanitize, + sanitizer_runtime, + saturating_add, + saturating_sub, + _Self, + self_in_typedefs, + self_struct_ctor, + send_trait, + should_panic, + simd, + simd_extract, + simd_ffi, + simd_insert, + since, + size, + size_of, + slice_patterns, + slicing_syntax, + soft, + Some, + specialization, + speed, + spotlight, + sse4a_target_feature, + stable, + staged_api, + start, + static_in_const, + staticlib, + static_nobundle, + static_recursion, + std, + std_inject, + str, + stringify, + stmt, + stmt_expr_attributes, + stop_after_dataflow, + struct_field_attributes, + struct_inherit, + structural_match, + struct_variant, + sty, + sub_with_overflow, + suggestion, + sync_trait, + target_feature, + target_has_atomic, + target_has_atomic_load_store, + target_thread_local, + task, + tbm_target_feature, + termination_trait, + termination_trait_test, + test, + test_2018_feature, + test_accepted_feature, + test_case, + test_removed_feature, + test_runner, + then_with, + thread_local, + tool_attributes, + tool_lints, + trace_macros, + track_caller, + trait_alias, + transmute, + transparent, + transparent_enums, + transparent_unions, + trivial_bounds, + Try, + try_blocks, + try_trait, + tt, + tuple_indexing, + Ty, + ty, + type_alias_impl_trait, + type_id, + type_name, + TyCtxt, + TyKind, + type_alias_enum_variants, + type_ascription, + type_length_limit, + type_macros, + u128, + u16, + u32, + u64, + u8, + unboxed_closures, + unchecked_shl, + unchecked_shr, + underscore_const_names, + underscore_imports, + underscore_lifetimes, + uniform_paths, + universal_impl_trait, + unmarked_api, + unreachable_code, + unrestricted_attribute_tokens, + unsafe_no_drop_flag, + unsized_locals, + unsized_tuple_coercion, + unstable, + untagged_unions, + unwind, + unwind_attributes, + unwrap_or, + used, + use_extern_macros, + use_nested_groups, + usize, + v1, + val, + var, + vec, + Vec, + vis, + visible_private_types, + volatile, + warn, + wasm_import_module, + wasm_target_feature, + while_let, + windows, + windows_subsystem, + wrapping_add, + wrapping_sub, + wrapping_mul, + Yield, + } +} + +#[derive(Copy, Clone, Eq, HashStable_Generic)] +pub struct Ident { + pub name: Symbol, + pub span: Span, +} + +impl Ident { + #[inline] + /// Constructs a new identifier from a symbol and a span. + pub const fn new(name: Symbol, span: Span) -> Ident { + Ident { name, span } + } + + /// Constructs a new identifier with a dummy span. + #[inline] + pub const fn with_dummy_span(name: Symbol) -> Ident { + Ident::new(name, DUMMY_SP) + } + + #[inline] + pub fn invalid() -> Ident { + Ident::with_dummy_span(kw::Invalid) + } + + /// Maps a string to an identifier with a dummy span. + pub fn from_str(string: &str) -> Ident { + Ident::with_dummy_span(Symbol::intern(string)) + } + + /// Maps a string and a span to an identifier. + pub fn from_str_and_span(string: &str, span: Span) -> Ident { + Ident::new(Symbol::intern(string), span) + } + + /// Replaces `lo` and `hi` with those from `span`, but keep hygiene context. + pub fn with_span_pos(self, span: Span) -> Ident { + Ident::new(self.name, span.with_ctxt(self.span.ctxt())) + } + + pub fn without_first_quote(self) -> Ident { + Ident::new(Symbol::intern(self.as_str().trim_start_matches('\'')), self.span) + } + + /// "Normalize" ident for use in comparisons using "item hygiene". + /// Identifiers with same string value become same if they came from the same "modern" macro + /// (e.g., `macro` item, but not `macro_rules` item) and stay different if they came from + /// different "modern" macros. + /// Technically, this operation strips all non-opaque marks from ident's syntactic context. + pub fn modern(self) -> Ident { + Ident::new(self.name, self.span.modern()) + } + + /// "Normalize" ident for use in comparisons using "local variable hygiene". + /// Identifiers with same string value become same if they came from the same non-transparent + /// macro (e.g., `macro` or `macro_rules!` items) and stay different if they came from different + /// non-transparent macros. + /// Technically, this operation strips all transparent marks from ident's syntactic context. + pub fn modern_and_legacy(self) -> Ident { + Ident::new(self.name, self.span.modern_and_legacy()) + } + + /// Convert the name to a `SymbolStr`. This is a slowish operation because + /// it requires locking the symbol interner. + pub fn as_str(self) -> SymbolStr { + self.name.as_str() + } +} + +impl PartialEq for Ident { + fn eq(&self, rhs: &Self) -> bool { + self.name == rhs.name && self.span.ctxt() == rhs.span.ctxt() + } +} + +impl Hash for Ident { + fn hash(&self, state: &mut H) { + self.name.hash(state); + self.span.ctxt().hash(state); + } +} + +impl fmt::Debug for Ident { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.is_raw_guess() { + write!(f, "r#")?; + } + write!(f, "{}{:?}", self.name, self.span.ctxt()) + } +} + +impl fmt::Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.is_raw_guess() { + write!(f, "r#")?; + } + fmt::Display::fmt(&self.name, f) + } +} + +impl UseSpecializedEncodable for Ident { + fn default_encode(&self, s: &mut S) -> Result<(), S::Error> { + s.emit_struct("Ident", 2, |s| { + s.emit_struct_field("name", 0, |s| self.name.encode(s))?; + s.emit_struct_field("span", 1, |s| self.span.encode(s)) + }) + } +} + +impl UseSpecializedDecodable for Ident { + fn default_decode(d: &mut D) -> Result { + d.read_struct("Ident", 2, |d| { + Ok(Ident { + name: d.read_struct_field("name", 0, Decodable::decode)?, + span: d.read_struct_field("span", 1, Decodable::decode)?, + }) + }) + } +} + +/// An interned string. +/// +/// Internally, a `Symbol` is implemented as an index, and all operations +/// (including hashing, equality, and ordering) operate on that index. The use +/// of `rustc_index::newtype_index!` means that `Option` only takes up 4 bytes, +/// because `rustc_index::newtype_index!` reserves the last 256 values for tagging purposes. +/// +/// Note that `Symbol` cannot directly be a `rustc_index::newtype_index!` because it +/// implements `fmt::Debug`, `Encodable`, and `Decodable` in special ways. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Symbol(SymbolIndex); + +rustc_index::newtype_index! { + pub struct SymbolIndex { .. } +} + +impl Symbol { + const fn new(n: u32) -> Self { + Symbol(SymbolIndex::from_u32_const(n)) + } + + /// Maps a string to its interned representation. + pub fn intern(string: &str) -> Self { + with_interner(|interner| interner.intern(string)) + } + + /// Access the symbol's chars. This is a slowish operation because it + /// requires locking the symbol interner. + pub fn with R, R>(self, f: F) -> R { + with_interner(|interner| f(interner.get(self))) + } + + /// Convert to a `SymbolStr`. This is a slowish operation because it + /// requires locking the symbol interner. + pub fn as_str(self) -> SymbolStr { + with_interner(|interner| unsafe { + SymbolStr { string: std::mem::transmute::<&str, &str>(interner.get(self)) } + }) + } + + pub fn as_u32(self) -> u32 { + self.0.as_u32() + } +} + +impl fmt::Debug for Symbol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.with(|str| fmt::Debug::fmt(&str, f)) + } +} + +impl fmt::Display for Symbol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.with(|str| fmt::Display::fmt(&str, f)) + } +} + +impl Encodable for Symbol { + fn encode(&self, s: &mut S) -> Result<(), S::Error> { + self.with(|string| s.emit_str(string)) + } +} + +impl Decodable for Symbol { + fn decode(d: &mut D) -> Result { + Ok(Symbol::intern(&d.read_str()?)) + } +} + +impl HashStable for Symbol { + #[inline] + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + self.as_str().hash_stable(hcx, hasher); + } +} + +impl ToStableHashKey for Symbol { + type KeyType = SymbolStr; + + #[inline] + fn to_stable_hash_key(&self, _: &CTX) -> SymbolStr { + self.as_str() + } +} + +// The `&'static str`s in this type actually point into the arena. +#[derive(Default)] +pub struct Interner { + arena: DroplessArena, + names: FxHashMap<&'static str, Symbol>, + strings: Vec<&'static str>, +} + +impl Interner { + fn prefill(init: &[&'static str]) -> Self { + Interner { + strings: init.into(), + names: init.iter().copied().zip((0..).map(Symbol::new)).collect(), + ..Default::default() + } + } + + pub fn intern(&mut self, string: &str) -> Symbol { + if let Some(&name) = self.names.get(string) { + return name; + } + + let name = Symbol::new(self.strings.len() as u32); + + // `from_utf8_unchecked` is safe since we just allocated a `&str` which is known to be + // UTF-8. + let string: &str = + unsafe { str::from_utf8_unchecked(self.arena.alloc_slice(string.as_bytes())) }; + // It is safe to extend the arena allocation to `'static` because we only access + // these while the arena is still alive. + let string: &'static str = unsafe { &*(string as *const str) }; + self.strings.push(string); + self.names.insert(string, name); + name + } + + // Get the symbol as a string. `Symbol::as_str()` should be used in + // preference to this function. + pub fn get(&self, symbol: Symbol) -> &str { + self.strings[symbol.0.as_usize()] + } +} + +// This module has a very short name because it's used a lot. +pub mod kw { + use super::Symbol; + keywords!(); +} + +// This module has a very short name because it's used a lot. +pub mod sym { + use super::Symbol; + use std::convert::TryInto; + + symbols!(); + + // Get the symbol for an integer. The first few non-negative integers each + // have a static symbol and therefore are fast. + pub fn integer + Copy + ToString>(n: N) -> Symbol { + if let Result::Ok(idx) = n.try_into() { + if let Option::Some(&sym) = digits_array.get(idx) { + return sym; + } + } + Symbol::intern(&n.to_string()) + } +} + +impl Symbol { + fn is_used_keyword_2018(self) -> bool { + self >= kw::Async && self <= kw::Dyn + } + + fn is_unused_keyword_2018(self) -> bool { + self == kw::Try + } + + /// Used for sanity checking rustdoc keyword sections. + pub fn is_doc_keyword(self) -> bool { + self <= kw::Union + } + + /// A keyword or reserved identifier that can be used as a path segment. + pub fn is_path_segment_keyword(self) -> bool { + self == kw::Super + || self == kw::SelfLower + || self == kw::SelfUpper + || self == kw::Crate + || self == kw::PathRoot + || self == kw::DollarCrate + } + + /// Returns `true` if the symbol is `true` or `false`. + pub fn is_bool_lit(self) -> bool { + self == kw::True || self == kw::False + } + + /// This symbol can be a raw identifier. + pub fn can_be_raw(self) -> bool { + self != kw::Invalid && self != kw::Underscore && !self.is_path_segment_keyword() + } +} + +impl Ident { + // Returns `true` for reserved identifiers used internally for elided lifetimes, + // unnamed method parameters, crate root module, error recovery etc. + pub fn is_special(self) -> bool { + self.name <= kw::Underscore + } + + /// Returns `true` if the token is a keyword used in the language. + pub fn is_used_keyword(self) -> bool { + // Note: `span.edition()` is relatively expensive, don't call it unless necessary. + self.name >= kw::As && self.name <= kw::While + || self.name.is_used_keyword_2018() && self.span.rust_2018() + } + + /// Returns `true` if the token is a keyword reserved for possible future use. + pub fn is_unused_keyword(self) -> bool { + // Note: `span.edition()` is relatively expensive, don't call it unless necessary. + self.name >= kw::Abstract && self.name <= kw::Yield + || self.name.is_unused_keyword_2018() && self.span.rust_2018() + } + + /// Returns `true` if the token is either a special identifier or a keyword. + pub fn is_reserved(self) -> bool { + self.is_special() || self.is_used_keyword() || self.is_unused_keyword() + } + + /// A keyword or reserved identifier that can be used as a path segment. + pub fn is_path_segment_keyword(self) -> bool { + self.name.is_path_segment_keyword() + } + + /// We see this identifier in a normal identifier position, like variable name or a type. + /// How was it written originally? Did it use the raw form? Let's try to guess. + pub fn is_raw_guess(self) -> bool { + self.name.can_be_raw() && self.is_reserved() + } +} + +#[inline] +fn with_interner T>(f: F) -> T { + GLOBALS.with(|globals| f(&mut *globals.symbol_interner.lock())) +} + +/// An alternative to `Symbol`, useful when the chars within the symbol need to +/// be accessed. It deliberately has limited functionality and should only be +/// used for temporary values. +/// +/// Because the interner outlives any thread which uses this type, we can +/// safely treat `string` which points to interner data, as an immortal string, +/// as long as this type never crosses between threads. +// +// FIXME: ensure that the interner outlives any thread which uses `SymbolStr`, +// by creating a new thread right after constructing the interner. +#[derive(Clone, Eq, PartialOrd, Ord)] +pub struct SymbolStr { + string: &'static str, +} + +// This impl allows a `SymbolStr` to be directly equated with a `String` or +// `&str`. +impl> std::cmp::PartialEq for SymbolStr { + fn eq(&self, other: &T) -> bool { + self.string == other.deref() + } +} + +impl !Send for SymbolStr {} +impl !Sync for SymbolStr {} + +/// This impl means that if `ss` is a `SymbolStr`: +/// - `*ss` is a `str`; +/// - `&*ss` is a `&str`; +/// - `&ss as &str` is a `&str`, which means that `&ss` can be passed to a +/// function expecting a `&str`. +impl std::ops::Deref for SymbolStr { + type Target = str; + #[inline] + fn deref(&self) -> &str { + self.string + } +} + +impl fmt::Debug for SymbolStr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(self.string, f) + } +} + +impl fmt::Display for SymbolStr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self.string, f) + } +} + +impl HashStable for SymbolStr { + #[inline] + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + self.string.hash_stable(hcx, hasher) + } +} + +impl ToStableHashKey for SymbolStr { + type KeyType = SymbolStr; + + #[inline] + fn to_stable_hash_key(&self, _: &CTX) -> SymbolStr { + self.clone() + } +} diff --git a/src/librustc_span/symbol/tests.rs b/src/librustc_span/symbol/tests.rs new file mode 100644 index 00000000000..f74b9a0cd1d --- /dev/null +++ b/src/librustc_span/symbol/tests.rs @@ -0,0 +1,25 @@ +use super::*; + +use crate::{edition, Globals}; + +#[test] +fn interner_tests() { + let mut i: Interner = Interner::default(); + // first one is zero: + assert_eq!(i.intern("dog"), Symbol::new(0)); + // re-use gets the same entry: + assert_eq!(i.intern("dog"), Symbol::new(0)); + // different string gets a different #: + assert_eq!(i.intern("cat"), Symbol::new(1)); + assert_eq!(i.intern("cat"), Symbol::new(1)); + // dog is still at zero + assert_eq!(i.intern("dog"), Symbol::new(0)); +} + +#[test] +fn without_first_quote_test() { + GLOBALS.set(&Globals::new(edition::DEFAULT_EDITION), || { + let i = Ident::from_str("'break"); + assert_eq!(i.without_first_quote().name, kw::Break); + }); +} diff --git a/src/librustc_span/tests.rs b/src/librustc_span/tests.rs new file mode 100644 index 00000000000..3c8eb8bcd31 --- /dev/null +++ b/src/librustc_span/tests.rs @@ -0,0 +1,40 @@ +use super::*; + +#[test] +fn test_lookup_line() { + let lines = &[BytePos(3), BytePos(17), BytePos(28)]; + + assert_eq!(lookup_line(lines, BytePos(0)), -1); + assert_eq!(lookup_line(lines, BytePos(3)), 0); + assert_eq!(lookup_line(lines, BytePos(4)), 0); + + assert_eq!(lookup_line(lines, BytePos(16)), 0); + assert_eq!(lookup_line(lines, BytePos(17)), 1); + assert_eq!(lookup_line(lines, BytePos(18)), 1); + + assert_eq!(lookup_line(lines, BytePos(28)), 2); + assert_eq!(lookup_line(lines, BytePos(29)), 2); +} + +#[test] +fn test_normalize_newlines() { + fn check(before: &str, after: &str, expected_positions: &[u32]) { + let mut actual = before.to_string(); + let mut actual_positions = vec![]; + normalize_newlines(&mut actual, &mut actual_positions); + let actual_positions: Vec<_> = actual_positions.into_iter().map(|nc| nc.pos.0).collect(); + assert_eq!(actual.as_str(), after); + assert_eq!(actual_positions, expected_positions); + } + check("", "", &[]); + check("\n", "\n", &[]); + check("\r", "\r", &[]); + check("\r\r", "\r\r", &[]); + check("\r\n", "\n", &[1]); + check("hello world", "hello world", &[]); + check("hello\nworld", "hello\nworld", &[]); + check("hello\r\nworld", "hello\nworld", &[6]); + check("\r\nhello\r\nworld\r\n", "\nhello\nworld\n", &[1, 7, 13]); + check("\r\r\n", "\r\n", &[2]); + check("hello\rworld", "hello\rworld", &[]); +} diff --git a/src/libsyntax_expand/Cargo.toml b/src/libsyntax_expand/Cargo.toml deleted file mode 100644 index 897d5a65ba3..00000000000 --- a/src/libsyntax_expand/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -authors = ["The Rust Project Developers"] -name = "syntax_expand" -version = "0.0.0" -edition = "2018" -build = false - -[lib] -name = "syntax_expand" -path = "lib.rs" -doctest = false - -[dependencies] -rustc_serialize = { path = "../libserialize", package = "serialize" } -log = "0.4" -syntax_pos = { path = "../libsyntax_pos" } -errors = { path = "../librustc_errors", package = "rustc_errors" } -rustc_data_structures = { path = "../librustc_data_structures" } -rustc_feature = { path = "../librustc_feature" } -rustc_lexer = { path = "../librustc_lexer" } -rustc_parse = { path = "../librustc_parse" } -smallvec = { version = "1.0", features = ["union", "may_dangle"] } -syntax = { path = "../libsyntax" } diff --git a/src/libsyntax_expand/base.rs b/src/libsyntax_expand/base.rs deleted file mode 100644 index 60bc591c095..00000000000 --- a/src/libsyntax_expand/base.rs +++ /dev/null @@ -1,1181 +0,0 @@ -use crate::expand::{self, AstFragment, Invocation}; - -use rustc_parse::{self, parser, DirectoryOwnership, MACRO_ARGUMENTS}; -use syntax::ast::{self, Attribute, Name, NodeId, PatKind}; -use syntax::attr::{self, Deprecation, HasAttrs, Stability}; -use syntax::edition::Edition; -use syntax::mut_visit::{self, MutVisitor}; -use syntax::ptr::P; -use syntax::sess::ParseSess; -use syntax::source_map::SourceMap; -use syntax::symbol::{kw, sym, Ident, Symbol}; -use syntax::token; -use syntax::tokenstream::{self, TokenStream}; -use syntax::visit::Visitor; - -use errors::{DiagnosticBuilder, DiagnosticId}; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::{self, Lrc}; -use smallvec::{smallvec, SmallVec}; -use syntax_pos::hygiene::{AstPass, ExpnData, ExpnId, ExpnKind}; -use syntax_pos::{FileName, MultiSpan, Span, DUMMY_SP}; - -use std::default::Default; -use std::iter; -use std::path::PathBuf; -use std::rc::Rc; - -crate use syntax_pos::hygiene::MacroKind; - -#[derive(Debug, Clone)] -pub enum Annotatable { - Item(P), - TraitItem(P), - ImplItem(P), - ForeignItem(P), - Stmt(P), - Expr(P), - Arm(ast::Arm), - Field(ast::Field), - FieldPat(ast::FieldPat), - GenericParam(ast::GenericParam), - Param(ast::Param), - StructField(ast::StructField), - Variant(ast::Variant), -} - -impl HasAttrs for Annotatable { - fn attrs(&self) -> &[Attribute] { - match *self { - Annotatable::Item(ref item) => &item.attrs, - Annotatable::TraitItem(ref trait_item) => &trait_item.attrs, - Annotatable::ImplItem(ref impl_item) => &impl_item.attrs, - Annotatable::ForeignItem(ref foreign_item) => &foreign_item.attrs, - Annotatable::Stmt(ref stmt) => stmt.attrs(), - Annotatable::Expr(ref expr) => &expr.attrs, - Annotatable::Arm(ref arm) => &arm.attrs, - Annotatable::Field(ref field) => &field.attrs, - Annotatable::FieldPat(ref fp) => &fp.attrs, - Annotatable::GenericParam(ref gp) => &gp.attrs, - Annotatable::Param(ref p) => &p.attrs, - Annotatable::StructField(ref sf) => &sf.attrs, - Annotatable::Variant(ref v) => &v.attrs(), - } - } - - fn visit_attrs)>(&mut self, f: F) { - match self { - Annotatable::Item(item) => item.visit_attrs(f), - Annotatable::TraitItem(trait_item) => trait_item.visit_attrs(f), - Annotatable::ImplItem(impl_item) => impl_item.visit_attrs(f), - Annotatable::ForeignItem(foreign_item) => foreign_item.visit_attrs(f), - Annotatable::Stmt(stmt) => stmt.visit_attrs(f), - Annotatable::Expr(expr) => expr.visit_attrs(f), - Annotatable::Arm(arm) => arm.visit_attrs(f), - Annotatable::Field(field) => field.visit_attrs(f), - Annotatable::FieldPat(fp) => fp.visit_attrs(f), - Annotatable::GenericParam(gp) => gp.visit_attrs(f), - Annotatable::Param(p) => p.visit_attrs(f), - Annotatable::StructField(sf) => sf.visit_attrs(f), - Annotatable::Variant(v) => v.visit_attrs(f), - } - } -} - -impl Annotatable { - pub fn span(&self) -> Span { - match *self { - Annotatable::Item(ref item) => item.span, - Annotatable::TraitItem(ref trait_item) => trait_item.span, - Annotatable::ImplItem(ref impl_item) => impl_item.span, - Annotatable::ForeignItem(ref foreign_item) => foreign_item.span, - Annotatable::Stmt(ref stmt) => stmt.span, - Annotatable::Expr(ref expr) => expr.span, - Annotatable::Arm(ref arm) => arm.span, - Annotatable::Field(ref field) => field.span, - Annotatable::FieldPat(ref fp) => fp.pat.span, - Annotatable::GenericParam(ref gp) => gp.ident.span, - Annotatable::Param(ref p) => p.span, - Annotatable::StructField(ref sf) => sf.span, - Annotatable::Variant(ref v) => v.span, - } - } - - pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { - match self { - Annotatable::Item(item) => visitor.visit_item(item), - Annotatable::TraitItem(trait_item) => visitor.visit_trait_item(trait_item), - Annotatable::ImplItem(impl_item) => visitor.visit_impl_item(impl_item), - Annotatable::ForeignItem(foreign_item) => visitor.visit_foreign_item(foreign_item), - Annotatable::Stmt(stmt) => visitor.visit_stmt(stmt), - Annotatable::Expr(expr) => visitor.visit_expr(expr), - Annotatable::Arm(arm) => visitor.visit_arm(arm), - Annotatable::Field(field) => visitor.visit_field(field), - Annotatable::FieldPat(fp) => visitor.visit_field_pattern(fp), - Annotatable::GenericParam(gp) => visitor.visit_generic_param(gp), - Annotatable::Param(p) => visitor.visit_param(p), - Annotatable::StructField(sf) => visitor.visit_struct_field(sf), - Annotatable::Variant(v) => visitor.visit_variant(v), - } - } - - pub fn expect_item(self) -> P { - match self { - Annotatable::Item(i) => i, - _ => panic!("expected Item"), - } - } - - pub fn map_item_or(self, mut f: F, mut or: G) -> Annotatable - where - F: FnMut(P) -> P, - G: FnMut(Annotatable) -> Annotatable, - { - match self { - Annotatable::Item(i) => Annotatable::Item(f(i)), - _ => or(self), - } - } - - pub fn expect_trait_item(self) -> ast::AssocItem { - match self { - Annotatable::TraitItem(i) => i.into_inner(), - _ => panic!("expected Item"), - } - } - - pub fn expect_impl_item(self) -> ast::AssocItem { - match self { - Annotatable::ImplItem(i) => i.into_inner(), - _ => panic!("expected Item"), - } - } - - pub fn expect_foreign_item(self) -> ast::ForeignItem { - match self { - Annotatable::ForeignItem(i) => i.into_inner(), - _ => panic!("expected foreign item"), - } - } - - pub fn expect_stmt(self) -> ast::Stmt { - match self { - Annotatable::Stmt(stmt) => stmt.into_inner(), - _ => panic!("expected statement"), - } - } - - pub fn expect_expr(self) -> P { - match self { - Annotatable::Expr(expr) => expr, - _ => panic!("expected expression"), - } - } - - pub fn expect_arm(self) -> ast::Arm { - match self { - Annotatable::Arm(arm) => arm, - _ => panic!("expected match arm"), - } - } - - pub fn expect_field(self) -> ast::Field { - match self { - Annotatable::Field(field) => field, - _ => panic!("expected field"), - } - } - - pub fn expect_field_pattern(self) -> ast::FieldPat { - match self { - Annotatable::FieldPat(fp) => fp, - _ => panic!("expected field pattern"), - } - } - - pub fn expect_generic_param(self) -> ast::GenericParam { - match self { - Annotatable::GenericParam(gp) => gp, - _ => panic!("expected generic parameter"), - } - } - - pub fn expect_param(self) -> ast::Param { - match self { - Annotatable::Param(param) => param, - _ => panic!("expected parameter"), - } - } - - pub fn expect_struct_field(self) -> ast::StructField { - match self { - Annotatable::StructField(sf) => sf, - _ => panic!("expected struct field"), - } - } - - pub fn expect_variant(self) -> ast::Variant { - match self { - Annotatable::Variant(v) => v, - _ => panic!("expected variant"), - } - } - - pub fn derive_allowed(&self) -> bool { - match *self { - Annotatable::Item(ref item) => match item.kind { - ast::ItemKind::Struct(..) | ast::ItemKind::Enum(..) | ast::ItemKind::Union(..) => { - true - } - _ => false, - }, - _ => false, - } - } -} - -// `meta_item` is the annotation, and `item` is the item being modified. -// FIXME Decorators should follow the same pattern too. -pub trait MultiItemModifier { - fn expand( - &self, - ecx: &mut ExtCtxt<'_>, - span: Span, - meta_item: &ast::MetaItem, - item: Annotatable, - ) -> Vec; -} - -impl MultiItemModifier for F -where - F: Fn(&mut ExtCtxt<'_>, Span, &ast::MetaItem, Annotatable) -> T, - T: Into>, -{ - fn expand( - &self, - ecx: &mut ExtCtxt<'_>, - span: Span, - meta_item: &ast::MetaItem, - item: Annotatable, - ) -> Vec { - (*self)(ecx, span, meta_item, item).into() - } -} - -impl Into> for Annotatable { - fn into(self) -> Vec { - vec![self] - } -} - -pub trait ProcMacro { - fn expand<'cx>(&self, ecx: &'cx mut ExtCtxt<'_>, span: Span, ts: TokenStream) -> TokenStream; -} - -impl ProcMacro for F -where - F: Fn(TokenStream) -> TokenStream, -{ - fn expand<'cx>(&self, _ecx: &'cx mut ExtCtxt<'_>, _span: Span, ts: TokenStream) -> TokenStream { - // FIXME setup implicit context in TLS before calling self. - (*self)(ts) - } -} - -pub trait AttrProcMacro { - fn expand<'cx>( - &self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - annotation: TokenStream, - annotated: TokenStream, - ) -> TokenStream; -} - -impl AttrProcMacro for F -where - F: Fn(TokenStream, TokenStream) -> TokenStream, -{ - fn expand<'cx>( - &self, - _ecx: &'cx mut ExtCtxt<'_>, - _span: Span, - annotation: TokenStream, - annotated: TokenStream, - ) -> TokenStream { - // FIXME setup implicit context in TLS before calling self. - (*self)(annotation, annotated) - } -} - -/// Represents a thing that maps token trees to Macro Results -pub trait TTMacroExpander { - fn expand<'cx>( - &self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - input: TokenStream, - ) -> Box; -} - -pub type MacroExpanderFn = - for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) -> Box; - -impl TTMacroExpander for F -where - F: for<'cx> Fn(&'cx mut ExtCtxt<'_>, Span, TokenStream) -> Box, -{ - fn expand<'cx>( - &self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - mut input: TokenStream, - ) -> Box { - struct AvoidInterpolatedIdents; - - impl MutVisitor for AvoidInterpolatedIdents { - fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) { - if let tokenstream::TokenTree::Token(token) = tt { - if let token::Interpolated(nt) = &token.kind { - if let token::NtIdent(ident, is_raw) = **nt { - *tt = tokenstream::TokenTree::token( - token::Ident(ident.name, is_raw), - ident.span, - ); - } - } - } - mut_visit::noop_visit_tt(tt, self) - } - - fn visit_mac(&mut self, mac: &mut ast::Mac) { - mut_visit::noop_visit_mac(mac, self) - } - } - AvoidInterpolatedIdents.visit_tts(&mut input); - (*self)(ecx, span, input) - } -} - -// Use a macro because forwarding to a simple function has type system issues -macro_rules! make_stmts_default { - ($me:expr) => { - $me.make_expr().map(|e| { - smallvec![ast::Stmt { - id: ast::DUMMY_NODE_ID, - span: e.span, - kind: ast::StmtKind::Expr(e), - }] - }) - }; -} - -/// The result of a macro expansion. The return values of the various -/// methods are spliced into the AST at the callsite of the macro. -pub trait MacResult { - /// Creates an expression. - fn make_expr(self: Box) -> Option> { - None - } - /// Creates zero or more items. - fn make_items(self: Box) -> Option; 1]>> { - None - } - - /// Creates zero or more impl items. - fn make_impl_items(self: Box) -> Option> { - None - } - - /// Creates zero or more trait items. - fn make_trait_items(self: Box) -> Option> { - None - } - - /// Creates zero or more items in an `extern {}` block - fn make_foreign_items(self: Box) -> Option> { - None - } - - /// Creates a pattern. - fn make_pat(self: Box) -> Option> { - None - } - - /// Creates zero or more statements. - /// - /// By default this attempts to create an expression statement, - /// returning None if that fails. - fn make_stmts(self: Box) -> Option> { - make_stmts_default!(self) - } - - fn make_ty(self: Box) -> Option> { - None - } - - fn make_arms(self: Box) -> Option> { - None - } - - fn make_fields(self: Box) -> Option> { - None - } - - fn make_field_patterns(self: Box) -> Option> { - None - } - - fn make_generic_params(self: Box) -> Option> { - None - } - - fn make_params(self: Box) -> Option> { - None - } - - fn make_struct_fields(self: Box) -> Option> { - None - } - - fn make_variants(self: Box) -> Option> { - None - } -} - -macro_rules! make_MacEager { - ( $( $fld:ident: $t:ty, )* ) => { - /// `MacResult` implementation for the common case where you've already - /// built each form of AST that you might return. - #[derive(Default)] - pub struct MacEager { - $( - pub $fld: Option<$t>, - )* - } - - impl MacEager { - $( - pub fn $fld(v: $t) -> Box { - Box::new(MacEager { - $fld: Some(v), - ..Default::default() - }) - } - )* - } - } -} - -make_MacEager! { - expr: P, - pat: P, - items: SmallVec<[P; 1]>, - impl_items: SmallVec<[ast::AssocItem; 1]>, - trait_items: SmallVec<[ast::AssocItem; 1]>, - foreign_items: SmallVec<[ast::ForeignItem; 1]>, - stmts: SmallVec<[ast::Stmt; 1]>, - ty: P, -} - -impl MacResult for MacEager { - fn make_expr(self: Box) -> Option> { - self.expr - } - - fn make_items(self: Box) -> Option; 1]>> { - self.items - } - - fn make_impl_items(self: Box) -> Option> { - self.impl_items - } - - fn make_trait_items(self: Box) -> Option> { - self.trait_items - } - - fn make_foreign_items(self: Box) -> Option> { - self.foreign_items - } - - fn make_stmts(self: Box) -> Option> { - match self.stmts.as_ref().map_or(0, |s| s.len()) { - 0 => make_stmts_default!(self), - _ => self.stmts, - } - } - - fn make_pat(self: Box) -> Option> { - if let Some(p) = self.pat { - return Some(p); - } - if let Some(e) = self.expr { - if let ast::ExprKind::Lit(_) = e.kind { - return Some(P(ast::Pat { - id: ast::DUMMY_NODE_ID, - span: e.span, - kind: PatKind::Lit(e), - })); - } - } - None - } - - fn make_ty(self: Box) -> Option> { - self.ty - } -} - -/// Fill-in macro expansion result, to allow compilation to continue -/// after hitting errors. -#[derive(Copy, Clone)] -pub struct DummyResult { - is_error: bool, - span: Span, -} - -impl DummyResult { - /// Creates a default MacResult that can be anything. - /// - /// Use this as a return value after hitting any errors and - /// calling `span_err`. - pub fn any(span: Span) -> Box { - Box::new(DummyResult { is_error: true, span }) - } - - /// Same as `any`, but must be a valid fragment, not error. - pub fn any_valid(span: Span) -> Box { - Box::new(DummyResult { is_error: false, span }) - } - - /// A plain dummy expression. - pub fn raw_expr(sp: Span, is_error: bool) -> P { - P(ast::Expr { - id: ast::DUMMY_NODE_ID, - kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) }, - span: sp, - attrs: ast::AttrVec::new(), - }) - } - - /// A plain dummy pattern. - pub fn raw_pat(sp: Span) -> ast::Pat { - ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp } - } - - /// A plain dummy type. - pub fn raw_ty(sp: Span, is_error: bool) -> P { - P(ast::Ty { - id: ast::DUMMY_NODE_ID, - kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) }, - span: sp, - }) - } -} - -impl MacResult for DummyResult { - fn make_expr(self: Box) -> Option> { - Some(DummyResult::raw_expr(self.span, self.is_error)) - } - - fn make_pat(self: Box) -> Option> { - Some(P(DummyResult::raw_pat(self.span))) - } - - fn make_items(self: Box) -> Option; 1]>> { - Some(SmallVec::new()) - } - - fn make_impl_items(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_trait_items(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_foreign_items(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_stmts(self: Box) -> Option> { - Some(smallvec![ast::Stmt { - id: ast::DUMMY_NODE_ID, - kind: ast::StmtKind::Expr(DummyResult::raw_expr(self.span, self.is_error)), - span: self.span, - }]) - } - - fn make_ty(self: Box) -> Option> { - Some(DummyResult::raw_ty(self.span, self.is_error)) - } - - fn make_arms(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_fields(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_field_patterns(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_generic_params(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_params(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_struct_fields(self: Box) -> Option> { - Some(SmallVec::new()) - } - - fn make_variants(self: Box) -> Option> { - Some(SmallVec::new()) - } -} - -/// A syntax extension kind. -pub enum SyntaxExtensionKind { - /// A token-based function-like macro. - Bang( - /// An expander with signature TokenStream -> TokenStream. - Box, - ), - - /// An AST-based function-like macro. - LegacyBang( - /// An expander with signature TokenStream -> AST. - Box, - ), - - /// A token-based attribute macro. - Attr( - /// An expander with signature (TokenStream, TokenStream) -> TokenStream. - /// The first TokenSteam is the attribute itself, the second is the annotated item. - /// The produced TokenSteam replaces the input TokenSteam. - Box, - ), - - /// An AST-based attribute macro. - LegacyAttr( - /// An expander with signature (AST, AST) -> AST. - /// The first AST fragment is the attribute itself, the second is the annotated item. - /// The produced AST fragment replaces the input AST fragment. - Box, - ), - - /// A trivial attribute "macro" that does nothing, - /// only keeps the attribute and marks it as inert, - /// thus making it ineligible for further expansion. - NonMacroAttr { - /// Suppresses the `unused_attributes` lint for this attribute. - mark_used: bool, - }, - - /// A token-based derive macro. - Derive( - /// An expander with signature TokenStream -> TokenStream (not yet). - /// The produced TokenSteam is appended to the input TokenSteam. - Box, - ), - - /// An AST-based derive macro. - LegacyDerive( - /// An expander with signature AST -> AST. - /// The produced AST fragment is appended to the input AST fragment. - Box, - ), -} - -/// A struct representing a macro definition in "lowered" form ready for expansion. -pub struct SyntaxExtension { - /// A syntax extension kind. - pub kind: SyntaxExtensionKind, - /// Span of the macro definition. - pub span: Span, - /// Whitelist of unstable features that are treated as stable inside this macro. - pub allow_internal_unstable: Option>, - /// Suppresses the `unsafe_code` lint for code produced by this macro. - pub allow_internal_unsafe: bool, - /// Enables the macro helper hack (`ident!(...)` -> `$crate::ident!(...)`) for this macro. - pub local_inner_macros: bool, - /// The macro's stability info. - pub stability: Option, - /// The macro's deprecation info. - pub deprecation: Option, - /// Names of helper attributes registered by this macro. - pub helper_attrs: Vec, - /// Edition of the crate in which this macro is defined. - pub edition: Edition, - /// Built-in macros have a couple of special properties like availability - /// in `#[no_implicit_prelude]` modules, so we have to keep this flag. - pub is_builtin: bool, - /// We have to identify macros providing a `Copy` impl early for compatibility reasons. - pub is_derive_copy: bool, -} - -impl SyntaxExtension { - /// Returns which kind of macro calls this syntax extension. - pub fn macro_kind(&self) -> MacroKind { - match self.kind { - SyntaxExtensionKind::Bang(..) | SyntaxExtensionKind::LegacyBang(..) => MacroKind::Bang, - SyntaxExtensionKind::Attr(..) - | SyntaxExtensionKind::LegacyAttr(..) - | SyntaxExtensionKind::NonMacroAttr { .. } => MacroKind::Attr, - SyntaxExtensionKind::Derive(..) | SyntaxExtensionKind::LegacyDerive(..) => { - MacroKind::Derive - } - } - } - - /// Constructs a syntax extension with default properties. - pub fn default(kind: SyntaxExtensionKind, edition: Edition) -> SyntaxExtension { - SyntaxExtension { - span: DUMMY_SP, - allow_internal_unstable: None, - allow_internal_unsafe: false, - local_inner_macros: false, - stability: None, - deprecation: None, - helper_attrs: Vec::new(), - edition, - is_builtin: false, - is_derive_copy: false, - kind, - } - } - - /// Constructs a syntax extension with the given properties - /// and other properties converted from attributes. - pub fn new( - sess: &ParseSess, - kind: SyntaxExtensionKind, - span: Span, - helper_attrs: Vec, - edition: Edition, - name: Name, - attrs: &[ast::Attribute], - ) -> SyntaxExtension { - let allow_internal_unstable = attr::allow_internal_unstable(&attrs, &sess.span_diagnostic) - .map(|features| features.collect::>().into()); - - let mut local_inner_macros = false; - if let Some(macro_export) = attr::find_by_name(attrs, sym::macro_export) { - if let Some(l) = macro_export.meta_item_list() { - local_inner_macros = attr::list_contains_name(&l, sym::local_inner_macros); - } - } - - let is_builtin = attr::contains_name(attrs, sym::rustc_builtin_macro); - let (stability, const_stability) = attr::find_stability(&sess, attrs, span); - if const_stability.is_some() { - sess.span_diagnostic.span_err(span, "macros cannot have const stability attributes"); - } - - SyntaxExtension { - kind, - span, - allow_internal_unstable, - allow_internal_unsafe: attr::contains_name(attrs, sym::allow_internal_unsafe), - local_inner_macros, - stability, - deprecation: attr::find_deprecation(&sess, attrs, span), - helper_attrs, - edition, - is_builtin, - is_derive_copy: is_builtin && name == sym::Copy, - } - } - - pub fn dummy_bang(edition: Edition) -> SyntaxExtension { - fn expander<'cx>( - _: &'cx mut ExtCtxt<'_>, - span: Span, - _: TokenStream, - ) -> Box { - DummyResult::any(span) - } - SyntaxExtension::default(SyntaxExtensionKind::LegacyBang(Box::new(expander)), edition) - } - - pub fn dummy_derive(edition: Edition) -> SyntaxExtension { - fn expander( - _: &mut ExtCtxt<'_>, - _: Span, - _: &ast::MetaItem, - _: Annotatable, - ) -> Vec { - Vec::new() - } - SyntaxExtension::default(SyntaxExtensionKind::Derive(Box::new(expander)), edition) - } - - pub fn non_macro_attr(mark_used: bool, edition: Edition) -> SyntaxExtension { - SyntaxExtension::default(SyntaxExtensionKind::NonMacroAttr { mark_used }, edition) - } - - pub fn expn_data(&self, parent: ExpnId, call_site: Span, descr: Symbol) -> ExpnData { - ExpnData { - kind: ExpnKind::Macro(self.macro_kind(), descr), - parent, - call_site, - def_site: self.span, - allow_internal_unstable: self.allow_internal_unstable.clone(), - allow_internal_unsafe: self.allow_internal_unsafe, - local_inner_macros: self.local_inner_macros, - edition: self.edition, - } - } -} - -/// Result of resolving a macro invocation. -pub enum InvocationRes { - Single(Lrc), - DeriveContainer(Vec>), -} - -/// Error type that denotes indeterminacy. -pub struct Indeterminate; - -pub trait Resolver { - fn next_node_id(&mut self) -> NodeId; - - fn resolve_dollar_crates(&mut self); - fn visit_ast_fragment_with_placeholders(&mut self, expn_id: ExpnId, fragment: &AstFragment); - fn register_builtin_macro(&mut self, ident: ast::Ident, ext: SyntaxExtension); - - fn expansion_for_ast_pass( - &mut self, - call_site: Span, - pass: AstPass, - features: &[Symbol], - parent_module_id: Option, - ) -> ExpnId; - - fn resolve_imports(&mut self); - - fn resolve_macro_invocation( - &mut self, - invoc: &Invocation, - eager_expansion_root: ExpnId, - force: bool, - ) -> Result; - - fn check_unused_macros(&mut self); - - fn has_derive_copy(&self, expn_id: ExpnId) -> bool; - fn add_derive_copy(&mut self, expn_id: ExpnId); -} - -#[derive(Clone)] -pub struct ModuleData { - pub mod_path: Vec, - pub directory: PathBuf, -} - -#[derive(Clone)] -pub struct ExpansionData { - pub id: ExpnId, - pub depth: usize, - pub module: Rc, - pub directory_ownership: DirectoryOwnership, - pub prior_type_ascription: Option<(Span, bool)>, -} - -/// One of these is made during expansion and incrementally updated as we go; -/// when a macro expansion occurs, the resulting nodes have the `backtrace() -/// -> expn_data` of their expansion context stored into their span. -pub struct ExtCtxt<'a> { - pub parse_sess: &'a ParseSess, - pub ecfg: expand::ExpansionConfig<'a>, - pub root_path: PathBuf, - pub resolver: &'a mut dyn Resolver, - pub current_expansion: ExpansionData, - pub expansions: FxHashMap>, -} - -impl<'a> ExtCtxt<'a> { - pub fn new( - parse_sess: &'a ParseSess, - ecfg: expand::ExpansionConfig<'a>, - resolver: &'a mut dyn Resolver, - ) -> ExtCtxt<'a> { - ExtCtxt { - parse_sess, - ecfg, - root_path: PathBuf::new(), - resolver, - current_expansion: ExpansionData { - id: ExpnId::root(), - depth: 0, - module: Rc::new(ModuleData { mod_path: Vec::new(), directory: PathBuf::new() }), - directory_ownership: DirectoryOwnership::Owned { relative: None }, - prior_type_ascription: None, - }, - expansions: FxHashMap::default(), - } - } - - /// Returns a `Folder` for deeply expanding all macros in an AST node. - pub fn expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { - expand::MacroExpander::new(self, false) - } - - /// Returns a `Folder` that deeply expands all macros and assigns all `NodeId`s in an AST node. - /// Once `NodeId`s are assigned, the node may not be expanded, removed, or otherwise modified. - pub fn monotonic_expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { - expand::MacroExpander::new(self, true) - } - pub fn new_parser_from_tts(&self, stream: TokenStream) -> parser::Parser<'a> { - rustc_parse::stream_to_parser(self.parse_sess, stream, MACRO_ARGUMENTS) - } - pub fn source_map(&self) -> &'a SourceMap { - self.parse_sess.source_map() - } - pub fn parse_sess(&self) -> &'a ParseSess { - self.parse_sess - } - pub fn call_site(&self) -> Span { - self.current_expansion.id.expn_data().call_site - } - - /// Equivalent of `Span::def_site` from the proc macro API, - /// except that the location is taken from the span passed as an argument. - pub fn with_def_site_ctxt(&self, span: Span) -> Span { - span.with_def_site_ctxt(self.current_expansion.id) - } - - /// Equivalent of `Span::call_site` from the proc macro API, - /// except that the location is taken from the span passed as an argument. - pub fn with_call_site_ctxt(&self, span: Span) -> Span { - span.with_call_site_ctxt(self.current_expansion.id) - } - - /// Equivalent of `Span::mixed_site` from the proc macro API, - /// except that the location is taken from the span passed as an argument. - pub fn with_mixed_site_ctxt(&self, span: Span) -> Span { - span.with_mixed_site_ctxt(self.current_expansion.id) - } - - /// Returns span for the macro which originally caused the current expansion to happen. - /// - /// Stops backtracing at include! boundary. - pub fn expansion_cause(&self) -> Option { - self.current_expansion.id.expansion_cause() - } - - pub fn struct_span_warn>(&self, sp: S, msg: &str) -> DiagnosticBuilder<'a> { - self.parse_sess.span_diagnostic.struct_span_warn(sp, msg) - } - pub fn struct_span_err>(&self, sp: S, msg: &str) -> DiagnosticBuilder<'a> { - self.parse_sess.span_diagnostic.struct_span_err(sp, msg) - } - pub fn struct_span_fatal>(&self, sp: S, msg: &str) -> DiagnosticBuilder<'a> { - self.parse_sess.span_diagnostic.struct_span_fatal(sp, msg) - } - - /// Emit `msg` attached to `sp`, and stop compilation immediately. - /// - /// `span_err` should be strongly preferred where-ever possible: - /// this should *only* be used when: - /// - /// - continuing has a high risk of flow-on errors (e.g., errors in - /// declaring a macro would cause all uses of that macro to - /// complain about "undefined macro"), or - /// - there is literally nothing else that can be done (however, - /// in most cases one can construct a dummy expression/item to - /// substitute; we never hit resolve/type-checking so the dummy - /// value doesn't have to match anything) - pub fn span_fatal>(&self, sp: S, msg: &str) -> ! { - self.parse_sess.span_diagnostic.span_fatal(sp, msg).raise(); - } - - /// Emit `msg` attached to `sp`, without immediately stopping - /// compilation. - /// - /// Compilation will be stopped in the near future (at the end of - /// the macro expansion phase). - pub fn span_err>(&self, sp: S, msg: &str) { - self.parse_sess.span_diagnostic.span_err(sp, msg); - } - pub fn span_err_with_code>(&self, sp: S, msg: &str, code: DiagnosticId) { - self.parse_sess.span_diagnostic.span_err_with_code(sp, msg, code); - } - pub fn span_warn>(&self, sp: S, msg: &str) { - self.parse_sess.span_diagnostic.span_warn(sp, msg); - } - pub fn span_bug>(&self, sp: S, msg: &str) -> ! { - self.parse_sess.span_diagnostic.span_bug(sp, msg); - } - pub fn trace_macros_diag(&mut self) { - for (sp, notes) in self.expansions.iter() { - let mut db = self.parse_sess.span_diagnostic.span_note_diag(*sp, "trace_macro"); - for note in notes { - db.note(note); - } - db.emit(); - } - // Fixme: does this result in errors? - self.expansions.clear(); - } - pub fn bug(&self, msg: &str) -> ! { - self.parse_sess.span_diagnostic.bug(msg); - } - pub fn trace_macros(&self) -> bool { - self.ecfg.trace_mac - } - pub fn set_trace_macros(&mut self, x: bool) { - self.ecfg.trace_mac = x - } - pub fn ident_of(&self, st: &str, sp: Span) -> ast::Ident { - ast::Ident::from_str_and_span(st, sp) - } - pub fn std_path(&self, components: &[Symbol]) -> Vec { - let def_site = self.with_def_site_ctxt(DUMMY_SP); - iter::once(Ident::new(kw::DollarCrate, def_site)) - .chain(components.iter().map(|&s| Ident::with_dummy_span(s))) - .collect() - } - pub fn name_of(&self, st: &str) -> ast::Name { - Symbol::intern(st) - } - - pub fn check_unused_macros(&mut self) { - self.resolver.check_unused_macros(); - } - - /// Resolves a path mentioned inside Rust code. - /// - /// This unifies the logic used for resolving `include_X!`, and `#[doc(include)]` file paths. - /// - /// Returns an absolute path to the file that `path` refers to. - pub fn resolve_path( - &self, - path: impl Into, - span: Span, - ) -> Result> { - let path = path.into(); - - // Relative paths are resolved relative to the file in which they are found - // after macro expansion (that is, they are unhygienic). - if !path.is_absolute() { - let callsite = span.source_callsite(); - let mut result = match self.source_map().span_to_unmapped_path(callsite) { - FileName::Real(path) => path, - FileName::DocTest(path, _) => path, - other => { - return Err(self.struct_span_err( - span, - &format!("cannot resolve relative path in non-file source `{}`", other), - )); - } - }; - result.pop(); - result.push(path); - Ok(result) - } else { - Ok(path) - } - } -} - -/// Extracts a string literal from the macro expanded version of `expr`, -/// emitting `err_msg` if `expr` is not a string literal. This does not stop -/// compilation on error, merely emits a non-fatal error and returns `None`. -pub fn expr_to_spanned_string<'a>( - cx: &'a mut ExtCtxt<'_>, - expr: P, - err_msg: &str, -) -> Result<(Symbol, ast::StrStyle, Span), Option>> { - // Perform eager expansion on the expression. - // We want to be able to handle e.g., `concat!("foo", "bar")`. - let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); - - Err(match expr.kind { - ast::ExprKind::Lit(ref l) => match l.kind { - ast::LitKind::Str(s, style) => return Ok((s, style, expr.span)), - ast::LitKind::Err(_) => None, - _ => Some(cx.struct_span_err(l.span, err_msg)), - }, - ast::ExprKind::Err => None, - _ => Some(cx.struct_span_err(expr.span, err_msg)), - }) -} - -pub fn expr_to_string( - cx: &mut ExtCtxt<'_>, - expr: P, - err_msg: &str, -) -> Option<(Symbol, ast::StrStyle)> { - expr_to_spanned_string(cx, expr, err_msg) - .map_err(|err| err.map(|mut err| err.emit())) - .ok() - .map(|(symbol, style, _)| (symbol, style)) -} - -/// Non-fatally assert that `tts` is empty. Note that this function -/// returns even when `tts` is non-empty, macros that *need* to stop -/// compilation should call -/// `cx.parse_sess.span_diagnostic.abort_if_errors()` (this should be -/// done as rarely as possible). -pub fn check_zero_tts(cx: &ExtCtxt<'_>, sp: Span, tts: TokenStream, name: &str) { - if !tts.is_empty() { - cx.span_err(sp, &format!("{} takes no arguments", name)); - } -} - -/// Interpreting `tts` as a comma-separated sequence of expressions, -/// expect exactly one string literal, or emit an error and return `None`. -pub fn get_single_str_from_tts( - cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, - name: &str, -) -> Option { - let mut p = cx.new_parser_from_tts(tts); - if p.token == token::Eof { - cx.span_err(sp, &format!("{} takes 1 argument", name)); - return None; - } - let ret = panictry!(p.parse_expr()); - let _ = p.eat(&token::Comma); - - if p.token != token::Eof { - cx.span_err(sp, &format!("{} takes 1 argument", name)); - } - expr_to_string(cx, ret, "argument must be a string literal").map(|(s, _)| s.to_string()) -} - -/// Extracts comma-separated expressions from `tts`. If there is a -/// parsing error, emit a non-fatal error and return `None`. -pub fn get_exprs_from_tts( - cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Option>> { - let mut p = cx.new_parser_from_tts(tts); - let mut es = Vec::new(); - while p.token != token::Eof { - let expr = panictry!(p.parse_expr()); - - // Perform eager expansion on the expression. - // We want to be able to handle e.g., `concat!("foo", "bar")`. - let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); - - es.push(expr); - if p.eat(&token::Comma) { - continue; - } - if p.token != token::Eof { - cx.span_err(sp, "expected token: `,`"); - return None; - } - } - Some(es) -} diff --git a/src/libsyntax_expand/build.rs b/src/libsyntax_expand/build.rs deleted file mode 100644 index 96020acb3b4..00000000000 --- a/src/libsyntax_expand/build.rs +++ /dev/null @@ -1,663 +0,0 @@ -use crate::base::ExtCtxt; - -use syntax::ast::{self, AttrVec, BlockCheckMode, Expr, Ident, PatKind, UnOp}; -use syntax::attr; -use syntax::ptr::P; -use syntax::source_map::{respan, Spanned}; -use syntax::symbol::{kw, sym, Symbol}; - -use syntax_pos::{Pos, Span}; - -impl<'a> ExtCtxt<'a> { - pub fn path(&self, span: Span, strs: Vec) -> ast::Path { - self.path_all(span, false, strs, vec![]) - } - pub fn path_ident(&self, span: Span, id: ast::Ident) -> ast::Path { - self.path(span, vec![id]) - } - pub fn path_global(&self, span: Span, strs: Vec) -> ast::Path { - self.path_all(span, true, strs, vec![]) - } - pub fn path_all( - &self, - span: Span, - global: bool, - mut idents: Vec, - args: Vec, - ) -> ast::Path { - assert!(!idents.is_empty()); - let add_root = global && !idents[0].is_path_segment_keyword(); - let mut segments = Vec::with_capacity(idents.len() + add_root as usize); - if add_root { - segments.push(ast::PathSegment::path_root(span)); - } - let last_ident = idents.pop().unwrap(); - segments.extend( - idents.into_iter().map(|ident| ast::PathSegment::from_ident(ident.with_span_pos(span))), - ); - let args = if !args.is_empty() { - ast::AngleBracketedArgs { args, constraints: Vec::new(), span }.into() - } else { - None - }; - segments.push(ast::PathSegment { - ident: last_ident.with_span_pos(span), - id: ast::DUMMY_NODE_ID, - args, - }); - ast::Path { span, segments } - } - - pub fn ty_mt(&self, ty: P, mutbl: ast::Mutability) -> ast::MutTy { - ast::MutTy { ty, mutbl } - } - - pub fn ty(&self, span: Span, kind: ast::TyKind) -> P { - P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind }) - } - - pub fn ty_path(&self, path: ast::Path) -> P { - self.ty(path.span, ast::TyKind::Path(None, path)) - } - - // Might need to take bounds as an argument in the future, if you ever want - // to generate a bounded existential trait type. - pub fn ty_ident(&self, span: Span, ident: ast::Ident) -> P { - self.ty_path(self.path_ident(span, ident)) - } - - pub fn anon_const(&self, span: Span, kind: ast::ExprKind) -> ast::AnonConst { - ast::AnonConst { - id: ast::DUMMY_NODE_ID, - value: P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() }), - } - } - - pub fn const_ident(&self, span: Span, ident: ast::Ident) -> ast::AnonConst { - self.anon_const(span, ast::ExprKind::Path(None, self.path_ident(span, ident))) - } - - pub fn ty_rptr( - &self, - span: Span, - ty: P, - lifetime: Option, - mutbl: ast::Mutability, - ) -> P { - self.ty(span, ast::TyKind::Rptr(lifetime, self.ty_mt(ty, mutbl))) - } - - pub fn ty_ptr(&self, span: Span, ty: P, mutbl: ast::Mutability) -> P { - self.ty(span, ast::TyKind::Ptr(self.ty_mt(ty, mutbl))) - } - - pub fn typaram( - &self, - span: Span, - ident: ast::Ident, - attrs: Vec, - bounds: ast::GenericBounds, - default: Option>, - ) -> ast::GenericParam { - ast::GenericParam { - ident: ident.with_span_pos(span), - id: ast::DUMMY_NODE_ID, - attrs: attrs.into(), - bounds, - kind: ast::GenericParamKind::Type { default }, - is_placeholder: false, - } - } - - pub fn trait_ref(&self, path: ast::Path) -> ast::TraitRef { - ast::TraitRef { path, ref_id: ast::DUMMY_NODE_ID } - } - - pub fn poly_trait_ref(&self, span: Span, path: ast::Path) -> ast::PolyTraitRef { - ast::PolyTraitRef { - bound_generic_params: Vec::new(), - trait_ref: self.trait_ref(path), - span, - } - } - - pub fn trait_bound(&self, path: ast::Path) -> ast::GenericBound { - ast::GenericBound::Trait( - self.poly_trait_ref(path.span, path), - ast::TraitBoundModifier::None, - ) - } - - pub fn lifetime(&self, span: Span, ident: ast::Ident) -> ast::Lifetime { - ast::Lifetime { id: ast::DUMMY_NODE_ID, ident: ident.with_span_pos(span) } - } - - pub fn lifetime_def( - &self, - span: Span, - ident: ast::Ident, - attrs: Vec, - bounds: ast::GenericBounds, - ) -> ast::GenericParam { - let lifetime = self.lifetime(span, ident); - ast::GenericParam { - ident: lifetime.ident, - id: lifetime.id, - attrs: attrs.into(), - bounds, - kind: ast::GenericParamKind::Lifetime, - is_placeholder: false, - } - } - - pub fn stmt_expr(&self, expr: P) -> ast::Stmt { - ast::Stmt { id: ast::DUMMY_NODE_ID, span: expr.span, kind: ast::StmtKind::Expr(expr) } - } - - pub fn stmt_let( - &self, - sp: Span, - mutbl: bool, - ident: ast::Ident, - ex: P, - ) -> ast::Stmt { - let pat = if mutbl { - let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Mut); - self.pat_ident_binding_mode(sp, ident, binding_mode) - } else { - self.pat_ident(sp, ident) - }; - let local = P(ast::Local { - pat, - ty: None, - init: Some(ex), - id: ast::DUMMY_NODE_ID, - span: sp, - attrs: AttrVec::new(), - }); - ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp } - } - - // Generates `let _: Type;`, which is usually used for type assertions. - pub fn stmt_let_type_only(&self, span: Span, ty: P) -> ast::Stmt { - let local = P(ast::Local { - pat: self.pat_wild(span), - ty: Some(ty), - init: None, - id: ast::DUMMY_NODE_ID, - span, - attrs: AttrVec::new(), - }); - ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span } - } - - pub fn stmt_item(&self, sp: Span, item: P) -> ast::Stmt { - ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Item(item), span: sp } - } - - pub fn block_expr(&self, expr: P) -> P { - self.block( - expr.span, - vec![ast::Stmt { - id: ast::DUMMY_NODE_ID, - span: expr.span, - kind: ast::StmtKind::Expr(expr), - }], - ) - } - pub fn block(&self, span: Span, stmts: Vec) -> P { - P(ast::Block { stmts, id: ast::DUMMY_NODE_ID, rules: BlockCheckMode::Default, span }) - } - - pub fn expr(&self, span: Span, kind: ast::ExprKind) -> P { - P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() }) - } - - pub fn expr_path(&self, path: ast::Path) -> P { - self.expr(path.span, ast::ExprKind::Path(None, path)) - } - - pub fn expr_ident(&self, span: Span, id: ast::Ident) -> P { - self.expr_path(self.path_ident(span, id)) - } - pub fn expr_self(&self, span: Span) -> P { - self.expr_ident(span, Ident::with_dummy_span(kw::SelfLower)) - } - - pub fn expr_binary( - &self, - sp: Span, - op: ast::BinOpKind, - lhs: P, - rhs: P, - ) -> P { - self.expr(sp, ast::ExprKind::Binary(Spanned { node: op, span: sp }, lhs, rhs)) - } - - pub fn expr_deref(&self, sp: Span, e: P) -> P { - self.expr(sp, ast::ExprKind::Unary(UnOp::Deref, e)) - } - - pub fn expr_addr_of(&self, sp: Span, e: P) -> P { - self.expr(sp, ast::ExprKind::AddrOf(ast::BorrowKind::Ref, ast::Mutability::Not, e)) - } - - pub fn expr_call( - &self, - span: Span, - expr: P, - args: Vec>, - ) -> P { - self.expr(span, ast::ExprKind::Call(expr, args)) - } - pub fn expr_call_ident( - &self, - span: Span, - id: ast::Ident, - args: Vec>, - ) -> P { - self.expr(span, ast::ExprKind::Call(self.expr_ident(span, id), args)) - } - pub fn expr_call_global( - &self, - sp: Span, - fn_path: Vec, - args: Vec>, - ) -> P { - let pathexpr = self.expr_path(self.path_global(sp, fn_path)); - self.expr_call(sp, pathexpr, args) - } - pub fn expr_method_call( - &self, - span: Span, - expr: P, - ident: ast::Ident, - mut args: Vec>, - ) -> P { - args.insert(0, expr); - let segment = ast::PathSegment::from_ident(ident.with_span_pos(span)); - self.expr(span, ast::ExprKind::MethodCall(segment, args)) - } - pub fn expr_block(&self, b: P) -> P { - self.expr(b.span, ast::ExprKind::Block(b, None)) - } - pub fn field_imm(&self, span: Span, ident: Ident, e: P) -> ast::Field { - ast::Field { - ident: ident.with_span_pos(span), - expr: e, - span, - is_shorthand: false, - attrs: AttrVec::new(), - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - } - } - pub fn expr_struct( - &self, - span: Span, - path: ast::Path, - fields: Vec, - ) -> P { - self.expr(span, ast::ExprKind::Struct(path, fields, None)) - } - pub fn expr_struct_ident( - &self, - span: Span, - id: ast::Ident, - fields: Vec, - ) -> P { - self.expr_struct(span, self.path_ident(span, id), fields) - } - - pub fn expr_lit(&self, span: Span, lit_kind: ast::LitKind) -> P { - let lit = ast::Lit::from_lit_kind(lit_kind, span); - self.expr(span, ast::ExprKind::Lit(lit)) - } - pub fn expr_usize(&self, span: Span, i: usize) -> P { - self.expr_lit( - span, - ast::LitKind::Int(i as u128, ast::LitIntType::Unsigned(ast::UintTy::Usize)), - ) - } - pub fn expr_u32(&self, sp: Span, u: u32) -> P { - self.expr_lit(sp, ast::LitKind::Int(u as u128, ast::LitIntType::Unsigned(ast::UintTy::U32))) - } - pub fn expr_bool(&self, sp: Span, value: bool) -> P { - self.expr_lit(sp, ast::LitKind::Bool(value)) - } - - pub fn expr_vec(&self, sp: Span, exprs: Vec>) -> P { - self.expr(sp, ast::ExprKind::Array(exprs)) - } - pub fn expr_vec_slice(&self, sp: Span, exprs: Vec>) -> P { - self.expr_addr_of(sp, self.expr_vec(sp, exprs)) - } - pub fn expr_str(&self, sp: Span, s: Symbol) -> P { - self.expr_lit(sp, ast::LitKind::Str(s, ast::StrStyle::Cooked)) - } - - pub fn expr_cast(&self, sp: Span, expr: P, ty: P) -> P { - self.expr(sp, ast::ExprKind::Cast(expr, ty)) - } - - pub fn expr_some(&self, sp: Span, expr: P) -> P { - let some = self.std_path(&[sym::option, sym::Option, sym::Some]); - self.expr_call_global(sp, some, vec![expr]) - } - - pub fn expr_tuple(&self, sp: Span, exprs: Vec>) -> P { - self.expr(sp, ast::ExprKind::Tup(exprs)) - } - - pub fn expr_fail(&self, span: Span, msg: Symbol) -> P { - let loc = self.source_map().lookup_char_pos(span.lo()); - let expr_file = self.expr_str(span, Symbol::intern(&loc.file.name.to_string())); - let expr_line = self.expr_u32(span, loc.line as u32); - let expr_col = self.expr_u32(span, loc.col.to_usize() as u32 + 1); - let expr_loc_tuple = self.expr_tuple(span, vec![expr_file, expr_line, expr_col]); - let expr_loc_ptr = self.expr_addr_of(span, expr_loc_tuple); - self.expr_call_global( - span, - [sym::std, sym::rt, sym::begin_panic].iter().map(|s| Ident::new(*s, span)).collect(), - vec![self.expr_str(span, msg), expr_loc_ptr], - ) - } - - pub fn expr_unreachable(&self, span: Span) -> P { - self.expr_fail(span, Symbol::intern("internal error: entered unreachable code")) - } - - pub fn expr_ok(&self, sp: Span, expr: P) -> P { - let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); - self.expr_call_global(sp, ok, vec![expr]) - } - - pub fn expr_try(&self, sp: Span, head: P) -> P { - let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); - let ok_path = self.path_global(sp, ok); - let err = self.std_path(&[sym::result, sym::Result, sym::Err]); - let err_path = self.path_global(sp, err); - - let binding_variable = self.ident_of("__try_var", sp); - let binding_pat = self.pat_ident(sp, binding_variable); - let binding_expr = self.expr_ident(sp, binding_variable); - - // `Ok(__try_var)` pattern - let ok_pat = self.pat_tuple_struct(sp, ok_path, vec![binding_pat.clone()]); - - // `Err(__try_var)` (pattern and expression respectively) - let err_pat = self.pat_tuple_struct(sp, err_path.clone(), vec![binding_pat]); - let err_inner_expr = - self.expr_call(sp, self.expr_path(err_path), vec![binding_expr.clone()]); - // `return Err(__try_var)` - let err_expr = self.expr(sp, ast::ExprKind::Ret(Some(err_inner_expr))); - - // `Ok(__try_var) => __try_var` - let ok_arm = self.arm(sp, ok_pat, binding_expr); - // `Err(__try_var) => return Err(__try_var)` - let err_arm = self.arm(sp, err_pat, err_expr); - - // `match head { Ok() => ..., Err() => ... }` - self.expr_match(sp, head, vec![ok_arm, err_arm]) - } - - pub fn pat(&self, span: Span, kind: PatKind) -> P { - P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span }) - } - pub fn pat_wild(&self, span: Span) -> P { - self.pat(span, PatKind::Wild) - } - pub fn pat_lit(&self, span: Span, expr: P) -> P { - self.pat(span, PatKind::Lit(expr)) - } - pub fn pat_ident(&self, span: Span, ident: ast::Ident) -> P { - let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Not); - self.pat_ident_binding_mode(span, ident, binding_mode) - } - - pub fn pat_ident_binding_mode( - &self, - span: Span, - ident: ast::Ident, - bm: ast::BindingMode, - ) -> P { - let pat = PatKind::Ident(bm, ident.with_span_pos(span), None); - self.pat(span, pat) - } - pub fn pat_path(&self, span: Span, path: ast::Path) -> P { - self.pat(span, PatKind::Path(None, path)) - } - pub fn pat_tuple_struct( - &self, - span: Span, - path: ast::Path, - subpats: Vec>, - ) -> P { - self.pat(span, PatKind::TupleStruct(path, subpats)) - } - pub fn pat_struct( - &self, - span: Span, - path: ast::Path, - field_pats: Vec, - ) -> P { - self.pat(span, PatKind::Struct(path, field_pats, false)) - } - pub fn pat_tuple(&self, span: Span, pats: Vec>) -> P { - self.pat(span, PatKind::Tuple(pats)) - } - - pub fn pat_some(&self, span: Span, pat: P) -> P { - let some = self.std_path(&[sym::option, sym::Option, sym::Some]); - let path = self.path_global(span, some); - self.pat_tuple_struct(span, path, vec![pat]) - } - - pub fn pat_none(&self, span: Span) -> P { - let some = self.std_path(&[sym::option, sym::Option, sym::None]); - let path = self.path_global(span, some); - self.pat_path(span, path) - } - - pub fn pat_ok(&self, span: Span, pat: P) -> P { - let some = self.std_path(&[sym::result, sym::Result, sym::Ok]); - let path = self.path_global(span, some); - self.pat_tuple_struct(span, path, vec![pat]) - } - - pub fn pat_err(&self, span: Span, pat: P) -> P { - let some = self.std_path(&[sym::result, sym::Result, sym::Err]); - let path = self.path_global(span, some); - self.pat_tuple_struct(span, path, vec![pat]) - } - - pub fn arm(&self, span: Span, pat: P, expr: P) -> ast::Arm { - ast::Arm { - attrs: vec![], - pat, - guard: None, - body: expr, - span, - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - } - } - - pub fn arm_unreachable(&self, span: Span) -> ast::Arm { - self.arm(span, self.pat_wild(span), self.expr_unreachable(span)) - } - - pub fn expr_match(&self, span: Span, arg: P, arms: Vec) -> P { - self.expr(span, ast::ExprKind::Match(arg, arms)) - } - - pub fn expr_if( - &self, - span: Span, - cond: P, - then: P, - els: Option>, - ) -> P { - let els = els.map(|x| self.expr_block(self.block_expr(x))); - self.expr(span, ast::ExprKind::If(cond, self.block_expr(then), els)) - } - - pub fn lambda_fn_decl( - &self, - span: Span, - fn_decl: P, - body: P, - fn_decl_span: Span, - ) -> P { - self.expr( - span, - ast::ExprKind::Closure( - ast::CaptureBy::Ref, - ast::IsAsync::NotAsync, - ast::Movability::Movable, - fn_decl, - body, - fn_decl_span, - ), - ) - } - - pub fn lambda(&self, span: Span, ids: Vec, body: P) -> P { - let fn_decl = self.fn_decl( - ids.iter().map(|id| self.param(span, *id, self.ty(span, ast::TyKind::Infer))).collect(), - ast::FunctionRetTy::Default(span), - ); - - // FIXME -- We are using `span` as the span of the `|...|` - // part of the lambda, but it probably (maybe?) corresponds to - // the entire lambda body. Probably we should extend the API - // here, but that's not entirely clear. - self.expr( - span, - ast::ExprKind::Closure( - ast::CaptureBy::Ref, - ast::IsAsync::NotAsync, - ast::Movability::Movable, - fn_decl, - body, - span, - ), - ) - } - - pub fn lambda0(&self, span: Span, body: P) -> P { - self.lambda(span, Vec::new(), body) - } - - pub fn lambda1(&self, span: Span, body: P, ident: ast::Ident) -> P { - self.lambda(span, vec![ident], body) - } - - pub fn lambda_stmts_1( - &self, - span: Span, - stmts: Vec, - ident: ast::Ident, - ) -> P { - self.lambda1(span, self.expr_block(self.block(span, stmts)), ident) - } - - pub fn param(&self, span: Span, ident: ast::Ident, ty: P) -> ast::Param { - let arg_pat = self.pat_ident(span, ident); - ast::Param { - attrs: AttrVec::default(), - id: ast::DUMMY_NODE_ID, - pat: arg_pat, - span, - ty, - is_placeholder: false, - } - } - - // FIXME: unused `self` - pub fn fn_decl(&self, inputs: Vec, output: ast::FunctionRetTy) -> P { - P(ast::FnDecl { inputs, output }) - } - - pub fn item( - &self, - span: Span, - name: Ident, - attrs: Vec, - kind: ast::ItemKind, - ) -> P { - // FIXME: Would be nice if our generated code didn't violate - // Rust coding conventions - P(ast::Item { - ident: name, - attrs, - id: ast::DUMMY_NODE_ID, - kind, - vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), - span, - tokens: None, - }) - } - - pub fn variant(&self, span: Span, ident: Ident, tys: Vec>) -> ast::Variant { - let vis_span = span.shrink_to_lo(); - let fields: Vec<_> = tys - .into_iter() - .map(|ty| ast::StructField { - span: ty.span, - ty, - ident: None, - vis: respan(vis_span, ast::VisibilityKind::Inherited), - attrs: Vec::new(), - id: ast::DUMMY_NODE_ID, - is_placeholder: false, - }) - .collect(); - - let vdata = if fields.is_empty() { - ast::VariantData::Unit(ast::DUMMY_NODE_ID) - } else { - ast::VariantData::Tuple(fields, ast::DUMMY_NODE_ID) - }; - - ast::Variant { - attrs: Vec::new(), - data: vdata, - disr_expr: None, - id: ast::DUMMY_NODE_ID, - ident, - vis: respan(vis_span, ast::VisibilityKind::Inherited), - span, - is_placeholder: false, - } - } - - pub fn item_static( - &self, - span: Span, - name: Ident, - ty: P, - mutbl: ast::Mutability, - expr: P, - ) -> P { - self.item(span, name, Vec::new(), ast::ItemKind::Static(ty, mutbl, expr)) - } - - pub fn item_const( - &self, - span: Span, - name: Ident, - ty: P, - expr: P, - ) -> P { - self.item(span, name, Vec::new(), ast::ItemKind::Const(ty, expr)) - } - - pub fn attribute(&self, mi: ast::MetaItem) -> ast::Attribute { - attr::mk_attr_outer(mi) - } - - pub fn meta_word(&self, sp: Span, w: ast::Name) -> ast::MetaItem { - attr::mk_word_item(Ident::new(w, sp)) - } -} diff --git a/src/libsyntax_expand/expand.rs b/src/libsyntax_expand/expand.rs deleted file mode 100644 index 089dd471f3b..00000000000 --- a/src/libsyntax_expand/expand.rs +++ /dev/null @@ -1,1725 +0,0 @@ -use crate::base::*; -use crate::config::StripUnconfigured; -use crate::hygiene::{ExpnData, ExpnId, ExpnKind, SyntaxContext}; -use crate::mbe::macro_rules::annotate_err_with_kind; -use crate::placeholders::{placeholder, PlaceholderExpander}; -use crate::proc_macro::collect_derives; - -use rustc_feature::Features; -use rustc_parse::configure; -use rustc_parse::parser::Parser; -use rustc_parse::validate_attr; -use rustc_parse::DirectoryOwnership; -use syntax::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path}; -use syntax::ast::{ItemKind, MacArgs, MacStmtStyle, StmtKind}; -use syntax::attr::{self, is_builtin_attr, HasAttrs}; -use syntax::feature_gate::{self, feature_err}; -use syntax::mut_visit::*; -use syntax::print::pprust; -use syntax::ptr::P; -use syntax::sess::ParseSess; -use syntax::source_map::respan; -use syntax::symbol::{sym, Symbol}; -use syntax::token; -use syntax::tokenstream::{TokenStream, TokenTree}; -use syntax::util::map_in_place::MapInPlace; -use syntax::visit::{self, Visitor}; - -use errors::{Applicability, FatalError, PResult}; -use smallvec::{smallvec, SmallVec}; -use syntax_pos::{FileName, Span, DUMMY_SP}; - -use rustc_data_structures::sync::Lrc; -use std::io::ErrorKind; -use std::ops::DerefMut; -use std::path::PathBuf; -use std::rc::Rc; -use std::{iter, mem, slice}; - -macro_rules! ast_fragments { - ( - $($Kind:ident($AstTy:ty) { - $kind_name:expr; - $(one fn $mut_visit_ast:ident; fn $visit_ast:ident;)? - $(many fn $flat_map_ast_elt:ident; fn $visit_ast_elt:ident;)? - fn $make_ast:ident; - })* - ) => { - /// A fragment of AST that can be produced by a single macro expansion. - /// Can also serve as an input and intermediate result for macro expansion operations. - pub enum AstFragment { - OptExpr(Option>), - $($Kind($AstTy),)* - } - - /// "Discriminant" of an AST fragment. - #[derive(Copy, Clone, PartialEq, Eq)] - pub enum AstFragmentKind { - OptExpr, - $($Kind,)* - } - - impl AstFragmentKind { - pub fn name(self) -> &'static str { - match self { - AstFragmentKind::OptExpr => "expression", - $(AstFragmentKind::$Kind => $kind_name,)* - } - } - - fn make_from<'a>(self, result: Box) -> Option { - match self { - AstFragmentKind::OptExpr => - result.make_expr().map(Some).map(AstFragment::OptExpr), - $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)* - } - } - } - - impl AstFragment { - pub fn add_placeholders(&mut self, placeholders: &[NodeId]) { - if placeholders.is_empty() { - return; - } - match self { - $($(AstFragment::$Kind(ast) => ast.extend(placeholders.iter().flat_map(|id| { - // We are repeating through arguments with `many`, to do that we have to - // mention some macro variable from those arguments even if it's not used. - macro _repeating($flat_map_ast_elt) {} - placeholder(AstFragmentKind::$Kind, *id, None).$make_ast() - })),)?)* - _ => panic!("unexpected AST fragment kind") - } - } - - pub fn make_opt_expr(self) -> Option> { - match self { - AstFragment::OptExpr(expr) => expr, - _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), - } - } - - $(pub fn $make_ast(self) -> $AstTy { - match self { - AstFragment::$Kind(ast) => ast, - _ => panic!("AstFragment::make_* called on the wrong kind of fragment"), - } - })* - - pub fn mut_visit_with(&mut self, vis: &mut F) { - match self { - AstFragment::OptExpr(opt_expr) => { - visit_clobber(opt_expr, |opt_expr| { - if let Some(expr) = opt_expr { - vis.filter_map_expr(expr) - } else { - None - } - }); - } - $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)* - $($(AstFragment::$Kind(ast) => - ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)* - } - } - - pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { - match *self { - AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr), - AstFragment::OptExpr(None) => {} - $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)* - $($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] { - visitor.$visit_ast_elt(ast_elt); - })?)* - } - } - } - - impl<'a> MacResult for crate::mbe::macro_rules::ParserAnyMacro<'a> { - $(fn $make_ast(self: Box>) - -> Option<$AstTy> { - Some(self.make(AstFragmentKind::$Kind).$make_ast()) - })* - } - } -} - -ast_fragments! { - Expr(P) { "expression"; one fn visit_expr; fn visit_expr; fn make_expr; } - Pat(P) { "pattern"; one fn visit_pat; fn visit_pat; fn make_pat; } - Ty(P) { "type"; one fn visit_ty; fn visit_ty; fn make_ty; } - Stmts(SmallVec<[ast::Stmt; 1]>) { - "statement"; many fn flat_map_stmt; fn visit_stmt; fn make_stmts; - } - Items(SmallVec<[P; 1]>) { - "item"; many fn flat_map_item; fn visit_item; fn make_items; - } - TraitItems(SmallVec<[ast::AssocItem; 1]>) { - "trait item"; many fn flat_map_trait_item; fn visit_trait_item; fn make_trait_items; - } - ImplItems(SmallVec<[ast::AssocItem; 1]>) { - "impl item"; many fn flat_map_impl_item; fn visit_impl_item; fn make_impl_items; - } - ForeignItems(SmallVec<[ast::ForeignItem; 1]>) { - "foreign item"; - many fn flat_map_foreign_item; - fn visit_foreign_item; - fn make_foreign_items; - } - Arms(SmallVec<[ast::Arm; 1]>) { - "match arm"; many fn flat_map_arm; fn visit_arm; fn make_arms; - } - Fields(SmallVec<[ast::Field; 1]>) { - "field expression"; many fn flat_map_field; fn visit_field; fn make_fields; - } - FieldPats(SmallVec<[ast::FieldPat; 1]>) { - "field pattern"; - many fn flat_map_field_pattern; - fn visit_field_pattern; - fn make_field_patterns; - } - GenericParams(SmallVec<[ast::GenericParam; 1]>) { - "generic parameter"; - many fn flat_map_generic_param; - fn visit_generic_param; - fn make_generic_params; - } - Params(SmallVec<[ast::Param; 1]>) { - "function parameter"; many fn flat_map_param; fn visit_param; fn make_params; - } - StructFields(SmallVec<[ast::StructField; 1]>) { - "field"; - many fn flat_map_struct_field; - fn visit_struct_field; - fn make_struct_fields; - } - Variants(SmallVec<[ast::Variant; 1]>) { - "variant"; many fn flat_map_variant; fn visit_variant; fn make_variants; - } -} - -impl AstFragmentKind { - fn dummy(self, span: Span) -> AstFragment { - self.make_from(DummyResult::any(span)).expect("couldn't create a dummy AST fragment") - } - - fn expect_from_annotatables>( - self, - items: I, - ) -> AstFragment { - let mut items = items.into_iter(); - match self { - AstFragmentKind::Arms => { - AstFragment::Arms(items.map(Annotatable::expect_arm).collect()) - } - AstFragmentKind::Fields => { - AstFragment::Fields(items.map(Annotatable::expect_field).collect()) - } - AstFragmentKind::FieldPats => { - AstFragment::FieldPats(items.map(Annotatable::expect_field_pattern).collect()) - } - AstFragmentKind::GenericParams => { - AstFragment::GenericParams(items.map(Annotatable::expect_generic_param).collect()) - } - AstFragmentKind::Params => { - AstFragment::Params(items.map(Annotatable::expect_param).collect()) - } - AstFragmentKind::StructFields => { - AstFragment::StructFields(items.map(Annotatable::expect_struct_field).collect()) - } - AstFragmentKind::Variants => { - AstFragment::Variants(items.map(Annotatable::expect_variant).collect()) - } - AstFragmentKind::Items => { - AstFragment::Items(items.map(Annotatable::expect_item).collect()) - } - AstFragmentKind::ImplItems => { - AstFragment::ImplItems(items.map(Annotatable::expect_impl_item).collect()) - } - AstFragmentKind::TraitItems => { - AstFragment::TraitItems(items.map(Annotatable::expect_trait_item).collect()) - } - AstFragmentKind::ForeignItems => { - AstFragment::ForeignItems(items.map(Annotatable::expect_foreign_item).collect()) - } - AstFragmentKind::Stmts => { - AstFragment::Stmts(items.map(Annotatable::expect_stmt).collect()) - } - AstFragmentKind::Expr => AstFragment::Expr( - items.next().expect("expected exactly one expression").expect_expr(), - ), - AstFragmentKind::OptExpr => { - AstFragment::OptExpr(items.next().map(Annotatable::expect_expr)) - } - AstFragmentKind::Pat | AstFragmentKind::Ty => { - panic!("patterns and types aren't annotatable") - } - } - } -} - -pub struct Invocation { - pub kind: InvocationKind, - pub fragment_kind: AstFragmentKind, - pub expansion_data: ExpansionData, -} - -pub enum InvocationKind { - Bang { - mac: ast::Mac, - span: Span, - }, - Attr { - attr: ast::Attribute, - item: Annotatable, - // Required for resolving derive helper attributes. - derives: Vec, - // We temporarily report errors for attribute macros placed after derives - after_derive: bool, - }, - Derive { - path: Path, - item: Annotatable, - }, - /// "Invocation" that contains all derives from an item, - /// broken into multiple `Derive` invocations when expanded. - /// FIXME: Find a way to remove it. - DeriveContainer { - derives: Vec, - item: Annotatable, - }, -} - -impl InvocationKind { - fn placeholder_visibility(&self) -> Option { - // HACK: For unnamed fields placeholders should have the same visibility as the actual - // fields because for tuple structs/variants resolve determines visibilities of their - // constructor using these field visibilities before attributes on them are are expanded. - // The assumption is that the attribute expansion cannot change field visibilities, - // and it holds because only inert attributes are supported in this position. - match self { - InvocationKind::Attr { item: Annotatable::StructField(field), .. } - | InvocationKind::Derive { item: Annotatable::StructField(field), .. } - | InvocationKind::DeriveContainer { item: Annotatable::StructField(field), .. } - if field.ident.is_none() => - { - Some(field.vis.clone()) - } - _ => None, - } - } -} - -impl Invocation { - pub fn span(&self) -> Span { - match &self.kind { - InvocationKind::Bang { span, .. } => *span, - InvocationKind::Attr { attr, .. } => attr.span, - InvocationKind::Derive { path, .. } => path.span, - InvocationKind::DeriveContainer { item, .. } => item.span(), - } - } -} - -pub struct MacroExpander<'a, 'b> { - pub cx: &'a mut ExtCtxt<'b>, - monotonic: bool, // cf. `cx.monotonic_expander()` -} - -impl<'a, 'b> MacroExpander<'a, 'b> { - pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { - MacroExpander { cx, monotonic } - } - - pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate { - let mut module = ModuleData { - mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)], - directory: match self.cx.source_map().span_to_unmapped_path(krate.span) { - FileName::Real(path) => path, - other => PathBuf::from(other.to_string()), - }, - }; - module.directory.pop(); - self.cx.root_path = module.directory.clone(); - self.cx.current_expansion.module = Rc::new(module); - - let orig_mod_span = krate.module.inner; - - let krate_item = AstFragment::Items(smallvec![P(ast::Item { - attrs: krate.attrs, - span: krate.span, - kind: ast::ItemKind::Mod(krate.module), - ident: Ident::invalid(), - id: ast::DUMMY_NODE_ID, - vis: respan(krate.span.shrink_to_lo(), ast::VisibilityKind::Public), - tokens: None, - })]); - - match self.fully_expand_fragment(krate_item).make_items().pop().map(P::into_inner) { - Some(ast::Item { attrs, kind: ast::ItemKind::Mod(module), .. }) => { - krate.attrs = attrs; - krate.module = module; - } - None => { - // Resolution failed so we return an empty expansion - krate.attrs = vec![]; - krate.module = ast::Mod { inner: orig_mod_span, items: vec![], inline: true }; - } - _ => unreachable!(), - }; - self.cx.trace_macros_diag(); - krate - } - - // Recursively expand all macro invocations in this AST fragment. - pub fn fully_expand_fragment(&mut self, input_fragment: AstFragment) -> AstFragment { - let orig_expansion_data = self.cx.current_expansion.clone(); - self.cx.current_expansion.depth = 0; - - // Collect all macro invocations and replace them with placeholders. - let (mut fragment_with_placeholders, mut invocations) = - self.collect_invocations(input_fragment, &[]); - - // Optimization: if we resolve all imports now, - // we'll be able to immediately resolve most of imported macros. - self.resolve_imports(); - - // Resolve paths in all invocations and produce output expanded fragments for them, but - // do not insert them into our input AST fragment yet, only store in `expanded_fragments`. - // The output fragments also go through expansion recursively until no invocations are left. - // Unresolved macros produce dummy outputs as a recovery measure. - invocations.reverse(); - let mut expanded_fragments = Vec::new(); - let mut undetermined_invocations = Vec::new(); - let (mut progress, mut force) = (false, !self.monotonic); - loop { - let invoc = if let Some(invoc) = invocations.pop() { - invoc - } else { - self.resolve_imports(); - if undetermined_invocations.is_empty() { - break; - } - invocations = mem::take(&mut undetermined_invocations); - force = !mem::replace(&mut progress, false); - continue; - }; - - let eager_expansion_root = - if self.monotonic { invoc.expansion_data.id } else { orig_expansion_data.id }; - let res = match self.cx.resolver.resolve_macro_invocation( - &invoc, - eager_expansion_root, - force, - ) { - Ok(res) => res, - Err(Indeterminate) => { - undetermined_invocations.push(invoc); - continue; - } - }; - - progress = true; - let ExpansionData { depth, id: expn_id, .. } = invoc.expansion_data; - self.cx.current_expansion = invoc.expansion_data.clone(); - - // FIXME(jseyfried): Refactor out the following logic - let (expanded_fragment, new_invocations) = match res { - InvocationRes::Single(ext) => { - let fragment = self.expand_invoc(invoc, &ext.kind); - self.collect_invocations(fragment, &[]) - } - InvocationRes::DeriveContainer(_exts) => { - // FIXME: Consider using the derive resolutions (`_exts`) immediately, - // instead of enqueuing the derives to be resolved again later. - let (derives, item) = match invoc.kind { - InvocationKind::DeriveContainer { derives, item } => (derives, item), - _ => unreachable!(), - }; - if !item.derive_allowed() { - let attr = attr::find_by_name(item.attrs(), sym::derive) - .expect("`derive` attribute should exist"); - let span = attr.span; - let mut err = self.cx.struct_span_err( - span, - "`derive` may only be applied to structs, enums and unions", - ); - if let ast::AttrStyle::Inner = attr.style { - let trait_list = derives - .iter() - .map(|t| pprust::path_to_string(t)) - .collect::>(); - let suggestion = format!("#[derive({})]", trait_list.join(", ")); - err.span_suggestion( - span, - "try an outer attribute", - suggestion, - // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT - Applicability::MaybeIncorrect, - ); - } - err.emit(); - } - - let mut item = self.fully_configure(item); - item.visit_attrs(|attrs| attrs.retain(|a| !a.has_name(sym::derive))); - - let mut derive_placeholders = Vec::with_capacity(derives.len()); - invocations.reserve(derives.len()); - for path in derives { - let expn_id = ExpnId::fresh(None); - derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id)); - invocations.push(Invocation { - kind: InvocationKind::Derive { path, item: item.clone() }, - fragment_kind: invoc.fragment_kind, - expansion_data: ExpansionData { - id: expn_id, - ..invoc.expansion_data.clone() - }, - }); - } - let fragment = - invoc.fragment_kind.expect_from_annotatables(::std::iter::once(item)); - self.collect_invocations(fragment, &derive_placeholders) - } - }; - - if expanded_fragments.len() < depth { - expanded_fragments.push(Vec::new()); - } - expanded_fragments[depth - 1].push((expn_id, expanded_fragment)); - if !self.cx.ecfg.single_step { - invocations.extend(new_invocations.into_iter().rev()); - } - } - - self.cx.current_expansion = orig_expansion_data; - - // Finally incorporate all the expanded macros into the input AST fragment. - let mut placeholder_expander = PlaceholderExpander::new(self.cx, self.monotonic); - while let Some(expanded_fragments) = expanded_fragments.pop() { - for (expn_id, expanded_fragment) in expanded_fragments.into_iter().rev() { - placeholder_expander - .add(NodeId::placeholder_from_expn_id(expn_id), expanded_fragment); - } - } - fragment_with_placeholders.mut_visit_with(&mut placeholder_expander); - fragment_with_placeholders - } - - fn resolve_imports(&mut self) { - if self.monotonic { - self.cx.resolver.resolve_imports(); - } - } - - /// Collects all macro invocations reachable at this time in this AST fragment, and replace - /// them with "placeholders" - dummy macro invocations with specially crafted `NodeId`s. - /// Then call into resolver that builds a skeleton ("reduced graph") of the fragment and - /// prepares data for resolving paths of macro invocations. - fn collect_invocations( - &mut self, - mut fragment: AstFragment, - extra_placeholders: &[NodeId], - ) -> (AstFragment, Vec) { - // Resolve `$crate`s in the fragment for pretty-printing. - self.cx.resolver.resolve_dollar_crates(); - - let invocations = { - let mut collector = InvocationCollector { - cfg: StripUnconfigured { - sess: self.cx.parse_sess, - features: self.cx.ecfg.features, - }, - cx: self.cx, - invocations: Vec::new(), - monotonic: self.monotonic, - }; - fragment.mut_visit_with(&mut collector); - fragment.add_placeholders(extra_placeholders); - collector.invocations - }; - - if self.monotonic { - self.cx - .resolver - .visit_ast_fragment_with_placeholders(self.cx.current_expansion.id, &fragment); - } - - (fragment, invocations) - } - - fn fully_configure(&mut self, item: Annotatable) -> Annotatable { - let mut cfg = - StripUnconfigured { sess: self.cx.parse_sess, features: self.cx.ecfg.features }; - // Since the item itself has already been configured by the InvocationCollector, - // we know that fold result vector will contain exactly one element - match item { - Annotatable::Item(item) => Annotatable::Item(cfg.flat_map_item(item).pop().unwrap()), - Annotatable::TraitItem(item) => Annotatable::TraitItem( - item.map(|item| cfg.flat_map_trait_item(item).pop().unwrap()), - ), - Annotatable::ImplItem(item) => { - Annotatable::ImplItem(item.map(|item| cfg.flat_map_impl_item(item).pop().unwrap())) - } - Annotatable::ForeignItem(item) => Annotatable::ForeignItem( - item.map(|item| cfg.flat_map_foreign_item(item).pop().unwrap()), - ), - Annotatable::Stmt(stmt) => { - Annotatable::Stmt(stmt.map(|stmt| cfg.flat_map_stmt(stmt).pop().unwrap())) - } - Annotatable::Expr(mut expr) => Annotatable::Expr({ - cfg.visit_expr(&mut expr); - expr - }), - Annotatable::Arm(arm) => Annotatable::Arm(cfg.flat_map_arm(arm).pop().unwrap()), - Annotatable::Field(field) => { - Annotatable::Field(cfg.flat_map_field(field).pop().unwrap()) - } - Annotatable::FieldPat(fp) => { - Annotatable::FieldPat(cfg.flat_map_field_pattern(fp).pop().unwrap()) - } - Annotatable::GenericParam(param) => { - Annotatable::GenericParam(cfg.flat_map_generic_param(param).pop().unwrap()) - } - Annotatable::Param(param) => { - Annotatable::Param(cfg.flat_map_param(param).pop().unwrap()) - } - Annotatable::StructField(sf) => { - Annotatable::StructField(cfg.flat_map_struct_field(sf).pop().unwrap()) - } - Annotatable::Variant(v) => Annotatable::Variant(cfg.flat_map_variant(v).pop().unwrap()), - } - } - - fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtensionKind) -> AstFragment { - if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit { - let expn_data = self.cx.current_expansion.id.expn_data(); - let suggested_limit = self.cx.ecfg.recursion_limit * 2; - let mut err = self.cx.struct_span_err( - expn_data.call_site, - &format!( - "recursion limit reached while expanding the macro `{}`", - expn_data.kind.descr() - ), - ); - err.help(&format!( - "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate", - suggested_limit - )); - err.emit(); - self.cx.trace_macros_diag(); - FatalError.raise(); - } - - let (fragment_kind, span) = (invoc.fragment_kind, invoc.span()); - match invoc.kind { - InvocationKind::Bang { mac, .. } => match ext { - SyntaxExtensionKind::Bang(expander) => { - self.gate_proc_macro_expansion_kind(span, fragment_kind); - let tok_result = expander.expand(self.cx, span, mac.args.inner_tokens()); - self.parse_ast_fragment(tok_result, fragment_kind, &mac.path, span) - } - SyntaxExtensionKind::LegacyBang(expander) => { - let prev = self.cx.current_expansion.prior_type_ascription; - self.cx.current_expansion.prior_type_ascription = mac.prior_type_ascription; - let tok_result = expander.expand(self.cx, span, mac.args.inner_tokens()); - let result = if let Some(result) = fragment_kind.make_from(tok_result) { - result - } else { - let msg = format!( - "non-{kind} macro in {kind} position: {path}", - kind = fragment_kind.name(), - path = pprust::path_to_string(&mac.path), - ); - self.cx.span_err(span, &msg); - self.cx.trace_macros_diag(); - fragment_kind.dummy(span) - }; - self.cx.current_expansion.prior_type_ascription = prev; - result - } - _ => unreachable!(), - }, - InvocationKind::Attr { attr, mut item, .. } => match ext { - SyntaxExtensionKind::Attr(expander) => { - self.gate_proc_macro_input(&item); - self.gate_proc_macro_attr_item(span, &item); - let item_tok = TokenTree::token( - token::Interpolated(Lrc::new(match item { - Annotatable::Item(item) => token::NtItem(item), - Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()), - Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()), - Annotatable::ForeignItem(item) => { - token::NtForeignItem(item.into_inner()) - } - Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()), - Annotatable::Expr(expr) => token::NtExpr(expr), - Annotatable::Arm(..) - | Annotatable::Field(..) - | Annotatable::FieldPat(..) - | Annotatable::GenericParam(..) - | Annotatable::Param(..) - | Annotatable::StructField(..) - | Annotatable::Variant(..) => panic!("unexpected annotatable"), - })), - DUMMY_SP, - ) - .into(); - let item = attr.unwrap_normal_item(); - if let MacArgs::Eq(..) = item.args { - self.cx.span_err(span, "key-value macro attributes are not supported"); - } - let tok_result = - expander.expand(self.cx, span, item.args.inner_tokens(), item_tok); - self.parse_ast_fragment(tok_result, fragment_kind, &item.path, span) - } - SyntaxExtensionKind::LegacyAttr(expander) => { - match validate_attr::parse_meta(self.cx.parse_sess, &attr) { - Ok(meta) => { - let item = expander.expand(self.cx, span, &meta, item); - fragment_kind.expect_from_annotatables(item) - } - Err(mut err) => { - err.emit(); - fragment_kind.dummy(span) - } - } - } - SyntaxExtensionKind::NonMacroAttr { mark_used } => { - attr::mark_known(&attr); - if *mark_used { - attr::mark_used(&attr); - } - item.visit_attrs(|attrs| attrs.push(attr)); - fragment_kind.expect_from_annotatables(iter::once(item)) - } - _ => unreachable!(), - }, - InvocationKind::Derive { path, item } => match ext { - SyntaxExtensionKind::Derive(expander) - | SyntaxExtensionKind::LegacyDerive(expander) => { - if !item.derive_allowed() { - return fragment_kind.dummy(span); - } - if let SyntaxExtensionKind::Derive(..) = ext { - self.gate_proc_macro_input(&item); - } - let meta = ast::MetaItem { kind: ast::MetaItemKind::Word, span, path }; - let items = expander.expand(self.cx, span, &meta, item); - fragment_kind.expect_from_annotatables(items) - } - _ => unreachable!(), - }, - InvocationKind::DeriveContainer { .. } => unreachable!(), - } - } - - fn gate_proc_macro_attr_item(&self, span: Span, item: &Annotatable) { - let kind = match item { - Annotatable::Item(_) - | Annotatable::TraitItem(_) - | Annotatable::ImplItem(_) - | Annotatable::ForeignItem(_) => return, - Annotatable::Stmt(_) => "statements", - Annotatable::Expr(_) => "expressions", - Annotatable::Arm(..) - | Annotatable::Field(..) - | Annotatable::FieldPat(..) - | Annotatable::GenericParam(..) - | Annotatable::Param(..) - | Annotatable::StructField(..) - | Annotatable::Variant(..) => panic!("unexpected annotatable"), - }; - if self.cx.ecfg.proc_macro_hygiene() { - return; - } - feature_err( - self.cx.parse_sess, - sym::proc_macro_hygiene, - span, - &format!("custom attributes cannot be applied to {}", kind), - ) - .emit(); - } - - fn gate_proc_macro_input(&self, annotatable: &Annotatable) { - struct GateProcMacroInput<'a> { - parse_sess: &'a ParseSess, - } - - impl<'ast, 'a> Visitor<'ast> for GateProcMacroInput<'a> { - fn visit_item(&mut self, item: &'ast ast::Item) { - match &item.kind { - ast::ItemKind::Mod(module) if !module.inline => { - feature_err( - self.parse_sess, - sym::proc_macro_hygiene, - item.span, - "non-inline modules in proc macro input are unstable", - ) - .emit(); - } - _ => {} - } - - visit::walk_item(self, item); - } - - fn visit_mac(&mut self, _: &'ast ast::Mac) {} - } - - if !self.cx.ecfg.proc_macro_hygiene() { - annotatable.visit_with(&mut GateProcMacroInput { parse_sess: self.cx.parse_sess }); - } - } - - fn gate_proc_macro_expansion_kind(&self, span: Span, kind: AstFragmentKind) { - let kind = match kind { - AstFragmentKind::Expr | AstFragmentKind::OptExpr => "expressions", - AstFragmentKind::Pat => "patterns", - AstFragmentKind::Stmts => "statements", - AstFragmentKind::Ty - | AstFragmentKind::Items - | AstFragmentKind::TraitItems - | AstFragmentKind::ImplItems - | AstFragmentKind::ForeignItems => return, - AstFragmentKind::Arms - | AstFragmentKind::Fields - | AstFragmentKind::FieldPats - | AstFragmentKind::GenericParams - | AstFragmentKind::Params - | AstFragmentKind::StructFields - | AstFragmentKind::Variants => panic!("unexpected AST fragment kind"), - }; - if self.cx.ecfg.proc_macro_hygiene() { - return; - } - feature_err( - self.cx.parse_sess, - sym::proc_macro_hygiene, - span, - &format!("procedural macros cannot be expanded to {}", kind), - ) - .emit(); - } - - fn parse_ast_fragment( - &mut self, - toks: TokenStream, - kind: AstFragmentKind, - path: &Path, - span: Span, - ) -> AstFragment { - let mut parser = self.cx.new_parser_from_tts(toks); - match parse_ast_fragment(&mut parser, kind, false) { - Ok(fragment) => { - ensure_complete_parse(&mut parser, path, kind.name(), span); - fragment - } - Err(mut err) => { - err.set_span(span); - annotate_err_with_kind(&mut err, kind, span); - err.emit(); - self.cx.trace_macros_diag(); - kind.dummy(span) - } - } - } -} - -pub fn parse_ast_fragment<'a>( - this: &mut Parser<'a>, - kind: AstFragmentKind, - macro_legacy_warnings: bool, -) -> PResult<'a, AstFragment> { - Ok(match kind { - AstFragmentKind::Items => { - let mut items = SmallVec::new(); - while let Some(item) = this.parse_item()? { - items.push(item); - } - AstFragment::Items(items) - } - AstFragmentKind::TraitItems => { - let mut items = SmallVec::new(); - while this.token != token::Eof { - items.push(this.parse_trait_item(&mut false)?); - } - AstFragment::TraitItems(items) - } - AstFragmentKind::ImplItems => { - let mut items = SmallVec::new(); - while this.token != token::Eof { - items.push(this.parse_impl_item(&mut false)?); - } - AstFragment::ImplItems(items) - } - AstFragmentKind::ForeignItems => { - let mut items = SmallVec::new(); - while this.token != token::Eof { - items.push(this.parse_foreign_item(DUMMY_SP)?); - } - AstFragment::ForeignItems(items) - } - AstFragmentKind::Stmts => { - let mut stmts = SmallVec::new(); - while this.token != token::Eof && - // won't make progress on a `}` - this.token != token::CloseDelim(token::Brace) - { - if let Some(stmt) = this.parse_full_stmt(macro_legacy_warnings)? { - stmts.push(stmt); - } - } - AstFragment::Stmts(stmts) - } - AstFragmentKind::Expr => AstFragment::Expr(this.parse_expr()?), - AstFragmentKind::OptExpr => { - if this.token != token::Eof { - AstFragment::OptExpr(Some(this.parse_expr()?)) - } else { - AstFragment::OptExpr(None) - } - } - AstFragmentKind::Ty => AstFragment::Ty(this.parse_ty()?), - AstFragmentKind::Pat => AstFragment::Pat(this.parse_pat(None)?), - AstFragmentKind::Arms - | AstFragmentKind::Fields - | AstFragmentKind::FieldPats - | AstFragmentKind::GenericParams - | AstFragmentKind::Params - | AstFragmentKind::StructFields - | AstFragmentKind::Variants => panic!("unexpected AST fragment kind"), - }) -} - -pub fn ensure_complete_parse<'a>( - this: &mut Parser<'a>, - macro_path: &Path, - kind_name: &str, - span: Span, -) { - if this.token != token::Eof { - let token = pprust::token_to_string(&this.token); - let msg = format!("macro expansion ignores token `{}` and any following", token); - // Avoid emitting backtrace info twice. - let def_site_span = this.token.span.with_ctxt(SyntaxContext::root()); - let mut err = this.struct_span_err(def_site_span, &msg); - err.span_label(span, "caused by the macro expansion here"); - let msg = format!( - "the usage of `{}!` is likely invalid in {} context", - pprust::path_to_string(macro_path), - kind_name, - ); - err.note(&msg); - let semi_span = this.sess.source_map().next_point(span); - - let semi_full_span = semi_span.to(this.sess.source_map().next_point(semi_span)); - match this.sess.source_map().span_to_snippet(semi_full_span) { - Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => { - err.span_suggestion( - semi_span, - "you might be missing a semicolon here", - ";".to_owned(), - Applicability::MaybeIncorrect, - ); - } - _ => {} - } - err.emit(); - } -} - -struct InvocationCollector<'a, 'b> { - cx: &'a mut ExtCtxt<'b>, - cfg: StripUnconfigured<'a>, - invocations: Vec, - monotonic: bool, -} - -impl<'a, 'b> InvocationCollector<'a, 'b> { - fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment { - // Expansion data for all the collected invocations is set upon their resolution, - // with exception of the derive container case which is not resolved and can get - // its expansion data immediately. - let expn_data = match &kind { - InvocationKind::DeriveContainer { item, .. } => Some(ExpnData { - parent: self.cx.current_expansion.id, - ..ExpnData::default( - ExpnKind::Macro(MacroKind::Attr, sym::derive), - item.span(), - self.cx.parse_sess.edition, - ) - }), - _ => None, - }; - let expn_id = ExpnId::fresh(expn_data); - let vis = kind.placeholder_visibility(); - self.invocations.push(Invocation { - kind, - fragment_kind, - expansion_data: ExpansionData { - id: expn_id, - depth: self.cx.current_expansion.depth + 1, - ..self.cx.current_expansion.clone() - }, - }); - placeholder(fragment_kind, NodeId::placeholder_from_expn_id(expn_id), vis) - } - - fn collect_bang(&mut self, mac: ast::Mac, span: Span, kind: AstFragmentKind) -> AstFragment { - self.collect(kind, InvocationKind::Bang { mac, span }) - } - - fn collect_attr( - &mut self, - attr: Option, - derives: Vec, - item: Annotatable, - kind: AstFragmentKind, - after_derive: bool, - ) -> AstFragment { - self.collect( - kind, - match attr { - Some(attr) => InvocationKind::Attr { attr, item, derives, after_derive }, - None => InvocationKind::DeriveContainer { derives, item }, - }, - ) - } - - fn find_attr_invoc( - &self, - attrs: &mut Vec, - after_derive: &mut bool, - ) -> Option { - let attr = attrs - .iter() - .position(|a| { - if a.has_name(sym::derive) { - *after_derive = true; - } - !attr::is_known(a) && !is_builtin_attr(a) - }) - .map(|i| attrs.remove(i)); - if let Some(attr) = &attr { - if !self.cx.ecfg.custom_inner_attributes() - && attr.style == ast::AttrStyle::Inner - && !attr.has_name(sym::test) - { - feature_err( - &self.cx.parse_sess, - sym::custom_inner_attributes, - attr.span, - "non-builtin inner attributes are unstable", - ) - .emit(); - } - } - attr - } - - /// If `item` is an attr invocation, remove and return the macro attribute and derive traits. - fn classify_item( - &mut self, - item: &mut T, - ) -> (Option, Vec, /* after_derive */ bool) - where - T: HasAttrs, - { - let (mut attr, mut traits, mut after_derive) = (None, Vec::new(), false); - - item.visit_attrs(|mut attrs| { - attr = self.find_attr_invoc(&mut attrs, &mut after_derive); - traits = collect_derives(&mut self.cx, &mut attrs); - }); - - (attr, traits, after_derive) - } - - /// Alternative to `classify_item()` that ignores `#[derive]` so invocations fallthrough - /// to the unused-attributes lint (making it an error on statements and expressions - /// is a breaking change) - fn classify_nonitem( - &mut self, - nonitem: &mut T, - ) -> (Option, /* after_derive */ bool) { - let (mut attr, mut after_derive) = (None, false); - - nonitem.visit_attrs(|mut attrs| { - attr = self.find_attr_invoc(&mut attrs, &mut after_derive); - }); - - (attr, after_derive) - } - - fn configure(&mut self, node: T) -> Option { - self.cfg.configure(node) - } - - // Detect use of feature-gated or invalid attributes on macro invocations - // since they will not be detected after macro expansion. - fn check_attributes(&mut self, attrs: &[ast::Attribute]) { - let features = self.cx.ecfg.features.unwrap(); - for attr in attrs.iter() { - feature_gate::check_attribute(attr, self.cx.parse_sess, features); - validate_attr::check_meta(self.cx.parse_sess, attr); - - // macros are expanded before any lint passes so this warning has to be hardcoded - if attr.has_name(sym::derive) { - self.cx - .struct_span_warn(attr.span, "`#[derive]` does nothing on macro invocations") - .note("this may become a hard error in a future release") - .emit(); - } - } - } -} - -impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { - fn visit_expr(&mut self, expr: &mut P) { - self.cfg.configure_expr(expr); - visit_clobber(expr.deref_mut(), |mut expr| { - self.cfg.configure_expr_kind(&mut expr.kind); - - // ignore derives so they remain unused - let (attr, after_derive) = self.classify_nonitem(&mut expr); - - if attr.is_some() { - // Collect the invoc regardless of whether or not attributes are permitted here - // expansion will eat the attribute so it won't error later. - attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); - - // AstFragmentKind::Expr requires the macro to emit an expression. - return self - .collect_attr( - attr, - vec![], - Annotatable::Expr(P(expr)), - AstFragmentKind::Expr, - after_derive, - ) - .make_expr() - .into_inner(); - } - - if let ast::ExprKind::Mac(mac) = expr.kind { - self.check_attributes(&expr.attrs); - self.collect_bang(mac, expr.span, AstFragmentKind::Expr).make_expr().into_inner() - } else { - noop_visit_expr(&mut expr, self); - expr - } - }); - } - - fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { - let mut arm = configure!(self, arm); - - let (attr, traits, after_derive) = self.classify_item(&mut arm); - if attr.is_some() || !traits.is_empty() { - return self - .collect_attr( - attr, - traits, - Annotatable::Arm(arm), - AstFragmentKind::Arms, - after_derive, - ) - .make_arms(); - } - - noop_flat_map_arm(arm, self) - } - - fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { - let mut field = configure!(self, field); - - let (attr, traits, after_derive) = self.classify_item(&mut field); - if attr.is_some() || !traits.is_empty() { - return self - .collect_attr( - attr, - traits, - Annotatable::Field(field), - AstFragmentKind::Fields, - after_derive, - ) - .make_fields(); - } - - noop_flat_map_field(field, self) - } - - fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { - let mut fp = configure!(self, fp); - - let (attr, traits, after_derive) = self.classify_item(&mut fp); - if attr.is_some() || !traits.is_empty() { - return self - .collect_attr( - attr, - traits, - Annotatable::FieldPat(fp), - AstFragmentKind::FieldPats, - after_derive, - ) - .make_field_patterns(); - } - - noop_flat_map_field_pattern(fp, self) - } - - fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { - let mut p = configure!(self, p); - - let (attr, traits, after_derive) = self.classify_item(&mut p); - if attr.is_some() || !traits.is_empty() { - return self - .collect_attr( - attr, - traits, - Annotatable::Param(p), - AstFragmentKind::Params, - after_derive, - ) - .make_params(); - } - - noop_flat_map_param(p, self) - } - - fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { - let mut sf = configure!(self, sf); - - let (attr, traits, after_derive) = self.classify_item(&mut sf); - if attr.is_some() || !traits.is_empty() { - return self - .collect_attr( - attr, - traits, - Annotatable::StructField(sf), - AstFragmentKind::StructFields, - after_derive, - ) - .make_struct_fields(); - } - - noop_flat_map_struct_field(sf, self) - } - - fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { - let mut variant = configure!(self, variant); - - let (attr, traits, after_derive) = self.classify_item(&mut variant); - if attr.is_some() || !traits.is_empty() { - return self - .collect_attr( - attr, - traits, - Annotatable::Variant(variant), - AstFragmentKind::Variants, - after_derive, - ) - .make_variants(); - } - - noop_flat_map_variant(variant, self) - } - - fn filter_map_expr(&mut self, expr: P) -> Option> { - let expr = configure!(self, expr); - expr.filter_map(|mut expr| { - self.cfg.configure_expr_kind(&mut expr.kind); - - // Ignore derives so they remain unused. - let (attr, after_derive) = self.classify_nonitem(&mut expr); - - if attr.is_some() { - attr.as_ref().map(|a| self.cfg.maybe_emit_expr_attr_err(a)); - - return self - .collect_attr( - attr, - vec![], - Annotatable::Expr(P(expr)), - AstFragmentKind::OptExpr, - after_derive, - ) - .make_opt_expr() - .map(|expr| expr.into_inner()); - } - - if let ast::ExprKind::Mac(mac) = expr.kind { - self.check_attributes(&expr.attrs); - self.collect_bang(mac, expr.span, AstFragmentKind::OptExpr) - .make_opt_expr() - .map(|expr| expr.into_inner()) - } else { - Some({ - noop_visit_expr(&mut expr, self); - expr - }) - } - }) - } - - fn visit_pat(&mut self, pat: &mut P) { - self.cfg.configure_pat(pat); - match pat.kind { - PatKind::Mac(_) => {} - _ => return noop_visit_pat(pat, self), - } - - visit_clobber(pat, |mut pat| match mem::replace(&mut pat.kind, PatKind::Wild) { - PatKind::Mac(mac) => self.collect_bang(mac, pat.span, AstFragmentKind::Pat).make_pat(), - _ => unreachable!(), - }); - } - - fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { - let mut stmt = configure!(self, stmt); - - // we'll expand attributes on expressions separately - if !stmt.is_expr() { - let (attr, derives, after_derive) = if stmt.is_item() { - self.classify_item(&mut stmt) - } else { - // ignore derives on non-item statements so it falls through - // to the unused-attributes lint - let (attr, after_derive) = self.classify_nonitem(&mut stmt); - (attr, vec![], after_derive) - }; - - if attr.is_some() || !derives.is_empty() { - return self - .collect_attr( - attr, - derives, - Annotatable::Stmt(P(stmt)), - AstFragmentKind::Stmts, - after_derive, - ) - .make_stmts(); - } - } - - if let StmtKind::Mac(mac) = stmt.kind { - let (mac, style, attrs) = mac.into_inner(); - self.check_attributes(&attrs); - let mut placeholder = - self.collect_bang(mac, stmt.span, AstFragmentKind::Stmts).make_stmts(); - - // If this is a macro invocation with a semicolon, then apply that - // semicolon to the final statement produced by expansion. - if style == MacStmtStyle::Semicolon { - if let Some(stmt) = placeholder.pop() { - placeholder.push(stmt.add_trailing_semicolon()); - } - } - - return placeholder; - } - - // The placeholder expander gives ids to statements, so we avoid folding the id here. - let ast::Stmt { id, kind, span } = stmt; - noop_flat_map_stmt_kind(kind, self) - .into_iter() - .map(|kind| ast::Stmt { id, kind, span }) - .collect() - } - - fn visit_block(&mut self, block: &mut P) { - let old_directory_ownership = self.cx.current_expansion.directory_ownership; - self.cx.current_expansion.directory_ownership = DirectoryOwnership::UnownedViaBlock; - noop_visit_block(block, self); - self.cx.current_expansion.directory_ownership = old_directory_ownership; - } - - fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { - let mut item = configure!(self, item); - - let (attr, traits, after_derive) = self.classify_item(&mut item); - if attr.is_some() || !traits.is_empty() { - return self - .collect_attr( - attr, - traits, - Annotatable::Item(item), - AstFragmentKind::Items, - after_derive, - ) - .make_items(); - } - - match item.kind { - ast::ItemKind::Mac(..) => { - self.check_attributes(&item.attrs); - item.and_then(|item| match item.kind { - ItemKind::Mac(mac) => self - .collect( - AstFragmentKind::Items, - InvocationKind::Bang { mac, span: item.span }, - ) - .make_items(), - _ => unreachable!(), - }) - } - ast::ItemKind::Mod(ast::Mod { inner, .. }) => { - if item.ident == Ident::invalid() { - return noop_flat_map_item(item, self); - } - - let orig_directory_ownership = self.cx.current_expansion.directory_ownership; - let mut module = (*self.cx.current_expansion.module).clone(); - module.mod_path.push(item.ident); - - // Detect if this is an inline module (`mod m { ... }` as opposed to `mod m;`). - // In the non-inline case, `inner` is never the dummy span (cf. `parse_item_mod`). - // Thus, if `inner` is the dummy span, we know the module is inline. - let inline_module = item.span.contains(inner) || inner.is_dummy(); - - if inline_module { - if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, sym::path) { - self.cx.current_expansion.directory_ownership = - DirectoryOwnership::Owned { relative: None }; - module.directory.push(&*path.as_str()); - } else { - module.directory.push(&*item.ident.as_str()); - } - } else { - let path = self.cx.parse_sess.source_map().span_to_unmapped_path(inner); - let mut path = match path { - FileName::Real(path) => path, - other => PathBuf::from(other.to_string()), - }; - let directory_ownership = match path.file_name().unwrap().to_str() { - Some("mod.rs") => DirectoryOwnership::Owned { relative: None }, - Some(_) => DirectoryOwnership::Owned { relative: Some(item.ident) }, - None => DirectoryOwnership::UnownedViaMod, - }; - path.pop(); - module.directory = path; - self.cx.current_expansion.directory_ownership = directory_ownership; - } - - let orig_module = - mem::replace(&mut self.cx.current_expansion.module, Rc::new(module)); - let result = noop_flat_map_item(item, self); - self.cx.current_expansion.module = orig_module; - self.cx.current_expansion.directory_ownership = orig_directory_ownership; - result - } - - _ => noop_flat_map_item(item, self), - } - } - - fn flat_map_trait_item(&mut self, item: ast::AssocItem) -> SmallVec<[ast::AssocItem; 1]> { - let mut item = configure!(self, item); - - let (attr, traits, after_derive) = self.classify_item(&mut item); - if attr.is_some() || !traits.is_empty() { - return self - .collect_attr( - attr, - traits, - Annotatable::TraitItem(P(item)), - AstFragmentKind::TraitItems, - after_derive, - ) - .make_trait_items(); - } - - match item.kind { - ast::AssocItemKind::Macro(mac) => { - let ast::AssocItem { attrs, span, .. } = item; - self.check_attributes(&attrs); - self.collect_bang(mac, span, AstFragmentKind::TraitItems).make_trait_items() - } - _ => noop_flat_map_assoc_item(item, self), - } - } - - fn flat_map_impl_item(&mut self, item: ast::AssocItem) -> SmallVec<[ast::AssocItem; 1]> { - let mut item = configure!(self, item); - - let (attr, traits, after_derive) = self.classify_item(&mut item); - if attr.is_some() || !traits.is_empty() { - return self - .collect_attr( - attr, - traits, - Annotatable::ImplItem(P(item)), - AstFragmentKind::ImplItems, - after_derive, - ) - .make_impl_items(); - } - - match item.kind { - ast::AssocItemKind::Macro(mac) => { - let ast::AssocItem { attrs, span, .. } = item; - self.check_attributes(&attrs); - self.collect_bang(mac, span, AstFragmentKind::ImplItems).make_impl_items() - } - _ => noop_flat_map_assoc_item(item, self), - } - } - - fn visit_ty(&mut self, ty: &mut P) { - match ty.kind { - ast::TyKind::Mac(_) => {} - _ => return noop_visit_ty(ty, self), - }; - - visit_clobber(ty, |mut ty| match mem::replace(&mut ty.kind, ast::TyKind::Err) { - ast::TyKind::Mac(mac) => self.collect_bang(mac, ty.span, AstFragmentKind::Ty).make_ty(), - _ => unreachable!(), - }); - } - - fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) { - self.cfg.configure_foreign_mod(foreign_mod); - noop_visit_foreign_mod(foreign_mod, self); - } - - fn flat_map_foreign_item( - &mut self, - mut foreign_item: ast::ForeignItem, - ) -> SmallVec<[ast::ForeignItem; 1]> { - let (attr, traits, after_derive) = self.classify_item(&mut foreign_item); - - if attr.is_some() || !traits.is_empty() { - return self - .collect_attr( - attr, - traits, - Annotatable::ForeignItem(P(foreign_item)), - AstFragmentKind::ForeignItems, - after_derive, - ) - .make_foreign_items(); - } - - if let ast::ForeignItemKind::Macro(mac) = foreign_item.kind { - self.check_attributes(&foreign_item.attrs); - return self - .collect_bang(mac, foreign_item.span, AstFragmentKind::ForeignItems) - .make_foreign_items(); - } - - noop_flat_map_foreign_item(foreign_item, self) - } - - fn visit_item_kind(&mut self, item: &mut ast::ItemKind) { - match item { - ast::ItemKind::MacroDef(..) => {} - _ => { - self.cfg.configure_item_kind(item); - noop_visit_item_kind(item, self); - } - } - } - - fn flat_map_generic_param( - &mut self, - param: ast::GenericParam, - ) -> SmallVec<[ast::GenericParam; 1]> { - let mut param = configure!(self, param); - - let (attr, traits, after_derive) = self.classify_item(&mut param); - if attr.is_some() || !traits.is_empty() { - return self - .collect_attr( - attr, - traits, - Annotatable::GenericParam(param), - AstFragmentKind::GenericParams, - after_derive, - ) - .make_generic_params(); - } - - noop_flat_map_generic_param(param, self) - } - - fn visit_attribute(&mut self, at: &mut ast::Attribute) { - // turn `#[doc(include="filename")]` attributes into `#[doc(include(file="filename", - // contents="file contents")]` attributes - if !at.check_name(sym::doc) { - return noop_visit_attribute(at, self); - } - - if let Some(list) = at.meta_item_list() { - if !list.iter().any(|it| it.check_name(sym::include)) { - return noop_visit_attribute(at, self); - } - - let mut items = vec![]; - - for mut it in list { - if !it.check_name(sym::include) { - items.push({ - noop_visit_meta_list_item(&mut it, self); - it - }); - continue; - } - - if let Some(file) = it.value_str() { - let err_count = self.cx.parse_sess.span_diagnostic.err_count(); - self.check_attributes(slice::from_ref(at)); - if self.cx.parse_sess.span_diagnostic.err_count() > err_count { - // avoid loading the file if they haven't enabled the feature - return noop_visit_attribute(at, self); - } - - let filename = match self.cx.resolve_path(&*file.as_str(), it.span()) { - Ok(filename) => filename, - Err(mut err) => { - err.emit(); - continue; - } - }; - - match self.cx.source_map().load_file(&filename) { - Ok(source_file) => { - let src = source_file - .src - .as_ref() - .expect("freshly loaded file should have a source"); - let src_interned = Symbol::intern(src.as_str()); - - let include_info = vec![ - ast::NestedMetaItem::MetaItem(attr::mk_name_value_item_str( - Ident::with_dummy_span(sym::file), - file, - DUMMY_SP, - )), - ast::NestedMetaItem::MetaItem(attr::mk_name_value_item_str( - Ident::with_dummy_span(sym::contents), - src_interned, - DUMMY_SP, - )), - ]; - - let include_ident = Ident::with_dummy_span(sym::include); - let item = attr::mk_list_item(include_ident, include_info); - items.push(ast::NestedMetaItem::MetaItem(item)); - } - Err(e) => { - let lit = - it.meta_item().and_then(|item| item.name_value_literal()).unwrap(); - - if e.kind() == ErrorKind::InvalidData { - self.cx - .struct_span_err( - lit.span, - &format!("{} wasn't a utf-8 file", filename.display()), - ) - .span_label(lit.span, "contains invalid utf-8") - .emit(); - } else { - let mut err = self.cx.struct_span_err( - lit.span, - &format!("couldn't read {}: {}", filename.display(), e), - ); - err.span_label(lit.span, "couldn't read file"); - - err.emit(); - } - } - } - } else { - let mut err = self.cx.struct_span_err( - it.span(), - &format!("expected path to external documentation"), - ); - - // Check if the user erroneously used `doc(include(...))` syntax. - let literal = it.meta_item_list().and_then(|list| { - if list.len() == 1 { - list[0].literal().map(|literal| &literal.kind) - } else { - None - } - }); - - let (path, applicability) = match &literal { - Some(LitKind::Str(path, ..)) => { - (path.to_string(), Applicability::MachineApplicable) - } - _ => (String::from(""), Applicability::HasPlaceholders), - }; - - err.span_suggestion( - it.span(), - "provide a file path with `=`", - format!("include = \"{}\"", path), - applicability, - ); - - err.emit(); - } - } - - let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items); - *at = attr::Attribute { - kind: ast::AttrKind::Normal(AttrItem { - path: meta.path, - args: meta.kind.mac_args(meta.span), - }), - span: at.span, - id: at.id, - style: at.style, - }; - } else { - noop_visit_attribute(at, self) - } - } - - fn visit_id(&mut self, id: &mut ast::NodeId) { - if self.monotonic { - debug_assert_eq!(*id, ast::DUMMY_NODE_ID); - *id = self.cx.resolver.next_node_id() - } - } - - fn visit_fn_decl(&mut self, mut fn_decl: &mut P) { - self.cfg.configure_fn_decl(&mut fn_decl); - noop_visit_fn_decl(fn_decl, self); - } -} - -pub struct ExpansionConfig<'feat> { - pub crate_name: String, - pub features: Option<&'feat Features>, - pub recursion_limit: usize, - pub trace_mac: bool, - pub should_test: bool, // If false, strip `#[test]` nodes - pub single_step: bool, - pub keep_macs: bool, -} - -impl<'feat> ExpansionConfig<'feat> { - pub fn default(crate_name: String) -> ExpansionConfig<'static> { - ExpansionConfig { - crate_name, - features: None, - recursion_limit: 1024, - trace_mac: false, - should_test: false, - single_step: false, - keep_macs: false, - } - } - - fn proc_macro_hygiene(&self) -> bool { - self.features.map_or(false, |features| features.proc_macro_hygiene) - } - fn custom_inner_attributes(&self) -> bool { - self.features.map_or(false, |features| features.custom_inner_attributes) - } -} diff --git a/src/libsyntax_expand/lib.rs b/src/libsyntax_expand/lib.rs deleted file mode 100644 index 258a7478329..00000000000 --- a/src/libsyntax_expand/lib.rs +++ /dev/null @@ -1,67 +0,0 @@ -#![feature(crate_visibility_modifier)] -#![feature(decl_macro)] -#![feature(proc_macro_diagnostic)] -#![feature(proc_macro_internals)] -#![feature(proc_macro_span)] - -extern crate proc_macro as pm; - -// A variant of 'try!' that panics on an Err. This is used as a crutch on the -// way towards a non-panic!-prone parser. It should be used for fatal parsing -// errors; eventually we plan to convert all code using panictry to just use -// normal try. -#[macro_export] -macro_rules! panictry { - ($e:expr) => {{ - use errors::FatalError; - use std::result::Result::{Err, Ok}; - match $e { - Ok(e) => e, - Err(mut e) => { - e.emit(); - FatalError.raise() - } - } - }}; -} - -mod placeholders; -mod proc_macro_server; - -pub use mbe::macro_rules::compile_declarative_macro; -crate use syntax_pos::hygiene; -pub mod base; -pub mod build; -pub mod expand; -pub use rustc_parse::config; -pub mod proc_macro; - -crate mod mbe; - -// HACK(Centril, #64197): These shouldn't really be here. -// Rather, they should be with their respective modules which are defined in other crates. -// However, since for now constructing a `ParseSess` sorta requires `config` from this crate, -// these tests will need to live here in the iterim. - -#[cfg(test)] -mod tests; -#[cfg(test)] -mod parse { - #[cfg(test)] - mod tests; - #[cfg(test)] - mod lexer { - #[cfg(test)] - mod tests; - } -} -#[cfg(test)] -mod tokenstream { - #[cfg(test)] - mod tests; -} -#[cfg(test)] -mod mut_visit { - #[cfg(test)] - mod tests; -} diff --git a/src/libsyntax_expand/mbe.rs b/src/libsyntax_expand/mbe.rs deleted file mode 100644 index 0473b653424..00000000000 --- a/src/libsyntax_expand/mbe.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! This module implements declarative macros: old `macro_rules` and the newer -//! `macro`. Declarative macros are also known as "macro by example", and that's -//! why we call this module `mbe`. For external documentation, prefer the -//! official terminology: "declarative macros". - -crate mod macro_check; -crate mod macro_parser; -crate mod macro_rules; -crate mod quoted; -crate mod transcribe; - -use syntax::ast; -use syntax::token::{self, Token, TokenKind}; -use syntax::tokenstream::DelimSpan; - -use syntax_pos::Span; - -use rustc_data_structures::sync::Lrc; - -/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note -/// that the delimiter itself might be `NoDelim`. -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -struct Delimited { - delim: token::DelimToken, - tts: Vec, -} - -impl Delimited { - /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter. - fn open_tt(&self, span: DelimSpan) -> TokenTree { - TokenTree::token(token::OpenDelim(self.delim), span.open) - } - - /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter. - fn close_tt(&self, span: DelimSpan) -> TokenTree { - TokenTree::token(token::CloseDelim(self.delim), span.close) - } -} - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -struct SequenceRepetition { - /// The sequence of token trees - tts: Vec, - /// The optional separator - separator: Option, - /// Whether the sequence can be repeated zero (*), or one or more times (+) - kleene: KleeneToken, - /// The number of `Match`s that appear in the sequence (and subsequences) - num_captures: usize, -} - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)] -struct KleeneToken { - span: Span, - op: KleeneOp, -} - -impl KleeneToken { - fn new(op: KleeneOp, span: Span) -> KleeneToken { - KleeneToken { span, op } - } -} - -/// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) -/// for token sequences. -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)] -enum KleeneOp { - /// Kleene star (`*`) for zero or more repetitions - ZeroOrMore, - /// Kleene plus (`+`) for one or more repetitions - OneOrMore, - /// Kleene optional (`?`) for zero or one reptitions - ZeroOrOne, -} - -/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)` -/// are "first-class" token trees. Useful for parsing macros. -#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)] -enum TokenTree { - Token(Token), - Delimited(DelimSpan, Lrc), - /// A kleene-style repetition sequence - Sequence(DelimSpan, Lrc), - /// e.g., `$var` - MetaVar(Span, ast::Ident), - /// e.g., `$var:expr`. This is only used in the left hand side of MBE macros. - MetaVarDecl( - Span, - ast::Ident, /* name to bind */ - ast::Ident, /* kind of nonterminal */ - ), -} - -impl TokenTree { - /// Return the number of tokens in the tree. - fn len(&self) -> usize { - match *self { - TokenTree::Delimited(_, ref delimed) => match delimed.delim { - token::NoDelim => delimed.tts.len(), - _ => delimed.tts.len() + 2, - }, - TokenTree::Sequence(_, ref seq) => seq.tts.len(), - _ => 0, - } - } - - /// Returns `true` if the given token tree is delimited. - fn is_delimited(&self) -> bool { - match *self { - TokenTree::Delimited(..) => true, - _ => false, - } - } - - /// Returns `true` if the given token tree is a token of the given kind. - fn is_token(&self, expected_kind: &TokenKind) -> bool { - match self { - TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind, - _ => false, - } - } - - /// Gets the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences. - fn get_tt(&self, index: usize) -> TokenTree { - match (self, index) { - (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => { - delimed.tts[index].clone() - } - (&TokenTree::Delimited(span, ref delimed), _) => { - if index == 0 { - return delimed.open_tt(span); - } - if index == delimed.tts.len() + 1 { - return delimed.close_tt(span); - } - delimed.tts[index - 1].clone() - } - (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(), - _ => panic!("Cannot expand a token tree"), - } - } - - /// Retrieves the `TokenTree`'s span. - fn span(&self) -> Span { - match *self { - TokenTree::Token(Token { span, .. }) - | TokenTree::MetaVar(span, _) - | TokenTree::MetaVarDecl(span, _, _) => span, - TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(), - } - } - - fn token(kind: TokenKind, span: Span) -> TokenTree { - TokenTree::Token(Token::new(kind, span)) - } -} diff --git a/src/libsyntax_expand/mbe/macro_check.rs b/src/libsyntax_expand/mbe/macro_check.rs deleted file mode 100644 index 616fddd3c1c..00000000000 --- a/src/libsyntax_expand/mbe/macro_check.rs +++ /dev/null @@ -1,627 +0,0 @@ -//! Checks that meta-variables in macro definition are correctly declared and used. -//! -//! # What is checked -//! -//! ## Meta-variables must not be bound twice -//! -//! ``` -//! macro_rules! foo { ($x:tt $x:tt) => { $x }; } -//! ``` -//! -//! This check is sound (no false-negative) and complete (no false-positive). -//! -//! ## Meta-variables must not be free -//! -//! ``` -//! macro_rules! foo { () => { $x }; } -//! ``` -//! -//! This check is also done at macro instantiation but only if the branch is taken. -//! -//! ## Meta-variables must repeat at least as many times as their binder -//! -//! ``` -//! macro_rules! foo { ($($x:tt)*) => { $x }; } -//! ``` -//! -//! This check is also done at macro instantiation but only if the branch is taken. -//! -//! ## Meta-variables must repeat with the same Kleene operators as their binder -//! -//! ``` -//! macro_rules! foo { ($($x:tt)+) => { $($x)* }; } -//! ``` -//! -//! This check is not done at macro instantiation. -//! -//! # Disclaimer -//! -//! In the presence of nested macros (a macro defined in a macro), those checks may have false -//! positives and false negatives. We try to detect those cases by recognizing potential macro -//! definitions in RHSes, but nested macros may be hidden through the use of particular values of -//! meta-variables. -//! -//! ## Examples of false positive -//! -//! False positives can come from cases where we don't recognize a nested macro, because it depends -//! on particular values of meta-variables. In the following example, we think both instances of -//! `$x` are free, which is a correct statement if `$name` is anything but `macro_rules`. But when -//! `$name` is `macro_rules`, like in the instantiation below, then `$x:tt` is actually a binder of -//! the nested macro and `$x` is bound to it. -//! -//! ``` -//! macro_rules! foo { ($name:ident) => { $name! bar { ($x:tt) => { $x }; } }; } -//! foo!(macro_rules); -//! ``` -//! -//! False positives can also come from cases where we think there is a nested macro while there -//! isn't. In the following example, we think `$x` is free, which is incorrect because `bar` is not -//! a nested macro since it is not evaluated as code by `stringify!`. -//! -//! ``` -//! macro_rules! foo { () => { stringify!(macro_rules! bar { () => { $x }; }) }; } -//! ``` -//! -//! ## Examples of false negative -//! -//! False negatives can come from cases where we don't recognize a meta-variable, because it depends -//! on particular values of meta-variables. In the following examples, we don't see that if `$d` is -//! instantiated with `$` then `$d z` becomes `$z` in the nested macro definition and is thus a free -//! meta-variable. Note however, that if `foo` is instantiated, then we would check the definition -//! of `bar` and would see the issue. -//! -//! ``` -//! macro_rules! foo { ($d:tt) => { macro_rules! bar { ($y:tt) => { $d z }; } }; } -//! ``` -//! -//! # How it is checked -//! -//! There are 3 main functions: `check_binders`, `check_occurrences`, and `check_nested_macro`. They -//! all need some kind of environment. -//! -//! ## Environments -//! -//! Environments are used to pass information. -//! -//! ### From LHS to RHS -//! -//! When checking a LHS with `check_binders`, we produce (and use) an environment for binders, -//! namely `Binders`. This is a mapping from binder name to information about that binder: the span -//! of the binder for error messages and the stack of Kleene operators under which it was bound in -//! the LHS. -//! -//! This environment is used by both the LHS and RHS. The LHS uses it to detect duplicate binders. -//! The RHS uses it to detect the other errors. -//! -//! ### From outer macro to inner macro -//! -//! When checking the RHS of an outer macro and we detect a nested macro definition, we push the -//! current state, namely `MacroState`, to an environment of nested macro definitions. Each state -//! stores the LHS binders when entering the macro definition as well as the stack of Kleene -//! operators under which the inner macro is defined in the RHS. -//! -//! This environment is a stack representing the nesting of macro definitions. As such, the stack of -//! Kleene operators under which a meta-variable is repeating is the concatenation of the stacks -//! stored when entering a macro definition starting from the state in which the meta-variable is -//! bound. -use crate::mbe::{KleeneToken, TokenTree}; - -use syntax::ast::NodeId; -use syntax::early_buffered_lints::META_VARIABLE_MISUSE; -use syntax::sess::ParseSess; -use syntax::symbol::{kw, sym}; -use syntax::token::{DelimToken, Token, TokenKind}; - -use rustc_data_structures::fx::FxHashMap; -use smallvec::SmallVec; -use syntax_pos::{symbol::Ident, MultiSpan, Span}; - -/// Stack represented as linked list. -/// -/// Those are used for environments because they grow incrementally and are not mutable. -enum Stack<'a, T> { - /// Empty stack. - Empty, - /// A non-empty stack. - Push { - /// The top element. - top: T, - /// The previous elements. - prev: &'a Stack<'a, T>, - }, -} - -impl<'a, T> Stack<'a, T> { - /// Returns whether a stack is empty. - fn is_empty(&self) -> bool { - match *self { - Stack::Empty => true, - _ => false, - } - } - - /// Returns a new stack with an element of top. - fn push(&'a self, top: T) -> Stack<'a, T> { - Stack::Push { top, prev: self } - } -} - -impl<'a, T> Iterator for &'a Stack<'a, T> { - type Item = &'a T; - - // Iterates from top to bottom of the stack. - fn next(&mut self) -> Option<&'a T> { - match *self { - Stack::Empty => None, - Stack::Push { ref top, ref prev } => { - *self = prev; - Some(top) - } - } - } -} - -impl From<&Stack<'_, KleeneToken>> for SmallVec<[KleeneToken; 1]> { - fn from(ops: &Stack<'_, KleeneToken>) -> SmallVec<[KleeneToken; 1]> { - let mut ops: SmallVec<[KleeneToken; 1]> = ops.cloned().collect(); - // The stack is innermost on top. We want outermost first. - ops.reverse(); - ops - } -} - -/// Information attached to a meta-variable binder in LHS. -struct BinderInfo { - /// The span of the meta-variable in LHS. - span: Span, - /// The stack of Kleene operators (outermost first). - ops: SmallVec<[KleeneToken; 1]>, -} - -/// An environment of meta-variables to their binder information. -type Binders = FxHashMap; - -/// The state at which we entered a macro definition in the RHS of another macro definition. -struct MacroState<'a> { - /// The binders of the branch where we entered the macro definition. - binders: &'a Binders, - /// The stack of Kleene operators (outermost first) where we entered the macro definition. - ops: SmallVec<[KleeneToken; 1]>, -} - -/// Checks that meta-variables are used correctly in a macro definition. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `span` is used when no spans are available -/// - `lhses` and `rhses` should have the same length and represent the macro definition -pub(super) fn check_meta_variables( - sess: &ParseSess, - node_id: NodeId, - span: Span, - lhses: &[TokenTree], - rhses: &[TokenTree], -) -> bool { - if lhses.len() != rhses.len() { - sess.span_diagnostic.span_bug(span, "length mismatch between LHSes and RHSes") - } - let mut valid = true; - for (lhs, rhs) in lhses.iter().zip(rhses.iter()) { - let mut binders = Binders::default(); - check_binders(sess, node_id, lhs, &Stack::Empty, &mut binders, &Stack::Empty, &mut valid); - check_occurrences(sess, node_id, rhs, &Stack::Empty, &binders, &Stack::Empty, &mut valid); - } - valid -} - -/// Checks `lhs` as part of the LHS of a macro definition, extends `binders` with new binders, and -/// sets `valid` to false in case of errors. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `lhs` is checked as part of a LHS -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the LHS -/// - `ops` is the stack of Kleene operators from the LHS -/// - `valid` is set in case of errors -fn check_binders( - sess: &ParseSess, - node_id: NodeId, - lhs: &TokenTree, - macros: &Stack<'_, MacroState<'_>>, - binders: &mut Binders, - ops: &Stack<'_, KleeneToken>, - valid: &mut bool, -) { - match *lhs { - TokenTree::Token(..) => {} - // This can only happen when checking a nested macro because this LHS is then in the RHS of - // the outer macro. See ui/macros/macro-of-higher-order.rs where $y:$fragment in the - // LHS of the nested macro (and RHS of the outer macro) is parsed as MetaVar(y) Colon - // MetaVar(fragment) and not as MetaVarDecl(y, fragment). - TokenTree::MetaVar(span, name) => { - if macros.is_empty() { - sess.span_diagnostic.span_bug(span, "unexpected MetaVar in lhs"); - } - // There are 3 possibilities: - if let Some(prev_info) = binders.get(&name) { - // 1. The meta-variable is already bound in the current LHS: This is an error. - let mut span = MultiSpan::from_span(span); - span.push_span_label(prev_info.span, "previous declaration".into()); - buffer_lint(sess, span, node_id, "duplicate matcher binding"); - } else if get_binder_info(macros, binders, name).is_none() { - // 2. The meta-variable is free: This is a binder. - binders.insert(name, BinderInfo { span, ops: ops.into() }); - } else { - // 3. The meta-variable is bound: This is an occurrence. - check_occurrences(sess, node_id, lhs, macros, binders, ops, valid); - } - } - // Similarly, this can only happen when checking a toplevel macro. - TokenTree::MetaVarDecl(span, name, _kind) => { - if !macros.is_empty() { - sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in nested lhs"); - } - if let Some(prev_info) = get_binder_info(macros, binders, name) { - // Duplicate binders at the top-level macro definition are errors. The lint is only - // for nested macro definitions. - sess.span_diagnostic - .struct_span_err(span, "duplicate matcher binding") - .span_label(span, "duplicate binding") - .span_label(prev_info.span, "previous binding") - .emit(); - *valid = false; - } else { - binders.insert(name, BinderInfo { span, ops: ops.into() }); - } - } - TokenTree::Delimited(_, ref del) => { - for tt in &del.tts { - check_binders(sess, node_id, tt, macros, binders, ops, valid); - } - } - TokenTree::Sequence(_, ref seq) => { - let ops = ops.push(seq.kleene); - for tt in &seq.tts { - check_binders(sess, node_id, tt, macros, binders, &ops, valid); - } - } - } -} - -/// Returns the binder information of a meta-variable. -/// -/// Arguments: -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the current binders -/// - `name` is the name of the meta-variable we are looking for -fn get_binder_info<'a>( - mut macros: &'a Stack<'a, MacroState<'a>>, - binders: &'a Binders, - name: Ident, -) -> Option<&'a BinderInfo> { - binders.get(&name).or_else(|| macros.find_map(|state| state.binders.get(&name))) -} - -/// Checks `rhs` as part of the RHS of a macro definition and sets `valid` to false in case of -/// errors. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `rhs` is checked as part of a RHS -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the associated LHS -/// - `ops` is the stack of Kleene operators from the RHS -/// - `valid` is set in case of errors -fn check_occurrences( - sess: &ParseSess, - node_id: NodeId, - rhs: &TokenTree, - macros: &Stack<'_, MacroState<'_>>, - binders: &Binders, - ops: &Stack<'_, KleeneToken>, - valid: &mut bool, -) { - match *rhs { - TokenTree::Token(..) => {} - TokenTree::MetaVarDecl(span, _name, _kind) => { - sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in rhs") - } - TokenTree::MetaVar(span, name) => { - check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name); - } - TokenTree::Delimited(_, ref del) => { - check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid); - } - TokenTree::Sequence(_, ref seq) => { - let ops = ops.push(seq.kleene); - check_nested_occurrences(sess, node_id, &seq.tts, macros, binders, &ops, valid); - } - } -} - -/// Represents the processed prefix of a nested macro. -#[derive(Clone, Copy, PartialEq, Eq)] -enum NestedMacroState { - /// Nothing that matches a nested macro definition was processed yet. - Empty, - /// The token `macro_rules` was processed. - MacroRules, - /// The tokens `macro_rules!` were processed. - MacroRulesNot, - /// The tokens `macro_rules!` followed by a name were processed. The name may be either directly - /// an identifier or a meta-variable (that hopefully would be instantiated by an identifier). - MacroRulesNotName, - /// The keyword `macro` was processed. - Macro, - /// The keyword `macro` followed by a name was processed. - MacroName, - /// The keyword `macro` followed by a name and a token delimited by parentheses was processed. - MacroNameParen, -} - -/// Checks `tts` as part of the RHS of a macro definition, tries to recognize nested macro -/// definitions, and sets `valid` to false in case of errors. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `tts` is checked as part of a RHS and may contain macro definitions -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the associated LHS -/// - `ops` is the stack of Kleene operators from the RHS -/// - `valid` is set in case of errors -fn check_nested_occurrences( - sess: &ParseSess, - node_id: NodeId, - tts: &[TokenTree], - macros: &Stack<'_, MacroState<'_>>, - binders: &Binders, - ops: &Stack<'_, KleeneToken>, - valid: &mut bool, -) { - let mut state = NestedMacroState::Empty; - let nested_macros = macros.push(MacroState { binders, ops: ops.into() }); - let mut nested_binders = Binders::default(); - for tt in tts { - match (state, tt) { - ( - NestedMacroState::Empty, - &TokenTree::Token(Token { kind: TokenKind::Ident(name, false), .. }), - ) => { - if name == sym::macro_rules { - state = NestedMacroState::MacroRules; - } else if name == kw::Macro { - state = NestedMacroState::Macro; - } - } - ( - NestedMacroState::MacroRules, - &TokenTree::Token(Token { kind: TokenKind::Not, .. }), - ) => { - state = NestedMacroState::MacroRulesNot; - } - ( - NestedMacroState::MacroRulesNot, - &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), - ) => { - state = NestedMacroState::MacroRulesNotName; - } - (NestedMacroState::MacroRulesNot, &TokenTree::MetaVar(..)) => { - state = NestedMacroState::MacroRulesNotName; - // We check that the meta-variable is correctly used. - check_occurrences(sess, node_id, tt, macros, binders, ops, valid); - } - (NestedMacroState::MacroRulesNotName, &TokenTree::Delimited(_, ref del)) - | (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) - if del.delim == DelimToken::Brace => - { - let legacy = state == NestedMacroState::MacroRulesNotName; - state = NestedMacroState::Empty; - let rest = - check_nested_macro(sess, node_id, legacy, &del.tts, &nested_macros, valid); - // If we did not check the whole macro definition, then check the rest as if outside - // the macro definition. - check_nested_occurrences( - sess, - node_id, - &del.tts[rest..], - macros, - binders, - ops, - valid, - ); - } - ( - NestedMacroState::Macro, - &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), - ) => { - state = NestedMacroState::MacroName; - } - (NestedMacroState::Macro, &TokenTree::MetaVar(..)) => { - state = NestedMacroState::MacroName; - // We check that the meta-variable is correctly used. - check_occurrences(sess, node_id, tt, macros, binders, ops, valid); - } - (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) - if del.delim == DelimToken::Paren => - { - state = NestedMacroState::MacroNameParen; - nested_binders = Binders::default(); - check_binders( - sess, - node_id, - tt, - &nested_macros, - &mut nested_binders, - &Stack::Empty, - valid, - ); - } - (NestedMacroState::MacroNameParen, &TokenTree::Delimited(_, ref del)) - if del.delim == DelimToken::Brace => - { - state = NestedMacroState::Empty; - check_occurrences( - sess, - node_id, - tt, - &nested_macros, - &nested_binders, - &Stack::Empty, - valid, - ); - } - (_, ref tt) => { - state = NestedMacroState::Empty; - check_occurrences(sess, node_id, tt, macros, binders, ops, valid); - } - } - } -} - -/// Checks the body of nested macro, returns where the check stopped, and sets `valid` to false in -/// case of errors. -/// -/// The token trees are checked as long as they look like a list of (LHS) => {RHS} token trees. This -/// check is a best-effort to detect a macro definition. It returns the position in `tts` where we -/// stopped checking because we detected we were not in a macro definition anymore. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `legacy` specifies whether the macro is legacy -/// - `tts` is checked as a list of (LHS) => {RHS} -/// - `macros` is the stack of outer macros -/// - `valid` is set in case of errors -fn check_nested_macro( - sess: &ParseSess, - node_id: NodeId, - legacy: bool, - tts: &[TokenTree], - macros: &Stack<'_, MacroState<'_>>, - valid: &mut bool, -) -> usize { - let n = tts.len(); - let mut i = 0; - let separator = if legacy { TokenKind::Semi } else { TokenKind::Comma }; - loop { - // We expect 3 token trees: `(LHS) => {RHS}`. The separator is checked after. - if i + 2 >= n - || !tts[i].is_delimited() - || !tts[i + 1].is_token(&TokenKind::FatArrow) - || !tts[i + 2].is_delimited() - { - break; - } - let lhs = &tts[i]; - let rhs = &tts[i + 2]; - let mut binders = Binders::default(); - check_binders(sess, node_id, lhs, macros, &mut binders, &Stack::Empty, valid); - check_occurrences(sess, node_id, rhs, macros, &binders, &Stack::Empty, valid); - // Since the last semicolon is optional for legacy macros and decl_macro are not terminated, - // we increment our checked position by how many token trees we already checked (the 3 - // above) before checking for the separator. - i += 3; - if i == n || !tts[i].is_token(&separator) { - break; - } - // We increment our checked position for the semicolon. - i += 1; - } - i -} - -/// Checks that a meta-variable occurrence is valid. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `macros` is the stack of possible outer macros -/// - `binders` contains the binders of the associated LHS -/// - `ops` is the stack of Kleene operators from the RHS -/// - `span` is the span of the meta-variable to check -/// - `name` is the name of the meta-variable to check -fn check_ops_is_prefix( - sess: &ParseSess, - node_id: NodeId, - macros: &Stack<'_, MacroState<'_>>, - binders: &Binders, - ops: &Stack<'_, KleeneToken>, - span: Span, - name: Ident, -) { - let macros = macros.push(MacroState { binders, ops: ops.into() }); - // Accumulates the stacks the operators of each state until (and including when) the - // meta-variable is found. The innermost stack is first. - let mut acc: SmallVec<[&SmallVec<[KleeneToken; 1]>; 1]> = SmallVec::new(); - for state in ¯os { - acc.push(&state.ops); - if let Some(binder) = state.binders.get(&name) { - // This variable concatenates the stack of operators from the RHS of the LHS where the - // meta-variable was defined to where it is used (in possibly nested macros). The - // outermost operator is first. - let mut occurrence_ops: SmallVec<[KleeneToken; 2]> = SmallVec::new(); - // We need to iterate from the end to start with outermost stack. - for ops in acc.iter().rev() { - occurrence_ops.extend_from_slice(ops); - } - ops_is_prefix(sess, node_id, span, name, &binder.ops, &occurrence_ops); - return; - } - } - buffer_lint(sess, span.into(), node_id, &format!("unknown macro variable `{}`", name)); -} - -/// Returns whether `binder_ops` is a prefix of `occurrence_ops`. -/// -/// The stack of Kleene operators of a meta-variable occurrence just needs to have the stack of -/// Kleene operators of its binder as a prefix. -/// -/// Consider $i in the following example: -/// -/// ( $( $i:ident = $($j:ident),+ );* ) => { $($( $i += $j; )+)* } -/// -/// It occurs under the Kleene stack ["*", "+"] and is bound under ["*"] only. -/// -/// Arguments: -/// - `sess` is used to emit diagnostics and lints -/// - `node_id` is used to emit lints -/// - `span` is the span of the meta-variable being check -/// - `name` is the name of the meta-variable being check -/// - `binder_ops` is the stack of Kleene operators for the binder -/// - `occurrence_ops` is the stack of Kleene operators for the occurrence -fn ops_is_prefix( - sess: &ParseSess, - node_id: NodeId, - span: Span, - name: Ident, - binder_ops: &[KleeneToken], - occurrence_ops: &[KleeneToken], -) { - for (i, binder) in binder_ops.iter().enumerate() { - if i >= occurrence_ops.len() { - let mut span = MultiSpan::from_span(span); - span.push_span_label(binder.span, "expected repetition".into()); - let message = &format!("variable '{}' is still repeating at this depth", name); - buffer_lint(sess, span, node_id, message); - return; - } - let occurrence = &occurrence_ops[i]; - if occurrence.op != binder.op { - let mut span = MultiSpan::from_span(span); - span.push_span_label(binder.span, "expected repetition".into()); - span.push_span_label(occurrence.span, "conflicting repetition".into()); - let message = "meta-variable repeats with different Kleene operator"; - buffer_lint(sess, span, node_id, message); - return; - } - } -} - -fn buffer_lint(sess: &ParseSess, span: MultiSpan, node_id: NodeId, message: &str) { - sess.buffer_lint(&META_VARIABLE_MISUSE, span, node_id, message); -} diff --git a/src/libsyntax_expand/mbe/macro_parser.rs b/src/libsyntax_expand/mbe/macro_parser.rs deleted file mode 100644 index 24253e1bdc2..00000000000 --- a/src/libsyntax_expand/mbe/macro_parser.rs +++ /dev/null @@ -1,930 +0,0 @@ -//! This is an NFA-based parser, which calls out to the main rust parser for named non-terminals -//! (which it commits to fully when it hits one in a grammar). There's a set of current NFA threads -//! and a set of next ones. Instead of NTs, we have a special case for Kleene star. The big-O, in -//! pathological cases, is worse than traditional use of NFA or Earley parsing, but it's an easier -//! fit for Macro-by-Example-style rules. -//! -//! (In order to prevent the pathological case, we'd need to lazily construct the resulting -//! `NamedMatch`es at the very end. It'd be a pain, and require more memory to keep around old -//! items, but it would also save overhead) -//! -//! We don't say this parser uses the Earley algorithm, because it's unnecessarily inaccurate. -//! The macro parser restricts itself to the features of finite state automata. Earley parsers -//! can be described as an extension of NFAs with completion rules, prediction rules, and recursion. -//! -//! Quick intro to how the parser works: -//! -//! A 'position' is a dot in the middle of a matcher, usually represented as a -//! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`. -//! -//! The parser walks through the input a character at a time, maintaining a list -//! of threads consistent with the current position in the input string: `cur_items`. -//! -//! As it processes them, it fills up `eof_items` with threads that would be valid if -//! the macro invocation is now over, `bb_items` with threads that are waiting on -//! a Rust non-terminal like `$e:expr`, and `next_items` with threads that are waiting -//! on a particular token. Most of the logic concerns moving the · through the -//! repetitions indicated by Kleene stars. The rules for moving the · without -//! consuming any input are called epsilon transitions. It only advances or calls -//! out to the real Rust parser when no `cur_items` threads remain. -//! -//! Example: -//! -//! ```text, ignore -//! Start parsing a a a a b against [· a $( a )* a b]. -//! -//! Remaining input: a a a a b -//! next: [· a $( a )* a b] -//! -//! - - - Advance over an a. - - - -//! -//! Remaining input: a a a b -//! cur: [a · $( a )* a b] -//! Descend/Skip (first item). -//! next: [a $( · a )* a b] [a $( a )* · a b]. -//! -//! - - - Advance over an a. - - - -//! -//! Remaining input: a a b -//! cur: [a $( a · )* a b] [a $( a )* a · b] -//! Follow epsilon transition: Finish/Repeat (first item) -//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] -//! -//! - - - Advance over an a. - - - (this looks exactly like the last step) -//! -//! Remaining input: a b -//! cur: [a $( a · )* a b] [a $( a )* a · b] -//! Follow epsilon transition: Finish/Repeat (first item) -//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] -//! -//! - - - Advance over an a. - - - (this looks exactly like the last step) -//! -//! Remaining input: b -//! cur: [a $( a · )* a b] [a $( a )* a · b] -//! Follow epsilon transition: Finish/Repeat (first item) -//! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b] -//! -//! - - - Advance over a b. - - - -//! -//! Remaining input: '' -//! eof: [a $( a )* a b ·] -//! ``` - -crate use NamedMatch::*; -crate use ParseResult::*; -use TokenTreeOrTokenTreeSlice::*; - -use crate::mbe::{self, TokenTree}; - -use rustc_parse::parser::{FollowedByType, Parser, PathStyle}; -use rustc_parse::Directory; -use syntax::ast::{Ident, Name}; -use syntax::print::pprust; -use syntax::sess::ParseSess; -use syntax::symbol::{kw, sym, Symbol}; -use syntax::token::{self, DocComment, Nonterminal, Token}; -use syntax::tokenstream::TokenStream; - -use errors::{FatalError, PResult}; -use smallvec::{smallvec, SmallVec}; -use syntax_pos::Span; - -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use std::collections::hash_map::Entry::{Occupied, Vacant}; -use std::mem; -use std::ops::{Deref, DerefMut}; - -// To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body. - -/// Either a sequence of token trees or a single one. This is used as the representation of the -/// sequence of tokens that make up a matcher. -#[derive(Clone)] -enum TokenTreeOrTokenTreeSlice<'tt> { - Tt(TokenTree), - TtSeq(&'tt [TokenTree]), -} - -impl<'tt> TokenTreeOrTokenTreeSlice<'tt> { - /// Returns the number of constituent top-level token trees of `self` (top-level in that it - /// will not recursively descend into subtrees). - fn len(&self) -> usize { - match *self { - TtSeq(ref v) => v.len(), - Tt(ref tt) => tt.len(), - } - } - - /// The `index`-th token tree of `self`. - fn get_tt(&self, index: usize) -> TokenTree { - match *self { - TtSeq(ref v) => v[index].clone(), - Tt(ref tt) => tt.get_tt(index), - } - } -} - -/// An unzipping of `TokenTree`s... see the `stack` field of `MatcherPos`. -/// -/// This is used by `inner_parse_loop` to keep track of delimited submatchers that we have -/// descended into. -#[derive(Clone)] -struct MatcherTtFrame<'tt> { - /// The "parent" matcher that we are descending into. - elts: TokenTreeOrTokenTreeSlice<'tt>, - /// The position of the "dot" in `elts` at the time we descended. - idx: usize, -} - -type NamedMatchVec = SmallVec<[NamedMatch; 4]>; - -/// Represents a single "position" (aka "matcher position", aka "item"), as -/// described in the module documentation. -/// -/// Here: -/// -/// - `'root` represents the lifetime of the stack slot that holds the root -/// `MatcherPos`. As described in `MatcherPosHandle`, the root `MatcherPos` -/// structure is stored on the stack, but subsequent instances are put into -/// the heap. -/// - `'tt` represents the lifetime of the token trees that this matcher -/// position refers to. -/// -/// It is important to distinguish these two lifetimes because we have a -/// `SmallVec>` below, and the destructor of -/// that is considered to possibly access the data from its elements (it lacks -/// a `#[may_dangle]` attribute). As a result, the compiler needs to know that -/// all the elements in that `SmallVec` strictly outlive the root stack slot -/// lifetime. By separating `'tt` from `'root`, we can show that. -#[derive(Clone)] -struct MatcherPos<'root, 'tt> { - /// The token or sequence of tokens that make up the matcher - top_elts: TokenTreeOrTokenTreeSlice<'tt>, - - /// The position of the "dot" in this matcher - idx: usize, - - /// For each named metavar in the matcher, we keep track of token trees matched against the - /// metavar by the black box parser. In particular, there may be more than one match per - /// metavar if we are in a repetition (each repetition matches each of the variables). - /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the - /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of - /// the current position of the `self` matcher position in the shared `matches` list. - /// - /// Also, note that while we are descending into a sequence, matchers are given their own - /// `matches` vector. Only once we reach the end of a full repetition of the sequence do we add - /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep` - /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one - /// wants the shared `matches`, one should use `up.matches`. - matches: Box<[Lrc]>, - /// The position in `matches` corresponding to the first metavar in this matcher's sequence of - /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds - /// to `matches[match_lo]`. - match_lo: usize, - /// The position in `matches` corresponding to the metavar we are currently trying to match - /// against the source token stream. `match_lo <= match_cur <= match_hi`. - match_cur: usize, - /// Similar to `match_lo` except `match_hi` is the position in `matches` of the _last_ metavar - /// in this matcher. - match_hi: usize, - - // The following fields are used if we are matching a repetition. If we aren't, they should be - // `None`. - /// The KleeneOp of this sequence if we are in a repetition. - seq_op: Option, - - /// The separator if we are in a repetition. - sep: Option, - - /// The "parent" matcher position if we are in a repetition. That is, the matcher position just - /// before we enter the sequence. - up: Option>, - - /// Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from - /// a delimited token tree (e.g., something wrapped in `(` `)`) or to get the contents of a doc - /// comment... - /// - /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. ) - /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does - /// that where the bottom of the stack is the outermost matcher. - /// Also, throughout the comments, this "descent" is often referred to as "unzipping"... - stack: SmallVec<[MatcherTtFrame<'tt>; 1]>, -} - -impl<'root, 'tt> MatcherPos<'root, 'tt> { - /// Adds `m` as a named match for the `idx`-th metavar. - fn push_match(&mut self, idx: usize, m: NamedMatch) { - let matches = Lrc::make_mut(&mut self.matches[idx]); - matches.push(m); - } -} - -// Lots of MatcherPos instances are created at runtime. Allocating them on the -// heap is slow. Furthermore, using SmallVec to allocate them all -// on the stack is also slow, because MatcherPos is quite a large type and -// instances get moved around a lot between vectors, which requires lots of -// slow memcpy calls. -// -// Therefore, the initial MatcherPos is always allocated on the stack, -// subsequent ones (of which there aren't that many) are allocated on the heap, -// and this type is used to encapsulate both cases. -enum MatcherPosHandle<'root, 'tt> { - Ref(&'root mut MatcherPos<'root, 'tt>), - Box(Box>), -} - -impl<'root, 'tt> Clone for MatcherPosHandle<'root, 'tt> { - // This always produces a new Box. - fn clone(&self) -> Self { - MatcherPosHandle::Box(match *self { - MatcherPosHandle::Ref(ref r) => Box::new((**r).clone()), - MatcherPosHandle::Box(ref b) => b.clone(), - }) - } -} - -impl<'root, 'tt> Deref for MatcherPosHandle<'root, 'tt> { - type Target = MatcherPos<'root, 'tt>; - fn deref(&self) -> &Self::Target { - match *self { - MatcherPosHandle::Ref(ref r) => r, - MatcherPosHandle::Box(ref b) => b, - } - } -} - -impl<'root, 'tt> DerefMut for MatcherPosHandle<'root, 'tt> { - fn deref_mut(&mut self) -> &mut MatcherPos<'root, 'tt> { - match *self { - MatcherPosHandle::Ref(ref mut r) => r, - MatcherPosHandle::Box(ref mut b) => b, - } - } -} - -/// Represents the possible results of an attempted parse. -crate enum ParseResult { - /// Parsed successfully. - Success(T), - /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected - /// end of macro invocation. Otherwise, it indicates that no rules expected the given token. - Failure(Token, &'static str), - /// Fatal error (malformed macro?). Abort compilation. - Error(syntax_pos::Span, String), -} - -/// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es. -/// This represents the mapping of metavars to the token trees they bind to. -crate type NamedParseResult = ParseResult>; - -/// Count how many metavars are named in the given matcher `ms`. -pub(super) fn count_names(ms: &[TokenTree]) -> usize { - ms.iter().fold(0, |count, elt| { - count - + match *elt { - TokenTree::Sequence(_, ref seq) => seq.num_captures, - TokenTree::Delimited(_, ref delim) => count_names(&delim.tts), - TokenTree::MetaVar(..) => 0, - TokenTree::MetaVarDecl(..) => 1, - TokenTree::Token(..) => 0, - } - }) -} - -/// `len` `Vec`s (initially shared and empty) that will store matches of metavars. -fn create_matches(len: usize) -> Box<[Lrc]> { - if len == 0 { - vec![] - } else { - let empty_matches = Lrc::new(SmallVec::new()); - vec![empty_matches; len] - } - .into_boxed_slice() -} - -/// Generates the top-level matcher position in which the "dot" is before the first token of the -/// matcher `ms`. -fn initial_matcher_pos<'root, 'tt>(ms: &'tt [TokenTree]) -> MatcherPos<'root, 'tt> { - let match_idx_hi = count_names(ms); - let matches = create_matches(match_idx_hi); - MatcherPos { - // Start with the top level matcher given to us - top_elts: TtSeq(ms), // "elts" is an abbr. for "elements" - // The "dot" is before the first token of the matcher - idx: 0, - - // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`. - // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since - // we haven't actually matched anything yet. - matches, - match_lo: 0, - match_cur: 0, - match_hi: match_idx_hi, - - // Haven't descended into any delimiters, so empty stack - stack: smallvec![], - - // Haven't descended into any sequences, so both of these are `None`. - seq_op: None, - sep: None, - up: None, - } -} - -/// `NamedMatch` is a pattern-match result for a single `token::MATCH_NONTERMINAL`: -/// so it is associated with a single ident in a parse, and all -/// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type -/// (expr, item, etc). Each leaf in a single `NamedMatch` corresponds to a -/// single `token::MATCH_NONTERMINAL` in the `TokenTree` that produced it. -/// -/// The in-memory structure of a particular `NamedMatch` represents the match -/// that occurred when a particular subset of a matcher was applied to a -/// particular token tree. -/// -/// The width of each `MatchedSeq` in the `NamedMatch`, and the identity of -/// the `MatchedNonterminal`s, will depend on the token tree it was applied -/// to: each `MatchedSeq` corresponds to a single `TTSeq` in the originating -/// token tree. The depth of the `NamedMatch` structure will therefore depend -/// only on the nesting depth of `ast::TTSeq`s in the originating -/// token tree it was derived from. -#[derive(Debug, Clone)] -crate enum NamedMatch { - MatchedSeq(Lrc), - MatchedNonterminal(Lrc), -} - -/// Takes a sequence of token trees `ms` representing a matcher which successfully matched input -/// and an iterator of items that matched input and produces a `NamedParseResult`. -fn nameize>( - sess: &ParseSess, - ms: &[TokenTree], - mut res: I, -) -> NamedParseResult { - // Recursively descend into each type of matcher (e.g., sequences, delimited, metavars) and make - // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one - // binding, then there is an error. If it does, then we insert the binding into the - // `NamedParseResult`. - fn n_rec>( - sess: &ParseSess, - m: &TokenTree, - res: &mut I, - ret_val: &mut FxHashMap, - ) -> Result<(), (syntax_pos::Span, String)> { - match *m { - TokenTree::Sequence(_, ref seq) => { - for next_m in &seq.tts { - n_rec(sess, next_m, res.by_ref(), ret_val)? - } - } - TokenTree::Delimited(_, ref delim) => { - for next_m in &delim.tts { - n_rec(sess, next_m, res.by_ref(), ret_val)?; - } - } - TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { - if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { - return Err((span, "missing fragment specifier".to_string())); - } - } - TokenTree::MetaVarDecl(sp, bind_name, _) => match ret_val.entry(bind_name) { - Vacant(spot) => { - spot.insert(res.next().unwrap()); - } - Occupied(..) => return Err((sp, format!("duplicated bind name: {}", bind_name))), - }, - TokenTree::MetaVar(..) | TokenTree::Token(..) => (), - } - - Ok(()) - } - - let mut ret_val = FxHashMap::default(); - for m in ms { - match n_rec(sess, m, res.by_ref(), &mut ret_val) { - Ok(_) => {} - Err((sp, msg)) => return Error(sp, msg), - } - } - - Success(ret_val) -} - -/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) -fn token_name_eq(t1: &Token, t2: &Token) -> bool { - if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) { - ident1.name == ident2.name && is_raw1 == is_raw2 - } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) { - ident1.name == ident2.name - } else { - t1.kind == t2.kind - } -} - -/// Process the matcher positions of `cur_items` until it is empty. In the process, this will -/// produce more items in `next_items`, `eof_items`, and `bb_items`. -/// -/// For more info about the how this happens, see the module-level doc comments and the inline -/// comments of this function. -/// -/// # Parameters -/// -/// - `sess`: the parsing session into which errors are emitted. -/// - `cur_items`: the set of current items to be processed. This should be empty by the end of a -/// successful execution of this function. -/// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in -/// the function `parse`. -/// - `eof_items`: the set of items that would be valid if this was the EOF. -/// - `bb_items`: the set of items that are waiting for the black-box parser. -/// - `token`: the current token of the parser. -/// - `span`: the `Span` in the source code corresponding to the token trees we are trying to match -/// against the matcher positions in `cur_items`. -/// -/// # Returns -/// -/// A `ParseResult`. Note that matches are kept track of through the items generated. -fn inner_parse_loop<'root, 'tt>( - sess: &ParseSess, - cur_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - next_items: &mut Vec>, - eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - token: &Token, -) -> ParseResult<()> { - // Pop items from `cur_items` until it is empty. - while let Some(mut item) = cur_items.pop() { - // When unzipped trees end, remove them. This corresponds to backtracking out of a - // delimited submatcher into which we already descended. In backtracking out again, we need - // to advance the "dot" past the delimiters in the outer matcher. - while item.idx >= item.top_elts.len() { - match item.stack.pop() { - Some(MatcherTtFrame { elts, idx }) => { - item.top_elts = elts; - item.idx = idx + 1; - } - None => break, - } - } - - // Get the current position of the "dot" (`idx`) in `item` and the number of token trees in - // the matcher (`len`). - let idx = item.idx; - let len = item.top_elts.len(); - - // If `idx >= len`, then we are at or past the end of the matcher of `item`. - if idx >= len { - // We are repeating iff there is a parent. If the matcher is inside of a repetition, - // then we could be at the end of a sequence or at the beginning of the next - // repetition. - if item.up.is_some() { - // At this point, regardless of whether there is a separator, we should add all - // matches from the complete repetition of the sequence to the shared, top-level - // `matches` list (actually, `up.matches`, which could itself not be the top-level, - // but anyway...). Moreover, we add another item to `cur_items` in which the "dot" - // is at the end of the `up` matcher. This ensures that the "dot" in the `up` - // matcher is also advanced sufficiently. - // - // NOTE: removing the condition `idx == len` allows trailing separators. - if idx == len { - // Get the `up` matcher - let mut new_pos = item.up.clone().unwrap(); - - // Add matches from this repetition to the `matches` of `up` - for idx in item.match_lo..item.match_hi { - let sub = item.matches[idx].clone(); - new_pos.push_match(idx, MatchedSeq(sub)); - } - - // Move the "dot" past the repetition in `up` - new_pos.match_cur = item.match_hi; - new_pos.idx += 1; - cur_items.push(new_pos); - } - - // Check if we need a separator. - if idx == len && item.sep.is_some() { - // We have a separator, and it is the current token. We can advance past the - // separator token. - if item.sep.as_ref().map(|sep| token_name_eq(token, sep)).unwrap_or(false) { - item.idx += 1; - next_items.push(item); - } - } - // We don't need a separator. Move the "dot" back to the beginning of the matcher - // and try to match again UNLESS we are only allowed to have _one_ repetition. - else if item.seq_op != Some(mbe::KleeneOp::ZeroOrOne) { - item.match_cur = item.match_lo; - item.idx = 0; - cur_items.push(item); - } - } - // If we are not in a repetition, then being at the end of a matcher means that we have - // reached the potential end of the input. - else { - eof_items.push(item); - } - } - // We are in the middle of a matcher. - else { - // Look at what token in the matcher we are trying to match the current token (`token`) - // against. Depending on that, we may generate new items. - match item.top_elts.get_tt(idx) { - // Need to descend into a sequence - TokenTree::Sequence(sp, seq) => { - // Examine the case where there are 0 matches of this sequence. We are - // implicitly disallowing OneOrMore from having 0 matches here. Thus, that will - // result in a "no rules expected token" error by virtue of this matcher not - // working. - if seq.kleene.op == mbe::KleeneOp::ZeroOrMore - || seq.kleene.op == mbe::KleeneOp::ZeroOrOne - { - let mut new_item = item.clone(); - new_item.match_cur += seq.num_captures; - new_item.idx += 1; - for idx in item.match_cur..item.match_cur + seq.num_captures { - new_item.push_match(idx, MatchedSeq(Lrc::new(smallvec![]))); - } - cur_items.push(new_item); - } - - let matches = create_matches(item.matches.len()); - cur_items.push(MatcherPosHandle::Box(Box::new(MatcherPos { - stack: smallvec![], - sep: seq.separator.clone(), - seq_op: Some(seq.kleene.op), - idx: 0, - matches, - match_lo: item.match_cur, - match_cur: item.match_cur, - match_hi: item.match_cur + seq.num_captures, - up: Some(item), - top_elts: Tt(TokenTree::Sequence(sp, seq)), - }))); - } - - // We need to match a metavar (but the identifier is invalid)... this is an error - TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { - if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { - return Error(span, "missing fragment specifier".to_string()); - } - } - - // We need to match a metavar with a valid ident... call out to the black-box - // parser by adding an item to `bb_items`. - TokenTree::MetaVarDecl(_, _, id) => { - // Built-in nonterminals never start with these tokens, - // so we can eliminate them from consideration. - if may_begin_with(token, id.name) { - bb_items.push(item); - } - } - - // We need to descend into a delimited submatcher or a doc comment. To do this, we - // push the current matcher onto a stack and push a new item containing the - // submatcher onto `cur_items`. - // - // At the beginning of the loop, if we reach the end of the delimited submatcher, - // we pop the stack to backtrack out of the descent. - seq @ TokenTree::Delimited(..) - | seq @ TokenTree::Token(Token { kind: DocComment(..), .. }) => { - let lower_elts = mem::replace(&mut item.top_elts, Tt(seq)); - let idx = item.idx; - item.stack.push(MatcherTtFrame { elts: lower_elts, idx }); - item.idx = 0; - cur_items.push(item); - } - - // We just matched a normal token. We can just advance the parser. - TokenTree::Token(t) if token_name_eq(&t, token) => { - item.idx += 1; - next_items.push(item); - } - - // There was another token that was not `token`... This means we can't add any - // rules. NOTE that this is not necessarily an error unless _all_ items in - // `cur_items` end up doing this. There may still be some other matchers that do - // end up working out. - TokenTree::Token(..) | TokenTree::MetaVar(..) => {} - } - } - } - - // Yay a successful parse (so far)! - Success(()) -} - -/// Use the given sequence of token trees (`ms`) as a matcher. Match the given token stream `tts` -/// against it and return the match. -/// -/// # Parameters -/// -/// - `sess`: The session into which errors are emitted -/// - `tts`: The tokenstream we are matching against the pattern `ms` -/// - `ms`: A sequence of token trees representing a pattern against which we are matching -/// - `directory`: Information about the file locations (needed for the black-box parser) -/// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box -/// parser) -pub(super) fn parse( - sess: &ParseSess, - tts: TokenStream, - ms: &[TokenTree], - directory: Option>, - recurse_into_modules: bool, -) -> NamedParseResult { - // Create a parser that can be used for the "black box" parts. - let mut parser = - Parser::new(sess, tts, directory, recurse_into_modules, true, rustc_parse::MACRO_ARGUMENTS); - - // A queue of possible matcher positions. We initialize it with the matcher position in which - // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then - // processes all of these possible matcher positions and produces possible next positions into - // `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items` - // and we start over again. - // - // This MatcherPos instance is allocated on the stack. All others -- and - // there are frequently *no* others! -- are allocated on the heap. - let mut initial = initial_matcher_pos(ms); - let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)]; - let mut next_items = Vec::new(); - - loop { - // Matcher positions black-box parsed by parser.rs (`parser`) - let mut bb_items = SmallVec::new(); - - // Matcher positions that would be valid if the macro invocation was over now - let mut eof_items = SmallVec::new(); - assert!(next_items.is_empty()); - - // Process `cur_items` until either we have finished the input or we need to get some - // parsing from the black-box parser done. The result is that `next_items` will contain a - // bunch of possible next matcher positions in `next_items`. - match inner_parse_loop( - sess, - &mut cur_items, - &mut next_items, - &mut eof_items, - &mut bb_items, - &parser.token, - ) { - Success(_) => {} - Failure(token, msg) => return Failure(token, msg), - Error(sp, msg) => return Error(sp, msg), - } - - // inner parse loop handled all cur_items, so it's empty - assert!(cur_items.is_empty()); - - // We need to do some post processing after the `inner_parser_loop`. - // - // Error messages here could be improved with links to original rules. - - // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise, - // either the parse is ambiguous (which should never happen) or there is a syntax error. - if parser.token == token::Eof { - if eof_items.len() == 1 { - let matches = - eof_items[0].matches.iter_mut().map(|dv| Lrc::make_mut(dv).pop().unwrap()); - return nameize(sess, ms, matches); - } else if eof_items.len() > 1 { - return Error( - parser.token.span, - "ambiguity: multiple successful parses".to_string(), - ); - } else { - return Failure( - Token::new( - token::Eof, - if parser.token.span.is_dummy() { - parser.token.span - } else { - sess.source_map().next_point(parser.token.span) - }, - ), - "missing tokens in macro arguments", - ); - } - } - // Performance hack: eof_items may share matchers via Rc with other things that we want - // to modify. Dropping eof_items now may drop these refcounts to 1, preventing an - // unnecessary implicit clone later in Rc::make_mut. - drop(eof_items); - - // Another possibility is that we need to call out to parse some rust nonterminal - // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong. - if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 { - let nts = bb_items - .iter() - .map(|item| match item.top_elts.get_tt(item.idx) { - TokenTree::MetaVarDecl(_, bind, name) => format!("{} ('{}')", name, bind), - _ => panic!(), - }) - .collect::>() - .join(" or "); - - return Error( - parser.token.span, - format!( - "local ambiguity: multiple parsing options: {}", - match next_items.len() { - 0 => format!("built-in NTs {}.", nts), - 1 => format!("built-in NTs {} or 1 other option.", nts), - n => format!("built-in NTs {} or {} other options.", nts, n), - } - ), - ); - } - // If there are no possible next positions AND we aren't waiting for the black-box parser, - // then there is a syntax error. - else if bb_items.is_empty() && next_items.is_empty() { - return Failure(parser.token.take(), "no rules expected this token in macro call"); - } - // Dump all possible `next_items` into `cur_items` for the next iteration. - else if !next_items.is_empty() { - // Now process the next token - cur_items.extend(next_items.drain(..)); - parser.bump(); - } - // Finally, we have the case where we need to call the black-box parser to get some - // nonterminal. - else { - assert_eq!(bb_items.len(), 1); - - let mut item = bb_items.pop().unwrap(); - if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) { - let match_cur = item.match_cur; - item.push_match( - match_cur, - MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, ident.name))), - ); - item.idx += 1; - item.match_cur += 1; - } else { - unreachable!() - } - cur_items.push(item); - } - - assert!(!cur_items.is_empty()); - } -} - -/// The token is an identifier, but not `_`. -/// We prohibit passing `_` to macros expecting `ident` for now. -fn get_macro_name(token: &Token) -> Option<(Name, bool)> { - match token.kind { - token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)), - _ => None, - } -} - -/// Checks whether a non-terminal may begin with a particular token. -/// -/// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that -/// token. Be conservative (return true) if not sure. -fn may_begin_with(token: &Token, name: Name) -> bool { - /// Checks whether the non-terminal may contain a single (non-keyword) identifier. - fn may_be_ident(nt: &token::Nonterminal) -> bool { - match *nt { - token::NtItem(_) | token::NtBlock(_) | token::NtVis(_) => false, - _ => true, - } - } - - match name { - sym::expr => { - token.can_begin_expr() - // This exception is here for backwards compatibility. - && !token.is_keyword(kw::Let) - } - sym::ty => token.can_begin_type(), - sym::ident => get_macro_name(token).is_some(), - sym::literal => token.can_begin_literal_or_bool(), - sym::vis => match token.kind { - // The follow-set of :vis + "priv" keyword + interpolated - token::Comma | token::Ident(..) | token::Interpolated(_) => true, - _ => token.can_begin_type(), - }, - sym::block => match token.kind { - token::OpenDelim(token::Brace) => true, - token::Interpolated(ref nt) => match **nt { - token::NtItem(_) - | token::NtPat(_) - | token::NtTy(_) - | token::NtIdent(..) - | token::NtMeta(_) - | token::NtPath(_) - | token::NtVis(_) => false, // none of these may start with '{'. - _ => true, - }, - _ => false, - }, - sym::path | sym::meta => match token.kind { - token::ModSep | token::Ident(..) => true, - token::Interpolated(ref nt) => match **nt { - token::NtPath(_) | token::NtMeta(_) => true, - _ => may_be_ident(&nt), - }, - _ => false, - }, - sym::pat => match token.kind { - token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) - token::OpenDelim(token::Paren) | // tuple pattern - token::OpenDelim(token::Bracket) | // slice pattern - token::BinOp(token::And) | // reference - token::BinOp(token::Minus) | // negative literal - token::AndAnd | // double reference - token::Literal(..) | // literal - token::DotDot | // range pattern (future compat) - token::DotDotDot | // range pattern (future compat) - token::ModSep | // path - token::Lt | // path (UFCS constant) - token::BinOp(token::Shl) => true, // path (double UFCS) - token::Interpolated(ref nt) => may_be_ident(nt), - _ => false, - }, - sym::lifetime => match token.kind { - token::Lifetime(_) => true, - token::Interpolated(ref nt) => match **nt { - token::NtLifetime(_) | token::NtTT(_) => true, - _ => false, - }, - _ => false, - }, - _ => match token.kind { - token::CloseDelim(_) => false, - _ => true, - }, - } -} - -/// A call to the "black-box" parser to parse some Rust non-terminal. -/// -/// # Parameters -/// -/// - `p`: the "black-box" parser to use -/// - `sp`: the `Span` we want to parse -/// - `name`: the name of the metavar _matcher_ we want to match (e.g., `tt`, `ident`, `block`, -/// etc...) -/// -/// # Returns -/// -/// The parsed non-terminal. -fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal { - // FIXME(Centril): Consider moving this to `parser.rs` to make - // the visibilities of the methods used below `pub(super)` at most. - - if name == sym::tt { - return token::NtTT(p.parse_token_tree()); - } - // check at the beginning and the parser checks after each bump - p.process_potential_macro_variable(); - match parse_nt_inner(p, sp, name) { - Ok(nt) => nt, - Err(mut err) => { - err.emit(); - FatalError.raise(); - } - } -} - -fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a, Nonterminal> { - Ok(match name { - sym::item => match p.parse_item()? { - Some(i) => token::NtItem(i), - None => return Err(p.fatal("expected an item keyword")), - }, - sym::block => token::NtBlock(p.parse_block()?), - sym::stmt => match p.parse_stmt()? { - Some(s) => token::NtStmt(s), - None => return Err(p.fatal("expected a statement")), - }, - sym::pat => token::NtPat(p.parse_pat(None)?), - sym::expr => token::NtExpr(p.parse_expr()?), - sym::literal => token::NtLiteral(p.parse_literal_maybe_minus()?), - sym::ty => token::NtTy(p.parse_ty()?), - // this could be handled like a token, since it is one - sym::ident => { - if let Some((name, is_raw)) = get_macro_name(&p.token) { - let span = p.token.span; - p.bump(); - token::NtIdent(Ident::new(name, span), is_raw) - } else { - let token_str = pprust::token_to_string(&p.token); - return Err(p.fatal(&format!("expected ident, found {}", &token_str))); - } - } - sym::path => token::NtPath(p.parse_path(PathStyle::Type)?), - sym::meta => token::NtMeta(p.parse_attr_item()?), - sym::vis => token::NtVis(p.parse_visibility(FollowedByType::Yes)?), - sym::lifetime => { - if p.check_lifetime() { - token::NtLifetime(p.expect_lifetime().ident) - } else { - let token_str = pprust::token_to_string(&p.token); - return Err(p.fatal(&format!("expected a lifetime, found `{}`", &token_str))); - } - } - // this is not supposed to happen, since it has been checked - // when compiling the macro. - _ => p.span_bug(sp, "invalid fragment specifier"), - }) -} diff --git a/src/libsyntax_expand/mbe/macro_rules.rs b/src/libsyntax_expand/mbe/macro_rules.rs deleted file mode 100644 index 2b2ed8c9248..00000000000 --- a/src/libsyntax_expand/mbe/macro_rules.rs +++ /dev/null @@ -1,1207 +0,0 @@ -use crate::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander}; -use crate::base::{SyntaxExtension, SyntaxExtensionKind}; -use crate::expand::{ensure_complete_parse, parse_ast_fragment, AstFragment, AstFragmentKind}; -use crate::mbe; -use crate::mbe::macro_check; -use crate::mbe::macro_parser::parse; -use crate::mbe::macro_parser::{Error, Failure, Success}; -use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedParseResult}; -use crate::mbe::transcribe::transcribe; - -use rustc_feature::Features; -use rustc_parse::parser::Parser; -use rustc_parse::Directory; -use syntax::ast; -use syntax::attr::{self, TransparencyError}; -use syntax::edition::Edition; -use syntax::print::pprust; -use syntax::sess::ParseSess; -use syntax::symbol::{kw, sym, Symbol}; -use syntax::token::{self, NtTT, Token, TokenKind::*}; -use syntax::tokenstream::{DelimSpan, TokenStream}; -use syntax_pos::hygiene::Transparency; -use syntax_pos::Span; - -use errors::{DiagnosticBuilder, FatalError}; -use log::debug; - -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use std::borrow::Cow; -use std::collections::hash_map::Entry; -use std::{mem, slice}; - -use errors::Applicability; - -const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \ - `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, \ - `literal`, `path`, `meta`, `tt`, `item` and `vis`"; - -crate struct ParserAnyMacro<'a> { - parser: Parser<'a>, - - /// Span of the expansion site of the macro this parser is for - site_span: Span, - /// The ident of the macro we're parsing - macro_ident: ast::Ident, - arm_span: Span, -} - -crate fn annotate_err_with_kind( - err: &mut DiagnosticBuilder<'_>, - kind: AstFragmentKind, - span: Span, -) { - match kind { - AstFragmentKind::Ty => { - err.span_label(span, "this macro call doesn't expand to a type"); - } - AstFragmentKind::Pat => { - err.span_label(span, "this macro call doesn't expand to a pattern"); - } - _ => {} - }; -} - -/// Instead of e.g. `vec![a, b, c]` in a pattern context, suggest `[a, b, c]`. -fn suggest_slice_pat(e: &mut DiagnosticBuilder<'_>, site_span: Span, parser: &Parser<'_>) { - let mut suggestion = None; - if let Ok(code) = parser.sess.source_map().span_to_snippet(site_span) { - if let Some(bang) = code.find('!') { - suggestion = Some(code[bang + 1..].to_string()); - } - } - if let Some(suggestion) = suggestion { - e.span_suggestion( - site_span, - "use a slice pattern here instead", - suggestion, - Applicability::MachineApplicable, - ); - } else { - e.span_label(site_span, "use a slice pattern here instead"); - } - e.help( - "for more information, see https://doc.rust-lang.org/edition-guide/\ - rust-2018/slice-patterns.html", - ); -} - -impl<'a> ParserAnyMacro<'a> { - crate fn make(mut self: Box>, kind: AstFragmentKind) -> AstFragment { - let ParserAnyMacro { site_span, macro_ident, ref mut parser, arm_span } = *self; - let fragment = panictry!(parse_ast_fragment(parser, kind, true).map_err(|mut e| { - if parser.token == token::Eof && e.message().ends_with(", found ``") { - if !e.span.is_dummy() { - // early end of macro arm (#52866) - e.replace_span_with(parser.sess.source_map().next_point(parser.token.span)); - } - let msg = &e.message[0]; - e.message[0] = ( - format!( - "macro expansion ends with an incomplete expression: {}", - msg.0.replace(", found ``", ""), - ), - msg.1, - ); - } - if e.span.is_dummy() { - // Get around lack of span in error (#30128) - e.replace_span_with(site_span); - if parser.sess.source_map().span_to_filename(arm_span).is_real() { - e.span_label(arm_span, "in this macro arm"); - } - } else if !parser.sess.source_map().span_to_filename(parser.token.span).is_real() { - e.span_label(site_span, "in this macro invocation"); - } - match kind { - AstFragmentKind::Pat if macro_ident.name == sym::vec => { - suggest_slice_pat(&mut e, site_span, parser); - } - _ => annotate_err_with_kind(&mut e, kind, site_span), - }; - e - })); - - // We allow semicolons at the end of expressions -- e.g., the semicolon in - // `macro_rules! m { () => { panic!(); } }` isn't parsed by `.parse_expr()`, - // but `m!()` is allowed in expression positions (cf. issue #34706). - if kind == AstFragmentKind::Expr && parser.token == token::Semi { - parser.bump(); - } - - // Make sure we don't have any tokens left to parse so we don't silently drop anything. - let path = ast::Path::from_ident(macro_ident.with_span_pos(site_span)); - ensure_complete_parse(parser, &path, kind.name(), site_span); - fragment - } -} - -struct MacroRulesMacroExpander { - name: ast::Ident, - span: Span, - transparency: Transparency, - lhses: Vec, - rhses: Vec, - valid: bool, -} - -impl TTMacroExpander for MacroRulesMacroExpander { - fn expand<'cx>( - &self, - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - input: TokenStream, - ) -> Box { - if !self.valid { - return DummyResult::any(sp); - } - generic_extension( - cx, - sp, - self.span, - self.name, - self.transparency, - input, - &self.lhses, - &self.rhses, - ) - } -} - -fn trace_macros_note(cx: &mut ExtCtxt<'_>, sp: Span, message: String) { - let sp = sp.macro_backtrace().last().map(|trace| trace.call_site).unwrap_or(sp); - cx.expansions.entry(sp).or_default().push(message); -} - -/// Given `lhses` and `rhses`, this is the new macro we create -fn generic_extension<'cx>( - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - def_span: Span, - name: ast::Ident, - transparency: Transparency, - arg: TokenStream, - lhses: &[mbe::TokenTree], - rhses: &[mbe::TokenTree], -) -> Box { - if cx.trace_macros() { - let msg = format!("expanding `{}! {{ {} }}`", name, pprust::tts_to_string(arg.clone())); - trace_macros_note(cx, sp, msg); - } - - // Which arm's failure should we report? (the one furthest along) - let mut best_failure: Option<(Token, &str)> = None; - for (i, lhs) in lhses.iter().enumerate() { - // try each arm's matchers - let lhs_tt = match *lhs { - mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], - _ => cx.span_bug(sp, "malformed macro lhs"), - }; - - // Take a snapshot of the state of pre-expansion gating at this point. - // This is used so that if a matcher is not `Success(..)`ful, - // then the spans which became gated when parsing the unsuccessful matcher - // are not recorded. On the first `Success(..)`ful matcher, the spans are merged. - let mut gated_spans_snaphot = mem::take(&mut *cx.parse_sess.gated_spans.spans.borrow_mut()); - - match parse_tt(cx, lhs_tt, arg.clone()) { - Success(named_matches) => { - // The matcher was `Success(..)`ful. - // Merge the gated spans from parsing the matcher with the pre-existing ones. - cx.parse_sess.gated_spans.merge(gated_spans_snaphot); - - let rhs = match rhses[i] { - // ignore delimiters - mbe::TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(), - _ => cx.span_bug(sp, "malformed macro rhs"), - }; - let arm_span = rhses[i].span(); - - let rhs_spans = rhs.iter().map(|t| t.span()).collect::>(); - // rhs has holes ( `$id` and `$(...)` that need filled) - let mut tts = transcribe(cx, &named_matches, rhs, transparency); - - // Replace all the tokens for the corresponding positions in the macro, to maintain - // proper positions in error reporting, while maintaining the macro_backtrace. - if rhs_spans.len() == tts.len() { - tts = tts.map_enumerated(|i, mut tt| { - let mut sp = rhs_spans[i]; - sp = sp.with_ctxt(tt.span().ctxt()); - tt.set_span(sp); - tt - }); - } - - if cx.trace_macros() { - let msg = format!("to `{}`", pprust::tts_to_string(tts.clone())); - trace_macros_note(cx, sp, msg); - } - - let directory = Directory { - path: Cow::from(cx.current_expansion.module.directory.as_path()), - ownership: cx.current_expansion.directory_ownership, - }; - let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None); - p.root_module_name = - cx.current_expansion.module.mod_path.last().map(|id| id.to_string()); - p.last_type_ascription = cx.current_expansion.prior_type_ascription; - - p.process_potential_macro_variable(); - // Let the context choose how to interpret the result. - // Weird, but useful for X-macros. - return Box::new(ParserAnyMacro { - parser: p, - - // Pass along the original expansion site and the name of the macro - // so we can print a useful error message if the parse of the expanded - // macro leaves unparsed tokens. - site_span: sp, - macro_ident: name, - arm_span, - }); - } - Failure(token, msg) => match best_failure { - Some((ref best_token, _)) if best_token.span.lo() >= token.span.lo() => {} - _ => best_failure = Some((token, msg)), - }, - Error(err_sp, ref msg) => cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]), - } - - // The matcher was not `Success(..)`ful. - // Restore to the state before snapshotting and maybe try again. - mem::swap(&mut gated_spans_snaphot, &mut cx.parse_sess.gated_spans.spans.borrow_mut()); - } - - let (token, label) = best_failure.expect("ran no matchers"); - let span = token.span.substitute_dummy(sp); - let mut err = cx.struct_span_err(span, &parse_failure_msg(&token)); - err.span_label(span, label); - if !def_span.is_dummy() && cx.source_map().span_to_filename(def_span).is_real() { - err.span_label(cx.source_map().def_span(def_span), "when calling this macro"); - } - - // Check whether there's a missing comma in this macro call, like `println!("{}" a);` - if let Some((arg, comma_span)) = arg.add_comma() { - for lhs in lhses { - // try each arm's matchers - let lhs_tt = match *lhs { - mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], - _ => continue, - }; - match parse_tt(cx, lhs_tt, arg.clone()) { - Success(_) => { - if comma_span.is_dummy() { - err.note("you might be missing a comma"); - } else { - err.span_suggestion_short( - comma_span, - "missing comma here", - ", ".to_string(), - Applicability::MachineApplicable, - ); - } - } - _ => {} - } - } - } - err.emit(); - cx.trace_macros_diag(); - DummyResult::any(sp) -} - -// Note that macro-by-example's input is also matched against a token tree: -// $( $lhs:tt => $rhs:tt );+ -// -// Holy self-referential! - -/// Converts a macro item into a syntax extension. -pub fn compile_declarative_macro( - sess: &ParseSess, - features: &Features, - def: &ast::Item, - edition: Edition, -) -> SyntaxExtension { - let diag = &sess.span_diagnostic; - let lhs_nm = ast::Ident::new(sym::lhs, def.span); - let rhs_nm = ast::Ident::new(sym::rhs, def.span); - let tt_spec = ast::Ident::new(sym::tt, def.span); - - // Parse the macro_rules! invocation - let (is_legacy, body) = match &def.kind { - ast::ItemKind::MacroDef(macro_def) => (macro_def.legacy, macro_def.body.inner_tokens()), - _ => unreachable!(), - }; - - // The pattern that macro_rules matches. - // The grammar for macro_rules! is: - // $( $lhs:tt => $rhs:tt );+ - // ...quasiquoting this would be nice. - // These spans won't matter, anyways - let argument_gram = vec![ - mbe::TokenTree::Sequence( - DelimSpan::dummy(), - Lrc::new(mbe::SequenceRepetition { - tts: vec![ - mbe::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec), - mbe::TokenTree::token(token::FatArrow, def.span), - mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec), - ], - separator: Some(Token::new( - if is_legacy { token::Semi } else { token::Comma }, - def.span, - )), - kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, def.span), - num_captures: 2, - }), - ), - // to phase into semicolon-termination instead of semicolon-separation - mbe::TokenTree::Sequence( - DelimSpan::dummy(), - Lrc::new(mbe::SequenceRepetition { - tts: vec![mbe::TokenTree::token( - if is_legacy { token::Semi } else { token::Comma }, - def.span, - )], - separator: None, - kleene: mbe::KleeneToken::new(mbe::KleeneOp::ZeroOrMore, def.span), - num_captures: 0, - }), - ), - ]; - - let argument_map = match parse(sess, body, &argument_gram, None, true) { - Success(m) => m, - Failure(token, msg) => { - let s = parse_failure_msg(&token); - let sp = token.span.substitute_dummy(def.span); - let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s); - err.span_label(sp, msg); - err.emit(); - FatalError.raise(); - } - Error(sp, s) => { - sess.span_diagnostic.span_fatal(sp.substitute_dummy(def.span), &s).raise(); - } - }; - - let mut valid = true; - - // Extract the arguments: - let lhses = match argument_map[&lhs_nm] { - MatchedSeq(ref s) => s - .iter() - .map(|m| { - if let MatchedNonterminal(ref nt) = *m { - if let NtTT(ref tt) = **nt { - let tt = mbe::quoted::parse(tt.clone().into(), true, sess).pop().unwrap(); - valid &= check_lhs_nt_follows(sess, features, &def.attrs, &tt); - return tt; - } - } - sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") - }) - .collect::>(), - _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs"), - }; - - let rhses = match argument_map[&rhs_nm] { - MatchedSeq(ref s) => s - .iter() - .map(|m| { - if let MatchedNonterminal(ref nt) = *m { - if let NtTT(ref tt) = **nt { - return mbe::quoted::parse(tt.clone().into(), false, sess).pop().unwrap(); - } - } - sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") - }) - .collect::>(), - _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs"), - }; - - for rhs in &rhses { - valid &= check_rhs(sess, rhs); - } - - // don't abort iteration early, so that errors for multiple lhses can be reported - for lhs in &lhses { - valid &= check_lhs_no_empty_seq(sess, slice::from_ref(lhs)); - } - - // We use CRATE_NODE_ID instead of `def.id` otherwise we may emit buffered lints for a node id - // that is not lint-checked and trigger the "failed to process buffered lint here" bug. - valid &= macro_check::check_meta_variables(sess, ast::CRATE_NODE_ID, def.span, &lhses, &rhses); - - let (transparency, transparency_error) = attr::find_transparency(&def.attrs, is_legacy); - match transparency_error { - Some(TransparencyError::UnknownTransparency(value, span)) => { - diag.span_err(span, &format!("unknown macro transparency: `{}`", value)) - } - Some(TransparencyError::MultipleTransparencyAttrs(old_span, new_span)) => { - diag.span_err(vec![old_span, new_span], "multiple macro transparency attributes") - } - None => {} - } - - let expander: Box<_> = Box::new(MacroRulesMacroExpander { - name: def.ident, - span: def.span, - transparency, - lhses, - rhses, - valid, - }); - - SyntaxExtension::new( - sess, - SyntaxExtensionKind::LegacyBang(expander), - def.span, - Vec::new(), - edition, - def.ident.name, - &def.attrs, - ) -} - -fn check_lhs_nt_follows( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - lhs: &mbe::TokenTree, -) -> bool { - // lhs is going to be like TokenTree::Delimited(...), where the - // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens. - if let mbe::TokenTree::Delimited(_, ref tts) = *lhs { - check_matcher(sess, features, attrs, &tts.tts) - } else { - let msg = "invalid macro matcher; matchers must be contained in balanced delimiters"; - sess.span_diagnostic.span_err(lhs.span(), msg); - false - } - // we don't abort on errors on rejection, the driver will do that for us - // after parsing/expansion. we can report every error in every macro this way. -} - -/// Checks that the lhs contains no repetition which could match an empty token -/// tree, because then the matcher would hang indefinitely. -fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool { - use mbe::TokenTree; - for tt in tts { - match *tt { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (), - TokenTree::Delimited(_, ref del) => { - if !check_lhs_no_empty_seq(sess, &del.tts) { - return false; - } - } - TokenTree::Sequence(span, ref seq) => { - if seq.separator.is_none() - && seq.tts.iter().all(|seq_tt| match *seq_tt { - TokenTree::MetaVarDecl(_, _, id) => id.name == sym::vis, - TokenTree::Sequence(_, ref sub_seq) => { - sub_seq.kleene.op == mbe::KleeneOp::ZeroOrMore - || sub_seq.kleene.op == mbe::KleeneOp::ZeroOrOne - } - _ => false, - }) - { - let sp = span.entire(); - sess.span_diagnostic.span_err(sp, "repetition matches empty token tree"); - return false; - } - if !check_lhs_no_empty_seq(sess, &seq.tts) { - return false; - } - } - } - } - - true -} - -fn check_rhs(sess: &ParseSess, rhs: &mbe::TokenTree) -> bool { - match *rhs { - mbe::TokenTree::Delimited(..) => return true, - _ => sess.span_diagnostic.span_err(rhs.span(), "macro rhs must be delimited"), - } - false -} - -fn check_matcher( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - matcher: &[mbe::TokenTree], -) -> bool { - let first_sets = FirstSets::new(matcher); - let empty_suffix = TokenSet::empty(); - let err = sess.span_diagnostic.err_count(); - check_matcher_core(sess, features, attrs, &first_sets, matcher, &empty_suffix); - err == sess.span_diagnostic.err_count() -} - -// `The FirstSets` for a matcher is a mapping from subsequences in the -// matcher to the FIRST set for that subsequence. -// -// This mapping is partially precomputed via a backwards scan over the -// token trees of the matcher, which provides a mapping from each -// repetition sequence to its *first* set. -// -// (Hypothetically, sequences should be uniquely identifiable via their -// spans, though perhaps that is false, e.g., for macro-generated macros -// that do not try to inject artificial span information. My plan is -// to try to catch such cases ahead of time and not include them in -// the precomputed mapping.) -struct FirstSets { - // this maps each TokenTree::Sequence `$(tt ...) SEP OP` that is uniquely identified by its - // span in the original matcher to the First set for the inner sequence `tt ...`. - // - // If two sequences have the same span in a matcher, then map that - // span to None (invalidating the mapping here and forcing the code to - // use a slow path). - first: FxHashMap>, -} - -impl FirstSets { - fn new(tts: &[mbe::TokenTree]) -> FirstSets { - use mbe::TokenTree; - - let mut sets = FirstSets { first: FxHashMap::default() }; - build_recur(&mut sets, tts); - return sets; - - // walks backward over `tts`, returning the FIRST for `tts` - // and updating `sets` at the same time for all sequence - // substructure we find within `tts`. - fn build_recur(sets: &mut FirstSets, tts: &[TokenTree]) -> TokenSet { - let mut first = TokenSet::empty(); - for tt in tts.iter().rev() { - match *tt { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { - first.replace_with(tt.clone()); - } - TokenTree::Delimited(span, ref delimited) => { - build_recur(sets, &delimited.tts[..]); - first.replace_with(delimited.open_tt(span)); - } - TokenTree::Sequence(sp, ref seq_rep) => { - let subfirst = build_recur(sets, &seq_rep.tts[..]); - - match sets.first.entry(sp.entire()) { - Entry::Vacant(vac) => { - vac.insert(Some(subfirst.clone())); - } - Entry::Occupied(mut occ) => { - // if there is already an entry, then a span must have collided. - // This should not happen with typical macro_rules macros, - // but syntax extensions need not maintain distinct spans, - // so distinct syntax trees can be assigned the same span. - // In such a case, the map cannot be trusted; so mark this - // entry as unusable. - occ.insert(None); - } - } - - // If the sequence contents can be empty, then the first - // token could be the separator token itself. - - if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { - first.add_one_maybe(TokenTree::Token(sep.clone())); - } - - // Reverse scan: Sequence comes before `first`. - if subfirst.maybe_empty - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne - { - // If sequence is potentially empty, then - // union them (preserving first emptiness). - first.add_all(&TokenSet { maybe_empty: true, ..subfirst }); - } else { - // Otherwise, sequence guaranteed - // non-empty; replace first. - first = subfirst; - } - } - } - } - - first - } - } - - // walks forward over `tts` until all potential FIRST tokens are - // identified. - fn first(&self, tts: &[mbe::TokenTree]) -> TokenSet { - use mbe::TokenTree; - - let mut first = TokenSet::empty(); - for tt in tts.iter() { - assert!(first.maybe_empty); - match *tt { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { - first.add_one(tt.clone()); - return first; - } - TokenTree::Delimited(span, ref delimited) => { - first.add_one(delimited.open_tt(span)); - return first; - } - TokenTree::Sequence(sp, ref seq_rep) => { - let subfirst_owned; - let subfirst = match self.first.get(&sp.entire()) { - Some(&Some(ref subfirst)) => subfirst, - Some(&None) => { - subfirst_owned = self.first(&seq_rep.tts[..]); - &subfirst_owned - } - None => { - panic!("We missed a sequence during FirstSets construction"); - } - }; - - // If the sequence contents can be empty, then the first - // token could be the separator token itself. - if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { - first.add_one_maybe(TokenTree::Token(sep.clone())); - } - - assert!(first.maybe_empty); - first.add_all(subfirst); - if subfirst.maybe_empty - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore - || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne - { - // Continue scanning for more first - // tokens, but also make sure we - // restore empty-tracking state. - first.maybe_empty = true; - continue; - } else { - return first; - } - } - } - } - - // we only exit the loop if `tts` was empty or if every - // element of `tts` matches the empty sequence. - assert!(first.maybe_empty); - first - } -} - -// A set of `mbe::TokenTree`s, which may include `TokenTree::Match`s -// (for macro-by-example syntactic variables). It also carries the -// `maybe_empty` flag; that is true if and only if the matcher can -// match an empty token sequence. -// -// The First set is computed on submatchers like `$($a:expr b),* $(c)* d`, -// which has corresponding FIRST = {$a:expr, c, d}. -// Likewise, `$($a:expr b),* $(c)+ d` has FIRST = {$a:expr, c}. -// -// (Notably, we must allow for *-op to occur zero times.) -#[derive(Clone, Debug)] -struct TokenSet { - tokens: Vec, - maybe_empty: bool, -} - -impl TokenSet { - // Returns a set for the empty sequence. - fn empty() -> Self { - TokenSet { tokens: Vec::new(), maybe_empty: true } - } - - // Returns the set `{ tok }` for the single-token (and thus - // non-empty) sequence [tok]. - fn singleton(tok: mbe::TokenTree) -> Self { - TokenSet { tokens: vec![tok], maybe_empty: false } - } - - // Changes self to be the set `{ tok }`. - // Since `tok` is always present, marks self as non-empty. - fn replace_with(&mut self, tok: mbe::TokenTree) { - self.tokens.clear(); - self.tokens.push(tok); - self.maybe_empty = false; - } - - // Changes self to be the empty set `{}`; meant for use when - // the particular token does not matter, but we want to - // record that it occurs. - fn replace_with_irrelevant(&mut self) { - self.tokens.clear(); - self.maybe_empty = false; - } - - // Adds `tok` to the set for `self`, marking sequence as non-empy. - fn add_one(&mut self, tok: mbe::TokenTree) { - if !self.tokens.contains(&tok) { - self.tokens.push(tok); - } - self.maybe_empty = false; - } - - // Adds `tok` to the set for `self`. (Leaves `maybe_empty` flag alone.) - fn add_one_maybe(&mut self, tok: mbe::TokenTree) { - if !self.tokens.contains(&tok) { - self.tokens.push(tok); - } - } - - // Adds all elements of `other` to this. - // - // (Since this is a set, we filter out duplicates.) - // - // If `other` is potentially empty, then preserves the previous - // setting of the empty flag of `self`. If `other` is guaranteed - // non-empty, then `self` is marked non-empty. - fn add_all(&mut self, other: &Self) { - for tok in &other.tokens { - if !self.tokens.contains(tok) { - self.tokens.push(tok.clone()); - } - } - if !other.maybe_empty { - self.maybe_empty = false; - } - } -} - -// Checks that `matcher` is internally consistent and that it -// can legally be followed by a token `N`, for all `N` in `follow`. -// (If `follow` is empty, then it imposes no constraint on -// the `matcher`.) -// -// Returns the set of NT tokens that could possibly come last in -// `matcher`. (If `matcher` matches the empty sequence, then -// `maybe_empty` will be set to true.) -// -// Requires that `first_sets` is pre-computed for `matcher`; -// see `FirstSets::new`. -fn check_matcher_core( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - first_sets: &FirstSets, - matcher: &[mbe::TokenTree], - follow: &TokenSet, -) -> TokenSet { - use mbe::TokenTree; - - let mut last = TokenSet::empty(); - - // 2. For each token and suffix [T, SUFFIX] in M: - // ensure that T can be followed by SUFFIX, and if SUFFIX may be empty, - // then ensure T can also be followed by any element of FOLLOW. - 'each_token: for i in 0..matcher.len() { - let token = &matcher[i]; - let suffix = &matcher[i + 1..]; - - let build_suffix_first = || { - let mut s = first_sets.first(suffix); - if s.maybe_empty { - s.add_all(follow); - } - s - }; - - // (we build `suffix_first` on demand below; you can tell - // which cases are supposed to fall through by looking for the - // initialization of this variable.) - let suffix_first; - - // First, update `last` so that it corresponds to the set - // of NT tokens that might end the sequence `... token`. - match *token { - TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => { - let can_be_followed_by_any; - if let Err(bad_frag) = has_legal_fragment_specifier(sess, features, attrs, token) { - let msg = format!("invalid fragment specifier `{}`", bad_frag); - sess.span_diagnostic - .struct_span_err(token.span(), &msg) - .help(VALID_FRAGMENT_NAMES_MSG) - .emit(); - // (This eliminates false positives and duplicates - // from error messages.) - can_be_followed_by_any = true; - } else { - can_be_followed_by_any = token_can_be_followed_by_any(token); - } - - if can_be_followed_by_any { - // don't need to track tokens that work with any, - last.replace_with_irrelevant(); - // ... and don't need to check tokens that can be - // followed by anything against SUFFIX. - continue 'each_token; - } else { - last.replace_with(token.clone()); - suffix_first = build_suffix_first(); - } - } - TokenTree::Delimited(span, ref d) => { - let my_suffix = TokenSet::singleton(d.close_tt(span)); - check_matcher_core(sess, features, attrs, first_sets, &d.tts, &my_suffix); - // don't track non NT tokens - last.replace_with_irrelevant(); - - // also, we don't need to check delimited sequences - // against SUFFIX - continue 'each_token; - } - TokenTree::Sequence(_, ref seq_rep) => { - suffix_first = build_suffix_first(); - // The trick here: when we check the interior, we want - // to include the separator (if any) as a potential - // (but not guaranteed) element of FOLLOW. So in that - // case, we make a temp copy of suffix and stuff - // delimiter in there. - // - // FIXME: Should I first scan suffix_first to see if - // delimiter is already in it before I go through the - // work of cloning it? But then again, this way I may - // get a "tighter" span? - let mut new; - let my_suffix = if let Some(sep) = &seq_rep.separator { - new = suffix_first.clone(); - new.add_one_maybe(TokenTree::Token(sep.clone())); - &new - } else { - &suffix_first - }; - - // At this point, `suffix_first` is built, and - // `my_suffix` is some TokenSet that we can use - // for checking the interior of `seq_rep`. - let next = - check_matcher_core(sess, features, attrs, first_sets, &seq_rep.tts, my_suffix); - if next.maybe_empty { - last.add_all(&next); - } else { - last = next; - } - - // the recursive call to check_matcher_core already ran the 'each_last - // check below, so we can just keep going forward here. - continue 'each_token; - } - } - - // (`suffix_first` guaranteed initialized once reaching here.) - - // Now `last` holds the complete set of NT tokens that could - // end the sequence before SUFFIX. Check that every one works with `suffix`. - 'each_last: for token in &last.tokens { - if let TokenTree::MetaVarDecl(_, name, frag_spec) = *token { - for next_token in &suffix_first.tokens { - match is_in_follow(next_token, frag_spec.name) { - IsInFollow::Invalid(msg, help) => { - sess.span_diagnostic - .struct_span_err(next_token.span(), &msg) - .help(help) - .emit(); - // don't bother reporting every source of - // conflict for a particular element of `last`. - continue 'each_last; - } - IsInFollow::Yes => {} - IsInFollow::No(possible) => { - let may_be = if last.tokens.len() == 1 && suffix_first.tokens.len() == 1 - { - "is" - } else { - "may be" - }; - - let sp = next_token.span(); - let mut err = sess.span_diagnostic.struct_span_err( - sp, - &format!( - "`${name}:{frag}` {may_be} followed by `{next}`, which \ - is not allowed for `{frag}` fragments", - name = name, - frag = frag_spec, - next = quoted_tt_to_string(next_token), - may_be = may_be - ), - ); - err.span_label( - sp, - format!("not allowed after `{}` fragments", frag_spec), - ); - let msg = "allowed there are: "; - match possible { - &[] => {} - &[t] => { - err.note(&format!( - "only {} is allowed after `{}` fragments", - t, frag_spec, - )); - } - ts => { - err.note(&format!( - "{}{} or {}", - msg, - ts[..ts.len() - 1] - .iter() - .map(|s| *s) - .collect::>() - .join(", "), - ts[ts.len() - 1], - )); - } - } - err.emit(); - } - } - } - } - } - } - last -} - -fn token_can_be_followed_by_any(tok: &mbe::TokenTree) -> bool { - if let mbe::TokenTree::MetaVarDecl(_, _, frag_spec) = *tok { - frag_can_be_followed_by_any(frag_spec.name) - } else { - // (Non NT's can always be followed by anthing in matchers.) - true - } -} - -/// Returns `true` if a fragment of type `frag` can be followed by any sort of -/// token. We use this (among other things) as a useful approximation -/// for when `frag` can be followed by a repetition like `$(...)*` or -/// `$(...)+`. In general, these can be a bit tricky to reason about, -/// so we adopt a conservative position that says that any fragment -/// specifier which consumes at most one token tree can be followed by -/// a fragment specifier (indeed, these fragments can be followed by -/// ANYTHING without fear of future compatibility hazards). -fn frag_can_be_followed_by_any(frag: Symbol) -> bool { - match frag { - sym::item | // always terminated by `}` or `;` - sym::block | // exactly one token tree - sym::ident | // exactly one token tree - sym::literal | // exactly one token tree - sym::meta | // exactly one token tree - sym::lifetime | // exactly one token tree - sym::tt => // exactly one token tree - true, - - _ => - false, - } -} - -enum IsInFollow { - Yes, - No(&'static [&'static str]), - Invalid(String, &'static str), -} - -/// Returns `true` if `frag` can legally be followed by the token `tok`. For -/// fragments that can consume an unbounded number of tokens, `tok` -/// must be within a well-defined follow set. This is intended to -/// guarantee future compatibility: for example, without this rule, if -/// we expanded `expr` to include a new binary operator, we might -/// break macros that were relying on that binary operator as a -/// separator. -// when changing this do not forget to update doc/book/macros.md! -fn is_in_follow(tok: &mbe::TokenTree, frag: Symbol) -> IsInFollow { - use mbe::TokenTree; - - if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok { - // closing a token tree can never be matched by any fragment; - // iow, we always require that `(` and `)` match, etc. - IsInFollow::Yes - } else { - match frag { - sym::item => { - // since items *must* be followed by either a `;` or a `}`, we can - // accept anything after them - IsInFollow::Yes - } - sym::block => { - // anything can follow block, the braces provide an easy boundary to - // maintain - IsInFollow::Yes - } - sym::stmt | sym::expr => { - const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"]; - match tok { - TokenTree::Token(token) => match token.kind { - FatArrow | Comma | Semi => IsInFollow::Yes, - _ => IsInFollow::No(TOKENS), - }, - _ => IsInFollow::No(TOKENS), - } - } - sym::pat => { - const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"]; - match tok { - TokenTree::Token(token) => match token.kind { - FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes, - Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes, - _ => IsInFollow::No(TOKENS), - }, - _ => IsInFollow::No(TOKENS), - } - } - sym::path | sym::ty => { - const TOKENS: &[&str] = &[ - "`{`", "`[`", "`=>`", "`,`", "`>`", "`=`", "`:`", "`;`", "`|`", "`as`", - "`where`", - ]; - match tok { - TokenTree::Token(token) => match token.kind { - OpenDelim(token::DelimToken::Brace) - | OpenDelim(token::DelimToken::Bracket) - | Comma - | FatArrow - | Colon - | Eq - | Gt - | BinOp(token::Shr) - | Semi - | BinOp(token::Or) => IsInFollow::Yes, - Ident(name, false) if name == kw::As || name == kw::Where => { - IsInFollow::Yes - } - _ => IsInFollow::No(TOKENS), - }, - TokenTree::MetaVarDecl(_, _, frag) if frag.name == sym::block => { - IsInFollow::Yes - } - _ => IsInFollow::No(TOKENS), - } - } - sym::ident | sym::lifetime => { - // being a single token, idents and lifetimes are harmless - IsInFollow::Yes - } - sym::literal => { - // literals may be of a single token, or two tokens (negative numbers) - IsInFollow::Yes - } - sym::meta | sym::tt => { - // being either a single token or a delimited sequence, tt is - // harmless - IsInFollow::Yes - } - sym::vis => { - // Explicitly disallow `priv`, on the off chance it comes back. - const TOKENS: &[&str] = &["`,`", "an ident", "a type"]; - match tok { - TokenTree::Token(token) => match token.kind { - Comma => IsInFollow::Yes, - Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes, - _ => { - if token.can_begin_type() { - IsInFollow::Yes - } else { - IsInFollow::No(TOKENS) - } - } - }, - TokenTree::MetaVarDecl(_, _, frag) - if frag.name == sym::ident - || frag.name == sym::ty - || frag.name == sym::path => - { - IsInFollow::Yes - } - _ => IsInFollow::No(TOKENS), - } - } - kw::Invalid => IsInFollow::Yes, - _ => IsInFollow::Invalid( - format!("invalid fragment specifier `{}`", frag), - VALID_FRAGMENT_NAMES_MSG, - ), - } - } -} - -fn has_legal_fragment_specifier( - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - tok: &mbe::TokenTree, -) -> Result<(), String> { - debug!("has_legal_fragment_specifier({:?})", tok); - if let mbe::TokenTree::MetaVarDecl(_, _, ref frag_spec) = *tok { - let frag_span = tok.span(); - if !is_legal_fragment_specifier(sess, features, attrs, frag_spec.name, frag_span) { - return Err(frag_spec.to_string()); - } - } - Ok(()) -} - -fn is_legal_fragment_specifier( - _sess: &ParseSess, - _features: &Features, - _attrs: &[ast::Attribute], - frag_name: Symbol, - _frag_span: Span, -) -> bool { - /* - * If new fragment specifiers are invented in nightly, `_sess`, - * `_features`, `_attrs`, and `_frag_span` will be useful here - * for checking against feature gates. See past versions of - * this function. - */ - match frag_name { - sym::item - | sym::block - | sym::stmt - | sym::expr - | sym::pat - | sym::lifetime - | sym::path - | sym::ty - | sym::ident - | sym::meta - | sym::tt - | sym::vis - | sym::literal - | kw::Invalid => true, - _ => false, - } -} - -fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String { - match *tt { - mbe::TokenTree::Token(ref token) => pprust::token_to_string(&token), - mbe::TokenTree::MetaVar(_, name) => format!("${}", name), - mbe::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind), - _ => panic!( - "unexpected mbe::TokenTree::{{Sequence or Delimited}} \ - in follow set checker" - ), - } -} - -/// Use this token tree as a matcher to parse given tts. -fn parse_tt(cx: &ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream) -> NamedParseResult { - // `None` is because we're not interpolating - let directory = Directory { - path: Cow::from(cx.current_expansion.module.directory.as_path()), - ownership: cx.current_expansion.directory_ownership, - }; - parse(cx.parse_sess(), tts, mtch, Some(directory), true) -} - -/// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For -/// other tokens, this is "unexpected token...". -fn parse_failure_msg(tok: &Token) -> String { - match tok.kind { - token::Eof => "unexpected end of macro invocation".to_string(), - _ => format!("no rules expected the token `{}`", pprust::token_to_string(tok),), - } -} diff --git a/src/libsyntax_expand/mbe/quoted.rs b/src/libsyntax_expand/mbe/quoted.rs deleted file mode 100644 index 56b97cbb7c6..00000000000 --- a/src/libsyntax_expand/mbe/quoted.rs +++ /dev/null @@ -1,248 +0,0 @@ -use crate::mbe::macro_parser; -use crate::mbe::{Delimited, KleeneOp, KleeneToken, SequenceRepetition, TokenTree}; - -use syntax::ast; -use syntax::print::pprust; -use syntax::sess::ParseSess; -use syntax::symbol::kw; -use syntax::token::{self, Token}; -use syntax::tokenstream; - -use syntax_pos::Span; - -use rustc_data_structures::sync::Lrc; - -/// Takes a `tokenstream::TokenStream` and returns a `Vec`. Specifically, this -/// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a -/// collection of `TokenTree` for use in parsing a macro. -/// -/// # Parameters -/// -/// - `input`: a token stream to read from, the contents of which we are parsing. -/// - `expect_matchers`: `parse` can be used to parse either the "patterns" or the "body" of a -/// macro. Both take roughly the same form _except_ that in a pattern, metavars are declared with -/// their "matcher" type. For example `$var:expr` or `$id:ident`. In this example, `expr` and -/// `ident` are "matchers". They are not present in the body of a macro rule -- just in the -/// pattern, so we pass a parameter to indicate whether to expect them or not. -/// - `sess`: the parsing session. Any errors will be emitted to this session. -/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use -/// unstable features or not. -/// - `edition`: which edition are we in. -/// - `macro_node_id`: the NodeId of the macro we are parsing. -/// -/// # Returns -/// -/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`. -pub(super) fn parse( - input: tokenstream::TokenStream, - expect_matchers: bool, - sess: &ParseSess, -) -> Vec { - // Will contain the final collection of `self::TokenTree` - let mut result = Vec::new(); - - // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming - // additional trees if need be. - let mut trees = input.trees(); - while let Some(tree) = trees.next() { - // Given the parsed tree, if there is a metavar and we are expecting matchers, actually - // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). - let tree = parse_tree(tree, &mut trees, expect_matchers, sess); - match tree { - TokenTree::MetaVar(start_sp, ident) if expect_matchers => { - let span = match trees.next() { - Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { - match trees.next() { - Some(tokenstream::TokenTree::Token(token)) => match token.ident() { - Some((kind, _)) => { - let span = token.span.with_lo(start_sp.lo()); - result.push(TokenTree::MetaVarDecl(span, ident, kind)); - continue; - } - _ => token.span, - }, - tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), - } - } - tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp), - }; - sess.missing_fragment_specifiers.borrow_mut().insert(span); - result.push(TokenTree::MetaVarDecl(span, ident, ast::Ident::invalid())); - } - - // Not a metavar or no matchers allowed, so just return the tree - _ => result.push(tree), - } - } - result -} - -/// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a -/// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree` -/// for use in parsing a macro. -/// -/// Converting the given tree may involve reading more tokens. -/// -/// # Parameters -/// -/// - `tree`: the tree we wish to convert. -/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish -/// converting `tree` -/// - `expect_matchers`: same as for `parse` (see above). -/// - `sess`: the parsing session. Any errors will be emitted to this session. -/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use -/// unstable features or not. -fn parse_tree( - tree: tokenstream::TokenTree, - trees: &mut impl Iterator, - expect_matchers: bool, - sess: &ParseSess, -) -> TokenTree { - // Depending on what `tree` is, we could be parsing different parts of a macro - match tree { - // `tree` is a `$` token. Look at the next token in `trees` - tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() { - // `tree` is followed by a delimited set of token trees. This indicates the beginning - // of a repetition sequence in the macro (e.g. `$(pat)*`). - Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { - // Must have `(` not `{` or `[` - if delim != token::Paren { - let tok = pprust::token_kind_to_string(&token::OpenDelim(delim)); - let msg = format!("expected `(`, found `{}`", tok); - sess.span_diagnostic.span_err(span.entire(), &msg); - } - // Parse the contents of the sequence itself - let sequence = parse(tts.into(), expect_matchers, sess); - // Get the Kleene operator and optional separator - let (separator, kleene) = parse_sep_and_kleene_op(trees, span.entire(), sess); - // Count the number of captured "names" (i.e., named metavars) - let name_captures = macro_parser::count_names(&sequence); - TokenTree::Sequence( - span, - Lrc::new(SequenceRepetition { - tts: sequence, - separator, - kleene, - num_captures: name_captures, - }), - ) - } - - // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special - // metavariable that names the crate of the invocation. - Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { - let (ident, is_raw) = token.ident().unwrap(); - let span = ident.span.with_lo(span.lo()); - if ident.name == kw::Crate && !is_raw { - TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span) - } else { - TokenTree::MetaVar(span, ident) - } - } - - // `tree` is followed by a random token. This is an error. - Some(tokenstream::TokenTree::Token(token)) => { - let msg = - format!("expected identifier, found `{}`", pprust::token_to_string(&token),); - sess.span_diagnostic.span_err(token.span, &msg); - TokenTree::MetaVar(token.span, ast::Ident::invalid()) - } - - // There are no more tokens. Just return the `$` we already have. - None => TokenTree::token(token::Dollar, span), - }, - - // `tree` is an arbitrary token. Keep it. - tokenstream::TokenTree::Token(token) => TokenTree::Token(token), - - // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to - // descend into the delimited set and further parse it. - tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( - span, - Lrc::new(Delimited { delim, tts: parse(tts.into(), expect_matchers, sess) }), - ), - } -} - -/// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return -/// `None`. -fn kleene_op(token: &Token) -> Option { - match token.kind { - token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), - token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), - token::Question => Some(KleeneOp::ZeroOrOne), - _ => None, - } -} - -/// Parse the next token tree of the input looking for a KleeneOp. Returns -/// -/// - Ok(Ok((op, span))) if the next token tree is a KleeneOp -/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp -/// - Err(span) if the next token tree is not a token -fn parse_kleene_op( - input: &mut impl Iterator, - span: Span, -) -> Result, Span> { - match input.next() { - Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) { - Some(op) => Ok(Ok((op, token.span))), - None => Ok(Err(token)), - }, - tree => Err(tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span)), - } -} - -/// Attempt to parse a single Kleene star, possibly with a separator. -/// -/// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the -/// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing -/// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator -/// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some -/// stream of tokens in an invocation of a macro. -/// -/// This function will take some input iterator `input` corresponding to `span` and a parsing -/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene -/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an -/// error with the appropriate span is emitted to `sess` and a dummy value is returned. -fn parse_sep_and_kleene_op( - input: &mut impl Iterator, - span: Span, - sess: &ParseSess, -) -> (Option, KleeneToken) { - // We basically look at two token trees here, denoted as #1 and #2 below - let span = match parse_kleene_op(input, span) { - // #1 is a `?`, `+`, or `*` KleeneOp - Ok(Ok((op, span))) => return (None, KleeneToken::new(op, span)), - - // #1 is a separator followed by #2, a KleeneOp - Ok(Err(token)) => match parse_kleene_op(input, token.span) { - // #2 is the `?` Kleene op, which does not take a separator (error) - Ok(Ok((KleeneOp::ZeroOrOne, span))) => { - // Error! - sess.span_diagnostic.span_err( - token.span, - "the `?` macro repetition operator does not take a separator", - ); - - // Return a dummy - return (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)); - } - - // #2 is a KleeneOp :D - Ok(Ok((op, span))) => return (Some(token), KleeneToken::new(op, span)), - - // #2 is a random token or not a token at all :( - Ok(Err(Token { span, .. })) | Err(span) => span, - }, - - // #1 is not a token - Err(span) => span, - }; - - // If we ever get to this point, we have experienced an "unexpected token" error - sess.span_diagnostic.span_err(span, "expected one of: `*`, `+`, or `?`"); - - // Return a dummy - (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)) -} diff --git a/src/libsyntax_expand/mbe/transcribe.rs b/src/libsyntax_expand/mbe/transcribe.rs deleted file mode 100644 index 0605f7ff36d..00000000000 --- a/src/libsyntax_expand/mbe/transcribe.rs +++ /dev/null @@ -1,392 +0,0 @@ -use crate::base::ExtCtxt; -use crate::mbe; -use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; - -use syntax::ast::{Ident, Mac}; -use syntax::mut_visit::{self, MutVisitor}; -use syntax::token::{self, NtTT, Token}; -use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; - -use smallvec::{smallvec, SmallVec}; - -use errors::pluralize; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use syntax_pos::hygiene::{ExpnId, Transparency}; -use syntax_pos::Span; - -use std::mem; - -// A Marker adds the given mark to the syntax context. -struct Marker(ExpnId, Transparency); - -impl MutVisitor for Marker { - fn visit_span(&mut self, span: &mut Span) { - *span = span.apply_mark(self.0, self.1) - } - - fn visit_mac(&mut self, mac: &mut Mac) { - mut_visit::noop_visit_mac(mac, self) - } -} - -/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). -enum Frame { - Delimited { forest: Lrc, idx: usize, span: DelimSpan }, - Sequence { forest: Lrc, idx: usize, sep: Option }, -} - -impl Frame { - /// Construct a new frame around the delimited set of tokens. - fn new(tts: Vec) -> Frame { - let forest = Lrc::new(mbe::Delimited { delim: token::NoDelim, tts }); - Frame::Delimited { forest, idx: 0, span: DelimSpan::dummy() } - } -} - -impl Iterator for Frame { - type Item = mbe::TokenTree; - - fn next(&mut self) -> Option { - match *self { - Frame::Delimited { ref forest, ref mut idx, .. } => { - *idx += 1; - forest.tts.get(*idx - 1).cloned() - } - Frame::Sequence { ref forest, ref mut idx, .. } => { - *idx += 1; - forest.tts.get(*idx - 1).cloned() - } - } - } -} - -/// This can do Macro-By-Example transcription. -/// - `interp` is a map of meta-variables to the tokens (non-terminals) they matched in the -/// invocation. We are assuming we already know there is a match. -/// - `src` is the RHS of the MBE, that is, the "example" we are filling in. -/// -/// For example, -/// -/// ```rust -/// macro_rules! foo { -/// ($id:ident) => { println!("{}", stringify!($id)); } -/// } -/// -/// foo!(bar); -/// ``` -/// -/// `interp` would contain `$id => bar` and `src` would contain `println!("{}", stringify!($id));`. -/// -/// `transcribe` would return a `TokenStream` containing `println!("{}", stringify!(bar));`. -/// -/// Along the way, we do some additional error checking. -pub(super) fn transcribe( - cx: &ExtCtxt<'_>, - interp: &FxHashMap, - src: Vec, - transparency: Transparency, -) -> TokenStream { - // Nothing for us to transcribe... - if src.is_empty() { - return TokenStream::default(); - } - - // We descend into the RHS (`src`), expanding things as we go. This stack contains the things - // we have yet to expand/are still expanding. We start the stack off with the whole RHS. - let mut stack: SmallVec<[Frame; 1]> = smallvec![Frame::new(src)]; - - // As we descend in the RHS, we will need to be able to match nested sequences of matchers. - // `repeats` keeps track of where we are in matching at each level, with the last element being - // the most deeply nested sequence. This is used as a stack. - let mut repeats = Vec::new(); - - // `result` contains resulting token stream from the TokenTree we just finished processing. At - // the end, this will contain the full result of transcription, but at arbitrary points during - // `transcribe`, `result` will contain subsets of the final result. - // - // Specifically, as we descend into each TokenTree, we will push the existing results onto the - // `result_stack` and clear `results`. We will then produce the results of transcribing the - // TokenTree into `results`. Then, as we unwind back out of the `TokenTree`, we will pop the - // `result_stack` and append `results` too it to produce the new `results` up to that point. - // - // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level - // again, and we are done transcribing. - let mut result: Vec = Vec::new(); - let mut result_stack = Vec::new(); - let mut marker = Marker(cx.current_expansion.id, transparency); - - loop { - // Look at the last frame on the stack. - let tree = if let Some(tree) = stack.last_mut().unwrap().next() { - // If it still has a TokenTree we have not looked at yet, use that tree. - tree - } - // The else-case never produces a value for `tree` (it `continue`s or `return`s). - else { - // Otherwise, if we have just reached the end of a sequence and we can keep repeating, - // go back to the beginning of the sequence. - if let Frame::Sequence { idx, sep, .. } = stack.last_mut().unwrap() { - let (repeat_idx, repeat_len) = repeats.last_mut().unwrap(); - *repeat_idx += 1; - if repeat_idx < repeat_len { - *idx = 0; - if let Some(sep) = sep { - result.push(TokenTree::Token(sep.clone()).into()); - } - continue; - } - } - - // We are done with the top of the stack. Pop it. Depending on what it was, we do - // different things. Note that the outermost item must be the delimited, wrapped RHS - // that was passed in originally to `transcribe`. - match stack.pop().unwrap() { - // Done with a sequence. Pop from repeats. - Frame::Sequence { .. } => { - repeats.pop(); - } - - // We are done processing a Delimited. If this is the top-level delimited, we are - // done. Otherwise, we unwind the result_stack to append what we have produced to - // any previous results. - Frame::Delimited { forest, span, .. } => { - if result_stack.is_empty() { - // No results left to compute! We are back at the top-level. - return TokenStream::new(result); - } - - // Step back into the parent Delimited. - let tree = - TokenTree::Delimited(span, forest.delim, TokenStream::new(result).into()); - result = result_stack.pop().unwrap(); - result.push(tree.into()); - } - } - continue; - }; - - // At this point, we know we are in the middle of a TokenTree (the last one on `stack`). - // `tree` contains the next `TokenTree` to be processed. - match tree { - // We are descending into a sequence. We first make sure that the matchers in the RHS - // and the matches in `interp` have the same shape. Otherwise, either the caller or the - // macro writer has made a mistake. - seq @ mbe::TokenTree::Sequence(..) => { - match lockstep_iter_size(&seq, interp, &repeats) { - LockstepIterSize::Unconstrained => { - cx.span_fatal( - seq.span(), /* blame macro writer */ - "attempted to repeat an expression containing no syntax variables \ - matched as repeating at this depth", - ); - } - - LockstepIterSize::Contradiction(ref msg) => { - // FIXME: this really ought to be caught at macro definition time... It - // happens when two meta-variables are used in the same repetition in a - // sequence, but they come from different sequence matchers and repeat - // different amounts. - cx.span_fatal(seq.span(), &msg[..]); - } - - LockstepIterSize::Constraint(len, _) => { - // We do this to avoid an extra clone above. We know that this is a - // sequence already. - let (sp, seq) = if let mbe::TokenTree::Sequence(sp, seq) = seq { - (sp, seq) - } else { - unreachable!() - }; - - // Is the repetition empty? - if len == 0 { - if seq.kleene.op == mbe::KleeneOp::OneOrMore { - // FIXME: this really ought to be caught at macro definition - // time... It happens when the Kleene operator in the matcher and - // the body for the same meta-variable do not match. - cx.span_fatal(sp.entire(), "this must repeat at least once"); - } - } else { - // 0 is the initial counter (we have done 0 repretitions so far). `len` - // is the total number of reptitions we should generate. - repeats.push((0, len)); - - // The first time we encounter the sequence we push it to the stack. It - // then gets reused (see the beginning of the loop) until we are done - // repeating. - stack.push(Frame::Sequence { - idx: 0, - sep: seq.separator.clone(), - forest: seq, - }); - } - } - } - } - - // Replace the meta-var with the matched token tree from the invocation. - mbe::TokenTree::MetaVar(mut sp, mut ident) => { - // Find the matched nonterminal from the macro invocation, and use it to replace - // the meta-var. - if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { - if let MatchedNonterminal(ref nt) = cur_matched { - // FIXME #2887: why do we apply a mark when matching a token tree meta-var - // (e.g. `$x:tt`), but not when we are matching any other type of token - // tree? - if let NtTT(ref tt) = **nt { - result.push(tt.clone().into()); - } else { - marker.visit_span(&mut sp); - let token = TokenTree::token(token::Interpolated(nt.clone()), sp); - result.push(token.into()); - } - } else { - // We were unable to descend far enough. This is an error. - cx.span_fatal( - sp, /* blame the macro writer */ - &format!("variable '{}' is still repeating at this depth", ident), - ); - } - } else { - // If we aren't able to match the meta-var, we push it back into the result but - // with modified syntax context. (I believe this supports nested macros). - marker.visit_span(&mut sp); - marker.visit_ident(&mut ident); - result.push(TokenTree::token(token::Dollar, sp).into()); - result.push(TokenTree::Token(Token::from_ast_ident(ident)).into()); - } - } - - // If we are entering a new delimiter, we push its contents to the `stack` to be - // processed, and we push all of the currently produced results to the `result_stack`. - // We will produce all of the results of the inside of the `Delimited` and then we will - // jump back out of the Delimited, pop the result_stack and add the new results back to - // the previous results (from outside the Delimited). - mbe::TokenTree::Delimited(mut span, delimited) => { - mut_visit::visit_delim_span(&mut span, &mut marker); - stack.push(Frame::Delimited { forest: delimited, idx: 0, span }); - result_stack.push(mem::take(&mut result)); - } - - // Nothing much to do here. Just push the token to the result, being careful to - // preserve syntax context. - mbe::TokenTree::Token(token) => { - let mut tt = TokenTree::Token(token); - marker.visit_tt(&mut tt); - result.push(tt.into()); - } - - // There should be no meta-var declarations in the invocation of a macro. - mbe::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"), - } - } -} - -/// Lookup the meta-var named `ident` and return the matched token tree from the invocation using -/// the set of matches `interpolations`. -/// -/// See the definition of `repeats` in the `transcribe` function. `repeats` is used to descend -/// into the right place in nested matchers. If we attempt to descend too far, the macro writer has -/// made a mistake, and we return `None`. -fn lookup_cur_matched<'a>( - ident: Ident, - interpolations: &'a FxHashMap, - repeats: &[(usize, usize)], -) -> Option<&'a NamedMatch> { - interpolations.get(&ident).map(|matched| { - let mut matched = matched; - for &(idx, _) in repeats { - match matched { - MatchedNonterminal(_) => break, - MatchedSeq(ref ads) => matched = ads.get(idx).unwrap(), - } - } - - matched - }) -} - -/// An accumulator over a TokenTree to be used with `fold`. During transcription, we need to make -/// sure that the size of each sequence and all of its nested sequences are the same as the sizes -/// of all the matched (nested) sequences in the macro invocation. If they don't match, somebody -/// has made a mistake (either the macro writer or caller). -#[derive(Clone)] -enum LockstepIterSize { - /// No constraints on length of matcher. This is true for any TokenTree variants except a - /// `MetaVar` with an actual `MatchedSeq` (as opposed to a `MatchedNonterminal`). - Unconstrained, - - /// A `MetaVar` with an actual `MatchedSeq`. The length of the match and the name of the - /// meta-var are returned. - Constraint(usize, Ident), - - /// Two `Constraint`s on the same sequence had different lengths. This is an error. - Contradiction(String), -} - -impl LockstepIterSize { - /// Find incompatibilities in matcher/invocation sizes. - /// - `Unconstrained` is compatible with everything. - /// - `Contradiction` is incompatible with everything. - /// - `Constraint(len)` is only compatible with other constraints of the same length. - fn with(self, other: LockstepIterSize) -> LockstepIterSize { - match self { - LockstepIterSize::Unconstrained => other, - LockstepIterSize::Contradiction(_) => self, - LockstepIterSize::Constraint(l_len, ref l_id) => match other { - LockstepIterSize::Unconstrained => self, - LockstepIterSize::Contradiction(_) => other, - LockstepIterSize::Constraint(r_len, _) if l_len == r_len => self, - LockstepIterSize::Constraint(r_len, r_id) => { - let msg = format!( - "meta-variable `{}` repeats {} time{}, but `{}` repeats {} time{}", - l_id, - l_len, - pluralize!(l_len), - r_id, - r_len, - pluralize!(r_len), - ); - LockstepIterSize::Contradiction(msg) - } - }, - } - } -} - -/// Given a `tree`, make sure that all sequences have the same length as the matches for the -/// appropriate meta-vars in `interpolations`. -/// -/// Note that if `repeats` does not match the exact correct depth of a meta-var, -/// `lookup_cur_matched` will return `None`, which is why this still works even in the presnece of -/// multiple nested matcher sequences. -fn lockstep_iter_size( - tree: &mbe::TokenTree, - interpolations: &FxHashMap, - repeats: &[(usize, usize)], -) -> LockstepIterSize { - use mbe::TokenTree; - match *tree { - TokenTree::Delimited(_, ref delimed) => { - delimed.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { - size.with(lockstep_iter_size(tt, interpolations, repeats)) - }) - } - TokenTree::Sequence(_, ref seq) => { - seq.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { - size.with(lockstep_iter_size(tt, interpolations, repeats)) - }) - } - TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => { - match lookup_cur_matched(name, interpolations, repeats) { - Some(matched) => match matched { - MatchedNonterminal(_) => LockstepIterSize::Unconstrained, - MatchedSeq(ref ads) => LockstepIterSize::Constraint(ads.len(), name), - }, - _ => LockstepIterSize::Unconstrained, - } - } - TokenTree::Token(..) => LockstepIterSize::Unconstrained, - } -} diff --git a/src/libsyntax_expand/mut_visit/tests.rs b/src/libsyntax_expand/mut_visit/tests.rs deleted file mode 100644 index 003ce0fcb1f..00000000000 --- a/src/libsyntax_expand/mut_visit/tests.rs +++ /dev/null @@ -1,72 +0,0 @@ -use crate::tests::{matches_codepattern, string_to_crate}; - -use syntax::ast::{self, Ident}; -use syntax::mut_visit::{self, MutVisitor}; -use syntax::print::pprust; -use syntax::with_default_globals; - -// This version doesn't care about getting comments or doc-strings in. -fn fake_print_crate(s: &mut pprust::State<'_>, krate: &ast::Crate) { - s.print_mod(&krate.module, &krate.attrs) -} - -// Change every identifier to "zz". -struct ToZzIdentMutVisitor; - -impl MutVisitor for ToZzIdentMutVisitor { - fn visit_ident(&mut self, ident: &mut ast::Ident) { - *ident = Ident::from_str("zz"); - } - fn visit_mac(&mut self, mac: &mut ast::Mac) { - mut_visit::noop_visit_mac(mac, self) - } -} - -// Maybe add to `expand.rs`. -macro_rules! assert_pred { - ($pred:expr, $predname:expr, $a:expr , $b:expr) => {{ - let pred_val = $pred; - let a_val = $a; - let b_val = $b; - if !(pred_val(&a_val, &b_val)) { - panic!("expected args satisfying {}, got {} and {}", $predname, a_val, b_val); - } - }}; -} - -// Make sure idents get transformed everywhere. -#[test] -fn ident_transformation() { - with_default_globals(|| { - let mut zz_visitor = ToZzIdentMutVisitor; - let mut krate = - string_to_crate("#[a] mod b {fn c (d : e, f : g) {h!(i,j,k);l;m}}".to_string()); - zz_visitor.visit_crate(&mut krate); - assert_pred!( - matches_codepattern, - "matches_codepattern", - pprust::to_string(|s| fake_print_crate(s, &krate)), - "#[zz]mod zz{fn zz(zz:zz,zz:zz){zz!(zz,zz,zz);zz;zz}}".to_string() - ); - }) -} - -// Make sure idents get transformed even inside macro defs. -#[test] -fn ident_transformation_in_defs() { - with_default_globals(|| { - let mut zz_visitor = ToZzIdentMutVisitor; - let mut krate = string_to_crate( - "macro_rules! a {(b $c:expr $(d $e:token)f+ => \ - (g $(d $d $e)+))} " - .to_string(), - ); - zz_visitor.visit_crate(&mut krate); - assert_pred!( - matches_codepattern, - "matches_codepattern", - pprust::to_string(|s| fake_print_crate(s, &krate)), - "macro_rules! zz{(zz$zz:zz$(zz $zz:zz)zz+=>(zz$(zz$zz$zz)+))}".to_string() - ); - }) -} diff --git a/src/libsyntax_expand/parse/lexer/tests.rs b/src/libsyntax_expand/parse/lexer/tests.rs deleted file mode 100644 index 2ca0224812b..00000000000 --- a/src/libsyntax_expand/parse/lexer/tests.rs +++ /dev/null @@ -1,256 +0,0 @@ -use rustc_data_structures::sync::Lrc; -use rustc_parse::lexer::StringReader; -use syntax::sess::ParseSess; -use syntax::source_map::{FilePathMapping, SourceMap}; -use syntax::token::{self, Token, TokenKind}; -use syntax::util::comments::is_doc_comment; -use syntax::with_default_globals; -use syntax_pos::symbol::Symbol; -use syntax_pos::{BytePos, Span}; - -use errors::{emitter::EmitterWriter, Handler}; -use std::io; -use std::path::PathBuf; - -fn mk_sess(sm: Lrc) -> ParseSess { - let emitter = EmitterWriter::new( - Box::new(io::sink()), - Some(sm.clone()), - false, - false, - false, - None, - false, - ); - ParseSess::with_span_handler(Handler::with_emitter(true, None, Box::new(emitter)), sm) -} - -// Creates a string reader for the given string. -fn setup<'a>(sm: &SourceMap, sess: &'a ParseSess, teststr: String) -> StringReader<'a> { - let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr); - StringReader::new(sess, sf, None) -} - -#[test] -fn t1() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - let mut string_reader = setup( - &sm, - &sh, - "/* my source file */ fn main() { println!(\"zebra\"); }\n".to_string(), - ); - assert_eq!(string_reader.next_token(), token::Comment); - assert_eq!(string_reader.next_token(), token::Whitespace); - let tok1 = string_reader.next_token(); - let tok2 = Token::new(mk_ident("fn"), Span::with_root_ctxt(BytePos(21), BytePos(23))); - assert_eq!(tok1.kind, tok2.kind); - assert_eq!(tok1.span, tok2.span); - assert_eq!(string_reader.next_token(), token::Whitespace); - // Read another token. - let tok3 = string_reader.next_token(); - assert_eq!(string_reader.pos.clone(), BytePos(28)); - let tok4 = Token::new(mk_ident("main"), Span::with_root_ctxt(BytePos(24), BytePos(28))); - assert_eq!(tok3.kind, tok4.kind); - assert_eq!(tok3.span, tok4.span); - - assert_eq!(string_reader.next_token(), token::OpenDelim(token::Paren)); - assert_eq!(string_reader.pos.clone(), BytePos(29)) - }) -} - -// Checks that the given reader produces the desired stream -// of tokens (stop checking after exhausting `expected`). -fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec) { - for expected_tok in &expected { - assert_eq!(&string_reader.next_token(), expected_tok); - } -} - -// Makes the identifier by looking up the string in the interner. -fn mk_ident(id: &str) -> TokenKind { - token::Ident(Symbol::intern(id), false) -} - -fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind { - TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern)) -} - -#[test] -fn doublecolon_parsing() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization( - setup(&sm, &sh, "a b".to_string()), - vec![mk_ident("a"), token::Whitespace, mk_ident("b")], - ); - }) -} - -#[test] -fn doublecolon_parsing_2() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization( - setup(&sm, &sh, "a::b".to_string()), - vec![mk_ident("a"), token::Colon, token::Colon, mk_ident("b")], - ); - }) -} - -#[test] -fn doublecolon_parsing_3() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization( - setup(&sm, &sh, "a ::b".to_string()), - vec![mk_ident("a"), token::Whitespace, token::Colon, token::Colon, mk_ident("b")], - ); - }) -} - -#[test] -fn doublecolon_parsing_4() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization( - setup(&sm, &sh, "a:: b".to_string()), - vec![mk_ident("a"), token::Colon, token::Colon, token::Whitespace, mk_ident("b")], - ); - }) -} - -#[test] -fn character_a() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!(setup(&sm, &sh, "'a'".to_string()).next_token(), mk_lit(token::Char, "a", None),); - }) -} - -#[test] -fn character_space() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!(setup(&sm, &sh, "' '".to_string()).next_token(), mk_lit(token::Char, " ", None),); - }) -} - -#[test] -fn character_escaped() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!( - setup(&sm, &sh, "'\\n'".to_string()).next_token(), - mk_lit(token::Char, "\\n", None), - ); - }) -} - -#[test] -fn lifetime_name() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!( - setup(&sm, &sh, "'abc".to_string()).next_token(), - token::Lifetime(Symbol::intern("'abc")), - ); - }) -} - -#[test] -fn raw_string() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!( - setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token(), - mk_lit(token::StrRaw(3), "\"#a\\b\x00c\"", None), - ); - }) -} - -#[test] -fn literal_suffixes() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - macro_rules! test { - ($input: expr, $tok_type: ident, $tok_contents: expr) => {{ - assert_eq!( - setup(&sm, &sh, format!("{}suffix", $input)).next_token(), - mk_lit(token::$tok_type, $tok_contents, Some("suffix")), - ); - // with a whitespace separator - assert_eq!( - setup(&sm, &sh, format!("{} suffix", $input)).next_token(), - mk_lit(token::$tok_type, $tok_contents, None), - ); - }}; - } - - test!("'a'", Char, "a"); - test!("b'a'", Byte, "a"); - test!("\"a\"", Str, "a"); - test!("b\"a\"", ByteStr, "a"); - test!("1234", Integer, "1234"); - test!("0b101", Integer, "0b101"); - test!("0xABC", Integer, "0xABC"); - test!("1.0", Float, "1.0"); - test!("1.0e10", Float, "1.0e10"); - - assert_eq!( - setup(&sm, &sh, "2us".to_string()).next_token(), - mk_lit(token::Integer, "2", Some("us")), - ); - assert_eq!( - setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token(), - mk_lit(token::StrRaw(3), "raw", Some("suffix")), - ); - assert_eq!( - setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token(), - mk_lit(token::ByteStrRaw(3), "raw", Some("suffix")), - ); - }) -} - -#[test] -fn line_doc_comments() { - assert!(is_doc_comment("///")); - assert!(is_doc_comment("/// blah")); - assert!(!is_doc_comment("////")); -} - -#[test] -fn nested_block_comments() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - let mut lexer = setup(&sm, &sh, "/* /* */ */'a'".to_string()); - assert_eq!(lexer.next_token(), token::Comment); - assert_eq!(lexer.next_token(), mk_lit(token::Char, "a", None)); - }) -} - -#[test] -fn crlf_comments() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - let mut lexer = setup(&sm, &sh, "// test\r\n/// test\r\n".to_string()); - let comment = lexer.next_token(); - assert_eq!(comment.kind, token::Comment); - assert_eq!((comment.span.lo(), comment.span.hi()), (BytePos(0), BytePos(7))); - assert_eq!(lexer.next_token(), token::Whitespace); - assert_eq!(lexer.next_token(), token::DocComment(Symbol::intern("/// test"))); - }) -} diff --git a/src/libsyntax_expand/parse/tests.rs b/src/libsyntax_expand/parse/tests.rs deleted file mode 100644 index 833fda6a2eb..00000000000 --- a/src/libsyntax_expand/parse/tests.rs +++ /dev/null @@ -1,348 +0,0 @@ -use crate::tests::{matches_codepattern, string_to_stream, with_error_checking_parse}; - -use errors::PResult; -use rustc_parse::new_parser_from_source_str; -use syntax::ast::{self, Name, PatKind}; -use syntax::print::pprust::item_to_string; -use syntax::ptr::P; -use syntax::sess::ParseSess; -use syntax::source_map::FilePathMapping; -use syntax::symbol::{kw, sym, Symbol}; -use syntax::token::{self, Token}; -use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree}; -use syntax::visit; -use syntax::with_default_globals; -use syntax_pos::{BytePos, FileName, Pos, Span}; - -use std::path::PathBuf; - -fn sess() -> ParseSess { - ParseSess::new(FilePathMapping::empty()) -} - -/// Parses an item. -/// -/// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and `Err` -/// when a syntax error occurred. -fn parse_item_from_source_str( - name: FileName, - source: String, - sess: &ParseSess, -) -> PResult<'_, Option>> { - new_parser_from_source_str(sess, name, source).parse_item() -} - -// Produces a `syntax_pos::span`. -fn sp(a: u32, b: u32) -> Span { - Span::with_root_ctxt(BytePos(a), BytePos(b)) -} - -/// Parses a string, return an expression. -fn string_to_expr(source_str: String) -> P { - with_error_checking_parse(source_str, &sess(), |p| p.parse_expr()) -} - -/// Parses a string, returns an item. -fn string_to_item(source_str: String) -> Option> { - with_error_checking_parse(source_str, &sess(), |p| p.parse_item()) -} - -#[should_panic] -#[test] -fn bad_path_expr_1() { - with_default_globals(|| { - string_to_expr("::abc::def::return".to_string()); - }) -} - -// Checks the token-tree-ization of macros. -#[test] -fn string_to_tts_macro() { - with_default_globals(|| { - let tts: Vec<_> = - string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect(); - let tts: &[TokenTree] = &tts[..]; - - match tts { - [TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }), TokenTree::Token(Token { kind: token::Not, .. }), TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }), TokenTree::Delimited(_, macro_delim, macro_tts)] - if name_macro_rules == &sym::macro_rules && name_zip.as_str() == "zip" => - { - let tts = ¯o_tts.trees().collect::>(); - match &tts[..] { - [TokenTree::Delimited(_, first_delim, first_tts), TokenTree::Token(Token { kind: token::FatArrow, .. }), TokenTree::Delimited(_, second_delim, second_tts)] - if macro_delim == &token::Paren => - { - let tts = &first_tts.trees().collect::>(); - match &tts[..] { - [TokenTree::Token(Token { kind: token::Dollar, .. }), TokenTree::Token(Token { kind: token::Ident(name, false), .. })] - if first_delim == &token::Paren && name.as_str() == "a" => {} - _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), - } - let tts = &second_tts.trees().collect::>(); - match &tts[..] { - [TokenTree::Token(Token { kind: token::Dollar, .. }), TokenTree::Token(Token { kind: token::Ident(name, false), .. })] - if second_delim == &token::Paren && name.as_str() == "a" => {} - _ => panic!("value 4: {:?} {:?}", second_delim, second_tts), - } - } - _ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts), - } - } - _ => panic!("value: {:?}", tts), - } - }) -} - -#[test] -fn string_to_tts_1() { - with_default_globals(|| { - let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); - - let expected = TokenStream::new(vec![ - TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(), - TokenTree::token(token::Ident(Name::intern("a"), false), sp(3, 4)).into(), - TokenTree::Delimited( - DelimSpan::from_pair(sp(5, 6), sp(13, 14)), - token::DelimToken::Paren, - TokenStream::new(vec![ - TokenTree::token(token::Ident(Name::intern("b"), false), sp(6, 7)).into(), - TokenTree::token(token::Colon, sp(8, 9)).into(), - TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(), - ]) - .into(), - ) - .into(), - TokenTree::Delimited( - DelimSpan::from_pair(sp(15, 16), sp(20, 21)), - token::DelimToken::Brace, - TokenStream::new(vec![ - TokenTree::token(token::Ident(Name::intern("b"), false), sp(17, 18)).into(), - TokenTree::token(token::Semi, sp(18, 19)).into(), - ]) - .into(), - ) - .into(), - ]); - - assert_eq!(tts, expected); - }) -} - -#[test] -fn parse_use() { - with_default_globals(|| { - let use_s = "use foo::bar::baz;"; - let vitem = string_to_item(use_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], use_s); - - let use_s = "use foo::bar as baz;"; - let vitem = string_to_item(use_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], use_s); - }) -} - -#[test] -fn parse_extern_crate() { - with_default_globals(|| { - let ex_s = "extern crate foo;"; - let vitem = string_to_item(ex_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], ex_s); - - let ex_s = "extern crate foo as bar;"; - let vitem = string_to_item(ex_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], ex_s); - }) -} - -fn get_spans_of_pat_idents(src: &str) -> Vec { - let item = string_to_item(src.to_string()).unwrap(); - - struct PatIdentVisitor { - spans: Vec, - } - impl<'a> visit::Visitor<'a> for PatIdentVisitor { - fn visit_pat(&mut self, p: &'a ast::Pat) { - match p.kind { - PatKind::Ident(_, ref ident, _) => { - self.spans.push(ident.span.clone()); - } - _ => { - visit::walk_pat(self, p); - } - } - } - } - let mut v = PatIdentVisitor { spans: Vec::new() }; - visit::walk_item(&mut v, &item); - return v.spans; -} - -#[test] -fn span_of_self_arg_pat_idents_are_correct() { - with_default_globals(|| { - let srcs = [ - "impl z { fn a (&self, &myarg: i32) {} }", - "impl z { fn a (&mut self, &myarg: i32) {} }", - "impl z { fn a (&'a self, &myarg: i32) {} }", - "impl z { fn a (self, &myarg: i32) {} }", - "impl z { fn a (self: Foo, &myarg: i32) {} }", - ]; - - for &src in &srcs { - let spans = get_spans_of_pat_idents(src); - let (lo, hi) = (spans[0].lo(), spans[0].hi()); - assert!( - "self" == &src[lo.to_usize()..hi.to_usize()], - "\"{}\" != \"self\". src=\"{}\"", - &src[lo.to_usize()..hi.to_usize()], - src - ) - } - }) -} - -#[test] -fn parse_exprs() { - with_default_globals(|| { - // just make sure that they parse.... - string_to_expr("3 + 4".to_string()); - string_to_expr("a::z.froob(b,&(987+3))".to_string()); - }) -} - -#[test] -fn attrs_fix_bug() { - with_default_globals(|| { - string_to_item( - "pub fn mk_file_writer(path: &Path, flags: &[FileFlag]) - -> Result, String> { -#[cfg(windows)] -fn wb() -> c_int { - (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int -} - -#[cfg(unix)] -fn wb() -> c_int { O_WRONLY as c_int } - -let mut fflags: c_int = wb(); -}" - .to_string(), - ); - }) -} - -#[test] -fn crlf_doc_comments() { - with_default_globals(|| { - let sess = sess(); - - let name_1 = FileName::Custom("crlf_source_1".to_string()); - let source = "/// doc comment\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name_1, source, &sess).unwrap().unwrap(); - let doc = item.attrs.iter().filter_map(|at| at.doc_str()).next().unwrap(); - assert_eq!(doc.as_str(), "/// doc comment"); - - let name_2 = FileName::Custom("crlf_source_2".to_string()); - let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name_2, source, &sess).unwrap().unwrap(); - let docs = item.attrs.iter().filter_map(|at| at.doc_str()).collect::>(); - let b: &[_] = &[Symbol::intern("/// doc comment"), Symbol::intern("/// line 2")]; - assert_eq!(&docs[..], b); - - let name_3 = FileName::Custom("clrf_source_3".to_string()); - let source = "/** doc comment\r\n * with CRLF */\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name_3, source, &sess).unwrap().unwrap(); - let doc = item.attrs.iter().filter_map(|at| at.doc_str()).next().unwrap(); - assert_eq!(doc.as_str(), "/** doc comment\n * with CRLF */"); - }); -} - -#[test] -fn ttdelim_span() { - fn parse_expr_from_source_str( - name: FileName, - source: String, - sess: &ParseSess, - ) -> PResult<'_, P> { - new_parser_from_source_str(sess, name, source).parse_expr() - } - - with_default_globals(|| { - let sess = sess(); - let expr = parse_expr_from_source_str( - PathBuf::from("foo").into(), - "foo!( fn main() { body } )".to_string(), - &sess, - ) - .unwrap(); - - let tts: Vec<_> = match expr.kind { - ast::ExprKind::Mac(ref mac) => mac.args.inner_tokens().trees().collect(), - _ => panic!("not a macro"), - }; - - let span = tts.iter().rev().next().unwrap().span(); - - match sess.source_map().span_to_snippet(span) { - Ok(s) => assert_eq!(&s[..], "{ body }"), - Err(_) => panic!("could not get snippet"), - } - }); -} - -// This tests that when parsing a string (rather than a file) we don't try -// and read in a file for a module declaration and just parse a stub. -// See `recurse_into_file_modules` in the parser. -#[test] -fn out_of_line_mod() { - with_default_globals(|| { - let item = parse_item_from_source_str( - PathBuf::from("foo").into(), - "mod foo { struct S; mod this_does_not_exist; }".to_owned(), - &sess(), - ) - .unwrap() - .unwrap(); - - if let ast::ItemKind::Mod(ref m) = item.kind { - assert!(m.items.len() == 2); - } else { - panic!(); - } - }); -} - -#[test] -fn eqmodws() { - assert_eq!(matches_codepattern("", ""), true); - assert_eq!(matches_codepattern("", "a"), false); - assert_eq!(matches_codepattern("a", ""), false); - assert_eq!(matches_codepattern("a", "a"), true); - assert_eq!(matches_codepattern("a b", "a \n\t\r b"), true); - assert_eq!(matches_codepattern("a b ", "a \n\t\r b"), true); - assert_eq!(matches_codepattern("a b", "a \n\t\r b "), false); - assert_eq!(matches_codepattern("a b", "a b"), true); - assert_eq!(matches_codepattern("ab", "a b"), false); - assert_eq!(matches_codepattern("a b", "ab"), true); - assert_eq!(matches_codepattern(" a b", "ab"), true); -} - -#[test] -fn pattern_whitespace() { - assert_eq!(matches_codepattern("", "\x0C"), false); - assert_eq!(matches_codepattern("a b ", "a \u{0085}\n\t\r b"), true); - assert_eq!(matches_codepattern("a b", "a \u{0085}\n\t\r b "), false); -} - -#[test] -fn non_pattern_whitespace() { - // These have the property 'White_Space' but not 'Pattern_White_Space' - assert_eq!(matches_codepattern("a b", "a\u{2002}b"), false); - assert_eq!(matches_codepattern("a b", "a\u{2002}b"), false); - assert_eq!(matches_codepattern("\u{205F}a b", "ab"), false); - assert_eq!(matches_codepattern("a \u{3000}b", "ab"), false); -} diff --git a/src/libsyntax_expand/placeholders.rs b/src/libsyntax_expand/placeholders.rs deleted file mode 100644 index 231a5a19cb6..00000000000 --- a/src/libsyntax_expand/placeholders.rs +++ /dev/null @@ -1,339 +0,0 @@ -use crate::base::ExtCtxt; -use crate::expand::{AstFragment, AstFragmentKind}; - -use syntax::ast; -use syntax::mut_visit::*; -use syntax::ptr::P; -use syntax::source_map::{dummy_spanned, DUMMY_SP}; - -use smallvec::{smallvec, SmallVec}; - -use rustc_data_structures::fx::FxHashMap; - -pub fn placeholder( - kind: AstFragmentKind, - id: ast::NodeId, - vis: Option, -) -> AstFragment { - fn mac_placeholder() -> ast::Mac { - ast::Mac { - path: ast::Path { span: DUMMY_SP, segments: Vec::new() }, - args: P(ast::MacArgs::Empty), - prior_type_ascription: None, - } - } - - let ident = ast::Ident::invalid(); - let attrs = Vec::new(); - let generics = ast::Generics::default(); - let vis = vis.unwrap_or_else(|| dummy_spanned(ast::VisibilityKind::Inherited)); - let span = DUMMY_SP; - let expr_placeholder = || { - P(ast::Expr { - id, - span, - attrs: ast::AttrVec::new(), - kind: ast::ExprKind::Mac(mac_placeholder()), - }) - }; - let ty = || P(ast::Ty { id, kind: ast::TyKind::Mac(mac_placeholder()), span }); - let pat = || P(ast::Pat { id, kind: ast::PatKind::Mac(mac_placeholder()), span }); - - match kind { - AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()), - AstFragmentKind::OptExpr => AstFragment::OptExpr(Some(expr_placeholder())), - AstFragmentKind::Items => AstFragment::Items(smallvec![P(ast::Item { - id, - span, - ident, - vis, - attrs, - kind: ast::ItemKind::Mac(mac_placeholder()), - tokens: None, - })]), - AstFragmentKind::TraitItems => AstFragment::TraitItems(smallvec![ast::AssocItem { - id, - span, - ident, - vis, - attrs, - generics, - kind: ast::AssocItemKind::Macro(mac_placeholder()), - defaultness: ast::Defaultness::Final, - tokens: None, - }]), - AstFragmentKind::ImplItems => AstFragment::ImplItems(smallvec![ast::AssocItem { - id, - span, - ident, - vis, - attrs, - generics, - kind: ast::AssocItemKind::Macro(mac_placeholder()), - defaultness: ast::Defaultness::Final, - tokens: None, - }]), - AstFragmentKind::ForeignItems => AstFragment::ForeignItems(smallvec![ast::ForeignItem { - id, - span, - ident, - vis, - attrs, - kind: ast::ForeignItemKind::Macro(mac_placeholder()), - tokens: None, - }]), - AstFragmentKind::Pat => { - AstFragment::Pat(P(ast::Pat { id, span, kind: ast::PatKind::Mac(mac_placeholder()) })) - } - AstFragmentKind::Ty => { - AstFragment::Ty(P(ast::Ty { id, span, kind: ast::TyKind::Mac(mac_placeholder()) })) - } - AstFragmentKind::Stmts => AstFragment::Stmts(smallvec![{ - let mac = P((mac_placeholder(), ast::MacStmtStyle::Braces, ast::AttrVec::new())); - ast::Stmt { id, span, kind: ast::StmtKind::Mac(mac) } - }]), - AstFragmentKind::Arms => AstFragment::Arms(smallvec![ast::Arm { - attrs: Default::default(), - body: expr_placeholder(), - guard: None, - id, - pat: pat(), - span, - is_placeholder: true, - }]), - AstFragmentKind::Fields => AstFragment::Fields(smallvec![ast::Field { - attrs: Default::default(), - expr: expr_placeholder(), - id, - ident, - is_shorthand: false, - span, - is_placeholder: true, - }]), - AstFragmentKind::FieldPats => AstFragment::FieldPats(smallvec![ast::FieldPat { - attrs: Default::default(), - id, - ident, - is_shorthand: false, - pat: pat(), - span, - is_placeholder: true, - }]), - AstFragmentKind::GenericParams => AstFragment::GenericParams(smallvec![{ - ast::GenericParam { - attrs: Default::default(), - bounds: Default::default(), - id, - ident, - is_placeholder: true, - kind: ast::GenericParamKind::Lifetime, - } - }]), - AstFragmentKind::Params => AstFragment::Params(smallvec![ast::Param { - attrs: Default::default(), - id, - pat: pat(), - span, - ty: ty(), - is_placeholder: true, - }]), - AstFragmentKind::StructFields => AstFragment::StructFields(smallvec![ast::StructField { - attrs: Default::default(), - id, - ident: None, - span, - ty: ty(), - vis, - is_placeholder: true, - }]), - AstFragmentKind::Variants => AstFragment::Variants(smallvec![ast::Variant { - attrs: Default::default(), - data: ast::VariantData::Struct(Default::default(), false), - disr_expr: None, - id, - ident, - span, - vis, - is_placeholder: true, - }]), - } -} - -pub struct PlaceholderExpander<'a, 'b> { - expanded_fragments: FxHashMap, - cx: &'a mut ExtCtxt<'b>, - monotonic: bool, -} - -impl<'a, 'b> PlaceholderExpander<'a, 'b> { - pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { - PlaceholderExpander { cx, expanded_fragments: FxHashMap::default(), monotonic } - } - - pub fn add(&mut self, id: ast::NodeId, mut fragment: AstFragment) { - fragment.mut_visit_with(self); - self.expanded_fragments.insert(id, fragment); - } - - fn remove(&mut self, id: ast::NodeId) -> AstFragment { - self.expanded_fragments.remove(&id).unwrap() - } -} - -impl<'a, 'b> MutVisitor for PlaceholderExpander<'a, 'b> { - fn flat_map_arm(&mut self, arm: ast::Arm) -> SmallVec<[ast::Arm; 1]> { - if arm.is_placeholder { - self.remove(arm.id).make_arms() - } else { - noop_flat_map_arm(arm, self) - } - } - - fn flat_map_field(&mut self, field: ast::Field) -> SmallVec<[ast::Field; 1]> { - if field.is_placeholder { - self.remove(field.id).make_fields() - } else { - noop_flat_map_field(field, self) - } - } - - fn flat_map_field_pattern(&mut self, fp: ast::FieldPat) -> SmallVec<[ast::FieldPat; 1]> { - if fp.is_placeholder { - self.remove(fp.id).make_field_patterns() - } else { - noop_flat_map_field_pattern(fp, self) - } - } - - fn flat_map_generic_param( - &mut self, - param: ast::GenericParam, - ) -> SmallVec<[ast::GenericParam; 1]> { - if param.is_placeholder { - self.remove(param.id).make_generic_params() - } else { - noop_flat_map_generic_param(param, self) - } - } - - fn flat_map_param(&mut self, p: ast::Param) -> SmallVec<[ast::Param; 1]> { - if p.is_placeholder { - self.remove(p.id).make_params() - } else { - noop_flat_map_param(p, self) - } - } - - fn flat_map_struct_field(&mut self, sf: ast::StructField) -> SmallVec<[ast::StructField; 1]> { - if sf.is_placeholder { - self.remove(sf.id).make_struct_fields() - } else { - noop_flat_map_struct_field(sf, self) - } - } - - fn flat_map_variant(&mut self, variant: ast::Variant) -> SmallVec<[ast::Variant; 1]> { - if variant.is_placeholder { - self.remove(variant.id).make_variants() - } else { - noop_flat_map_variant(variant, self) - } - } - - fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { - match item.kind { - ast::ItemKind::Mac(_) => return self.remove(item.id).make_items(), - ast::ItemKind::MacroDef(_) => return smallvec![item], - _ => {} - } - - noop_flat_map_item(item, self) - } - - fn flat_map_trait_item(&mut self, item: ast::AssocItem) -> SmallVec<[ast::AssocItem; 1]> { - match item.kind { - ast::AssocItemKind::Macro(_) => self.remove(item.id).make_trait_items(), - _ => noop_flat_map_assoc_item(item, self), - } - } - - fn flat_map_impl_item(&mut self, item: ast::AssocItem) -> SmallVec<[ast::AssocItem; 1]> { - match item.kind { - ast::AssocItemKind::Macro(_) => self.remove(item.id).make_impl_items(), - _ => noop_flat_map_assoc_item(item, self), - } - } - - fn flat_map_foreign_item(&mut self, item: ast::ForeignItem) -> SmallVec<[ast::ForeignItem; 1]> { - match item.kind { - ast::ForeignItemKind::Macro(_) => self.remove(item.id).make_foreign_items(), - _ => noop_flat_map_foreign_item(item, self), - } - } - - fn visit_expr(&mut self, expr: &mut P) { - match expr.kind { - ast::ExprKind::Mac(_) => *expr = self.remove(expr.id).make_expr(), - _ => noop_visit_expr(expr, self), - } - } - - fn filter_map_expr(&mut self, expr: P) -> Option> { - match expr.kind { - ast::ExprKind::Mac(_) => self.remove(expr.id).make_opt_expr(), - _ => noop_filter_map_expr(expr, self), - } - } - - fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { - let (style, mut stmts) = match stmt.kind { - ast::StmtKind::Mac(mac) => (mac.1, self.remove(stmt.id).make_stmts()), - _ => return noop_flat_map_stmt(stmt, self), - }; - - if style == ast::MacStmtStyle::Semicolon { - if let Some(stmt) = stmts.pop() { - stmts.push(stmt.add_trailing_semicolon()); - } - } - - stmts - } - - fn visit_pat(&mut self, pat: &mut P) { - match pat.kind { - ast::PatKind::Mac(_) => *pat = self.remove(pat.id).make_pat(), - _ => noop_visit_pat(pat, self), - } - } - - fn visit_ty(&mut self, ty: &mut P) { - match ty.kind { - ast::TyKind::Mac(_) => *ty = self.remove(ty.id).make_ty(), - _ => noop_visit_ty(ty, self), - } - } - - fn visit_block(&mut self, block: &mut P) { - noop_visit_block(block, self); - - for stmt in block.stmts.iter_mut() { - if self.monotonic { - assert_eq!(stmt.id, ast::DUMMY_NODE_ID); - stmt.id = self.cx.resolver.next_node_id(); - } - } - } - - fn visit_mod(&mut self, module: &mut ast::Mod) { - noop_visit_mod(module, self); - module.items.retain(|item| match item.kind { - ast::ItemKind::Mac(_) if !self.cx.ecfg.keep_macs => false, // remove macro definitions - _ => true, - }); - } - - fn visit_mac(&mut self, _mac: &mut ast::Mac) { - // Do nothing. - } -} diff --git a/src/libsyntax_expand/proc_macro.rs b/src/libsyntax_expand/proc_macro.rs deleted file mode 100644 index 9f42ec13b56..00000000000 --- a/src/libsyntax_expand/proc_macro.rs +++ /dev/null @@ -1,239 +0,0 @@ -use crate::base::{self, *}; -use crate::proc_macro_server; - -use syntax::ast::{self, ItemKind, MetaItemKind, NestedMetaItem}; -use syntax::errors::{Applicability, FatalError}; -use syntax::symbol::sym; -use syntax::token; -use syntax::tokenstream::{self, TokenStream}; - -use rustc_data_structures::sync::Lrc; -use syntax_pos::{Span, DUMMY_SP}; - -const EXEC_STRATEGY: pm::bridge::server::SameThread = pm::bridge::server::SameThread; - -pub struct BangProcMacro { - pub client: pm::bridge::client::Client pm::TokenStream>, -} - -impl base::ProcMacro for BangProcMacro { - fn expand<'cx>( - &self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - input: TokenStream, - ) -> TokenStream { - let server = proc_macro_server::Rustc::new(ecx); - match self.client.run(&EXEC_STRATEGY, server, input) { - Ok(stream) => stream, - Err(e) => { - let msg = "proc macro panicked"; - let mut err = ecx.struct_span_fatal(span, msg); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - - err.emit(); - FatalError.raise(); - } - } - } -} - -pub struct AttrProcMacro { - pub client: pm::bridge::client::Client pm::TokenStream>, -} - -impl base::AttrProcMacro for AttrProcMacro { - fn expand<'cx>( - &self, - ecx: &'cx mut ExtCtxt<'_>, - span: Span, - annotation: TokenStream, - annotated: TokenStream, - ) -> TokenStream { - let server = proc_macro_server::Rustc::new(ecx); - match self.client.run(&EXEC_STRATEGY, server, annotation, annotated) { - Ok(stream) => stream, - Err(e) => { - let msg = "custom attribute panicked"; - let mut err = ecx.struct_span_fatal(span, msg); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - - err.emit(); - FatalError.raise(); - } - } - } -} - -pub struct ProcMacroDerive { - pub client: pm::bridge::client::Client pm::TokenStream>, -} - -impl MultiItemModifier for ProcMacroDerive { - fn expand( - &self, - ecx: &mut ExtCtxt<'_>, - span: Span, - _meta_item: &ast::MetaItem, - item: Annotatable, - ) -> Vec { - let item = match item { - Annotatable::Arm(..) - | Annotatable::Field(..) - | Annotatable::FieldPat(..) - | Annotatable::GenericParam(..) - | Annotatable::Param(..) - | Annotatable::StructField(..) - | Annotatable::Variant(..) => panic!("unexpected annotatable"), - Annotatable::Item(item) => item, - Annotatable::ImplItem(_) - | Annotatable::TraitItem(_) - | Annotatable::ForeignItem(_) - | Annotatable::Stmt(_) - | Annotatable::Expr(_) => { - ecx.span_err( - span, - "proc-macro derives may only be \ - applied to a struct, enum, or union", - ); - return Vec::new(); - } - }; - match item.kind { - ItemKind::Struct(..) | ItemKind::Enum(..) | ItemKind::Union(..) => {} - _ => { - ecx.span_err( - span, - "proc-macro derives may only be \ - applied to a struct, enum, or union", - ); - return Vec::new(); - } - } - - let token = token::Interpolated(Lrc::new(token::NtItem(item))); - let input = tokenstream::TokenTree::token(token, DUMMY_SP).into(); - - let server = proc_macro_server::Rustc::new(ecx); - let stream = match self.client.run(&EXEC_STRATEGY, server, input) { - Ok(stream) => stream, - Err(e) => { - let msg = "proc-macro derive panicked"; - let mut err = ecx.struct_span_fatal(span, msg); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - - err.emit(); - FatalError.raise(); - } - }; - - let error_count_before = ecx.parse_sess.span_diagnostic.err_count(); - let msg = "proc-macro derive produced unparseable tokens"; - - let mut parser = - rustc_parse::stream_to_parser(ecx.parse_sess, stream, Some("proc-macro derive")); - let mut items = vec![]; - - loop { - match parser.parse_item() { - Ok(None) => break, - Ok(Some(item)) => items.push(Annotatable::Item(item)), - Err(mut err) => { - // FIXME: handle this better - err.cancel(); - ecx.struct_span_fatal(span, msg).emit(); - FatalError.raise(); - } - } - } - - // fail if there have been errors emitted - if ecx.parse_sess.span_diagnostic.err_count() > error_count_before { - ecx.struct_span_fatal(span, msg).emit(); - FatalError.raise(); - } - - items - } -} - -crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec) -> Vec { - let mut result = Vec::new(); - attrs.retain(|attr| { - if !attr.has_name(sym::derive) { - return true; - } - - // 1) First let's ensure that it's a meta item. - let nmis = match attr.meta_item_list() { - None => { - cx.struct_span_err(attr.span, "malformed `derive` attribute input") - .span_suggestion( - attr.span, - "missing traits to be derived", - "#[derive(Trait1, Trait2, ...)]".to_owned(), - Applicability::HasPlaceholders, - ) - .emit(); - return false; - } - Some(x) => x, - }; - - let mut error_reported_filter_map = false; - let mut error_reported_map = false; - let traits = nmis - .into_iter() - // 2) Moreover, let's ensure we have a path and not `#[derive("foo")]`. - .filter_map(|nmi| match nmi { - NestedMetaItem::Literal(lit) => { - error_reported_filter_map = true; - cx.struct_span_err(lit.span, "expected path to a trait, found literal") - .help("for example, write `#[derive(Debug)]` for `Debug`") - .emit(); - None - } - NestedMetaItem::MetaItem(mi) => Some(mi), - }) - // 3) Finally, we only accept `#[derive($path_0, $path_1, ..)]` - // but not e.g. `#[derive($path_0 = "value", $path_1(abc))]`. - // In this case we can still at least determine that the user - // wanted this trait to be derived, so let's keep it. - .map(|mi| { - let mut traits_dont_accept = |title, action| { - error_reported_map = true; - let sp = mi.span.with_lo(mi.path.span.hi()); - cx.struct_span_err(sp, title) - .span_suggestion( - sp, - action, - String::new(), - Applicability::MachineApplicable, - ) - .emit(); - }; - match &mi.kind { - MetaItemKind::List(..) => traits_dont_accept( - "traits in `#[derive(...)]` don't accept arguments", - "remove the arguments", - ), - MetaItemKind::NameValue(..) => traits_dont_accept( - "traits in `#[derive(...)]` don't accept values", - "remove the value", - ), - MetaItemKind::Word => {} - } - mi.path - }); - - result.extend(traits); - !error_reported_filter_map && !error_reported_map - }); - result -} diff --git a/src/libsyntax_expand/proc_macro_server.rs b/src/libsyntax_expand/proc_macro_server.rs deleted file mode 100644 index 790e1f0edc0..00000000000 --- a/src/libsyntax_expand/proc_macro_server.rs +++ /dev/null @@ -1,689 +0,0 @@ -use crate::base::ExtCtxt; - -use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str}; -use syntax::ast; -use syntax::print::pprust; -use syntax::sess::ParseSess; -use syntax::token; -use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; -use syntax::util::comments; -use syntax_pos::symbol::{kw, sym, Symbol}; -use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span}; - -use errors::Diagnostic; -use rustc_data_structures::sync::Lrc; - -use pm::bridge::{server, TokenTree}; -use pm::{Delimiter, Level, LineColumn, Spacing}; -use std::ops::Bound; -use std::{ascii, panic}; - -trait FromInternal { - fn from_internal(x: T) -> Self; -} - -trait ToInternal { - fn to_internal(self) -> T; -} - -impl FromInternal for Delimiter { - fn from_internal(delim: token::DelimToken) -> Delimiter { - match delim { - token::Paren => Delimiter::Parenthesis, - token::Brace => Delimiter::Brace, - token::Bracket => Delimiter::Bracket, - token::NoDelim => Delimiter::None, - } - } -} - -impl ToInternal for Delimiter { - fn to_internal(self) -> token::DelimToken { - match self { - Delimiter::Parenthesis => token::Paren, - Delimiter::Brace => token::Brace, - Delimiter::Bracket => token::Bracket, - Delimiter::None => token::NoDelim, - } - } -} - -impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> - for TokenTree -{ - fn from_internal( - ((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec), - ) -> Self { - use syntax::token::*; - - let joint = is_joint == Joint; - let Token { kind, span } = match tree { - tokenstream::TokenTree::Delimited(span, delim, tts) => { - let delimiter = Delimiter::from_internal(delim); - return TokenTree::Group(Group { delimiter, stream: tts.into(), span }); - } - tokenstream::TokenTree::Token(token) => token, - }; - - macro_rules! tt { - ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( - TokenTree::$ty(self::$ty { - $($field $(: $value)*,)+ - span, - }) - ); - ($ty:ident::$method:ident($($value:expr),*)) => ( - TokenTree::$ty(self::$ty::$method($($value,)* span)) - ); - } - macro_rules! op { - ($a:expr) => { - tt!(Punct::new($a, joint)) - }; - ($a:expr, $b:expr) => {{ - stack.push(tt!(Punct::new($b, joint))); - tt!(Punct::new($a, true)) - }}; - ($a:expr, $b:expr, $c:expr) => {{ - stack.push(tt!(Punct::new($c, joint))); - stack.push(tt!(Punct::new($b, true))); - tt!(Punct::new($a, true)) - }}; - } - - match kind { - Eq => op!('='), - Lt => op!('<'), - Le => op!('<', '='), - EqEq => op!('=', '='), - Ne => op!('!', '='), - Ge => op!('>', '='), - Gt => op!('>'), - AndAnd => op!('&', '&'), - OrOr => op!('|', '|'), - Not => op!('!'), - Tilde => op!('~'), - BinOp(Plus) => op!('+'), - BinOp(Minus) => op!('-'), - BinOp(Star) => op!('*'), - BinOp(Slash) => op!('/'), - BinOp(Percent) => op!('%'), - BinOp(Caret) => op!('^'), - BinOp(And) => op!('&'), - BinOp(Or) => op!('|'), - BinOp(Shl) => op!('<', '<'), - BinOp(Shr) => op!('>', '>'), - BinOpEq(Plus) => op!('+', '='), - BinOpEq(Minus) => op!('-', '='), - BinOpEq(Star) => op!('*', '='), - BinOpEq(Slash) => op!('/', '='), - BinOpEq(Percent) => op!('%', '='), - BinOpEq(Caret) => op!('^', '='), - BinOpEq(And) => op!('&', '='), - BinOpEq(Or) => op!('|', '='), - BinOpEq(Shl) => op!('<', '<', '='), - BinOpEq(Shr) => op!('>', '>', '='), - At => op!('@'), - Dot => op!('.'), - DotDot => op!('.', '.'), - DotDotDot => op!('.', '.', '.'), - DotDotEq => op!('.', '.', '='), - Comma => op!(','), - Semi => op!(';'), - Colon => op!(':'), - ModSep => op!(':', ':'), - RArrow => op!('-', '>'), - LArrow => op!('<', '-'), - FatArrow => op!('=', '>'), - Pound => op!('#'), - Dollar => op!('$'), - Question => op!('?'), - SingleQuote => op!('\''), - - Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), - Ident(name, is_raw) => tt!(Ident::new(name, is_raw)), - Lifetime(name) => { - let ident = ast::Ident::new(name, span).without_first_quote(); - stack.push(tt!(Ident::new(ident.name, false))); - tt!(Punct::new('\'', true)) - } - Literal(lit) => tt!(Literal { lit }), - DocComment(c) => { - let style = comments::doc_comment_style(&c.as_str()); - let stripped = comments::strip_doc_comment_decoration(&c.as_str()); - let mut escaped = String::new(); - for ch in stripped.chars() { - escaped.extend(ch.escape_debug()); - } - let stream = vec![ - Ident(sym::doc, false), - Eq, - TokenKind::lit(token::Str, Symbol::intern(&escaped), None), - ] - .into_iter() - .map(|kind| tokenstream::TokenTree::token(kind, span)) - .collect(); - stack.push(TokenTree::Group(Group { - delimiter: Delimiter::Bracket, - stream, - span: DelimSpan::from_single(span), - })); - if style == ast::AttrStyle::Inner { - stack.push(tt!(Punct::new('!', false))); - } - tt!(Punct::new('#', false)) - } - - Interpolated(nt) => { - let stream = nt_to_tokenstream(&nt, sess, span); - TokenTree::Group(Group { - delimiter: Delimiter::None, - stream, - span: DelimSpan::from_single(span), - }) - } - - OpenDelim(..) | CloseDelim(..) => unreachable!(), - Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(), - } - } -} - -impl ToInternal for TokenTree { - fn to_internal(self) -> TokenStream { - use syntax::token::*; - - let (ch, joint, span) = match self { - TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), - TokenTree::Group(Group { delimiter, stream, span }) => { - return tokenstream::TokenTree::Delimited( - span, - delimiter.to_internal(), - stream.into(), - ) - .into(); - } - TokenTree::Ident(self::Ident { sym, is_raw, span }) => { - return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); - } - TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Integer, symbol, suffix }, - span, - }) if symbol.as_str().starts_with("-") => { - let minus = BinOp(BinOpToken::Minus); - let symbol = Symbol::intern(&symbol.as_str()[1..]); - let integer = TokenKind::lit(token::Integer, symbol, suffix); - let a = tokenstream::TokenTree::token(minus, span); - let b = tokenstream::TokenTree::token(integer, span); - return vec![a, b].into_iter().collect(); - } - TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Float, symbol, suffix }, - span, - }) if symbol.as_str().starts_with("-") => { - let minus = BinOp(BinOpToken::Minus); - let symbol = Symbol::intern(&symbol.as_str()[1..]); - let float = TokenKind::lit(token::Float, symbol, suffix); - let a = tokenstream::TokenTree::token(minus, span); - let b = tokenstream::TokenTree::token(float, span); - return vec![a, b].into_iter().collect(); - } - TokenTree::Literal(self::Literal { lit, span }) => { - return tokenstream::TokenTree::token(Literal(lit), span).into(); - } - }; - - let kind = match ch { - '=' => Eq, - '<' => Lt, - '>' => Gt, - '!' => Not, - '~' => Tilde, - '+' => BinOp(Plus), - '-' => BinOp(Minus), - '*' => BinOp(Star), - '/' => BinOp(Slash), - '%' => BinOp(Percent), - '^' => BinOp(Caret), - '&' => BinOp(And), - '|' => BinOp(Or), - '@' => At, - '.' => Dot, - ',' => Comma, - ';' => Semi, - ':' => Colon, - '#' => Pound, - '$' => Dollar, - '?' => Question, - '\'' => SingleQuote, - _ => unreachable!(), - }; - - let tree = tokenstream::TokenTree::token(kind, span); - TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })]) - } -} - -impl ToInternal for Level { - fn to_internal(self) -> errors::Level { - match self { - Level::Error => errors::Level::Error, - Level::Warning => errors::Level::Warning, - Level::Note => errors::Level::Note, - Level::Help => errors::Level::Help, - _ => unreachable!("unknown proc_macro::Level variant: {:?}", self), - } - } -} - -#[derive(Clone)] -pub struct TokenStreamIter { - cursor: tokenstream::Cursor, - stack: Vec>, -} - -#[derive(Clone)] -pub struct Group { - delimiter: Delimiter, - stream: TokenStream, - span: DelimSpan, -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Punct { - ch: char, - // NB. not using `Spacing` here because it doesn't implement `Hash`. - joint: bool, - span: Span, -} - -impl Punct { - fn new(ch: char, joint: bool, span: Span) -> Punct { - const LEGAL_CHARS: &[char] = &[ - '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', - ':', '#', '$', '?', '\'', - ]; - if !LEGAL_CHARS.contains(&ch) { - panic!("unsupported character `{:?}`", ch) - } - Punct { ch, joint, span } - } -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Ident { - sym: Symbol, - is_raw: bool, - span: Span, -} - -impl Ident { - fn is_valid(string: &str) -> bool { - let mut chars = string.chars(); - if let Some(start) = chars.next() { - rustc_lexer::is_id_start(start) && chars.all(rustc_lexer::is_id_continue) - } else { - false - } - } - fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident { - let string = sym.as_str(); - if !Self::is_valid(&string) { - panic!("`{:?}` is not a valid identifier", string) - } - if is_raw && !sym.can_be_raw() { - panic!("`{}` cannot be a raw identifier", string); - } - Ident { sym, is_raw, span } - } - fn dollar_crate(span: Span) -> Ident { - // `$crate` is accepted as an ident only if it comes from the compiler. - Ident { sym: kw::DollarCrate, is_raw: false, span } - } -} - -// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. -#[derive(Clone, Debug)] -pub struct Literal { - lit: token::Lit, - span: Span, -} - -pub(crate) struct Rustc<'a> { - sess: &'a ParseSess, - def_site: Span, - call_site: Span, - mixed_site: Span, -} - -impl<'a> Rustc<'a> { - pub fn new(cx: &'a ExtCtxt<'_>) -> Self { - let expn_data = cx.current_expansion.id.expn_data(); - Rustc { - sess: cx.parse_sess, - def_site: cx.with_def_site_ctxt(expn_data.def_site), - call_site: cx.with_call_site_ctxt(expn_data.call_site), - mixed_site: cx.with_mixed_site_ctxt(expn_data.call_site), - } - } - - fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option) -> Literal { - Literal { lit: token::Lit::new(kind, symbol, suffix), span: server::Span::call_site(self) } - } -} - -impl server::Types for Rustc<'_> { - type TokenStream = TokenStream; - type TokenStreamBuilder = tokenstream::TokenStreamBuilder; - type TokenStreamIter = TokenStreamIter; - type Group = Group; - type Punct = Punct; - type Ident = Ident; - type Literal = Literal; - type SourceFile = Lrc; - type MultiSpan = Vec; - type Diagnostic = Diagnostic; - type Span = Span; -} - -impl server::TokenStream for Rustc<'_> { - fn new(&mut self) -> Self::TokenStream { - TokenStream::default() - } - fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { - stream.is_empty() - } - fn from_str(&mut self, src: &str) -> Self::TokenStream { - parse_stream_from_source_str( - FileName::proc_macro_source_code(src), - src.to_string(), - self.sess, - Some(self.call_site), - ) - } - fn to_string(&mut self, stream: &Self::TokenStream) -> String { - pprust::tts_to_string(stream.clone()) - } - fn from_token_tree( - &mut self, - tree: TokenTree, - ) -> Self::TokenStream { - tree.to_internal() - } - fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { - TokenStreamIter { cursor: stream.trees(), stack: vec![] } - } -} - -impl server::TokenStreamBuilder for Rustc<'_> { - fn new(&mut self) -> Self::TokenStreamBuilder { - tokenstream::TokenStreamBuilder::new() - } - fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) { - builder.push(stream); - } - fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream { - builder.build() - } -} - -impl server::TokenStreamIter for Rustc<'_> { - fn next( - &mut self, - iter: &mut Self::TokenStreamIter, - ) -> Option> { - loop { - let tree = iter.stack.pop().or_else(|| { - let next = iter.cursor.next_with_joint()?; - Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) - })?; - // HACK: The condition "dummy span + group with empty delimiter" represents an AST - // fragment approximately converted into a token stream. This may happen, for - // example, with inputs to proc macro attributes, including derives. Such "groups" - // need to flattened during iteration over stream's token trees. - // Eventually this needs to be removed in favor of keeping original token trees - // and not doing the roundtrip through AST. - if let TokenTree::Group(ref group) = tree { - if group.delimiter == Delimiter::None && group.span.entire().is_dummy() { - iter.cursor.append(group.stream.clone()); - continue; - } - } - return Some(tree); - } - } -} - -impl server::Group for Rustc<'_> { - fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group { - Group { delimiter, stream, span: DelimSpan::from_single(server::Span::call_site(self)) } - } - fn delimiter(&mut self, group: &Self::Group) -> Delimiter { - group.delimiter - } - fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { - group.stream.clone() - } - fn span(&mut self, group: &Self::Group) -> Self::Span { - group.span.entire() - } - fn span_open(&mut self, group: &Self::Group) -> Self::Span { - group.span.open - } - fn span_close(&mut self, group: &Self::Group) -> Self::Span { - group.span.close - } - fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { - group.span = DelimSpan::from_single(span); - } -} - -impl server::Punct for Rustc<'_> { - fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { - Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) - } - fn as_char(&mut self, punct: Self::Punct) -> char { - punct.ch - } - fn spacing(&mut self, punct: Self::Punct) -> Spacing { - if punct.joint { Spacing::Joint } else { Spacing::Alone } - } - fn span(&mut self, punct: Self::Punct) -> Self::Span { - punct.span - } - fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { - Punct { span, ..punct } - } -} - -impl server::Ident for Rustc<'_> { - fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { - Ident::new(Symbol::intern(string), is_raw, span) - } - fn span(&mut self, ident: Self::Ident) -> Self::Span { - ident.span - } - fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { - Ident { span, ..ident } - } -} - -impl server::Literal for Rustc<'_> { - // FIXME(eddyb) `Literal` should not expose internal `Debug` impls. - fn debug(&mut self, literal: &Self::Literal) -> String { - format!("{:?}", literal) - } - fn integer(&mut self, n: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), None) - } - fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) - } - fn float(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), None) - } - fn f32(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) - } - fn f64(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) - } - fn string(&mut self, string: &str) -> Self::Literal { - let mut escaped = String::new(); - for ch in string.chars() { - escaped.extend(ch.escape_debug()); - } - self.lit(token::Str, Symbol::intern(&escaped), None) - } - fn character(&mut self, ch: char) -> Self::Literal { - let mut escaped = String::new(); - escaped.extend(ch.escape_unicode()); - self.lit(token::Char, Symbol::intern(&escaped), None) - } - fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { - let string = bytes - .iter() - .cloned() - .flat_map(ascii::escape_default) - .map(Into::::into) - .collect::(); - self.lit(token::ByteStr, Symbol::intern(&string), None) - } - fn span(&mut self, literal: &Self::Literal) -> Self::Span { - literal.span - } - fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { - literal.span = span; - } - fn subspan( - &mut self, - literal: &Self::Literal, - start: Bound, - end: Bound, - ) -> Option { - let span = literal.span; - let length = span.hi().to_usize() - span.lo().to_usize(); - - let start = match start { - Bound::Included(lo) => lo, - Bound::Excluded(lo) => lo + 1, - Bound::Unbounded => 0, - }; - - let end = match end { - Bound::Included(hi) => hi + 1, - Bound::Excluded(hi) => hi, - Bound::Unbounded => length, - }; - - // Bounds check the values, preventing addition overflow and OOB spans. - if start > u32::max_value() as usize - || end > u32::max_value() as usize - || (u32::max_value() - start as u32) < span.lo().to_u32() - || (u32::max_value() - end as u32) < span.lo().to_u32() - || start >= end - || end > length - { - return None; - } - - let new_lo = span.lo() + BytePos::from_usize(start); - let new_hi = span.lo() + BytePos::from_usize(end); - Some(span.with_lo(new_lo).with_hi(new_hi)) - } -} - -impl server::SourceFile for Rustc<'_> { - fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { - Lrc::ptr_eq(file1, file2) - } - fn path(&mut self, file: &Self::SourceFile) -> String { - match file.name { - FileName::Real(ref path) => path - .to_str() - .expect("non-UTF8 file path in `proc_macro::SourceFile::path`") - .to_string(), - _ => file.name.to_string(), - } - } - fn is_real(&mut self, file: &Self::SourceFile) -> bool { - file.is_real_file() - } -} - -impl server::MultiSpan for Rustc<'_> { - fn new(&mut self) -> Self::MultiSpan { - vec![] - } - fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) { - spans.push(span) - } -} - -impl server::Diagnostic for Rustc<'_> { - fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic { - let mut diag = Diagnostic::new(level.to_internal(), msg); - diag.set_span(MultiSpan::from_spans(spans)); - diag - } - fn sub( - &mut self, - diag: &mut Self::Diagnostic, - level: Level, - msg: &str, - spans: Self::MultiSpan, - ) { - diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None); - } - fn emit(&mut self, diag: Self::Diagnostic) { - self.sess.span_diagnostic.emit_diagnostic(&diag); - } -} - -impl server::Span for Rustc<'_> { - fn debug(&mut self, span: Self::Span) -> String { - format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) - } - fn def_site(&mut self) -> Self::Span { - self.def_site - } - fn call_site(&mut self) -> Self::Span { - self.call_site - } - fn mixed_site(&mut self) -> Self::Span { - self.mixed_site - } - fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { - self.sess.source_map().lookup_char_pos(span.lo()).file - } - fn parent(&mut self, span: Self::Span) -> Option { - span.parent() - } - fn source(&mut self, span: Self::Span) -> Self::Span { - span.source_callsite() - } - fn start(&mut self, span: Self::Span) -> LineColumn { - let loc = self.sess.source_map().lookup_char_pos(span.lo()); - LineColumn { line: loc.line, column: loc.col.to_usize() } - } - fn end(&mut self, span: Self::Span) -> LineColumn { - let loc = self.sess.source_map().lookup_char_pos(span.hi()); - LineColumn { line: loc.line, column: loc.col.to_usize() } - } - fn join(&mut self, first: Self::Span, second: Self::Span) -> Option { - let self_loc = self.sess.source_map().lookup_char_pos(first.lo()); - let other_loc = self.sess.source_map().lookup_char_pos(second.lo()); - - if self_loc.file.name != other_loc.file.name { - return None; - } - - Some(first.to(second)) - } - fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { - span.with_ctxt(at.ctxt()) - } - fn source_text(&mut self, span: Self::Span) -> Option { - self.sess.source_map().span_to_snippet(span).ok() - } -} diff --git a/src/libsyntax_expand/tests.rs b/src/libsyntax_expand/tests.rs deleted file mode 100644 index 4f5ff97e48d..00000000000 --- a/src/libsyntax_expand/tests.rs +++ /dev/null @@ -1,1012 +0,0 @@ -use rustc_parse::{new_parser_from_source_str, parser::Parser, source_file_to_stream}; -use syntax::ast; -use syntax::sess::ParseSess; -use syntax::source_map::{FilePathMapping, SourceMap}; -use syntax::tokenstream::TokenStream; -use syntax::with_default_globals; -use syntax_pos::{BytePos, MultiSpan, Span}; - -use errors::emitter::EmitterWriter; -use errors::{Handler, PResult}; -use rustc_data_structures::sync::Lrc; - -use std::io; -use std::io::prelude::*; -use std::iter::Peekable; -use std::path::{Path, PathBuf}; -use std::str; -use std::sync::{Arc, Mutex}; - -/// Map string to parser (via tts). -fn string_to_parser(ps: &ParseSess, source_str: String) -> Parser<'_> { - new_parser_from_source_str(ps, PathBuf::from("bogofile").into(), source_str) -} - -crate fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T -where - F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>, -{ - let mut p = string_to_parser(&ps, s); - let x = f(&mut p).unwrap(); - p.sess.span_diagnostic.abort_if_errors(); - x -} - -/// Maps a string to tts, using a made-up filename. -crate fn string_to_stream(source_str: String) -> TokenStream { - let ps = ParseSess::new(FilePathMapping::empty()); - source_file_to_stream( - &ps, - ps.source_map().new_source_file(PathBuf::from("bogofile").into(), source_str), - None, - ) - .0 -} - -/// Parses a string, returns a crate. -crate fn string_to_crate(source_str: String) -> ast::Crate { - let ps = ParseSess::new(FilePathMapping::empty()); - with_error_checking_parse(source_str, &ps, |p| p.parse_crate_mod()) -} - -/// Does the given string match the pattern? whitespace in the first string -/// may be deleted or replaced with other whitespace to match the pattern. -/// This function is relatively Unicode-ignorant; fortunately, the careful design -/// of UTF-8 mitigates this ignorance. It doesn't do NKF-normalization(?). -crate fn matches_codepattern(a: &str, b: &str) -> bool { - let mut a_iter = a.chars().peekable(); - let mut b_iter = b.chars().peekable(); - - loop { - let (a, b) = match (a_iter.peek(), b_iter.peek()) { - (None, None) => return true, - (None, _) => return false, - (Some(&a), None) => { - if rustc_lexer::is_whitespace(a) { - break; // Trailing whitespace check is out of loop for borrowck. - } else { - return false; - } - } - (Some(&a), Some(&b)) => (a, b), - }; - - if rustc_lexer::is_whitespace(a) && rustc_lexer::is_whitespace(b) { - // Skip whitespace for `a` and `b`. - scan_for_non_ws_or_end(&mut a_iter); - scan_for_non_ws_or_end(&mut b_iter); - } else if rustc_lexer::is_whitespace(a) { - // Skip whitespace for `a`. - scan_for_non_ws_or_end(&mut a_iter); - } else if a == b { - a_iter.next(); - b_iter.next(); - } else { - return false; - } - } - - // Check if a has *only* trailing whitespace. - a_iter.all(rustc_lexer::is_whitespace) -} - -/// Advances the given peekable `Iterator` until it reaches a non-whitespace character. -fn scan_for_non_ws_or_end>(iter: &mut Peekable) { - while iter.peek().copied().map(|c| rustc_lexer::is_whitespace(c)) == Some(true) { - iter.next(); - } -} - -/// Identifies a position in the text by the n'th occurrence of a string. -struct Position { - string: &'static str, - count: usize, -} - -struct SpanLabel { - start: Position, - end: Position, - label: &'static str, -} - -crate struct Shared { - pub data: Arc>, -} - -impl Write for Shared { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.data.lock().unwrap().write(buf) - } - - fn flush(&mut self) -> io::Result<()> { - self.data.lock().unwrap().flush() - } -} - -fn test_harness(file_text: &str, span_labels: Vec, expected_output: &str) { - with_default_globals(|| { - let output = Arc::new(Mutex::new(Vec::new())); - - let source_map = Lrc::new(SourceMap::new(FilePathMapping::empty())); - source_map.new_source_file(Path::new("test.rs").to_owned().into(), file_text.to_owned()); - - let primary_span = make_span(&file_text, &span_labels[0].start, &span_labels[0].end); - let mut msp = MultiSpan::from_span(primary_span); - for span_label in span_labels { - let span = make_span(&file_text, &span_label.start, &span_label.end); - msp.push_span_label(span, span_label.label.to_string()); - println!("span: {:?} label: {:?}", span, span_label.label); - println!("text: {:?}", source_map.span_to_snippet(span)); - } - - let emitter = EmitterWriter::new( - Box::new(Shared { data: output.clone() }), - Some(source_map.clone()), - false, - false, - false, - None, - false, - ); - let handler = Handler::with_emitter(true, None, Box::new(emitter)); - handler.span_err(msp, "foo"); - - assert!( - expected_output.chars().next() == Some('\n'), - "expected output should begin with newline" - ); - let expected_output = &expected_output[1..]; - - let bytes = output.lock().unwrap(); - let actual_output = str::from_utf8(&bytes).unwrap(); - println!("expected output:\n------\n{}------", expected_output); - println!("actual output:\n------\n{}------", actual_output); - - assert!(expected_output == actual_output) - }) -} - -fn make_span(file_text: &str, start: &Position, end: &Position) -> Span { - let start = make_pos(file_text, start); - let end = make_pos(file_text, end) + end.string.len(); // just after matching thing ends - assert!(start <= end); - Span::with_root_ctxt(BytePos(start as u32), BytePos(end as u32)) -} - -fn make_pos(file_text: &str, pos: &Position) -> usize { - let mut remainder = file_text; - let mut offset = 0; - for _ in 0..pos.count { - if let Some(n) = remainder.find(&pos.string) { - offset += n; - remainder = &remainder[n + 1..]; - } else { - panic!("failed to find {} instances of {:?} in {:?}", pos.count, pos.string, file_text); - } - } - offset -} - -#[test] -fn ends_on_col0() { - test_harness( - r#" -fn foo() { -} -"#, - vec![SpanLabel { - start: Position { string: "{", count: 1 }, - end: Position { string: "}", count: 1 }, - label: "test", - }], - r#" -error: foo - --> test.rs:2:10 - | -2 | fn foo() { - | __________^ -3 | | } - | |_^ test - -"#, - ); -} - -#[test] -fn ends_on_col2() { - test_harness( - r#" -fn foo() { - - - } -"#, - vec![SpanLabel { - start: Position { string: "{", count: 1 }, - end: Position { string: "}", count: 1 }, - label: "test", - }], - r#" -error: foo - --> test.rs:2:10 - | -2 | fn foo() { - | __________^ -3 | | -4 | | -5 | | } - | |___^ test - -"#, - ); -} -#[test] -fn non_nested() { - test_harness( - r#" -fn foo() { - X0 Y0 - X1 Y1 - X2 Y2 -} -"#, - vec![ - SpanLabel { - start: Position { string: "X0", count: 1 }, - end: Position { string: "X2", count: 1 }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { string: "Y0", count: 1 }, - end: Position { string: "Y2", count: 1 }, - label: "`Y` is a good letter too", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | X0 Y0 - | ____^__- - | | ___| - | || -4 | || X1 Y1 -5 | || X2 Y2 - | ||____^__- `Y` is a good letter too - | |____| - | `X` is a good letter - -"#, - ); -} - -#[test] -fn nested() { - test_harness( - r#" -fn foo() { - X0 Y0 - Y1 X1 -} -"#, - vec![ - SpanLabel { - start: Position { string: "X0", count: 1 }, - end: Position { string: "X1", count: 1 }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { string: "Y0", count: 1 }, - end: Position { string: "Y1", count: 1 }, - label: "`Y` is a good letter too", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | X0 Y0 - | ____^__- - | | ___| - | || -4 | || Y1 X1 - | ||____-__^ `X` is a good letter - | |_____| - | `Y` is a good letter too - -"#, - ); -} - -#[test] -fn different_overlap() { - test_harness( - r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 - X2 Y2 Z2 - X3 Y3 Z3 -} -"#, - vec![ - SpanLabel { - start: Position { string: "Y0", count: 1 }, - end: Position { string: "X2", count: 1 }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { string: "Z1", count: 1 }, - end: Position { string: "X3", count: 1 }, - label: "`Y` is a good letter too", - }, - ], - r#" -error: foo - --> test.rs:3:6 - | -3 | X0 Y0 Z0 - | ______^ -4 | | X1 Y1 Z1 - | |_________- -5 | || X2 Y2 Z2 - | ||____^ `X` is a good letter -6 | | X3 Y3 Z3 - | |_____- `Y` is a good letter too - -"#, - ); -} - -#[test] -fn triple_overlap() { - test_harness( - r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 - X2 Y2 Z2 -} -"#, - vec![ - SpanLabel { - start: Position { string: "X0", count: 1 }, - end: Position { string: "X2", count: 1 }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { string: "Y0", count: 1 }, - end: Position { string: "Y2", count: 1 }, - label: "`Y` is a good letter too", - }, - SpanLabel { - start: Position { string: "Z0", count: 1 }, - end: Position { string: "Z2", count: 1 }, - label: "`Z` label", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | X0 Y0 Z0 - | _____^__-__- - | | ____|__| - | || ___| - | ||| -4 | ||| X1 Y1 Z1 -5 | ||| X2 Y2 Z2 - | |||____^__-__- `Z` label - | ||____|__| - | |____| `Y` is a good letter too - | `X` is a good letter - -"#, - ); -} - -#[test] -fn triple_exact_overlap() { - test_harness( - r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 - X2 Y2 Z2 -} -"#, - vec![ - SpanLabel { - start: Position { string: "X0", count: 1 }, - end: Position { string: "X2", count: 1 }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { string: "X0", count: 1 }, - end: Position { string: "X2", count: 1 }, - label: "`Y` is a good letter too", - }, - SpanLabel { - start: Position { string: "X0", count: 1 }, - end: Position { string: "X2", count: 1 }, - label: "`Z` label", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | / X0 Y0 Z0 -4 | | X1 Y1 Z1 -5 | | X2 Y2 Z2 - | | ^ - | | | - | | `X` is a good letter - | |____`Y` is a good letter too - | `Z` label - -"#, - ); -} - -#[test] -fn minimum_depth() { - test_harness( - r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 - X2 Y2 Z2 - X3 Y3 Z3 -} -"#, - vec![ - SpanLabel { - start: Position { string: "Y0", count: 1 }, - end: Position { string: "X1", count: 1 }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { string: "Y1", count: 1 }, - end: Position { string: "Z2", count: 1 }, - label: "`Y` is a good letter too", - }, - SpanLabel { - start: Position { string: "X2", count: 1 }, - end: Position { string: "Y3", count: 1 }, - label: "`Z`", - }, - ], - r#" -error: foo - --> test.rs:3:6 - | -3 | X0 Y0 Z0 - | ______^ -4 | | X1 Y1 Z1 - | |____^_- - | ||____| - | | `X` is a good letter -5 | | X2 Y2 Z2 - | |____-______- `Y` is a good letter too - | ____| - | | -6 | | X3 Y3 Z3 - | |________- `Z` - -"#, - ); -} - -#[test] -fn non_overlaping() { - test_harness( - r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 - X2 Y2 Z2 - X3 Y3 Z3 -} -"#, - vec![ - SpanLabel { - start: Position { string: "X0", count: 1 }, - end: Position { string: "X1", count: 1 }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { string: "Y2", count: 1 }, - end: Position { string: "Z3", count: 1 }, - label: "`Y` is a good letter too", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | / X0 Y0 Z0 -4 | | X1 Y1 Z1 - | |____^ `X` is a good letter -5 | X2 Y2 Z2 - | ______- -6 | | X3 Y3 Z3 - | |__________- `Y` is a good letter too - -"#, - ); -} - -#[test] -fn overlaping_start_and_end() { - test_harness( - r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 - X2 Y2 Z2 - X3 Y3 Z3 -} -"#, - vec![ - SpanLabel { - start: Position { string: "Y0", count: 1 }, - end: Position { string: "X1", count: 1 }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { string: "Z1", count: 1 }, - end: Position { string: "Z3", count: 1 }, - label: "`Y` is a good letter too", - }, - ], - r#" -error: foo - --> test.rs:3:6 - | -3 | X0 Y0 Z0 - | ______^ -4 | | X1 Y1 Z1 - | |____^____- - | ||____| - | | `X` is a good letter -5 | | X2 Y2 Z2 -6 | | X3 Y3 Z3 - | |___________- `Y` is a good letter too - -"#, - ); -} - -#[test] -fn multiple_labels_primary_without_message() { - test_harness( - r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { string: "b", count: 1 }, - end: Position { string: "}", count: 1 }, - label: "", - }, - SpanLabel { - start: Position { string: "a", count: 1 }, - end: Position { string: "d", count: 1 }, - label: "`a` is a good letter", - }, - SpanLabel { - start: Position { string: "c", count: 1 }, - end: Position { string: "c", count: 1 }, - label: "", - }, - ], - r#" -error: foo - --> test.rs:3:7 - | -3 | a { b { c } d } - | ----^^^^-^^-- `a` is a good letter - -"#, - ); -} - -#[test] -fn multiple_labels_secondary_without_message() { - test_harness( - r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { string: "a", count: 1 }, - end: Position { string: "d", count: 1 }, - label: "`a` is a good letter", - }, - SpanLabel { - start: Position { string: "b", count: 1 }, - end: Position { string: "}", count: 1 }, - label: "", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | a { b { c } d } - | ^^^^-------^^ `a` is a good letter - -"#, - ); -} - -#[test] -fn multiple_labels_primary_without_message_2() { - test_harness( - r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { string: "b", count: 1 }, - end: Position { string: "}", count: 1 }, - label: "`b` is a good letter", - }, - SpanLabel { - start: Position { string: "a", count: 1 }, - end: Position { string: "d", count: 1 }, - label: "", - }, - SpanLabel { - start: Position { string: "c", count: 1 }, - end: Position { string: "c", count: 1 }, - label: "", - }, - ], - r#" -error: foo - --> test.rs:3:7 - | -3 | a { b { c } d } - | ----^^^^-^^-- - | | - | `b` is a good letter - -"#, - ); -} - -#[test] -fn multiple_labels_secondary_without_message_2() { - test_harness( - r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { string: "a", count: 1 }, - end: Position { string: "d", count: 1 }, - label: "", - }, - SpanLabel { - start: Position { string: "b", count: 1 }, - end: Position { string: "}", count: 1 }, - label: "`b` is a good letter", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | a { b { c } d } - | ^^^^-------^^ - | | - | `b` is a good letter - -"#, - ); -} - -#[test] -fn multiple_labels_secondary_without_message_3() { - test_harness( - r#" -fn foo() { - a bc d -} -"#, - vec![ - SpanLabel { - start: Position { string: "a", count: 1 }, - end: Position { string: "b", count: 1 }, - label: "`a` is a good letter", - }, - SpanLabel { - start: Position { string: "c", count: 1 }, - end: Position { string: "d", count: 1 }, - label: "", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | a bc d - | ^^^^---- - | | - | `a` is a good letter - -"#, - ); -} - -#[test] -fn multiple_labels_without_message() { - test_harness( - r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { string: "a", count: 1 }, - end: Position { string: "d", count: 1 }, - label: "", - }, - SpanLabel { - start: Position { string: "b", count: 1 }, - end: Position { string: "}", count: 1 }, - label: "", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | a { b { c } d } - | ^^^^-------^^ - -"#, - ); -} - -#[test] -fn multiple_labels_without_message_2() { - test_harness( - r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { string: "b", count: 1 }, - end: Position { string: "}", count: 1 }, - label: "", - }, - SpanLabel { - start: Position { string: "a", count: 1 }, - end: Position { string: "d", count: 1 }, - label: "", - }, - SpanLabel { - start: Position { string: "c", count: 1 }, - end: Position { string: "c", count: 1 }, - label: "", - }, - ], - r#" -error: foo - --> test.rs:3:7 - | -3 | a { b { c } d } - | ----^^^^-^^-- - -"#, - ); -} - -#[test] -fn multiple_labels_with_message() { - test_harness( - r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { string: "a", count: 1 }, - end: Position { string: "d", count: 1 }, - label: "`a` is a good letter", - }, - SpanLabel { - start: Position { string: "b", count: 1 }, - end: Position { string: "}", count: 1 }, - label: "`b` is a good letter", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | a { b { c } d } - | ^^^^-------^^ - | | | - | | `b` is a good letter - | `a` is a good letter - -"#, - ); -} - -#[test] -fn single_label_with_message() { - test_harness( - r#" -fn foo() { - a { b { c } d } -} -"#, - vec![SpanLabel { - start: Position { string: "a", count: 1 }, - end: Position { string: "d", count: 1 }, - label: "`a` is a good letter", - }], - r#" -error: foo - --> test.rs:3:3 - | -3 | a { b { c } d } - | ^^^^^^^^^^^^^ `a` is a good letter - -"#, - ); -} - -#[test] -fn single_label_without_message() { - test_harness( - r#" -fn foo() { - a { b { c } d } -} -"#, - vec![SpanLabel { - start: Position { string: "a", count: 1 }, - end: Position { string: "d", count: 1 }, - label: "", - }], - r#" -error: foo - --> test.rs:3:3 - | -3 | a { b { c } d } - | ^^^^^^^^^^^^^ - -"#, - ); -} - -#[test] -fn long_snippet() { - test_harness( - r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 - X2 Y2 Z2 - X3 Y3 Z3 -} -"#, - vec![ - SpanLabel { - start: Position { string: "Y0", count: 1 }, - end: Position { string: "X1", count: 1 }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { string: "Z1", count: 1 }, - end: Position { string: "Z3", count: 1 }, - label: "`Y` is a good letter too", - }, - ], - r#" -error: foo - --> test.rs:3:6 - | -3 | X0 Y0 Z0 - | ______^ -4 | | X1 Y1 Z1 - | |____^____- - | ||____| - | | `X` is a good letter -5 | | 1 -6 | | 2 -7 | | 3 -... | -15 | | X2 Y2 Z2 -16 | | X3 Y3 Z3 - | |___________- `Y` is a good letter too - -"#, - ); -} - -#[test] -fn long_snippet_multiple_spans() { - test_harness( - r#" -fn foo() { - X0 Y0 Z0 -1 -2 -3 - X1 Y1 Z1 -4 -5 -6 - X2 Y2 Z2 -7 -8 -9 -10 - X3 Y3 Z3 -} -"#, - vec![ - SpanLabel { - start: Position { string: "Y0", count: 1 }, - end: Position { string: "Y3", count: 1 }, - label: "`Y` is a good letter", - }, - SpanLabel { - start: Position { string: "Z1", count: 1 }, - end: Position { string: "Z2", count: 1 }, - label: "`Z` is a good letter too", - }, - ], - r#" -error: foo - --> test.rs:3:6 - | -3 | X0 Y0 Z0 - | ______^ -4 | | 1 -5 | | 2 -6 | | 3 -7 | | X1 Y1 Z1 - | |_________- -8 | || 4 -9 | || 5 -10 | || 6 -11 | || X2 Y2 Z2 - | ||__________- `Z` is a good letter too -... | -15 | | 10 -16 | | X3 Y3 Z3 - | |_______^ `Y` is a good letter - -"#, - ); -} diff --git a/src/libsyntax_expand/tokenstream/tests.rs b/src/libsyntax_expand/tokenstream/tests.rs deleted file mode 100644 index e13999320df..00000000000 --- a/src/libsyntax_expand/tokenstream/tests.rs +++ /dev/null @@ -1,110 +0,0 @@ -use crate::tests::string_to_stream; - -use smallvec::smallvec; -use syntax::ast::Name; -use syntax::token; -use syntax::tokenstream::{TokenStream, TokenStreamBuilder, TokenTree}; -use syntax::with_default_globals; -use syntax_pos::{BytePos, Span}; - -fn string_to_ts(string: &str) -> TokenStream { - string_to_stream(string.to_owned()) -} - -fn sp(a: u32, b: u32) -> Span { - Span::with_root_ctxt(BytePos(a), BytePos(b)) -} - -#[test] -fn test_concat() { - with_default_globals(|| { - let test_res = string_to_ts("foo::bar::baz"); - let test_fst = string_to_ts("foo::bar"); - let test_snd = string_to_ts("::baz"); - let eq_res = TokenStream::from_streams(smallvec![test_fst, test_snd]); - assert_eq!(test_res.trees().count(), 5); - assert_eq!(eq_res.trees().count(), 5); - assert_eq!(test_res.eq_unspanned(&eq_res), true); - }) -} - -#[test] -fn test_to_from_bijection() { - with_default_globals(|| { - let test_start = string_to_ts("foo::bar(baz)"); - let test_end = test_start.trees().collect(); - assert_eq!(test_start, test_end) - }) -} - -#[test] -fn test_eq_0() { - with_default_globals(|| { - let test_res = string_to_ts("foo"); - let test_eqs = string_to_ts("foo"); - assert_eq!(test_res, test_eqs) - }) -} - -#[test] -fn test_eq_1() { - with_default_globals(|| { - let test_res = string_to_ts("::bar::baz"); - let test_eqs = string_to_ts("::bar::baz"); - assert_eq!(test_res, test_eqs) - }) -} - -#[test] -fn test_eq_3() { - with_default_globals(|| { - let test_res = string_to_ts(""); - let test_eqs = string_to_ts(""); - assert_eq!(test_res, test_eqs) - }) -} - -#[test] -fn test_diseq_0() { - with_default_globals(|| { - let test_res = string_to_ts("::bar::baz"); - let test_eqs = string_to_ts("bar::baz"); - assert_eq!(test_res == test_eqs, false) - }) -} - -#[test] -fn test_diseq_1() { - with_default_globals(|| { - let test_res = string_to_ts("(bar,baz)"); - let test_eqs = string_to_ts("bar,baz"); - assert_eq!(test_res == test_eqs, false) - }) -} - -#[test] -fn test_is_empty() { - with_default_globals(|| { - let test0: TokenStream = Vec::::new().into_iter().collect(); - let test1: TokenStream = - TokenTree::token(token::Ident(Name::intern("a"), false), sp(0, 1)).into(); - let test2 = string_to_ts("foo(bar::baz)"); - - assert_eq!(test0.is_empty(), true); - assert_eq!(test1.is_empty(), false); - assert_eq!(test2.is_empty(), false); - }) -} - -#[test] -fn test_dotdotdot() { - with_default_globals(|| { - let mut builder = TokenStreamBuilder::new(); - builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint()); - builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint()); - builder.push(TokenTree::token(token::Dot, sp(2, 3))); - let stream = builder.build(); - assert!(stream.eq_unspanned(&string_to_ts("..."))); - assert_eq!(stream.trees().count(), 1); - }) -} diff --git a/src/libsyntax_ext/Cargo.toml b/src/libsyntax_ext/Cargo.toml deleted file mode 100644 index d73a9ea6cdb..00000000000 --- a/src/libsyntax_ext/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -authors = ["The Rust Project Developers"] -name = "syntax_ext" -version = "0.0.0" -edition = "2018" - -[lib] -name = "syntax_ext" -path = "lib.rs" -doctest = false - -[dependencies] -errors = { path = "../librustc_errors", package = "rustc_errors" } -fmt_macros = { path = "../libfmt_macros" } -log = "0.4" -rustc_data_structures = { path = "../librustc_data_structures" } -rustc_feature = { path = "../librustc_feature" } -rustc_parse = { path = "../librustc_parse" } -rustc_target = { path = "../librustc_target" } -smallvec = { version = "1.0", features = ["union", "may_dangle"] } -syntax = { path = "../libsyntax" } -syntax_expand = { path = "../libsyntax_expand" } -syntax_pos = { path = "../libsyntax_pos" } -rustc_error_codes = { path = "../librustc_error_codes" } diff --git a/src/libsyntax_ext/asm.rs b/src/libsyntax_ext/asm.rs deleted file mode 100644 index 324bef9cbb8..00000000000 --- a/src/libsyntax_ext/asm.rs +++ /dev/null @@ -1,289 +0,0 @@ -// Inline assembly support. -// -use State::*; - -use errors::{DiagnosticBuilder, PResult}; -use rustc_parse::parser::Parser; -use syntax::ast::{self, AsmDialect}; -use syntax::ptr::P; -use syntax::symbol::{kw, sym, Symbol}; -use syntax::token::{self, Token}; -use syntax::tokenstream::{self, TokenStream}; -use syntax::{span_err, struct_span_err}; -use syntax_expand::base::*; -use syntax_pos::Span; - -use rustc_error_codes::*; - -enum State { - Asm, - Outputs, - Inputs, - Clobbers, - Options, - StateNone, -} - -impl State { - fn next(&self) -> State { - match *self { - Asm => Outputs, - Outputs => Inputs, - Inputs => Clobbers, - Clobbers => Options, - Options => StateNone, - StateNone => StateNone, - } - } -} - -const OPTIONS: &[Symbol] = &[sym::volatile, sym::alignstack, sym::intel]; - -pub fn expand_asm<'cx>( - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let mut inline_asm = match parse_inline_asm(cx, sp, tts) { - Ok(Some(inline_asm)) => inline_asm, - Ok(None) => return DummyResult::any(sp), - Err(mut err) => { - err.emit(); - return DummyResult::any(sp); - } - }; - - // If there are no outputs, the inline assembly is executed just for its side effects, - // so ensure that it is volatile - if inline_asm.outputs.is_empty() { - inline_asm.volatile = true; - } - - MacEager::expr(P(ast::Expr { - id: ast::DUMMY_NODE_ID, - kind: ast::ExprKind::InlineAsm(P(inline_asm)), - span: cx.with_def_site_ctxt(sp), - attrs: ast::AttrVec::new(), - })) -} - -fn parse_asm_str<'a>(p: &mut Parser<'a>) -> PResult<'a, Symbol> { - match p.parse_str_lit() { - Ok(str_lit) => Ok(str_lit.symbol_unescaped), - Err(opt_lit) => { - let span = opt_lit.map_or(p.token.span, |lit| lit.span); - let mut err = p.sess.span_diagnostic.struct_span_err(span, "expected string literal"); - err.span_label(span, "not a string literal"); - Err(err) - } - } -} - -fn parse_inline_asm<'a>( - cx: &mut ExtCtxt<'a>, - sp: Span, - tts: TokenStream, -) -> Result, DiagnosticBuilder<'a>> { - // Split the tts before the first colon, to avoid `asm!("x": y)` being - // parsed as `asm!(z)` with `z = "x": y` which is type ascription. - let first_colon = tts - .trees() - .position(|tt| match tt { - tokenstream::TokenTree::Token(Token { kind: token::Colon, .. }) - | tokenstream::TokenTree::Token(Token { kind: token::ModSep, .. }) => true, - _ => false, - }) - .unwrap_or(tts.len()); - let mut p = cx.new_parser_from_tts(tts.trees().skip(first_colon).collect()); - let mut asm = kw::Invalid; - let mut asm_str_style = None; - let mut outputs = Vec::new(); - let mut inputs = Vec::new(); - let mut clobs = Vec::new(); - let mut volatile = false; - let mut alignstack = false; - let mut dialect = AsmDialect::Att; - - let mut state = Asm; - - 'statement: loop { - match state { - Asm => { - if asm_str_style.is_some() { - // If we already have a string with instructions, - // ending up in Asm state again is an error. - return Err(struct_span_err!( - cx.parse_sess.span_diagnostic, - sp, - E0660, - "malformed inline assembly" - )); - } - // Nested parser, stop before the first colon (see above). - let mut p2 = cx.new_parser_from_tts(tts.trees().take(first_colon).collect()); - - if p2.token == token::Eof { - let mut err = - cx.struct_span_err(sp, "macro requires a string literal as an argument"); - err.span_label(sp, "string literal required"); - return Err(err); - } - - let expr = p2.parse_expr()?; - let (s, style) = - match expr_to_string(cx, expr, "inline assembly must be a string literal") { - Some((s, st)) => (s, st), - None => return Ok(None), - }; - - // This is most likely malformed. - if p2.token != token::Eof { - let mut extra_tts = p2.parse_all_token_trees()?; - extra_tts.extend(tts.trees().skip(first_colon)); - p = cx.new_parser_from_tts(extra_tts.into_iter().collect()); - } - - asm = s; - asm_str_style = Some(style); - } - Outputs => { - while p.token != token::Eof && p.token != token::Colon && p.token != token::ModSep { - if !outputs.is_empty() { - p.eat(&token::Comma); - } - - let constraint = parse_asm_str(&mut p)?; - - let span = p.prev_span; - - p.expect(&token::OpenDelim(token::Paren))?; - let expr = p.parse_expr()?; - p.expect(&token::CloseDelim(token::Paren))?; - - // Expands a read+write operand into two operands. - // - // Use '+' modifier when you want the same expression - // to be both an input and an output at the same time. - // It's the opposite of '=&' which means that the memory - // cannot be shared with any other operand (usually when - // a register is clobbered early.) - let constraint_str = constraint.as_str(); - let mut ch = constraint_str.chars(); - let output = match ch.next() { - Some('=') => None, - Some('+') => Some(Symbol::intern(&format!("={}", ch.as_str()))), - _ => { - span_err!( - cx, - span, - E0661, - "output operand constraint lacks '=' or '+'" - ); - None - } - }; - - let is_rw = output.is_some(); - let is_indirect = constraint_str.contains("*"); - outputs.push(ast::InlineAsmOutput { - constraint: output.unwrap_or(constraint), - expr, - is_rw, - is_indirect, - }); - } - } - Inputs => { - while p.token != token::Eof && p.token != token::Colon && p.token != token::ModSep { - if !inputs.is_empty() { - p.eat(&token::Comma); - } - - let constraint = parse_asm_str(&mut p)?; - - if constraint.as_str().starts_with("=") { - span_err!(cx, p.prev_span, E0662, "input operand constraint contains '='"); - } else if constraint.as_str().starts_with("+") { - span_err!(cx, p.prev_span, E0663, "input operand constraint contains '+'"); - } - - p.expect(&token::OpenDelim(token::Paren))?; - let input = p.parse_expr()?; - p.expect(&token::CloseDelim(token::Paren))?; - - inputs.push((constraint, input)); - } - } - Clobbers => { - while p.token != token::Eof && p.token != token::Colon && p.token != token::ModSep { - if !clobs.is_empty() { - p.eat(&token::Comma); - } - - let s = parse_asm_str(&mut p)?; - - if OPTIONS.iter().any(|&opt| s == opt) { - cx.span_warn(p.prev_span, "expected a clobber, found an option"); - } else if s.as_str().starts_with("{") || s.as_str().ends_with("}") { - span_err!( - cx, - p.prev_span, - E0664, - "clobber should not be surrounded by braces" - ); - } - - clobs.push(s); - } - } - Options => { - let option = parse_asm_str(&mut p)?; - - if option == sym::volatile { - // Indicates that the inline assembly has side effects - // and must not be optimized out along with its outputs. - volatile = true; - } else if option == sym::alignstack { - alignstack = true; - } else if option == sym::intel { - dialect = AsmDialect::Intel; - } else { - cx.span_warn(p.prev_span, "unrecognized option"); - } - - if p.token == token::Comma { - p.eat(&token::Comma); - } - } - StateNone => (), - } - - loop { - // MOD_SEP is a double colon '::' without space in between. - // When encountered, the state must be advanced twice. - match (&p.token.kind, state.next(), state.next().next()) { - (&token::Colon, StateNone, _) | (&token::ModSep, _, StateNone) => { - p.bump(); - break 'statement; - } - (&token::Colon, st, _) | (&token::ModSep, _, st) => { - p.bump(); - state = st; - } - (&token::Eof, ..) => break 'statement, - _ => break, - } - } - } - - Ok(Some(ast::InlineAsm { - asm, - asm_str_style: asm_str_style.unwrap(), - outputs, - inputs, - clobbers: clobs, - volatile, - alignstack, - dialect, - })) -} diff --git a/src/libsyntax_ext/assert.rs b/src/libsyntax_ext/assert.rs deleted file mode 100644 index 331e9fa61d0..00000000000 --- a/src/libsyntax_ext/assert.rs +++ /dev/null @@ -1,137 +0,0 @@ -use errors::{Applicability, DiagnosticBuilder}; - -use rustc_parse::parser::Parser; -use syntax::ast::{self, *}; -use syntax::print::pprust; -use syntax::ptr::P; -use syntax::symbol::{sym, Symbol}; -use syntax::token::{self, TokenKind}; -use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree}; -use syntax_expand::base::*; -use syntax_pos::{Span, DUMMY_SP}; - -pub fn expand_assert<'cx>( - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let Assert { cond_expr, custom_message } = match parse_assert(cx, sp, tts) { - Ok(assert) => assert, - Err(mut err) => { - err.emit(); - return DummyResult::any(sp); - } - }; - - // `core::panic` and `std::panic` are different macros, so we use call-site - // context to pick up whichever is currently in scope. - let sp = cx.with_call_site_ctxt(sp); - let tokens = custom_message.unwrap_or_else(|| { - TokenStream::from(TokenTree::token( - TokenKind::lit( - token::Str, - Symbol::intern(&format!( - "assertion failed: {}", - pprust::expr_to_string(&cond_expr).escape_debug() - )), - None, - ), - DUMMY_SP, - )) - }); - let args = P(MacArgs::Delimited(DelimSpan::from_single(sp), MacDelimiter::Parenthesis, tokens)); - let panic_call = Mac { - path: Path::from_ident(Ident::new(sym::panic, sp)), - args, - prior_type_ascription: None, - }; - let if_expr = cx.expr_if( - sp, - cx.expr(sp, ExprKind::Unary(UnOp::Not, cond_expr)), - cx.expr(sp, ExprKind::Mac(panic_call)), - None, - ); - MacEager::expr(if_expr) -} - -struct Assert { - cond_expr: P, - custom_message: Option, -} - -fn parse_assert<'a>( - cx: &mut ExtCtxt<'a>, - sp: Span, - stream: TokenStream, -) -> Result> { - let mut parser = cx.new_parser_from_tts(stream); - - if parser.token == token::Eof { - let mut err = cx.struct_span_err(sp, "macro requires a boolean expression as an argument"); - err.span_label(sp, "boolean expression required"); - return Err(err); - } - - let cond_expr = parser.parse_expr()?; - - // Some crates use the `assert!` macro in the following form (note extra semicolon): - // - // assert!( - // my_function(); - // ); - // - // Warn about semicolon and suggest removing it. Eventually, this should be turned into an - // error. - if parser.token == token::Semi { - let mut err = cx.struct_span_warn(sp, "macro requires an expression as an argument"); - err.span_suggestion( - parser.token.span, - "try removing semicolon", - String::new(), - Applicability::MaybeIncorrect, - ); - err.note("this is going to be an error in the future"); - err.emit(); - - parser.bump(); - } - - // Some crates use the `assert!` macro in the following form (note missing comma before - // message): - // - // assert!(true "error message"); - // - // Parse this as an actual message, and suggest inserting a comma. Eventually, this should be - // turned into an error. - let custom_message = - if let token::Literal(token::Lit { kind: token::Str, .. }) = parser.token.kind { - let mut err = cx.struct_span_warn(parser.token.span, "unexpected string literal"); - let comma_span = cx.source_map().next_point(parser.prev_span); - err.span_suggestion_short( - comma_span, - "try adding a comma", - ", ".to_string(), - Applicability::MaybeIncorrect, - ); - err.note("this is going to be an error in the future"); - err.emit(); - - parse_custom_message(&mut parser) - } else if parser.eat(&token::Comma) { - parse_custom_message(&mut parser) - } else { - None - }; - - if parser.token != token::Eof { - parser.expect_one_of(&[], &[])?; - unreachable!(); - } - - Ok(Assert { cond_expr, custom_message }) -} - -fn parse_custom_message(parser: &mut Parser<'_>) -> Option { - let ts = parser.parse_tokens(); - if !ts.is_empty() { Some(ts) } else { None } -} diff --git a/src/libsyntax_ext/cfg.rs b/src/libsyntax_ext/cfg.rs deleted file mode 100644 index 7b1dbcc7762..00000000000 --- a/src/libsyntax_ext/cfg.rs +++ /dev/null @@ -1,54 +0,0 @@ -/// The compiler code necessary to support the cfg! extension, which expands to -/// a literal `true` or `false` based on whether the given cfg matches the -/// current compilation environment. -use errors::DiagnosticBuilder; - -use syntax::ast; -use syntax::attr; -use syntax::token; -use syntax::tokenstream::TokenStream; -use syntax_expand::base::{self, *}; -use syntax_pos::Span; - -pub fn expand_cfg( - cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let sp = cx.with_def_site_ctxt(sp); - - match parse_cfg(cx, sp, tts) { - Ok(cfg) => { - let matches_cfg = attr::cfg_matches(&cfg, cx.parse_sess, cx.ecfg.features); - MacEager::expr(cx.expr_bool(sp, matches_cfg)) - } - Err(mut err) => { - err.emit(); - DummyResult::any(sp) - } - } -} - -fn parse_cfg<'a>( - cx: &mut ExtCtxt<'a>, - sp: Span, - tts: TokenStream, -) -> Result> { - let mut p = cx.new_parser_from_tts(tts); - - if p.token == token::Eof { - let mut err = cx.struct_span_err(sp, "macro requires a cfg-pattern as an argument"); - err.span_label(sp, "cfg-pattern required"); - return Err(err); - } - - let cfg = p.parse_meta_item()?; - - let _ = p.eat(&token::Comma); - - if !p.eat(&token::Eof) { - return Err(cx.struct_span_err(sp, "expected 1 cfg-pattern")); - } - - Ok(cfg) -} diff --git a/src/libsyntax_ext/cmdline_attrs.rs b/src/libsyntax_ext/cmdline_attrs.rs deleted file mode 100644 index 1ce083112a8..00000000000 --- a/src/libsyntax_ext/cmdline_attrs.rs +++ /dev/null @@ -1,30 +0,0 @@ -//! Attributes injected into the crate root from command line using `-Z crate-attr`. - -use syntax::ast::{self, AttrItem, AttrStyle}; -use syntax::attr::mk_attr; -use syntax::sess::ParseSess; -use syntax::token; -use syntax_expand::panictry; -use syntax_pos::FileName; - -pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -> ast::Crate { - for raw_attr in attrs { - let mut parser = rustc_parse::new_parser_from_source_str( - parse_sess, - FileName::cli_crate_attr_source_code(&raw_attr), - raw_attr.clone(), - ); - - let start_span = parser.token.span; - let AttrItem { path, args } = panictry!(parser.parse_attr_item()); - let end_span = parser.token.span; - if parser.token != token::Eof { - parse_sess.span_diagnostic.span_err(start_span.to(end_span), "invalid crate attribute"); - continue; - } - - krate.attrs.push(mk_attr(AttrStyle::Inner, path, args, start_span.to(end_span))); - } - - krate -} diff --git a/src/libsyntax_ext/compile_error.rs b/src/libsyntax_ext/compile_error.rs deleted file mode 100644 index 394259fc67b..00000000000 --- a/src/libsyntax_ext/compile_error.rs +++ /dev/null @@ -1,20 +0,0 @@ -// The compiler code necessary to support the compile_error! extension. - -use syntax::tokenstream::TokenStream; -use syntax_expand::base::{self, *}; -use syntax_pos::Span; - -pub fn expand_compile_error<'cx>( - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let var = match get_single_str_from_tts(cx, sp, tts, "compile_error!") { - None => return DummyResult::any(sp), - Some(v) => v, - }; - - cx.span_err(sp, &var); - - DummyResult::any(sp) -} diff --git a/src/libsyntax_ext/concat.rs b/src/libsyntax_ext/concat.rs deleted file mode 100644 index 0cc8e205ae9..00000000000 --- a/src/libsyntax_ext/concat.rs +++ /dev/null @@ -1,62 +0,0 @@ -use syntax::ast; -use syntax::symbol::Symbol; -use syntax::tokenstream::TokenStream; -use syntax_expand::base::{self, DummyResult}; - -use std::string::String; - -pub fn expand_concat( - cx: &mut base::ExtCtxt<'_>, - sp: syntax_pos::Span, - tts: TokenStream, -) -> Box { - let es = match base::get_exprs_from_tts(cx, sp, tts) { - Some(e) => e, - None => return DummyResult::any(sp), - }; - let mut accumulator = String::new(); - let mut missing_literal = vec![]; - let mut has_errors = false; - for e in es { - match e.kind { - ast::ExprKind::Lit(ref lit) => match lit.kind { - ast::LitKind::Str(ref s, _) | ast::LitKind::Float(ref s, _) => { - accumulator.push_str(&s.as_str()); - } - ast::LitKind::Char(c) => { - accumulator.push(c); - } - ast::LitKind::Int(i, ast::LitIntType::Unsigned(_)) - | ast::LitKind::Int(i, ast::LitIntType::Signed(_)) - | ast::LitKind::Int(i, ast::LitIntType::Unsuffixed) => { - accumulator.push_str(&i.to_string()); - } - ast::LitKind::Bool(b) => { - accumulator.push_str(&b.to_string()); - } - ast::LitKind::Byte(..) | ast::LitKind::ByteStr(..) => { - cx.span_err(e.span, "cannot concatenate a byte string literal"); - } - ast::LitKind::Err(_) => { - has_errors = true; - } - }, - ast::ExprKind::Err => { - has_errors = true; - } - _ => { - missing_literal.push(e.span); - } - } - } - if missing_literal.len() > 0 { - let mut err = cx.struct_span_err(missing_literal, "expected a literal"); - err.note("only literals (like `\"foo\"`, `42` and `3.14`) can be passed to `concat!()`"); - err.emit(); - return DummyResult::any(sp); - } else if has_errors { - return DummyResult::any(sp); - } - let sp = cx.with_def_site_ctxt(sp); - base::MacEager::expr(cx.expr_str(sp, Symbol::intern(&accumulator))) -} diff --git a/src/libsyntax_ext/concat_idents.rs b/src/libsyntax_ext/concat_idents.rs deleted file mode 100644 index d870e858bea..00000000000 --- a/src/libsyntax_ext/concat_idents.rs +++ /dev/null @@ -1,68 +0,0 @@ -use syntax::ast; -use syntax::ptr::P; -use syntax::token::{self, Token}; -use syntax::tokenstream::{TokenStream, TokenTree}; -use syntax_expand::base::{self, *}; -use syntax_pos::symbol::Symbol; -use syntax_pos::Span; - -pub fn expand_concat_idents<'cx>( - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - if tts.is_empty() { - cx.span_err(sp, "concat_idents! takes 1 or more arguments."); - return DummyResult::any(sp); - } - - let mut res_str = String::new(); - for (i, e) in tts.into_trees().enumerate() { - if i & 1 == 1 { - match e { - TokenTree::Token(Token { kind: token::Comma, .. }) => {} - _ => { - cx.span_err(sp, "concat_idents! expecting comma."); - return DummyResult::any(sp); - } - } - } else { - match e { - TokenTree::Token(Token { kind: token::Ident(name, _), .. }) => { - res_str.push_str(&name.as_str()) - } - _ => { - cx.span_err(sp, "concat_idents! requires ident args."); - return DummyResult::any(sp); - } - } - } - } - - let ident = ast::Ident::new(Symbol::intern(&res_str), cx.with_call_site_ctxt(sp)); - - struct ConcatIdentsResult { - ident: ast::Ident, - } - - impl base::MacResult for ConcatIdentsResult { - fn make_expr(self: Box) -> Option> { - Some(P(ast::Expr { - id: ast::DUMMY_NODE_ID, - kind: ast::ExprKind::Path(None, ast::Path::from_ident(self.ident)), - span: self.ident.span, - attrs: ast::AttrVec::new(), - })) - } - - fn make_ty(self: Box) -> Option> { - Some(P(ast::Ty { - id: ast::DUMMY_NODE_ID, - kind: ast::TyKind::Path(None, ast::Path::from_ident(self.ident)), - span: self.ident.span, - })) - } - } - - Box::new(ConcatIdentsResult { ident }) -} diff --git a/src/libsyntax_ext/deriving/bounds.rs b/src/libsyntax_ext/deriving/bounds.rs deleted file mode 100644 index 9793ac1ca08..00000000000 --- a/src/libsyntax_ext/deriving/bounds.rs +++ /dev/null @@ -1,29 +0,0 @@ -use crate::deriving::generic::ty::*; -use crate::deriving::generic::*; -use crate::deriving::path_std; - -use syntax::ast::MetaItem; -use syntax_expand::base::{Annotatable, ExtCtxt}; -use syntax_pos::Span; - -pub fn expand_deriving_copy( - cx: &mut ExtCtxt<'_>, - span: Span, - mitem: &MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable), -) { - let trait_def = TraitDef { - span, - attributes: Vec::new(), - path: path_std!(cx, marker::Copy), - additional_bounds: Vec::new(), - generics: LifetimeBounds::empty(), - is_unsafe: false, - supports_unions: true, - methods: Vec::new(), - associated_types: Vec::new(), - }; - - trait_def.expand(cx, mitem, item, push); -} diff --git a/src/libsyntax_ext/deriving/clone.rs b/src/libsyntax_ext/deriving/clone.rs deleted file mode 100644 index 171e4104c0a..00000000000 --- a/src/libsyntax_ext/deriving/clone.rs +++ /dev/null @@ -1,225 +0,0 @@ -use crate::deriving::generic::ty::*; -use crate::deriving::generic::*; -use crate::deriving::path_std; - -use syntax::ast::{self, Expr, GenericArg, Generics, ItemKind, MetaItem, VariantData}; -use syntax::ptr::P; -use syntax::symbol::{kw, sym, Symbol}; -use syntax_expand::base::{Annotatable, ExtCtxt}; -use syntax_pos::Span; - -pub fn expand_deriving_clone( - cx: &mut ExtCtxt<'_>, - span: Span, - mitem: &MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable), -) { - // check if we can use a short form - // - // the short form is `fn clone(&self) -> Self { *self }` - // - // we can use the short form if: - // - the item is Copy (unfortunately, all we can check is whether it's also deriving Copy) - // - there are no generic parameters (after specialization this limitation can be removed) - // if we used the short form with generics, we'd have to bound the generics with - // Clone + Copy, and then there'd be no Clone impl at all if the user fills in something - // that is Clone but not Copy. and until specialization we can't write both impls. - // - the item is a union with Copy fields - // Unions with generic parameters still can derive Clone because they require Copy - // for deriving, Clone alone is not enough. - // Whever Clone is implemented for fields is irrelevant so we don't assert it. - let bounds; - let substructure; - let is_shallow; - match *item { - Annotatable::Item(ref annitem) => match annitem.kind { - ItemKind::Struct(_, Generics { ref params, .. }) - | ItemKind::Enum(_, Generics { ref params, .. }) => { - let container_id = cx.current_expansion.id.expn_data().parent; - if cx.resolver.has_derive_copy(container_id) - && !params.iter().any(|param| match param.kind { - ast::GenericParamKind::Type { .. } => true, - _ => false, - }) - { - bounds = vec![]; - is_shallow = true; - substructure = combine_substructure(Box::new(|c, s, sub| { - cs_clone_shallow("Clone", c, s, sub, false) - })); - } else { - bounds = vec![]; - is_shallow = false; - substructure = - combine_substructure(Box::new(|c, s, sub| cs_clone("Clone", c, s, sub))); - } - } - ItemKind::Union(..) => { - bounds = vec![Literal(path_std!(cx, marker::Copy))]; - is_shallow = true; - substructure = combine_substructure(Box::new(|c, s, sub| { - cs_clone_shallow("Clone", c, s, sub, true) - })); - } - _ => { - bounds = vec![]; - is_shallow = false; - substructure = - combine_substructure(Box::new(|c, s, sub| cs_clone("Clone", c, s, sub))); - } - }, - - _ => cx.span_bug(span, "`#[derive(Clone)]` on trait item or impl item"), - } - - let inline = cx.meta_word(span, sym::inline); - let attrs = vec![cx.attribute(inline)]; - let trait_def = TraitDef { - span, - attributes: Vec::new(), - path: path_std!(cx, clone::Clone), - additional_bounds: bounds, - generics: LifetimeBounds::empty(), - is_unsafe: false, - supports_unions: true, - methods: vec![MethodDef { - name: "clone", - generics: LifetimeBounds::empty(), - explicit_self: borrowed_explicit_self(), - args: Vec::new(), - ret_ty: Self_, - attributes: attrs, - is_unsafe: false, - unify_fieldless_variants: false, - combine_substructure: substructure, - }], - associated_types: Vec::new(), - }; - - trait_def.expand_ext(cx, mitem, item, push, is_shallow) -} - -fn cs_clone_shallow( - name: &str, - cx: &mut ExtCtxt<'_>, - trait_span: Span, - substr: &Substructure<'_>, - is_union: bool, -) -> P { - fn assert_ty_bounds( - cx: &mut ExtCtxt<'_>, - stmts: &mut Vec, - ty: P, - span: Span, - helper_name: &str, - ) { - // Generate statement `let _: helper_name;`, - // set the expn ID so we can use the unstable struct. - let span = cx.with_def_site_ctxt(span); - let assert_path = cx.path_all( - span, - true, - cx.std_path(&[sym::clone, Symbol::intern(helper_name)]), - vec![GenericArg::Type(ty)], - ); - stmts.push(cx.stmt_let_type_only(span, cx.ty_path(assert_path))); - } - fn process_variant(cx: &mut ExtCtxt<'_>, stmts: &mut Vec, variant: &VariantData) { - for field in variant.fields() { - // let _: AssertParamIsClone; - assert_ty_bounds(cx, stmts, field.ty.clone(), field.span, "AssertParamIsClone"); - } - } - - let mut stmts = Vec::new(); - if is_union { - // let _: AssertParamIsCopy; - let self_ty = - cx.ty_path(cx.path_ident(trait_span, ast::Ident::with_dummy_span(kw::SelfUpper))); - assert_ty_bounds(cx, &mut stmts, self_ty, trait_span, "AssertParamIsCopy"); - } else { - match *substr.fields { - StaticStruct(vdata, ..) => { - process_variant(cx, &mut stmts, vdata); - } - StaticEnum(enum_def, ..) => { - for variant in &enum_def.variants { - process_variant(cx, &mut stmts, &variant.data); - } - } - _ => cx.span_bug( - trait_span, - &format!( - "unexpected substructure in \ - shallow `derive({})`", - name - ), - ), - } - } - stmts.push(cx.stmt_expr(cx.expr_deref(trait_span, cx.expr_self(trait_span)))); - cx.expr_block(cx.block(trait_span, stmts)) -} - -fn cs_clone( - name: &str, - cx: &mut ExtCtxt<'_>, - trait_span: Span, - substr: &Substructure<'_>, -) -> P { - let ctor_path; - let all_fields; - let fn_path = cx.std_path(&[sym::clone, sym::Clone, sym::clone]); - let subcall = |cx: &mut ExtCtxt<'_>, field: &FieldInfo<'_>| { - let args = vec![cx.expr_addr_of(field.span, field.self_.clone())]; - cx.expr_call_global(field.span, fn_path.clone(), args) - }; - - let vdata; - match *substr.fields { - Struct(vdata_, ref af) => { - ctor_path = cx.path(trait_span, vec![substr.type_ident]); - all_fields = af; - vdata = vdata_; - } - EnumMatching(.., variant, ref af) => { - ctor_path = cx.path(trait_span, vec![substr.type_ident, variant.ident]); - all_fields = af; - vdata = &variant.data; - } - EnumNonMatchingCollapsed(..) => { - cx.span_bug(trait_span, &format!("non-matching enum variants in `derive({})`", name,)) - } - StaticEnum(..) | StaticStruct(..) => { - cx.span_bug(trait_span, &format!("associated function in `derive({})`", name)) - } - } - - match *vdata { - VariantData::Struct(..) => { - let fields = all_fields - .iter() - .map(|field| { - let ident = match field.name { - Some(i) => i, - None => cx.span_bug( - trait_span, - &format!("unnamed field in normal struct in `derive({})`", name,), - ), - }; - let call = subcall(cx, field); - cx.field_imm(field.span, ident, call) - }) - .collect::>(); - - cx.expr_struct(trait_span, ctor_path, fields) - } - VariantData::Tuple(..) => { - let subcalls = all_fields.iter().map(|f| subcall(cx, f)).collect(); - let path = cx.expr_path(ctor_path); - cx.expr_call(trait_span, path, subcalls) - } - VariantData::Unit(..) => cx.expr_path(ctor_path), - } -} diff --git a/src/libsyntax_ext/deriving/cmp/eq.rs b/src/libsyntax_ext/deriving/cmp/eq.rs deleted file mode 100644 index f292ec0e428..00000000000 --- a/src/libsyntax_ext/deriving/cmp/eq.rs +++ /dev/null @@ -1,104 +0,0 @@ -use crate::deriving::generic::ty::*; -use crate::deriving::generic::*; -use crate::deriving::path_std; - -use syntax::ast::{self, Expr, GenericArg, Ident, MetaItem}; -use syntax::ptr::P; -use syntax::symbol::{sym, Symbol}; -use syntax_expand::base::{Annotatable, ExtCtxt}; -use syntax_pos::Span; - -pub fn expand_deriving_eq( - cx: &mut ExtCtxt<'_>, - span: Span, - mitem: &MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable), -) { - let inline = cx.meta_word(span, sym::inline); - let hidden = syntax::attr::mk_nested_word_item(Ident::new(sym::hidden, span)); - let doc = syntax::attr::mk_list_item(Ident::new(sym::doc, span), vec![hidden]); - let attrs = vec![cx.attribute(inline), cx.attribute(doc)]; - let trait_def = TraitDef { - span, - attributes: Vec::new(), - path: path_std!(cx, cmp::Eq), - additional_bounds: Vec::new(), - generics: LifetimeBounds::empty(), - is_unsafe: false, - supports_unions: true, - methods: vec![MethodDef { - name: "assert_receiver_is_total_eq", - generics: LifetimeBounds::empty(), - explicit_self: borrowed_explicit_self(), - args: vec![], - ret_ty: nil_ty(), - attributes: attrs, - is_unsafe: false, - unify_fieldless_variants: true, - combine_substructure: combine_substructure(Box::new(|a, b, c| { - cs_total_eq_assert(a, b, c) - })), - }], - associated_types: Vec::new(), - }; - - super::inject_impl_of_structural_trait( - cx, - span, - item, - path_std!(cx, marker::StructuralEq), - push, - ); - - trait_def.expand_ext(cx, mitem, item, push, true) -} - -fn cs_total_eq_assert( - cx: &mut ExtCtxt<'_>, - trait_span: Span, - substr: &Substructure<'_>, -) -> P { - fn assert_ty_bounds( - cx: &mut ExtCtxt<'_>, - stmts: &mut Vec, - ty: P, - span: Span, - helper_name: &str, - ) { - // Generate statement `let _: helper_name;`, - // set the expn ID so we can use the unstable struct. - let span = cx.with_def_site_ctxt(span); - let assert_path = cx.path_all( - span, - true, - cx.std_path(&[sym::cmp, Symbol::intern(helper_name)]), - vec![GenericArg::Type(ty)], - ); - stmts.push(cx.stmt_let_type_only(span, cx.ty_path(assert_path))); - } - fn process_variant( - cx: &mut ExtCtxt<'_>, - stmts: &mut Vec, - variant: &ast::VariantData, - ) { - for field in variant.fields() { - // let _: AssertParamIsEq; - assert_ty_bounds(cx, stmts, field.ty.clone(), field.span, "AssertParamIsEq"); - } - } - - let mut stmts = Vec::new(); - match *substr.fields { - StaticStruct(vdata, ..) => { - process_variant(cx, &mut stmts, vdata); - } - StaticEnum(enum_def, ..) => { - for variant in &enum_def.variants { - process_variant(cx, &mut stmts, &variant.data); - } - } - _ => cx.span_bug(trait_span, "unexpected substructure in `derive(Eq)`"), - } - cx.expr_block(cx.block(trait_span, stmts)) -} diff --git a/src/libsyntax_ext/deriving/cmp/ord.rs b/src/libsyntax_ext/deriving/cmp/ord.rs deleted file mode 100644 index e009763da1b..00000000000 --- a/src/libsyntax_ext/deriving/cmp/ord.rs +++ /dev/null @@ -1,113 +0,0 @@ -use crate::deriving::generic::ty::*; -use crate::deriving::generic::*; -use crate::deriving::path_std; - -use syntax::ast::{self, Expr, MetaItem}; -use syntax::ptr::P; -use syntax::symbol::sym; -use syntax_expand::base::{Annotatable, ExtCtxt}; -use syntax_pos::Span; - -pub fn expand_deriving_ord( - cx: &mut ExtCtxt<'_>, - span: Span, - mitem: &MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable), -) { - let inline = cx.meta_word(span, sym::inline); - let attrs = vec![cx.attribute(inline)]; - let trait_def = TraitDef { - span, - attributes: Vec::new(), - path: path_std!(cx, cmp::Ord), - additional_bounds: Vec::new(), - generics: LifetimeBounds::empty(), - is_unsafe: false, - supports_unions: false, - methods: vec![MethodDef { - name: "cmp", - generics: LifetimeBounds::empty(), - explicit_self: borrowed_explicit_self(), - args: vec![(borrowed_self(), "other")], - ret_ty: Literal(path_std!(cx, cmp::Ordering)), - attributes: attrs, - is_unsafe: false, - unify_fieldless_variants: true, - combine_substructure: combine_substructure(Box::new(|a, b, c| cs_cmp(a, b, c))), - }], - associated_types: Vec::new(), - }; - - trait_def.expand(cx, mitem, item, push) -} - -pub fn ordering_collapsed( - cx: &mut ExtCtxt<'_>, - span: Span, - self_arg_tags: &[ast::Ident], -) -> P { - let lft = cx.expr_ident(span, self_arg_tags[0]); - let rgt = cx.expr_addr_of(span, cx.expr_ident(span, self_arg_tags[1])); - cx.expr_method_call(span, lft, ast::Ident::new(sym::cmp, span), vec![rgt]) -} - -pub fn cs_cmp(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> P { - let test_id = ast::Ident::new(sym::cmp, span); - let equals_path = cx.path_global(span, cx.std_path(&[sym::cmp, sym::Ordering, sym::Equal])); - - let cmp_path = cx.std_path(&[sym::cmp, sym::Ord, sym::cmp]); - - // Builds: - // - // match ::std::cmp::Ord::cmp(&self_field1, &other_field1) { - // ::std::cmp::Ordering::Equal => - // match ::std::cmp::Ord::cmp(&self_field2, &other_field2) { - // ::std::cmp::Ordering::Equal => { - // ... - // } - // cmp => cmp - // }, - // cmp => cmp - // } - // - cs_fold( - // foldr nests the if-elses correctly, leaving the first field - // as the outermost one, and the last as the innermost. - false, - |cx, span, old, self_f, other_fs| { - // match new { - // ::std::cmp::Ordering::Equal => old, - // cmp => cmp - // } - - let new = { - let other_f = match other_fs { - [o_f] => o_f, - _ => cx.span_bug(span, "not exactly 2 arguments in `derive(Ord)`"), - }; - - let args = - vec![cx.expr_addr_of(span, self_f), cx.expr_addr_of(span, other_f.clone())]; - - cx.expr_call_global(span, cmp_path.clone(), args) - }; - - let eq_arm = cx.arm(span, cx.pat_path(span, equals_path.clone()), old); - let neq_arm = cx.arm(span, cx.pat_ident(span, test_id), cx.expr_ident(span, test_id)); - - cx.expr_match(span, new, vec![eq_arm, neq_arm]) - }, - cx.expr_path(equals_path.clone()), - Box::new(|cx, span, (self_args, tag_tuple), _non_self_args| { - if self_args.len() != 2 { - cx.span_bug(span, "not exactly 2 arguments in `derive(Ord)`") - } else { - ordering_collapsed(cx, span, tag_tuple) - } - }), - cx, - span, - substr, - ) -} diff --git a/src/libsyntax_ext/deriving/cmp/partial_eq.rs b/src/libsyntax_ext/deriving/cmp/partial_eq.rs deleted file mode 100644 index 91c13b76a00..00000000000 --- a/src/libsyntax_ext/deriving/cmp/partial_eq.rs +++ /dev/null @@ -1,112 +0,0 @@ -use crate::deriving::generic::ty::*; -use crate::deriving::generic::*; -use crate::deriving::{path_local, path_std}; - -use syntax::ast::{BinOpKind, Expr, MetaItem}; -use syntax::ptr::P; -use syntax::symbol::sym; -use syntax_expand::base::{Annotatable, ExtCtxt}; -use syntax_pos::Span; - -pub fn expand_deriving_partial_eq( - cx: &mut ExtCtxt<'_>, - span: Span, - mitem: &MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable), -) { - // structures are equal if all fields are equal, and non equal, if - // any fields are not equal or if the enum variants are different - fn cs_op( - cx: &mut ExtCtxt<'_>, - span: Span, - substr: &Substructure<'_>, - op: BinOpKind, - combiner: BinOpKind, - base: bool, - ) -> P { - let op = |cx: &mut ExtCtxt<'_>, span: Span, self_f: P, other_fs: &[P]| { - let other_f = match other_fs { - [o_f] => o_f, - _ => cx.span_bug(span, "not exactly 2 arguments in `derive(PartialEq)`"), - }; - - cx.expr_binary(span, op, self_f, other_f.clone()) - }; - - cs_fold1( - true, // use foldl - |cx, span, subexpr, self_f, other_fs| { - let eq = op(cx, span, self_f, other_fs); - cx.expr_binary(span, combiner, subexpr, eq) - }, - |cx, args| { - match args { - Some((span, self_f, other_fs)) => { - // Special-case the base case to generate cleaner code. - op(cx, span, self_f, other_fs) - } - None => cx.expr_bool(span, base), - } - }, - Box::new(|cx, span, _, _| cx.expr_bool(span, !base)), - cx, - span, - substr, - ) - } - - fn cs_eq(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> P { - cs_op(cx, span, substr, BinOpKind::Eq, BinOpKind::And, true) - } - fn cs_ne(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> P { - cs_op(cx, span, substr, BinOpKind::Ne, BinOpKind::Or, false) - } - - macro_rules! md { - ($name:expr, $f:ident) => {{ - let inline = cx.meta_word(span, sym::inline); - let attrs = vec![cx.attribute(inline)]; - MethodDef { - name: $name, - generics: LifetimeBounds::empty(), - explicit_self: borrowed_explicit_self(), - args: vec![(borrowed_self(), "other")], - ret_ty: Literal(path_local!(bool)), - attributes: attrs, - is_unsafe: false, - unify_fieldless_variants: true, - combine_substructure: combine_substructure(Box::new(|a, b, c| $f(a, b, c))), - } - }}; - } - - super::inject_impl_of_structural_trait( - cx, - span, - item, - path_std!(cx, marker::StructuralPartialEq), - push, - ); - - // avoid defining `ne` if we can - // c-like enums, enums without any fields and structs without fields - // can safely define only `eq`. - let mut methods = vec![md!("eq", cs_eq)]; - if !is_type_without_fields(item) { - methods.push(md!("ne", cs_ne)); - } - - let trait_def = TraitDef { - span, - attributes: Vec::new(), - path: path_std!(cx, cmp::PartialEq), - additional_bounds: Vec::new(), - generics: LifetimeBounds::empty(), - is_unsafe: false, - supports_unions: false, - methods, - associated_types: Vec::new(), - }; - trait_def.expand(cx, mitem, item, push) -} diff --git a/src/libsyntax_ext/deriving/cmp/partial_ord.rs b/src/libsyntax_ext/deriving/cmp/partial_ord.rs deleted file mode 100644 index 760ed325f36..00000000000 --- a/src/libsyntax_ext/deriving/cmp/partial_ord.rs +++ /dev/null @@ -1,302 +0,0 @@ -pub use OrderingOp::*; - -use crate::deriving::generic::ty::*; -use crate::deriving::generic::*; -use crate::deriving::{path_local, path_std, pathvec_std}; - -use syntax::ast::{self, BinOpKind, Expr, MetaItem}; -use syntax::ptr::P; -use syntax::symbol::{sym, Symbol}; -use syntax_expand::base::{Annotatable, ExtCtxt}; -use syntax_pos::Span; - -pub fn expand_deriving_partial_ord( - cx: &mut ExtCtxt<'_>, - span: Span, - mitem: &MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable), -) { - macro_rules! md { - ($name:expr, $op:expr, $equal:expr) => {{ - let inline = cx.meta_word(span, sym::inline); - let attrs = vec![cx.attribute(inline)]; - MethodDef { - name: $name, - generics: LifetimeBounds::empty(), - explicit_self: borrowed_explicit_self(), - args: vec![(borrowed_self(), "other")], - ret_ty: Literal(path_local!(bool)), - attributes: attrs, - is_unsafe: false, - unify_fieldless_variants: true, - combine_substructure: combine_substructure(Box::new(|cx, span, substr| { - cs_op($op, $equal, cx, span, substr) - })), - } - }}; - } - - let ordering_ty = Literal(path_std!(cx, cmp::Ordering)); - let ret_ty = Literal(Path::new_( - pathvec_std!(cx, option::Option), - None, - vec![Box::new(ordering_ty)], - PathKind::Std, - )); - - let inline = cx.meta_word(span, sym::inline); - let attrs = vec![cx.attribute(inline)]; - - let partial_cmp_def = MethodDef { - name: "partial_cmp", - generics: LifetimeBounds::empty(), - explicit_self: borrowed_explicit_self(), - args: vec![(borrowed_self(), "other")], - ret_ty, - attributes: attrs, - is_unsafe: false, - unify_fieldless_variants: true, - combine_substructure: combine_substructure(Box::new(|cx, span, substr| { - cs_partial_cmp(cx, span, substr) - })), - }; - - // avoid defining extra methods if we can - // c-like enums, enums without any fields and structs without fields - // can safely define only `partial_cmp`. - let methods = if is_type_without_fields(item) { - vec![partial_cmp_def] - } else { - vec![ - partial_cmp_def, - md!("lt", true, false), - md!("le", true, true), - md!("gt", false, false), - md!("ge", false, true), - ] - }; - - let trait_def = TraitDef { - span, - attributes: vec![], - path: path_std!(cx, cmp::PartialOrd), - additional_bounds: vec![], - generics: LifetimeBounds::empty(), - is_unsafe: false, - supports_unions: false, - methods, - associated_types: Vec::new(), - }; - trait_def.expand(cx, mitem, item, push) -} - -#[derive(Copy, Clone)] -pub enum OrderingOp { - PartialCmpOp, - LtOp, - LeOp, - GtOp, - GeOp, -} - -pub fn some_ordering_collapsed( - cx: &mut ExtCtxt<'_>, - span: Span, - op: OrderingOp, - self_arg_tags: &[ast::Ident], -) -> P { - let lft = cx.expr_ident(span, self_arg_tags[0]); - let rgt = cx.expr_addr_of(span, cx.expr_ident(span, self_arg_tags[1])); - let op_str = match op { - PartialCmpOp => "partial_cmp", - LtOp => "lt", - LeOp => "le", - GtOp => "gt", - GeOp => "ge", - }; - cx.expr_method_call(span, lft, cx.ident_of(op_str, span), vec![rgt]) -} - -pub fn cs_partial_cmp(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> P { - let test_id = ast::Ident::new(sym::cmp, span); - let ordering = cx.path_global(span, cx.std_path(&[sym::cmp, sym::Ordering, sym::Equal])); - let ordering_expr = cx.expr_path(ordering.clone()); - let equals_expr = cx.expr_some(span, ordering_expr); - - let partial_cmp_path = cx.std_path(&[sym::cmp, sym::PartialOrd, sym::partial_cmp]); - - // Builds: - // - // match ::std::cmp::PartialOrd::partial_cmp(&self_field1, &other_field1) { - // ::std::option::Option::Some(::std::cmp::Ordering::Equal) => - // match ::std::cmp::PartialOrd::partial_cmp(&self_field2, &other_field2) { - // ::std::option::Option::Some(::std::cmp::Ordering::Equal) => { - // ... - // } - // cmp => cmp - // }, - // cmp => cmp - // } - // - cs_fold( - // foldr nests the if-elses correctly, leaving the first field - // as the outermost one, and the last as the innermost. - false, - |cx, span, old, self_f, other_fs| { - // match new { - // Some(::std::cmp::Ordering::Equal) => old, - // cmp => cmp - // } - - let new = { - let other_f = match other_fs { - [o_f] => o_f, - _ => cx.span_bug(span, "not exactly 2 arguments in `derive(PartialOrd)`"), - }; - - let args = - vec![cx.expr_addr_of(span, self_f), cx.expr_addr_of(span, other_f.clone())]; - - cx.expr_call_global(span, partial_cmp_path.clone(), args) - }; - - let eq_arm = cx.arm(span, cx.pat_some(span, cx.pat_path(span, ordering.clone())), old); - let neq_arm = cx.arm(span, cx.pat_ident(span, test_id), cx.expr_ident(span, test_id)); - - cx.expr_match(span, new, vec![eq_arm, neq_arm]) - }, - equals_expr, - Box::new(|cx, span, (self_args, tag_tuple), _non_self_args| { - if self_args.len() != 2 { - cx.span_bug(span, "not exactly 2 arguments in `derive(PartialOrd)`") - } else { - some_ordering_collapsed(cx, span, PartialCmpOp, tag_tuple) - } - }), - cx, - span, - substr, - ) -} - -/// Strict inequality. -fn cs_op( - less: bool, - inclusive: bool, - cx: &mut ExtCtxt<'_>, - span: Span, - substr: &Substructure<'_>, -) -> P { - let ordering_path = |cx: &mut ExtCtxt<'_>, name: &str| { - cx.expr_path( - cx.path_global(span, cx.std_path(&[sym::cmp, sym::Ordering, Symbol::intern(name)])), - ) - }; - - let par_cmp = |cx: &mut ExtCtxt<'_>, span, self_f: P, other_fs: &[P], default| { - let other_f = match other_fs { - [o_f] => o_f, - _ => cx.span_bug(span, "not exactly 2 arguments in `derive(PartialOrd)`"), - }; - - // `PartialOrd::partial_cmp(self.fi, other.fi)` - let cmp_path = cx.expr_path( - cx.path_global(span, cx.std_path(&[sym::cmp, sym::PartialOrd, sym::partial_cmp])), - ); - let cmp = cx.expr_call( - span, - cmp_path, - vec![cx.expr_addr_of(span, self_f), cx.expr_addr_of(span, other_f.clone())], - ); - - let default = ordering_path(cx, default); - // `Option::unwrap_or(_, Ordering::Equal)` - let unwrap_path = cx.expr_path( - cx.path_global(span, cx.std_path(&[sym::option, sym::Option, sym::unwrap_or])), - ); - cx.expr_call(span, unwrap_path, vec![cmp, default]) - }; - - let fold = cs_fold1( - false, // need foldr - |cx, span, subexpr, self_f, other_fs| { - // build up a series of `partial_cmp`s from the inside - // out (hence foldr) to get lexical ordering, i.e., for op == - // `ast::lt` - // - // ``` - // Ordering::then_with( - // Option::unwrap_or( - // PartialOrd::partial_cmp(self.f1, other.f1), Ordering::Equal) - // ), - // Option::unwrap_or( - // PartialOrd::partial_cmp(self.f2, other.f2), Ordering::Greater) - // ) - // ) - // == Ordering::Less - // ``` - // - // and for op == - // `ast::le` - // - // ``` - // Ordering::then_with( - // Option::unwrap_or( - // PartialOrd::partial_cmp(self.f1, other.f1), Ordering::Equal) - // ), - // Option::unwrap_or( - // PartialOrd::partial_cmp(self.f2, other.f2), Ordering::Greater) - // ) - // ) - // != Ordering::Greater - // ``` - // - // The optimiser should remove the redundancy. We explicitly - // get use the binops to avoid auto-deref dereferencing too many - // layers of pointers, if the type includes pointers. - - // `Option::unwrap_or(PartialOrd::partial_cmp(self.fi, other.fi), Ordering::Equal)` - let par_cmp = par_cmp(cx, span, self_f, other_fs, "Equal"); - - // `Ordering::then_with(Option::unwrap_or(..), ..)` - let then_with_path = cx.expr_path( - cx.path_global(span, cx.std_path(&[sym::cmp, sym::Ordering, sym::then_with])), - ); - cx.expr_call(span, then_with_path, vec![par_cmp, cx.lambda0(span, subexpr)]) - }, - |cx, args| match args { - Some((span, self_f, other_fs)) => { - let opposite = if less { "Greater" } else { "Less" }; - par_cmp(cx, span, self_f, other_fs, opposite) - } - None => cx.expr_bool(span, inclusive), - }, - Box::new(|cx, span, (self_args, tag_tuple), _non_self_args| { - if self_args.len() != 2 { - cx.span_bug(span, "not exactly 2 arguments in `derive(PartialOrd)`") - } else { - let op = match (less, inclusive) { - (false, false) => GtOp, - (false, true) => GeOp, - (true, false) => LtOp, - (true, true) => LeOp, - }; - some_ordering_collapsed(cx, span, op, tag_tuple) - } - }), - cx, - span, - substr, - ); - - match *substr.fields { - EnumMatching(.., ref all_fields) | Struct(.., ref all_fields) if !all_fields.is_empty() => { - let ordering = ordering_path(cx, if less ^ inclusive { "Less" } else { "Greater" }); - let comp_op = if inclusive { BinOpKind::Ne } else { BinOpKind::Eq }; - - cx.expr_binary(span, comp_op, fold, ordering) - } - _ => fold, - } -} diff --git a/src/libsyntax_ext/deriving/debug.rs b/src/libsyntax_ext/deriving/debug.rs deleted file mode 100644 index c145b63274e..00000000000 --- a/src/libsyntax_ext/deriving/debug.rs +++ /dev/null @@ -1,137 +0,0 @@ -use crate::deriving::generic::ty::*; -use crate::deriving::generic::*; -use crate::deriving::path_std; - -use syntax::ast::{self, Ident}; -use syntax::ast::{Expr, MetaItem}; -use syntax::ptr::P; -use syntax::symbol::sym; -use syntax_expand::base::{Annotatable, ExtCtxt}; -use syntax_pos::{Span, DUMMY_SP}; - -pub fn expand_deriving_debug( - cx: &mut ExtCtxt<'_>, - span: Span, - mitem: &MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable), -) { - // &mut ::std::fmt::Formatter - let fmtr = - Ptr(Box::new(Literal(path_std!(cx, fmt::Formatter))), Borrowed(None, ast::Mutability::Mut)); - - let trait_def = TraitDef { - span, - attributes: Vec::new(), - path: path_std!(cx, fmt::Debug), - additional_bounds: Vec::new(), - generics: LifetimeBounds::empty(), - is_unsafe: false, - supports_unions: false, - methods: vec![MethodDef { - name: "fmt", - generics: LifetimeBounds::empty(), - explicit_self: borrowed_explicit_self(), - args: vec![(fmtr, "f")], - ret_ty: Literal(path_std!(cx, fmt::Result)), - attributes: Vec::new(), - is_unsafe: false, - unify_fieldless_variants: false, - combine_substructure: combine_substructure(Box::new(|a, b, c| { - show_substructure(a, b, c) - })), - }], - associated_types: Vec::new(), - }; - trait_def.expand(cx, mitem, item, push) -} - -/// We use the debug builders to do the heavy lifting here -fn show_substructure(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> P { - // build fmt.debug_struct().field(, &)....build() - // or fmt.debug_tuple().field(&)....build() - // based on the "shape". - let (ident, vdata, fields) = match substr.fields { - Struct(vdata, fields) => (substr.type_ident, *vdata, fields), - EnumMatching(_, _, v, fields) => (v.ident, &v.data, fields), - EnumNonMatchingCollapsed(..) | StaticStruct(..) | StaticEnum(..) => { - cx.span_bug(span, "nonsensical .fields in `#[derive(Debug)]`") - } - }; - - // We want to make sure we have the ctxt set so that we can use unstable methods - let span = cx.with_def_site_ctxt(span); - let name = cx.expr_lit(span, ast::LitKind::Str(ident.name, ast::StrStyle::Cooked)); - let builder = cx.ident_of("debug_trait_builder", span); - let builder_expr = cx.expr_ident(span, builder.clone()); - - let fmt = substr.nonself_args[0].clone(); - - let mut stmts = vec![]; - match vdata { - ast::VariantData::Tuple(..) | ast::VariantData::Unit(..) => { - // tuple struct/"normal" variant - let expr = cx.expr_method_call(span, fmt, cx.ident_of("debug_tuple", span), vec![name]); - stmts.push(cx.stmt_let(span, true, builder, expr)); - - for field in fields { - // Use double indirection to make sure this works for unsized types - let field = cx.expr_addr_of(field.span, field.self_.clone()); - let field = cx.expr_addr_of(field.span, field); - - let expr = cx.expr_method_call( - span, - builder_expr.clone(), - Ident::new(sym::field, span), - vec![field], - ); - - // Use `let _ = expr;` to avoid triggering the - // unused_results lint. - stmts.push(stmt_let_undescore(cx, span, expr)); - } - } - ast::VariantData::Struct(..) => { - // normal struct/struct variant - let expr = - cx.expr_method_call(span, fmt, cx.ident_of("debug_struct", span), vec![name]); - stmts.push(cx.stmt_let(DUMMY_SP, true, builder, expr)); - - for field in fields { - let name = cx.expr_lit( - field.span, - ast::LitKind::Str(field.name.unwrap().name, ast::StrStyle::Cooked), - ); - - // Use double indirection to make sure this works for unsized types - let field = cx.expr_addr_of(field.span, field.self_.clone()); - let field = cx.expr_addr_of(field.span, field); - let expr = cx.expr_method_call( - span, - builder_expr.clone(), - Ident::new(sym::field, span), - vec![name, field], - ); - stmts.push(stmt_let_undescore(cx, span, expr)); - } - } - } - - let expr = cx.expr_method_call(span, builder_expr, cx.ident_of("finish", span), vec![]); - - stmts.push(cx.stmt_expr(expr)); - let block = cx.block(span, stmts); - cx.expr_block(block) -} - -fn stmt_let_undescore(cx: &mut ExtCtxt<'_>, sp: Span, expr: P) -> ast::Stmt { - let local = P(ast::Local { - pat: cx.pat_wild(sp), - ty: None, - init: Some(expr), - id: ast::DUMMY_NODE_ID, - span: sp, - attrs: ast::AttrVec::new(), - }); - ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp } -} diff --git a/src/libsyntax_ext/deriving/decodable.rs b/src/libsyntax_ext/deriving/decodable.rs deleted file mode 100644 index 7f21440d49a..00000000000 --- a/src/libsyntax_ext/deriving/decodable.rs +++ /dev/null @@ -1,225 +0,0 @@ -//! The compiler code necessary for `#[derive(RustcDecodable)]`. See encodable.rs for more. - -use crate::deriving::generic::ty::*; -use crate::deriving::generic::*; -use crate::deriving::pathvec_std; - -use syntax::ast; -use syntax::ast::{Expr, MetaItem, Mutability}; -use syntax::ptr::P; -use syntax::symbol::Symbol; -use syntax_expand::base::{Annotatable, ExtCtxt}; -use syntax_pos::Span; - -pub fn expand_deriving_rustc_decodable( - cx: &mut ExtCtxt<'_>, - span: Span, - mitem: &MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable), -) { - let krate = "rustc_serialize"; - let typaram = "__D"; - - let trait_def = TraitDef { - span, - attributes: Vec::new(), - path: Path::new_(vec![krate, "Decodable"], None, vec![], PathKind::Global), - additional_bounds: Vec::new(), - generics: LifetimeBounds::empty(), - is_unsafe: false, - supports_unions: false, - methods: vec![MethodDef { - name: "decode", - generics: LifetimeBounds { - lifetimes: Vec::new(), - bounds: vec![( - typaram, - vec![Path::new_(vec![krate, "Decoder"], None, vec![], PathKind::Global)], - )], - }, - explicit_self: None, - args: vec![( - Ptr(Box::new(Literal(Path::new_local(typaram))), Borrowed(None, Mutability::Mut)), - "d", - )], - ret_ty: Literal(Path::new_( - pathvec_std!(cx, result::Result), - None, - vec![ - Box::new(Self_), - Box::new(Literal(Path::new_( - vec![typaram, "Error"], - None, - vec![], - PathKind::Local, - ))), - ], - PathKind::Std, - )), - attributes: Vec::new(), - is_unsafe: false, - unify_fieldless_variants: false, - combine_substructure: combine_substructure(Box::new(|a, b, c| { - decodable_substructure(a, b, c, krate) - })), - }], - associated_types: Vec::new(), - }; - - trait_def.expand(cx, mitem, item, push) -} - -fn decodable_substructure( - cx: &mut ExtCtxt<'_>, - trait_span: Span, - substr: &Substructure<'_>, - krate: &str, -) -> P { - let decoder = substr.nonself_args[0].clone(); - let recurse = vec![ - cx.ident_of(krate, trait_span), - cx.ident_of("Decodable", trait_span), - cx.ident_of("decode", trait_span), - ]; - let exprdecode = cx.expr_path(cx.path_global(trait_span, recurse)); - // throw an underscore in front to suppress unused variable warnings - let blkarg = cx.ident_of("_d", trait_span); - let blkdecoder = cx.expr_ident(trait_span, blkarg); - - return match *substr.fields { - StaticStruct(_, ref summary) => { - let nfields = match *summary { - Unnamed(ref fields, _) => fields.len(), - Named(ref fields) => fields.len(), - }; - let read_struct_field = cx.ident_of("read_struct_field", trait_span); - - let path = cx.path_ident(trait_span, substr.type_ident); - let result = - decode_static_fields(cx, trait_span, path, summary, |cx, span, name, field| { - cx.expr_try( - span, - cx.expr_method_call( - span, - blkdecoder.clone(), - read_struct_field, - vec![ - cx.expr_str(span, name), - cx.expr_usize(span, field), - exprdecode.clone(), - ], - ), - ) - }); - let result = cx.expr_ok(trait_span, result); - cx.expr_method_call( - trait_span, - decoder, - cx.ident_of("read_struct", trait_span), - vec![ - cx.expr_str(trait_span, substr.type_ident.name), - cx.expr_usize(trait_span, nfields), - cx.lambda1(trait_span, result, blkarg), - ], - ) - } - StaticEnum(_, ref fields) => { - let variant = cx.ident_of("i", trait_span); - - let mut arms = Vec::with_capacity(fields.len() + 1); - let mut variants = Vec::with_capacity(fields.len()); - let rvariant_arg = cx.ident_of("read_enum_variant_arg", trait_span); - - for (i, &(ident, v_span, ref parts)) in fields.iter().enumerate() { - variants.push(cx.expr_str(v_span, ident.name)); - - let path = cx.path(trait_span, vec![substr.type_ident, ident]); - let decoded = - decode_static_fields(cx, v_span, path, parts, |cx, span, _, field| { - let idx = cx.expr_usize(span, field); - cx.expr_try( - span, - cx.expr_method_call( - span, - blkdecoder.clone(), - rvariant_arg, - vec![idx, exprdecode.clone()], - ), - ) - }); - - arms.push(cx.arm(v_span, cx.pat_lit(v_span, cx.expr_usize(v_span, i)), decoded)); - } - - arms.push(cx.arm_unreachable(trait_span)); - - let result = cx.expr_ok( - trait_span, - cx.expr_match(trait_span, cx.expr_ident(trait_span, variant), arms), - ); - let lambda = cx.lambda(trait_span, vec![blkarg, variant], result); - let variant_vec = cx.expr_vec(trait_span, variants); - let variant_vec = cx.expr_addr_of(trait_span, variant_vec); - let result = cx.expr_method_call( - trait_span, - blkdecoder, - cx.ident_of("read_enum_variant", trait_span), - vec![variant_vec, lambda], - ); - cx.expr_method_call( - trait_span, - decoder, - cx.ident_of("read_enum", trait_span), - vec![ - cx.expr_str(trait_span, substr.type_ident.name), - cx.lambda1(trait_span, result, blkarg), - ], - ) - } - _ => cx.bug("expected StaticEnum or StaticStruct in derive(Decodable)"), - }; -} - -/// Creates a decoder for a single enum variant/struct: -/// - `outer_pat_path` is the path to this enum variant/struct -/// - `getarg` should retrieve the `usize`-th field with name `@str`. -fn decode_static_fields( - cx: &mut ExtCtxt<'_>, - trait_span: Span, - outer_pat_path: ast::Path, - fields: &StaticFields, - mut getarg: F, -) -> P -where - F: FnMut(&mut ExtCtxt<'_>, Span, Symbol, usize) -> P, -{ - match *fields { - Unnamed(ref fields, is_tuple) => { - let path_expr = cx.expr_path(outer_pat_path); - if !is_tuple { - path_expr - } else { - let fields = fields - .iter() - .enumerate() - .map(|(i, &span)| getarg(cx, span, Symbol::intern(&format!("_field{}", i)), i)) - .collect(); - - cx.expr_call(trait_span, path_expr, fields) - } - } - Named(ref fields) => { - // use the field's span to get nicer error messages. - let fields = fields - .iter() - .enumerate() - .map(|(i, &(ident, span))| { - let arg = getarg(cx, span, ident.name, i); - cx.field_imm(span, ident, arg) - }) - .collect(); - cx.expr_struct(trait_span, outer_pat_path, fields) - } - } -} diff --git a/src/libsyntax_ext/deriving/default.rs b/src/libsyntax_ext/deriving/default.rs deleted file mode 100644 index d623e1fa4cc..00000000000 --- a/src/libsyntax_ext/deriving/default.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::deriving::generic::ty::*; -use crate::deriving::generic::*; -use crate::deriving::path_std; - -use syntax::ast::{Expr, MetaItem}; -use syntax::ptr::P; -use syntax::span_err; -use syntax::symbol::{kw, sym}; -use syntax_expand::base::{Annotatable, DummyResult, ExtCtxt}; -use syntax_pos::Span; - -use rustc_error_codes::*; - -pub fn expand_deriving_default( - cx: &mut ExtCtxt<'_>, - span: Span, - mitem: &MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable), -) { - let inline = cx.meta_word(span, sym::inline); - let attrs = vec![cx.attribute(inline)]; - let trait_def = TraitDef { - span, - attributes: Vec::new(), - path: path_std!(cx, default::Default), - additional_bounds: Vec::new(), - generics: LifetimeBounds::empty(), - is_unsafe: false, - supports_unions: false, - methods: vec![MethodDef { - name: "default", - generics: LifetimeBounds::empty(), - explicit_self: None, - args: Vec::new(), - ret_ty: Self_, - attributes: attrs, - is_unsafe: false, - unify_fieldless_variants: false, - combine_substructure: combine_substructure(Box::new(|a, b, c| { - default_substructure(a, b, c) - })), - }], - associated_types: Vec::new(), - }; - trait_def.expand(cx, mitem, item, push) -} - -fn default_substructure( - cx: &mut ExtCtxt<'_>, - trait_span: Span, - substr: &Substructure<'_>, -) -> P { - // Note that `kw::Default` is "default" and `sym::Default` is "Default"! - let default_ident = cx.std_path(&[kw::Default, sym::Default, kw::Default]); - let default_call = |span| cx.expr_call_global(span, default_ident.clone(), Vec::new()); - - return match *substr.fields { - StaticStruct(_, ref summary) => match *summary { - Unnamed(ref fields, is_tuple) => { - if !is_tuple { - cx.expr_ident(trait_span, substr.type_ident) - } else { - let exprs = fields.iter().map(|sp| default_call(*sp)).collect(); - cx.expr_call_ident(trait_span, substr.type_ident, exprs) - } - } - Named(ref fields) => { - let default_fields = fields - .iter() - .map(|&(ident, span)| cx.field_imm(span, ident, default_call(span))) - .collect(); - cx.expr_struct_ident(trait_span, substr.type_ident, default_fields) - } - }, - StaticEnum(..) => { - span_err!(cx, trait_span, E0665, "`Default` cannot be derived for enums, only structs"); - // let compilation continue - DummyResult::raw_expr(trait_span, true) - } - _ => cx.span_bug(trait_span, "method in `derive(Default)`"), - }; -} diff --git a/src/libsyntax_ext/deriving/encodable.rs b/src/libsyntax_ext/deriving/encodable.rs deleted file mode 100644 index 98b0160d6e8..00000000000 --- a/src/libsyntax_ext/deriving/encodable.rs +++ /dev/null @@ -1,287 +0,0 @@ -//! The compiler code necessary to implement the `#[derive(RustcEncodable)]` -//! (and `RustcDecodable`, in `decodable.rs`) extension. The idea here is that -//! type-defining items may be tagged with -//! `#[derive(RustcEncodable, RustcDecodable)]`. -//! -//! For example, a type like: -//! -//! ``` -//! #[derive(RustcEncodable, RustcDecodable)] -//! struct Node { id: usize } -//! ``` -//! -//! would generate two implementations like: -//! -//! ``` -//! # struct Node { id: usize } -//! impl, E> Encodable for Node { -//! fn encode(&self, s: &mut S) -> Result<(), E> { -//! s.emit_struct("Node", 1, |this| { -//! this.emit_struct_field("id", 0, |this| { -//! Encodable::encode(&self.id, this) -//! /* this.emit_usize(self.id) can also be used */ -//! }) -//! }) -//! } -//! } -//! -//! impl, E> Decodable for Node { -//! fn decode(d: &mut D) -> Result { -//! d.read_struct("Node", 1, |this| { -//! match this.read_struct_field("id", 0, |this| Decodable::decode(this)) { -//! Ok(id) => Ok(Node { id: id }), -//! Err(e) => Err(e), -//! } -//! }) -//! } -//! } -//! ``` -//! -//! Other interesting scenarios are when the item has type parameters or -//! references other non-built-in types. A type definition like: -//! -//! ``` -//! # #[derive(RustcEncodable, RustcDecodable)] -//! # struct Span; -//! #[derive(RustcEncodable, RustcDecodable)] -//! struct Spanned { node: T, span: Span } -//! ``` -//! -//! would yield functions like: -//! -//! ``` -//! # #[derive(RustcEncodable, RustcDecodable)] -//! # struct Span; -//! # struct Spanned { node: T, span: Span } -//! impl< -//! S: Encoder, -//! E, -//! T: Encodable -//! > Encodable for Spanned { -//! fn encode(&self, s: &mut S) -> Result<(), E> { -//! s.emit_struct("Spanned", 2, |this| { -//! this.emit_struct_field("node", 0, |this| self.node.encode(this)) -//! .unwrap(); -//! this.emit_struct_field("span", 1, |this| self.span.encode(this)) -//! }) -//! } -//! } -//! -//! impl< -//! D: Decoder, -//! E, -//! T: Decodable -//! > Decodable for Spanned { -//! fn decode(d: &mut D) -> Result, E> { -//! d.read_struct("Spanned", 2, |this| { -//! Ok(Spanned { -//! node: this.read_struct_field("node", 0, |this| Decodable::decode(this)) -//! .unwrap(), -//! span: this.read_struct_field("span", 1, |this| Decodable::decode(this)) -//! .unwrap(), -//! }) -//! }) -//! } -//! } -//! ``` - -use crate::deriving::generic::ty::*; -use crate::deriving::generic::*; -use crate::deriving::pathvec_std; - -use syntax::ast::{Expr, ExprKind, MetaItem, Mutability}; -use syntax::ptr::P; -use syntax::symbol::Symbol; -use syntax_expand::base::{Annotatable, ExtCtxt}; -use syntax_pos::Span; - -pub fn expand_deriving_rustc_encodable( - cx: &mut ExtCtxt<'_>, - span: Span, - mitem: &MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable), -) { - let krate = "rustc_serialize"; - let typaram = "__S"; - - let trait_def = TraitDef { - span, - attributes: Vec::new(), - path: Path::new_(vec![krate, "Encodable"], None, vec![], PathKind::Global), - additional_bounds: Vec::new(), - generics: LifetimeBounds::empty(), - is_unsafe: false, - supports_unions: false, - methods: vec![MethodDef { - name: "encode", - generics: LifetimeBounds { - lifetimes: Vec::new(), - bounds: vec![( - typaram, - vec![Path::new_(vec![krate, "Encoder"], None, vec![], PathKind::Global)], - )], - }, - explicit_self: borrowed_explicit_self(), - args: vec![( - Ptr(Box::new(Literal(Path::new_local(typaram))), Borrowed(None, Mutability::Mut)), - "s", - )], - ret_ty: Literal(Path::new_( - pathvec_std!(cx, result::Result), - None, - vec![ - Box::new(Tuple(Vec::new())), - Box::new(Literal(Path::new_( - vec![typaram, "Error"], - None, - vec![], - PathKind::Local, - ))), - ], - PathKind::Std, - )), - attributes: Vec::new(), - is_unsafe: false, - unify_fieldless_variants: false, - combine_substructure: combine_substructure(Box::new(|a, b, c| { - encodable_substructure(a, b, c, krate) - })), - }], - associated_types: Vec::new(), - }; - - trait_def.expand(cx, mitem, item, push) -} - -fn encodable_substructure( - cx: &mut ExtCtxt<'_>, - trait_span: Span, - substr: &Substructure<'_>, - krate: &'static str, -) -> P { - let encoder = substr.nonself_args[0].clone(); - // throw an underscore in front to suppress unused variable warnings - let blkarg = cx.ident_of("_e", trait_span); - let blkencoder = cx.expr_ident(trait_span, blkarg); - let fn_path = cx.expr_path(cx.path_global( - trait_span, - vec![ - cx.ident_of(krate, trait_span), - cx.ident_of("Encodable", trait_span), - cx.ident_of("encode", trait_span), - ], - )); - - return match *substr.fields { - Struct(_, ref fields) => { - let emit_struct_field = cx.ident_of("emit_struct_field", trait_span); - let mut stmts = Vec::new(); - for (i, &FieldInfo { name, ref self_, span, .. }) in fields.iter().enumerate() { - let name = match name { - Some(id) => id.name, - None => Symbol::intern(&format!("_field{}", i)), - }; - let self_ref = cx.expr_addr_of(span, self_.clone()); - let enc = cx.expr_call(span, fn_path.clone(), vec![self_ref, blkencoder.clone()]); - let lambda = cx.lambda1(span, enc, blkarg); - let call = cx.expr_method_call( - span, - blkencoder.clone(), - emit_struct_field, - vec![cx.expr_str(span, name), cx.expr_usize(span, i), lambda], - ); - - // last call doesn't need a try! - let last = fields.len() - 1; - let call = if i != last { - cx.expr_try(span, call) - } else { - cx.expr(span, ExprKind::Ret(Some(call))) - }; - - let stmt = cx.stmt_expr(call); - stmts.push(stmt); - } - - // unit structs have no fields and need to return Ok() - let blk = if stmts.is_empty() { - let ok = cx.expr_ok(trait_span, cx.expr_tuple(trait_span, vec![])); - cx.lambda1(trait_span, ok, blkarg) - } else { - cx.lambda_stmts_1(trait_span, stmts, blkarg) - }; - - cx.expr_method_call( - trait_span, - encoder, - cx.ident_of("emit_struct", trait_span), - vec![ - cx.expr_str(trait_span, substr.type_ident.name), - cx.expr_usize(trait_span, fields.len()), - blk, - ], - ) - } - - EnumMatching(idx, _, variant, ref fields) => { - // We're not generating an AST that the borrow checker is expecting, - // so we need to generate a unique local variable to take the - // mutable loan out on, otherwise we get conflicts which don't - // actually exist. - let me = cx.stmt_let(trait_span, false, blkarg, encoder); - let encoder = cx.expr_ident(trait_span, blkarg); - let emit_variant_arg = cx.ident_of("emit_enum_variant_arg", trait_span); - let mut stmts = Vec::new(); - if !fields.is_empty() { - let last = fields.len() - 1; - for (i, &FieldInfo { ref self_, span, .. }) in fields.iter().enumerate() { - let self_ref = cx.expr_addr_of(span, self_.clone()); - let enc = - cx.expr_call(span, fn_path.clone(), vec![self_ref, blkencoder.clone()]); - let lambda = cx.lambda1(span, enc, blkarg); - let call = cx.expr_method_call( - span, - blkencoder.clone(), - emit_variant_arg, - vec![cx.expr_usize(span, i), lambda], - ); - let call = if i != last { - cx.expr_try(span, call) - } else { - cx.expr(span, ExprKind::Ret(Some(call))) - }; - stmts.push(cx.stmt_expr(call)); - } - } else { - let ok = cx.expr_ok(trait_span, cx.expr_tuple(trait_span, vec![])); - let ret_ok = cx.expr(trait_span, ExprKind::Ret(Some(ok))); - stmts.push(cx.stmt_expr(ret_ok)); - } - - let blk = cx.lambda_stmts_1(trait_span, stmts, blkarg); - let name = cx.expr_str(trait_span, variant.ident.name); - let call = cx.expr_method_call( - trait_span, - blkencoder, - cx.ident_of("emit_enum_variant", trait_span), - vec![ - name, - cx.expr_usize(trait_span, idx), - cx.expr_usize(trait_span, fields.len()), - blk, - ], - ); - let blk = cx.lambda1(trait_span, call, blkarg); - let ret = cx.expr_method_call( - trait_span, - encoder, - cx.ident_of("emit_enum", trait_span), - vec![cx.expr_str(trait_span, substr.type_ident.name), blk], - ); - cx.expr_block(cx.block(trait_span, vec![me, cx.stmt_expr(ret)])) - } - - _ => cx.bug("expected Struct or EnumMatching in derive(Encodable)"), - }; -} diff --git a/src/libsyntax_ext/deriving/generic/mod.rs b/src/libsyntax_ext/deriving/generic/mod.rs deleted file mode 100644 index 7d7b73ebb42..00000000000 --- a/src/libsyntax_ext/deriving/generic/mod.rs +++ /dev/null @@ -1,1812 +0,0 @@ -//! Some code that abstracts away much of the boilerplate of writing -//! `derive` instances for traits. Among other things it manages getting -//! access to the fields of the 4 different sorts of structs and enum -//! variants, as well as creating the method and impl ast instances. -//! -//! Supported features (fairly exhaustive): -//! -//! - Methods taking any number of parameters of any type, and returning -//! any type, other than vectors, bottom and closures. -//! - Generating `impl`s for types with type parameters and lifetimes -//! (e.g., `Option`), the parameters are automatically given the -//! current trait as a bound. (This includes separate type parameters -//! and lifetimes for methods.) -//! - Additional bounds on the type parameters (`TraitDef.additional_bounds`) -//! -//! The most important thing for implementors is the `Substructure` and -//! `SubstructureFields` objects. The latter groups 5 possibilities of the -//! arguments: -//! -//! - `Struct`, when `Self` is a struct (including tuple structs, e.g -//! `struct T(i32, char)`). -//! - `EnumMatching`, when `Self` is an enum and all the arguments are the -//! same variant of the enum (e.g., `Some(1)`, `Some(3)` and `Some(4)`) -//! - `EnumNonMatchingCollapsed` when `Self` is an enum and the arguments -//! are not the same variant (e.g., `None`, `Some(1)` and `None`). -//! - `StaticEnum` and `StaticStruct` for static methods, where the type -//! being derived upon is either an enum or struct respectively. (Any -//! argument with type Self is just grouped among the non-self -//! arguments.) -//! -//! In the first two cases, the values from the corresponding fields in -//! all the arguments are grouped together. For `EnumNonMatchingCollapsed` -//! this isn't possible (different variants have different fields), so the -//! fields are inaccessible. (Previous versions of the deriving infrastructure -//! had a way to expand into code that could access them, at the cost of -//! generating exponential amounts of code; see issue #15375). There are no -//! fields with values in the static cases, so these are treated entirely -//! differently. -//! -//! The non-static cases have `Option` in several places associated -//! with field `expr`s. This represents the name of the field it is -//! associated with. It is only not `None` when the associated field has -//! an identifier in the source code. For example, the `x`s in the -//! following snippet -//! -//! ```rust -//! # #![allow(dead_code)] -//! struct A { x : i32 } -//! -//! struct B(i32); -//! -//! enum C { -//! C0(i32), -//! C1 { x: i32 } -//! } -//! ``` -//! -//! The `i32`s in `B` and `C0` don't have an identifier, so the -//! `Option`s would be `None` for them. -//! -//! In the static cases, the structure is summarized, either into the just -//! spans of the fields or a list of spans and the field idents (for tuple -//! structs and record structs, respectively), or a list of these, for -//! enums (one for each variant). For empty struct and empty enum -//! variants, it is represented as a count of 0. -//! -//! # "`cs`" functions -//! -//! The `cs_...` functions ("combine substructure) are designed to -//! make life easier by providing some pre-made recipes for common -//! threads; mostly calling the function being derived on all the -//! arguments and then combining them back together in some way (or -//! letting the user chose that). They are not meant to be the only -//! way to handle the structures that this code creates. -//! -//! # Examples -//! -//! The following simplified `PartialEq` is used for in-code examples: -//! -//! ```rust -//! trait PartialEq { -//! fn eq(&self, other: &Self) -> bool; -//! } -//! impl PartialEq for i32 { -//! fn eq(&self, other: &i32) -> bool { -//! *self == *other -//! } -//! } -//! ``` -//! -//! Some examples of the values of `SubstructureFields` follow, using the -//! above `PartialEq`, `A`, `B` and `C`. -//! -//! ## Structs -//! -//! When generating the `expr` for the `A` impl, the `SubstructureFields` is -//! -//! ```{.text} -//! Struct(vec![FieldInfo { -//! span: -//! name: Some(), -//! self_: , -//! other: vec![, -//! name: None, -//! self_: -//! other: vec![] -//! }]) -//! ``` -//! -//! ## Enums -//! -//! When generating the `expr` for a call with `self == C0(a)` and `other -//! == C0(b)`, the SubstructureFields is -//! -//! ```{.text} -//! EnumMatching(0, , -//! vec![FieldInfo { -//! span: -//! name: None, -//! self_: , -//! other: vec![] -//! }]) -//! ``` -//! -//! For `C1 {x}` and `C1 {x}`, -//! -//! ```{.text} -//! EnumMatching(1, , -//! vec![FieldInfo { -//! span: -//! name: Some(), -//! self_: , -//! other: vec![] -//! }]) -//! ``` -//! -//! For `C0(a)` and `C1 {x}` , -//! -//! ```{.text} -//! EnumNonMatchingCollapsed( -//! vec![, ], -//! &[, ], -//! &[, ]) -//! ``` -//! -//! It is the same for when the arguments are flipped to `C1 {x}` and -//! `C0(a)`; the only difference is what the values of the identifiers -//! and will -//! be in the generated code. -//! -//! `EnumNonMatchingCollapsed` deliberately provides far less information -//! than is generally available for a given pair of variants; see #15375 -//! for discussion. -//! -//! ## Static -//! -//! A static method on the types above would result in, -//! -//! ```{.text} -//! StaticStruct(, Named(vec![(, )])) -//! -//! StaticStruct(, Unnamed(vec![])) -//! -//! StaticEnum(, -//! vec![(, , Unnamed(vec![])), -//! (, , Named(vec![(, )]))]) -//! ``` - -pub use StaticFields::*; -pub use SubstructureFields::*; - -use std::cell::RefCell; -use std::iter; -use std::vec; - -use syntax::ast::{self, BinOpKind, EnumDef, Expr, Generics, Ident, PatKind}; -use syntax::ast::{GenericArg, GenericParamKind, VariantData}; -use syntax::attr; -use syntax::ptr::P; -use syntax::sess::ParseSess; -use syntax::source_map::respan; -use syntax::symbol::{kw, sym, Symbol}; -use syntax::util::map_in_place::MapInPlace; -use syntax_expand::base::{Annotatable, ExtCtxt}; -use syntax_pos::Span; - -use ty::{LifetimeBounds, Path, Ptr, PtrTy, Self_, Ty}; - -use crate::deriving; - -pub mod ty; - -pub struct TraitDef<'a> { - /// The span for the current #[derive(Foo)] header. - pub span: Span, - - pub attributes: Vec, - - /// Path of the trait, including any type parameters - pub path: Path<'a>, - - /// Additional bounds required of any type parameters of the type, - /// other than the current trait - pub additional_bounds: Vec>, - - /// Any extra lifetimes and/or bounds, e.g., `D: serialize::Decoder` - pub generics: LifetimeBounds<'a>, - - /// Is it an `unsafe` trait? - pub is_unsafe: bool, - - /// Can this trait be derived for unions? - pub supports_unions: bool, - - pub methods: Vec>, - - pub associated_types: Vec<(ast::Ident, Ty<'a>)>, -} - -pub struct MethodDef<'a> { - /// name of the method - pub name: &'a str, - /// List of generics, e.g., `R: rand::Rng` - pub generics: LifetimeBounds<'a>, - - /// Whether there is a self argument (outer Option) i.e., whether - /// this is a static function, and whether it is a pointer (inner - /// Option) - pub explicit_self: Option>, - - /// Arguments other than the self argument - pub args: Vec<(Ty<'a>, &'a str)>, - - /// Returns type - pub ret_ty: Ty<'a>, - - pub attributes: Vec, - - // Is it an `unsafe fn`? - pub is_unsafe: bool, - - /// Can we combine fieldless variants for enums into a single match arm? - pub unify_fieldless_variants: bool, - - pub combine_substructure: RefCell>, -} - -/// All the data about the data structure/method being derived upon. -pub struct Substructure<'a> { - /// ident of self - pub type_ident: Ident, - /// ident of the method - pub method_ident: Ident, - /// dereferenced access to any `Self_` or `Ptr(Self_, _)` arguments - pub self_args: &'a [P], - /// verbatim access to any other arguments - pub nonself_args: &'a [P], - pub fields: &'a SubstructureFields<'a>, -} - -/// Summary of the relevant parts of a struct/enum field. -pub struct FieldInfo<'a> { - pub span: Span, - /// None for tuple structs/normal enum variants, Some for normal - /// structs/struct enum variants. - pub name: Option, - /// The expression corresponding to this field of `self` - /// (specifically, a reference to it). - pub self_: P, - /// The expressions corresponding to references to this field in - /// the other `Self` arguments. - pub other: Vec>, - /// The attributes on the field - pub attrs: &'a [ast::Attribute], -} - -/// Fields for a static method -pub enum StaticFields { - /// Tuple and unit structs/enum variants like this. - Unnamed(Vec, bool /*is tuple*/), - /// Normal structs/struct variants. - Named(Vec<(Ident, Span)>), -} - -/// A summary of the possible sets of fields. -pub enum SubstructureFields<'a> { - Struct(&'a ast::VariantData, Vec>), - /// Matching variants of the enum: variant index, variant count, ast::Variant, - /// fields: the field name is only non-`None` in the case of a struct - /// variant. - EnumMatching(usize, usize, &'a ast::Variant, Vec>), - - /// Non-matching variants of the enum, but with all state hidden from - /// the consequent code. The first component holds `Ident`s for all of - /// the `Self` arguments; the second component is a slice of all of the - /// variants for the enum itself, and the third component is a list of - /// `Ident`s bound to the variant index values for each of the actual - /// input `Self` arguments. - EnumNonMatchingCollapsed(Vec, &'a [ast::Variant], &'a [Ident]), - - /// A static method where `Self` is a struct. - StaticStruct(&'a ast::VariantData, StaticFields), - /// A static method where `Self` is an enum. - StaticEnum(&'a ast::EnumDef, Vec<(Ident, Span, StaticFields)>), -} - -/// Combine the values of all the fields together. The last argument is -/// all the fields of all the structures. -pub type CombineSubstructureFunc<'a> = - Box, Span, &Substructure<'_>) -> P + 'a>; - -/// Deal with non-matching enum variants. The tuple is a list of -/// identifiers (one for each `Self` argument, which could be any of the -/// variants since they have been collapsed together) and the identifiers -/// holding the variant index value for each of the `Self` arguments. The -/// last argument is all the non-`Self` args of the method being derived. -pub type EnumNonMatchCollapsedFunc<'a> = - Box, Span, (&[Ident], &[Ident]), &[P]) -> P + 'a>; - -pub fn combine_substructure( - f: CombineSubstructureFunc<'_>, -) -> RefCell> { - RefCell::new(f) -} - -/// This method helps to extract all the type parameters referenced from a -/// type. For a type parameter ``, it looks for either a `TyPath` that -/// is not global and starts with `T`, or a `TyQPath`. -fn find_type_parameters( - ty: &ast::Ty, - ty_param_names: &[ast::Name], - cx: &ExtCtxt<'_>, -) -> Vec> { - use syntax::visit; - - struct Visitor<'a, 'b> { - cx: &'a ExtCtxt<'b>, - ty_param_names: &'a [ast::Name], - types: Vec>, - } - - impl<'a, 'b> visit::Visitor<'a> for Visitor<'a, 'b> { - fn visit_ty(&mut self, ty: &'a ast::Ty) { - if let ast::TyKind::Path(_, ref path) = ty.kind { - if let Some(segment) = path.segments.first() { - if self.ty_param_names.contains(&segment.ident.name) { - self.types.push(P(ty.clone())); - } - } - } - - visit::walk_ty(self, ty) - } - - fn visit_mac(&mut self, mac: &ast::Mac) { - self.cx.span_err(mac.span(), "`derive` cannot be used on items with type macros"); - } - } - - let mut visitor = Visitor { cx, ty_param_names, types: Vec::new() }; - visit::Visitor::visit_ty(&mut visitor, ty); - - visitor.types -} - -impl<'a> TraitDef<'a> { - pub fn expand( - self, - cx: &mut ExtCtxt<'_>, - mitem: &ast::MetaItem, - item: &'a Annotatable, - push: &mut dyn FnMut(Annotatable), - ) { - self.expand_ext(cx, mitem, item, push, false); - } - - pub fn expand_ext( - self, - cx: &mut ExtCtxt<'_>, - mitem: &ast::MetaItem, - item: &'a Annotatable, - push: &mut dyn FnMut(Annotatable), - from_scratch: bool, - ) { - match *item { - Annotatable::Item(ref item) => { - let is_packed = item.attrs.iter().any(|attr| { - for r in attr::find_repr_attrs(&cx.parse_sess, attr) { - if let attr::ReprPacked(_) = r { - return true; - } - } - false - }); - let has_no_type_params = match item.kind { - ast::ItemKind::Struct(_, ref generics) - | ast::ItemKind::Enum(_, ref generics) - | ast::ItemKind::Union(_, ref generics) => { - !generics.params.iter().any(|param| match param.kind { - ast::GenericParamKind::Type { .. } => true, - _ => false, - }) - } - _ => { - // Non-ADT derive is an error, but it should have been - // set earlier; see - // libsyntax_expand/expand.rs:MacroExpander::fully_expand_fragment() - // libsyntax_expand/base.rs:Annotatable::derive_allowed() - return; - } - }; - let container_id = cx.current_expansion.id.expn_data().parent; - let always_copy = has_no_type_params && cx.resolver.has_derive_copy(container_id); - let use_temporaries = is_packed && always_copy; - - let newitem = match item.kind { - ast::ItemKind::Struct(ref struct_def, ref generics) => self.expand_struct_def( - cx, - &struct_def, - item.ident, - generics, - from_scratch, - use_temporaries, - ), - ast::ItemKind::Enum(ref enum_def, ref generics) => { - // We ignore `use_temporaries` here, because - // `repr(packed)` enums cause an error later on. - // - // This can only cause further compilation errors - // downstream in blatantly illegal code, so it - // is fine. - self.expand_enum_def( - cx, - enum_def, - &item.attrs, - item.ident, - generics, - from_scratch, - ) - } - ast::ItemKind::Union(ref struct_def, ref generics) => { - if self.supports_unions { - self.expand_struct_def( - cx, - &struct_def, - item.ident, - generics, - from_scratch, - use_temporaries, - ) - } else { - cx.span_err(mitem.span, "this trait cannot be derived for unions"); - return; - } - } - _ => unreachable!(), - }; - // Keep the lint attributes of the previous item to control how the - // generated implementations are linted - let mut attrs = newitem.attrs.clone(); - attrs.extend( - item.attrs - .iter() - .filter(|a| { - [ - sym::allow, - sym::warn, - sym::deny, - sym::forbid, - sym::stable, - sym::unstable, - ] - .contains(&a.name_or_empty()) - }) - .cloned(), - ); - push(Annotatable::Item(P(ast::Item { attrs: attrs, ..(*newitem).clone() }))) - } - _ => { - // Non-Item derive is an error, but it should have been - // set earlier; see - // libsyntax_expand/expand.rs:MacroExpander::fully_expand_fragment() - // libsyntax_expand/base.rs:Annotatable::derive_allowed() - return; - } - } - } - - /// Given that we are deriving a trait `DerivedTrait` for a type like: - /// - /// ```ignore (only-for-syntax-highlight) - /// struct Struct<'a, ..., 'z, A, B: DeclaredTrait, C, ..., Z> where C: WhereTrait { - /// a: A, - /// b: B::Item, - /// b1: ::Item, - /// c1: ::Item, - /// c2: Option<::Item>, - /// ... - /// } - /// ``` - /// - /// create an impl like: - /// - /// ```ignore (only-for-syntax-highlight) - /// impl<'a, ..., 'z, A, B: DeclaredTrait, C, ... Z> where - /// C: WhereTrait, - /// A: DerivedTrait + B1 + ... + BN, - /// B: DerivedTrait + B1 + ... + BN, - /// C: DerivedTrait + B1 + ... + BN, - /// B::Item: DerivedTrait + B1 + ... + BN, - /// ::Item: DerivedTrait + B1 + ... + BN, - /// ... - /// { - /// ... - /// } - /// ``` - /// - /// where B1, ..., BN are the bounds given by `bounds_paths`.'. Z is a phantom type, and - /// therefore does not get bound by the derived trait. - fn create_derived_impl( - &self, - cx: &mut ExtCtxt<'_>, - type_ident: Ident, - generics: &Generics, - field_tys: Vec>, - methods: Vec, - ) -> P { - let trait_path = self.path.to_path(cx, self.span, type_ident, generics); - - // Transform associated types from `deriving::ty::Ty` into `ast::AssocItem` - let associated_types = - self.associated_types.iter().map(|&(ident, ref type_def)| ast::AssocItem { - id: ast::DUMMY_NODE_ID, - span: self.span, - ident, - vis: respan(self.span.shrink_to_lo(), ast::VisibilityKind::Inherited), - defaultness: ast::Defaultness::Final, - attrs: Vec::new(), - generics: Generics::default(), - kind: ast::AssocItemKind::TyAlias( - Vec::new(), - Some(type_def.to_ty(cx, self.span, type_ident, generics)), - ), - tokens: None, - }); - - let Generics { mut params, mut where_clause, span } = - self.generics.to_generics(cx, self.span, type_ident, generics); - - // Create the generic parameters - params.extend(generics.params.iter().map(|param| match param.kind { - GenericParamKind::Lifetime { .. } => param.clone(), - GenericParamKind::Type { .. } => { - // I don't think this can be moved out of the loop, since - // a GenericBound requires an ast id - let bounds: Vec<_> = - // extra restrictions on the generics parameters to the - // type being derived upon - self.additional_bounds.iter().map(|p| { - cx.trait_bound(p.to_path(cx, self.span, type_ident, generics)) - }).chain( - // require the current trait - iter::once(cx.trait_bound(trait_path.clone())) - ).chain( - // also add in any bounds from the declaration - param.bounds.iter().cloned() - ).collect(); - - cx.typaram(self.span, param.ident, vec![], bounds, None) - } - GenericParamKind::Const { .. } => param.clone(), - })); - - // and similarly for where clauses - where_clause.predicates.extend(generics.where_clause.predicates.iter().map(|clause| { - match *clause { - ast::WherePredicate::BoundPredicate(ref wb) => { - ast::WherePredicate::BoundPredicate(ast::WhereBoundPredicate { - span: self.span, - bound_generic_params: wb.bound_generic_params.clone(), - bounded_ty: wb.bounded_ty.clone(), - bounds: wb.bounds.iter().cloned().collect(), - }) - } - ast::WherePredicate::RegionPredicate(ref rb) => { - ast::WherePredicate::RegionPredicate(ast::WhereRegionPredicate { - span: self.span, - lifetime: rb.lifetime, - bounds: rb.bounds.iter().cloned().collect(), - }) - } - ast::WherePredicate::EqPredicate(ref we) => { - ast::WherePredicate::EqPredicate(ast::WhereEqPredicate { - id: ast::DUMMY_NODE_ID, - span: self.span, - lhs_ty: we.lhs_ty.clone(), - rhs_ty: we.rhs_ty.clone(), - }) - } - } - })); - - { - // Extra scope required here so ty_params goes out of scope before params is moved - - let mut ty_params = params - .iter() - .filter_map(|param| match param.kind { - ast::GenericParamKind::Type { .. } => Some(param), - _ => None, - }) - .peekable(); - - if ty_params.peek().is_some() { - let ty_param_names: Vec = - ty_params.map(|ty_param| ty_param.ident.name).collect(); - - for field_ty in field_tys { - let tys = find_type_parameters(&field_ty, &ty_param_names, cx); - - for ty in tys { - // if we have already handled this type, skip it - if let ast::TyKind::Path(_, ref p) = ty.kind { - if p.segments.len() == 1 - && ty_param_names.contains(&p.segments[0].ident.name) - { - continue; - }; - } - let mut bounds: Vec<_> = self - .additional_bounds - .iter() - .map(|p| cx.trait_bound(p.to_path(cx, self.span, type_ident, generics))) - .collect(); - - // require the current trait - bounds.push(cx.trait_bound(trait_path.clone())); - - let predicate = ast::WhereBoundPredicate { - span: self.span, - bound_generic_params: Vec::new(), - bounded_ty: ty, - bounds, - }; - - let predicate = ast::WherePredicate::BoundPredicate(predicate); - where_clause.predicates.push(predicate); - } - } - } - } - - let trait_generics = Generics { params, where_clause, span }; - - // Create the reference to the trait. - let trait_ref = cx.trait_ref(trait_path); - - let self_params: Vec<_> = generics - .params - .iter() - .map(|param| match param.kind { - GenericParamKind::Lifetime { .. } => { - GenericArg::Lifetime(cx.lifetime(self.span, param.ident)) - } - GenericParamKind::Type { .. } => { - GenericArg::Type(cx.ty_ident(self.span, param.ident)) - } - GenericParamKind::Const { .. } => { - GenericArg::Const(cx.const_ident(self.span, param.ident)) - } - }) - .collect(); - - // Create the type of `self`. - let path = cx.path_all(self.span, false, vec![type_ident], self_params); - let self_type = cx.ty_path(path); - - let attr = cx.attribute(cx.meta_word(self.span, sym::automatically_derived)); - // Just mark it now since we know that it'll end up used downstream - attr::mark_used(&attr); - let opt_trait_ref = Some(trait_ref); - let unused_qual = { - let word = syntax::attr::mk_nested_word_item(Ident::new( - Symbol::intern("unused_qualifications"), - self.span, - )); - let list = syntax::attr::mk_list_item(Ident::new(sym::allow, self.span), vec![word]); - cx.attribute(list) - }; - - let mut a = vec![attr, unused_qual]; - a.extend(self.attributes.iter().cloned()); - - let unsafety = if self.is_unsafe { ast::Unsafety::Unsafe } else { ast::Unsafety::Normal }; - - cx.item( - self.span, - Ident::invalid(), - a, - ast::ItemKind::Impl( - unsafety, - ast::ImplPolarity::Positive, - ast::Defaultness::Final, - trait_generics, - opt_trait_ref, - self_type, - methods.into_iter().chain(associated_types).collect(), - ), - ) - } - - fn expand_struct_def( - &self, - cx: &mut ExtCtxt<'_>, - struct_def: &'a VariantData, - type_ident: Ident, - generics: &Generics, - from_scratch: bool, - use_temporaries: bool, - ) -> P { - let field_tys: Vec> = - struct_def.fields().iter().map(|field| field.ty.clone()).collect(); - - let methods = self - .methods - .iter() - .map(|method_def| { - let (explicit_self, self_args, nonself_args, tys) = - method_def.split_self_nonself_args(cx, self, type_ident, generics); - - let body = if from_scratch || method_def.is_static() { - method_def.expand_static_struct_method_body( - cx, - self, - struct_def, - type_ident, - &self_args[..], - &nonself_args[..], - ) - } else { - method_def.expand_struct_method_body( - cx, - self, - struct_def, - type_ident, - &self_args[..], - &nonself_args[..], - use_temporaries, - ) - }; - - method_def.create_method(cx, self, type_ident, generics, explicit_self, tys, body) - }) - .collect(); - - self.create_derived_impl(cx, type_ident, generics, field_tys, methods) - } - - fn expand_enum_def( - &self, - cx: &mut ExtCtxt<'_>, - enum_def: &'a EnumDef, - type_attrs: &[ast::Attribute], - type_ident: Ident, - generics: &Generics, - from_scratch: bool, - ) -> P { - let mut field_tys = Vec::new(); - - for variant in &enum_def.variants { - field_tys.extend(variant.data.fields().iter().map(|field| field.ty.clone())); - } - - let methods = self - .methods - .iter() - .map(|method_def| { - let (explicit_self, self_args, nonself_args, tys) = - method_def.split_self_nonself_args(cx, self, type_ident, generics); - - let body = if from_scratch || method_def.is_static() { - method_def.expand_static_enum_method_body( - cx, - self, - enum_def, - type_ident, - &self_args[..], - &nonself_args[..], - ) - } else { - method_def.expand_enum_method_body( - cx, - self, - enum_def, - type_attrs, - type_ident, - self_args, - &nonself_args[..], - ) - }; - - method_def.create_method(cx, self, type_ident, generics, explicit_self, tys, body) - }) - .collect(); - - self.create_derived_impl(cx, type_ident, generics, field_tys, methods) - } -} - -fn find_repr_type_name(sess: &ParseSess, type_attrs: &[ast::Attribute]) -> &'static str { - let mut repr_type_name = "isize"; - for a in type_attrs { - for r in &attr::find_repr_attrs(sess, a) { - repr_type_name = match *r { - attr::ReprPacked(_) - | attr::ReprSimd - | attr::ReprAlign(_) - | attr::ReprTransparent => continue, - - attr::ReprC => "i32", - - attr::ReprInt(attr::SignedInt(ast::IntTy::Isize)) => "isize", - attr::ReprInt(attr::SignedInt(ast::IntTy::I8)) => "i8", - attr::ReprInt(attr::SignedInt(ast::IntTy::I16)) => "i16", - attr::ReprInt(attr::SignedInt(ast::IntTy::I32)) => "i32", - attr::ReprInt(attr::SignedInt(ast::IntTy::I64)) => "i64", - attr::ReprInt(attr::SignedInt(ast::IntTy::I128)) => "i128", - - attr::ReprInt(attr::UnsignedInt(ast::UintTy::Usize)) => "usize", - attr::ReprInt(attr::UnsignedInt(ast::UintTy::U8)) => "u8", - attr::ReprInt(attr::UnsignedInt(ast::UintTy::U16)) => "u16", - attr::ReprInt(attr::UnsignedInt(ast::UintTy::U32)) => "u32", - attr::ReprInt(attr::UnsignedInt(ast::UintTy::U64)) => "u64", - attr::ReprInt(attr::UnsignedInt(ast::UintTy::U128)) => "u128", - } - } - } - repr_type_name -} - -impl<'a> MethodDef<'a> { - fn call_substructure_method( - &self, - cx: &mut ExtCtxt<'_>, - trait_: &TraitDef<'_>, - type_ident: Ident, - self_args: &[P], - nonself_args: &[P], - fields: &SubstructureFields<'_>, - ) -> P { - let substructure = Substructure { - type_ident, - method_ident: cx.ident_of(self.name, trait_.span), - self_args, - nonself_args, - fields, - }; - let mut f = self.combine_substructure.borrow_mut(); - let f: &mut CombineSubstructureFunc<'_> = &mut *f; - f(cx, trait_.span, &substructure) - } - - fn get_ret_ty( - &self, - cx: &mut ExtCtxt<'_>, - trait_: &TraitDef<'_>, - generics: &Generics, - type_ident: Ident, - ) -> P { - self.ret_ty.to_ty(cx, trait_.span, type_ident, generics) - } - - fn is_static(&self) -> bool { - self.explicit_self.is_none() - } - - fn split_self_nonself_args( - &self, - cx: &mut ExtCtxt<'_>, - trait_: &TraitDef<'_>, - type_ident: Ident, - generics: &Generics, - ) -> (Option, Vec>, Vec>, Vec<(Ident, P)>) { - let mut self_args = Vec::new(); - let mut nonself_args = Vec::new(); - let mut arg_tys = Vec::new(); - let mut nonstatic = false; - - let ast_explicit_self = self.explicit_self.as_ref().map(|self_ptr| { - let (self_expr, explicit_self) = ty::get_explicit_self(cx, trait_.span, self_ptr); - - self_args.push(self_expr); - nonstatic = true; - - explicit_self - }); - - for (ty, name) in self.args.iter() { - let ast_ty = ty.to_ty(cx, trait_.span, type_ident, generics); - let ident = cx.ident_of(name, trait_.span); - arg_tys.push((ident, ast_ty)); - - let arg_expr = cx.expr_ident(trait_.span, ident); - - match *ty { - // for static methods, just treat any Self - // arguments as a normal arg - Self_ if nonstatic => { - self_args.push(arg_expr); - } - Ptr(ref ty, _) if (if let Self_ = **ty { true } else { false }) && nonstatic => { - self_args.push(cx.expr_deref(trait_.span, arg_expr)) - } - _ => { - nonself_args.push(arg_expr); - } - } - } - - (ast_explicit_self, self_args, nonself_args, arg_tys) - } - - fn create_method( - &self, - cx: &mut ExtCtxt<'_>, - trait_: &TraitDef<'_>, - type_ident: Ident, - generics: &Generics, - explicit_self: Option, - arg_types: Vec<(Ident, P)>, - body: P, - ) -> ast::AssocItem { - // Create the generics that aren't for `Self`. - let fn_generics = self.generics.to_generics(cx, trait_.span, type_ident, generics); - - let args = { - let self_args = explicit_self.map(|explicit_self| { - let ident = Ident::with_dummy_span(kw::SelfLower).with_span_pos(trait_.span); - ast::Param::from_self(ast::AttrVec::default(), explicit_self, ident) - }); - let nonself_args = - arg_types.into_iter().map(|(name, ty)| cx.param(trait_.span, name, ty)); - self_args.into_iter().chain(nonself_args).collect() - }; - - let ret_type = self.get_ret_ty(cx, trait_, generics, type_ident); - - let method_ident = cx.ident_of(self.name, trait_.span); - let fn_decl = cx.fn_decl(args, ast::FunctionRetTy::Ty(ret_type)); - let body_block = cx.block_expr(body); - - let unsafety = if self.is_unsafe { ast::Unsafety::Unsafe } else { ast::Unsafety::Normal }; - - let trait_lo_sp = trait_.span.shrink_to_lo(); - - let sig = ast::FnSig { - header: ast::FnHeader { unsafety, ext: ast::Extern::None, ..ast::FnHeader::default() }, - decl: fn_decl, - }; - - // Create the method. - ast::AssocItem { - id: ast::DUMMY_NODE_ID, - attrs: self.attributes.clone(), - generics: fn_generics, - span: trait_.span, - vis: respan(trait_lo_sp, ast::VisibilityKind::Inherited), - defaultness: ast::Defaultness::Final, - ident: method_ident, - kind: ast::AssocItemKind::Fn(sig, Some(body_block)), - tokens: None, - } - } - - /// ``` - /// #[derive(PartialEq)] - /// # struct Dummy; - /// struct A { x: i32, y: i32 } - /// - /// // equivalent to: - /// impl PartialEq for A { - /// fn eq(&self, other: &A) -> bool { - /// match *self { - /// A {x: ref __self_0_0, y: ref __self_0_1} => { - /// match *other { - /// A {x: ref __self_1_0, y: ref __self_1_1} => { - /// __self_0_0.eq(__self_1_0) && __self_0_1.eq(__self_1_1) - /// } - /// } - /// } - /// } - /// } - /// } - /// - /// // or if A is repr(packed) - note fields are matched by-value - /// // instead of by-reference. - /// impl PartialEq for A { - /// fn eq(&self, other: &A) -> bool { - /// match *self { - /// A {x: __self_0_0, y: __self_0_1} => { - /// match other { - /// A {x: __self_1_0, y: __self_1_1} => { - /// __self_0_0.eq(&__self_1_0) && __self_0_1.eq(&__self_1_1) - /// } - /// } - /// } - /// } - /// } - /// } - /// ``` - fn expand_struct_method_body<'b>( - &self, - cx: &mut ExtCtxt<'_>, - trait_: &TraitDef<'b>, - struct_def: &'b VariantData, - type_ident: Ident, - self_args: &[P], - nonself_args: &[P], - use_temporaries: bool, - ) -> P { - let mut raw_fields = Vec::new(); // Vec<[fields of self], - // [fields of next Self arg], [etc]> - let mut patterns = Vec::new(); - for i in 0..self_args.len() { - let struct_path = cx.path(trait_.span, vec![type_ident]); - let (pat, ident_expr) = trait_.create_struct_pattern( - cx, - struct_path, - struct_def, - &format!("__self_{}", i), - ast::Mutability::Not, - use_temporaries, - ); - patterns.push(pat); - raw_fields.push(ident_expr); - } - - // transpose raw_fields - let fields = if !raw_fields.is_empty() { - let mut raw_fields = raw_fields.into_iter().map(|v| v.into_iter()); - let first_field = raw_fields.next().unwrap(); - let mut other_fields: Vec> = raw_fields.collect(); - first_field - .map(|(span, opt_id, field, attrs)| FieldInfo { - span, - name: opt_id, - self_: field, - other: other_fields - .iter_mut() - .map(|l| match l.next().unwrap() { - (.., ex, _) => ex, - }) - .collect(), - attrs, - }) - .collect() - } else { - cx.span_bug(trait_.span, "no `self` parameter for method in generic `derive`") - }; - - // body of the inner most destructuring match - let mut body = self.call_substructure_method( - cx, - trait_, - type_ident, - self_args, - nonself_args, - &Struct(struct_def, fields), - ); - - // make a series of nested matches, to destructure the - // structs. This is actually right-to-left, but it shouldn't - // matter. - for (arg_expr, pat) in self_args.iter().zip(patterns) { - body = cx.expr_match( - trait_.span, - arg_expr.clone(), - vec![cx.arm(trait_.span, pat.clone(), body)], - ) - } - - body - } - - fn expand_static_struct_method_body( - &self, - cx: &mut ExtCtxt<'_>, - trait_: &TraitDef<'_>, - struct_def: &VariantData, - type_ident: Ident, - self_args: &[P], - nonself_args: &[P], - ) -> P { - let summary = trait_.summarise_struct(cx, struct_def); - - self.call_substructure_method( - cx, - trait_, - type_ident, - self_args, - nonself_args, - &StaticStruct(struct_def, summary), - ) - } - - /// ``` - /// #[derive(PartialEq)] - /// # struct Dummy; - /// enum A { - /// A1, - /// A2(i32) - /// } - /// - /// // is equivalent to - /// - /// impl PartialEq for A { - /// fn eq(&self, other: &A) -> ::bool { - /// match (&*self, &*other) { - /// (&A1, &A1) => true, - /// (&A2(ref self_0), - /// &A2(ref __arg_1_0)) => (*self_0).eq(&(*__arg_1_0)), - /// _ => { - /// let __self_vi = match *self { A1(..) => 0, A2(..) => 1 }; - /// let __arg_1_vi = match *other { A1(..) => 0, A2(..) => 1 }; - /// false - /// } - /// } - /// } - /// } - /// ``` - /// - /// (Of course `__self_vi` and `__arg_1_vi` are unused for - /// `PartialEq`, and those subcomputations will hopefully be removed - /// as their results are unused. The point of `__self_vi` and - /// `__arg_1_vi` is for `PartialOrd`; see #15503.) - fn expand_enum_method_body<'b>( - &self, - cx: &mut ExtCtxt<'_>, - trait_: &TraitDef<'b>, - enum_def: &'b EnumDef, - type_attrs: &[ast::Attribute], - type_ident: Ident, - self_args: Vec>, - nonself_args: &[P], - ) -> P { - self.build_enum_match_tuple( - cx, - trait_, - enum_def, - type_attrs, - type_ident, - self_args, - nonself_args, - ) - } - - /// Creates a match for a tuple of all `self_args`, where either all - /// variants match, or it falls into a catch-all for when one variant - /// does not match. - - /// There are N + 1 cases because is a case for each of the N - /// variants where all of the variants match, and one catch-all for - /// when one does not match. - - /// As an optimization we generate code which checks whether all variants - /// match first which makes llvm see that C-like enums can be compiled into - /// a simple equality check (for PartialEq). - - /// The catch-all handler is provided access the variant index values - /// for each of the self-args, carried in precomputed variables. - - /// ```{.text} - /// let __self0_vi = unsafe { - /// std::intrinsics::discriminant_value(&self) } as i32; - /// let __self1_vi = unsafe { - /// std::intrinsics::discriminant_value(&arg1) } as i32; - /// let __self2_vi = unsafe { - /// std::intrinsics::discriminant_value(&arg2) } as i32; - /// - /// if __self0_vi == __self1_vi && __self0_vi == __self2_vi && ... { - /// match (...) { - /// (Variant1, Variant1, ...) => Body1 - /// (Variant2, Variant2, ...) => Body2, - /// ... - /// _ => ::core::intrinsics::unreachable() - /// } - /// } - /// else { - /// ... // catch-all remainder can inspect above variant index values. - /// } - /// ``` - fn build_enum_match_tuple<'b>( - &self, - cx: &mut ExtCtxt<'_>, - trait_: &TraitDef<'b>, - enum_def: &'b EnumDef, - type_attrs: &[ast::Attribute], - type_ident: Ident, - mut self_args: Vec>, - nonself_args: &[P], - ) -> P { - let sp = trait_.span; - let variants = &enum_def.variants; - - let self_arg_names = iter::once("__self".to_string()) - .chain( - self_args - .iter() - .enumerate() - .skip(1) - .map(|(arg_count, _self_arg)| format!("__arg_{}", arg_count)), - ) - .collect::>(); - - let self_arg_idents = - self_arg_names.iter().map(|name| cx.ident_of(name, sp)).collect::>(); - - // The `vi_idents` will be bound, solely in the catch-all, to - // a series of let statements mapping each self_arg to an int - // value corresponding to its discriminant. - let vi_idents = self_arg_names - .iter() - .map(|name| { - let vi_suffix = format!("{}_vi", &name[..]); - cx.ident_of(&vi_suffix[..], trait_.span) - }) - .collect::>(); - - // Builds, via callback to call_substructure_method, the - // delegated expression that handles the catch-all case, - // using `__variants_tuple` to drive logic if necessary. - let catch_all_substructure = - EnumNonMatchingCollapsed(self_arg_idents, &variants[..], &vi_idents[..]); - - let first_fieldless = variants.iter().find(|v| v.data.fields().is_empty()); - - // These arms are of the form: - // (Variant1, Variant1, ...) => Body1 - // (Variant2, Variant2, ...) => Body2 - // ... - // where each tuple has length = self_args.len() - let mut match_arms: Vec = variants - .iter() - .enumerate() - .filter(|&(_, v)| !(self.unify_fieldless_variants && v.data.fields().is_empty())) - .map(|(index, variant)| { - let mk_self_pat = |cx: &mut ExtCtxt<'_>, self_arg_name: &str| { - let (p, idents) = trait_.create_enum_variant_pattern( - cx, - type_ident, - variant, - self_arg_name, - ast::Mutability::Not, - ); - (cx.pat(sp, PatKind::Ref(p, ast::Mutability::Not)), idents) - }; - - // A single arm has form (&VariantK, &VariantK, ...) => BodyK - // (see "Final wrinkle" note below for why.) - let mut subpats = Vec::with_capacity(self_arg_names.len()); - let mut self_pats_idents = Vec::with_capacity(self_arg_names.len() - 1); - let first_self_pat_idents = { - let (p, idents) = mk_self_pat(cx, &self_arg_names[0]); - subpats.push(p); - idents - }; - for self_arg_name in &self_arg_names[1..] { - let (p, idents) = mk_self_pat(cx, &self_arg_name[..]); - subpats.push(p); - self_pats_idents.push(idents); - } - - // Here is the pat = `(&VariantK, &VariantK, ...)` - let single_pat = cx.pat_tuple(sp, subpats); - - // For the BodyK, we need to delegate to our caller, - // passing it an EnumMatching to indicate which case - // we are in. - - // All of the Self args have the same variant in these - // cases. So we transpose the info in self_pats_idents - // to gather the getter expressions together, in the - // form that EnumMatching expects. - - // The transposition is driven by walking across the - // arg fields of the variant for the first self pat. - let field_tuples = first_self_pat_idents - .into_iter() - .enumerate() - // For each arg field of self, pull out its getter expr ... - .map(|(field_index, (sp, opt_ident, self_getter_expr, attrs))| { - // ... but FieldInfo also wants getter expr - // for matching other arguments of Self type; - // so walk across the *other* self_pats_idents - // and pull out getter for same field in each - // of them (using `field_index` tracked above). - // That is the heart of the transposition. - let others = self_pats_idents - .iter() - .map(|fields| { - let (_, _opt_ident, ref other_getter_expr, _) = fields[field_index]; - - // All Self args have same variant, so - // opt_idents are the same. (Assert - // here to make it self-evident that - // it is okay to ignore `_opt_ident`.) - assert!(opt_ident == _opt_ident); - - other_getter_expr.clone() - }) - .collect::>>(); - - FieldInfo { - span: sp, - name: opt_ident, - self_: self_getter_expr, - other: others, - attrs, - } - }) - .collect::>>(); - - // Now, for some given VariantK, we have built up - // expressions for referencing every field of every - // Self arg, assuming all are instances of VariantK. - // Build up code associated with such a case. - let substructure = EnumMatching(index, variants.len(), variant, field_tuples); - let arm_expr = self.call_substructure_method( - cx, - trait_, - type_ident, - &self_args[..], - nonself_args, - &substructure, - ); - - cx.arm(sp, single_pat, arm_expr) - }) - .collect(); - - let default = match first_fieldless { - Some(v) if self.unify_fieldless_variants => { - // We need a default case that handles the fieldless variants. - // The index and actual variant aren't meaningful in this case, - // so just use whatever - let substructure = EnumMatching(0, variants.len(), v, Vec::new()); - Some(self.call_substructure_method( - cx, - trait_, - type_ident, - &self_args[..], - nonself_args, - &substructure, - )) - } - _ if variants.len() > 1 && self_args.len() > 1 => { - // Since we know that all the arguments will match if we reach - // the match expression we add the unreachable intrinsics as the - // result of the catch all which should help llvm in optimizing it - Some(deriving::call_intrinsic(cx, sp, "unreachable", vec![])) - } - _ => None, - }; - if let Some(arm) = default { - match_arms.push(cx.arm(sp, cx.pat_wild(sp), arm)); - } - - // We will usually need the catch-all after matching the - // tuples `(VariantK, VariantK, ...)` for each VariantK of the - // enum. But: - // - // * when there is only one Self arg, the arms above suffice - // (and the deriving we call back into may not be prepared to - // handle EnumNonMatchCollapsed), and, - // - // * when the enum has only one variant, the single arm that - // is already present always suffices. - // - // * In either of the two cases above, if we *did* add a - // catch-all `_` match, it would trigger the - // unreachable-pattern error. - // - if variants.len() > 1 && self_args.len() > 1 { - // Build a series of let statements mapping each self_arg - // to its discriminant value. If this is a C-style enum - // with a specific repr type, then casts the values to - // that type. Otherwise casts to `i32` (the default repr - // type). - // - // i.e., for `enum E { A, B(1), C(T, T) }`, and a deriving - // with three Self args, builds three statements: - // - // ``` - // let __self0_vi = unsafe { - // std::intrinsics::discriminant_value(&self) } as i32; - // let __self1_vi = unsafe { - // std::intrinsics::discriminant_value(&arg1) } as i32; - // let __self2_vi = unsafe { - // std::intrinsics::discriminant_value(&arg2) } as i32; - // ``` - let mut index_let_stmts: Vec = Vec::with_capacity(vi_idents.len() + 1); - - // We also build an expression which checks whether all discriminants are equal - // discriminant_test = __self0_vi == __self1_vi && __self0_vi == __self2_vi && ... - let mut discriminant_test = cx.expr_bool(sp, true); - - let target_type_name = find_repr_type_name(&cx.parse_sess, type_attrs); - - let mut first_ident = None; - for (&ident, self_arg) in vi_idents.iter().zip(&self_args) { - let self_addr = cx.expr_addr_of(sp, self_arg.clone()); - let variant_value = - deriving::call_intrinsic(cx, sp, "discriminant_value", vec![self_addr]); - - let target_ty = cx.ty_ident(sp, cx.ident_of(target_type_name, sp)); - let variant_disr = cx.expr_cast(sp, variant_value, target_ty); - let let_stmt = cx.stmt_let(sp, false, ident, variant_disr); - index_let_stmts.push(let_stmt); - - match first_ident { - Some(first) => { - let first_expr = cx.expr_ident(sp, first); - let id = cx.expr_ident(sp, ident); - let test = cx.expr_binary(sp, BinOpKind::Eq, first_expr, id); - discriminant_test = - cx.expr_binary(sp, BinOpKind::And, discriminant_test, test) - } - None => { - first_ident = Some(ident); - } - } - } - - let arm_expr = self.call_substructure_method( - cx, - trait_, - type_ident, - &self_args[..], - nonself_args, - &catch_all_substructure, - ); - - // Final wrinkle: the self_args are expressions that deref - // down to desired places, but we cannot actually deref - // them when they are fed as r-values into a tuple - // expression; here add a layer of borrowing, turning - // `(*self, *__arg_0, ...)` into `(&*self, &*__arg_0, ...)`. - self_args.map_in_place(|self_arg| cx.expr_addr_of(sp, self_arg)); - let match_arg = cx.expr(sp, ast::ExprKind::Tup(self_args)); - - // Lastly we create an expression which branches on all discriminants being equal - // if discriminant_test { - // match (...) { - // (Variant1, Variant1, ...) => Body1 - // (Variant2, Variant2, ...) => Body2, - // ... - // _ => ::core::intrinsics::unreachable() - // } - // } - // else { - // - // } - let all_match = cx.expr_match(sp, match_arg, match_arms); - let arm_expr = cx.expr_if(sp, discriminant_test, all_match, Some(arm_expr)); - index_let_stmts.push(cx.stmt_expr(arm_expr)); - cx.expr_block(cx.block(sp, index_let_stmts)) - } else if variants.is_empty() { - // As an additional wrinkle, For a zero-variant enum A, - // currently the compiler - // will accept `fn (a: &Self) { match *a { } }` - // but rejects `fn (a: &Self) { match (&*a,) { } }` - // as well as `fn (a: &Self) { match ( *a,) { } }` - // - // This means that the strategy of building up a tuple of - // all Self arguments fails when Self is a zero variant - // enum: rustc rejects the expanded program, even though - // the actual code tends to be impossible to execute (at - // least safely), according to the type system. - // - // The most expedient fix for this is to just let the - // code fall through to the catch-all. But even this is - // error-prone, since the catch-all as defined above would - // generate code like this: - // - // _ => { let __self0 = match *self { }; - // let __self1 = match *__arg_0 { }; - // } - // - // Which is yields bindings for variables which type - // inference cannot resolve to unique types. - // - // One option to the above might be to add explicit type - // annotations. But the *only* reason to go down that path - // would be to try to make the expanded output consistent - // with the case when the number of enum variants >= 1. - // - // That just isn't worth it. In fact, trying to generate - // sensible code for *any* deriving on a zero-variant enum - // does not make sense. But at the same time, for now, we - // do not want to cause a compile failure just because the - // user happened to attach a deriving to their - // zero-variant enum. - // - // Instead, just generate a failing expression for the - // zero variant case, skipping matches and also skipping - // delegating back to the end user code entirely. - // - // (See also #4499 and #12609; note that some of the - // discussions there influence what choice we make here; - // e.g., if we feature-gate `match x { ... }` when x refers - // to an uninhabited type (e.g., a zero-variant enum or a - // type holding such an enum), but do not feature-gate - // zero-variant enums themselves, then attempting to - // derive Debug on such a type could here generate code - // that needs the feature gate enabled.) - - deriving::call_intrinsic(cx, sp, "unreachable", vec![]) - } else { - // Final wrinkle: the self_args are expressions that deref - // down to desired places, but we cannot actually deref - // them when they are fed as r-values into a tuple - // expression; here add a layer of borrowing, turning - // `(*self, *__arg_0, ...)` into `(&*self, &*__arg_0, ...)`. - self_args.map_in_place(|self_arg| cx.expr_addr_of(sp, self_arg)); - let match_arg = cx.expr(sp, ast::ExprKind::Tup(self_args)); - cx.expr_match(sp, match_arg, match_arms) - } - } - - fn expand_static_enum_method_body( - &self, - cx: &mut ExtCtxt<'_>, - trait_: &TraitDef<'_>, - enum_def: &EnumDef, - type_ident: Ident, - self_args: &[P], - nonself_args: &[P], - ) -> P { - let summary = enum_def - .variants - .iter() - .map(|v| { - let sp = v.span.with_ctxt(trait_.span.ctxt()); - let summary = trait_.summarise_struct(cx, &v.data); - (v.ident, sp, summary) - }) - .collect(); - self.call_substructure_method( - cx, - trait_, - type_ident, - self_args, - nonself_args, - &StaticEnum(enum_def, summary), - ) - } -} - -// general helper methods. -impl<'a> TraitDef<'a> { - fn summarise_struct(&self, cx: &mut ExtCtxt<'_>, struct_def: &VariantData) -> StaticFields { - let mut named_idents = Vec::new(); - let mut just_spans = Vec::new(); - for field in struct_def.fields() { - let sp = field.span.with_ctxt(self.span.ctxt()); - match field.ident { - Some(ident) => named_idents.push((ident, sp)), - _ => just_spans.push(sp), - } - } - - let is_tuple = if let ast::VariantData::Tuple(..) = struct_def { true } else { false }; - match (just_spans.is_empty(), named_idents.is_empty()) { - (false, false) => cx.span_bug( - self.span, - "a struct with named and unnamed \ - fields in generic `derive`", - ), - // named fields - (_, false) => Named(named_idents), - // unnamed fields - (false, _) => Unnamed(just_spans, is_tuple), - // empty - _ => Named(Vec::new()), - } - } - - fn create_subpatterns( - &self, - cx: &mut ExtCtxt<'_>, - field_paths: Vec, - mutbl: ast::Mutability, - use_temporaries: bool, - ) -> Vec> { - field_paths - .iter() - .map(|path| { - let binding_mode = if use_temporaries { - ast::BindingMode::ByValue(ast::Mutability::Not) - } else { - ast::BindingMode::ByRef(mutbl) - }; - cx.pat(path.span, PatKind::Ident(binding_mode, (*path).clone(), None)) - }) - .collect() - } - - fn create_struct_pattern( - &self, - cx: &mut ExtCtxt<'_>, - struct_path: ast::Path, - struct_def: &'a VariantData, - prefix: &str, - mutbl: ast::Mutability, - use_temporaries: bool, - ) -> (P, Vec<(Span, Option, P, &'a [ast::Attribute])>) { - let mut paths = Vec::new(); - let mut ident_exprs = Vec::new(); - for (i, struct_field) in struct_def.fields().iter().enumerate() { - let sp = struct_field.span.with_ctxt(self.span.ctxt()); - let ident = cx.ident_of(&format!("{}_{}", prefix, i), self.span); - paths.push(ident.with_span_pos(sp)); - let val = cx.expr_path(cx.path_ident(sp, ident)); - let val = if use_temporaries { val } else { cx.expr_deref(sp, val) }; - let val = cx.expr(sp, ast::ExprKind::Paren(val)); - - ident_exprs.push((sp, struct_field.ident, val, &struct_field.attrs[..])); - } - - let subpats = self.create_subpatterns(cx, paths, mutbl, use_temporaries); - let pattern = match *struct_def { - VariantData::Struct(..) => { - let field_pats = subpats - .into_iter() - .zip(&ident_exprs) - .map(|(pat, &(sp, ident, ..))| { - if ident.is_none() { - cx.span_bug(sp, "a braced struct with unnamed fields in `derive`"); - } - ast::FieldPat { - ident: ident.unwrap(), - is_shorthand: false, - attrs: ast::AttrVec::new(), - id: ast::DUMMY_NODE_ID, - span: pat.span.with_ctxt(self.span.ctxt()), - pat, - is_placeholder: false, - } - }) - .collect(); - cx.pat_struct(self.span, struct_path, field_pats) - } - VariantData::Tuple(..) => cx.pat_tuple_struct(self.span, struct_path, subpats), - VariantData::Unit(..) => cx.pat_path(self.span, struct_path), - }; - - (pattern, ident_exprs) - } - - fn create_enum_variant_pattern( - &self, - cx: &mut ExtCtxt<'_>, - enum_ident: ast::Ident, - variant: &'a ast::Variant, - prefix: &str, - mutbl: ast::Mutability, - ) -> (P, Vec<(Span, Option, P, &'a [ast::Attribute])>) { - let sp = variant.span.with_ctxt(self.span.ctxt()); - let variant_path = cx.path(sp, vec![enum_ident, variant.ident]); - let use_temporaries = false; // enums can't be repr(packed) - self.create_struct_pattern(cx, variant_path, &variant.data, prefix, mutbl, use_temporaries) - } -} - -// helpful premade recipes - -pub fn cs_fold_fields<'a, F>( - use_foldl: bool, - mut f: F, - base: P, - cx: &mut ExtCtxt<'_>, - all_fields: &[FieldInfo<'a>], -) -> P -where - F: FnMut(&mut ExtCtxt<'_>, Span, P, P, &[P]) -> P, -{ - if use_foldl { - all_fields - .iter() - .fold(base, |old, field| f(cx, field.span, old, field.self_.clone(), &field.other)) - } else { - all_fields - .iter() - .rev() - .fold(base, |old, field| f(cx, field.span, old, field.self_.clone(), &field.other)) - } -} - -pub fn cs_fold_enumnonmatch( - mut enum_nonmatch_f: EnumNonMatchCollapsedFunc<'_>, - cx: &mut ExtCtxt<'_>, - trait_span: Span, - substructure: &Substructure<'_>, -) -> P { - match *substructure.fields { - EnumNonMatchingCollapsed(ref all_args, _, tuple) => { - enum_nonmatch_f(cx, trait_span, (&all_args[..], tuple), substructure.nonself_args) - } - _ => cx.span_bug(trait_span, "cs_fold_enumnonmatch expected an EnumNonMatchingCollapsed"), - } -} - -pub fn cs_fold_static(cx: &mut ExtCtxt<'_>, trait_span: Span) -> P { - cx.span_bug(trait_span, "static function in `derive`") -} - -/// Fold the fields. `use_foldl` controls whether this is done -/// left-to-right (`true`) or right-to-left (`false`). -pub fn cs_fold( - use_foldl: bool, - f: F, - base: P, - enum_nonmatch_f: EnumNonMatchCollapsedFunc<'_>, - cx: &mut ExtCtxt<'_>, - trait_span: Span, - substructure: &Substructure<'_>, -) -> P -where - F: FnMut(&mut ExtCtxt<'_>, Span, P, P, &[P]) -> P, -{ - match *substructure.fields { - EnumMatching(.., ref all_fields) | Struct(_, ref all_fields) => { - cs_fold_fields(use_foldl, f, base, cx, all_fields) - } - EnumNonMatchingCollapsed(..) => { - cs_fold_enumnonmatch(enum_nonmatch_f, cx, trait_span, substructure) - } - StaticEnum(..) | StaticStruct(..) => cs_fold_static(cx, trait_span), - } -} - -/// Function to fold over fields, with three cases, to generate more efficient and concise code. -/// When the `substructure` has grouped fields, there are two cases: -/// Zero fields: call the base case function with `None` (like the usual base case of `cs_fold`). -/// One or more fields: call the base case function on the first value (which depends on -/// `use_fold`), and use that as the base case. Then perform `cs_fold` on the remainder of the -/// fields. -/// When the `substructure` is a `EnumNonMatchingCollapsed`, the result of `enum_nonmatch_f` -/// is returned. Statics may not be folded over. -/// See `cs_op` in `partial_ord.rs` for a model example. -pub fn cs_fold1( - use_foldl: bool, - f: F, - mut b: B, - enum_nonmatch_f: EnumNonMatchCollapsedFunc<'_>, - cx: &mut ExtCtxt<'_>, - trait_span: Span, - substructure: &Substructure<'_>, -) -> P -where - F: FnMut(&mut ExtCtxt<'_>, Span, P, P, &[P]) -> P, - B: FnMut(&mut ExtCtxt<'_>, Option<(Span, P, &[P])>) -> P, -{ - match *substructure.fields { - EnumMatching(.., ref all_fields) | Struct(_, ref all_fields) => { - let (base, all_fields) = match (all_fields.is_empty(), use_foldl) { - (false, true) => { - let field = &all_fields[0]; - let args = (field.span, field.self_.clone(), &field.other[..]); - (b(cx, Some(args)), &all_fields[1..]) - } - (false, false) => { - let idx = all_fields.len() - 1; - let field = &all_fields[idx]; - let args = (field.span, field.self_.clone(), &field.other[..]); - (b(cx, Some(args)), &all_fields[..idx]) - } - (true, _) => (b(cx, None), &all_fields[..]), - }; - - cs_fold_fields(use_foldl, f, base, cx, all_fields) - } - EnumNonMatchingCollapsed(..) => { - cs_fold_enumnonmatch(enum_nonmatch_f, cx, trait_span, substructure) - } - StaticEnum(..) | StaticStruct(..) => cs_fold_static(cx, trait_span), - } -} - -/// Returns `true` if the type has no value fields -/// (for an enum, no variant has any fields) -pub fn is_type_without_fields(item: &Annotatable) -> bool { - if let Annotatable::Item(ref item) = *item { - match item.kind { - ast::ItemKind::Enum(ref enum_def, _) => { - enum_def.variants.iter().all(|v| v.data.fields().is_empty()) - } - ast::ItemKind::Struct(ref variant_data, _) => variant_data.fields().is_empty(), - _ => false, - } - } else { - false - } -} diff --git a/src/libsyntax_ext/deriving/generic/ty.rs b/src/libsyntax_ext/deriving/generic/ty.rs deleted file mode 100644 index 7eab15aff77..00000000000 --- a/src/libsyntax_ext/deriving/generic/ty.rs +++ /dev/null @@ -1,283 +0,0 @@ -//! A mini version of ast::Ty, which is easier to use, and features an explicit `Self` type to use -//! when specifying impls to be derived. - -pub use PtrTy::*; -pub use Ty::*; - -use syntax::ast::{self, Expr, GenericArg, GenericParamKind, Generics, Ident, SelfKind}; -use syntax::ptr::P; -use syntax::source_map::{respan, DUMMY_SP}; -use syntax_expand::base::ExtCtxt; -use syntax_pos::symbol::kw; -use syntax_pos::Span; - -/// The types of pointers -#[derive(Clone)] -pub enum PtrTy { - /// &'lifetime mut - Borrowed(Option, ast::Mutability), - /// *mut - #[allow(dead_code)] - Raw(ast::Mutability), -} - -/// A path, e.g., `::std::option::Option::` (global). Has support -/// for type parameters and a lifetime. -#[derive(Clone)] -pub struct Path<'a> { - path: Vec<&'a str>, - lifetime: Option, - params: Vec>>, - kind: PathKind, -} - -#[derive(Clone)] -pub enum PathKind { - Local, - Global, - Std, -} - -impl<'a> Path<'a> { - pub fn new(path: Vec<&str>) -> Path<'_> { - Path::new_(path, None, Vec::new(), PathKind::Std) - } - pub fn new_local(path: &str) -> Path<'_> { - Path::new_(vec![path], None, Vec::new(), PathKind::Local) - } - pub fn new_<'r>( - path: Vec<&'r str>, - lifetime: Option, - params: Vec>>, - kind: PathKind, - ) -> Path<'r> { - Path { path, lifetime, params, kind } - } - - pub fn to_ty( - &self, - cx: &ExtCtxt<'_>, - span: Span, - self_ty: Ident, - self_generics: &Generics, - ) -> P { - cx.ty_path(self.to_path(cx, span, self_ty, self_generics)) - } - pub fn to_path( - &self, - cx: &ExtCtxt<'_>, - span: Span, - self_ty: Ident, - self_generics: &Generics, - ) -> ast::Path { - let mut idents = self.path.iter().map(|s| cx.ident_of(*s, span)).collect(); - let lt = mk_lifetimes(cx, span, &self.lifetime); - let tys: Vec> = - self.params.iter().map(|t| t.to_ty(cx, span, self_ty, self_generics)).collect(); - let params = lt - .into_iter() - .map(|lt| GenericArg::Lifetime(lt)) - .chain(tys.into_iter().map(|ty| GenericArg::Type(ty))) - .collect(); - - match self.kind { - PathKind::Global => cx.path_all(span, true, idents, params), - PathKind::Local => cx.path_all(span, false, idents, params), - PathKind::Std => { - let def_site = cx.with_def_site_ctxt(DUMMY_SP); - idents.insert(0, Ident::new(kw::DollarCrate, def_site)); - cx.path_all(span, false, idents, params) - } - } - } -} - -/// A type. Supports pointers, Self, and literals. -#[derive(Clone)] -pub enum Ty<'a> { - Self_, - /// &/Box/ Ty - Ptr(Box>, PtrTy), - /// mod::mod::Type<[lifetime], [Params...]>, including a plain type - /// parameter, and things like `i32` - Literal(Path<'a>), - /// includes unit - Tuple(Vec>), -} - -pub fn borrowed_ptrty() -> PtrTy { - Borrowed(None, ast::Mutability::Not) -} -pub fn borrowed(ty: Box>) -> Ty<'_> { - Ptr(ty, borrowed_ptrty()) -} - -pub fn borrowed_explicit_self() -> Option> { - Some(Some(borrowed_ptrty())) -} - -pub fn borrowed_self<'r>() -> Ty<'r> { - borrowed(Box::new(Self_)) -} - -pub fn nil_ty<'r>() -> Ty<'r> { - Tuple(Vec::new()) -} - -fn mk_lifetime(cx: &ExtCtxt<'_>, span: Span, lt: &Option) -> Option { - lt.map(|ident| cx.lifetime(span, ident)) -} - -fn mk_lifetimes(cx: &ExtCtxt<'_>, span: Span, lt: &Option) -> Vec { - mk_lifetime(cx, span, lt).into_iter().collect() -} - -impl<'a> Ty<'a> { - pub fn to_ty( - &self, - cx: &ExtCtxt<'_>, - span: Span, - self_ty: Ident, - self_generics: &Generics, - ) -> P { - match *self { - Ptr(ref ty, ref ptr) => { - let raw_ty = ty.to_ty(cx, span, self_ty, self_generics); - match *ptr { - Borrowed(ref lt, mutbl) => { - let lt = mk_lifetime(cx, span, lt); - cx.ty_rptr(span, raw_ty, lt, mutbl) - } - Raw(mutbl) => cx.ty_ptr(span, raw_ty, mutbl), - } - } - Literal(ref p) => p.to_ty(cx, span, self_ty, self_generics), - Self_ => cx.ty_path(self.to_path(cx, span, self_ty, self_generics)), - Tuple(ref fields) => { - let ty = ast::TyKind::Tup( - fields.iter().map(|f| f.to_ty(cx, span, self_ty, self_generics)).collect(), - ); - cx.ty(span, ty) - } - } - } - - pub fn to_path( - &self, - cx: &ExtCtxt<'_>, - span: Span, - self_ty: Ident, - generics: &Generics, - ) -> ast::Path { - match *self { - Self_ => { - let params: Vec<_> = generics - .params - .iter() - .map(|param| match param.kind { - GenericParamKind::Lifetime { .. } => { - GenericArg::Lifetime(ast::Lifetime { id: param.id, ident: param.ident }) - } - GenericParamKind::Type { .. } => { - GenericArg::Type(cx.ty_ident(span, param.ident)) - } - GenericParamKind::Const { .. } => { - GenericArg::Const(cx.const_ident(span, param.ident)) - } - }) - .collect(); - - cx.path_all(span, false, vec![self_ty], params) - } - Literal(ref p) => p.to_path(cx, span, self_ty, generics), - Ptr(..) => cx.span_bug(span, "pointer in a path in generic `derive`"), - Tuple(..) => cx.span_bug(span, "tuple in a path in generic `derive`"), - } - } -} - -fn mk_ty_param( - cx: &ExtCtxt<'_>, - span: Span, - name: &str, - attrs: &[ast::Attribute], - bounds: &[Path<'_>], - self_ident: Ident, - self_generics: &Generics, -) -> ast::GenericParam { - let bounds = bounds - .iter() - .map(|b| { - let path = b.to_path(cx, span, self_ident, self_generics); - cx.trait_bound(path) - }) - .collect(); - cx.typaram(span, cx.ident_of(name, span), attrs.to_owned(), bounds, None) -} - -fn mk_generics(params: Vec, span: Span) -> Generics { - Generics { params, where_clause: ast::WhereClause { predicates: Vec::new(), span }, span } -} - -/// Lifetimes and bounds on type parameters -#[derive(Clone)] -pub struct LifetimeBounds<'a> { - pub lifetimes: Vec<(&'a str, Vec<&'a str>)>, - pub bounds: Vec<(&'a str, Vec>)>, -} - -impl<'a> LifetimeBounds<'a> { - pub fn empty() -> LifetimeBounds<'a> { - LifetimeBounds { lifetimes: Vec::new(), bounds: Vec::new() } - } - pub fn to_generics( - &self, - cx: &ExtCtxt<'_>, - span: Span, - self_ty: Ident, - self_generics: &Generics, - ) -> Generics { - let generic_params = self - .lifetimes - .iter() - .map(|&(lt, ref bounds)| { - let bounds = bounds - .iter() - .map(|b| ast::GenericBound::Outlives(cx.lifetime(span, Ident::from_str(b)))); - cx.lifetime_def(span, Ident::from_str(lt), vec![], bounds.collect()) - }) - .chain(self.bounds.iter().map(|t| { - let (name, ref bounds) = *t; - mk_ty_param(cx, span, name, &[], &bounds, self_ty, self_generics) - })) - .collect(); - - mk_generics(generic_params, span) - } -} - -pub fn get_explicit_self( - cx: &ExtCtxt<'_>, - span: Span, - self_ptr: &Option, -) -> (P, ast::ExplicitSelf) { - // this constructs a fresh `self` path - let self_path = cx.expr_self(span); - match *self_ptr { - None => (self_path, respan(span, SelfKind::Value(ast::Mutability::Not))), - Some(ref ptr) => { - let self_ty = respan( - span, - match *ptr { - Borrowed(ref lt, mutbl) => { - let lt = lt.map(|s| cx.lifetime(span, s)); - SelfKind::Region(lt, mutbl) - } - Raw(_) => cx.span_bug(span, "attempted to use *self in deriving definition"), - }, - ); - let self_expr = cx.expr_deref(span, self_path); - (self_expr, self_ty) - } - } -} diff --git a/src/libsyntax_ext/deriving/hash.rs b/src/libsyntax_ext/deriving/hash.rs deleted file mode 100644 index acf18ac70e6..00000000000 --- a/src/libsyntax_ext/deriving/hash.rs +++ /dev/null @@ -1,92 +0,0 @@ -use crate::deriving::generic::ty::*; -use crate::deriving::generic::*; -use crate::deriving::{self, path_std, pathvec_std}; - -use syntax::ast::{Expr, MetaItem, Mutability}; -use syntax::ptr::P; -use syntax::symbol::sym; -use syntax_expand::base::{Annotatable, ExtCtxt}; -use syntax_pos::Span; - -pub fn expand_deriving_hash( - cx: &mut ExtCtxt<'_>, - span: Span, - mitem: &MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable), -) { - let path = Path::new_(pathvec_std!(cx, hash::Hash), None, vec![], PathKind::Std); - - let typaram = "__H"; - - let arg = Path::new_local(typaram); - let hash_trait_def = TraitDef { - span, - attributes: Vec::new(), - path, - additional_bounds: Vec::new(), - generics: LifetimeBounds::empty(), - is_unsafe: false, - supports_unions: false, - methods: vec![MethodDef { - name: "hash", - generics: LifetimeBounds { - lifetimes: Vec::new(), - bounds: vec![(typaram, vec![path_std!(cx, hash::Hasher)])], - }, - explicit_self: borrowed_explicit_self(), - args: vec![(Ptr(Box::new(Literal(arg)), Borrowed(None, Mutability::Mut)), "state")], - ret_ty: nil_ty(), - attributes: vec![], - is_unsafe: false, - unify_fieldless_variants: true, - combine_substructure: combine_substructure(Box::new(|a, b, c| { - hash_substructure(a, b, c) - })), - }], - associated_types: Vec::new(), - }; - - hash_trait_def.expand(cx, mitem, item, push); -} - -fn hash_substructure(cx: &mut ExtCtxt<'_>, trait_span: Span, substr: &Substructure<'_>) -> P { - let state_expr = match &substr.nonself_args { - &[o_f] => o_f, - _ => cx.span_bug(trait_span, "incorrect number of arguments in `derive(Hash)`"), - }; - let call_hash = |span, thing_expr| { - let hash_path = { - let strs = cx.std_path(&[sym::hash, sym::Hash, sym::hash]); - - cx.expr_path(cx.path_global(span, strs)) - }; - let ref_thing = cx.expr_addr_of(span, thing_expr); - let expr = cx.expr_call(span, hash_path, vec![ref_thing, state_expr.clone()]); - cx.stmt_expr(expr) - }; - let mut stmts = Vec::new(); - - let fields = match *substr.fields { - Struct(_, ref fs) | EnumMatching(_, 1, .., ref fs) => fs, - EnumMatching(.., ref fs) => { - let variant_value = deriving::call_intrinsic( - cx, - trait_span, - "discriminant_value", - vec![cx.expr_self(trait_span)], - ); - - stmts.push(call_hash(trait_span, variant_value)); - - fs - } - _ => cx.span_bug(trait_span, "impossible substructure in `derive(Hash)`"), - }; - - stmts.extend( - fields.iter().map(|FieldInfo { ref self_, span, .. }| call_hash(*span, self_.clone())), - ); - - cx.expr_block(cx.block(trait_span, stmts)) -} diff --git a/src/libsyntax_ext/deriving/mod.rs b/src/libsyntax_ext/deriving/mod.rs deleted file mode 100644 index ca4d4fbc5bd..00000000000 --- a/src/libsyntax_ext/deriving/mod.rs +++ /dev/null @@ -1,171 +0,0 @@ -//! The compiler code necessary to implement the `#[derive]` extensions. - -use syntax::ast::{self, ItemKind, MetaItem}; -use syntax::ptr::P; -use syntax::symbol::{sym, Symbol}; -use syntax_expand::base::{Annotatable, ExtCtxt, MultiItemModifier}; -use syntax_pos::Span; - -macro path_local($x:ident) { - generic::ty::Path::new_local(stringify!($x)) -} - -macro pathvec_std($cx:expr, $($rest:ident)::+) {{ - vec![ $( stringify!($rest) ),+ ] -}} - -macro path_std($($x:tt)*) { - generic::ty::Path::new( pathvec_std!( $($x)* ) ) -} - -pub mod bounds; -pub mod clone; -pub mod debug; -pub mod decodable; -pub mod default; -pub mod encodable; -pub mod hash; - -#[path = "cmp/eq.rs"] -pub mod eq; -#[path = "cmp/ord.rs"] -pub mod ord; -#[path = "cmp/partial_eq.rs"] -pub mod partial_eq; -#[path = "cmp/partial_ord.rs"] -pub mod partial_ord; - -pub mod generic; - -crate struct BuiltinDerive( - crate fn(&mut ExtCtxt<'_>, Span, &MetaItem, &Annotatable, &mut dyn FnMut(Annotatable)), -); - -impl MultiItemModifier for BuiltinDerive { - fn expand( - &self, - ecx: &mut ExtCtxt<'_>, - span: Span, - meta_item: &MetaItem, - item: Annotatable, - ) -> Vec { - // FIXME: Built-in derives often forget to give spans contexts, - // so we are doing it here in a centralized way. - let span = ecx.with_def_site_ctxt(span); - let mut items = Vec::new(); - (self.0)(ecx, span, meta_item, &item, &mut |a| items.push(a)); - items - } -} - -/// Constructs an expression that calls an intrinsic -fn call_intrinsic( - cx: &ExtCtxt<'_>, - span: Span, - intrinsic: &str, - args: Vec>, -) -> P { - let span = cx.with_def_site_ctxt(span); - let path = cx.std_path(&[sym::intrinsics, Symbol::intern(intrinsic)]); - let call = cx.expr_call_global(span, path, args); - - cx.expr_block(P(ast::Block { - stmts: vec![cx.stmt_expr(call)], - id: ast::DUMMY_NODE_ID, - rules: ast::BlockCheckMode::Unsafe(ast::CompilerGenerated), - span, - })) -} - -// Injects `impl<...> Structural for ItemType<...> { }`. In particular, -// does *not* add `where T: Structural` for parameters `T` in `...`. -// (That's the main reason we cannot use TraitDef here.) -fn inject_impl_of_structural_trait( - cx: &mut ExtCtxt<'_>, - span: Span, - item: &Annotatable, - structural_path: generic::ty::Path<'_>, - push: &mut dyn FnMut(Annotatable), -) { - let item = match *item { - Annotatable::Item(ref item) => item, - _ => { - // Non-Item derive is an error, but it should have been - // set earlier; see - // libsyntax_expand/expand.rs:MacroExpander::fully_expand_fragment() - // libsyntax_expand/base.rs:Annotatable::derive_allowed() - return; - } - }; - - let generics = match item.kind { - ItemKind::Struct(_, ref generics) | ItemKind::Enum(_, ref generics) => generics, - // Do not inject `impl Structural for Union`. (`PartialEq` does not - // support unions, so we will see error downstream.) - ItemKind::Union(..) => return, - _ => unreachable!(), - }; - - // Create generics param list for where clauses and impl headers - let mut generics = generics.clone(); - - // Create the type of `self`. - // - // in addition, remove defaults from type params (impls cannot have them). - let self_params: Vec<_> = generics - .params - .iter_mut() - .map(|param| match &mut param.kind { - ast::GenericParamKind::Lifetime => { - ast::GenericArg::Lifetime(cx.lifetime(span, param.ident)) - } - ast::GenericParamKind::Type { default } => { - *default = None; - ast::GenericArg::Type(cx.ty_ident(span, param.ident)) - } - ast::GenericParamKind::Const { ty: _ } => { - ast::GenericArg::Const(cx.const_ident(span, param.ident)) - } - }) - .collect(); - - let type_ident = item.ident; - - let trait_ref = cx.trait_ref(structural_path.to_path(cx, span, type_ident, &generics)); - let self_type = cx.ty_path(cx.path_all(span, false, vec![type_ident], self_params)); - - // It would be nice to also encode constraint `where Self: Eq` (by adding it - // onto `generics` cloned above). Unfortunately, that strategy runs afoul of - // rust-lang/rust#48214. So we perform that additional check in the compiler - // itself, instead of encoding it here. - - // Keep the lint and stability attributes of the original item, to control - // how the generated implementation is linted. - let mut attrs = Vec::new(); - attrs.extend( - item.attrs - .iter() - .filter(|a| { - [sym::allow, sym::warn, sym::deny, sym::forbid, sym::stable, sym::unstable] - .contains(&a.name_or_empty()) - }) - .cloned(), - ); - - let newitem = cx.item( - span, - ast::Ident::invalid(), - attrs, - ItemKind::Impl( - ast::Unsafety::Normal, - ast::ImplPolarity::Positive, - ast::Defaultness::Final, - generics, - Some(trait_ref), - self_type, - Vec::new(), - ), - ); - - push(Annotatable::Item(newitem)); -} diff --git a/src/libsyntax_ext/env.rs b/src/libsyntax_ext/env.rs deleted file mode 100644 index c9ecbabc8ff..00000000000 --- a/src/libsyntax_ext/env.rs +++ /dev/null @@ -1,88 +0,0 @@ -// The compiler code necessary to support the env! extension. Eventually this -// should all get sucked into either the compiler syntax extension plugin -// interface. -// - -use syntax::ast::{self, GenericArg, Ident}; -use syntax::symbol::{kw, sym, Symbol}; -use syntax::tokenstream::TokenStream; -use syntax_expand::base::{self, *}; -use syntax_pos::Span; - -use std::env; - -pub fn expand_option_env<'cx>( - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let var = match get_single_str_from_tts(cx, sp, tts, "option_env!") { - None => return DummyResult::any(sp), - Some(v) => v, - }; - - let sp = cx.with_def_site_ctxt(sp); - let e = match env::var(&var.as_str()) { - Err(..) => { - let lt = cx.lifetime(sp, Ident::new(kw::StaticLifetime, sp)); - cx.expr_path(cx.path_all( - sp, - true, - cx.std_path(&[sym::option, sym::Option, sym::None]), - vec![GenericArg::Type(cx.ty_rptr( - sp, - cx.ty_ident(sp, Ident::new(sym::str, sp)), - Some(lt), - ast::Mutability::Not, - ))], - )) - } - Ok(s) => cx.expr_call_global( - sp, - cx.std_path(&[sym::option, sym::Option, sym::Some]), - vec![cx.expr_str(sp, Symbol::intern(&s))], - ), - }; - MacEager::expr(e) -} - -pub fn expand_env<'cx>( - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let mut exprs = match get_exprs_from_tts(cx, sp, tts) { - Some(ref exprs) if exprs.is_empty() => { - cx.span_err(sp, "env! takes 1 or 2 arguments"); - return DummyResult::any(sp); - } - None => return DummyResult::any(sp), - Some(exprs) => exprs.into_iter(), - }; - - let var = match expr_to_string(cx, exprs.next().unwrap(), "expected string literal") { - None => return DummyResult::any(sp), - Some((v, _style)) => v, - }; - let msg = match exprs.next() { - None => Symbol::intern(&format!("environment variable `{}` not defined", var)), - Some(second) => match expr_to_string(cx, second, "expected string literal") { - None => return DummyResult::any(sp), - Some((s, _style)) => s, - }, - }; - - if exprs.next().is_some() { - cx.span_err(sp, "env! takes 1 or 2 arguments"); - return DummyResult::any(sp); - } - - let e = match env::var(&*var.as_str()) { - Err(_) => { - cx.span_err(sp, &msg.as_str()); - return DummyResult::any(sp); - } - Ok(s) => cx.expr_str(sp, Symbol::intern(&s)), - }; - MacEager::expr(e) -} diff --git a/src/libsyntax_ext/format.rs b/src/libsyntax_ext/format.rs deleted file mode 100644 index 1d1f68a4906..00000000000 --- a/src/libsyntax_ext/format.rs +++ /dev/null @@ -1,1233 +0,0 @@ -use ArgumentType::*; -use Position::*; - -use fmt_macros as parse; - -use errors::pluralize; -use errors::Applicability; -use errors::DiagnosticBuilder; - -use syntax::ast; -use syntax::ptr::P; -use syntax::symbol::{sym, Symbol}; -use syntax::token; -use syntax::tokenstream::TokenStream; -use syntax_expand::base::{self, *}; -use syntax_pos::{MultiSpan, Span}; - -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; -use std::borrow::Cow; -use std::collections::hash_map::Entry; - -#[derive(PartialEq)] -enum ArgumentType { - Placeholder(&'static str), - Count, -} - -enum Position { - Exact(usize), - Named(Symbol), -} - -struct Context<'a, 'b> { - ecx: &'a mut ExtCtxt<'b>, - /// The macro's call site. References to unstable formatting internals must - /// use this span to pass the stability checker. - macsp: Span, - /// The span of the format string literal. - fmtsp: Span, - - /// List of parsed argument expressions. - /// Named expressions are resolved early, and are appended to the end of - /// argument expressions. - /// - /// Example showing the various data structures in motion: - /// - /// * Original: `"{foo:o} {:o} {foo:x} {0:x} {1:o} {:x} {1:x} {0:o}"` - /// * Implicit argument resolution: `"{foo:o} {0:o} {foo:x} {0:x} {1:o} {1:x} {1:x} {0:o}"` - /// * Name resolution: `"{2:o} {0:o} {2:x} {0:x} {1:o} {1:x} {1:x} {0:o}"` - /// * `arg_types` (in JSON): `[[0, 1, 0], [0, 1, 1], [0, 1]]` - /// * `arg_unique_types` (in simplified JSON): `[["o", "x"], ["o", "x"], ["o", "x"]]` - /// * `names` (in JSON): `{"foo": 2}` - args: Vec>, - /// Placeholder slot numbers indexed by argument. - arg_types: Vec>, - /// Unique format specs seen for each argument. - arg_unique_types: Vec>, - /// Map from named arguments to their resolved indices. - names: FxHashMap, - - /// The latest consecutive literal strings, or empty if there weren't any. - literal: String, - - /// Collection of the compiled `rt::Argument` structures - pieces: Vec>, - /// Collection of string literals - str_pieces: Vec>, - /// Stays `true` if all formatting parameters are default (as in "{}{}"). - all_pieces_simple: bool, - - /// Mapping between positional argument references and indices into the - /// final generated static argument array. We record the starting indices - /// corresponding to each positional argument, and number of references - /// consumed so far for each argument, to facilitate correct `Position` - /// mapping in `build_piece`. In effect this can be seen as a "flattened" - /// version of `arg_unique_types`. - /// - /// Again with the example described above in docstring for `args`: - /// - /// * `arg_index_map` (in JSON): `[[0, 1, 0], [2, 3, 3], [4, 5]]` - arg_index_map: Vec>, - - /// Starting offset of count argument slots. - count_args_index_offset: usize, - - /// Count argument slots and tracking data structures. - /// Count arguments are separately tracked for de-duplication in case - /// multiple references are made to one argument. For example, in this - /// format string: - /// - /// * Original: `"{:.*} {:.foo$} {1:.*} {:.0$}"` - /// * Implicit argument resolution: `"{1:.0$} {2:.foo$} {1:.3$} {4:.0$}"` - /// * Name resolution: `"{1:.0$} {2:.5$} {1:.3$} {4:.0$}"` - /// * `count_positions` (in JSON): `{0: 0, 5: 1, 3: 2}` - /// * `count_args`: `vec![Exact(0), Exact(5), Exact(3)]` - count_args: Vec, - /// Relative slot numbers for count arguments. - count_positions: FxHashMap, - /// Number of count slots assigned. - count_positions_count: usize, - - /// Current position of the implicit positional arg pointer, as if it - /// still existed in this phase of processing. - /// Used only for `all_pieces_simple` tracking in `build_piece`. - curarg: usize, - /// Current piece being evaluated, used for error reporting. - curpiece: usize, - /// Keep track of invalid references to positional arguments. - invalid_refs: Vec<(usize, usize)>, - /// Spans of all the formatting arguments, in order. - arg_spans: Vec, - /// All the formatting arguments that have formatting flags set, in order for diagnostics. - arg_with_formatting: Vec>, - /// Whether this formatting string is a literal or it comes from a macro. - is_literal: bool, -} - -/// Parses the arguments from the given list of tokens, returning the diagnostic -/// if there's a parse error so we can continue parsing other format! -/// expressions. -/// -/// If parsing succeeds, the return value is: -/// -/// ```text -/// Some((fmtstr, parsed arguments, index map for named arguments)) -/// ``` -fn parse_args<'a>( - ecx: &mut ExtCtxt<'a>, - sp: Span, - tts: TokenStream, -) -> Result<(P, Vec>, FxHashMap), DiagnosticBuilder<'a>> { - let mut args = Vec::>::new(); - let mut names = FxHashMap::::default(); - - let mut p = ecx.new_parser_from_tts(tts); - - if p.token == token::Eof { - return Err(ecx.struct_span_err(sp, "requires at least a format string argument")); - } - - let fmtstr = p.parse_expr()?; - let mut first = true; - let mut named = false; - - while p.token != token::Eof { - if !p.eat(&token::Comma) { - if first { - // After `format!(""` we always expect *only* a comma... - let mut err = ecx.struct_span_err(p.token.span, "expected token: `,`"); - err.span_label(p.token.span, "expected `,`"); - p.maybe_annotate_with_ascription(&mut err, false); - return Err(err); - } else { - // ...after that delegate to `expect` to also include the other expected tokens. - return Err(p.expect(&token::Comma).err().unwrap()); - } - } - first = false; - if p.token == token::Eof { - break; - } // accept trailing commas - if p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq) { - named = true; - let name = if let token::Ident(name, _) = p.token.kind { - p.bump(); - name - } else { - unreachable!(); - }; - - p.expect(&token::Eq)?; - let e = p.parse_expr()?; - if let Some(prev) = names.get(&name) { - ecx.struct_span_err(e.span, &format!("duplicate argument named `{}`", name)) - .span_label(args[*prev].span, "previously here") - .span_label(e.span, "duplicate argument") - .emit(); - continue; - } - - // Resolve names into slots early. - // Since all the positional args are already seen at this point - // if the input is valid, we can simply append to the positional - // args. And remember the names. - let slot = args.len(); - names.insert(name, slot); - args.push(e); - } else { - let e = p.parse_expr()?; - if named { - let mut err = ecx - .struct_span_err(e.span, "positional arguments cannot follow named arguments"); - err.span_label(e.span, "positional arguments must be before named arguments"); - for (_, pos) in &names { - err.span_label(args[*pos].span, "named argument"); - } - err.emit(); - } - args.push(e); - } - } - Ok((fmtstr, args, names)) -} - -impl<'a, 'b> Context<'a, 'b> { - fn resolve_name_inplace(&self, p: &mut parse::Piece<'_>) { - // NOTE: the `unwrap_or` branch is needed in case of invalid format - // arguments, e.g., `format_args!("{foo}")`. - let lookup = |s: Symbol| *self.names.get(&s).unwrap_or(&0); - - match *p { - parse::String(_) => {} - parse::NextArgument(ref mut arg) => { - if let parse::ArgumentNamed(s) = arg.position { - arg.position = parse::ArgumentIs(lookup(s)); - } - if let parse::CountIsName(s) = arg.format.width { - arg.format.width = parse::CountIsParam(lookup(s)); - } - if let parse::CountIsName(s) = arg.format.precision { - arg.format.precision = parse::CountIsParam(lookup(s)); - } - } - } - } - - /// Verifies one piece of a parse string, and remembers it if valid. - /// All errors are not emitted as fatal so we can continue giving errors - /// about this and possibly other format strings. - fn verify_piece(&mut self, p: &parse::Piece<'_>) { - match *p { - parse::String(..) => {} - parse::NextArgument(ref arg) => { - // width/precision first, if they have implicit positional - // parameters it makes more sense to consume them first. - self.verify_count(arg.format.width); - self.verify_count(arg.format.precision); - - // argument second, if it's an implicit positional parameter - // it's written second, so it should come after width/precision. - let pos = match arg.position { - parse::ArgumentIs(i) | parse::ArgumentImplicitlyIs(i) => Exact(i), - parse::ArgumentNamed(s) => Named(s), - }; - - let ty = Placeholder(match &arg.format.ty[..] { - "" => "Display", - "?" => "Debug", - "e" => "LowerExp", - "E" => "UpperExp", - "o" => "Octal", - "p" => "Pointer", - "b" => "Binary", - "x" => "LowerHex", - "X" => "UpperHex", - _ => { - let fmtsp = self.fmtsp; - let sp = arg.format.ty_span.map(|sp| fmtsp.from_inner(sp)); - let mut err = self.ecx.struct_span_err( - sp.unwrap_or(fmtsp), - &format!("unknown format trait `{}`", arg.format.ty), - ); - err.note( - "the only appropriate formatting traits are:\n\ - - ``, which uses the `Display` trait\n\ - - `?`, which uses the `Debug` trait\n\ - - `e`, which uses the `LowerExp` trait\n\ - - `E`, which uses the `UpperExp` trait\n\ - - `o`, which uses the `Octal` trait\n\ - - `p`, which uses the `Pointer` trait\n\ - - `b`, which uses the `Binary` trait\n\ - - `x`, which uses the `LowerHex` trait\n\ - - `X`, which uses the `UpperHex` trait", - ); - if let Some(sp) = sp { - for (fmt, name) in &[ - ("", "Display"), - ("?", "Debug"), - ("e", "LowerExp"), - ("E", "UpperExp"), - ("o", "Octal"), - ("p", "Pointer"), - ("b", "Binary"), - ("x", "LowerHex"), - ("X", "UpperHex"), - ] { - err.tool_only_span_suggestion( - sp, - &format!("use the `{}` trait", name), - fmt.to_string(), - Applicability::MaybeIncorrect, - ); - } - } - err.emit(); - "" - } - }); - self.verify_arg_type(pos, ty); - self.curpiece += 1; - } - } - } - - fn verify_count(&mut self, c: parse::Count) { - match c { - parse::CountImplied | parse::CountIs(..) => {} - parse::CountIsParam(i) => { - self.verify_arg_type(Exact(i), Count); - } - parse::CountIsName(s) => { - self.verify_arg_type(Named(s), Count); - } - } - } - - fn describe_num_args(&self) -> Cow<'_, str> { - match self.args.len() { - 0 => "no arguments were given".into(), - 1 => "there is 1 argument".into(), - x => format!("there are {} arguments", x).into(), - } - } - - /// Handle invalid references to positional arguments. Output different - /// errors for the case where all arguments are positional and for when - /// there are named arguments or numbered positional arguments in the - /// format string. - fn report_invalid_references(&self, numbered_position_args: bool) { - let mut e; - let sp = if self.is_literal { - // Point at the formatting arguments. - MultiSpan::from_spans(self.arg_spans.clone()) - } else { - MultiSpan::from_span(self.fmtsp) - }; - let refs = - self.invalid_refs.iter().map(|(r, pos)| (r.to_string(), self.arg_spans.get(*pos))); - - let mut zero_based_note = false; - - let count = self.pieces.len() - + self.arg_with_formatting.iter().filter(|fmt| fmt.precision_span.is_some()).count(); - if self.names.is_empty() && !numbered_position_args && count != self.args.len() { - e = self.ecx.struct_span_err( - sp, - &format!( - "{} positional argument{} in format string, but {}", - count, - pluralize!(count), - self.describe_num_args(), - ), - ); - for arg in &self.args { - // Point at the arguments that will be formatted. - e.span_label(arg.span, ""); - } - } else { - let (mut refs, spans): (Vec<_>, Vec<_>) = refs.unzip(); - // Avoid `invalid reference to positional arguments 7 and 7 (there is 1 argument)` - // for `println!("{7:7$}", 1);` - refs.sort(); - refs.dedup(); - let (arg_list, mut sp) = if refs.len() == 1 { - let spans: Vec<_> = spans.into_iter().filter_map(|sp| sp.map(|sp| *sp)).collect(); - ( - format!("argument {}", refs[0]), - if spans.is_empty() { - MultiSpan::from_span(self.fmtsp) - } else { - MultiSpan::from_spans(spans) - }, - ) - } else { - let pos = MultiSpan::from_spans(spans.into_iter().map(|s| *s.unwrap()).collect()); - let reg = refs.pop().unwrap(); - (format!("arguments {head} and {tail}", head = refs.join(", "), tail = reg,), pos) - }; - if !self.is_literal { - sp = MultiSpan::from_span(self.fmtsp); - } - - e = self.ecx.struct_span_err( - sp, - &format!( - "invalid reference to positional {} ({})", - arg_list, - self.describe_num_args() - ), - ); - zero_based_note = true; - }; - - for fmt in &self.arg_with_formatting { - if let Some(span) = fmt.precision_span { - let span = self.fmtsp.from_inner(span); - match fmt.precision { - parse::CountIsParam(pos) if pos > self.args.len() => { - e.span_label( - span, - &format!( - "this precision flag expects an `usize` argument at position {}, \ - but {}", - pos, - self.describe_num_args(), - ), - ); - zero_based_note = true; - } - parse::CountIsParam(pos) => { - let count = self.pieces.len() - + self - .arg_with_formatting - .iter() - .filter(|fmt| fmt.precision_span.is_some()) - .count(); - e.span_label(span, &format!( - "this precision flag adds an extra required argument at position {}, \ - which is why there {} expected", - pos, - if count == 1 { - "is 1 argument".to_string() - } else { - format!("are {} arguments", count) - }, - )); - if let Some(arg) = self.args.get(pos) { - e.span_label( - arg.span, - "this parameter corresponds to the precision flag", - ); - } - zero_based_note = true; - } - _ => {} - } - } - if let Some(span) = fmt.width_span { - let span = self.fmtsp.from_inner(span); - match fmt.width { - parse::CountIsParam(pos) if pos > self.args.len() => { - e.span_label( - span, - &format!( - "this width flag expects an `usize` argument at position {}, \ - but {}", - pos, - self.describe_num_args(), - ), - ); - zero_based_note = true; - } - _ => {} - } - } - } - if zero_based_note { - e.note("positional arguments are zero-based"); - } - if !self.arg_with_formatting.is_empty() { - e.note( - "for information about formatting flags, visit \ - https://doc.rust-lang.org/std/fmt/index.html", - ); - } - - e.emit(); - } - - /// Actually verifies and tracks a given format placeholder - /// (a.k.a. argument). - fn verify_arg_type(&mut self, arg: Position, ty: ArgumentType) { - match arg { - Exact(arg) => { - if self.args.len() <= arg { - self.invalid_refs.push((arg, self.curpiece)); - return; - } - match ty { - Placeholder(_) => { - // record every (position, type) combination only once - let ref mut seen_ty = self.arg_unique_types[arg]; - let i = seen_ty.iter().position(|x| *x == ty).unwrap_or_else(|| { - let i = seen_ty.len(); - seen_ty.push(ty); - i - }); - self.arg_types[arg].push(i); - } - Count => { - if let Entry::Vacant(e) = self.count_positions.entry(arg) { - let i = self.count_positions_count; - e.insert(i); - self.count_args.push(Exact(arg)); - self.count_positions_count += 1; - } - } - } - } - - Named(name) => { - match self.names.get(&name) { - Some(&idx) => { - // Treat as positional arg. - self.verify_arg_type(Exact(idx), ty) - } - None => { - let msg = format!("there is no argument named `{}`", name); - let sp = if self.is_literal { - *self.arg_spans.get(self.curpiece).unwrap_or(&self.fmtsp) - } else { - self.fmtsp - }; - let mut err = self.ecx.struct_span_err(sp, &msg[..]); - err.emit(); - } - } - } - } - } - - /// Builds the mapping between format placeholders and argument objects. - fn build_index_map(&mut self) { - // NOTE: Keep the ordering the same as `into_expr`'s expansion would do! - let args_len = self.args.len(); - self.arg_index_map.reserve(args_len); - - let mut sofar = 0usize; - - // Map the arguments - for i in 0..args_len { - let ref arg_types = self.arg_types[i]; - let arg_offsets = arg_types.iter().map(|offset| sofar + *offset).collect::>(); - self.arg_index_map.push(arg_offsets); - sofar += self.arg_unique_types[i].len(); - } - - // Record starting index for counts, which appear just after arguments - self.count_args_index_offset = sofar; - } - - fn rtpath(ecx: &ExtCtxt<'_>, s: &str) -> Vec { - ecx.std_path(&[sym::fmt, sym::rt, sym::v1, Symbol::intern(s)]) - } - - fn build_count(&self, c: parse::Count) -> P { - let sp = self.macsp; - let count = |c, arg| { - let mut path = Context::rtpath(self.ecx, "Count"); - path.push(self.ecx.ident_of(c, sp)); - match arg { - Some(arg) => self.ecx.expr_call_global(sp, path, vec![arg]), - None => self.ecx.expr_path(self.ecx.path_global(sp, path)), - } - }; - match c { - parse::CountIs(i) => count("Is", Some(self.ecx.expr_usize(sp, i))), - parse::CountIsParam(i) => { - // This needs mapping too, as `i` is referring to a macro - // argument. If `i` is not found in `count_positions` then - // the error had already been emitted elsewhere. - let i = self.count_positions.get(&i).cloned().unwrap_or(0) - + self.count_args_index_offset; - count("Param", Some(self.ecx.expr_usize(sp, i))) - } - parse::CountImplied => count("Implied", None), - // should never be the case, names are already resolved - parse::CountIsName(_) => panic!("should never happen"), - } - } - - /// Build a literal expression from the accumulated string literals - fn build_literal_string(&mut self) -> P { - let sp = self.fmtsp; - let s = Symbol::intern(&self.literal); - self.literal.clear(); - self.ecx.expr_str(sp, s) - } - - /// Builds a static `rt::Argument` from a `parse::Piece` or append - /// to the `literal` string. - fn build_piece( - &mut self, - piece: &parse::Piece<'a>, - arg_index_consumed: &mut Vec, - ) -> Option> { - let sp = self.macsp; - match *piece { - parse::String(s) => { - self.literal.push_str(s); - None - } - parse::NextArgument(ref arg) => { - // Build the position - let pos = { - let pos = |c, arg| { - let mut path = Context::rtpath(self.ecx, "Position"); - path.push(self.ecx.ident_of(c, sp)); - match arg { - Some(i) => { - let arg = self.ecx.expr_usize(sp, i); - self.ecx.expr_call_global(sp, path, vec![arg]) - } - None => self.ecx.expr_path(self.ecx.path_global(sp, path)), - } - }; - match arg.position { - parse::ArgumentIs(i) | parse::ArgumentImplicitlyIs(i) => { - // Map to index in final generated argument array - // in case of multiple types specified - let arg_idx = match arg_index_consumed.get_mut(i) { - None => 0, // error already emitted elsewhere - Some(offset) => { - let ref idx_map = self.arg_index_map[i]; - // unwrap_or branch: error already emitted elsewhere - let arg_idx = *idx_map.get(*offset).unwrap_or(&0); - *offset += 1; - arg_idx - } - }; - pos("At", Some(arg_idx)) - } - - // should never be the case, because names are already - // resolved. - parse::ArgumentNamed(_) => panic!("should never happen"), - } - }; - - let simple_arg = parse::Argument { - position: { - // We don't have ArgumentNext any more, so we have to - // track the current argument ourselves. - let i = self.curarg; - self.curarg += 1; - parse::ArgumentIs(i) - }, - format: parse::FormatSpec { - fill: arg.format.fill, - align: parse::AlignUnknown, - flags: 0, - precision: parse::CountImplied, - precision_span: None, - width: parse::CountImplied, - width_span: None, - ty: arg.format.ty, - ty_span: arg.format.ty_span, - }, - }; - - let fill = arg.format.fill.unwrap_or(' '); - - let pos_simple = arg.position.index() == simple_arg.position.index(); - - if arg.format.precision_span.is_some() || arg.format.width_span.is_some() { - self.arg_with_formatting.push(arg.format); - } - if !pos_simple || arg.format != simple_arg.format || fill != ' ' { - self.all_pieces_simple = false; - } - - // Build the format - let fill = self.ecx.expr_lit(sp, ast::LitKind::Char(fill)); - let align = |name| { - let mut p = Context::rtpath(self.ecx, "Alignment"); - p.push(self.ecx.ident_of(name, sp)); - self.ecx.path_global(sp, p) - }; - let align = match arg.format.align { - parse::AlignLeft => align("Left"), - parse::AlignRight => align("Right"), - parse::AlignCenter => align("Center"), - parse::AlignUnknown => align("Unknown"), - }; - let align = self.ecx.expr_path(align); - let flags = self.ecx.expr_u32(sp, arg.format.flags); - let prec = self.build_count(arg.format.precision); - let width = self.build_count(arg.format.width); - let path = self.ecx.path_global(sp, Context::rtpath(self.ecx, "FormatSpec")); - let fmt = self.ecx.expr_struct( - sp, - path, - vec![ - self.ecx.field_imm(sp, self.ecx.ident_of("fill", sp), fill), - self.ecx.field_imm(sp, self.ecx.ident_of("align", sp), align), - self.ecx.field_imm(sp, self.ecx.ident_of("flags", sp), flags), - self.ecx.field_imm(sp, self.ecx.ident_of("precision", sp), prec), - self.ecx.field_imm(sp, self.ecx.ident_of("width", sp), width), - ], - ); - - let path = self.ecx.path_global(sp, Context::rtpath(self.ecx, "Argument")); - Some(self.ecx.expr_struct( - sp, - path, - vec![ - self.ecx.field_imm(sp, self.ecx.ident_of("position", sp), pos), - self.ecx.field_imm(sp, self.ecx.ident_of("format", sp), fmt), - ], - )) - } - } - } - - /// Actually builds the expression which the format_args! block will be - /// expanded to. - fn into_expr(self) -> P { - let mut locals = - Vec::with_capacity((0..self.args.len()).map(|i| self.arg_unique_types[i].len()).sum()); - let mut counts = Vec::with_capacity(self.count_args.len()); - let mut pats = Vec::with_capacity(self.args.len()); - let mut heads = Vec::with_capacity(self.args.len()); - - let names_pos: Vec<_> = (0..self.args.len()) - .map(|i| self.ecx.ident_of(&format!("arg{}", i), self.macsp)) - .collect(); - - // First, build up the static array which will become our precompiled - // format "string" - let pieces = self.ecx.expr_vec_slice(self.fmtsp, self.str_pieces); - - // Before consuming the expressions, we have to remember spans for - // count arguments as they are now generated separate from other - // arguments, hence have no access to the `P`'s. - let spans_pos: Vec<_> = self.args.iter().map(|e| e.span.clone()).collect(); - - // Right now there is a bug such that for the expression: - // foo(bar(&1)) - // the lifetime of `1` doesn't outlast the call to `bar`, so it's not - // valid for the call to `foo`. To work around this all arguments to the - // format! string are shoved into locals. Furthermore, we shove the address - // of each variable because we don't want to move out of the arguments - // passed to this function. - for (i, e) in self.args.into_iter().enumerate() { - let name = names_pos[i]; - let span = self.ecx.with_def_site_ctxt(e.span); - pats.push(self.ecx.pat_ident(span, name)); - for ref arg_ty in self.arg_unique_types[i].iter() { - locals.push(Context::format_arg(self.ecx, self.macsp, e.span, arg_ty, name)); - } - heads.push(self.ecx.expr_addr_of(e.span, e)); - } - for pos in self.count_args { - let index = match pos { - Exact(i) => i, - _ => panic!("should never happen"), - }; - let name = names_pos[index]; - let span = spans_pos[index]; - counts.push(Context::format_arg(self.ecx, self.macsp, span, &Count, name)); - } - - // Now create a vector containing all the arguments - let args = locals.into_iter().chain(counts.into_iter()); - - let args_array = self.ecx.expr_vec(self.macsp, args.collect()); - - // Constructs an AST equivalent to: - // - // match (&arg0, &arg1) { - // (tmp0, tmp1) => args_array - // } - // - // It was: - // - // let tmp0 = &arg0; - // let tmp1 = &arg1; - // args_array - // - // Because of #11585 the new temporary lifetime rule, the enclosing - // statements for these temporaries become the let's themselves. - // If one or more of them are RefCell's, RefCell borrow() will also - // end there; they don't last long enough for args_array to use them. - // The match expression solves the scope problem. - // - // Note, it may also very well be transformed to: - // - // match arg0 { - // ref tmp0 => { - // match arg1 => { - // ref tmp1 => args_array } } } - // - // But the nested match expression is proved to perform not as well - // as series of let's; the first approach does. - let pat = self.ecx.pat_tuple(self.macsp, pats); - let arm = self.ecx.arm(self.macsp, pat, args_array); - let head = self.ecx.expr(self.macsp, ast::ExprKind::Tup(heads)); - let result = self.ecx.expr_match(self.macsp, head, vec![arm]); - - let args_slice = self.ecx.expr_addr_of(self.macsp, result); - - // Now create the fmt::Arguments struct with all our locals we created. - let (fn_name, fn_args) = if self.all_pieces_simple { - ("new_v1", vec![pieces, args_slice]) - } else { - // Build up the static array which will store our precompiled - // nonstandard placeholders, if there are any. - let fmt = self.ecx.expr_vec_slice(self.macsp, self.pieces); - - ("new_v1_formatted", vec![pieces, args_slice, fmt]) - }; - - let path = self.ecx.std_path(&[sym::fmt, sym::Arguments, Symbol::intern(fn_name)]); - self.ecx.expr_call_global(self.macsp, path, fn_args) - } - - fn format_arg( - ecx: &ExtCtxt<'_>, - macsp: Span, - mut sp: Span, - ty: &ArgumentType, - arg: ast::Ident, - ) -> P { - sp = ecx.with_def_site_ctxt(sp); - let arg = ecx.expr_ident(sp, arg); - let trait_ = match *ty { - Placeholder(trait_) if trait_ == "" => return DummyResult::raw_expr(sp, true), - Placeholder(trait_) => trait_, - Count => { - let path = ecx.std_path(&[sym::fmt, sym::ArgumentV1, sym::from_usize]); - return ecx.expr_call_global(macsp, path, vec![arg]); - } - }; - - let path = ecx.std_path(&[sym::fmt, Symbol::intern(trait_), sym::fmt]); - let format_fn = ecx.path_global(sp, path); - let path = ecx.std_path(&[sym::fmt, sym::ArgumentV1, sym::new]); - ecx.expr_call_global(macsp, path, vec![arg, ecx.expr_path(format_fn)]) - } -} - -fn expand_format_args_impl<'cx>( - ecx: &'cx mut ExtCtxt<'_>, - mut sp: Span, - tts: TokenStream, - nl: bool, -) -> Box { - sp = ecx.with_def_site_ctxt(sp); - match parse_args(ecx, sp, tts) { - Ok((efmt, args, names)) => { - MacEager::expr(expand_preparsed_format_args(ecx, sp, efmt, args, names, nl)) - } - Err(mut err) => { - err.emit(); - DummyResult::any(sp) - } - } -} - -pub fn expand_format_args<'cx>( - ecx: &'cx mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - expand_format_args_impl(ecx, sp, tts, false) -} - -pub fn expand_format_args_nl<'cx>( - ecx: &'cx mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - expand_format_args_impl(ecx, sp, tts, true) -} - -/// Take the various parts of `format_args!(efmt, args..., name=names...)` -/// and construct the appropriate formatting expression. -pub fn expand_preparsed_format_args( - ecx: &mut ExtCtxt<'_>, - sp: Span, - efmt: P, - args: Vec>, - names: FxHashMap, - append_newline: bool, -) -> P { - // NOTE: this verbose way of initializing `Vec>` is because - // `ArgumentType` does not derive `Clone`. - let arg_types: Vec<_> = (0..args.len()).map(|_| Vec::new()).collect(); - let arg_unique_types: Vec<_> = (0..args.len()).map(|_| Vec::new()).collect(); - - let mut macsp = ecx.call_site(); - macsp = ecx.with_def_site_ctxt(macsp); - - let msg = "format argument must be a string literal"; - let fmt_sp = efmt.span; - let (fmt_str, fmt_style, fmt_span) = match expr_to_spanned_string(ecx, efmt, msg) { - Ok(mut fmt) if append_newline => { - fmt.0 = Symbol::intern(&format!("{}\n", fmt.0)); - fmt - } - Ok(fmt) => fmt, - Err(err) => { - if let Some(mut err) = err { - let sugg_fmt = match args.len() { - 0 => "{}".to_string(), - _ => format!("{}{{}}", "{} ".repeat(args.len())), - }; - err.span_suggestion( - fmt_sp.shrink_to_lo(), - "you might be missing a string literal to format with", - format!("\"{}\", ", sugg_fmt), - Applicability::MaybeIncorrect, - ); - err.emit(); - } - return DummyResult::raw_expr(sp, true); - } - }; - - let (is_literal, fmt_snippet) = match ecx.source_map().span_to_snippet(fmt_sp) { - Ok(s) => (s.starts_with("\"") || s.starts_with("r#"), Some(s)), - _ => (false, None), - }; - - let str_style = match fmt_style { - ast::StrStyle::Cooked => None, - ast::StrStyle::Raw(raw) => Some(raw as usize), - }; - - /// Finds the indices of all characters that have been processed and differ between the actual - /// written code (code snippet) and the `InternedString` that get's processed in the `Parser` - /// in order to properly synthethise the intra-string `Span`s for error diagnostics. - fn find_skips(snippet: &str, is_raw: bool) -> Vec { - let mut eat_ws = false; - let mut s = snippet.chars().enumerate().peekable(); - let mut skips = vec![]; - while let Some((pos, c)) = s.next() { - match (c, s.peek()) { - // skip whitespace and empty lines ending in '\\' - ('\\', Some((next_pos, '\n'))) if !is_raw => { - eat_ws = true; - skips.push(pos); - skips.push(*next_pos); - let _ = s.next(); - } - ('\\', Some((next_pos, '\n'))) - | ('\\', Some((next_pos, 'n'))) - | ('\\', Some((next_pos, 't'))) - if eat_ws => - { - skips.push(pos); - skips.push(*next_pos); - let _ = s.next(); - } - (' ', _) | ('\n', _) | ('\t', _) if eat_ws => { - skips.push(pos); - } - ('\\', Some((next_pos, 'n'))) - | ('\\', Some((next_pos, 't'))) - | ('\\', Some((next_pos, '0'))) - | ('\\', Some((next_pos, '\\'))) - | ('\\', Some((next_pos, '\''))) - | ('\\', Some((next_pos, '\"'))) => { - skips.push(*next_pos); - let _ = s.next(); - } - ('\\', Some((_, 'x'))) if !is_raw => { - for _ in 0..3 { - // consume `\xAB` literal - if let Some((pos, _)) = s.next() { - skips.push(pos); - } else { - break; - } - } - } - ('\\', Some((_, 'u'))) if !is_raw => { - if let Some((pos, _)) = s.next() { - skips.push(pos); - } - if let Some((next_pos, next_c)) = s.next() { - if next_c == '{' { - skips.push(next_pos); - let mut i = 0; // consume up to 6 hexanumeric chars + closing `}` - while let (Some((next_pos, c)), true) = (s.next(), i < 7) { - if c.is_digit(16) { - skips.push(next_pos); - } else if c == '}' { - skips.push(next_pos); - break; - } else { - break; - } - i += 1; - } - } else if next_c.is_digit(16) { - skips.push(next_pos); - // We suggest adding `{` and `}` when appropriate, accept it here as if - // it were correct - let mut i = 0; // consume up to 6 hexanumeric chars - while let (Some((next_pos, c)), _) = (s.next(), i < 6) { - if c.is_digit(16) { - skips.push(next_pos); - } else { - break; - } - i += 1; - } - } - } - } - _ if eat_ws => { - // `take_while(|c| c.is_whitespace())` - eat_ws = false; - } - _ => {} - } - } - skips - } - - let skips = if let (true, Some(ref snippet)) = (is_literal, fmt_snippet.as_ref()) { - let r_start = str_style.map(|r| r + 1).unwrap_or(0); - let r_end = str_style.map(|r| r).unwrap_or(0); - let s = &snippet[r_start + 1..snippet.len() - r_end - 1]; - find_skips(s, str_style.is_some()) - } else { - vec![] - }; - - let fmt_str = &fmt_str.as_str(); // for the suggestions below - let mut parser = parse::Parser::new(fmt_str, str_style, skips, append_newline); - - let mut unverified_pieces = Vec::new(); - while let Some(piece) = parser.next() { - if !parser.errors.is_empty() { - break; - } else { - unverified_pieces.push(piece); - } - } - - if !parser.errors.is_empty() { - let err = parser.errors.remove(0); - let sp = fmt_span.from_inner(err.span); - let mut e = ecx.struct_span_err(sp, &format!("invalid format string: {}", err.description)); - e.span_label(sp, err.label + " in format string"); - if let Some(note) = err.note { - e.note(¬e); - } - if let Some((label, span)) = err.secondary_label { - let sp = fmt_span.from_inner(span); - e.span_label(sp, label); - } - e.emit(); - return DummyResult::raw_expr(sp, true); - } - - let arg_spans = parser.arg_places.iter().map(|span| fmt_span.from_inner(*span)).collect(); - - let named_pos: FxHashSet = names.values().cloned().collect(); - - let mut cx = Context { - ecx, - args, - arg_types, - arg_unique_types, - names, - curarg: 0, - curpiece: 0, - arg_index_map: Vec::new(), - count_args: Vec::new(), - count_positions: FxHashMap::default(), - count_positions_count: 0, - count_args_index_offset: 0, - literal: String::new(), - pieces: Vec::with_capacity(unverified_pieces.len()), - str_pieces: Vec::with_capacity(unverified_pieces.len()), - all_pieces_simple: true, - macsp, - fmtsp: fmt_span, - invalid_refs: Vec::new(), - arg_spans, - arg_with_formatting: Vec::new(), - is_literal, - }; - - // This needs to happen *after* the Parser has consumed all pieces to create all the spans - let pieces = unverified_pieces - .into_iter() - .map(|mut piece| { - cx.verify_piece(&piece); - cx.resolve_name_inplace(&mut piece); - piece - }) - .collect::>(); - - let numbered_position_args = pieces.iter().any(|arg: &parse::Piece<'_>| match *arg { - parse::String(_) => false, - parse::NextArgument(arg) => match arg.position { - parse::Position::ArgumentIs(_) => true, - _ => false, - }, - }); - - cx.build_index_map(); - - let mut arg_index_consumed = vec![0usize; cx.arg_index_map.len()]; - - for piece in pieces { - if let Some(piece) = cx.build_piece(&piece, &mut arg_index_consumed) { - let s = cx.build_literal_string(); - cx.str_pieces.push(s); - cx.pieces.push(piece); - } - } - - if !cx.literal.is_empty() { - let s = cx.build_literal_string(); - cx.str_pieces.push(s); - } - - if cx.invalid_refs.len() >= 1 { - cx.report_invalid_references(numbered_position_args); - } - - // Make sure that all arguments were used and all arguments have types. - let errs = cx - .arg_types - .iter() - .enumerate() - .filter(|(i, ty)| ty.is_empty() && !cx.count_positions.contains_key(&i)) - .map(|(i, _)| { - let msg = if named_pos.contains(&i) { - // named argument - "named argument never used" - } else { - // positional argument - "argument never used" - }; - (cx.args[i].span, msg) - }) - .collect::>(); - - let errs_len = errs.len(); - if !errs.is_empty() { - let args_used = cx.arg_types.len() - errs_len; - let args_unused = errs_len; - - let mut diag = { - if errs_len == 1 { - let (sp, msg) = errs.into_iter().next().unwrap(); - let mut diag = cx.ecx.struct_span_err(sp, msg); - diag.span_label(sp, msg); - diag - } else { - let mut diag = cx.ecx.struct_span_err( - errs.iter().map(|&(sp, _)| sp).collect::>(), - "multiple unused formatting arguments", - ); - diag.span_label(cx.fmtsp, "multiple missing formatting specifiers"); - for (sp, msg) in errs { - diag.span_label(sp, msg); - } - diag - } - }; - - // Used to ensure we only report translations for *one* kind of foreign format. - let mut found_foreign = false; - // Decide if we want to look for foreign formatting directives. - if args_used < args_unused { - use super::format_foreign as foreign; - - // The set of foreign substitutions we've explained. This prevents spamming the user - // with `%d should be written as {}` over and over again. - let mut explained = FxHashSet::default(); - - macro_rules! check_foreign { - ($kind:ident) => {{ - let mut show_doc_note = false; - - let mut suggestions = vec![]; - // account for `"` and account for raw strings `r#` - let padding = str_style.map(|i| i + 2).unwrap_or(1); - for sub in foreign::$kind::iter_subs(fmt_str, padding) { - let trn = match sub.translate() { - Some(trn) => trn, - - // If it has no translation, don't call it out specifically. - None => continue, - }; - - let pos = sub.position(); - let sub = String::from(sub.as_str()); - if explained.contains(&sub) { - continue; - } - explained.insert(sub.clone()); - - if !found_foreign { - found_foreign = true; - show_doc_note = true; - } - - if let Some(inner_sp) = pos { - let sp = fmt_sp.from_inner(inner_sp); - suggestions.push((sp, trn)); - } else { - diag.help(&format!("`{}` should be written as `{}`", sub, trn)); - } - } - - if show_doc_note { - diag.note(concat!( - stringify!($kind), - " formatting not supported; see the documentation for `std::fmt`", - )); - } - if suggestions.len() > 0 { - diag.multipart_suggestion( - "format specifiers use curly braces", - suggestions, - Applicability::MachineApplicable, - ); - } - }}; - } - - check_foreign!(printf); - if !found_foreign { - check_foreign!(shell); - } - } - if !found_foreign && errs_len == 1 { - diag.span_label(cx.fmtsp, "formatting specifier missing"); - } - - diag.emit(); - } - - cx.into_expr() -} diff --git a/src/libsyntax_ext/format_foreign.rs b/src/libsyntax_ext/format_foreign.rs deleted file mode 100644 index 9c151cf94b4..00000000000 --- a/src/libsyntax_ext/format_foreign.rs +++ /dev/null @@ -1,827 +0,0 @@ -pub mod printf { - use super::strcursor::StrCursor as Cur; - use syntax_pos::InnerSpan; - - /// Represents a single `printf`-style substitution. - #[derive(Clone, PartialEq, Debug)] - pub enum Substitution<'a> { - /// A formatted output substitution with its internal byte offset. - Format(Format<'a>), - /// A literal `%%` escape. - Escape, - } - - impl<'a> Substitution<'a> { - pub fn as_str(&self) -> &str { - match *self { - Substitution::Format(ref fmt) => fmt.span, - Substitution::Escape => "%%", - } - } - - pub fn position(&self) -> Option { - match *self { - Substitution::Format(ref fmt) => Some(fmt.position), - _ => None, - } - } - - pub fn set_position(&mut self, start: usize, end: usize) { - match self { - Substitution::Format(ref mut fmt) => { - fmt.position = InnerSpan::new(start, end); - } - _ => {} - } - } - - /// Translate this substitution into an equivalent Rust formatting directive. - /// - /// This ignores cases where the substitution does not have an exact equivalent, or where - /// the substitution would be unnecessary. - pub fn translate(&self) -> Option { - match *self { - Substitution::Format(ref fmt) => fmt.translate(), - Substitution::Escape => None, - } - } - } - - #[derive(Clone, PartialEq, Debug)] - /// A single `printf`-style formatting directive. - pub struct Format<'a> { - /// The entire original formatting directive. - pub span: &'a str, - /// The (1-based) parameter to be converted. - pub parameter: Option, - /// Formatting flags. - pub flags: &'a str, - /// Minimum width of the output. - pub width: Option, - /// Precision of the conversion. - pub precision: Option, - /// Length modifier for the conversion. - pub length: Option<&'a str>, - /// Type of parameter being converted. - pub type_: &'a str, - /// Byte offset for the start and end of this formatting directive. - pub position: InnerSpan, - } - - impl Format<'_> { - /// Translate this directive into an equivalent Rust formatting directive. - /// - /// Returns `None` in cases where the `printf` directive does not have an exact Rust - /// equivalent, rather than guessing. - pub fn translate(&self) -> Option { - use std::fmt::Write; - - let (c_alt, c_zero, c_left, c_plus) = { - let mut c_alt = false; - let mut c_zero = false; - let mut c_left = false; - let mut c_plus = false; - for c in self.flags.chars() { - match c { - '#' => c_alt = true, - '0' => c_zero = true, - '-' => c_left = true, - '+' => c_plus = true, - _ => return None, - } - } - (c_alt, c_zero, c_left, c_plus) - }; - - // Has a special form in Rust for numbers. - let fill = c_zero.then_some("0"); - - let align = c_left.then_some("<"); - - // Rust doesn't have an equivalent to the `' '` flag. - let sign = c_plus.then_some("+"); - - // Not *quite* the same, depending on the type... - let alt = c_alt; - - let width = match self.width { - Some(Num::Next) => { - // NOTE: Rust doesn't support this. - return None; - } - w @ Some(Num::Arg(_)) => w, - w @ Some(Num::Num(_)) => w, - None => None, - }; - - let precision = self.precision; - - // NOTE: although length *can* have an effect, we can't duplicate the effect in Rust, so - // we just ignore it. - - let (type_, use_zero_fill, is_int) = match self.type_ { - "d" | "i" | "u" => (None, true, true), - "f" | "F" => (None, false, false), - "s" | "c" => (None, false, false), - "e" | "E" => (Some(self.type_), true, false), - "x" | "X" | "o" => (Some(self.type_), true, true), - "p" => (Some(self.type_), false, true), - "g" => (Some("e"), true, false), - "G" => (Some("E"), true, false), - _ => return None, - }; - - let (fill, width, precision) = match (is_int, width, precision) { - (true, Some(_), Some(_)) => { - // Rust can't duplicate this insanity. - return None; - } - (true, None, Some(p)) => (Some("0"), Some(p), None), - (true, w, None) => (fill, w, None), - (false, w, p) => (fill, w, p), - }; - - let align = match (self.type_, width.is_some(), align.is_some()) { - ("s", true, false) => Some(">"), - _ => align, - }; - - let (fill, zero_fill) = match (fill, use_zero_fill) { - (Some("0"), true) => (None, true), - (fill, _) => (fill, false), - }; - - let alt = match type_ { - Some("x") | Some("X") => alt, - _ => false, - }; - - let has_options = fill.is_some() - || align.is_some() - || sign.is_some() - || alt - || zero_fill - || width.is_some() - || precision.is_some() - || type_.is_some(); - - // Initialise with a rough guess. - let cap = self.span.len() + if has_options { 2 } else { 0 }; - let mut s = String::with_capacity(cap); - - s.push_str("{"); - - if let Some(arg) = self.parameter { - write!(s, "{}", arg.checked_sub(1)?).ok()?; - } - - if has_options { - s.push_str(":"); - - let align = if let Some(fill) = fill { - s.push_str(fill); - align.or(Some(">")) - } else { - align - }; - - if let Some(align) = align { - s.push_str(align); - } - - if let Some(sign) = sign { - s.push_str(sign); - } - - if alt { - s.push_str("#"); - } - - if zero_fill { - s.push_str("0"); - } - - if let Some(width) = width { - width.translate(&mut s).ok()?; - } - - if let Some(precision) = precision { - s.push_str("."); - precision.translate(&mut s).ok()?; - } - - if let Some(type_) = type_ { - s.push_str(type_); - } - } - - s.push_str("}"); - Some(s) - } - } - - /// A general number used in a `printf` formatting directive. - #[derive(Copy, Clone, PartialEq, Debug)] - pub enum Num { - // The range of these values is technically bounded by `NL_ARGMAX`... but, at least for GNU - // libc, it apparently has no real fixed limit. A `u16` is used here on the basis that it - // is *vanishingly* unlikely that *anyone* is going to try formatting something wider, or - // with more precision, than 32 thousand positions which is so wide it couldn't possibly fit - // on a screen. - /// A specific, fixed value. - Num(u16), - /// The value is derived from a positional argument. - Arg(u16), - /// The value is derived from the "next" unconverted argument. - Next, - } - - impl Num { - fn from_str(s: &str, arg: Option<&str>) -> Self { - if let Some(arg) = arg { - Num::Arg(arg.parse().unwrap_or_else(|_| panic!("invalid format arg `{:?}`", arg))) - } else if s == "*" { - Num::Next - } else { - Num::Num(s.parse().unwrap_or_else(|_| panic!("invalid format num `{:?}`", s))) - } - } - - fn translate(&self, s: &mut String) -> std::fmt::Result { - use std::fmt::Write; - match *self { - Num::Num(n) => write!(s, "{}", n), - Num::Arg(n) => { - let n = n.checked_sub(1).ok_or(std::fmt::Error)?; - write!(s, "{}$", n) - } - Num::Next => write!(s, "*"), - } - } - } - - /// Returns an iterator over all substitutions in a given string. - pub fn iter_subs(s: &str, start_pos: usize) -> Substitutions<'_> { - Substitutions { s, pos: start_pos } - } - - /// Iterator over substitutions in a string. - pub struct Substitutions<'a> { - s: &'a str, - pos: usize, - } - - impl<'a> Iterator for Substitutions<'a> { - type Item = Substitution<'a>; - fn next(&mut self) -> Option { - let (mut sub, tail) = parse_next_substitution(self.s)?; - self.s = tail; - match sub { - Substitution::Format(_) => { - if let Some(inner_span) = sub.position() { - sub.set_position(inner_span.start + self.pos, inner_span.end + self.pos); - self.pos += inner_span.end; - } - } - Substitution::Escape => self.pos += 2, - } - Some(sub) - } - - fn size_hint(&self) -> (usize, Option) { - // Substitutions are at least 2 characters long. - (0, Some(self.s.len() / 2)) - } - } - - enum State { - Start, - Flags, - Width, - WidthArg, - Prec, - PrecInner, - Length, - Type, - } - - /// Parse the next substitution from the input string. - pub fn parse_next_substitution(s: &str) -> Option<(Substitution<'_>, &str)> { - use self::State::*; - - let at = { - let start = s.find('%')?; - match s[start + 1..].chars().next()? { - '%' => return Some((Substitution::Escape, &s[start + 2..])), - _ => { /* fall-through */ } - } - - Cur::new_at(&s[..], start) - }; - - // This is meant to be a translation of the following regex: - // - // ```regex - // (?x) - // ^ % - // (?: (?P \d+) \$ )? - // (?P [-+ 0\#']* ) - // (?P \d+ | \* (?: (?P \d+) \$ )? )? - // (?: \. (?P \d+ | \* (?: (?P \d+) \$ )? ) )? - // (?P - // # Standard - // hh | h | ll | l | L | z | j | t - // - // # Other - // | I32 | I64 | I | q - // )? - // (?P . ) - // ``` - - // Used to establish the full span at the end. - let start = at; - // The current position within the string. - let mut at = at.at_next_cp()?; - // `c` is the next codepoint, `next` is a cursor after it. - let (mut c, mut next) = at.next_cp()?; - - // Update `at`, `c`, and `next`, exiting if we're out of input. - macro_rules! move_to { - ($cur:expr) => {{ - at = $cur; - let (c_, next_) = at.next_cp()?; - c = c_; - next = next_; - }}; - } - - // Constructs a result when parsing fails. - // - // Note: `move` used to capture copies of the cursors as they are *now*. - let fallback = move || { - return Some(( - Substitution::Format(Format { - span: start.slice_between(next).unwrap(), - parameter: None, - flags: "", - width: None, - precision: None, - length: None, - type_: at.slice_between(next).unwrap(), - position: InnerSpan::new(start.at, next.at), - }), - next.slice_after(), - )); - }; - - // Next parsing state. - let mut state = Start; - - // Sadly, Rust isn't *quite* smart enough to know these *must* be initialised by the end. - let mut parameter: Option = None; - let mut flags: &str = ""; - let mut width: Option = None; - let mut precision: Option = None; - let mut length: Option<&str> = None; - let mut type_: &str = ""; - let end: Cur<'_>; - - if let Start = state { - match c { - '1'..='9' => { - let end = at_next_cp_while(next, is_digit); - match end.next_cp() { - // Yes, this *is* the parameter. - Some(('$', end2)) => { - state = Flags; - parameter = Some(at.slice_between(end).unwrap().parse().unwrap()); - move_to!(end2); - } - // Wait, no, actually, it's the width. - Some(_) => { - state = Prec; - parameter = None; - flags = ""; - width = Some(Num::from_str(at.slice_between(end).unwrap(), None)); - move_to!(end); - } - // It's invalid, is what it is. - None => return fallback(), - } - } - _ => { - state = Flags; - parameter = None; - move_to!(at); - } - } - } - - if let Flags = state { - let end = at_next_cp_while(at, is_flag); - state = Width; - flags = at.slice_between(end).unwrap(); - move_to!(end); - } - - if let Width = state { - match c { - '*' => { - state = WidthArg; - move_to!(next); - } - '1'..='9' => { - let end = at_next_cp_while(next, is_digit); - state = Prec; - width = Some(Num::from_str(at.slice_between(end).unwrap(), None)); - move_to!(end); - } - _ => { - state = Prec; - width = None; - move_to!(at); - } - } - } - - if let WidthArg = state { - let end = at_next_cp_while(at, is_digit); - match end.next_cp() { - Some(('$', end2)) => { - state = Prec; - width = Some(Num::from_str("", Some(at.slice_between(end).unwrap()))); - move_to!(end2); - } - _ => { - state = Prec; - width = Some(Num::Next); - move_to!(end); - } - } - } - - if let Prec = state { - match c { - '.' => { - state = PrecInner; - move_to!(next); - } - _ => { - state = Length; - precision = None; - move_to!(at); - } - } - } - - if let PrecInner = state { - match c { - '*' => { - let end = at_next_cp_while(next, is_digit); - match end.next_cp() { - Some(('$', end2)) => { - state = Length; - precision = Some(Num::from_str("*", next.slice_between(end))); - move_to!(end2); - } - _ => { - state = Length; - precision = Some(Num::Next); - move_to!(end); - } - } - } - '0'..='9' => { - let end = at_next_cp_while(next, is_digit); - state = Length; - precision = Some(Num::from_str(at.slice_between(end).unwrap(), None)); - move_to!(end); - } - _ => return fallback(), - } - } - - if let Length = state { - let c1_next1 = next.next_cp(); - match (c, c1_next1) { - ('h', Some(('h', next1))) | ('l', Some(('l', next1))) => { - state = Type; - length = Some(at.slice_between(next1).unwrap()); - move_to!(next1); - } - - ('h', _) | ('l', _) | ('L', _) | ('z', _) | ('j', _) | ('t', _) | ('q', _) => { - state = Type; - length = Some(at.slice_between(next).unwrap()); - move_to!(next); - } - - ('I', _) => { - let end = next - .at_next_cp() - .and_then(|end| end.at_next_cp()) - .map(|end| (next.slice_between(end).unwrap(), end)); - let end = match end { - Some(("32", end)) => end, - Some(("64", end)) => end, - _ => next, - }; - state = Type; - length = Some(at.slice_between(end).unwrap()); - move_to!(end); - } - - _ => { - state = Type; - length = None; - move_to!(at); - } - } - } - - if let Type = state { - drop(c); - type_ = at.slice_between(next).unwrap(); - - // Don't use `move_to!` here, as we *can* be at the end of the input. - at = next; - } - - drop(c); - drop(next); - - end = at; - let position = InnerSpan::new(start.at, end.at); - - let f = Format { - span: start.slice_between(end).unwrap(), - parameter, - flags, - width, - precision, - length, - type_, - position, - }; - Some((Substitution::Format(f), end.slice_after())) - } - - fn at_next_cp_while(mut cur: Cur<'_>, mut pred: F) -> Cur<'_> - where - F: FnMut(char) -> bool, - { - loop { - match cur.next_cp() { - Some((c, next)) => { - if pred(c) { - cur = next; - } else { - return cur; - } - } - None => return cur, - } - } - } - - fn is_digit(c: char) -> bool { - match c { - '0'..='9' => true, - _ => false, - } - } - - fn is_flag(c: char) -> bool { - match c { - '0' | '-' | '+' | ' ' | '#' | '\'' => true, - _ => false, - } - } - - #[cfg(test)] - mod tests; -} - -pub mod shell { - use super::strcursor::StrCursor as Cur; - use syntax_pos::InnerSpan; - - #[derive(Clone, PartialEq, Debug)] - pub enum Substitution<'a> { - Ordinal(u8, (usize, usize)), - Name(&'a str, (usize, usize)), - Escape((usize, usize)), - } - - impl Substitution<'_> { - pub fn as_str(&self) -> String { - match self { - Substitution::Ordinal(n, _) => format!("${}", n), - Substitution::Name(n, _) => format!("${}", n), - Substitution::Escape(_) => "$$".into(), - } - } - - pub fn position(&self) -> Option { - match self { - Substitution::Ordinal(_, pos) - | Substitution::Name(_, pos) - | Substitution::Escape(pos) => Some(InnerSpan::new(pos.0, pos.1)), - } - } - - pub fn set_position(&mut self, start: usize, end: usize) { - match self { - Substitution::Ordinal(_, ref mut pos) - | Substitution::Name(_, ref mut pos) - | Substitution::Escape(ref mut pos) => *pos = (start, end), - } - } - - pub fn translate(&self) -> Option { - match *self { - Substitution::Ordinal(n, _) => Some(format!("{{{}}}", n)), - Substitution::Name(n, _) => Some(format!("{{{}}}", n)), - Substitution::Escape(_) => None, - } - } - } - - /// Returns an iterator over all substitutions in a given string. - pub fn iter_subs(s: &str, start_pos: usize) -> Substitutions<'_> { - Substitutions { s, pos: start_pos } - } - - /// Iterator over substitutions in a string. - pub struct Substitutions<'a> { - s: &'a str, - pos: usize, - } - - impl<'a> Iterator for Substitutions<'a> { - type Item = Substitution<'a>; - fn next(&mut self) -> Option { - match parse_next_substitution(self.s) { - Some((mut sub, tail)) => { - self.s = tail; - if let Some(InnerSpan { start, end }) = sub.position() { - sub.set_position(start + self.pos, end + self.pos); - self.pos += end; - } - Some(sub) - } - None => None, - } - } - - fn size_hint(&self) -> (usize, Option) { - (0, Some(self.s.len())) - } - } - - /// Parse the next substitution from the input string. - pub fn parse_next_substitution(s: &str) -> Option<(Substitution<'_>, &str)> { - let at = { - let start = s.find('$')?; - match s[start + 1..].chars().next()? { - '$' => return Some((Substitution::Escape((start, start + 2)), &s[start + 2..])), - c @ '0'..='9' => { - let n = (c as u8) - b'0'; - return Some((Substitution::Ordinal(n, (start, start + 2)), &s[start + 2..])); - } - _ => { /* fall-through */ } - } - - Cur::new_at(&s[..], start) - }; - - let at = at.at_next_cp()?; - let (c, inner) = at.next_cp()?; - - if !is_ident_head(c) { - None - } else { - let end = at_next_cp_while(inner, is_ident_tail); - let slice = at.slice_between(end).unwrap(); - let start = at.at - 1; - let end_pos = at.at + slice.len(); - Some((Substitution::Name(slice, (start, end_pos)), end.slice_after())) - } - } - - fn at_next_cp_while(mut cur: Cur<'_>, mut pred: F) -> Cur<'_> - where - F: FnMut(char) -> bool, - { - loop { - match cur.next_cp() { - Some((c, next)) => { - if pred(c) { - cur = next; - } else { - return cur; - } - } - None => return cur, - } - } - } - - fn is_ident_head(c: char) -> bool { - match c { - 'a'..='z' | 'A'..='Z' | '_' => true, - _ => false, - } - } - - fn is_ident_tail(c: char) -> bool { - match c { - '0'..='9' => true, - c => is_ident_head(c), - } - } - - #[cfg(test)] - mod tests; -} - -mod strcursor { - pub struct StrCursor<'a> { - s: &'a str, - pub at: usize, - } - - impl<'a> StrCursor<'a> { - pub fn new_at(s: &'a str, at: usize) -> StrCursor<'a> { - StrCursor { s, at } - } - - pub fn at_next_cp(mut self) -> Option> { - match self.try_seek_right_cp() { - true => Some(self), - false => None, - } - } - - pub fn next_cp(mut self) -> Option<(char, StrCursor<'a>)> { - let cp = self.cp_after()?; - self.seek_right(cp.len_utf8()); - Some((cp, self)) - } - - fn slice_before(&self) -> &'a str { - &self.s[0..self.at] - } - - pub fn slice_after(&self) -> &'a str { - &self.s[self.at..] - } - - pub fn slice_between(&self, until: StrCursor<'a>) -> Option<&'a str> { - if !str_eq_literal(self.s, until.s) { - None - } else { - use std::cmp::{max, min}; - let beg = min(self.at, until.at); - let end = max(self.at, until.at); - Some(&self.s[beg..end]) - } - } - - fn cp_after(&self) -> Option { - self.slice_after().chars().next() - } - - fn try_seek_right_cp(&mut self) -> bool { - match self.slice_after().chars().next() { - Some(c) => { - self.at += c.len_utf8(); - true - } - None => false, - } - } - - fn seek_right(&mut self, bytes: usize) { - self.at += bytes; - } - } - - impl Copy for StrCursor<'_> {} - - impl<'a> Clone for StrCursor<'a> { - fn clone(&self) -> StrCursor<'a> { - *self - } - } - - impl std::fmt::Debug for StrCursor<'_> { - fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(fmt, "StrCursor({:?} | {:?})", self.slice_before(), self.slice_after()) - } - } - - fn str_eq_literal(a: &str, b: &str) -> bool { - a.as_bytes().as_ptr() == b.as_bytes().as_ptr() && a.len() == b.len() - } -} diff --git a/src/libsyntax_ext/format_foreign/printf/tests.rs b/src/libsyntax_ext/format_foreign/printf/tests.rs deleted file mode 100644 index b9a85a84d6c..00000000000 --- a/src/libsyntax_ext/format_foreign/printf/tests.rs +++ /dev/null @@ -1,145 +0,0 @@ -use super::{iter_subs, parse_next_substitution as pns, Format as F, Num as N, Substitution as S}; - -macro_rules! assert_eq_pnsat { - ($lhs:expr, $rhs:expr) => { - assert_eq!( - pns($lhs).and_then(|(s, _)| s.translate()), - $rhs.map(>::from) - ) - }; -} - -#[test] -fn test_escape() { - assert_eq!(pns("has no escapes"), None); - assert_eq!(pns("has no escapes, either %"), None); - assert_eq!(pns("*so* has a %% escape"), Some((S::Escape, " escape"))); - assert_eq!(pns("%% leading escape"), Some((S::Escape, " leading escape"))); - assert_eq!(pns("trailing escape %%"), Some((S::Escape, ""))); -} - -#[test] -fn test_parse() { - macro_rules! assert_pns_eq_sub { - ($in_:expr, { - $param:expr, $flags:expr, - $width:expr, $prec:expr, $len:expr, $type_:expr, - $pos:expr, - }) => { - assert_eq!( - pns(concat!($in_, "!")), - Some(( - S::Format(F { - span: $in_, - parameter: $param, - flags: $flags, - width: $width, - precision: $prec, - length: $len, - type_: $type_, - position: syntax_pos::InnerSpan::new($pos.0, $pos.1), - }), - "!" - )) - ) - }; - } - - assert_pns_eq_sub!("%!", - { None, "", None, None, None, "!", (0, 2), }); - assert_pns_eq_sub!("%c", - { None, "", None, None, None, "c", (0, 2), }); - assert_pns_eq_sub!("%s", - { None, "", None, None, None, "s", (0, 2), }); - assert_pns_eq_sub!("%06d", - { None, "0", Some(N::Num(6)), None, None, "d", (0, 4), }); - assert_pns_eq_sub!("%4.2f", - { None, "", Some(N::Num(4)), Some(N::Num(2)), None, "f", (0, 5), }); - assert_pns_eq_sub!("%#x", - { None, "#", None, None, None, "x", (0, 3), }); - assert_pns_eq_sub!("%-10s", - { None, "-", Some(N::Num(10)), None, None, "s", (0, 5), }); - assert_pns_eq_sub!("%*s", - { None, "", Some(N::Next), None, None, "s", (0, 3), }); - assert_pns_eq_sub!("%-10.*s", - { None, "-", Some(N::Num(10)), Some(N::Next), None, "s", (0, 7), }); - assert_pns_eq_sub!("%-*.*s", - { None, "-", Some(N::Next), Some(N::Next), None, "s", (0, 6), }); - assert_pns_eq_sub!("%.6i", - { None, "", None, Some(N::Num(6)), None, "i", (0, 4), }); - assert_pns_eq_sub!("%+i", - { None, "+", None, None, None, "i", (0, 3), }); - assert_pns_eq_sub!("%08X", - { None, "0", Some(N::Num(8)), None, None, "X", (0, 4), }); - assert_pns_eq_sub!("%lu", - { None, "", None, None, Some("l"), "u", (0, 3), }); - assert_pns_eq_sub!("%Iu", - { None, "", None, None, Some("I"), "u", (0, 3), }); - assert_pns_eq_sub!("%I32u", - { None, "", None, None, Some("I32"), "u", (0, 5), }); - assert_pns_eq_sub!("%I64u", - { None, "", None, None, Some("I64"), "u", (0, 5), }); - assert_pns_eq_sub!("%'d", - { None, "'", None, None, None, "d", (0, 3), }); - assert_pns_eq_sub!("%10s", - { None, "", Some(N::Num(10)), None, None, "s", (0, 4), }); - assert_pns_eq_sub!("%-10.10s", - { None, "-", Some(N::Num(10)), Some(N::Num(10)), None, "s", (0, 8), }); - assert_pns_eq_sub!("%1$d", - { Some(1), "", None, None, None, "d", (0, 4), }); - assert_pns_eq_sub!("%2$.*3$d", - { Some(2), "", None, Some(N::Arg(3)), None, "d", (0, 8), }); - assert_pns_eq_sub!("%1$*2$.*3$d", - { Some(1), "", Some(N::Arg(2)), Some(N::Arg(3)), None, "d", (0, 11), }); - assert_pns_eq_sub!("%-8ld", - { None, "-", Some(N::Num(8)), None, Some("l"), "d", (0, 5), }); -} - -#[test] -fn test_iter() { - let s = "The %d'th word %% is: `%.*s` %!\n"; - let subs: Vec<_> = iter_subs(s, 0).map(|sub| sub.translate()).collect(); - assert_eq!( - subs.iter().map(|ms| ms.as_ref().map(|s| &s[..])).collect::>(), - vec![Some("{}"), None, Some("{:.*}"), None] - ); -} - -/// Checks that the translations are what we expect. -#[test] -fn test_translation() { - assert_eq_pnsat!("%c", Some("{}")); - assert_eq_pnsat!("%d", Some("{}")); - assert_eq_pnsat!("%u", Some("{}")); - assert_eq_pnsat!("%x", Some("{:x}")); - assert_eq_pnsat!("%X", Some("{:X}")); - assert_eq_pnsat!("%e", Some("{:e}")); - assert_eq_pnsat!("%E", Some("{:E}")); - assert_eq_pnsat!("%f", Some("{}")); - assert_eq_pnsat!("%g", Some("{:e}")); - assert_eq_pnsat!("%G", Some("{:E}")); - assert_eq_pnsat!("%s", Some("{}")); - assert_eq_pnsat!("%p", Some("{:p}")); - - assert_eq_pnsat!("%06d", Some("{:06}")); - assert_eq_pnsat!("%4.2f", Some("{:4.2}")); - assert_eq_pnsat!("%#x", Some("{:#x}")); - assert_eq_pnsat!("%-10s", Some("{:<10}")); - assert_eq_pnsat!("%*s", None); - assert_eq_pnsat!("%-10.*s", Some("{:<10.*}")); - assert_eq_pnsat!("%-*.*s", None); - assert_eq_pnsat!("%.6i", Some("{:06}")); - assert_eq_pnsat!("%+i", Some("{:+}")); - assert_eq_pnsat!("%08X", Some("{:08X}")); - assert_eq_pnsat!("%lu", Some("{}")); - assert_eq_pnsat!("%Iu", Some("{}")); - assert_eq_pnsat!("%I32u", Some("{}")); - assert_eq_pnsat!("%I64u", Some("{}")); - assert_eq_pnsat!("%'d", None); - assert_eq_pnsat!("%10s", Some("{:>10}")); - assert_eq_pnsat!("%-10.10s", Some("{:<10.10}")); - assert_eq_pnsat!("%1$d", Some("{0}")); - assert_eq_pnsat!("%2$.*3$d", Some("{1:02$}")); - assert_eq_pnsat!("%1$*2$.*3$s", Some("{0:>1$.2$}")); - assert_eq_pnsat!("%-8ld", Some("{:<8}")); -} diff --git a/src/libsyntax_ext/format_foreign/shell/tests.rs b/src/libsyntax_ext/format_foreign/shell/tests.rs deleted file mode 100644 index ed8fe81dfcd..00000000000 --- a/src/libsyntax_ext/format_foreign/shell/tests.rs +++ /dev/null @@ -1,56 +0,0 @@ -use super::{parse_next_substitution as pns, Substitution as S}; - -macro_rules! assert_eq_pnsat { - ($lhs:expr, $rhs:expr) => { - assert_eq!( - pns($lhs).and_then(|(f, _)| f.translate()), - $rhs.map(>::from) - ) - }; -} - -#[test] -fn test_escape() { - assert_eq!(pns("has no escapes"), None); - assert_eq!(pns("has no escapes, either $"), None); - assert_eq!(pns("*so* has a $$ escape"), Some((S::Escape((11, 13)), " escape"))); - assert_eq!(pns("$$ leading escape"), Some((S::Escape((0, 2)), " leading escape"))); - assert_eq!(pns("trailing escape $$"), Some((S::Escape((16, 18)), ""))); -} - -#[test] -fn test_parse() { - macro_rules! assert_pns_eq_sub { - ($in_:expr, $kind:ident($arg:expr, $pos:expr)) => { - assert_eq!(pns(concat!($in_, "!")), Some((S::$kind($arg.into(), $pos), "!"))) - }; - } - - assert_pns_eq_sub!("$0", Ordinal(0, (0, 2))); - assert_pns_eq_sub!("$1", Ordinal(1, (0, 2))); - assert_pns_eq_sub!("$9", Ordinal(9, (0, 2))); - assert_pns_eq_sub!("$N", Name("N", (0, 2))); - assert_pns_eq_sub!("$NAME", Name("NAME", (0, 5))); -} - -#[test] -fn test_iter() { - use super::iter_subs; - let s = "The $0'th word $$ is: `$WORD` $!\n"; - let subs: Vec<_> = iter_subs(s, 0).map(|sub| sub.translate()).collect(); - assert_eq!( - subs.iter().map(|ms| ms.as_ref().map(|s| &s[..])).collect::>(), - vec![Some("{0}"), None, Some("{WORD}")] - ); -} - -#[test] -fn test_translation() { - assert_eq_pnsat!("$0", Some("{0}")); - assert_eq_pnsat!("$9", Some("{9}")); - assert_eq_pnsat!("$1", Some("{1}")); - assert_eq_pnsat!("$10", Some("{1}")); - assert_eq_pnsat!("$stuff", Some("{stuff}")); - assert_eq_pnsat!("$NAME", Some("{NAME}")); - assert_eq_pnsat!("$PREFIX/bin", Some("{PREFIX}")); -} diff --git a/src/libsyntax_ext/global_allocator.rs b/src/libsyntax_ext/global_allocator.rs deleted file mode 100644 index edfdda4703c..00000000000 --- a/src/libsyntax_ext/global_allocator.rs +++ /dev/null @@ -1,175 +0,0 @@ -use crate::util::check_builtin_macro_attribute; - -use syntax::ast::{self, Attribute, Expr, FnHeader, FnSig, Generics, Ident, Param}; -use syntax::ast::{ItemKind, Mutability, Stmt, Ty, TyKind, Unsafety}; -use syntax::expand::allocator::{AllocatorKind, AllocatorMethod, AllocatorTy, ALLOCATOR_METHODS}; -use syntax::ptr::P; -use syntax::symbol::{kw, sym, Symbol}; -use syntax_expand::base::{Annotatable, ExtCtxt}; -use syntax_pos::Span; - -pub fn expand( - ecx: &mut ExtCtxt<'_>, - _span: Span, - meta_item: &ast::MetaItem, - item: Annotatable, -) -> Vec { - check_builtin_macro_attribute(ecx, meta_item, sym::global_allocator); - - let not_static = |item: Annotatable| { - ecx.parse_sess.span_diagnostic.span_err(item.span(), "allocators must be statics"); - vec![item] - }; - let item = match item { - Annotatable::Item(item) => match item.kind { - ItemKind::Static(..) => item, - _ => return not_static(Annotatable::Item(item)), - }, - _ => return not_static(item), - }; - - // Generate a bunch of new items using the AllocFnFactory - let span = ecx.with_def_site_ctxt(item.span); - let f = AllocFnFactory { span, kind: AllocatorKind::Global, global: item.ident, cx: ecx }; - - // Generate item statements for the allocator methods. - let stmts = ALLOCATOR_METHODS.iter().map(|method| f.allocator_fn(method)).collect(); - - // Generate anonymous constant serving as container for the allocator methods. - let const_ty = ecx.ty(span, TyKind::Tup(Vec::new())); - let const_body = ecx.expr_block(ecx.block(span, stmts)); - let const_item = ecx.item_const(span, Ident::new(kw::Underscore, span), const_ty, const_body); - - // Return the original item and the new methods. - vec![Annotatable::Item(item), Annotatable::Item(const_item)] -} - -struct AllocFnFactory<'a, 'b> { - span: Span, - kind: AllocatorKind, - global: Ident, - cx: &'b ExtCtxt<'a>, -} - -impl AllocFnFactory<'_, '_> { - fn allocator_fn(&self, method: &AllocatorMethod) -> Stmt { - let mut abi_args = Vec::new(); - let mut i = 0; - let ref mut mk = || { - let name = self.cx.ident_of(&format!("arg{}", i), self.span); - i += 1; - name - }; - let args = method.inputs.iter().map(|ty| self.arg_ty(ty, &mut abi_args, mk)).collect(); - let result = self.call_allocator(method.name, args); - let (output_ty, output_expr) = self.ret_ty(&method.output, result); - let decl = self.cx.fn_decl(abi_args, ast::FunctionRetTy::Ty(output_ty)); - let header = FnHeader { unsafety: Unsafety::Unsafe, ..FnHeader::default() }; - let sig = FnSig { decl, header }; - let kind = ItemKind::Fn(sig, Generics::default(), self.cx.block_expr(output_expr)); - let item = self.cx.item( - self.span, - self.cx.ident_of(&self.kind.fn_name(method.name), self.span), - self.attrs(), - kind, - ); - self.cx.stmt_item(self.span, item) - } - - fn call_allocator(&self, method: &str, mut args: Vec>) -> P { - let method = self.cx.std_path(&[ - Symbol::intern("alloc"), - Symbol::intern("GlobalAlloc"), - Symbol::intern(method), - ]); - let method = self.cx.expr_path(self.cx.path(self.span, method)); - let allocator = self.cx.path_ident(self.span, self.global); - let allocator = self.cx.expr_path(allocator); - let allocator = self.cx.expr_addr_of(self.span, allocator); - args.insert(0, allocator); - - self.cx.expr_call(self.span, method, args) - } - - fn attrs(&self) -> Vec { - let special = sym::rustc_std_internal_symbol; - let special = self.cx.meta_word(self.span, special); - vec![self.cx.attribute(special)] - } - - fn arg_ty( - &self, - ty: &AllocatorTy, - args: &mut Vec, - ident: &mut dyn FnMut() -> Ident, - ) -> P { - match *ty { - AllocatorTy::Layout => { - let usize = self.cx.path_ident(self.span, Ident::new(sym::usize, self.span)); - let ty_usize = self.cx.ty_path(usize); - let size = ident(); - let align = ident(); - args.push(self.cx.param(self.span, size, ty_usize.clone())); - args.push(self.cx.param(self.span, align, ty_usize)); - - let layout_new = self.cx.std_path(&[ - Symbol::intern("alloc"), - Symbol::intern("Layout"), - Symbol::intern("from_size_align_unchecked"), - ]); - let layout_new = self.cx.expr_path(self.cx.path(self.span, layout_new)); - let size = self.cx.expr_ident(self.span, size); - let align = self.cx.expr_ident(self.span, align); - let layout = self.cx.expr_call(self.span, layout_new, vec![size, align]); - layout - } - - AllocatorTy::Ptr => { - let ident = ident(); - args.push(self.cx.param(self.span, ident, self.ptr_u8())); - let arg = self.cx.expr_ident(self.span, ident); - self.cx.expr_cast(self.span, arg, self.ptr_u8()) - } - - AllocatorTy::Usize => { - let ident = ident(); - args.push(self.cx.param(self.span, ident, self.usize())); - self.cx.expr_ident(self.span, ident) - } - - AllocatorTy::ResultPtr | AllocatorTy::Unit => { - panic!("can't convert AllocatorTy to an argument") - } - } - } - - fn ret_ty(&self, ty: &AllocatorTy, expr: P) -> (P, P) { - match *ty { - AllocatorTy::ResultPtr => { - // We're creating: - // - // #expr as *mut u8 - - let expr = self.cx.expr_cast(self.span, expr, self.ptr_u8()); - (self.ptr_u8(), expr) - } - - AllocatorTy::Unit => (self.cx.ty(self.span, TyKind::Tup(Vec::new())), expr), - - AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { - panic!("can't convert `AllocatorTy` to an output") - } - } - } - - fn usize(&self) -> P { - let usize = self.cx.path_ident(self.span, Ident::new(sym::usize, self.span)); - self.cx.ty_path(usize) - } - - fn ptr_u8(&self) -> P { - let u8 = self.cx.path_ident(self.span, Ident::new(sym::u8, self.span)); - let ty_u8 = self.cx.ty_path(u8); - self.cx.ty_ptr(self.span, ty_u8, Mutability::Mut) - } -} diff --git a/src/libsyntax_ext/global_asm.rs b/src/libsyntax_ext/global_asm.rs deleted file mode 100644 index fc933e4673a..00000000000 --- a/src/libsyntax_ext/global_asm.rs +++ /dev/null @@ -1,64 +0,0 @@ -/// Module-level assembly support. -/// -/// The macro defined here allows you to specify "top-level", -/// "file-scoped", or "module-level" assembly. These synonyms -/// all correspond to LLVM's module-level inline assembly instruction. -/// -/// For example, `global_asm!("some assembly here")` codegens to -/// LLVM's `module asm "some assembly here"`. All of LLVM's caveats -/// therefore apply. -use errors::DiagnosticBuilder; - -use smallvec::smallvec; -use syntax::ast; -use syntax::ptr::P; -use syntax::source_map::respan; -use syntax::token; -use syntax::tokenstream::TokenStream; -use syntax_expand::base::{self, *}; -use syntax_pos::Span; - -pub fn expand_global_asm<'cx>( - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - match parse_global_asm(cx, sp, tts) { - Ok(Some(global_asm)) => MacEager::items(smallvec![P(ast::Item { - ident: ast::Ident::invalid(), - attrs: Vec::new(), - id: ast::DUMMY_NODE_ID, - kind: ast::ItemKind::GlobalAsm(P(global_asm)), - vis: respan(sp.shrink_to_lo(), ast::VisibilityKind::Inherited), - span: cx.with_def_site_ctxt(sp), - tokens: None, - })]), - Ok(None) => DummyResult::any(sp), - Err(mut err) => { - err.emit(); - DummyResult::any(sp) - } - } -} - -fn parse_global_asm<'a>( - cx: &mut ExtCtxt<'a>, - sp: Span, - tts: TokenStream, -) -> Result, DiagnosticBuilder<'a>> { - let mut p = cx.new_parser_from_tts(tts); - - if p.token == token::Eof { - let mut err = cx.struct_span_err(sp, "macro requires a string literal as an argument"); - err.span_label(sp, "string literal required"); - return Err(err); - } - - let expr = p.parse_expr()?; - let (asm, _) = match expr_to_string(cx, expr, "inline assembly must be a string literal") { - Some((s, st)) => (s, st), - None => return Ok(None), - }; - - Ok(Some(ast::GlobalAsm { asm })) -} diff --git a/src/libsyntax_ext/lib.rs b/src/libsyntax_ext/lib.rs deleted file mode 100644 index 40aafece8c6..00000000000 --- a/src/libsyntax_ext/lib.rs +++ /dev/null @@ -1,109 +0,0 @@ -//! This crate contains implementations of built-in macros and other code generating facilities -//! injecting code into the crate before it is lowered to HIR. - -#![doc(html_root_url = "https://doc.rust-lang.org/nightly/")] -#![feature(bool_to_option)] -#![feature(crate_visibility_modifier)] -#![feature(decl_macro)] -#![feature(nll)] -#![feature(proc_macro_internals)] -#![feature(proc_macro_quote)] - -extern crate proc_macro; - -use crate::deriving::*; - -use syntax::ast::Ident; -use syntax::edition::Edition; -use syntax::symbol::sym; -use syntax_expand::base::{MacroExpanderFn, Resolver, SyntaxExtension, SyntaxExtensionKind}; -use syntax_expand::proc_macro::BangProcMacro; - -mod asm; -mod assert; -mod cfg; -mod compile_error; -mod concat; -mod concat_idents; -mod deriving; -mod env; -mod format; -mod format_foreign; -mod global_allocator; -mod global_asm; -mod log_syntax; -mod source_util; -mod test; -mod trace_macros; -mod util; - -pub mod cmdline_attrs; -pub mod proc_macro_harness; -pub mod standard_library_imports; -pub mod test_harness; - -pub fn register_builtin_macros(resolver: &mut dyn Resolver, edition: Edition) { - let mut register = |name, kind| { - resolver.register_builtin_macro( - Ident::with_dummy_span(name), - SyntaxExtension { is_builtin: true, ..SyntaxExtension::default(kind, edition) }, - ) - }; - macro register_bang($($name:ident: $f:expr,)*) { - $(register(sym::$name, SyntaxExtensionKind::LegacyBang(Box::new($f as MacroExpanderFn)));)* - } - macro register_attr($($name:ident: $f:expr,)*) { - $(register(sym::$name, SyntaxExtensionKind::LegacyAttr(Box::new($f)));)* - } - macro register_derive($($name:ident: $f:expr,)*) { - $(register(sym::$name, SyntaxExtensionKind::LegacyDerive(Box::new(BuiltinDerive($f))));)* - } - - register_bang! { - asm: asm::expand_asm, - assert: assert::expand_assert, - cfg: cfg::expand_cfg, - column: source_util::expand_column, - compile_error: compile_error::expand_compile_error, - concat_idents: concat_idents::expand_concat_idents, - concat: concat::expand_concat, - env: env::expand_env, - file: source_util::expand_file, - format_args_nl: format::expand_format_args_nl, - format_args: format::expand_format_args, - global_asm: global_asm::expand_global_asm, - include_bytes: source_util::expand_include_bytes, - include_str: source_util::expand_include_str, - include: source_util::expand_include, - line: source_util::expand_line, - log_syntax: log_syntax::expand_log_syntax, - module_path: source_util::expand_mod, - option_env: env::expand_option_env, - stringify: source_util::expand_stringify, - trace_macros: trace_macros::expand_trace_macros, - } - - register_attr! { - bench: test::expand_bench, - global_allocator: global_allocator::expand, - test: test::expand_test, - test_case: test::expand_test_case, - } - - register_derive! { - Clone: clone::expand_deriving_clone, - Copy: bounds::expand_deriving_copy, - Debug: debug::expand_deriving_debug, - Default: default::expand_deriving_default, - Eq: eq::expand_deriving_eq, - Hash: hash::expand_deriving_hash, - Ord: ord::expand_deriving_ord, - PartialEq: partial_eq::expand_deriving_partial_eq, - PartialOrd: partial_ord::expand_deriving_partial_ord, - RustcDecodable: decodable::expand_deriving_rustc_decodable, - RustcEncodable: encodable::expand_deriving_rustc_encodable, - } - - let client = proc_macro::bridge::client::Client::expand1(proc_macro::quote); - register(sym::quote, SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client }))); -} diff --git a/src/libsyntax_ext/log_syntax.rs b/src/libsyntax_ext/log_syntax.rs deleted file mode 100644 index 111226be877..00000000000 --- a/src/libsyntax_ext/log_syntax.rs +++ /dev/null @@ -1,15 +0,0 @@ -use syntax::print; -use syntax::tokenstream::TokenStream; -use syntax_expand::base; -use syntax_pos; - -pub fn expand_log_syntax<'cx>( - _cx: &'cx mut base::ExtCtxt<'_>, - sp: syntax_pos::Span, - tts: TokenStream, -) -> Box { - println!("{}", print::pprust::tts_to_string(tts)); - - // any so that `log_syntax` can be invoked as an expression and item. - base::DummyResult::any_valid(sp) -} diff --git a/src/libsyntax_ext/proc_macro_harness.rs b/src/libsyntax_ext/proc_macro_harness.rs deleted file mode 100644 index b6436cc1646..00000000000 --- a/src/libsyntax_ext/proc_macro_harness.rs +++ /dev/null @@ -1,463 +0,0 @@ -use std::mem; - -use smallvec::smallvec; -use syntax::ast::{self, Ident}; -use syntax::attr; -use syntax::expand::is_proc_macro_attr; -use syntax::print::pprust; -use syntax::ptr::P; -use syntax::sess::ParseSess; -use syntax::symbol::{kw, sym}; -use syntax::visit::{self, Visitor}; -use syntax_expand::base::{ExtCtxt, Resolver}; -use syntax_expand::expand::{AstFragment, ExpansionConfig}; -use syntax_pos::hygiene::AstPass; -use syntax_pos::{Span, DUMMY_SP}; - -struct ProcMacroDerive { - trait_name: ast::Name, - function_name: Ident, - span: Span, - attrs: Vec, -} - -enum ProcMacroDefType { - Attr, - Bang, -} - -struct ProcMacroDef { - function_name: Ident, - span: Span, - def_type: ProcMacroDefType, -} - -enum ProcMacro { - Derive(ProcMacroDerive), - Def(ProcMacroDef), -} - -struct CollectProcMacros<'a> { - macros: Vec, - in_root: bool, - handler: &'a errors::Handler, - is_proc_macro_crate: bool, - is_test_crate: bool, -} - -pub fn inject( - sess: &ParseSess, - resolver: &mut dyn Resolver, - mut krate: ast::Crate, - is_proc_macro_crate: bool, - has_proc_macro_decls: bool, - is_test_crate: bool, - num_crate_types: usize, - handler: &errors::Handler, -) -> ast::Crate { - let ecfg = ExpansionConfig::default("proc_macro".to_string()); - let mut cx = ExtCtxt::new(sess, ecfg, resolver); - - let mut collect = CollectProcMacros { - macros: Vec::new(), - in_root: true, - handler, - is_proc_macro_crate, - is_test_crate, - }; - - if has_proc_macro_decls || is_proc_macro_crate { - visit::walk_crate(&mut collect, &krate); - } - // NOTE: If you change the order of macros in this vec - // for any reason, you must also update 'raw_proc_macro' - // in src/librustc_metadata/decoder.rs - let macros = collect.macros; - - if !is_proc_macro_crate { - return krate; - } - - if num_crate_types > 1 { - handler.err("cannot mix `proc-macro` crate type with others"); - } - - if is_test_crate { - return krate; - } - - krate.module.items.push(mk_decls(&mut cx, ¯os)); - - krate -} - -impl<'a> CollectProcMacros<'a> { - fn check_not_pub_in_root(&self, vis: &ast::Visibility, sp: Span) { - if self.is_proc_macro_crate && self.in_root && vis.node.is_pub() { - self.handler.span_err( - sp, - "`proc-macro` crate types currently cannot export any items other \ - than functions tagged with `#[proc_macro]`, `#[proc_macro_derive]`, \ - or `#[proc_macro_attribute]`", - ); - } - } - - fn collect_custom_derive(&mut self, item: &'a ast::Item, attr: &'a ast::Attribute) { - // Once we've located the `#[proc_macro_derive]` attribute, verify - // that it's of the form `#[proc_macro_derive(Foo)]` or - // `#[proc_macro_derive(Foo, attributes(A, ..))]` - let list = match attr.meta_item_list() { - Some(list) => list, - None => return, - }; - if list.len() != 1 && list.len() != 2 { - self.handler.span_err(attr.span, "attribute must have either one or two arguments"); - return; - } - let trait_attr = match list[0].meta_item() { - Some(meta_item) => meta_item, - _ => { - self.handler.span_err(list[0].span(), "not a meta item"); - return; - } - }; - let trait_ident = match trait_attr.ident() { - Some(trait_ident) if trait_attr.is_word() => trait_ident, - _ => { - self.handler.span_err(trait_attr.span, "must only be one word"); - return; - } - }; - - if !trait_ident.name.can_be_raw() { - self.handler.span_err( - trait_attr.span, - &format!("`{}` cannot be a name of derive macro", trait_ident), - ); - } - - let attributes_attr = list.get(1); - let proc_attrs: Vec<_> = if let Some(attr) = attributes_attr { - if !attr.check_name(sym::attributes) { - self.handler.span_err(attr.span(), "second argument must be `attributes`") - } - attr.meta_item_list() - .unwrap_or_else(|| { - self.handler - .span_err(attr.span(), "attribute must be of form: `attributes(foo, bar)`"); - &[] - }) - .into_iter() - .filter_map(|attr| { - let attr = match attr.meta_item() { - Some(meta_item) => meta_item, - _ => { - self.handler.span_err(attr.span(), "not a meta item"); - return None; - } - }; - - let ident = match attr.ident() { - Some(ident) if attr.is_word() => ident, - _ => { - self.handler.span_err(attr.span, "must only be one word"); - return None; - } - }; - if !ident.name.can_be_raw() { - self.handler.span_err( - attr.span, - &format!("`{}` cannot be a name of derive helper attribute", ident), - ); - } - - Some(ident.name) - }) - .collect() - } else { - Vec::new() - }; - - if self.in_root && item.vis.node.is_pub() { - self.macros.push(ProcMacro::Derive(ProcMacroDerive { - span: item.span, - trait_name: trait_ident.name, - function_name: item.ident, - attrs: proc_attrs, - })); - } else { - let msg = if !self.in_root { - "functions tagged with `#[proc_macro_derive]` must \ - currently reside in the root of the crate" - } else { - "functions tagged with `#[proc_macro_derive]` must be `pub`" - }; - self.handler.span_err(item.span, msg); - } - } - - fn collect_attr_proc_macro(&mut self, item: &'a ast::Item) { - if self.in_root && item.vis.node.is_pub() { - self.macros.push(ProcMacro::Def(ProcMacroDef { - span: item.span, - function_name: item.ident, - def_type: ProcMacroDefType::Attr, - })); - } else { - let msg = if !self.in_root { - "functions tagged with `#[proc_macro_attribute]` must \ - currently reside in the root of the crate" - } else { - "functions tagged with `#[proc_macro_attribute]` must be `pub`" - }; - self.handler.span_err(item.span, msg); - } - } - - fn collect_bang_proc_macro(&mut self, item: &'a ast::Item) { - if self.in_root && item.vis.node.is_pub() { - self.macros.push(ProcMacro::Def(ProcMacroDef { - span: item.span, - function_name: item.ident, - def_type: ProcMacroDefType::Bang, - })); - } else { - let msg = if !self.in_root { - "functions tagged with `#[proc_macro]` must \ - currently reside in the root of the crate" - } else { - "functions tagged with `#[proc_macro]` must be `pub`" - }; - self.handler.span_err(item.span, msg); - } - } -} - -impl<'a> Visitor<'a> for CollectProcMacros<'a> { - fn visit_item(&mut self, item: &'a ast::Item) { - if let ast::ItemKind::MacroDef(..) = item.kind { - if self.is_proc_macro_crate && attr::contains_name(&item.attrs, sym::macro_export) { - let msg = - "cannot export macro_rules! macros from a `proc-macro` crate type currently"; - self.handler.span_err(item.span, msg); - } - } - - // First up, make sure we're checking a bare function. If we're not then - // we're just not interested in this item. - // - // If we find one, try to locate a `#[proc_macro_derive]` attribute on it. - let is_fn = match item.kind { - ast::ItemKind::Fn(..) => true, - _ => false, - }; - - let mut found_attr: Option<&'a ast::Attribute> = None; - - for attr in &item.attrs { - if is_proc_macro_attr(&attr) { - if let Some(prev_attr) = found_attr { - let prev_item = prev_attr.get_normal_item(); - let item = attr.get_normal_item(); - let path_str = pprust::path_to_string(&item.path); - let msg = if item.path.segments[0].ident.name - == prev_item.path.segments[0].ident.name - { - format!( - "only one `#[{}]` attribute is allowed on any given function", - path_str, - ) - } else { - format!( - "`#[{}]` and `#[{}]` attributes cannot both be applied - to the same function", - path_str, - pprust::path_to_string(&prev_item.path), - ) - }; - - self.handler - .struct_span_err(attr.span, &msg) - .span_label(prev_attr.span, "previous attribute here") - .emit(); - - return; - } - - found_attr = Some(attr); - } - } - - let attr = match found_attr { - None => { - self.check_not_pub_in_root(&item.vis, item.span); - let prev_in_root = mem::replace(&mut self.in_root, false); - visit::walk_item(self, item); - self.in_root = prev_in_root; - return; - } - Some(attr) => attr, - }; - - if !is_fn { - let msg = format!( - "the `#[{}]` attribute may only be used on bare functions", - pprust::path_to_string(&attr.get_normal_item().path), - ); - - self.handler.span_err(attr.span, &msg); - return; - } - - if self.is_test_crate { - return; - } - - if !self.is_proc_macro_crate { - let msg = format!( - "the `#[{}]` attribute is only usable with crates of the `proc-macro` crate type", - pprust::path_to_string(&attr.get_normal_item().path), - ); - - self.handler.span_err(attr.span, &msg); - return; - } - - if attr.check_name(sym::proc_macro_derive) { - self.collect_custom_derive(item, attr); - } else if attr.check_name(sym::proc_macro_attribute) { - self.collect_attr_proc_macro(item); - } else if attr.check_name(sym::proc_macro) { - self.collect_bang_proc_macro(item); - }; - - let prev_in_root = mem::replace(&mut self.in_root, false); - visit::walk_item(self, item); - self.in_root = prev_in_root; - } - - fn visit_mac(&mut self, mac: &'a ast::Mac) { - visit::walk_mac(self, mac) - } -} - -// Creates a new module which looks like: -// -// const _: () = { -// extern crate proc_macro; -// -// use proc_macro::bridge::client::ProcMacro; -// -// #[rustc_proc_macro_decls] -// #[allow(deprecated)] -// static DECLS: &[ProcMacro] = &[ -// ProcMacro::custom_derive($name_trait1, &[], ::$name1); -// ProcMacro::custom_derive($name_trait2, &["attribute_name"], ::$name2); -// // ... -// ]; -// } -fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> P { - let expn_id = cx.resolver.expansion_for_ast_pass( - DUMMY_SP, - AstPass::ProcMacroHarness, - &[sym::rustc_attrs, sym::proc_macro_internals], - None, - ); - let span = DUMMY_SP.with_def_site_ctxt(expn_id); - - let proc_macro = Ident::new(sym::proc_macro, span); - let krate = cx.item(span, proc_macro, Vec::new(), ast::ItemKind::ExternCrate(None)); - - let bridge = cx.ident_of("bridge", span); - let client = cx.ident_of("client", span); - let proc_macro_ty = cx.ident_of("ProcMacro", span); - let custom_derive = cx.ident_of("custom_derive", span); - let attr = cx.ident_of("attr", span); - let bang = cx.ident_of("bang", span); - - let decls = { - let local_path = - |sp: Span, name| cx.expr_path(cx.path(sp.with_ctxt(span.ctxt()), vec![name])); - let proc_macro_ty_method_path = |method| { - cx.expr_path(cx.path(span, vec![proc_macro, bridge, client, proc_macro_ty, method])) - }; - macros - .iter() - .map(|m| match m { - ProcMacro::Derive(cd) => cx.expr_call( - span, - proc_macro_ty_method_path(custom_derive), - vec![ - cx.expr_str(cd.span, cd.trait_name), - cx.expr_vec_slice( - span, - cd.attrs.iter().map(|&s| cx.expr_str(cd.span, s)).collect::>(), - ), - local_path(cd.span, cd.function_name), - ], - ), - ProcMacro::Def(ca) => { - let ident = match ca.def_type { - ProcMacroDefType::Attr => attr, - ProcMacroDefType::Bang => bang, - }; - - cx.expr_call( - span, - proc_macro_ty_method_path(ident), - vec![ - cx.expr_str(ca.span, ca.function_name.name), - local_path(ca.span, ca.function_name), - ], - ) - } - }) - .collect() - }; - - let decls_static = cx - .item_static( - span, - cx.ident_of("_DECLS", span), - cx.ty_rptr( - span, - cx.ty( - span, - ast::TyKind::Slice( - cx.ty_path(cx.path(span, vec![proc_macro, bridge, client, proc_macro_ty])), - ), - ), - None, - ast::Mutability::Not, - ), - ast::Mutability::Not, - cx.expr_vec_slice(span, decls), - ) - .map(|mut i| { - let attr = cx.meta_word(span, sym::rustc_proc_macro_decls); - i.attrs.push(cx.attribute(attr)); - - let deprecated_attr = attr::mk_nested_word_item(Ident::new(sym::deprecated, span)); - let allow_deprecated_attr = - attr::mk_list_item(Ident::new(sym::allow, span), vec![deprecated_attr]); - i.attrs.push(cx.attribute(allow_deprecated_attr)); - - i - }); - - let block = cx.expr_block( - cx.block(span, vec![cx.stmt_item(span, krate), cx.stmt_item(span, decls_static)]), - ); - - let anon_constant = cx.item_const( - span, - ast::Ident::new(kw::Underscore, span), - cx.ty(span, ast::TyKind::Tup(Vec::new())), - block, - ); - - // Integrate the new item into existing module structures. - let items = AstFragment::Items(smallvec![anon_constant]); - cx.monotonic_expander().fully_expand_fragment(items).make_items().pop().unwrap() -} diff --git a/src/libsyntax_ext/source_util.rs b/src/libsyntax_ext/source_util.rs deleted file mode 100644 index fccc36e2ea8..00000000000 --- a/src/libsyntax_ext/source_util.rs +++ /dev/null @@ -1,216 +0,0 @@ -use rustc_parse::{self, new_sub_parser_from_file, parser::Parser, DirectoryOwnership}; -use syntax::ast; -use syntax::early_buffered_lints::INCOMPLETE_INCLUDE; -use syntax::print::pprust; -use syntax::ptr::P; -use syntax::symbol::Symbol; -use syntax::token; -use syntax::tokenstream::TokenStream; -use syntax_expand::base::{self, *}; -use syntax_expand::panictry; - -use smallvec::SmallVec; -use syntax_pos::{self, Pos, Span}; - -use rustc_data_structures::sync::Lrc; - -// These macros all relate to the file system; they either return -// the column/row/filename of the expression, or they include -// a given file into the current one. - -/// line!(): expands to the current line number -pub fn expand_line( - cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let sp = cx.with_def_site_ctxt(sp); - base::check_zero_tts(cx, sp, tts, "line!"); - - let topmost = cx.expansion_cause().unwrap_or(sp); - let loc = cx.source_map().lookup_char_pos(topmost.lo()); - - base::MacEager::expr(cx.expr_u32(topmost, loc.line as u32)) -} - -/* column!(): expands to the current column number */ -pub fn expand_column( - cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let sp = cx.with_def_site_ctxt(sp); - base::check_zero_tts(cx, sp, tts, "column!"); - - let topmost = cx.expansion_cause().unwrap_or(sp); - let loc = cx.source_map().lookup_char_pos(topmost.lo()); - - base::MacEager::expr(cx.expr_u32(topmost, loc.col.to_usize() as u32 + 1)) -} - -/// file!(): expands to the current filename */ -/// The source_file (`loc.file`) contains a bunch more information we could spit -/// out if we wanted. -pub fn expand_file( - cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let sp = cx.with_def_site_ctxt(sp); - base::check_zero_tts(cx, sp, tts, "file!"); - - let topmost = cx.expansion_cause().unwrap_or(sp); - let loc = cx.source_map().lookup_char_pos(topmost.lo()); - base::MacEager::expr(cx.expr_str(topmost, Symbol::intern(&loc.file.name.to_string()))) -} - -pub fn expand_stringify( - cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let sp = cx.with_def_site_ctxt(sp); - let s = pprust::tts_to_string(tts); - base::MacEager::expr(cx.expr_str(sp, Symbol::intern(&s))) -} - -pub fn expand_mod( - cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let sp = cx.with_def_site_ctxt(sp); - base::check_zero_tts(cx, sp, tts, "module_path!"); - let mod_path = &cx.current_expansion.module.mod_path; - let string = mod_path.iter().map(|x| x.to_string()).collect::>().join("::"); - - base::MacEager::expr(cx.expr_str(sp, Symbol::intern(&string))) -} - -/// include! : parse the given file as an expr -/// This is generally a bad idea because it's going to behave -/// unhygienically. -pub fn expand_include<'cx>( - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let sp = cx.with_def_site_ctxt(sp); - let file = match get_single_str_from_tts(cx, sp, tts, "include!") { - Some(f) => f, - None => return DummyResult::any(sp), - }; - // The file will be added to the code map by the parser - let file = match cx.resolve_path(file, sp) { - Ok(f) => f, - Err(mut err) => { - err.emit(); - return DummyResult::any(sp); - } - }; - let directory_ownership = DirectoryOwnership::Owned { relative: None }; - let p = new_sub_parser_from_file(cx.parse_sess(), &file, directory_ownership, None, sp); - - struct ExpandResult<'a> { - p: Parser<'a>, - } - impl<'a> base::MacResult for ExpandResult<'a> { - fn make_expr(mut self: Box>) -> Option> { - let r = panictry!(self.p.parse_expr()); - if self.p.token != token::Eof { - self.p.sess.buffer_lint( - &INCOMPLETE_INCLUDE, - self.p.token.span, - ast::CRATE_NODE_ID, - "include macro expected single expression in source", - ); - } - Some(r) - } - - fn make_items(mut self: Box>) -> Option; 1]>> { - let mut ret = SmallVec::new(); - while self.p.token != token::Eof { - match panictry!(self.p.parse_item()) { - Some(item) => ret.push(item), - None => { - let token = pprust::token_to_string(&self.p.token); - self.p - .sess - .span_diagnostic - .span_fatal( - self.p.token.span, - &format!("expected item, found `{}`", token), - ) - .raise(); - } - } - } - Some(ret) - } - } - - Box::new(ExpandResult { p }) -} - -// include_str! : read the given file, insert it as a literal string expr -pub fn expand_include_str( - cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let sp = cx.with_def_site_ctxt(sp); - let file = match get_single_str_from_tts(cx, sp, tts, "include_str!") { - Some(f) => f, - None => return DummyResult::any(sp), - }; - let file = match cx.resolve_path(file, sp) { - Ok(f) => f, - Err(mut err) => { - err.emit(); - return DummyResult::any(sp); - } - }; - match cx.source_map().load_binary_file(&file) { - Ok(bytes) => match std::str::from_utf8(&bytes) { - Ok(src) => { - let interned_src = Symbol::intern(&src); - base::MacEager::expr(cx.expr_str(sp, interned_src)) - } - Err(_) => { - cx.span_err(sp, &format!("{} wasn't a utf-8 file", file.display())); - DummyResult::any(sp) - } - }, - Err(e) => { - cx.span_err(sp, &format!("couldn't read {}: {}", file.display(), e)); - DummyResult::any(sp) - } - } -} - -pub fn expand_include_bytes( - cx: &mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - let sp = cx.with_def_site_ctxt(sp); - let file = match get_single_str_from_tts(cx, sp, tts, "include_bytes!") { - Some(f) => f, - None => return DummyResult::any(sp), - }; - let file = match cx.resolve_path(file, sp) { - Ok(f) => f, - Err(mut err) => { - err.emit(); - return DummyResult::any(sp); - } - }; - match cx.source_map().load_binary_file(&file) { - Ok(bytes) => base::MacEager::expr(cx.expr_lit(sp, ast::LitKind::ByteStr(Lrc::new(bytes)))), - Err(e) => { - cx.span_err(sp, &format!("couldn't read {}: {}", file.display(), e)); - DummyResult::any(sp) - } - } -} diff --git a/src/libsyntax_ext/standard_library_imports.rs b/src/libsyntax_ext/standard_library_imports.rs deleted file mode 100644 index 50f86a0f3ec..00000000000 --- a/src/libsyntax_ext/standard_library_imports.rs +++ /dev/null @@ -1,85 +0,0 @@ -use syntax::edition::Edition; -use syntax::ptr::P; -use syntax::sess::ParseSess; -use syntax::symbol::{kw, sym, Ident, Symbol}; -use syntax::{ast, attr}; -use syntax_expand::base::{ExtCtxt, Resolver}; -use syntax_expand::expand::ExpansionConfig; -use syntax_pos::hygiene::AstPass; -use syntax_pos::DUMMY_SP; - -pub fn inject( - mut krate: ast::Crate, - resolver: &mut dyn Resolver, - sess: &ParseSess, - alt_std_name: Option, -) -> (ast::Crate, Option) { - let rust_2018 = sess.edition >= Edition::Edition2018; - - // the first name in this list is the crate name of the crate with the prelude - let names: &[Symbol] = if attr::contains_name(&krate.attrs, sym::no_core) { - return (krate, None); - } else if attr::contains_name(&krate.attrs, sym::no_std) { - if attr::contains_name(&krate.attrs, sym::compiler_builtins) { - &[sym::core] - } else { - &[sym::core, sym::compiler_builtins] - } - } else { - &[sym::std] - }; - - let expn_id = resolver.expansion_for_ast_pass( - DUMMY_SP, - AstPass::StdImports, - &[sym::prelude_import], - None, - ); - let span = DUMMY_SP.with_def_site_ctxt(expn_id); - let call_site = DUMMY_SP.with_call_site_ctxt(expn_id); - - let ecfg = ExpansionConfig::default("std_lib_injection".to_string()); - let cx = ExtCtxt::new(sess, ecfg, resolver); - - // .rev() to preserve ordering above in combination with insert(0, ...) - for &name in names.iter().rev() { - let ident = if rust_2018 { Ident::new(name, span) } else { Ident::new(name, call_site) }; - krate.module.items.insert( - 0, - cx.item( - span, - ident, - vec![cx.attribute(cx.meta_word(span, sym::macro_use))], - ast::ItemKind::ExternCrate(alt_std_name), - ), - ); - } - - // The crates have been injected, the assumption is that the first one is - // the one with the prelude. - let name = names[0]; - - let import_path = if rust_2018 { - [name, sym::prelude, sym::v1].iter().map(|symbol| ast::Ident::new(*symbol, span)).collect() - } else { - [kw::PathRoot, name, sym::prelude, sym::v1] - .iter() - .map(|symbol| ast::Ident::new(*symbol, span)) - .collect() - }; - - let use_item = cx.item( - span, - ast::Ident::invalid(), - vec![cx.attribute(cx.meta_word(span, sym::prelude_import))], - ast::ItemKind::Use(P(ast::UseTree { - prefix: cx.path(span, import_path), - kind: ast::UseTreeKind::Glob, - span, - })), - ); - - krate.module.items.insert(0, use_item); - - (krate, Some(name)) -} diff --git a/src/libsyntax_ext/test.rs b/src/libsyntax_ext/test.rs deleted file mode 100644 index edf427edaae..00000000000 --- a/src/libsyntax_ext/test.rs +++ /dev/null @@ -1,439 +0,0 @@ -/// The expansion from a test function to the appropriate test struct for libtest -/// Ideally, this code would be in libtest but for efficiency and error messages it lives here. -use crate::util::check_builtin_macro_attribute; - -use syntax::ast; -use syntax::attr; -use syntax::print::pprust; -use syntax::source_map::respan; -use syntax::symbol::{sym, Symbol}; -use syntax_expand::base::*; -use syntax_pos::Span; - -use std::iter; - -// #[test_case] is used by custom test authors to mark tests -// When building for test, it needs to make the item public and gensym the name -// Otherwise, we'll omit the item. This behavior means that any item annotated -// with #[test_case] is never addressable. -// -// We mark item with an inert attribute "rustc_test_marker" which the test generation -// logic will pick up on. -pub fn expand_test_case( - ecx: &mut ExtCtxt<'_>, - attr_sp: Span, - meta_item: &ast::MetaItem, - anno_item: Annotatable, -) -> Vec { - check_builtin_macro_attribute(ecx, meta_item, sym::test_case); - - if !ecx.ecfg.should_test { - return vec![]; - } - - let sp = ecx.with_def_site_ctxt(attr_sp); - let mut item = anno_item.expect_item(); - item = item.map(|mut item| { - item.vis = respan(item.vis.span, ast::VisibilityKind::Public); - item.ident.span = item.ident.span.with_ctxt(sp.ctxt()); - item.attrs.push(ecx.attribute(ecx.meta_word(sp, sym::rustc_test_marker))); - item - }); - - return vec![Annotatable::Item(item)]; -} - -pub fn expand_test( - cx: &mut ExtCtxt<'_>, - attr_sp: Span, - meta_item: &ast::MetaItem, - item: Annotatable, -) -> Vec { - check_builtin_macro_attribute(cx, meta_item, sym::test); - expand_test_or_bench(cx, attr_sp, item, false) -} - -pub fn expand_bench( - cx: &mut ExtCtxt<'_>, - attr_sp: Span, - meta_item: &ast::MetaItem, - item: Annotatable, -) -> Vec { - check_builtin_macro_attribute(cx, meta_item, sym::bench); - expand_test_or_bench(cx, attr_sp, item, true) -} - -pub fn expand_test_or_bench( - cx: &mut ExtCtxt<'_>, - attr_sp: Span, - item: Annotatable, - is_bench: bool, -) -> Vec { - // If we're not in test configuration, remove the annotated item - if !cx.ecfg.should_test { - return vec![]; - } - - let item = if let Annotatable::Item(i) = item { - i - } else { - cx.parse_sess - .span_diagnostic - .span_fatal( - item.span(), - "`#[test]` attribute is only allowed on non associated functions", - ) - .raise(); - }; - - if let ast::ItemKind::Mac(_) = item.kind { - cx.parse_sess.span_diagnostic.span_warn( - item.span, - "`#[test]` attribute should not be used on macros. Use `#[cfg(test)]` instead.", - ); - return vec![Annotatable::Item(item)]; - } - - // has_*_signature will report any errors in the type so compilation - // will fail. We shouldn't try to expand in this case because the errors - // would be spurious. - if (!is_bench && !has_test_signature(cx, &item)) - || (is_bench && !has_bench_signature(cx, &item)) - { - return vec![Annotatable::Item(item)]; - } - - let (sp, attr_sp) = (cx.with_def_site_ctxt(item.span), cx.with_def_site_ctxt(attr_sp)); - - let test_id = ast::Ident::new(sym::test, attr_sp); - - // creates test::$name - let test_path = |name| cx.path(sp, vec![test_id, cx.ident_of(name, sp)]); - - // creates test::ShouldPanic::$name - let should_panic_path = - |name| cx.path(sp, vec![test_id, cx.ident_of("ShouldPanic", sp), cx.ident_of(name, sp)]); - - // creates test::TestType::$name - let test_type_path = - |name| cx.path(sp, vec![test_id, cx.ident_of("TestType", sp), cx.ident_of(name, sp)]); - - // creates $name: $expr - let field = |name, expr| cx.field_imm(sp, cx.ident_of(name, sp), expr); - - let test_fn = if is_bench { - // A simple ident for a lambda - let b = cx.ident_of("b", attr_sp); - - cx.expr_call( - sp, - cx.expr_path(test_path("StaticBenchFn")), - vec![ - // |b| self::test::assert_test_result( - cx.lambda1( - sp, - cx.expr_call( - sp, - cx.expr_path(test_path("assert_test_result")), - vec![ - // super::$test_fn(b) - cx.expr_call( - sp, - cx.expr_path(cx.path(sp, vec![item.ident])), - vec![cx.expr_ident(sp, b)], - ), - ], - ), - b, - ), // ) - ], - ) - } else { - cx.expr_call( - sp, - cx.expr_path(test_path("StaticTestFn")), - vec![ - // || { - cx.lambda0( - sp, - // test::assert_test_result( - cx.expr_call( - sp, - cx.expr_path(test_path("assert_test_result")), - vec![ - // $test_fn() - cx.expr_call(sp, cx.expr_path(cx.path(sp, vec![item.ident])), vec![]), // ) - ], - ), // } - ), // ) - ], - ) - }; - - let mut test_const = cx.item( - sp, - ast::Ident::new(item.ident.name, sp), - vec![ - // #[cfg(test)] - cx.attribute(attr::mk_list_item( - ast::Ident::new(sym::cfg, attr_sp), - vec![attr::mk_nested_word_item(ast::Ident::new(sym::test, attr_sp))], - )), - // #[rustc_test_marker] - cx.attribute(cx.meta_word(attr_sp, sym::rustc_test_marker)), - ], - // const $ident: test::TestDescAndFn = - ast::ItemKind::Const( - cx.ty(sp, ast::TyKind::Path(None, test_path("TestDescAndFn"))), - // test::TestDescAndFn { - cx.expr_struct( - sp, - test_path("TestDescAndFn"), - vec![ - // desc: test::TestDesc { - field( - "desc", - cx.expr_struct( - sp, - test_path("TestDesc"), - vec![ - // name: "path::to::test" - field( - "name", - cx.expr_call( - sp, - cx.expr_path(test_path("StaticTestName")), - vec![cx.expr_str( - sp, - Symbol::intern(&item_path( - // skip the name of the root module - &cx.current_expansion.module.mod_path[1..], - &item.ident, - )), - )], - ), - ), - // ignore: true | false - field("ignore", cx.expr_bool(sp, should_ignore(&item))), - // allow_fail: true | false - field("allow_fail", cx.expr_bool(sp, should_fail(&item))), - // should_panic: ... - field( - "should_panic", - match should_panic(cx, &item) { - // test::ShouldPanic::No - ShouldPanic::No => cx.expr_path(should_panic_path("No")), - // test::ShouldPanic::Yes - ShouldPanic::Yes(None) => { - cx.expr_path(should_panic_path("Yes")) - } - // test::ShouldPanic::YesWithMessage("...") - ShouldPanic::Yes(Some(sym)) => cx.expr_call( - sp, - cx.expr_path(should_panic_path("YesWithMessage")), - vec![cx.expr_str(sp, sym)], - ), - }, - ), - // test_type: ... - field( - "test_type", - match test_type(cx) { - // test::TestType::UnitTest - TestType::UnitTest => { - cx.expr_path(test_type_path("UnitTest")) - } - // test::TestType::IntegrationTest - TestType::IntegrationTest => { - cx.expr_path(test_type_path("IntegrationTest")) - } - // test::TestPath::Unknown - TestType::Unknown => { - cx.expr_path(test_type_path("Unknown")) - } - }, - ), - // }, - ], - ), - ), - // testfn: test::StaticTestFn(...) | test::StaticBenchFn(...) - field("testfn", test_fn), // } - ], - ), // } - ), - ); - test_const = test_const.map(|mut tc| { - tc.vis.node = ast::VisibilityKind::Public; - tc - }); - - // extern crate test - let test_extern = cx.item(sp, test_id, vec![], ast::ItemKind::ExternCrate(None)); - - log::debug!("synthetic test item:\n{}\n", pprust::item_to_string(&test_const)); - - vec![ - // Access to libtest under a hygienic name - Annotatable::Item(test_extern), - // The generated test case - Annotatable::Item(test_const), - // The original item - Annotatable::Item(item), - ] -} - -fn item_path(mod_path: &[ast::Ident], item_ident: &ast::Ident) -> String { - mod_path - .iter() - .chain(iter::once(item_ident)) - .map(|x| x.to_string()) - .collect::>() - .join("::") -} - -enum ShouldPanic { - No, - Yes(Option), -} - -fn should_ignore(i: &ast::Item) -> bool { - attr::contains_name(&i.attrs, sym::ignore) -} - -fn should_fail(i: &ast::Item) -> bool { - attr::contains_name(&i.attrs, sym::allow_fail) -} - -fn should_panic(cx: &ExtCtxt<'_>, i: &ast::Item) -> ShouldPanic { - match attr::find_by_name(&i.attrs, sym::should_panic) { - Some(attr) => { - let ref sd = cx.parse_sess.span_diagnostic; - - match attr.meta_item_list() { - // Handle #[should_panic(expected = "foo")] - Some(list) => { - let msg = list - .iter() - .find(|mi| mi.check_name(sym::expected)) - .and_then(|mi| mi.meta_item()) - .and_then(|mi| mi.value_str()); - if list.len() != 1 || msg.is_none() { - sd.struct_span_warn( - attr.span, - "argument must be of the form: \ - `expected = \"error message\"`", - ) - .note( - "Errors in this attribute were erroneously \ - allowed and will become a hard error in a \ - future release.", - ) - .emit(); - ShouldPanic::Yes(None) - } else { - ShouldPanic::Yes(msg) - } - } - // Handle #[should_panic] and #[should_panic = "expected"] - None => ShouldPanic::Yes(attr.value_str()), - } - } - None => ShouldPanic::No, - } -} - -enum TestType { - UnitTest, - IntegrationTest, - Unknown, -} - -/// Attempts to determine the type of test. -/// Since doctests are created without macro expanding, only possible variants here -/// are `UnitTest`, `IntegrationTest` or `Unknown`. -fn test_type(cx: &ExtCtxt<'_>) -> TestType { - // Root path from context contains the topmost sources directory of the crate. - // I.e., for `project` with sources in `src` and tests in `tests` folders - // (no matter how many nested folders lie inside), - // there will be two different root paths: `/project/src` and `/project/tests`. - let crate_path = cx.root_path.as_path(); - - if crate_path.ends_with("src") { - // `/src` folder contains unit-tests. - TestType::UnitTest - } else if crate_path.ends_with("tests") { - // `/tests` folder contains integration tests. - TestType::IntegrationTest - } else { - // Crate layout doesn't match expected one, test type is unknown. - TestType::Unknown - } -} - -fn has_test_signature(cx: &ExtCtxt<'_>, i: &ast::Item) -> bool { - let has_should_panic_attr = attr::contains_name(&i.attrs, sym::should_panic); - let ref sd = cx.parse_sess.span_diagnostic; - if let ast::ItemKind::Fn(ref sig, ref generics, _) = i.kind { - if sig.header.unsafety == ast::Unsafety::Unsafe { - sd.span_err(i.span, "unsafe functions cannot be used for tests"); - return false; - } - if sig.header.asyncness.node.is_async() { - sd.span_err(i.span, "async functions cannot be used for tests"); - return false; - } - - // If the termination trait is active, the compiler will check that the output - // type implements the `Termination` trait as `libtest` enforces that. - let has_output = match sig.decl.output { - ast::FunctionRetTy::Default(..) => false, - ast::FunctionRetTy::Ty(ref t) if t.kind.is_unit() => false, - _ => true, - }; - - if !sig.decl.inputs.is_empty() { - sd.span_err(i.span, "functions used as tests can not have any arguments"); - return false; - } - - match (has_output, has_should_panic_attr) { - (true, true) => { - sd.span_err(i.span, "functions using `#[should_panic]` must return `()`"); - false - } - (true, false) => { - if !generics.params.is_empty() { - sd.span_err(i.span, "functions used as tests must have signature fn() -> ()"); - false - } else { - true - } - } - (false, _) => true, - } - } else { - sd.span_err(i.span, "only functions may be used as tests"); - false - } -} - -fn has_bench_signature(cx: &ExtCtxt<'_>, i: &ast::Item) -> bool { - let has_sig = if let ast::ItemKind::Fn(ref sig, _, _) = i.kind { - // N.B., inadequate check, but we're running - // well before resolve, can't get too deep. - sig.decl.inputs.len() == 1 - } else { - false - }; - - if !has_sig { - cx.parse_sess.span_diagnostic.span_err( - i.span, - "functions used as benches must have \ - signature `fn(&mut Bencher) -> impl Termination`", - ); - } - - has_sig -} diff --git a/src/libsyntax_ext/test_harness.rs b/src/libsyntax_ext/test_harness.rs deleted file mode 100644 index b00fc3d26c1..00000000000 --- a/src/libsyntax_ext/test_harness.rs +++ /dev/null @@ -1,366 +0,0 @@ -// Code that generates a test runner to run all the tests in a crate - -use log::debug; -use rustc_feature::Features; -use rustc_target::spec::PanicStrategy; -use smallvec::{smallvec, SmallVec}; -use syntax::ast::{self, Ident}; -use syntax::attr; -use syntax::entry::{self, EntryPointType}; -use syntax::mut_visit::{ExpectOne, *}; -use syntax::ptr::P; -use syntax::sess::ParseSess; -use syntax::source_map::respan; -use syntax::symbol::{sym, Symbol}; -use syntax_expand::base::{ExtCtxt, Resolver}; -use syntax_expand::expand::{AstFragment, ExpansionConfig}; -use syntax_pos::hygiene::{AstPass, SyntaxContext, Transparency}; -use syntax_pos::{Span, DUMMY_SP}; - -use std::{iter, mem}; - -struct Test { - span: Span, - ident: Ident, -} - -struct TestCtxt<'a> { - ext_cx: ExtCtxt<'a>, - panic_strategy: PanicStrategy, - def_site: Span, - test_cases: Vec, - reexport_test_harness_main: Option, - test_runner: Option, -} - -// Traverse the crate, collecting all the test functions, eliding any -// existing main functions, and synthesizing a main test harness -pub fn inject( - sess: &ParseSess, - resolver: &mut dyn Resolver, - should_test: bool, - krate: &mut ast::Crate, - span_diagnostic: &errors::Handler, - features: &Features, - panic_strategy: PanicStrategy, - platform_panic_strategy: PanicStrategy, - enable_panic_abort_tests: bool, -) { - // Check for #![reexport_test_harness_main = "some_name"] which gives the - // main test function the name `some_name` without hygiene. This needs to be - // unconditional, so that the attribute is still marked as used in - // non-test builds. - let reexport_test_harness_main = - attr::first_attr_value_str_by_name(&krate.attrs, sym::reexport_test_harness_main); - - // Do this here so that the test_runner crate attribute gets marked as used - // even in non-test builds - let test_runner = get_test_runner(span_diagnostic, &krate); - - if should_test { - let panic_strategy = match (panic_strategy, enable_panic_abort_tests) { - (PanicStrategy::Abort, true) => PanicStrategy::Abort, - (PanicStrategy::Abort, false) if panic_strategy == platform_panic_strategy => { - // Silently allow compiling with panic=abort on these platforms, - // but with old behavior (abort if a test fails). - PanicStrategy::Unwind - } - (PanicStrategy::Abort, false) => { - span_diagnostic.err( - "building tests with panic=abort is not supported \ - without `-Zpanic_abort_tests`", - ); - PanicStrategy::Unwind - } - (PanicStrategy::Unwind, _) => PanicStrategy::Unwind, - }; - generate_test_harness( - sess, - resolver, - reexport_test_harness_main, - krate, - features, - panic_strategy, - test_runner, - ) - } -} - -struct TestHarnessGenerator<'a> { - cx: TestCtxt<'a>, - tests: Vec, -} - -impl<'a> MutVisitor for TestHarnessGenerator<'a> { - fn visit_crate(&mut self, c: &mut ast::Crate) { - noop_visit_crate(c, self); - - // Create a main function to run our tests - c.module.items.push(mk_main(&mut self.cx)); - } - - fn flat_map_item(&mut self, i: P) -> SmallVec<[P; 1]> { - let mut item = i.into_inner(); - if is_test_case(&item) { - debug!("this is a test item"); - - let test = Test { span: item.span, ident: item.ident }; - self.tests.push(test); - } - - // We don't want to recurse into anything other than mods, since - // mods or tests inside of functions will break things - if let ast::ItemKind::Mod(mut module) = item.kind { - let tests = mem::take(&mut self.tests); - noop_visit_mod(&mut module, self); - let mut tests = mem::replace(&mut self.tests, tests); - - if !tests.is_empty() { - let parent = - if item.id == ast::DUMMY_NODE_ID { ast::CRATE_NODE_ID } else { item.id }; - // Create an identifier that will hygienically resolve the test - // case name, even in another module. - let expn_id = self.cx.ext_cx.resolver.expansion_for_ast_pass( - module.inner, - AstPass::TestHarness, - &[], - Some(parent), - ); - for test in &mut tests { - // See the comment on `mk_main` for why we're using - // `apply_mark` directly. - test.ident.span = test.ident.span.apply_mark(expn_id, Transparency::Opaque); - } - self.cx.test_cases.extend(tests); - } - item.kind = ast::ItemKind::Mod(module); - } - smallvec![P(item)] - } - - fn visit_mac(&mut self, _mac: &mut ast::Mac) { - // Do nothing. - } -} - -/// A folder used to remove any entry points (like fn main) because the harness -/// generator will provide its own -struct EntryPointCleaner { - // Current depth in the ast - depth: usize, - def_site: Span, -} - -impl MutVisitor for EntryPointCleaner { - fn flat_map_item(&mut self, i: P) -> SmallVec<[P; 1]> { - self.depth += 1; - let item = noop_flat_map_item(i, self).expect_one("noop did something"); - self.depth -= 1; - - // Remove any #[main] or #[start] from the AST so it doesn't - // clash with the one we're going to add, but mark it as - // #[allow(dead_code)] to avoid printing warnings. - let item = match entry::entry_point_type(&item, self.depth) { - EntryPointType::MainNamed | EntryPointType::MainAttr | EntryPointType::Start => item - .map(|ast::Item { id, ident, attrs, kind, vis, span, tokens }| { - let allow_ident = Ident::new(sym::allow, self.def_site); - let dc_nested = attr::mk_nested_word_item(Ident::from_str_and_span( - "dead_code", - self.def_site, - )); - let allow_dead_code_item = attr::mk_list_item(allow_ident, vec![dc_nested]); - let allow_dead_code = attr::mk_attr_outer(allow_dead_code_item); - - ast::Item { - id, - ident, - attrs: attrs - .into_iter() - .filter(|attr| { - !attr.check_name(sym::main) && !attr.check_name(sym::start) - }) - .chain(iter::once(allow_dead_code)) - .collect(), - kind, - vis, - span, - tokens, - } - }), - EntryPointType::None | EntryPointType::OtherMain => item, - }; - - smallvec![item] - } - - fn visit_mac(&mut self, _mac: &mut ast::Mac) { - // Do nothing. - } -} - -/// Crawl over the crate, inserting test reexports and the test main function -fn generate_test_harness( - sess: &ParseSess, - resolver: &mut dyn Resolver, - reexport_test_harness_main: Option, - krate: &mut ast::Crate, - features: &Features, - panic_strategy: PanicStrategy, - test_runner: Option, -) { - let mut econfig = ExpansionConfig::default("test".to_string()); - econfig.features = Some(features); - - let ext_cx = ExtCtxt::new(sess, econfig, resolver); - - let expn_id = ext_cx.resolver.expansion_for_ast_pass( - DUMMY_SP, - AstPass::TestHarness, - &[sym::main, sym::test, sym::rustc_attrs], - None, - ); - let def_site = DUMMY_SP.with_def_site_ctxt(expn_id); - - // Remove the entry points - let mut cleaner = EntryPointCleaner { depth: 0, def_site }; - cleaner.visit_crate(krate); - - let cx = TestCtxt { - ext_cx, - panic_strategy, - def_site, - test_cases: Vec::new(), - reexport_test_harness_main, - test_runner, - }; - - TestHarnessGenerator { cx, tests: Vec::new() }.visit_crate(krate); -} - -/// Creates a function item for use as the main function of a test build. -/// This function will call the `test_runner` as specified by the crate attribute -/// -/// By default this expands to -/// -/// #[main] -/// pub fn main() { -/// extern crate test; -/// test::test_main_static(&[ -/// &test_const1, -/// &test_const2, -/// &test_const3, -/// ]); -/// } -/// -/// Most of the Ident have the usual def-site hygiene for the AST pass. The -/// exception is the `test_const`s. These have a syntax context that has two -/// opaque marks: one from the expansion of `test` or `test_case`, and one -/// generated in `TestHarnessGenerator::flat_map_item`. When resolving this -/// identifier after failing to find a matching identifier in the root module -/// we remove the outer mark, and try resolving at its def-site, which will -/// then resolve to `test_const`. -/// -/// The expansion here can be controlled by two attributes: -/// -/// `reexport_test_harness_main` provides a different name for the `main` -/// function and `test_runner` provides a path that replaces -/// `test::test_main_static`. -fn mk_main(cx: &mut TestCtxt<'_>) -> P { - let sp = cx.def_site; - let ecx = &cx.ext_cx; - let test_id = Ident::new(sym::test, sp); - - let runner_name = match cx.panic_strategy { - PanicStrategy::Unwind => "test_main_static", - PanicStrategy::Abort => "test_main_static_abort", - }; - - // test::test_main_static(...) - let mut test_runner = cx - .test_runner - .clone() - .unwrap_or(ecx.path(sp, vec![test_id, ecx.ident_of(runner_name, sp)])); - - test_runner.span = sp; - - let test_main_path_expr = ecx.expr_path(test_runner); - let call_test_main = ecx.expr_call(sp, test_main_path_expr, vec![mk_tests_slice(cx, sp)]); - let call_test_main = ecx.stmt_expr(call_test_main); - - // extern crate test - let test_extern_stmt = - ecx.stmt_item(sp, ecx.item(sp, test_id, vec![], ast::ItemKind::ExternCrate(None))); - - // #[main] - let main_meta = ecx.meta_word(sp, sym::main); - let main_attr = ecx.attribute(main_meta); - - // pub fn main() { ... } - let main_ret_ty = ecx.ty(sp, ast::TyKind::Tup(vec![])); - - // If no test runner is provided we need to import the test crate - let main_body = if cx.test_runner.is_none() { - ecx.block(sp, vec![test_extern_stmt, call_test_main]) - } else { - ecx.block(sp, vec![call_test_main]) - }; - - let decl = ecx.fn_decl(vec![], ast::FunctionRetTy::Ty(main_ret_ty)); - let sig = ast::FnSig { decl, header: ast::FnHeader::default() }; - let main = ast::ItemKind::Fn(sig, ast::Generics::default(), main_body); - - // Honor the reexport_test_harness_main attribute - let main_id = match cx.reexport_test_harness_main { - Some(sym) => Ident::new(sym, sp.with_ctxt(SyntaxContext::root())), - None => Ident::new(sym::main, sp), - }; - - let main = P(ast::Item { - ident: main_id, - attrs: vec![main_attr], - id: ast::DUMMY_NODE_ID, - kind: main, - vis: respan(sp, ast::VisibilityKind::Public), - span: sp, - tokens: None, - }); - - // Integrate the new item into existing module structures. - let main = AstFragment::Items(smallvec![main]); - cx.ext_cx.monotonic_expander().fully_expand_fragment(main).make_items().pop().unwrap() -} - -/// Creates a slice containing every test like so: -/// &[&test1, &test2] -fn mk_tests_slice(cx: &TestCtxt<'_>, sp: Span) -> P { - debug!("building test vector from {} tests", cx.test_cases.len()); - let ref ecx = cx.ext_cx; - - ecx.expr_vec_slice( - sp, - cx.test_cases - .iter() - .map(|test| { - ecx.expr_addr_of(test.span, ecx.expr_path(ecx.path(test.span, vec![test.ident]))) - }) - .collect(), - ) -} - -fn is_test_case(i: &ast::Item) -> bool { - attr::contains_name(&i.attrs, sym::rustc_test_marker) -} - -fn get_test_runner(sd: &errors::Handler, krate: &ast::Crate) -> Option { - let test_attr = attr::find_by_name(&krate.attrs, sym::test_runner)?; - test_attr.meta_item_list().map(|meta_list| { - if meta_list.len() != 1 { - sd.span_fatal(test_attr.span, "`#![test_runner(..)]` accepts exactly 1 argument") - .raise() - } - match meta_list[0].meta_item() { - Some(meta_item) if meta_item.is_word() => meta_item.path.clone(), - _ => sd.span_fatal(test_attr.span, "`test_runner` argument must be a path").raise(), - } - }) -} diff --git a/src/libsyntax_ext/trace_macros.rs b/src/libsyntax_ext/trace_macros.rs deleted file mode 100644 index 96ae5bf5b4e..00000000000 --- a/src/libsyntax_ext/trace_macros.rs +++ /dev/null @@ -1,29 +0,0 @@ -use syntax::symbol::kw; -use syntax::tokenstream::{TokenStream, TokenTree}; -use syntax_expand::base::{self, ExtCtxt}; -use syntax_pos::Span; - -pub fn expand_trace_macros( - cx: &mut ExtCtxt<'_>, - sp: Span, - tt: TokenStream, -) -> Box { - let mut cursor = tt.into_trees(); - let mut err = false; - let value = match &cursor.next() { - Some(TokenTree::Token(token)) if token.is_keyword(kw::True) => true, - Some(TokenTree::Token(token)) if token.is_keyword(kw::False) => false, - _ => { - err = true; - false - } - }; - err |= cursor.next().is_some(); - if err { - cx.span_err(sp, "trace_macros! accepts only `true` or `false`") - } else { - cx.set_trace_macros(value); - } - - base::DummyResult::any_valid(sp) -} diff --git a/src/libsyntax_ext/util.rs b/src/libsyntax_ext/util.rs deleted file mode 100644 index aedd5aac1a9..00000000000 --- a/src/libsyntax_ext/util.rs +++ /dev/null @@ -1,12 +0,0 @@ -use rustc_feature::AttributeTemplate; -use rustc_parse::validate_attr; -use syntax::ast::MetaItem; -use syntax_expand::base::ExtCtxt; -use syntax_pos::Symbol; - -pub fn check_builtin_macro_attribute(ecx: &ExtCtxt<'_>, meta_item: &MetaItem, name: Symbol) { - // All the built-in macro attributes are "words" at the moment. - let template = AttributeTemplate::only_word(); - let attr = ecx.attribute(meta_item.clone()); - validate_attr::check_builtin_attribute(ecx.parse_sess, &attr, name, template); -} diff --git a/src/libsyntax_pos/Cargo.toml b/src/libsyntax_pos/Cargo.toml deleted file mode 100644 index 2cac76085d2..00000000000 --- a/src/libsyntax_pos/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -authors = ["The Rust Project Developers"] -name = "syntax_pos" -version = "0.0.0" -edition = "2018" - -[lib] -name = "syntax_pos" -path = "lib.rs" -doctest = false - -[dependencies] -rustc_serialize = { path = "../libserialize", package = "serialize" } -rustc_macros = { path = "../librustc_macros" } -rustc_data_structures = { path = "../librustc_data_structures" } -rustc_index = { path = "../librustc_index" } -arena = { path = "../libarena" } -scoped-tls = "1.0" -unicode-width = "0.1.4" -cfg-if = "0.1.2" -log = "0.4" diff --git a/src/libsyntax_pos/analyze_source_file.rs b/src/libsyntax_pos/analyze_source_file.rs deleted file mode 100644 index b4beb3dc376..00000000000 --- a/src/libsyntax_pos/analyze_source_file.rs +++ /dev/null @@ -1,274 +0,0 @@ -use super::*; -use unicode_width::UnicodeWidthChar; - -#[cfg(test)] -mod tests; - -/// Finds all newlines, multi-byte characters, and non-narrow characters in a -/// SourceFile. -/// -/// This function will use an SSE2 enhanced implementation if hardware support -/// is detected at runtime. -pub fn analyze_source_file( - src: &str, - source_file_start_pos: BytePos, -) -> (Vec, Vec, Vec) { - let mut lines = vec![source_file_start_pos]; - let mut multi_byte_chars = vec![]; - let mut non_narrow_chars = vec![]; - - // Calls the right implementation, depending on hardware support available. - analyze_source_file_dispatch( - src, - source_file_start_pos, - &mut lines, - &mut multi_byte_chars, - &mut non_narrow_chars, - ); - - // The code above optimistically registers a new line *after* each \n - // it encounters. If that point is already outside the source_file, remove - // it again. - if let Some(&last_line_start) = lines.last() { - let source_file_end = source_file_start_pos + BytePos::from_usize(src.len()); - assert!(source_file_end >= last_line_start); - if last_line_start == source_file_end { - lines.pop(); - } - } - - (lines, multi_byte_chars, non_narrow_chars) -} - -cfg_if::cfg_if! { - if #[cfg(all(any(target_arch = "x86", target_arch = "x86_64")))] { - fn analyze_source_file_dispatch(src: &str, - source_file_start_pos: BytePos, - lines: &mut Vec, - multi_byte_chars: &mut Vec, - non_narrow_chars: &mut Vec) { - if is_x86_feature_detected!("sse2") { - unsafe { - analyze_source_file_sse2(src, - source_file_start_pos, - lines, - multi_byte_chars, - non_narrow_chars); - } - } else { - analyze_source_file_generic(src, - src.len(), - source_file_start_pos, - lines, - multi_byte_chars, - non_narrow_chars); - - } - } - - /// Checks 16 byte chunks of text at a time. If the chunk contains - /// something other than printable ASCII characters and newlines, the - /// function falls back to the generic implementation. Otherwise it uses - /// SSE2 intrinsics to quickly find all newlines. - #[target_feature(enable = "sse2")] - unsafe fn analyze_source_file_sse2(src: &str, - output_offset: BytePos, - lines: &mut Vec, - multi_byte_chars: &mut Vec, - non_narrow_chars: &mut Vec) { - #[cfg(target_arch = "x86")] - use std::arch::x86::*; - #[cfg(target_arch = "x86_64")] - use std::arch::x86_64::*; - - const CHUNK_SIZE: usize = 16; - - let src_bytes = src.as_bytes(); - - let chunk_count = src.len() / CHUNK_SIZE; - - // This variable keeps track of where we should start decoding a - // chunk. If a multi-byte character spans across chunk boundaries, - // we need to skip that part in the next chunk because we already - // handled it. - let mut intra_chunk_offset = 0; - - for chunk_index in 0 .. chunk_count { - let ptr = src_bytes.as_ptr() as *const __m128i; - // We don't know if the pointer is aligned to 16 bytes, so we - // use `loadu`, which supports unaligned loading. - let chunk = _mm_loadu_si128(ptr.offset(chunk_index as isize)); - - // For character in the chunk, see if its byte value is < 0, which - // indicates that it's part of a UTF-8 char. - let multibyte_test = _mm_cmplt_epi8(chunk, _mm_set1_epi8(0)); - // Create a bit mask from the comparison results. - let multibyte_mask = _mm_movemask_epi8(multibyte_test); - - // If the bit mask is all zero, we only have ASCII chars here: - if multibyte_mask == 0 { - assert!(intra_chunk_offset == 0); - - // Check if there are any control characters in the chunk. All - // control characters that we can encounter at this point have a - // byte value less than 32 or ... - let control_char_test0 = _mm_cmplt_epi8(chunk, _mm_set1_epi8(32)); - let control_char_mask0 = _mm_movemask_epi8(control_char_test0); - - // ... it's the ASCII 'DEL' character with a value of 127. - let control_char_test1 = _mm_cmpeq_epi8(chunk, _mm_set1_epi8(127)); - let control_char_mask1 = _mm_movemask_epi8(control_char_test1); - - let control_char_mask = control_char_mask0 | control_char_mask1; - - if control_char_mask != 0 { - // Check for newlines in the chunk - let newlines_test = _mm_cmpeq_epi8(chunk, _mm_set1_epi8(b'\n' as i8)); - let newlines_mask = _mm_movemask_epi8(newlines_test); - - if control_char_mask == newlines_mask { - // All control characters are newlines, record them - let mut newlines_mask = 0xFFFF0000 | newlines_mask as u32; - let output_offset = output_offset + - BytePos::from_usize(chunk_index * CHUNK_SIZE + 1); - - loop { - let index = newlines_mask.trailing_zeros(); - - if index >= CHUNK_SIZE as u32 { - // We have arrived at the end of the chunk. - break - } - - lines.push(BytePos(index) + output_offset); - - // Clear the bit, so we can find the next one. - newlines_mask &= (!1) << index; - } - - // We are done for this chunk. All control characters were - // newlines and we took care of those. - continue - } else { - // Some of the control characters are not newlines, - // fall through to the slow path below. - } - } else { - // No control characters, nothing to record for this chunk - continue - } - } - - // The slow path. - // There are control chars in here, fallback to generic decoding. - let scan_start = chunk_index * CHUNK_SIZE + intra_chunk_offset; - intra_chunk_offset = analyze_source_file_generic( - &src[scan_start .. ], - CHUNK_SIZE - intra_chunk_offset, - BytePos::from_usize(scan_start) + output_offset, - lines, - multi_byte_chars, - non_narrow_chars - ); - } - - // There might still be a tail left to analyze - let tail_start = chunk_count * CHUNK_SIZE + intra_chunk_offset; - if tail_start < src.len() { - analyze_source_file_generic(&src[tail_start as usize ..], - src.len() - tail_start, - output_offset + BytePos::from_usize(tail_start), - lines, - multi_byte_chars, - non_narrow_chars); - } - } - } else { - - // The target (or compiler version) does not support SSE2 ... - fn analyze_source_file_dispatch(src: &str, - source_file_start_pos: BytePos, - lines: &mut Vec, - multi_byte_chars: &mut Vec, - non_narrow_chars: &mut Vec) { - analyze_source_file_generic(src, - src.len(), - source_file_start_pos, - lines, - multi_byte_chars, - non_narrow_chars); - } - } -} - -// `scan_len` determines the number of bytes in `src` to scan. Note that the -// function can read past `scan_len` if a multi-byte character start within the -// range but extends past it. The overflow is returned by the function. -fn analyze_source_file_generic( - src: &str, - scan_len: usize, - output_offset: BytePos, - lines: &mut Vec, - multi_byte_chars: &mut Vec, - non_narrow_chars: &mut Vec, -) -> usize { - assert!(src.len() >= scan_len); - let mut i = 0; - let src_bytes = src.as_bytes(); - - while i < scan_len { - let byte = unsafe { - // We verified that i < scan_len <= src.len() - *src_bytes.get_unchecked(i as usize) - }; - - // How much to advance in order to get to the next UTF-8 char in the - // string. - let mut char_len = 1; - - if byte < 32 { - // This is an ASCII control character, it could be one of the cases - // that are interesting to us. - - let pos = BytePos::from_usize(i) + output_offset; - - match byte { - b'\n' => { - lines.push(pos + BytePos(1)); - } - b'\t' => { - non_narrow_chars.push(NonNarrowChar::Tab(pos)); - } - _ => { - non_narrow_chars.push(NonNarrowChar::ZeroWidth(pos)); - } - } - } else if byte >= 127 { - // The slow path: - // This is either ASCII control character "DEL" or the beginning of - // a multibyte char. Just decode to `char`. - let c = (&src[i..]).chars().next().unwrap(); - char_len = c.len_utf8(); - - let pos = BytePos::from_usize(i) + output_offset; - - if char_len > 1 { - assert!(char_len >= 2 && char_len <= 4); - let mbc = MultiByteChar { pos, bytes: char_len as u8 }; - multi_byte_chars.push(mbc); - } - - // Assume control characters are zero width. - // FIXME: How can we decide between `width` and `width_cjk`? - let char_width = UnicodeWidthChar::width(c).unwrap_or(0); - - if char_width != 1 { - non_narrow_chars.push(NonNarrowChar::new(pos, char_width)); - } - } - - i += char_len; - } - - i - scan_len -} diff --git a/src/libsyntax_pos/analyze_source_file/tests.rs b/src/libsyntax_pos/analyze_source_file/tests.rs deleted file mode 100644 index cb418a4bdaf..00000000000 --- a/src/libsyntax_pos/analyze_source_file/tests.rs +++ /dev/null @@ -1,142 +0,0 @@ -use super::*; - -macro_rules! test { - (case: $test_name:ident, - text: $text:expr, - source_file_start_pos: $source_file_start_pos:expr, - lines: $lines:expr, - multi_byte_chars: $multi_byte_chars:expr, - non_narrow_chars: $non_narrow_chars:expr,) => { - #[test] - fn $test_name() { - let (lines, multi_byte_chars, non_narrow_chars) = - analyze_source_file($text, BytePos($source_file_start_pos)); - - let expected_lines: Vec = $lines.into_iter().map(|pos| BytePos(pos)).collect(); - - assert_eq!(lines, expected_lines); - - let expected_mbcs: Vec = $multi_byte_chars - .into_iter() - .map(|(pos, bytes)| MultiByteChar { pos: BytePos(pos), bytes }) - .collect(); - - assert_eq!(multi_byte_chars, expected_mbcs); - - let expected_nncs: Vec = $non_narrow_chars - .into_iter() - .map(|(pos, width)| NonNarrowChar::new(BytePos(pos), width)) - .collect(); - - assert_eq!(non_narrow_chars, expected_nncs); - } - }; -} - -test!( - case: empty_text, - text: "", - source_file_start_pos: 0, - lines: vec![], - multi_byte_chars: vec![], - non_narrow_chars: vec![], -); - -test!( - case: newlines_short, - text: "a\nc", - source_file_start_pos: 0, - lines: vec![0, 2], - multi_byte_chars: vec![], - non_narrow_chars: vec![], -); - -test!( - case: newlines_long, - text: "012345678\nabcdef012345678\na", - source_file_start_pos: 0, - lines: vec![0, 10, 26], - multi_byte_chars: vec![], - non_narrow_chars: vec![], -); - -test!( - case: newline_and_multi_byte_char_in_same_chunk, - text: "01234β789\nbcdef0123456789abcdef", - source_file_start_pos: 0, - lines: vec![0, 11], - multi_byte_chars: vec![(5, 2)], - non_narrow_chars: vec![], -); - -test!( - case: newline_and_control_char_in_same_chunk, - text: "01234\u{07}6789\nbcdef0123456789abcdef", - source_file_start_pos: 0, - lines: vec![0, 11], - multi_byte_chars: vec![], - non_narrow_chars: vec![(5, 0)], -); - -test!( - case: multi_byte_char_short, - text: "aβc", - source_file_start_pos: 0, - lines: vec![0], - multi_byte_chars: vec![(1, 2)], - non_narrow_chars: vec![], -); - -test!( - case: multi_byte_char_long, - text: "0123456789abcΔf012345β", - source_file_start_pos: 0, - lines: vec![0], - multi_byte_chars: vec![(13, 2), (22, 2)], - non_narrow_chars: vec![], -); - -test!( - case: multi_byte_char_across_chunk_boundary, - text: "0123456789abcdeΔ123456789abcdef01234", - source_file_start_pos: 0, - lines: vec![0], - multi_byte_chars: vec![(15, 2)], - non_narrow_chars: vec![], -); - -test!( - case: multi_byte_char_across_chunk_boundary_tail, - text: "0123456789abcdeΔ....", - source_file_start_pos: 0, - lines: vec![0], - multi_byte_chars: vec![(15, 2)], - non_narrow_chars: vec![], -); - -test!( - case: non_narrow_short, - text: "0\t2", - source_file_start_pos: 0, - lines: vec![0], - multi_byte_chars: vec![], - non_narrow_chars: vec![(1, 4)], -); - -test!( - case: non_narrow_long, - text: "01\t3456789abcdef01234567\u{07}9", - source_file_start_pos: 0, - lines: vec![0], - multi_byte_chars: vec![], - non_narrow_chars: vec![(2, 4), (24, 0)], -); - -test!( - case: output_offset_all, - text: "01\t345\n789abcΔf01234567\u{07}9\nbcΔf", - source_file_start_pos: 1000, - lines: vec![0 + 1000, 7 + 1000, 27 + 1000], - multi_byte_chars: vec![(13 + 1000, 2), (29 + 1000, 2)], - non_narrow_chars: vec![(2 + 1000, 4), (24 + 1000, 0)], -); diff --git a/src/libsyntax_pos/caching_source_map_view.rs b/src/libsyntax_pos/caching_source_map_view.rs deleted file mode 100644 index c329f2225b0..00000000000 --- a/src/libsyntax_pos/caching_source_map_view.rs +++ /dev/null @@ -1,108 +0,0 @@ -use crate::source_map::SourceMap; -use crate::{BytePos, SourceFile}; -use rustc_data_structures::sync::Lrc; - -#[derive(Clone)] -struct CacheEntry { - time_stamp: usize, - line_number: usize, - line_start: BytePos, - line_end: BytePos, - file: Lrc, - file_index: usize, -} - -#[derive(Clone)] -pub struct CachingSourceMapView<'cm> { - source_map: &'cm SourceMap, - line_cache: [CacheEntry; 3], - time_stamp: usize, -} - -impl<'cm> CachingSourceMapView<'cm> { - pub fn new(source_map: &'cm SourceMap) -> CachingSourceMapView<'cm> { - let files = source_map.files(); - let first_file = files[0].clone(); - let entry = CacheEntry { - time_stamp: 0, - line_number: 0, - line_start: BytePos(0), - line_end: BytePos(0), - file: first_file, - file_index: 0, - }; - - CachingSourceMapView { - source_map, - line_cache: [entry.clone(), entry.clone(), entry], - time_stamp: 0, - } - } - - pub fn byte_pos_to_line_and_col( - &mut self, - pos: BytePos, - ) -> Option<(Lrc, usize, BytePos)> { - self.time_stamp += 1; - - // Check if the position is in one of the cached lines - for cache_entry in self.line_cache.iter_mut() { - if pos >= cache_entry.line_start && pos < cache_entry.line_end { - cache_entry.time_stamp = self.time_stamp; - - return Some(( - cache_entry.file.clone(), - cache_entry.line_number, - pos - cache_entry.line_start, - )); - } - } - - // No cache hit ... - let mut oldest = 0; - for index in 1..self.line_cache.len() { - if self.line_cache[index].time_stamp < self.line_cache[oldest].time_stamp { - oldest = index; - } - } - - let cache_entry = &mut self.line_cache[oldest]; - - // If the entry doesn't point to the correct file, fix it up - if pos < cache_entry.file.start_pos || pos >= cache_entry.file.end_pos { - let file_valid; - if self.source_map.files().len() > 0 { - let file_index = self.source_map.lookup_source_file_idx(pos); - let file = self.source_map.files()[file_index].clone(); - - if pos >= file.start_pos && pos < file.end_pos { - cache_entry.file = file; - cache_entry.file_index = file_index; - file_valid = true; - } else { - file_valid = false; - } - } else { - file_valid = false; - } - - if !file_valid { - return None; - } - } - - let line_index = cache_entry.file.lookup_line(pos).unwrap(); - let line_bounds = cache_entry.file.line_bounds(line_index); - - cache_entry.line_number = line_index + 1; - cache_entry.line_start = line_bounds.0; - cache_entry.line_end = line_bounds.1; - cache_entry.time_stamp = self.time_stamp; - - return Some(( - cache_entry.file.clone(), - cache_entry.line_number, - pos - cache_entry.line_start, - )); - } -} diff --git a/src/libsyntax_pos/edition.rs b/src/libsyntax_pos/edition.rs deleted file mode 100644 index 3017191563b..00000000000 --- a/src/libsyntax_pos/edition.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::symbol::{sym, Symbol}; -use std::fmt; -use std::str::FromStr; - -use rustc_macros::HashStable_Generic; - -/// The edition of the compiler (RFC 2052) -#[derive( - Clone, - Copy, - Hash, - PartialEq, - PartialOrd, - Debug, - RustcEncodable, - RustcDecodable, - Eq, - HashStable_Generic -)] -pub enum Edition { - // editions must be kept in order, oldest to newest - /// The 2015 edition - Edition2015, - /// The 2018 edition - Edition2018, - // when adding new editions, be sure to update: - // - // - Update the `ALL_EDITIONS` const - // - Update the EDITION_NAME_LIST const - // - add a `rust_####()` function to the session - // - update the enum in Cargo's sources as well -} - -// must be in order from oldest to newest -pub const ALL_EDITIONS: &[Edition] = &[Edition::Edition2015, Edition::Edition2018]; - -pub const EDITION_NAME_LIST: &str = "2015|2018"; - -pub const DEFAULT_EDITION: Edition = Edition::Edition2015; - -impl fmt::Display for Edition { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s = match *self { - Edition::Edition2015 => "2015", - Edition::Edition2018 => "2018", - }; - write!(f, "{}", s) - } -} - -impl Edition { - pub fn lint_name(&self) -> &'static str { - match *self { - Edition::Edition2015 => "rust_2015_compatibility", - Edition::Edition2018 => "rust_2018_compatibility", - } - } - - pub fn feature_name(&self) -> Symbol { - match *self { - Edition::Edition2015 => sym::rust_2015_preview, - Edition::Edition2018 => sym::rust_2018_preview, - } - } - - pub fn is_stable(&self) -> bool { - match *self { - Edition::Edition2015 => true, - Edition::Edition2018 => true, - } - } -} - -impl FromStr for Edition { - type Err = (); - fn from_str(s: &str) -> Result { - match s { - "2015" => Ok(Edition::Edition2015), - "2018" => Ok(Edition::Edition2018), - _ => Err(()), - } - } -} diff --git a/src/libsyntax_pos/fatal_error.rs b/src/libsyntax_pos/fatal_error.rs deleted file mode 100644 index 718c0ddbc63..00000000000 --- a/src/libsyntax_pos/fatal_error.rs +++ /dev/null @@ -1,26 +0,0 @@ -/// Used as a return value to signify a fatal error occurred. (It is also -/// used as the argument to panic at the moment, but that will eventually -/// not be true.) -#[derive(Copy, Clone, Debug)] -#[must_use] -pub struct FatalError; - -pub struct FatalErrorMarker; - -// Don't implement Send on FatalError. This makes it impossible to panic!(FatalError). -// We don't want to invoke the panic handler and print a backtrace for fatal errors. -impl !Send for FatalError {} - -impl FatalError { - pub fn raise(self) -> ! { - std::panic::resume_unwind(Box::new(FatalErrorMarker)) - } -} - -impl std::fmt::Display for FatalError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "parser fatal error") - } -} - -impl std::error::Error for FatalError {} diff --git a/src/libsyntax_pos/hygiene.rs b/src/libsyntax_pos/hygiene.rs deleted file mode 100644 index fd1f07c743b..00000000000 --- a/src/libsyntax_pos/hygiene.rs +++ /dev/null @@ -1,850 +0,0 @@ -//! Machinery for hygienic macros, inspired by the `MTWT[1]` paper. -//! -//! `[1]` Matthew Flatt, Ryan Culpepper, David Darais, and Robert Bruce Findler. 2012. -//! *Macros that work together: Compile-time bindings, partial expansion, -//! and definition contexts*. J. Funct. Program. 22, 2 (March 2012), 181-216. -//! DOI=10.1017/S0956796812000093 - -// Hygiene data is stored in a global variable and accessed via TLS, which -// means that accesses are somewhat expensive. (`HygieneData::with` -// encapsulates a single access.) Therefore, on hot code paths it is worth -// ensuring that multiple HygieneData accesses are combined into a single -// `HygieneData::with`. -// -// This explains why `HygieneData`, `SyntaxContext` and `ExpnId` have interfaces -// with a certain amount of redundancy in them. For example, -// `SyntaxContext::outer_expn_data` combines `SyntaxContext::outer` and -// `ExpnId::expn_data` so that two `HygieneData` accesses can be performed within -// a single `HygieneData::with` call. -// -// It also explains why many functions appear in `HygieneData` and again in -// `SyntaxContext` or `ExpnId`. For example, `HygieneData::outer` and -// `SyntaxContext::outer` do the same thing, but the former is for use within a -// `HygieneData::with` call while the latter is for use outside such a call. -// When modifying this file it is important to understand this distinction, -// because getting it wrong can lead to nested `HygieneData::with` calls that -// trigger runtime aborts. (Fortunately these are obvious and easy to fix.) - -use crate::edition::Edition; -use crate::symbol::{kw, sym, Symbol}; -use crate::GLOBALS; -use crate::{Span, DUMMY_SP}; - -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use rustc_macros::HashStable_Generic; -use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; -use std::fmt; - -/// A `SyntaxContext` represents a chain of pairs `(ExpnId, Transparency)` named "marks". -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct SyntaxContext(u32); - -#[derive(Debug)] -struct SyntaxContextData { - outer_expn: ExpnId, - outer_transparency: Transparency, - parent: SyntaxContext, - /// This context, but with all transparent and semi-transparent expansions filtered away. - opaque: SyntaxContext, - /// This context, but with all transparent expansions filtered away. - opaque_and_semitransparent: SyntaxContext, - /// Name of the crate to which `$crate` with this context would resolve. - dollar_crate_name: Symbol, -} - -/// A unique ID associated with a macro invocation and expansion. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] -pub struct ExpnId(u32); - -/// A property of a macro expansion that determines how identifiers -/// produced by that expansion are resolved. -#[derive( - Copy, - Clone, - PartialEq, - Eq, - PartialOrd, - Hash, - Debug, - RustcEncodable, - RustcDecodable, - HashStable_Generic -)] -pub enum Transparency { - /// Identifier produced by a transparent expansion is always resolved at call-site. - /// Call-site spans in procedural macros, hygiene opt-out in `macro` should use this. - Transparent, - /// Identifier produced by a semi-transparent expansion may be resolved - /// either at call-site or at definition-site. - /// If it's a local variable, label or `$crate` then it's resolved at def-site. - /// Otherwise it's resolved at call-site. - /// `macro_rules` macros behave like this, built-in macros currently behave like this too, - /// but that's an implementation detail. - SemiTransparent, - /// Identifier produced by an opaque expansion is always resolved at definition-site. - /// Def-site spans in procedural macros, identifiers from `macro` by default use this. - Opaque, -} - -impl ExpnId { - pub fn fresh(expn_data: Option) -> Self { - HygieneData::with(|data| data.fresh_expn(expn_data)) - } - - /// The ID of the theoretical expansion that generates freshly parsed, unexpanded AST. - #[inline] - pub fn root() -> Self { - ExpnId(0) - } - - #[inline] - pub fn as_u32(self) -> u32 { - self.0 - } - - #[inline] - pub fn from_u32(raw: u32) -> ExpnId { - ExpnId(raw) - } - - #[inline] - pub fn expn_data(self) -> ExpnData { - HygieneData::with(|data| data.expn_data(self).clone()) - } - - #[inline] - pub fn set_expn_data(self, expn_data: ExpnData) { - HygieneData::with(|data| { - let old_expn_data = &mut data.expn_data[self.0 as usize]; - assert!(old_expn_data.is_none(), "expansion data is reset for an expansion ID"); - *old_expn_data = Some(expn_data); - }) - } - - pub fn is_descendant_of(self, ancestor: ExpnId) -> bool { - HygieneData::with(|data| data.is_descendant_of(self, ancestor)) - } - - /// `expn_id.outer_expn_is_descendant_of(ctxt)` is equivalent to but faster than - /// `expn_id.is_descendant_of(ctxt.outer_expn())`. - pub fn outer_expn_is_descendant_of(self, ctxt: SyntaxContext) -> bool { - HygieneData::with(|data| data.is_descendant_of(self, data.outer_expn(ctxt))) - } - - /// Returns span for the macro which originally caused this expansion to happen. - /// - /// Stops backtracing at include! boundary. - pub fn expansion_cause(mut self) -> Option { - let mut last_macro = None; - loop { - let expn_data = self.expn_data(); - // Stop going up the backtrace once include! is encountered - if expn_data.is_root() || expn_data.kind.descr() == sym::include { - break; - } - self = expn_data.call_site.ctxt().outer_expn(); - last_macro = Some(expn_data.call_site); - } - last_macro - } -} - -#[derive(Debug)] -crate struct HygieneData { - /// Each expansion should have an associated expansion data, but sometimes there's a delay - /// between creation of an expansion ID and obtaining its data (e.g. macros are collected - /// first and then resolved later), so we use an `Option` here. - expn_data: Vec>, - syntax_context_data: Vec, - syntax_context_map: FxHashMap<(SyntaxContext, ExpnId, Transparency), SyntaxContext>, -} - -impl HygieneData { - crate fn new(edition: Edition) -> Self { - HygieneData { - expn_data: vec![Some(ExpnData::default(ExpnKind::Root, DUMMY_SP, edition))], - syntax_context_data: vec![SyntaxContextData { - outer_expn: ExpnId::root(), - outer_transparency: Transparency::Opaque, - parent: SyntaxContext(0), - opaque: SyntaxContext(0), - opaque_and_semitransparent: SyntaxContext(0), - dollar_crate_name: kw::DollarCrate, - }], - syntax_context_map: FxHashMap::default(), - } - } - - fn with T>(f: F) -> T { - GLOBALS.with(|globals| f(&mut *globals.hygiene_data.borrow_mut())) - } - - fn fresh_expn(&mut self, expn_data: Option) -> ExpnId { - self.expn_data.push(expn_data); - ExpnId(self.expn_data.len() as u32 - 1) - } - - fn expn_data(&self, expn_id: ExpnId) -> &ExpnData { - self.expn_data[expn_id.0 as usize].as_ref().expect("no expansion data for an expansion ID") - } - - fn is_descendant_of(&self, mut expn_id: ExpnId, ancestor: ExpnId) -> bool { - while expn_id != ancestor { - if expn_id == ExpnId::root() { - return false; - } - expn_id = self.expn_data(expn_id).parent; - } - true - } - - fn modern(&self, ctxt: SyntaxContext) -> SyntaxContext { - self.syntax_context_data[ctxt.0 as usize].opaque - } - - fn modern_and_legacy(&self, ctxt: SyntaxContext) -> SyntaxContext { - self.syntax_context_data[ctxt.0 as usize].opaque_and_semitransparent - } - - fn outer_expn(&self, ctxt: SyntaxContext) -> ExpnId { - self.syntax_context_data[ctxt.0 as usize].outer_expn - } - - fn outer_mark(&self, ctxt: SyntaxContext) -> (ExpnId, Transparency) { - let data = &self.syntax_context_data[ctxt.0 as usize]; - (data.outer_expn, data.outer_transparency) - } - - fn parent_ctxt(&self, ctxt: SyntaxContext) -> SyntaxContext { - self.syntax_context_data[ctxt.0 as usize].parent - } - - fn remove_mark(&self, ctxt: &mut SyntaxContext) -> (ExpnId, Transparency) { - let outer_mark = self.outer_mark(*ctxt); - *ctxt = self.parent_ctxt(*ctxt); - outer_mark - } - - fn marks(&self, mut ctxt: SyntaxContext) -> Vec<(ExpnId, Transparency)> { - let mut marks = Vec::new(); - while ctxt != SyntaxContext::root() { - marks.push(self.outer_mark(ctxt)); - ctxt = self.parent_ctxt(ctxt); - } - marks.reverse(); - marks - } - - fn walk_chain(&self, mut span: Span, to: SyntaxContext) -> Span { - while span.from_expansion() && span.ctxt() != to { - span = self.expn_data(self.outer_expn(span.ctxt())).call_site; - } - span - } - - fn adjust(&self, ctxt: &mut SyntaxContext, expn_id: ExpnId) -> Option { - let mut scope = None; - while !self.is_descendant_of(expn_id, self.outer_expn(*ctxt)) { - scope = Some(self.remove_mark(ctxt).0); - } - scope - } - - fn apply_mark( - &mut self, - ctxt: SyntaxContext, - expn_id: ExpnId, - transparency: Transparency, - ) -> SyntaxContext { - assert_ne!(expn_id, ExpnId::root()); - if transparency == Transparency::Opaque { - return self.apply_mark_internal(ctxt, expn_id, transparency); - } - - let call_site_ctxt = self.expn_data(expn_id).call_site.ctxt(); - let mut call_site_ctxt = if transparency == Transparency::SemiTransparent { - self.modern(call_site_ctxt) - } else { - self.modern_and_legacy(call_site_ctxt) - }; - - if call_site_ctxt == SyntaxContext::root() { - return self.apply_mark_internal(ctxt, expn_id, transparency); - } - - // Otherwise, `expn_id` is a macros 1.0 definition and the call site is in a - // macros 2.0 expansion, i.e., a macros 1.0 invocation is in a macros 2.0 definition. - // - // In this case, the tokens from the macros 1.0 definition inherit the hygiene - // at their invocation. That is, we pretend that the macros 1.0 definition - // was defined at its invocation (i.e., inside the macros 2.0 definition) - // so that the macros 2.0 definition remains hygienic. - // - // See the example at `test/ui/hygiene/legacy_interaction.rs`. - for (expn_id, transparency) in self.marks(ctxt) { - call_site_ctxt = self.apply_mark_internal(call_site_ctxt, expn_id, transparency); - } - self.apply_mark_internal(call_site_ctxt, expn_id, transparency) - } - - fn apply_mark_internal( - &mut self, - ctxt: SyntaxContext, - expn_id: ExpnId, - transparency: Transparency, - ) -> SyntaxContext { - let syntax_context_data = &mut self.syntax_context_data; - let mut opaque = syntax_context_data[ctxt.0 as usize].opaque; - let mut opaque_and_semitransparent = - syntax_context_data[ctxt.0 as usize].opaque_and_semitransparent; - - if transparency >= Transparency::Opaque { - let parent = opaque; - opaque = *self - .syntax_context_map - .entry((parent, expn_id, transparency)) - .or_insert_with(|| { - let new_opaque = SyntaxContext(syntax_context_data.len() as u32); - syntax_context_data.push(SyntaxContextData { - outer_expn: expn_id, - outer_transparency: transparency, - parent, - opaque: new_opaque, - opaque_and_semitransparent: new_opaque, - dollar_crate_name: kw::DollarCrate, - }); - new_opaque - }); - } - - if transparency >= Transparency::SemiTransparent { - let parent = opaque_and_semitransparent; - opaque_and_semitransparent = *self - .syntax_context_map - .entry((parent, expn_id, transparency)) - .or_insert_with(|| { - let new_opaque_and_semitransparent = - SyntaxContext(syntax_context_data.len() as u32); - syntax_context_data.push(SyntaxContextData { - outer_expn: expn_id, - outer_transparency: transparency, - parent, - opaque, - opaque_and_semitransparent: new_opaque_and_semitransparent, - dollar_crate_name: kw::DollarCrate, - }); - new_opaque_and_semitransparent - }); - } - - let parent = ctxt; - *self.syntax_context_map.entry((parent, expn_id, transparency)).or_insert_with(|| { - let new_opaque_and_semitransparent_and_transparent = - SyntaxContext(syntax_context_data.len() as u32); - syntax_context_data.push(SyntaxContextData { - outer_expn: expn_id, - outer_transparency: transparency, - parent, - opaque, - opaque_and_semitransparent, - dollar_crate_name: kw::DollarCrate, - }); - new_opaque_and_semitransparent_and_transparent - }) - } -} - -pub fn clear_syntax_context_map() { - HygieneData::with(|data| data.syntax_context_map = FxHashMap::default()); -} - -pub fn walk_chain(span: Span, to: SyntaxContext) -> Span { - HygieneData::with(|data| data.walk_chain(span, to)) -} - -pub fn update_dollar_crate_names(mut get_name: impl FnMut(SyntaxContext) -> Symbol) { - // The new contexts that need updating are at the end of the list and have `$crate` as a name. - let (len, to_update) = HygieneData::with(|data| { - ( - data.syntax_context_data.len(), - data.syntax_context_data - .iter() - .rev() - .take_while(|scdata| scdata.dollar_crate_name == kw::DollarCrate) - .count(), - ) - }); - // The callback must be called from outside of the `HygieneData` lock, - // since it will try to acquire it too. - let range_to_update = len - to_update..len; - let names: Vec<_> = - range_to_update.clone().map(|idx| get_name(SyntaxContext::from_u32(idx as u32))).collect(); - HygieneData::with(|data| { - range_to_update.zip(names.into_iter()).for_each(|(idx, name)| { - data.syntax_context_data[idx].dollar_crate_name = name; - }) - }) -} - -pub fn debug_hygiene_data(verbose: bool) -> String { - HygieneData::with(|data| { - if verbose { - format!("{:#?}", data) - } else { - let mut s = String::from(""); - s.push_str("Expansions:"); - data.expn_data.iter().enumerate().for_each(|(id, expn_info)| { - let expn_info = expn_info.as_ref().expect("no expansion data for an expansion ID"); - s.push_str(&format!( - "\n{}: parent: {:?}, call_site_ctxt: {:?}, kind: {:?}", - id, - expn_info.parent, - expn_info.call_site.ctxt(), - expn_info.kind, - )); - }); - s.push_str("\n\nSyntaxContexts:"); - data.syntax_context_data.iter().enumerate().for_each(|(id, ctxt)| { - s.push_str(&format!( - "\n#{}: parent: {:?}, outer_mark: ({:?}, {:?})", - id, ctxt.parent, ctxt.outer_expn, ctxt.outer_transparency, - )); - }); - s - } - }) -} - -impl SyntaxContext { - #[inline] - pub const fn root() -> Self { - SyntaxContext(0) - } - - #[inline] - crate fn as_u32(self) -> u32 { - self.0 - } - - #[inline] - crate fn from_u32(raw: u32) -> SyntaxContext { - SyntaxContext(raw) - } - - /// Extend a syntax context with a given expansion and transparency. - crate fn apply_mark(self, expn_id: ExpnId, transparency: Transparency) -> SyntaxContext { - HygieneData::with(|data| data.apply_mark(self, expn_id, transparency)) - } - - /// Pulls a single mark off of the syntax context. This effectively moves the - /// context up one macro definition level. That is, if we have a nested macro - /// definition as follows: - /// - /// ```rust - /// macro_rules! f { - /// macro_rules! g { - /// ... - /// } - /// } - /// ``` - /// - /// and we have a SyntaxContext that is referring to something declared by an invocation - /// of g (call it g1), calling remove_mark will result in the SyntaxContext for the - /// invocation of f that created g1. - /// Returns the mark that was removed. - pub fn remove_mark(&mut self) -> ExpnId { - HygieneData::with(|data| data.remove_mark(self).0) - } - - pub fn marks(self) -> Vec<(ExpnId, Transparency)> { - HygieneData::with(|data| data.marks(self)) - } - - /// Adjust this context for resolution in a scope created by the given expansion. - /// For example, consider the following three resolutions of `f`: - /// - /// ```rust - /// mod foo { pub fn f() {} } // `f`'s `SyntaxContext` is empty. - /// m!(f); - /// macro m($f:ident) { - /// mod bar { - /// pub fn f() {} // `f`'s `SyntaxContext` has a single `ExpnId` from `m`. - /// pub fn $f() {} // `$f`'s `SyntaxContext` is empty. - /// } - /// foo::f(); // `f`'s `SyntaxContext` has a single `ExpnId` from `m` - /// //^ Since `mod foo` is outside this expansion, `adjust` removes the mark from `f`, - /// //| and it resolves to `::foo::f`. - /// bar::f(); // `f`'s `SyntaxContext` has a single `ExpnId` from `m` - /// //^ Since `mod bar` not outside this expansion, `adjust` does not change `f`, - /// //| and it resolves to `::bar::f`. - /// bar::$f(); // `f`'s `SyntaxContext` is empty. - /// //^ Since `mod bar` is not outside this expansion, `adjust` does not change `$f`, - /// //| and it resolves to `::bar::$f`. - /// } - /// ``` - /// This returns the expansion whose definition scope we use to privacy check the resolution, - /// or `None` if we privacy check as usual (i.e., not w.r.t. a macro definition scope). - pub fn adjust(&mut self, expn_id: ExpnId) -> Option { - HygieneData::with(|data| data.adjust(self, expn_id)) - } - - /// Like `SyntaxContext::adjust`, but also modernizes `self`. - pub fn modernize_and_adjust(&mut self, expn_id: ExpnId) -> Option { - HygieneData::with(|data| { - *self = data.modern(*self); - data.adjust(self, expn_id) - }) - } - - /// Adjust this context for resolution in a scope created by the given expansion - /// via a glob import with the given `SyntaxContext`. - /// For example: - /// - /// ```rust - /// m!(f); - /// macro m($i:ident) { - /// mod foo { - /// pub fn f() {} // `f`'s `SyntaxContext` has a single `ExpnId` from `m`. - /// pub fn $i() {} // `$i`'s `SyntaxContext` is empty. - /// } - /// n(f); - /// macro n($j:ident) { - /// use foo::*; - /// f(); // `f`'s `SyntaxContext` has a mark from `m` and a mark from `n` - /// //^ `glob_adjust` removes the mark from `n`, so this resolves to `foo::f`. - /// $i(); // `$i`'s `SyntaxContext` has a mark from `n` - /// //^ `glob_adjust` removes the mark from `n`, so this resolves to `foo::$i`. - /// $j(); // `$j`'s `SyntaxContext` has a mark from `m` - /// //^ This cannot be glob-adjusted, so this is a resolution error. - /// } - /// } - /// ``` - /// This returns `None` if the context cannot be glob-adjusted. - /// Otherwise, it returns the scope to use when privacy checking (see `adjust` for details). - pub fn glob_adjust(&mut self, expn_id: ExpnId, glob_span: Span) -> Option> { - HygieneData::with(|data| { - let mut scope = None; - let mut glob_ctxt = data.modern(glob_span.ctxt()); - while !data.is_descendant_of(expn_id, data.outer_expn(glob_ctxt)) { - scope = Some(data.remove_mark(&mut glob_ctxt).0); - if data.remove_mark(self).0 != scope.unwrap() { - return None; - } - } - if data.adjust(self, expn_id).is_some() { - return None; - } - Some(scope) - }) - } - - /// Undo `glob_adjust` if possible: - /// - /// ```rust - /// if let Some(privacy_checking_scope) = self.reverse_glob_adjust(expansion, glob_ctxt) { - /// assert!(self.glob_adjust(expansion, glob_ctxt) == Some(privacy_checking_scope)); - /// } - /// ``` - pub fn reverse_glob_adjust( - &mut self, - expn_id: ExpnId, - glob_span: Span, - ) -> Option> { - HygieneData::with(|data| { - if data.adjust(self, expn_id).is_some() { - return None; - } - - let mut glob_ctxt = data.modern(glob_span.ctxt()); - let mut marks = Vec::new(); - while !data.is_descendant_of(expn_id, data.outer_expn(glob_ctxt)) { - marks.push(data.remove_mark(&mut glob_ctxt)); - } - - let scope = marks.last().map(|mark| mark.0); - while let Some((expn_id, transparency)) = marks.pop() { - *self = data.apply_mark(*self, expn_id, transparency); - } - Some(scope) - }) - } - - pub fn hygienic_eq(self, other: SyntaxContext, expn_id: ExpnId) -> bool { - HygieneData::with(|data| { - let mut self_modern = data.modern(self); - data.adjust(&mut self_modern, expn_id); - self_modern == data.modern(other) - }) - } - - #[inline] - pub fn modern(self) -> SyntaxContext { - HygieneData::with(|data| data.modern(self)) - } - - #[inline] - pub fn modern_and_legacy(self) -> SyntaxContext { - HygieneData::with(|data| data.modern_and_legacy(self)) - } - - #[inline] - pub fn outer_expn(self) -> ExpnId { - HygieneData::with(|data| data.outer_expn(self)) - } - - /// `ctxt.outer_expn_data()` is equivalent to but faster than - /// `ctxt.outer_expn().expn_data()`. - #[inline] - pub fn outer_expn_data(self) -> ExpnData { - HygieneData::with(|data| data.expn_data(data.outer_expn(self)).clone()) - } - - #[inline] - pub fn outer_mark_with_data(self) -> (ExpnId, Transparency, ExpnData) { - HygieneData::with(|data| { - let (expn_id, transparency) = data.outer_mark(self); - (expn_id, transparency, data.expn_data(expn_id).clone()) - }) - } - - pub fn dollar_crate_name(self) -> Symbol { - HygieneData::with(|data| data.syntax_context_data[self.0 as usize].dollar_crate_name) - } -} - -impl fmt::Debug for SyntaxContext { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "#{}", self.0) - } -} - -impl Span { - /// Creates a fresh expansion with given properties. - /// Expansions are normally created by macros, but in some cases expansions are created for - /// other compiler-generated code to set per-span properties like allowed unstable features. - /// The returned span belongs to the created expansion and has the new properties, - /// but its location is inherited from the current span. - pub fn fresh_expansion(self, expn_data: ExpnData) -> Span { - self.fresh_expansion_with_transparency(expn_data, Transparency::Transparent) - } - - pub fn fresh_expansion_with_transparency( - self, - expn_data: ExpnData, - transparency: Transparency, - ) -> Span { - HygieneData::with(|data| { - let expn_id = data.fresh_expn(Some(expn_data)); - self.with_ctxt(data.apply_mark(SyntaxContext::root(), expn_id, transparency)) - }) - } -} - -/// A subset of properties from both macro definition and macro call available through global data. -/// Avoid using this if you have access to the original definition or call structures. -#[derive(Clone, Debug, RustcEncodable, RustcDecodable, HashStable_Generic)] -pub struct ExpnData { - // --- The part unique to each expansion. - /// The kind of this expansion - macro or compiler desugaring. - pub kind: ExpnKind, - /// The expansion that produced this expansion. - #[stable_hasher(ignore)] - pub parent: ExpnId, - /// The location of the actual macro invocation or syntax sugar , e.g. - /// `let x = foo!();` or `if let Some(y) = x {}` - /// - /// This may recursively refer to other macro invocations, e.g., if - /// `foo!()` invoked `bar!()` internally, and there was an - /// expression inside `bar!`; the call_site of the expression in - /// the expansion would point to the `bar!` invocation; that - /// call_site span would have its own ExpnData, with the call_site - /// pointing to the `foo!` invocation. - pub call_site: Span, - - // --- The part specific to the macro/desugaring definition. - // --- It may be reasonable to share this part between expansions with the same definition, - // --- but such sharing is known to bring some minor inconveniences without also bringing - // --- noticeable perf improvements (PR #62898). - /// The span of the macro definition (possibly dummy). - /// This span serves only informational purpose and is not used for resolution. - pub def_site: Span, - /// List of #[unstable]/feature-gated features that the macro is allowed to use - /// internally without forcing the whole crate to opt-in - /// to them. - pub allow_internal_unstable: Option>, - /// Whether the macro is allowed to use `unsafe` internally - /// even if the user crate has `#![forbid(unsafe_code)]`. - pub allow_internal_unsafe: bool, - /// Enables the macro helper hack (`ident!(...)` -> `$crate::ident!(...)`) - /// for a given macro. - pub local_inner_macros: bool, - /// Edition of the crate in which the macro is defined. - pub edition: Edition, -} - -impl ExpnData { - /// Constructs expansion data with default properties. - pub fn default(kind: ExpnKind, call_site: Span, edition: Edition) -> ExpnData { - ExpnData { - kind, - parent: ExpnId::root(), - call_site, - def_site: DUMMY_SP, - allow_internal_unstable: None, - allow_internal_unsafe: false, - local_inner_macros: false, - edition, - } - } - - pub fn allow_unstable( - kind: ExpnKind, - call_site: Span, - edition: Edition, - allow_internal_unstable: Lrc<[Symbol]>, - ) -> ExpnData { - ExpnData { - allow_internal_unstable: Some(allow_internal_unstable), - ..ExpnData::default(kind, call_site, edition) - } - } - - #[inline] - pub fn is_root(&self) -> bool { - if let ExpnKind::Root = self.kind { true } else { false } - } -} - -/// Expansion kind. -#[derive(Clone, Debug, RustcEncodable, RustcDecodable, HashStable_Generic)] -pub enum ExpnKind { - /// No expansion, aka root expansion. Only `ExpnId::root()` has this kind. - Root, - /// Expansion produced by a macro. - Macro(MacroKind, Symbol), - /// Transform done by the compiler on the AST. - AstPass(AstPass), - /// Desugaring done by the compiler during HIR lowering. - Desugaring(DesugaringKind), -} - -impl ExpnKind { - pub fn descr(&self) -> Symbol { - match *self { - ExpnKind::Root => kw::PathRoot, - ExpnKind::Macro(_, descr) => descr, - ExpnKind::AstPass(kind) => Symbol::intern(kind.descr()), - ExpnKind::Desugaring(kind) => Symbol::intern(kind.descr()), - } - } -} - -/// The kind of macro invocation or definition. -#[derive( - Clone, - Copy, - PartialEq, - Eq, - RustcEncodable, - RustcDecodable, - Hash, - Debug, - HashStable_Generic -)] -pub enum MacroKind { - /// A bang macro `foo!()`. - Bang, - /// An attribute macro `#[foo]`. - Attr, - /// A derive macro `#[derive(Foo)]` - Derive, -} - -impl MacroKind { - pub fn descr(self) -> &'static str { - match self { - MacroKind::Bang => "macro", - MacroKind::Attr => "attribute macro", - MacroKind::Derive => "derive macro", - } - } - - pub fn descr_expected(self) -> &'static str { - match self { - MacroKind::Attr => "attribute", - _ => self.descr(), - } - } - - pub fn article(self) -> &'static str { - match self { - MacroKind::Attr => "an", - _ => "a", - } - } -} - -/// The kind of AST transform. -#[derive(Clone, Copy, PartialEq, Debug, RustcEncodable, RustcDecodable, HashStable_Generic)] -pub enum AstPass { - StdImports, - TestHarness, - ProcMacroHarness, -} - -impl AstPass { - fn descr(self) -> &'static str { - match self { - AstPass::StdImports => "standard library imports", - AstPass::TestHarness => "test harness", - AstPass::ProcMacroHarness => "proc macro harness", - } - } -} - -/// The kind of compiler desugaring. -#[derive(Clone, Copy, PartialEq, Debug, RustcEncodable, RustcDecodable, HashStable_Generic)] -pub enum DesugaringKind { - /// We desugar `if c { i } else { e }` to `match $ExprKind::Use(c) { true => i, _ => e }`. - /// However, we do not want to blame `c` for unreachability but rather say that `i` - /// is unreachable. This desugaring kind allows us to avoid blaming `c`. - /// This also applies to `while` loops. - CondTemporary, - QuestionMark, - TryBlock, - /// Desugaring of an `impl Trait` in return type position - /// to an `type Foo = impl Trait;` and replacing the - /// `impl Trait` with `Foo`. - OpaqueTy, - Async, - Await, - ForLoop, -} - -impl DesugaringKind { - /// The description wording should combine well with "desugaring of {}". - fn descr(self) -> &'static str { - match self { - DesugaringKind::CondTemporary => "`if` or `while` condition", - DesugaringKind::Async => "`async` block or function", - DesugaringKind::Await => "`await` expression", - DesugaringKind::QuestionMark => "operator `?`", - DesugaringKind::TryBlock => "`try` block", - DesugaringKind::OpaqueTy => "`impl Trait`", - DesugaringKind::ForLoop => "`for` loop", - } - } -} - -impl Encodable for ExpnId { - fn encode(&self, _: &mut E) -> Result<(), E::Error> { - Ok(()) // FIXME(jseyfried) intercrate hygiene - } -} - -impl Decodable for ExpnId { - fn decode(_: &mut D) -> Result { - Ok(ExpnId::root()) // FIXME(jseyfried) intercrate hygiene - } -} diff --git a/src/libsyntax_pos/lib.rs b/src/libsyntax_pos/lib.rs deleted file mode 100644 index a58c12f2350..00000000000 --- a/src/libsyntax_pos/lib.rs +++ /dev/null @@ -1,1672 +0,0 @@ -//! The source positions and related helper functions. -//! -//! ## Note -//! -//! This API is completely unstable and subject to change. - -#![doc(html_root_url = "https://doc.rust-lang.org/nightly/")] -#![feature(const_fn)] -#![feature(crate_visibility_modifier)] -#![feature(nll)] -#![feature(optin_builtin_traits)] -#![feature(rustc_attrs)] -#![feature(specialization)] -#![feature(step_trait)] - -use rustc_data_structures::AtomicRef; -use rustc_macros::HashStable_Generic; -use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; - -mod caching_source_map_view; -pub mod source_map; -pub use self::caching_source_map_view::CachingSourceMapView; - -pub mod edition; -use edition::Edition; -pub mod hygiene; -use hygiene::Transparency; -pub use hygiene::{DesugaringKind, ExpnData, ExpnId, ExpnKind, MacroKind, SyntaxContext}; - -mod span_encoding; -pub use span_encoding::{Span, DUMMY_SP}; - -pub mod symbol; -pub use symbol::{sym, Symbol}; - -mod analyze_source_file; -pub mod fatal_error; - -use rustc_data_structures::fingerprint::Fingerprint; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; -use rustc_data_structures::sync::{Lock, Lrc}; - -use std::borrow::Cow; -use std::cell::RefCell; -use std::cmp::{self, Ordering}; -use std::fmt; -use std::hash::{Hash, Hasher}; -use std::ops::{Add, Sub}; -use std::path::PathBuf; - -#[cfg(test)] -mod tests; - -pub struct Globals { - symbol_interner: Lock, - span_interner: Lock, - hygiene_data: Lock, -} - -impl Globals { - pub fn new(edition: Edition) -> Globals { - Globals { - symbol_interner: Lock::new(symbol::Interner::fresh()), - span_interner: Lock::new(span_encoding::SpanInterner::default()), - hygiene_data: Lock::new(hygiene::HygieneData::new(edition)), - } - } -} - -scoped_tls::scoped_thread_local!(pub static GLOBALS: Globals); - -/// Differentiates between real files and common virtual files. -#[derive( - Debug, - Eq, - PartialEq, - Clone, - Ord, - PartialOrd, - Hash, - RustcDecodable, - RustcEncodable, - HashStable_Generic -)] -pub enum FileName { - Real(PathBuf), - /// A macro. This includes the full name of the macro, so that there are no clashes. - Macros(String), - /// Call to `quote!`. - QuoteExpansion(u64), - /// Command line. - Anon(u64), - /// Hack in `src/libsyntax/parse.rs`. - // FIXME(jseyfried) - MacroExpansion(u64), - ProcMacroSourceCode(u64), - /// Strings provided as `--cfg [cfgspec]` stored in a `crate_cfg`. - CfgSpec(u64), - /// Strings provided as crate attributes in the CLI. - CliCrateAttr(u64), - /// Custom sources for explicit parser calls from plugins and drivers. - Custom(String), - DocTest(PathBuf, isize), -} - -impl std::fmt::Display for FileName { - fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use FileName::*; - match *self { - Real(ref path) => write!(fmt, "{}", path.display()), - Macros(ref name) => write!(fmt, "<{} macros>", name), - QuoteExpansion(_) => write!(fmt, ""), - MacroExpansion(_) => write!(fmt, ""), - Anon(_) => write!(fmt, ""), - ProcMacroSourceCode(_) => write!(fmt, ""), - CfgSpec(_) => write!(fmt, ""), - CliCrateAttr(_) => write!(fmt, ""), - Custom(ref s) => write!(fmt, "<{}>", s), - DocTest(ref path, _) => write!(fmt, "{}", path.display()), - } - } -} - -impl From for FileName { - fn from(p: PathBuf) -> Self { - assert!(!p.to_string_lossy().ends_with('>')); - FileName::Real(p) - } -} - -impl FileName { - pub fn is_real(&self) -> bool { - use FileName::*; - match *self { - Real(_) => true, - Macros(_) - | Anon(_) - | MacroExpansion(_) - | ProcMacroSourceCode(_) - | CfgSpec(_) - | CliCrateAttr(_) - | Custom(_) - | QuoteExpansion(_) - | DocTest(_, _) => false, - } - } - - pub fn is_macros(&self) -> bool { - use FileName::*; - match *self { - Real(_) - | Anon(_) - | MacroExpansion(_) - | ProcMacroSourceCode(_) - | CfgSpec(_) - | CliCrateAttr(_) - | Custom(_) - | QuoteExpansion(_) - | DocTest(_, _) => false, - Macros(_) => true, - } - } - - pub fn quote_expansion_source_code(src: &str) -> FileName { - let mut hasher = StableHasher::new(); - src.hash(&mut hasher); - FileName::QuoteExpansion(hasher.finish()) - } - - pub fn macro_expansion_source_code(src: &str) -> FileName { - let mut hasher = StableHasher::new(); - src.hash(&mut hasher); - FileName::MacroExpansion(hasher.finish()) - } - - pub fn anon_source_code(src: &str) -> FileName { - let mut hasher = StableHasher::new(); - src.hash(&mut hasher); - FileName::Anon(hasher.finish()) - } - - pub fn proc_macro_source_code(src: &str) -> FileName { - let mut hasher = StableHasher::new(); - src.hash(&mut hasher); - FileName::ProcMacroSourceCode(hasher.finish()) - } - - pub fn cfg_spec_source_code(src: &str) -> FileName { - let mut hasher = StableHasher::new(); - src.hash(&mut hasher); - FileName::QuoteExpansion(hasher.finish()) - } - - pub fn cli_crate_attr_source_code(src: &str) -> FileName { - let mut hasher = StableHasher::new(); - src.hash(&mut hasher); - FileName::CliCrateAttr(hasher.finish()) - } - - pub fn doc_test_source_code(path: PathBuf, line: isize) -> FileName { - FileName::DocTest(path, line) - } -} - -/// Spans represent a region of code, used for error reporting. Positions in spans -/// are *absolute* positions from the beginning of the source_map, not positions -/// relative to `SourceFile`s. Methods on the `SourceMap` can be used to relate spans back -/// to the original source. -/// You must be careful if the span crosses more than one file - you will not be -/// able to use many of the functions on spans in source_map and you cannot assume -/// that the length of the `span = hi - lo`; there may be space in the `BytePos` -/// range between files. -/// -/// `SpanData` is public because `Span` uses a thread-local interner and can't be -/// sent to other threads, but some pieces of performance infra run in a separate thread. -/// Using `Span` is generally preferred. -#[derive(Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd)] -pub struct SpanData { - pub lo: BytePos, - pub hi: BytePos, - /// Information about where the macro came from, if this piece of - /// code was created by a macro expansion. - pub ctxt: SyntaxContext, -} - -impl SpanData { - #[inline] - pub fn with_lo(&self, lo: BytePos) -> Span { - Span::new(lo, self.hi, self.ctxt) - } - #[inline] - pub fn with_hi(&self, hi: BytePos) -> Span { - Span::new(self.lo, hi, self.ctxt) - } - #[inline] - pub fn with_ctxt(&self, ctxt: SyntaxContext) -> Span { - Span::new(self.lo, self.hi, ctxt) - } -} - -// The interner is pointed to by a thread local value which is only set on the main thread -// with parallelization is disabled. So we don't allow `Span` to transfer between threads -// to avoid panics and other errors, even though it would be memory safe to do so. -#[cfg(not(parallel_compiler))] -impl !Send for Span {} -#[cfg(not(parallel_compiler))] -impl !Sync for Span {} - -impl PartialOrd for Span { - fn partial_cmp(&self, rhs: &Self) -> Option { - PartialOrd::partial_cmp(&self.data(), &rhs.data()) - } -} -impl Ord for Span { - fn cmp(&self, rhs: &Self) -> Ordering { - Ord::cmp(&self.data(), &rhs.data()) - } -} - -/// A collection of spans. Spans have two orthogonal attributes: -/// -/// - They can be *primary spans*. In this case they are the locus of -/// the error, and would be rendered with `^^^`. -/// - They can have a *label*. In this case, the label is written next -/// to the mark in the snippet when we render. -#[derive(Clone, Debug, Hash, PartialEq, Eq, RustcEncodable, RustcDecodable)] -pub struct MultiSpan { - primary_spans: Vec, - span_labels: Vec<(Span, String)>, -} - -impl Span { - #[inline] - pub fn lo(self) -> BytePos { - self.data().lo - } - #[inline] - pub fn with_lo(self, lo: BytePos) -> Span { - self.data().with_lo(lo) - } - #[inline] - pub fn hi(self) -> BytePos { - self.data().hi - } - #[inline] - pub fn with_hi(self, hi: BytePos) -> Span { - self.data().with_hi(hi) - } - #[inline] - pub fn ctxt(self) -> SyntaxContext { - self.data().ctxt - } - #[inline] - pub fn with_ctxt(self, ctxt: SyntaxContext) -> Span { - self.data().with_ctxt(ctxt) - } - - /// Returns `true` if this is a dummy span with any hygienic context. - #[inline] - pub fn is_dummy(self) -> bool { - let span = self.data(); - span.lo.0 == 0 && span.hi.0 == 0 - } - - /// Returns `true` if this span comes from a macro or desugaring. - #[inline] - pub fn from_expansion(self) -> bool { - self.ctxt() != SyntaxContext::root() - } - - #[inline] - pub fn with_root_ctxt(lo: BytePos, hi: BytePos) -> Span { - Span::new(lo, hi, SyntaxContext::root()) - } - - /// Returns a new span representing an empty span at the beginning of this span - #[inline] - pub fn shrink_to_lo(self) -> Span { - let span = self.data(); - span.with_hi(span.lo) - } - /// Returns a new span representing an empty span at the end of this span. - #[inline] - pub fn shrink_to_hi(self) -> Span { - let span = self.data(); - span.with_lo(span.hi) - } - - /// Returns `self` if `self` is not the dummy span, and `other` otherwise. - pub fn substitute_dummy(self, other: Span) -> Span { - if self.is_dummy() { other } else { self } - } - - /// Returns `true` if `self` fully encloses `other`. - pub fn contains(self, other: Span) -> bool { - let span = self.data(); - let other = other.data(); - span.lo <= other.lo && other.hi <= span.hi - } - - /// Returns `true` if `self` touches `other`. - pub fn overlaps(self, other: Span) -> bool { - let span = self.data(); - let other = other.data(); - span.lo < other.hi && other.lo < span.hi - } - - /// Returns `true` if the spans are equal with regards to the source text. - /// - /// Use this instead of `==` when either span could be generated code, - /// and you only care that they point to the same bytes of source text. - pub fn source_equal(&self, other: &Span) -> bool { - let span = self.data(); - let other = other.data(); - span.lo == other.lo && span.hi == other.hi - } - - /// Returns `Some(span)`, where the start is trimmed by the end of `other`. - pub fn trim_start(self, other: Span) -> Option { - let span = self.data(); - let other = other.data(); - if span.hi > other.hi { Some(span.with_lo(cmp::max(span.lo, other.hi))) } else { None } - } - - /// Returns the source span -- this is either the supplied span, or the span for - /// the macro callsite that expanded to it. - pub fn source_callsite(self) -> Span { - let expn_data = self.ctxt().outer_expn_data(); - if !expn_data.is_root() { expn_data.call_site.source_callsite() } else { self } - } - - /// The `Span` for the tokens in the previous macro expansion from which `self` was generated, - /// if any. - pub fn parent(self) -> Option { - let expn_data = self.ctxt().outer_expn_data(); - if !expn_data.is_root() { Some(expn_data.call_site) } else { None } - } - - /// Edition of the crate from which this span came. - pub fn edition(self) -> edition::Edition { - self.ctxt().outer_expn_data().edition - } - - #[inline] - pub fn rust_2015(&self) -> bool { - self.edition() == edition::Edition::Edition2015 - } - - #[inline] - pub fn rust_2018(&self) -> bool { - self.edition() >= edition::Edition::Edition2018 - } - - /// Returns the source callee. - /// - /// Returns `None` if the supplied span has no expansion trace, - /// else returns the `ExpnData` for the macro definition - /// corresponding to the source callsite. - pub fn source_callee(self) -> Option { - fn source_callee(expn_data: ExpnData) -> ExpnData { - let next_expn_data = expn_data.call_site.ctxt().outer_expn_data(); - if !next_expn_data.is_root() { source_callee(next_expn_data) } else { expn_data } - } - let expn_data = self.ctxt().outer_expn_data(); - if !expn_data.is_root() { Some(source_callee(expn_data)) } else { None } - } - - /// Checks if a span is "internal" to a macro in which `#[unstable]` - /// items can be used (that is, a macro marked with - /// `#[allow_internal_unstable]`). - pub fn allows_unstable(&self, feature: Symbol) -> bool { - self.ctxt().outer_expn_data().allow_internal_unstable.map_or(false, |features| { - features - .iter() - .any(|&f| f == feature || f == sym::allow_internal_unstable_backcompat_hack) - }) - } - - /// Checks if this span arises from a compiler desugaring of kind `kind`. - pub fn is_desugaring(&self, kind: DesugaringKind) -> bool { - match self.ctxt().outer_expn_data().kind { - ExpnKind::Desugaring(k) => k == kind, - _ => false, - } - } - - /// Returns the compiler desugaring that created this span, or `None` - /// if this span is not from a desugaring. - pub fn desugaring_kind(&self) -> Option { - match self.ctxt().outer_expn_data().kind { - ExpnKind::Desugaring(k) => Some(k), - _ => None, - } - } - - /// Checks if a span is "internal" to a macro in which `unsafe` - /// can be used without triggering the `unsafe_code` lint - // (that is, a macro marked with `#[allow_internal_unsafe]`). - pub fn allows_unsafe(&self) -> bool { - self.ctxt().outer_expn_data().allow_internal_unsafe - } - - pub fn macro_backtrace(mut self) -> Vec { - let mut prev_span = DUMMY_SP; - let mut result = vec![]; - loop { - let expn_data = self.ctxt().outer_expn_data(); - if expn_data.is_root() { - break; - } - // Don't print recursive invocations. - if !expn_data.call_site.source_equal(&prev_span) { - let (pre, post) = match expn_data.kind { - ExpnKind::Root => break, - ExpnKind::Desugaring(..) => ("desugaring of ", ""), - ExpnKind::AstPass(..) => ("", ""), - ExpnKind::Macro(macro_kind, _) => match macro_kind { - MacroKind::Bang => ("", "!"), - MacroKind::Attr => ("#[", "]"), - MacroKind::Derive => ("#[derive(", ")]"), - }, - }; - result.push(MacroBacktrace { - call_site: expn_data.call_site, - macro_decl_name: format!("{}{}{}", pre, expn_data.kind.descr(), post), - def_site_span: expn_data.def_site, - }); - } - - prev_span = self; - self = expn_data.call_site; - } - result - } - - /// Returns a `Span` that would enclose both `self` and `end`. - pub fn to(self, end: Span) -> Span { - let span_data = self.data(); - let end_data = end.data(); - // FIXME(jseyfried): `self.ctxt` should always equal `end.ctxt` here (cf. issue #23480). - // Return the macro span on its own to avoid weird diagnostic output. It is preferable to - // have an incomplete span than a completely nonsensical one. - if span_data.ctxt != end_data.ctxt { - if span_data.ctxt == SyntaxContext::root() { - return end; - } else if end_data.ctxt == SyntaxContext::root() { - return self; - } - // Both spans fall within a macro. - // FIXME(estebank): check if it is the *same* macro. - } - Span::new( - cmp::min(span_data.lo, end_data.lo), - cmp::max(span_data.hi, end_data.hi), - if span_data.ctxt == SyntaxContext::root() { end_data.ctxt } else { span_data.ctxt }, - ) - } - - /// Returns a `Span` between the end of `self` to the beginning of `end`. - pub fn between(self, end: Span) -> Span { - let span = self.data(); - let end = end.data(); - Span::new( - span.hi, - end.lo, - if end.ctxt == SyntaxContext::root() { end.ctxt } else { span.ctxt }, - ) - } - - /// Returns a `Span` between the beginning of `self` to the beginning of `end`. - pub fn until(self, end: Span) -> Span { - let span = self.data(); - let end = end.data(); - Span::new( - span.lo, - end.lo, - if end.ctxt == SyntaxContext::root() { end.ctxt } else { span.ctxt }, - ) - } - - pub fn from_inner(self, inner: InnerSpan) -> Span { - let span = self.data(); - Span::new( - span.lo + BytePos::from_usize(inner.start), - span.lo + BytePos::from_usize(inner.end), - span.ctxt, - ) - } - - /// Equivalent of `Span::def_site` from the proc macro API, - /// except that the location is taken from the `self` span. - pub fn with_def_site_ctxt(self, expn_id: ExpnId) -> Span { - self.with_ctxt_from_mark(expn_id, Transparency::Opaque) - } - - /// Equivalent of `Span::call_site` from the proc macro API, - /// except that the location is taken from the `self` span. - pub fn with_call_site_ctxt(&self, expn_id: ExpnId) -> Span { - self.with_ctxt_from_mark(expn_id, Transparency::Transparent) - } - - /// Equivalent of `Span::mixed_site` from the proc macro API, - /// except that the location is taken from the `self` span. - pub fn with_mixed_site_ctxt(&self, expn_id: ExpnId) -> Span { - self.with_ctxt_from_mark(expn_id, Transparency::SemiTransparent) - } - - /// Produces a span with the same location as `self` and context produced by a macro with the - /// given ID and transparency, assuming that macro was defined directly and not produced by - /// some other macro (which is the case for built-in and procedural macros). - pub fn with_ctxt_from_mark(self, expn_id: ExpnId, transparency: Transparency) -> Span { - self.with_ctxt(SyntaxContext::root().apply_mark(expn_id, transparency)) - } - - #[inline] - pub fn apply_mark(self, expn_id: ExpnId, transparency: Transparency) -> Span { - let span = self.data(); - span.with_ctxt(span.ctxt.apply_mark(expn_id, transparency)) - } - - #[inline] - pub fn remove_mark(&mut self) -> ExpnId { - let mut span = self.data(); - let mark = span.ctxt.remove_mark(); - *self = Span::new(span.lo, span.hi, span.ctxt); - mark - } - - #[inline] - pub fn adjust(&mut self, expn_id: ExpnId) -> Option { - let mut span = self.data(); - let mark = span.ctxt.adjust(expn_id); - *self = Span::new(span.lo, span.hi, span.ctxt); - mark - } - - #[inline] - pub fn modernize_and_adjust(&mut self, expn_id: ExpnId) -> Option { - let mut span = self.data(); - let mark = span.ctxt.modernize_and_adjust(expn_id); - *self = Span::new(span.lo, span.hi, span.ctxt); - mark - } - - #[inline] - pub fn glob_adjust(&mut self, expn_id: ExpnId, glob_span: Span) -> Option> { - let mut span = self.data(); - let mark = span.ctxt.glob_adjust(expn_id, glob_span); - *self = Span::new(span.lo, span.hi, span.ctxt); - mark - } - - #[inline] - pub fn reverse_glob_adjust( - &mut self, - expn_id: ExpnId, - glob_span: Span, - ) -> Option> { - let mut span = self.data(); - let mark = span.ctxt.reverse_glob_adjust(expn_id, glob_span); - *self = Span::new(span.lo, span.hi, span.ctxt); - mark - } - - #[inline] - pub fn modern(self) -> Span { - let span = self.data(); - span.with_ctxt(span.ctxt.modern()) - } - - #[inline] - pub fn modern_and_legacy(self) -> Span { - let span = self.data(); - span.with_ctxt(span.ctxt.modern_and_legacy()) - } -} - -#[derive(Clone, Debug)] -pub struct SpanLabel { - /// The span we are going to include in the final snippet. - pub span: Span, - - /// Is this a primary span? This is the "locus" of the message, - /// and is indicated with a `^^^^` underline, versus `----`. - pub is_primary: bool, - - /// What label should we attach to this span (if any)? - pub label: Option, -} - -impl Default for Span { - fn default() -> Self { - DUMMY_SP - } -} - -impl rustc_serialize::UseSpecializedEncodable for Span { - fn default_encode(&self, s: &mut S) -> Result<(), S::Error> { - let span = self.data(); - s.emit_struct("Span", 2, |s| { - s.emit_struct_field("lo", 0, |s| span.lo.encode(s))?; - - s.emit_struct_field("hi", 1, |s| span.hi.encode(s)) - }) - } -} - -impl rustc_serialize::UseSpecializedDecodable for Span { - fn default_decode(d: &mut D) -> Result { - d.read_struct("Span", 2, |d| { - let lo = d.read_struct_field("lo", 0, Decodable::decode)?; - let hi = d.read_struct_field("hi", 1, Decodable::decode)?; - Ok(Span::with_root_ctxt(lo, hi)) - }) - } -} - -pub fn default_span_debug(span: Span, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Span") - .field("lo", &span.lo()) - .field("hi", &span.hi()) - .field("ctxt", &span.ctxt()) - .finish() -} - -impl fmt::Debug for Span { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - (*SPAN_DEBUG)(*self, f) - } -} - -impl fmt::Debug for SpanData { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - (*SPAN_DEBUG)(Span::new(self.lo, self.hi, self.ctxt), f) - } -} - -impl MultiSpan { - #[inline] - pub fn new() -> MultiSpan { - MultiSpan { primary_spans: vec![], span_labels: vec![] } - } - - pub fn from_span(primary_span: Span) -> MultiSpan { - MultiSpan { primary_spans: vec![primary_span], span_labels: vec![] } - } - - pub fn from_spans(vec: Vec) -> MultiSpan { - MultiSpan { primary_spans: vec, span_labels: vec![] } - } - - pub fn push_span_label(&mut self, span: Span, label: String) { - self.span_labels.push((span, label)); - } - - /// Selects the first primary span (if any). - pub fn primary_span(&self) -> Option { - self.primary_spans.first().cloned() - } - - /// Returns all primary spans. - pub fn primary_spans(&self) -> &[Span] { - &self.primary_spans - } - - /// Returns `true` if any of the primary spans are displayable. - pub fn has_primary_spans(&self) -> bool { - self.primary_spans.iter().any(|sp| !sp.is_dummy()) - } - - /// Returns `true` if this contains only a dummy primary span with any hygienic context. - pub fn is_dummy(&self) -> bool { - let mut is_dummy = true; - for span in &self.primary_spans { - if !span.is_dummy() { - is_dummy = false; - } - } - is_dummy - } - - /// Replaces all occurrences of one Span with another. Used to move `Span`s in areas that don't - /// display well (like std macros). Returns whether replacements occurred. - pub fn replace(&mut self, before: Span, after: Span) -> bool { - let mut replacements_occurred = false; - for primary_span in &mut self.primary_spans { - if *primary_span == before { - *primary_span = after; - replacements_occurred = true; - } - } - for span_label in &mut self.span_labels { - if span_label.0 == before { - span_label.0 = after; - replacements_occurred = true; - } - } - replacements_occurred - } - - /// Returns the strings to highlight. We always ensure that there - /// is an entry for each of the primary spans -- for each primary - /// span `P`, if there is at least one label with span `P`, we return - /// those labels (marked as primary). But otherwise we return - /// `SpanLabel` instances with empty labels. - pub fn span_labels(&self) -> Vec { - let is_primary = |span| self.primary_spans.contains(&span); - - let mut span_labels = self - .span_labels - .iter() - .map(|&(span, ref label)| SpanLabel { - span, - is_primary: is_primary(span), - label: Some(label.clone()), - }) - .collect::>(); - - for &span in &self.primary_spans { - if !span_labels.iter().any(|sl| sl.span == span) { - span_labels.push(SpanLabel { span, is_primary: true, label: None }); - } - } - - span_labels - } - - /// Returns `true` if any of the span labels is displayable. - pub fn has_span_labels(&self) -> bool { - self.span_labels.iter().any(|(sp, _)| !sp.is_dummy()) - } -} - -impl From for MultiSpan { - fn from(span: Span) -> MultiSpan { - MultiSpan::from_span(span) - } -} - -impl From> for MultiSpan { - fn from(spans: Vec) -> MultiSpan { - MultiSpan::from_spans(spans) - } -} - -/// Identifies an offset of a multi-byte character in a `SourceFile`. -#[derive(Copy, Clone, RustcEncodable, RustcDecodable, Eq, PartialEq, Debug)] -pub struct MultiByteChar { - /// The absolute offset of the character in the `SourceMap`. - pub pos: BytePos, - /// The number of bytes, `>= 2`. - pub bytes: u8, -} - -/// Identifies an offset of a non-narrow character in a `SourceFile`. -#[derive(Copy, Clone, RustcEncodable, RustcDecodable, Eq, PartialEq, Debug)] -pub enum NonNarrowChar { - /// Represents a zero-width character. - ZeroWidth(BytePos), - /// Represents a wide (full-width) character. - Wide(BytePos), - /// Represents a tab character, represented visually with a width of 4 characters. - Tab(BytePos), -} - -impl NonNarrowChar { - fn new(pos: BytePos, width: usize) -> Self { - match width { - 0 => NonNarrowChar::ZeroWidth(pos), - 2 => NonNarrowChar::Wide(pos), - 4 => NonNarrowChar::Tab(pos), - _ => panic!("width {} given for non-narrow character", width), - } - } - - /// Returns the absolute offset of the character in the `SourceMap`. - pub fn pos(&self) -> BytePos { - match *self { - NonNarrowChar::ZeroWidth(p) | NonNarrowChar::Wide(p) | NonNarrowChar::Tab(p) => p, - } - } - - /// Returns the width of the character, 0 (zero-width) or 2 (wide). - pub fn width(&self) -> usize { - match *self { - NonNarrowChar::ZeroWidth(_) => 0, - NonNarrowChar::Wide(_) => 2, - NonNarrowChar::Tab(_) => 4, - } - } -} - -impl Add for NonNarrowChar { - type Output = Self; - - fn add(self, rhs: BytePos) -> Self { - match self { - NonNarrowChar::ZeroWidth(pos) => NonNarrowChar::ZeroWidth(pos + rhs), - NonNarrowChar::Wide(pos) => NonNarrowChar::Wide(pos + rhs), - NonNarrowChar::Tab(pos) => NonNarrowChar::Tab(pos + rhs), - } - } -} - -impl Sub for NonNarrowChar { - type Output = Self; - - fn sub(self, rhs: BytePos) -> Self { - match self { - NonNarrowChar::ZeroWidth(pos) => NonNarrowChar::ZeroWidth(pos - rhs), - NonNarrowChar::Wide(pos) => NonNarrowChar::Wide(pos - rhs), - NonNarrowChar::Tab(pos) => NonNarrowChar::Tab(pos - rhs), - } - } -} - -/// Identifies an offset of a character that was normalized away from `SourceFile`. -#[derive(Copy, Clone, RustcEncodable, RustcDecodable, Eq, PartialEq, Debug)] -pub struct NormalizedPos { - /// The absolute offset of the character in the `SourceMap`. - pub pos: BytePos, - /// The difference between original and normalized string at position. - pub diff: u32, -} - -/// The state of the lazy external source loading mechanism of a `SourceFile`. -#[derive(PartialEq, Eq, Clone)] -pub enum ExternalSource { - /// The external source has been loaded already. - Present(String), - /// No attempt has been made to load the external source. - AbsentOk, - /// A failed attempt has been made to load the external source. - AbsentErr, - /// No external source has to be loaded, since the `SourceFile` represents a local crate. - Unneeded, -} - -impl ExternalSource { - pub fn is_absent(&self) -> bool { - match *self { - ExternalSource::Present(_) => false, - _ => true, - } - } - - pub fn get_source(&self) -> Option<&str> { - match *self { - ExternalSource::Present(ref src) => Some(src), - _ => None, - } - } -} - -#[derive(Debug)] -pub struct OffsetOverflowError; - -/// A single source in the `SourceMap`. -#[derive(Clone)] -pub struct SourceFile { - /// The name of the file that the source came from. Source that doesn't - /// originate from files has names between angle brackets by convention - /// (e.g., ``). - pub name: FileName, - /// `true` if the `name` field above has been modified by `--remap-path-prefix`. - pub name_was_remapped: bool, - /// The unmapped path of the file that the source came from. - /// Set to `None` if the `SourceFile` was imported from an external crate. - pub unmapped_path: Option, - /// Indicates which crate this `SourceFile` was imported from. - pub crate_of_origin: u32, - /// The complete source code. - pub src: Option>, - /// The source code's hash. - pub src_hash: u128, - /// The external source code (used for external crates, which will have a `None` - /// value as `self.src`. - pub external_src: Lock, - /// The start position of this source in the `SourceMap`. - pub start_pos: BytePos, - /// The end position of this source in the `SourceMap`. - pub end_pos: BytePos, - /// Locations of lines beginnings in the source code. - pub lines: Vec, - /// Locations of multi-byte characters in the source code. - pub multibyte_chars: Vec, - /// Width of characters that are not narrow in the source code. - pub non_narrow_chars: Vec, - /// Locations of characters removed during normalization. - pub normalized_pos: Vec, - /// A hash of the filename, used for speeding up hashing in incremental compilation. - pub name_hash: u128, -} - -impl Encodable for SourceFile { - fn encode(&self, s: &mut S) -> Result<(), S::Error> { - s.emit_struct("SourceFile", 8, |s| { - s.emit_struct_field("name", 0, |s| self.name.encode(s))?; - s.emit_struct_field("name_was_remapped", 1, |s| self.name_was_remapped.encode(s))?; - s.emit_struct_field("src_hash", 2, |s| self.src_hash.encode(s))?; - s.emit_struct_field("start_pos", 3, |s| self.start_pos.encode(s))?; - s.emit_struct_field("end_pos", 4, |s| self.end_pos.encode(s))?; - s.emit_struct_field("lines", 5, |s| { - let lines = &self.lines[..]; - // Store the length. - s.emit_u32(lines.len() as u32)?; - - if !lines.is_empty() { - // In order to preserve some space, we exploit the fact that - // the lines list is sorted and individual lines are - // probably not that long. Because of that we can store lines - // as a difference list, using as little space as possible - // for the differences. - let max_line_length = if lines.len() == 1 { - 0 - } else { - lines.windows(2).map(|w| w[1] - w[0]).map(|bp| bp.to_usize()).max().unwrap() - }; - - let bytes_per_diff: u8 = match max_line_length { - 0..=0xFF => 1, - 0x100..=0xFFFF => 2, - _ => 4, - }; - - // Encode the number of bytes used per diff. - bytes_per_diff.encode(s)?; - - // Encode the first element. - lines[0].encode(s)?; - - let diff_iter = (&lines[..]).windows(2).map(|w| (w[1] - w[0])); - - match bytes_per_diff { - 1 => { - for diff in diff_iter { - (diff.0 as u8).encode(s)? - } - } - 2 => { - for diff in diff_iter { - (diff.0 as u16).encode(s)? - } - } - 4 => { - for diff in diff_iter { - diff.0.encode(s)? - } - } - _ => unreachable!(), - } - } - - Ok(()) - })?; - s.emit_struct_field("multibyte_chars", 6, |s| self.multibyte_chars.encode(s))?; - s.emit_struct_field("non_narrow_chars", 7, |s| self.non_narrow_chars.encode(s))?; - s.emit_struct_field("name_hash", 8, |s| self.name_hash.encode(s))?; - s.emit_struct_field("normalized_pos", 9, |s| self.normalized_pos.encode(s)) - }) - } -} - -impl Decodable for SourceFile { - fn decode(d: &mut D) -> Result { - d.read_struct("SourceFile", 8, |d| { - let name: FileName = d.read_struct_field("name", 0, |d| Decodable::decode(d))?; - let name_was_remapped: bool = - d.read_struct_field("name_was_remapped", 1, |d| Decodable::decode(d))?; - let src_hash: u128 = d.read_struct_field("src_hash", 2, |d| Decodable::decode(d))?; - let start_pos: BytePos = - d.read_struct_field("start_pos", 3, |d| Decodable::decode(d))?; - let end_pos: BytePos = d.read_struct_field("end_pos", 4, |d| Decodable::decode(d))?; - let lines: Vec = d.read_struct_field("lines", 5, |d| { - let num_lines: u32 = Decodable::decode(d)?; - let mut lines = Vec::with_capacity(num_lines as usize); - - if num_lines > 0 { - // Read the number of bytes used per diff. - let bytes_per_diff: u8 = Decodable::decode(d)?; - - // Read the first element. - let mut line_start: BytePos = Decodable::decode(d)?; - lines.push(line_start); - - for _ in 1..num_lines { - let diff = match bytes_per_diff { - 1 => d.read_u8()? as u32, - 2 => d.read_u16()? as u32, - 4 => d.read_u32()?, - _ => unreachable!(), - }; - - line_start = line_start + BytePos(diff); - - lines.push(line_start); - } - } - - Ok(lines) - })?; - let multibyte_chars: Vec = - d.read_struct_field("multibyte_chars", 6, |d| Decodable::decode(d))?; - let non_narrow_chars: Vec = - d.read_struct_field("non_narrow_chars", 7, |d| Decodable::decode(d))?; - let name_hash: u128 = d.read_struct_field("name_hash", 8, |d| Decodable::decode(d))?; - let normalized_pos: Vec = - d.read_struct_field("normalized_pos", 9, |d| Decodable::decode(d))?; - Ok(SourceFile { - name, - name_was_remapped, - unmapped_path: None, - // `crate_of_origin` has to be set by the importer. - // This value matches up with `rustc::hir::def_id::INVALID_CRATE`. - // That constant is not available here, unfortunately. - crate_of_origin: std::u32::MAX - 1, - start_pos, - end_pos, - src: None, - src_hash, - external_src: Lock::new(ExternalSource::AbsentOk), - lines, - multibyte_chars, - non_narrow_chars, - normalized_pos, - name_hash, - }) - }) - } -} - -impl fmt::Debug for SourceFile { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(fmt, "SourceFile({})", self.name) - } -} - -impl SourceFile { - pub fn new( - name: FileName, - name_was_remapped: bool, - unmapped_path: FileName, - mut src: String, - start_pos: BytePos, - ) -> Result { - let normalized_pos = normalize_src(&mut src, start_pos); - - let src_hash = { - let mut hasher: StableHasher = StableHasher::new(); - hasher.write(src.as_bytes()); - hasher.finish::() - }; - let name_hash = { - let mut hasher: StableHasher = StableHasher::new(); - name.hash(&mut hasher); - hasher.finish::() - }; - let end_pos = start_pos.to_usize() + src.len(); - if end_pos > u32::max_value() as usize { - return Err(OffsetOverflowError); - } - - let (lines, multibyte_chars, non_narrow_chars) = - analyze_source_file::analyze_source_file(&src[..], start_pos); - - Ok(SourceFile { - name, - name_was_remapped, - unmapped_path: Some(unmapped_path), - crate_of_origin: 0, - src: Some(Lrc::new(src)), - src_hash, - external_src: Lock::new(ExternalSource::Unneeded), - start_pos, - end_pos: Pos::from_usize(end_pos), - lines, - multibyte_chars, - non_narrow_chars, - normalized_pos, - name_hash, - }) - } - - /// Returns the `BytePos` of the beginning of the current line. - pub fn line_begin_pos(&self, pos: BytePos) -> BytePos { - let line_index = self.lookup_line(pos).unwrap(); - self.lines[line_index] - } - - /// Add externally loaded source. - /// If the hash of the input doesn't match or no input is supplied via None, - /// it is interpreted as an error and the corresponding enum variant is set. - /// The return value signifies whether some kind of source is present. - pub fn add_external_src(&self, get_src: F) -> bool - where - F: FnOnce() -> Option, - { - if *self.external_src.borrow() == ExternalSource::AbsentOk { - let src = get_src(); - let mut external_src = self.external_src.borrow_mut(); - // Check that no-one else have provided the source while we were getting it - if *external_src == ExternalSource::AbsentOk { - if let Some(src) = src { - let mut hasher: StableHasher = StableHasher::new(); - hasher.write(src.as_bytes()); - - if hasher.finish::() == self.src_hash { - *external_src = ExternalSource::Present(src); - return true; - } - } else { - *external_src = ExternalSource::AbsentErr; - } - - false - } else { - self.src.is_some() || external_src.get_source().is_some() - } - } else { - self.src.is_some() || self.external_src.borrow().get_source().is_some() - } - } - - /// Gets a line from the list of pre-computed line-beginnings. - /// The line number here is 0-based. - pub fn get_line(&self, line_number: usize) -> Option> { - fn get_until_newline(src: &str, begin: usize) -> &str { - // We can't use `lines.get(line_number+1)` because we might - // be parsing when we call this function and thus the current - // line is the last one we have line info for. - let slice = &src[begin..]; - match slice.find('\n') { - Some(e) => &slice[..e], - None => slice, - } - } - - let begin = { - let line = if let Some(line) = self.lines.get(line_number) { - line - } else { - return None; - }; - let begin: BytePos = *line - self.start_pos; - begin.to_usize() - }; - - if let Some(ref src) = self.src { - Some(Cow::from(get_until_newline(src, begin))) - } else if let Some(src) = self.external_src.borrow().get_source() { - Some(Cow::Owned(String::from(get_until_newline(src, begin)))) - } else { - None - } - } - - pub fn is_real_file(&self) -> bool { - self.name.is_real() - } - - pub fn is_imported(&self) -> bool { - self.src.is_none() - } - - pub fn byte_length(&self) -> u32 { - self.end_pos.0 - self.start_pos.0 - } - pub fn count_lines(&self) -> usize { - self.lines.len() - } - - /// Finds the line containing the given position. The return value is the - /// index into the `lines` array of this `SourceFile`, not the 1-based line - /// number. If the source_file is empty or the position is located before the - /// first line, `None` is returned. - pub fn lookup_line(&self, pos: BytePos) -> Option { - if self.lines.len() == 0 { - return None; - } - - let line_index = lookup_line(&self.lines[..], pos); - assert!(line_index < self.lines.len() as isize); - if line_index >= 0 { Some(line_index as usize) } else { None } - } - - pub fn line_bounds(&self, line_index: usize) -> (BytePos, BytePos) { - if self.start_pos == self.end_pos { - return (self.start_pos, self.end_pos); - } - - assert!(line_index < self.lines.len()); - if line_index == (self.lines.len() - 1) { - (self.lines[line_index], self.end_pos) - } else { - (self.lines[line_index], self.lines[line_index + 1]) - } - } - - #[inline] - pub fn contains(&self, byte_pos: BytePos) -> bool { - byte_pos >= self.start_pos && byte_pos <= self.end_pos - } - - /// Calculates the original byte position relative to the start of the file - /// based on the given byte position. - pub fn original_relative_byte_pos(&self, pos: BytePos) -> BytePos { - // Diff before any records is 0. Otherwise use the previously recorded - // diff as that applies to the following characters until a new diff - // is recorded. - let diff = match self.normalized_pos.binary_search_by(|np| np.pos.cmp(&pos)) { - Ok(i) => self.normalized_pos[i].diff, - Err(i) if i == 0 => 0, - Err(i) => self.normalized_pos[i - 1].diff, - }; - - BytePos::from_u32(pos.0 - self.start_pos.0 + diff) - } -} - -/// Normalizes the source code and records the normalizations. -fn normalize_src(src: &mut String, start_pos: BytePos) -> Vec { - let mut normalized_pos = vec![]; - remove_bom(src, &mut normalized_pos); - normalize_newlines(src, &mut normalized_pos); - - // Offset all the positions by start_pos to match the final file positions. - for np in &mut normalized_pos { - np.pos.0 += start_pos.0; - } - - normalized_pos -} - -/// Removes UTF-8 BOM, if any. -fn remove_bom(src: &mut String, normalized_pos: &mut Vec) { - if src.starts_with("\u{feff}") { - src.drain(..3); - normalized_pos.push(NormalizedPos { pos: BytePos(0), diff: 3 }); - } -} - -/// Replaces `\r\n` with `\n` in-place in `src`. -/// -/// Returns error if there's a lone `\r` in the string -fn normalize_newlines(src: &mut String, normalized_pos: &mut Vec) { - if !src.as_bytes().contains(&b'\r') { - return; - } - - // We replace `\r\n` with `\n` in-place, which doesn't break utf-8 encoding. - // While we *can* call `as_mut_vec` and do surgery on the live string - // directly, let's rather steal the contents of `src`. This makes the code - // safe even if a panic occurs. - - let mut buf = std::mem::replace(src, String::new()).into_bytes(); - let mut gap_len = 0; - let mut tail = buf.as_mut_slice(); - let mut cursor = 0; - let original_gap = normalized_pos.last().map_or(0, |l| l.diff); - loop { - let idx = match find_crlf(&tail[gap_len..]) { - None => tail.len(), - Some(idx) => idx + gap_len, - }; - tail.copy_within(gap_len..idx, 0); - tail = &mut tail[idx - gap_len..]; - if tail.len() == gap_len { - break; - } - cursor += idx - gap_len; - gap_len += 1; - normalized_pos.push(NormalizedPos { - pos: BytePos::from_usize(cursor + 1), - diff: original_gap + gap_len as u32, - }); - } - - // Account for removed `\r`. - // After `set_len`, `buf` is guaranteed to contain utf-8 again. - let new_len = buf.len() - gap_len; - unsafe { - buf.set_len(new_len); - *src = String::from_utf8_unchecked(buf); - } - - fn find_crlf(src: &[u8]) -> Option { - let mut search_idx = 0; - while let Some(idx) = find_cr(&src[search_idx..]) { - if src[search_idx..].get(idx + 1) != Some(&b'\n') { - search_idx += idx + 1; - continue; - } - return Some(search_idx + idx); - } - None - } - - fn find_cr(src: &[u8]) -> Option { - src.iter().position(|&b| b == b'\r') - } -} - -// _____________________________________________________________________________ -// Pos, BytePos, CharPos -// - -pub trait Pos { - fn from_usize(n: usize) -> Self; - fn to_usize(&self) -> usize; - fn from_u32(n: u32) -> Self; - fn to_u32(&self) -> u32; -} - -/// A byte offset. Keep this small (currently 32-bits), as AST contains -/// a lot of them. -#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] -pub struct BytePos(pub u32); - -/// A character offset. Because of multibyte UTF-8 characters, a byte offset -/// is not equivalent to a character offset. The `SourceMap` will convert `BytePos` -/// values to `CharPos` values as necessary. -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] -pub struct CharPos(pub usize); - -// FIXME: lots of boilerplate in these impls, but so far my attempts to fix -// have been unsuccessful. - -impl Pos for BytePos { - #[inline(always)] - fn from_usize(n: usize) -> BytePos { - BytePos(n as u32) - } - - #[inline(always)] - fn to_usize(&self) -> usize { - self.0 as usize - } - - #[inline(always)] - fn from_u32(n: u32) -> BytePos { - BytePos(n) - } - - #[inline(always)] - fn to_u32(&self) -> u32 { - self.0 - } -} - -impl Add for BytePos { - type Output = BytePos; - - #[inline(always)] - fn add(self, rhs: BytePos) -> BytePos { - BytePos((self.to_usize() + rhs.to_usize()) as u32) - } -} - -impl Sub for BytePos { - type Output = BytePos; - - #[inline(always)] - fn sub(self, rhs: BytePos) -> BytePos { - BytePos((self.to_usize() - rhs.to_usize()) as u32) - } -} - -impl Encodable for BytePos { - fn encode(&self, s: &mut S) -> Result<(), S::Error> { - s.emit_u32(self.0) - } -} - -impl Decodable for BytePos { - fn decode(d: &mut D) -> Result { - Ok(BytePos(d.read_u32()?)) - } -} - -impl Pos for CharPos { - #[inline(always)] - fn from_usize(n: usize) -> CharPos { - CharPos(n) - } - - #[inline(always)] - fn to_usize(&self) -> usize { - self.0 - } - - #[inline(always)] - fn from_u32(n: u32) -> CharPos { - CharPos(n as usize) - } - - #[inline(always)] - fn to_u32(&self) -> u32 { - self.0 as u32 - } -} - -impl Add for CharPos { - type Output = CharPos; - - #[inline(always)] - fn add(self, rhs: CharPos) -> CharPos { - CharPos(self.to_usize() + rhs.to_usize()) - } -} - -impl Sub for CharPos { - type Output = CharPos; - - #[inline(always)] - fn sub(self, rhs: CharPos) -> CharPos { - CharPos(self.to_usize() - rhs.to_usize()) - } -} - -// _____________________________________________________________________________ -// Loc, SourceFileAndLine, SourceFileAndBytePos -// - -/// A source code location used for error reporting. -#[derive(Debug, Clone)] -pub struct Loc { - /// Information about the original source. - pub file: Lrc, - /// The (1-based) line number. - pub line: usize, - /// The (0-based) column offset. - pub col: CharPos, - /// The (0-based) column offset when displayed. - pub col_display: usize, -} - -// Used to be structural records. -#[derive(Debug)] -pub struct SourceFileAndLine { - pub sf: Lrc, - pub line: usize, -} -#[derive(Debug)] -pub struct SourceFileAndBytePos { - pub sf: Lrc, - pub pos: BytePos, -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub struct LineInfo { - /// Index of line, starting from 0. - pub line_index: usize, - - /// Column in line where span begins, starting from 0. - pub start_col: CharPos, - - /// Column in line where span ends, starting from 0, exclusive. - pub end_col: CharPos, -} - -pub struct FileLines { - pub file: Lrc, - pub lines: Vec, -} - -pub static SPAN_DEBUG: AtomicRef) -> fmt::Result> = - AtomicRef::new(&(default_span_debug as fn(_, &mut fmt::Formatter<'_>) -> _)); - -#[derive(Debug)] -pub struct MacroBacktrace { - /// span where macro was applied to generate this code - pub call_site: Span, - - /// name of macro that was applied (e.g., "foo!" or "#[derive(Eq)]") - pub macro_decl_name: String, - - /// span where macro was defined (possibly dummy) - pub def_site_span: Span, -} - -// _____________________________________________________________________________ -// SpanLinesError, SpanSnippetError, DistinctSources, MalformedSourceMapPositions -// - -pub type FileLinesResult = Result; - -#[derive(Clone, PartialEq, Eq, Debug)] -pub enum SpanLinesError { - DistinctSources(DistinctSources), -} - -#[derive(Clone, PartialEq, Eq, Debug)] -pub enum SpanSnippetError { - IllFormedSpan(Span), - DistinctSources(DistinctSources), - MalformedForSourcemap(MalformedSourceMapPositions), - SourceNotAvailable { filename: FileName }, -} - -#[derive(Clone, PartialEq, Eq, Debug)] -pub struct DistinctSources { - pub begin: (FileName, BytePos), - pub end: (FileName, BytePos), -} - -#[derive(Clone, PartialEq, Eq, Debug)] -pub struct MalformedSourceMapPositions { - pub name: FileName, - pub source_len: usize, - pub begin_pos: BytePos, - pub end_pos: BytePos, -} - -/// Range inside of a `Span` used for diagnostics when we only have access to relative positions. -#[derive(Copy, Clone, PartialEq, Eq, Debug)] -pub struct InnerSpan { - pub start: usize, - pub end: usize, -} - -impl InnerSpan { - pub fn new(start: usize, end: usize) -> InnerSpan { - InnerSpan { start, end } - } -} - -// Given a slice of line start positions and a position, returns the index of -// the line the position is on. Returns -1 if the position is located before -// the first line. -fn lookup_line(lines: &[BytePos], pos: BytePos) -> isize { - match lines.binary_search(&pos) { - Ok(line) => line as isize, - Err(line) => line as isize - 1, - } -} - -/// Requirements for a `StableHashingContext` to be used in this crate. -/// This is a hack to allow using the `HashStable_Generic` derive macro -/// instead of implementing everything in librustc. -pub trait HashStableContext { - fn hash_spans(&self) -> bool; - fn byte_pos_to_line_and_col( - &mut self, - byte: BytePos, - ) -> Option<(Lrc, usize, BytePos)>; -} - -impl HashStable for Span -where - CTX: HashStableContext, -{ - /// Hashes a span in a stable way. We can't directly hash the span's `BytePos` - /// fields (that would be similar to hashing pointers, since those are just - /// offsets into the `SourceMap`). Instead, we hash the (file name, line, column) - /// triple, which stays the same even if the containing `SourceFile` has moved - /// within the `SourceMap`. - /// Also note that we are hashing byte offsets for the column, not unicode - /// codepoint offsets. For the purpose of the hash that's sufficient. - /// Also, hashing filenames is expensive so we avoid doing it twice when the - /// span starts and ends in the same file, which is almost always the case. - fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { - const TAG_VALID_SPAN: u8 = 0; - const TAG_INVALID_SPAN: u8 = 1; - const TAG_EXPANSION: u8 = 0; - const TAG_NO_EXPANSION: u8 = 1; - - if !ctx.hash_spans() { - return; - } - - if *self == DUMMY_SP { - return std::hash::Hash::hash(&TAG_INVALID_SPAN, hasher); - } - - // If this is not an empty or invalid span, we want to hash the last - // position that belongs to it, as opposed to hashing the first - // position past it. - let span = self.data(); - let (file_lo, line_lo, col_lo) = match ctx.byte_pos_to_line_and_col(span.lo) { - Some(pos) => pos, - None => { - return std::hash::Hash::hash(&TAG_INVALID_SPAN, hasher); - } - }; - - if !file_lo.contains(span.hi) { - return std::hash::Hash::hash(&TAG_INVALID_SPAN, hasher); - } - - std::hash::Hash::hash(&TAG_VALID_SPAN, hasher); - // We truncate the stable ID hash and line and column numbers. The chances - // of causing a collision this way should be minimal. - std::hash::Hash::hash(&(file_lo.name_hash as u64), hasher); - - let col = (col_lo.0 as u64) & 0xFF; - let line = ((line_lo as u64) & 0xFF_FF_FF) << 8; - let len = ((span.hi - span.lo).0 as u64) << 32; - let line_col_len = col | line | len; - std::hash::Hash::hash(&line_col_len, hasher); - - if span.ctxt == SyntaxContext::root() { - TAG_NO_EXPANSION.hash_stable(ctx, hasher); - } else { - TAG_EXPANSION.hash_stable(ctx, hasher); - - // Since the same expansion context is usually referenced many - // times, we cache a stable hash of it and hash that instead of - // recursing every time. - thread_local! { - static CACHE: RefCell> = Default::default(); - } - - let sub_hash: u64 = CACHE.with(|cache| { - let expn_id = span.ctxt.outer_expn(); - - if let Some(&sub_hash) = cache.borrow().get(&expn_id) { - return sub_hash; - } - - let mut hasher = StableHasher::new(); - expn_id.expn_data().hash_stable(ctx, &mut hasher); - let sub_hash: Fingerprint = hasher.finish(); - let sub_hash = sub_hash.to_smaller_hash(); - cache.borrow_mut().insert(expn_id, sub_hash); - sub_hash - }); - - sub_hash.hash_stable(ctx, hasher); - } - } -} diff --git a/src/libsyntax_pos/source_map.rs b/src/libsyntax_pos/source_map.rs deleted file mode 100644 index 0b9b9fe7887..00000000000 --- a/src/libsyntax_pos/source_map.rs +++ /dev/null @@ -1,984 +0,0 @@ -//! The `SourceMap` tracks all the source code used within a single crate, mapping -//! from integer byte positions to the original source code location. Each bit -//! of source parsed during crate parsing (typically files, in-memory strings, -//! or various bits of macro expansion) cover a continuous range of bytes in the -//! `SourceMap` and are represented by `SourceFile`s. Byte positions are stored in -//! `Span` and used pervasively in the compiler. They are absolute positions -//! within the `SourceMap`, which upon request can be converted to line and column -//! information, source code snippets, etc. - -pub use crate::hygiene::{ExpnData, ExpnKind}; -pub use crate::*; - -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::stable_hasher::StableHasher; -use rustc_data_structures::sync::{Lock, LockGuard, Lrc, MappedLockGuard}; -use std::cmp; -use std::hash::Hash; -use std::path::{Path, PathBuf}; - -use log::debug; -use std::env; -use std::fs; -use std::io; - -#[cfg(test)] -mod tests; - -/// Returns the span itself if it doesn't come from a macro expansion, -/// otherwise return the call site span up to the `enclosing_sp` by -/// following the `expn_data` chain. -pub fn original_sp(sp: Span, enclosing_sp: Span) -> Span { - let expn_data1 = sp.ctxt().outer_expn_data(); - let expn_data2 = enclosing_sp.ctxt().outer_expn_data(); - if expn_data1.is_root() || !expn_data2.is_root() && expn_data1.call_site == expn_data2.call_site - { - sp - } else { - original_sp(expn_data1.call_site, enclosing_sp) - } -} - -#[derive(Clone, RustcEncodable, RustcDecodable, Debug, Copy, HashStable_Generic)] -pub struct Spanned { - pub node: T, - pub span: Span, -} - -pub fn respan(sp: Span, t: T) -> Spanned { - Spanned { node: t, span: sp } -} - -pub fn dummy_spanned(t: T) -> Spanned { - respan(DUMMY_SP, t) -} - -// _____________________________________________________________________________ -// SourceFile, MultiByteChar, FileName, FileLines -// - -/// An abstraction over the fs operations used by the Parser. -pub trait FileLoader { - /// Query the existence of a file. - fn file_exists(&self, path: &Path) -> bool; - - /// Returns an absolute path to a file, if possible. - fn abs_path(&self, path: &Path) -> Option; - - /// Read the contents of an UTF-8 file into memory. - fn read_file(&self, path: &Path) -> io::Result; -} - -/// A FileLoader that uses std::fs to load real files. -pub struct RealFileLoader; - -impl FileLoader for RealFileLoader { - fn file_exists(&self, path: &Path) -> bool { - fs::metadata(path).is_ok() - } - - fn abs_path(&self, path: &Path) -> Option { - if path.is_absolute() { - Some(path.to_path_buf()) - } else { - env::current_dir().ok().map(|cwd| cwd.join(path)) - } - } - - fn read_file(&self, path: &Path) -> io::Result { - fs::read_to_string(path) - } -} - -// This is a `SourceFile` identifier that is used to correlate `SourceFile`s between -// subsequent compilation sessions (which is something we need to do during -// incremental compilation). -#[derive(Copy, Clone, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable, Debug)] -pub struct StableSourceFileId(u128); - -impl StableSourceFileId { - pub fn new(source_file: &SourceFile) -> StableSourceFileId { - StableSourceFileId::new_from_pieces( - &source_file.name, - source_file.name_was_remapped, - source_file.unmapped_path.as_ref(), - ) - } - - pub fn new_from_pieces( - name: &FileName, - name_was_remapped: bool, - unmapped_path: Option<&FileName>, - ) -> StableSourceFileId { - let mut hasher = StableHasher::new(); - - name.hash(&mut hasher); - name_was_remapped.hash(&mut hasher); - unmapped_path.hash(&mut hasher); - - StableSourceFileId(hasher.finish()) - } -} - -// _____________________________________________________________________________ -// SourceMap -// - -#[derive(Default)] -pub(super) struct SourceMapFiles { - source_files: Vec>, - stable_id_to_source_file: FxHashMap>, -} - -pub struct SourceMap { - files: Lock, - file_loader: Box, - // This is used to apply the file path remapping as specified via - // `--remap-path-prefix` to all `SourceFile`s allocated within this `SourceMap`. - path_mapping: FilePathMapping, -} - -impl SourceMap { - pub fn new(path_mapping: FilePathMapping) -> SourceMap { - SourceMap { files: Default::default(), file_loader: Box::new(RealFileLoader), path_mapping } - } - - pub fn with_file_loader( - file_loader: Box, - path_mapping: FilePathMapping, - ) -> SourceMap { - SourceMap { files: Default::default(), file_loader, path_mapping } - } - - pub fn path_mapping(&self) -> &FilePathMapping { - &self.path_mapping - } - - pub fn file_exists(&self, path: &Path) -> bool { - self.file_loader.file_exists(path) - } - - pub fn load_file(&self, path: &Path) -> io::Result> { - let src = self.file_loader.read_file(path)?; - let filename = path.to_owned().into(); - Ok(self.new_source_file(filename, src)) - } - - /// Loads source file as a binary blob. - /// - /// Unlike `load_file`, guarantees that no normalization like BOM-removal - /// takes place. - pub fn load_binary_file(&self, path: &Path) -> io::Result> { - // Ideally, this should use `self.file_loader`, but it can't - // deal with binary files yet. - let bytes = fs::read(path)?; - - // We need to add file to the `SourceMap`, so that it is present - // in dep-info. There's also an edge case that file might be both - // loaded as a binary via `include_bytes!` and as proper `SourceFile` - // via `mod`, so we try to use real file contents and not just an - // empty string. - let text = std::str::from_utf8(&bytes).unwrap_or("").to_string(); - self.new_source_file(path.to_owned().into(), text); - Ok(bytes) - } - - pub fn files(&self) -> MappedLockGuard<'_, Vec>> { - LockGuard::map(self.files.borrow(), |files| &mut files.source_files) - } - - pub fn source_file_by_stable_id( - &self, - stable_id: StableSourceFileId, - ) -> Option> { - self.files.borrow().stable_id_to_source_file.get(&stable_id).map(|sf| sf.clone()) - } - - fn next_start_pos(&self) -> usize { - match self.files.borrow().source_files.last() { - None => 0, - // Add one so there is some space between files. This lets us distinguish - // positions in the `SourceMap`, even in the presence of zero-length files. - Some(last) => last.end_pos.to_usize() + 1, - } - } - - /// Creates a new `SourceFile`. - /// If a file already exists in the `SourceMap` with the same ID, that file is returned - /// unmodified. - pub fn new_source_file(&self, filename: FileName, src: String) -> Lrc { - self.try_new_source_file(filename, src).unwrap_or_else(|OffsetOverflowError| { - eprintln!("fatal error: rustc does not support files larger than 4GB"); - crate::fatal_error::FatalError.raise() - }) - } - - fn try_new_source_file( - &self, - filename: FileName, - src: String, - ) -> Result, OffsetOverflowError> { - let start_pos = self.next_start_pos(); - - // The path is used to determine the directory for loading submodules and - // include files, so it must be before remapping. - // Note that filename may not be a valid path, eg it may be `` etc, - // but this is okay because the directory determined by `path.pop()` will - // be empty, so the working directory will be used. - let unmapped_path = filename.clone(); - - let (filename, was_remapped) = match filename { - FileName::Real(filename) => { - let (filename, was_remapped) = self.path_mapping.map_prefix(filename); - (FileName::Real(filename), was_remapped) - } - other => (other, false), - }; - - let file_id = - StableSourceFileId::new_from_pieces(&filename, was_remapped, Some(&unmapped_path)); - - let lrc_sf = match self.source_file_by_stable_id(file_id) { - Some(lrc_sf) => lrc_sf, - None => { - let source_file = Lrc::new(SourceFile::new( - filename, - was_remapped, - unmapped_path, - src, - Pos::from_usize(start_pos), - )?); - - let mut files = self.files.borrow_mut(); - - files.source_files.push(source_file.clone()); - files.stable_id_to_source_file.insert(file_id, source_file.clone()); - - source_file - } - }; - Ok(lrc_sf) - } - - /// Allocates a new `SourceFile` representing a source file from an external - /// crate. The source code of such an "imported `SourceFile`" is not available, - /// but we still know enough to generate accurate debuginfo location - /// information for things inlined from other crates. - pub fn new_imported_source_file( - &self, - filename: FileName, - name_was_remapped: bool, - crate_of_origin: u32, - src_hash: u128, - name_hash: u128, - source_len: usize, - mut file_local_lines: Vec, - mut file_local_multibyte_chars: Vec, - mut file_local_non_narrow_chars: Vec, - mut file_local_normalized_pos: Vec, - ) -> Lrc { - let start_pos = self.next_start_pos(); - - let end_pos = Pos::from_usize(start_pos + source_len); - let start_pos = Pos::from_usize(start_pos); - - for pos in &mut file_local_lines { - *pos = *pos + start_pos; - } - - for mbc in &mut file_local_multibyte_chars { - mbc.pos = mbc.pos + start_pos; - } - - for swc in &mut file_local_non_narrow_chars { - *swc = *swc + start_pos; - } - - for nc in &mut file_local_normalized_pos { - nc.pos = nc.pos + start_pos; - } - - let source_file = Lrc::new(SourceFile { - name: filename, - name_was_remapped, - unmapped_path: None, - crate_of_origin, - src: None, - src_hash, - external_src: Lock::new(ExternalSource::AbsentOk), - start_pos, - end_pos, - lines: file_local_lines, - multibyte_chars: file_local_multibyte_chars, - non_narrow_chars: file_local_non_narrow_chars, - normalized_pos: file_local_normalized_pos, - name_hash, - }); - - let mut files = self.files.borrow_mut(); - - files.source_files.push(source_file.clone()); - files - .stable_id_to_source_file - .insert(StableSourceFileId::new(&source_file), source_file.clone()); - - source_file - } - - pub fn mk_substr_filename(&self, sp: Span) -> String { - let pos = self.lookup_char_pos(sp.lo()); - format!("<{}:{}:{}>", pos.file.name, pos.line, pos.col.to_usize() + 1) - } - - // If there is a doctest offset, applies it to the line. - pub fn doctest_offset_line(&self, file: &FileName, orig: usize) -> usize { - return match file { - FileName::DocTest(_, offset) => { - return if *offset >= 0 { - orig + *offset as usize - } else { - orig - (-(*offset)) as usize - }; - } - _ => orig, - }; - } - - /// Looks up source information about a `BytePos`. - pub fn lookup_char_pos(&self, pos: BytePos) -> Loc { - let chpos = self.bytepos_to_file_charpos(pos); - match self.lookup_line(pos) { - Ok(SourceFileAndLine { sf: f, line: a }) => { - let line = a + 1; // Line numbers start at 1 - let linebpos = f.lines[a]; - let linechpos = self.bytepos_to_file_charpos(linebpos); - let col = chpos - linechpos; - - let col_display = { - let start_width_idx = f - .non_narrow_chars - .binary_search_by_key(&linebpos, |x| x.pos()) - .unwrap_or_else(|x| x); - let end_width_idx = f - .non_narrow_chars - .binary_search_by_key(&pos, |x| x.pos()) - .unwrap_or_else(|x| x); - let special_chars = end_width_idx - start_width_idx; - let non_narrow: usize = f.non_narrow_chars[start_width_idx..end_width_idx] - .into_iter() - .map(|x| x.width()) - .sum(); - col.0 - special_chars + non_narrow - }; - debug!("byte pos {:?} is on the line at byte pos {:?}", pos, linebpos); - debug!("char pos {:?} is on the line at char pos {:?}", chpos, linechpos); - debug!("byte is on line: {}", line); - assert!(chpos >= linechpos); - Loc { file: f, line, col, col_display } - } - Err(f) => { - let col_display = { - let end_width_idx = f - .non_narrow_chars - .binary_search_by_key(&pos, |x| x.pos()) - .unwrap_or_else(|x| x); - let non_narrow: usize = - f.non_narrow_chars[0..end_width_idx].into_iter().map(|x| x.width()).sum(); - chpos.0 - end_width_idx + non_narrow - }; - Loc { file: f, line: 0, col: chpos, col_display } - } - } - } - - // If the corresponding `SourceFile` is empty, does not return a line number. - pub fn lookup_line(&self, pos: BytePos) -> Result> { - let idx = self.lookup_source_file_idx(pos); - - let f = (*self.files.borrow().source_files)[idx].clone(); - - match f.lookup_line(pos) { - Some(line) => Ok(SourceFileAndLine { sf: f, line }), - None => Err(f), - } - } - - /// Returns `Some(span)`, a union of the LHS and RHS span. The LHS must precede the RHS. If - /// there are gaps between LHS and RHS, the resulting union will cross these gaps. - /// For this to work, - /// - /// * the syntax contexts of both spans much match, - /// * the LHS span needs to end on the same line the RHS span begins, - /// * the LHS span must start at or before the RHS span. - pub fn merge_spans(&self, sp_lhs: Span, sp_rhs: Span) -> Option { - // Ensure we're at the same expansion ID. - if sp_lhs.ctxt() != sp_rhs.ctxt() { - return None; - } - - let lhs_end = match self.lookup_line(sp_lhs.hi()) { - Ok(x) => x, - Err(_) => return None, - }; - let rhs_begin = match self.lookup_line(sp_rhs.lo()) { - Ok(x) => x, - Err(_) => return None, - }; - - // If we must cross lines to merge, don't merge. - if lhs_end.line != rhs_begin.line { - return None; - } - - // Ensure these follow the expected order and that we don't overlap. - if (sp_lhs.lo() <= sp_rhs.lo()) && (sp_lhs.hi() <= sp_rhs.lo()) { - Some(sp_lhs.to(sp_rhs)) - } else { - None - } - } - - pub fn span_to_string(&self, sp: Span) -> String { - if self.files.borrow().source_files.is_empty() && sp.is_dummy() { - return "no-location".to_string(); - } - - let lo = self.lookup_char_pos(sp.lo()); - let hi = self.lookup_char_pos(sp.hi()); - format!( - "{}:{}:{}: {}:{}", - lo.file.name, - lo.line, - lo.col.to_usize() + 1, - hi.line, - hi.col.to_usize() + 1, - ) - } - - pub fn span_to_filename(&self, sp: Span) -> FileName { - self.lookup_char_pos(sp.lo()).file.name.clone() - } - - pub fn span_to_unmapped_path(&self, sp: Span) -> FileName { - self.lookup_char_pos(sp.lo()) - .file - .unmapped_path - .clone() - .expect("`SourceMap::span_to_unmapped_path` called for imported `SourceFile`?") - } - - pub fn is_multiline(&self, sp: Span) -> bool { - let lo = self.lookup_char_pos(sp.lo()); - let hi = self.lookup_char_pos(sp.hi()); - lo.line != hi.line - } - - pub fn span_to_lines(&self, sp: Span) -> FileLinesResult { - debug!("span_to_lines(sp={:?})", sp); - - let lo = self.lookup_char_pos(sp.lo()); - debug!("span_to_lines: lo={:?}", lo); - let hi = self.lookup_char_pos(sp.hi()); - debug!("span_to_lines: hi={:?}", hi); - - if lo.file.start_pos != hi.file.start_pos { - return Err(SpanLinesError::DistinctSources(DistinctSources { - begin: (lo.file.name.clone(), lo.file.start_pos), - end: (hi.file.name.clone(), hi.file.start_pos), - })); - } - assert!(hi.line >= lo.line); - - let mut lines = Vec::with_capacity(hi.line - lo.line + 1); - - // The span starts partway through the first line, - // but after that it starts from offset 0. - let mut start_col = lo.col; - - // For every line but the last, it extends from `start_col` - // and to the end of the line. Be careful because the line - // numbers in Loc are 1-based, so we subtract 1 to get 0-based - // lines. - for line_index in lo.line - 1..hi.line - 1 { - let line_len = lo.file.get_line(line_index).map(|s| s.chars().count()).unwrap_or(0); - lines.push(LineInfo { line_index, start_col, end_col: CharPos::from_usize(line_len) }); - start_col = CharPos::from_usize(0); - } - - // For the last line, it extends from `start_col` to `hi.col`: - lines.push(LineInfo { line_index: hi.line - 1, start_col, end_col: hi.col }); - - Ok(FileLines { file: lo.file, lines }) - } - - /// Extracts the source surrounding the given `Span` using the `extract_source` function. The - /// extract function takes three arguments: a string slice containing the source, an index in - /// the slice for the beginning of the span and an index in the slice for the end of the span. - fn span_to_source(&self, sp: Span, extract_source: F) -> Result - where - F: Fn(&str, usize, usize) -> Result, - { - let local_begin = self.lookup_byte_offset(sp.lo()); - let local_end = self.lookup_byte_offset(sp.hi()); - - if local_begin.sf.start_pos != local_end.sf.start_pos { - return Err(SpanSnippetError::DistinctSources(DistinctSources { - begin: (local_begin.sf.name.clone(), local_begin.sf.start_pos), - end: (local_end.sf.name.clone(), local_end.sf.start_pos), - })); - } else { - self.ensure_source_file_source_present(local_begin.sf.clone()); - - let start_index = local_begin.pos.to_usize(); - let end_index = local_end.pos.to_usize(); - let source_len = (local_begin.sf.end_pos - local_begin.sf.start_pos).to_usize(); - - if start_index > end_index || end_index > source_len { - return Err(SpanSnippetError::MalformedForSourcemap(MalformedSourceMapPositions { - name: local_begin.sf.name.clone(), - source_len, - begin_pos: local_begin.pos, - end_pos: local_end.pos, - })); - } - - if let Some(ref src) = local_begin.sf.src { - return extract_source(src, start_index, end_index); - } else if let Some(src) = local_begin.sf.external_src.borrow().get_source() { - return extract_source(src, start_index, end_index); - } else { - return Err(SpanSnippetError::SourceNotAvailable { - filename: local_begin.sf.name.clone(), - }); - } - } - } - - /// Returns the source snippet as `String` corresponding to the given `Span`. - pub fn span_to_snippet(&self, sp: Span) -> Result { - self.span_to_source(sp, |src, start_index, end_index| { - src.get(start_index..end_index) - .map(|s| s.to_string()) - .ok_or_else(|| SpanSnippetError::IllFormedSpan(sp)) - }) - } - - pub fn span_to_margin(&self, sp: Span) -> Option { - match self.span_to_prev_source(sp) { - Err(_) => None, - Ok(source) => source - .split('\n') - .last() - .map(|last_line| last_line.len() - last_line.trim_start().len()), - } - } - - /// Returns the source snippet as `String` before the given `Span`. - pub fn span_to_prev_source(&self, sp: Span) -> Result { - self.span_to_source(sp, |src, start_index, _| { - src.get(..start_index) - .map(|s| s.to_string()) - .ok_or_else(|| SpanSnippetError::IllFormedSpan(sp)) - }) - } - - /// Extends the given `Span` to just after the previous occurrence of `c`. Return the same span - /// if no character could be found or if an error occurred while retrieving the code snippet. - pub fn span_extend_to_prev_char(&self, sp: Span, c: char) -> Span { - if let Ok(prev_source) = self.span_to_prev_source(sp) { - let prev_source = prev_source.rsplit(c).nth(0).unwrap_or("").trim_start(); - if !prev_source.is_empty() && !prev_source.contains('\n') { - return sp.with_lo(BytePos(sp.lo().0 - prev_source.len() as u32)); - } - } - - sp - } - - /// Extends the given `Span` to just after the previous occurrence of `pat` when surrounded by - /// whitespace. Returns the same span if no character could be found or if an error occurred - /// while retrieving the code snippet. - pub fn span_extend_to_prev_str(&self, sp: Span, pat: &str, accept_newlines: bool) -> Span { - // assure that the pattern is delimited, to avoid the following - // fn my_fn() - // ^^^^ returned span without the check - // ---------- correct span - for ws in &[" ", "\t", "\n"] { - let pat = pat.to_owned() + ws; - if let Ok(prev_source) = self.span_to_prev_source(sp) { - let prev_source = prev_source.rsplit(&pat).nth(0).unwrap_or("").trim_start(); - if !prev_source.is_empty() && (!prev_source.contains('\n') || accept_newlines) { - return sp.with_lo(BytePos(sp.lo().0 - prev_source.len() as u32)); - } - } - } - - sp - } - - /// Given a `Span`, tries to get a shorter span ending before the first occurrence of `char` - /// `c`. - pub fn span_until_char(&self, sp: Span, c: char) -> Span { - match self.span_to_snippet(sp) { - Ok(snippet) => { - let snippet = snippet.split(c).nth(0).unwrap_or("").trim_end(); - if !snippet.is_empty() && !snippet.contains('\n') { - sp.with_hi(BytePos(sp.lo().0 + snippet.len() as u32)) - } else { - sp - } - } - _ => sp, - } - } - - /// Given a `Span`, tries to get a shorter span ending just after the first occurrence of `char` - /// `c`. - pub fn span_through_char(&self, sp: Span, c: char) -> Span { - if let Ok(snippet) = self.span_to_snippet(sp) { - if let Some(offset) = snippet.find(c) { - return sp.with_hi(BytePos(sp.lo().0 + (offset + c.len_utf8()) as u32)); - } - } - sp - } - - /// Given a `Span`, gets a new `Span` covering the first token and all its trailing whitespace - /// or the original `Span`. - /// - /// If `sp` points to `"let mut x"`, then a span pointing at `"let "` will be returned. - pub fn span_until_non_whitespace(&self, sp: Span) -> Span { - let mut whitespace_found = false; - - self.span_take_while(sp, |c| { - if !whitespace_found && c.is_whitespace() { - whitespace_found = true; - } - - if whitespace_found && !c.is_whitespace() { false } else { true } - }) - } - - /// Given a `Span`, gets a new `Span` covering the first token without its trailing whitespace - /// or the original `Span` in case of error. - /// - /// If `sp` points to `"let mut x"`, then a span pointing at `"let"` will be returned. - pub fn span_until_whitespace(&self, sp: Span) -> Span { - self.span_take_while(sp, |c| !c.is_whitespace()) - } - - /// Given a `Span`, gets a shorter one until `predicate` yields `false`. - pub fn span_take_while

(&self, sp: Span, predicate: P) -> Span - where - P: for<'r> FnMut(&'r char) -> bool, - { - if let Ok(snippet) = self.span_to_snippet(sp) { - let offset = snippet.chars().take_while(predicate).map(|c| c.len_utf8()).sum::(); - - sp.with_hi(BytePos(sp.lo().0 + (offset as u32))) - } else { - sp - } - } - - pub fn def_span(&self, sp: Span) -> Span { - self.span_until_char(sp, '{') - } - - /// Returns a new span representing just the start point of this span. - pub fn start_point(&self, sp: Span) -> Span { - let pos = sp.lo().0; - let width = self.find_width_of_character_at_span(sp, false); - let corrected_start_position = pos.checked_add(width).unwrap_or(pos); - let end_point = BytePos(cmp::max(corrected_start_position, sp.lo().0)); - sp.with_hi(end_point) - } - - /// Returns a new span representing just the end point of this span. - pub fn end_point(&self, sp: Span) -> Span { - let pos = sp.hi().0; - - let width = self.find_width_of_character_at_span(sp, false); - let corrected_end_position = pos.checked_sub(width).unwrap_or(pos); - - let end_point = BytePos(cmp::max(corrected_end_position, sp.lo().0)); - sp.with_lo(end_point) - } - - /// Returns a new span representing the next character after the end-point of this span. - pub fn next_point(&self, sp: Span) -> Span { - let start_of_next_point = sp.hi().0; - - let width = self.find_width_of_character_at_span(sp, true); - // If the width is 1, then the next span should point to the same `lo` and `hi`. However, - // in the case of a multibyte character, where the width != 1, the next span should - // span multiple bytes to include the whole character. - let end_of_next_point = - start_of_next_point.checked_add(width - 1).unwrap_or(start_of_next_point); - - let end_of_next_point = BytePos(cmp::max(sp.lo().0 + 1, end_of_next_point)); - Span::new(BytePos(start_of_next_point), end_of_next_point, sp.ctxt()) - } - - /// Finds the width of a character, either before or after the provided span. - fn find_width_of_character_at_span(&self, sp: Span, forwards: bool) -> u32 { - let sp = sp.data(); - if sp.lo == sp.hi { - debug!("find_width_of_character_at_span: early return empty span"); - return 1; - } - - let local_begin = self.lookup_byte_offset(sp.lo); - let local_end = self.lookup_byte_offset(sp.hi); - debug!( - "find_width_of_character_at_span: local_begin=`{:?}`, local_end=`{:?}`", - local_begin, local_end - ); - - if local_begin.sf.start_pos != local_end.sf.start_pos { - debug!("find_width_of_character_at_span: begin and end are in different files"); - return 1; - } - - let start_index = local_begin.pos.to_usize(); - let end_index = local_end.pos.to_usize(); - debug!( - "find_width_of_character_at_span: start_index=`{:?}`, end_index=`{:?}`", - start_index, end_index - ); - - // Disregard indexes that are at the start or end of their spans, they can't fit bigger - // characters. - if (!forwards && end_index == usize::min_value()) - || (forwards && start_index == usize::max_value()) - { - debug!("find_width_of_character_at_span: start or end of span, cannot be multibyte"); - return 1; - } - - let source_len = (local_begin.sf.end_pos - local_begin.sf.start_pos).to_usize(); - debug!("find_width_of_character_at_span: source_len=`{:?}`", source_len); - // Ensure indexes are also not malformed. - if start_index > end_index || end_index > source_len { - debug!("find_width_of_character_at_span: source indexes are malformed"); - return 1; - } - - let src = local_begin.sf.external_src.borrow(); - - // We need to extend the snippet to the end of the src rather than to end_index so when - // searching forwards for boundaries we've got somewhere to search. - let snippet = if let Some(ref src) = local_begin.sf.src { - let len = src.len(); - (&src[start_index..len]) - } else if let Some(src) = src.get_source() { - let len = src.len(); - (&src[start_index..len]) - } else { - return 1; - }; - debug!("find_width_of_character_at_span: snippet=`{:?}`", snippet); - - let mut target = if forwards { end_index + 1 } else { end_index - 1 }; - debug!("find_width_of_character_at_span: initial target=`{:?}`", target); - - while !snippet.is_char_boundary(target - start_index) && target < source_len { - target = if forwards { - target + 1 - } else { - match target.checked_sub(1) { - Some(target) => target, - None => { - break; - } - } - }; - debug!("find_width_of_character_at_span: target=`{:?}`", target); - } - debug!("find_width_of_character_at_span: final target=`{:?}`", target); - - if forwards { (target - end_index) as u32 } else { (end_index - target) as u32 } - } - - pub fn get_source_file(&self, filename: &FileName) -> Option> { - for sf in self.files.borrow().source_files.iter() { - if *filename == sf.name { - return Some(sf.clone()); - } - } - None - } - - /// For a global `BytePos`, computes the local offset within the containing `SourceFile`. - pub fn lookup_byte_offset(&self, bpos: BytePos) -> SourceFileAndBytePos { - let idx = self.lookup_source_file_idx(bpos); - let sf = (*self.files.borrow().source_files)[idx].clone(); - let offset = bpos - sf.start_pos; - SourceFileAndBytePos { sf, pos: offset } - } - - /// Converts an absolute `BytePos` to a `CharPos` relative to the `SourceFile`. - pub fn bytepos_to_file_charpos(&self, bpos: BytePos) -> CharPos { - let idx = self.lookup_source_file_idx(bpos); - let map = &(*self.files.borrow().source_files)[idx]; - - // The number of extra bytes due to multibyte chars in the `SourceFile`. - let mut total_extra_bytes = 0; - - for mbc in map.multibyte_chars.iter() { - debug!("{}-byte char at {:?}", mbc.bytes, mbc.pos); - if mbc.pos < bpos { - // Every character is at least one byte, so we only - // count the actual extra bytes. - total_extra_bytes += mbc.bytes as u32 - 1; - // We should never see a byte position in the middle of a - // character. - assert!(bpos.to_u32() >= mbc.pos.to_u32() + mbc.bytes as u32); - } else { - break; - } - } - - assert!(map.start_pos.to_u32() + total_extra_bytes <= bpos.to_u32()); - CharPos(bpos.to_usize() - map.start_pos.to_usize() - total_extra_bytes as usize) - } - - // Returns the index of the `SourceFile` (in `self.files`) that contains `pos`. - pub fn lookup_source_file_idx(&self, pos: BytePos) -> usize { - self.files - .borrow() - .source_files - .binary_search_by_key(&pos, |key| key.start_pos) - .unwrap_or_else(|p| p - 1) - } - - pub fn count_lines(&self) -> usize { - self.files().iter().fold(0, |a, f| a + f.count_lines()) - } - - pub fn generate_fn_name_span(&self, span: Span) -> Option { - let prev_span = self.span_extend_to_prev_str(span, "fn", true); - self.span_to_snippet(prev_span) - .map(|snippet| { - let len = snippet - .find(|c: char| !c.is_alphanumeric() && c != '_') - .expect("no label after fn"); - prev_span.with_hi(BytePos(prev_span.lo().0 + len as u32)) - }) - .ok() - } - - /// Takes the span of a type parameter in a function signature and try to generate a span for - /// the function name (with generics) and a new snippet for this span with the pointed type - /// parameter as a new local type parameter. - /// - /// For instance: - /// ```rust,ignore (pseudo-Rust) - /// // Given span - /// fn my_function(param: T) - /// // ^ Original span - /// - /// // Result - /// fn my_function(param: T) - /// // ^^^^^^^^^^^ Generated span with snippet `my_function` - /// ``` - /// - /// Attention: The method used is very fragile since it essentially duplicates the work of the - /// parser. If you need to use this function or something similar, please consider updating the - /// `SourceMap` functions and this function to something more robust. - pub fn generate_local_type_param_snippet(&self, span: Span) -> Option<(Span, String)> { - // Try to extend the span to the previous "fn" keyword to retrieve the function - // signature. - let sugg_span = self.span_extend_to_prev_str(span, "fn", false); - if sugg_span != span { - if let Ok(snippet) = self.span_to_snippet(sugg_span) { - // Consume the function name. - let mut offset = snippet - .find(|c: char| !c.is_alphanumeric() && c != '_') - .expect("no label after fn"); - - // Consume the generics part of the function signature. - let mut bracket_counter = 0; - let mut last_char = None; - for c in snippet[offset..].chars() { - match c { - '<' => bracket_counter += 1, - '>' => bracket_counter -= 1, - '(' => { - if bracket_counter == 0 { - break; - } - } - _ => {} - } - offset += c.len_utf8(); - last_char = Some(c); - } - - // Adjust the suggestion span to encompass the function name with its generics. - let sugg_span = sugg_span.with_hi(BytePos(sugg_span.lo().0 + offset as u32)); - - // Prepare the new suggested snippet to append the type parameter that triggered - // the error in the generics of the function signature. - let mut new_snippet = if last_char == Some('>') { - format!("{}, ", &snippet[..(offset - '>'.len_utf8())]) - } else { - format!("{}<", &snippet[..offset]) - }; - new_snippet - .push_str(&self.span_to_snippet(span).unwrap_or_else(|_| "T".to_string())); - new_snippet.push('>'); - - return Some((sugg_span, new_snippet)); - } - } - - None - } - pub fn ensure_source_file_source_present(&self, source_file: Lrc) -> bool { - source_file.add_external_src(|| match source_file.name { - FileName::Real(ref name) => self.file_loader.read_file(name).ok(), - _ => None, - }) - } - pub fn call_span_if_macro(&self, sp: Span) -> Span { - if self.span_to_filename(sp.clone()).is_macros() { - let v = sp.macro_backtrace(); - if let Some(use_site) = v.last() { - return use_site.call_site; - } - } - sp - } -} - -#[derive(Clone)] -pub struct FilePathMapping { - mapping: Vec<(PathBuf, PathBuf)>, -} - -impl FilePathMapping { - pub fn empty() -> FilePathMapping { - FilePathMapping { mapping: vec![] } - } - - pub fn new(mapping: Vec<(PathBuf, PathBuf)>) -> FilePathMapping { - FilePathMapping { mapping } - } - - /// Applies any path prefix substitution as defined by the mapping. - /// The return value is the remapped path and a boolean indicating whether - /// the path was affected by the mapping. - pub fn map_prefix(&self, path: PathBuf) -> (PathBuf, bool) { - // NOTE: We are iterating over the mapping entries from last to first - // because entries specified later on the command line should - // take precedence. - for &(ref from, ref to) in self.mapping.iter().rev() { - if let Ok(rest) = path.strip_prefix(from) { - return (to.join(rest), true); - } - } - - (path, false) - } -} diff --git a/src/libsyntax_pos/source_map/tests.rs b/src/libsyntax_pos/source_map/tests.rs deleted file mode 100644 index 79df1884f0d..00000000000 --- a/src/libsyntax_pos/source_map/tests.rs +++ /dev/null @@ -1,216 +0,0 @@ -use super::*; - -use rustc_data_structures::sync::Lrc; - -fn init_source_map() -> SourceMap { - let sm = SourceMap::new(FilePathMapping::empty()); - sm.new_source_file(PathBuf::from("blork.rs").into(), "first line.\nsecond line".to_string()); - sm.new_source_file(PathBuf::from("empty.rs").into(), String::new()); - sm.new_source_file(PathBuf::from("blork2.rs").into(), "first line.\nsecond line".to_string()); - sm -} - -/// Tests `lookup_byte_offset`. -#[test] -fn t3() { - let sm = init_source_map(); - - let srcfbp1 = sm.lookup_byte_offset(BytePos(23)); - assert_eq!(srcfbp1.sf.name, PathBuf::from("blork.rs").into()); - assert_eq!(srcfbp1.pos, BytePos(23)); - - let srcfbp1 = sm.lookup_byte_offset(BytePos(24)); - assert_eq!(srcfbp1.sf.name, PathBuf::from("empty.rs").into()); - assert_eq!(srcfbp1.pos, BytePos(0)); - - let srcfbp2 = sm.lookup_byte_offset(BytePos(25)); - assert_eq!(srcfbp2.sf.name, PathBuf::from("blork2.rs").into()); - assert_eq!(srcfbp2.pos, BytePos(0)); -} - -/// Tests `bytepos_to_file_charpos`. -#[test] -fn t4() { - let sm = init_source_map(); - - let cp1 = sm.bytepos_to_file_charpos(BytePos(22)); - assert_eq!(cp1, CharPos(22)); - - let cp2 = sm.bytepos_to_file_charpos(BytePos(25)); - assert_eq!(cp2, CharPos(0)); -} - -/// Tests zero-length `SourceFile`s. -#[test] -fn t5() { - let sm = init_source_map(); - - let loc1 = sm.lookup_char_pos(BytePos(22)); - assert_eq!(loc1.file.name, PathBuf::from("blork.rs").into()); - assert_eq!(loc1.line, 2); - assert_eq!(loc1.col, CharPos(10)); - - let loc2 = sm.lookup_char_pos(BytePos(25)); - assert_eq!(loc2.file.name, PathBuf::from("blork2.rs").into()); - assert_eq!(loc2.line, 1); - assert_eq!(loc2.col, CharPos(0)); -} - -fn init_source_map_mbc() -> SourceMap { - let sm = SourceMap::new(FilePathMapping::empty()); - // "€" is a three-byte UTF8 char. - sm.new_source_file( - PathBuf::from("blork.rs").into(), - "fir€st €€€€ line.\nsecond line".to_string(), - ); - sm.new_source_file( - PathBuf::from("blork2.rs").into(), - "first line€€.\n€ second line".to_string(), - ); - sm -} - -/// Tests `bytepos_to_file_charpos` in the presence of multi-byte chars. -#[test] -fn t6() { - let sm = init_source_map_mbc(); - - let cp1 = sm.bytepos_to_file_charpos(BytePos(3)); - assert_eq!(cp1, CharPos(3)); - - let cp2 = sm.bytepos_to_file_charpos(BytePos(6)); - assert_eq!(cp2, CharPos(4)); - - let cp3 = sm.bytepos_to_file_charpos(BytePos(56)); - assert_eq!(cp3, CharPos(12)); - - let cp4 = sm.bytepos_to_file_charpos(BytePos(61)); - assert_eq!(cp4, CharPos(15)); -} - -/// Test `span_to_lines` for a span ending at the end of a `SourceFile`. -#[test] -fn t7() { - let sm = init_source_map(); - let span = Span::with_root_ctxt(BytePos(12), BytePos(23)); - let file_lines = sm.span_to_lines(span).unwrap(); - - assert_eq!(file_lines.file.name, PathBuf::from("blork.rs").into()); - assert_eq!(file_lines.lines.len(), 1); - assert_eq!(file_lines.lines[0].line_index, 1); -} - -/// Given a string like " ~~~~~~~~~~~~ ", produces a span -/// converting that range. The idea is that the string has the same -/// length as the input, and we uncover the byte positions. Note -/// that this can span lines and so on. -fn span_from_selection(input: &str, selection: &str) -> Span { - assert_eq!(input.len(), selection.len()); - let left_index = selection.find('~').unwrap() as u32; - let right_index = selection.rfind('~').map(|x| x as u32).unwrap_or(left_index); - Span::with_root_ctxt(BytePos(left_index), BytePos(right_index + 1)) -} - -/// Tests `span_to_snippet` and `span_to_lines` for a span converting 3 -/// lines in the middle of a file. -#[test] -fn span_to_snippet_and_lines_spanning_multiple_lines() { - let sm = SourceMap::new(FilePathMapping::empty()); - let inputtext = "aaaaa\nbbbbBB\nCCC\nDDDDDddddd\neee\n"; - let selection = " \n ~~\n~~~\n~~~~~ \n \n"; - sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_string()); - let span = span_from_selection(inputtext, selection); - - // Check that we are extracting the text we thought we were extracting. - assert_eq!(&sm.span_to_snippet(span).unwrap(), "BB\nCCC\nDDDDD"); - - // Check that span_to_lines gives us the complete result with the lines/cols we expected. - let lines = sm.span_to_lines(span).unwrap(); - let expected = vec![ - LineInfo { line_index: 1, start_col: CharPos(4), end_col: CharPos(6) }, - LineInfo { line_index: 2, start_col: CharPos(0), end_col: CharPos(3) }, - LineInfo { line_index: 3, start_col: CharPos(0), end_col: CharPos(5) }, - ]; - assert_eq!(lines.lines, expected); -} - -/// Test span_to_snippet for a span ending at the end of a `SourceFile`. -#[test] -fn t8() { - let sm = init_source_map(); - let span = Span::with_root_ctxt(BytePos(12), BytePos(23)); - let snippet = sm.span_to_snippet(span); - - assert_eq!(snippet, Ok("second line".to_string())); -} - -/// Test `span_to_str` for a span ending at the end of a `SourceFile`. -#[test] -fn t9() { - let sm = init_source_map(); - let span = Span::with_root_ctxt(BytePos(12), BytePos(23)); - let sstr = sm.span_to_string(span); - - assert_eq!(sstr, "blork.rs:2:1: 2:12"); -} - -/// Tests failing to merge two spans on different lines. -#[test] -fn span_merging_fail() { - let sm = SourceMap::new(FilePathMapping::empty()); - let inputtext = "bbbb BB\ncc CCC\n"; - let selection1 = " ~~\n \n"; - let selection2 = " \n ~~~\n"; - sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_owned()); - let span1 = span_from_selection(inputtext, selection1); - let span2 = span_from_selection(inputtext, selection2); - - assert!(sm.merge_spans(span1, span2).is_none()); -} - -/// Returns the span corresponding to the `n`th occurrence of `substring` in `source_text`. -trait SourceMapExtension { - fn span_substr( - &self, - file: &Lrc, - source_text: &str, - substring: &str, - n: usize, - ) -> Span; -} - -impl SourceMapExtension for SourceMap { - fn span_substr( - &self, - file: &Lrc, - source_text: &str, - substring: &str, - n: usize, - ) -> Span { - println!( - "span_substr(file={:?}/{:?}, substring={:?}, n={})", - file.name, file.start_pos, substring, n - ); - let mut i = 0; - let mut hi = 0; - loop { - let offset = source_text[hi..].find(substring).unwrap_or_else(|| { - panic!( - "source_text `{}` does not have {} occurrences of `{}`, only {}", - source_text, n, substring, i - ); - }); - let lo = hi + offset; - hi = lo + substring.len(); - if i == n { - let span = Span::with_root_ctxt( - BytePos(lo as u32 + file.start_pos.0), - BytePos(hi as u32 + file.start_pos.0), - ); - assert_eq!(&self.span_to_snippet(span).unwrap()[..], substring); - return span; - } - i += 1; - } - } -} diff --git a/src/libsyntax_pos/span_encoding.rs b/src/libsyntax_pos/span_encoding.rs deleted file mode 100644 index d769cf83a03..00000000000 --- a/src/libsyntax_pos/span_encoding.rs +++ /dev/null @@ -1,140 +0,0 @@ -// Spans are encoded using 1-bit tag and 2 different encoding formats (one for each tag value). -// One format is used for keeping span data inline, -// another contains index into an out-of-line span interner. -// The encoding format for inline spans were obtained by optimizing over crates in rustc/libstd. -// See https://internals.rust-lang.org/t/rfc-compiler-refactoring-spans/1357/28 - -use crate::hygiene::SyntaxContext; -use crate::GLOBALS; -use crate::{BytePos, SpanData}; - -use rustc_data_structures::fx::FxHashMap; - -/// A compressed span. -/// -/// `SpanData` is 12 bytes, which is a bit too big to stick everywhere. `Span` -/// is a form that only takes up 8 bytes, with less space for the length and -/// context. The vast majority (99.9%+) of `SpanData` instances will fit within -/// those 8 bytes; any `SpanData` whose fields don't fit into a `Span` are -/// stored in a separate interner table, and the `Span` will index into that -/// table. Interning is rare enough that the cost is low, but common enough -/// that the code is exercised regularly. -/// -/// An earlier version of this code used only 4 bytes for `Span`, but that was -/// slower because only 80--90% of spans could be stored inline (even less in -/// very large crates) and so the interner was used a lot more. -/// -/// Inline (compressed) format: -/// - `span.base_or_index == span_data.lo` -/// - `span.len_or_tag == len == span_data.hi - span_data.lo` (must be `<= MAX_LEN`) -/// - `span.ctxt == span_data.ctxt` (must be `<= MAX_CTXT`) -/// -/// Interned format: -/// - `span.base_or_index == index` (indexes into the interner table) -/// - `span.len_or_tag == LEN_TAG` (high bit set, all other bits are zero) -/// - `span.ctxt == 0` -/// -/// The inline form uses 0 for the tag value (rather than 1) so that we don't -/// need to mask out the tag bit when getting the length, and so that the -/// dummy span can be all zeroes. -/// -/// Notes about the choice of field sizes: -/// - `base` is 32 bits in both `Span` and `SpanData`, which means that `base` -/// values never cause interning. The number of bits needed for `base` -/// depends on the crate size. 32 bits allows up to 4 GiB of code in a crate. -/// `script-servo` is the largest crate in `rustc-perf`, requiring 26 bits -/// for some spans. -/// - `len` is 15 bits in `Span` (a u16, minus 1 bit for the tag) and 32 bits -/// in `SpanData`, which means that large `len` values will cause interning. -/// The number of bits needed for `len` does not depend on the crate size. -/// The most common number of bits for `len` are 0--7, with a peak usually at -/// 3 or 4, and then it drops off quickly from 8 onwards. 15 bits is enough -/// for 99.99%+ of cases, but larger values (sometimes 20+ bits) might occur -/// dozens of times in a typical crate. -/// - `ctxt` is 16 bits in `Span` and 32 bits in `SpanData`, which means that -/// large `ctxt` values will cause interning. The number of bits needed for -/// `ctxt` values depend partly on the crate size and partly on the form of -/// the code. No crates in `rustc-perf` need more than 15 bits for `ctxt`, -/// but larger crates might need more than 16 bits. -/// -#[derive(Clone, Copy, Eq, PartialEq, Hash)] -pub struct Span { - base_or_index: u32, - len_or_tag: u16, - ctxt_or_zero: u16, -} - -const LEN_TAG: u16 = 0b1000_0000_0000_0000; -const MAX_LEN: u32 = 0b0111_1111_1111_1111; -const MAX_CTXT: u32 = 0b1111_1111_1111_1111; - -/// Dummy span, both position and length are zero, syntax context is zero as well. -pub const DUMMY_SP: Span = Span { base_or_index: 0, len_or_tag: 0, ctxt_or_zero: 0 }; - -impl Span { - #[inline] - pub fn new(mut lo: BytePos, mut hi: BytePos, ctxt: SyntaxContext) -> Self { - if lo > hi { - std::mem::swap(&mut lo, &mut hi); - } - - let (base, len, ctxt2) = (lo.0, hi.0 - lo.0, ctxt.as_u32()); - - if len <= MAX_LEN && ctxt2 <= MAX_CTXT { - // Inline format. - Span { base_or_index: base, len_or_tag: len as u16, ctxt_or_zero: ctxt2 as u16 } - } else { - // Interned format. - let index = with_span_interner(|interner| interner.intern(&SpanData { lo, hi, ctxt })); - Span { base_or_index: index, len_or_tag: LEN_TAG, ctxt_or_zero: 0 } - } - } - - #[inline] - pub fn data(self) -> SpanData { - if self.len_or_tag != LEN_TAG { - // Inline format. - debug_assert!(self.len_or_tag as u32 <= MAX_LEN); - SpanData { - lo: BytePos(self.base_or_index), - hi: BytePos(self.base_or_index + self.len_or_tag as u32), - ctxt: SyntaxContext::from_u32(self.ctxt_or_zero as u32), - } - } else { - // Interned format. - debug_assert!(self.ctxt_or_zero == 0); - let index = self.base_or_index; - with_span_interner(|interner| *interner.get(index)) - } - } -} - -#[derive(Default)] -pub struct SpanInterner { - spans: FxHashMap, - span_data: Vec, -} - -impl SpanInterner { - fn intern(&mut self, span_data: &SpanData) -> u32 { - if let Some(index) = self.spans.get(span_data) { - return *index; - } - - let index = self.spans.len() as u32; - self.span_data.push(*span_data); - self.spans.insert(*span_data, index); - index - } - - #[inline] - fn get(&self, index: u32) -> &SpanData { - &self.span_data[index as usize] - } -} - -// If an interner exists, return it. Otherwise, prepare a fresh one. -#[inline] -fn with_span_interner T>(f: F) -> T { - GLOBALS.with(|globals| f(&mut *globals.span_interner.lock())) -} diff --git a/src/libsyntax_pos/symbol.rs b/src/libsyntax_pos/symbol.rs deleted file mode 100644 index 7ae037faf15..00000000000 --- a/src/libsyntax_pos/symbol.rs +++ /dev/null @@ -1,1213 +0,0 @@ -//! An "interner" is a data structure that associates values with usize tags and -//! allows bidirectional lookup; i.e., given a value, one can easily find the -//! type, and vice versa. - -use arena::DroplessArena; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHashKey}; -use rustc_index::vec::Idx; -use rustc_macros::{symbols, HashStable_Generic}; -use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; -use rustc_serialize::{UseSpecializedDecodable, UseSpecializedEncodable}; - -use std::cmp::{Ord, PartialEq, PartialOrd}; -use std::fmt; -use std::hash::{Hash, Hasher}; -use std::str; - -use crate::{Span, DUMMY_SP, GLOBALS}; - -#[cfg(test)] -mod tests; - -symbols! { - // After modifying this list adjust `is_special`, `is_used_keyword`/`is_unused_keyword`, - // this should be rarely necessary though if the keywords are kept in alphabetic order. - Keywords { - // Special reserved identifiers used internally for elided lifetimes, - // unnamed method parameters, crate root module, error recovery etc. - Invalid: "", - PathRoot: "{{root}}", - DollarCrate: "$crate", - Underscore: "_", - - // Keywords that are used in stable Rust. - As: "as", - Break: "break", - Const: "const", - Continue: "continue", - Crate: "crate", - Else: "else", - Enum: "enum", - Extern: "extern", - False: "false", - Fn: "fn", - For: "for", - If: "if", - Impl: "impl", - In: "in", - Let: "let", - Loop: "loop", - Match: "match", - Mod: "mod", - Move: "move", - Mut: "mut", - Pub: "pub", - Ref: "ref", - Return: "return", - SelfLower: "self", - SelfUpper: "Self", - Static: "static", - Struct: "struct", - Super: "super", - Trait: "trait", - True: "true", - Type: "type", - Unsafe: "unsafe", - Use: "use", - Where: "where", - While: "while", - - // Keywords that are used in unstable Rust or reserved for future use. - Abstract: "abstract", - Become: "become", - Box: "box", - Do: "do", - Final: "final", - Macro: "macro", - Override: "override", - Priv: "priv", - Typeof: "typeof", - Unsized: "unsized", - Virtual: "virtual", - Yield: "yield", - - // Edition-specific keywords that are used in stable Rust. - Async: "async", // >= 2018 Edition only - Await: "await", // >= 2018 Edition only - Dyn: "dyn", // >= 2018 Edition only - - // Edition-specific keywords that are used in unstable Rust or reserved for future use. - Try: "try", // >= 2018 Edition only - - // Special lifetime names - UnderscoreLifetime: "'_", - StaticLifetime: "'static", - - // Weak keywords, have special meaning only in specific contexts. - Auto: "auto", - Catch: "catch", - Default: "default", - Raw: "raw", - Union: "union", - } - - // Symbols that can be referred to with syntax_pos::sym::*. The symbol is - // the stringified identifier unless otherwise specified (e.g. - // `proc_dash_macro` represents "proc-macro"). - // - // As well as the symbols listed, there are symbols for the the strings - // "0", "1", ..., "9", which are accessible via `sym::integer`. - Symbols { - aarch64_target_feature, - abi, - abi_amdgpu_kernel, - abi_efiapi, - abi_msp430_interrupt, - abi_ptx, - abi_sysv64, - abi_thiscall, - abi_unadjusted, - abi_vectorcall, - abi_x86_interrupt, - aborts, - add_with_overflow, - advanced_slice_patterns, - adx_target_feature, - alias, - align, - alignstack, - all, - allocator, - allocator_internals, - alloc_error_handler, - allow, - allowed, - allow_fail, - allow_internal_unsafe, - allow_internal_unstable, - allow_internal_unstable_backcompat_hack, - always, - and, - any, - arbitrary_enum_discriminant, - arbitrary_self_types, - Arguments, - ArgumentV1, - arm_target_feature, - asm, - assert, - associated_consts, - associated_type_bounds, - associated_type_defaults, - associated_types, - assume_init, - async_await, - async_closure, - attr, - attributes, - attr_literals, - augmented_assignments, - automatically_derived, - avx512_target_feature, - await_macro, - begin_panic, - bench, - bin, - bind_by_move_pattern_guards, - bindings_after_at, - block, - bool, - borrowck_graphviz_postflow, - borrowck_graphviz_preflow, - box_patterns, - box_syntax, - braced_empty_structs, - bswap, - bitreverse, - C, - caller_location, - cdylib, - cfg, - cfg_attr, - cfg_attr_multi, - cfg_doctest, - cfg_sanitize, - cfg_target_feature, - cfg_target_has_atomic, - cfg_target_thread_local, - cfg_target_vendor, - char, - clippy, - clone, - Clone, - clone_closures, - clone_from, - closure_to_fn_coercion, - cmp, - cmpxchg16b_target_feature, - cold, - column, - compile_error, - compiler_builtins, - concat, - concat_idents, - conservative_impl_trait, - console, - const_compare_raw_pointers, - const_constructor, - const_extern_fn, - const_fn, - const_fn_union, - const_generics, - const_if_match, - const_indexing, - const_in_array_repeat_expressions, - const_let, - const_loop, - const_mut_refs, - const_panic, - const_raw_ptr_deref, - const_raw_ptr_to_usize_cast, - const_transmute, - contents, - context, - convert, - Copy, - copy_closures, - core, - core_intrinsics, - crate_id, - crate_in_paths, - crate_local, - crate_name, - crate_type, - crate_visibility_modifier, - ctpop, - cttz, - cttz_nonzero, - ctlz, - ctlz_nonzero, - custom_attribute, - custom_derive, - custom_inner_attributes, - custom_test_frameworks, - c_variadic, - debug_trait, - declare_lint_pass, - decl_macro, - Debug, - Decodable, - Default, - default_lib_allocator, - default_type_parameter_fallback, - default_type_params, - delay_span_bug_from_inside_query, - deny, - deprecated, - deref, - deref_mut, - derive, - diagnostic, - direct, - doc, - doc_alias, - doc_cfg, - doc_keyword, - doc_masked, - doc_spotlight, - doctest, - document_private_items, - dotdoteq_in_patterns, - dotdot_in_tuple_patterns, - double_braced_crate: "{{crate}}", - double_braced_impl: "{{impl}}", - double_braced_misc: "{{misc}}", - double_braced_closure: "{{closure}}", - double_braced_constructor: "{{constructor}}", - double_braced_constant: "{{constant}}", - double_braced_opaque: "{{opaque}}", - dropck_eyepatch, - dropck_parametricity, - drop_types_in_const, - dylib, - dyn_trait, - eh_personality, - eh_unwind_resume, - enable, - Encodable, - env, - eq, - err, - Err, - Eq, - Equal, - enclosing_scope, - except, - exclusive_range_pattern, - exhaustive_integer_patterns, - exhaustive_patterns, - existential_type, - expected, - export_name, - expr, - extern_absolute_paths, - external_doc, - extern_crate_item_prelude, - extern_crate_self, - extern_in_paths, - extern_prelude, - extern_types, - f16c_target_feature, - f32, - f64, - feature, - ffi_returns_twice, - field, - field_init_shorthand, - file, - fmt, - fmt_internals, - fn_must_use, - forbid, - format_args, - format_args_nl, - from, - From, - from_desugaring, - from_error, - from_generator, - from_method, - from_ok, - from_usize, - fundamental, - future, - Future, - FxHashSet, - FxHashMap, - gen_future, - generators, - generic_associated_types, - generic_param_attrs, - global_allocator, - global_asm, - globs, - hash, - Hash, - HashSet, - HashMap, - hexagon_target_feature, - hidden, - homogeneous_aggregate, - html_favicon_url, - html_logo_url, - html_no_source, - html_playground_url, - html_root_url, - i128, - i128_type, - i16, - i32, - i64, - i8, - ident, - if_let, - if_while_or_patterns, - ignore, - impl_header_lifetime_elision, - impl_lint_pass, - impl_trait_in_bindings, - import_shadowing, - index, - index_mut, - in_band_lifetimes, - include, - include_bytes, - include_str, - inclusive_range_syntax, - infer_outlives_requirements, - infer_static_outlives_requirements, - inline, - intel, - into_future, - IntoFuture, - into_iter, - IntoIterator, - into_result, - intrinsics, - irrefutable_let_patterns, - isize, - issue, - issue_5723_bootstrap, - issue_tracker_base_url, - item, - item_context: "ItemContext", - item_like_imports, - iter, - Iterator, - keyword, - kind, - label, - label_break_value, - lang, - lang_items, - let_chains, - lhs, - lib, - lifetime, - line, - link, - linkage, - link_args, - link_cfg, - link_llvm_intrinsics, - link_name, - link_ordinal, - link_section, - LintPass, - lint_reasons, - literal, - local_inner_macros, - log_syntax, - loop_break_value, - macro_at_most_once_rep, - macro_escape, - macro_export, - macro_lifetime_matcher, - macro_literal_matcher, - macro_reexport, - macro_rules, - macros_in_extern, - macro_use, - macro_vis_matcher, - main, - managed_boxes, - marker, - marker_trait_attr, - masked, - match_beginning_vert, - match_default_bindings, - may_dangle, - maybe_uninit_uninit, - maybe_uninit_zeroed, - mem_uninitialized, - mem_zeroed, - member_constraints, - message, - meta, - min_align_of, - min_const_fn, - min_const_unsafe_fn, - mips_target_feature, - mmx_target_feature, - module, - module_path, - more_struct_aliases, - move_val_init, - movbe_target_feature, - mul_with_overflow, - must_use, - naked, - naked_functions, - name, - needs_allocator, - needs_drop, - needs_panic_runtime, - negate_unsigned, - never, - never_type, - never_type_fallback, - new, - next, - __next, - nll, - no_builtins, - no_core, - no_crate_inject, - no_debug, - no_default_passes, - no_implicit_prelude, - no_inline, - no_link, - no_main, - no_mangle, - non_ascii_idents, - None, - non_exhaustive, - non_modrs_mods, - no_stack_check, - no_start, - no_std, - not, - note, - object_safe_for_dispatch, - Ok, - omit_gdb_pretty_printer_section, - on, - on_unimplemented, - oom, - ops, - optimize, - optimize_attribute, - optin_builtin_traits, - option, - Option, - option_env, - opt_out_copy, - or, - or_patterns, - Ord, - Ordering, - Output, - overlapping_marker_traits, - packed, - panic, - panic_handler, - panic_impl, - panic_implementation, - panic_runtime, - parent_trait, - partial_cmp, - param_attrs, - PartialEq, - PartialOrd, - passes, - pat, - path, - pattern_parentheses, - Pending, - pin, - Pin, - pinned, - platform_intrinsics, - plugin, - plugin_registrar, - plugins, - Poll, - poll_with_tls_context, - powerpc_target_feature, - precise_pointer_size_matching, - pref_align_of, - prelude, - prelude_import, - primitive, - proc_dash_macro: "proc-macro", - proc_macro, - proc_macro_attribute, - proc_macro_def_site, - proc_macro_derive, - proc_macro_expr, - proc_macro_gen, - proc_macro_hygiene, - proc_macro_internals, - proc_macro_mod, - proc_macro_non_items, - proc_macro_path_invoc, - profiler_runtime, - ptr_offset_from, - pub_restricted, - pushpop_unsafe, - quad_precision_float, - question_mark, - quote, - Range, - RangeFrom, - RangeFull, - RangeInclusive, - RangeTo, - RangeToInclusive, - raw_dylib, - raw_identifiers, - raw_ref_op, - Ready, - reason, - recursion_limit, - reexport_test_harness_main, - reflect, - register_attr, - register_tool, - relaxed_adts, - repr, - repr128, - repr_align, - repr_align_enum, - repr_packed, - repr_simd, - repr_transparent, - re_rebalance_coherence, - result, - Result, - Return, - rhs, - rlib, - rotate_left, - rotate_right, - rt, - rtm_target_feature, - rust, - rust_2015_preview, - rust_2018_preview, - rust_begin_unwind, - rustc, - RustcDecodable, - RustcEncodable, - rustc_allocator, - rustc_allocator_nounwind, - rustc_allow_const_fn_ptr, - rustc_args_required_const, - rustc_attrs, - rustc_builtin_macro, - rustc_clean, - rustc_const_unstable, - rustc_const_stable, - rustc_conversion_suggestion, - rustc_def_path, - rustc_deprecated, - rustc_diagnostic_item, - rustc_diagnostic_macros, - rustc_dirty, - rustc_dummy, - rustc_dump_env_program_clauses, - rustc_dump_program_clauses, - rustc_dump_user_substs, - rustc_error, - rustc_expected_cgu_reuse, - rustc_if_this_changed, - rustc_inherit_overflow_checks, - rustc_layout, - rustc_layout_scalar_valid_range_end, - rustc_layout_scalar_valid_range_start, - rustc_macro_transparency, - rustc_mir, - rustc_nonnull_optimization_guaranteed, - rustc_object_lifetime_default, - rustc_on_unimplemented, - rustc_outlives, - rustc_paren_sugar, - rustc_partition_codegened, - rustc_partition_reused, - rustc_peek, - rustc_peek_definite_init, - rustc_peek_maybe_init, - rustc_peek_maybe_uninit, - rustc_peek_indirectly_mutable, - rustc_private, - rustc_proc_macro_decls, - rustc_promotable, - rustc_regions, - rustc_stable, - rustc_std_internal_symbol, - rustc_symbol_name, - rustc_synthetic, - rustc_reservation_impl, - rustc_test_marker, - rustc_then_this_would_need, - rustc_variance, - rustfmt, - rust_eh_personality, - rust_eh_unwind_resume, - rust_oom, - rvalue_static_promotion, - sanitize, - sanitizer_runtime, - saturating_add, - saturating_sub, - _Self, - self_in_typedefs, - self_struct_ctor, - send_trait, - should_panic, - simd, - simd_extract, - simd_ffi, - simd_insert, - since, - size, - size_of, - slice_patterns, - slicing_syntax, - soft, - Some, - specialization, - speed, - spotlight, - sse4a_target_feature, - stable, - staged_api, - start, - static_in_const, - staticlib, - static_nobundle, - static_recursion, - std, - std_inject, - str, - stringify, - stmt, - stmt_expr_attributes, - stop_after_dataflow, - struct_field_attributes, - struct_inherit, - structural_match, - struct_variant, - sty, - sub_with_overflow, - suggestion, - sync_trait, - target_feature, - target_has_atomic, - target_has_atomic_load_store, - target_thread_local, - task, - tbm_target_feature, - termination_trait, - termination_trait_test, - test, - test_2018_feature, - test_accepted_feature, - test_case, - test_removed_feature, - test_runner, - then_with, - thread_local, - tool_attributes, - tool_lints, - trace_macros, - track_caller, - trait_alias, - transmute, - transparent, - transparent_enums, - transparent_unions, - trivial_bounds, - Try, - try_blocks, - try_trait, - tt, - tuple_indexing, - Ty, - ty, - type_alias_impl_trait, - type_id, - type_name, - TyCtxt, - TyKind, - type_alias_enum_variants, - type_ascription, - type_length_limit, - type_macros, - u128, - u16, - u32, - u64, - u8, - unboxed_closures, - unchecked_shl, - unchecked_shr, - underscore_const_names, - underscore_imports, - underscore_lifetimes, - uniform_paths, - universal_impl_trait, - unmarked_api, - unreachable_code, - unrestricted_attribute_tokens, - unsafe_no_drop_flag, - unsized_locals, - unsized_tuple_coercion, - unstable, - untagged_unions, - unwind, - unwind_attributes, - unwrap_or, - used, - use_extern_macros, - use_nested_groups, - usize, - v1, - val, - var, - vec, - Vec, - vis, - visible_private_types, - volatile, - warn, - wasm_import_module, - wasm_target_feature, - while_let, - windows, - windows_subsystem, - wrapping_add, - wrapping_sub, - wrapping_mul, - Yield, - } -} - -#[derive(Copy, Clone, Eq, HashStable_Generic)] -pub struct Ident { - pub name: Symbol, - pub span: Span, -} - -impl Ident { - #[inline] - /// Constructs a new identifier from a symbol and a span. - pub const fn new(name: Symbol, span: Span) -> Ident { - Ident { name, span } - } - - /// Constructs a new identifier with a dummy span. - #[inline] - pub const fn with_dummy_span(name: Symbol) -> Ident { - Ident::new(name, DUMMY_SP) - } - - #[inline] - pub fn invalid() -> Ident { - Ident::with_dummy_span(kw::Invalid) - } - - /// Maps a string to an identifier with a dummy span. - pub fn from_str(string: &str) -> Ident { - Ident::with_dummy_span(Symbol::intern(string)) - } - - /// Maps a string and a span to an identifier. - pub fn from_str_and_span(string: &str, span: Span) -> Ident { - Ident::new(Symbol::intern(string), span) - } - - /// Replaces `lo` and `hi` with those from `span`, but keep hygiene context. - pub fn with_span_pos(self, span: Span) -> Ident { - Ident::new(self.name, span.with_ctxt(self.span.ctxt())) - } - - pub fn without_first_quote(self) -> Ident { - Ident::new(Symbol::intern(self.as_str().trim_start_matches('\'')), self.span) - } - - /// "Normalize" ident for use in comparisons using "item hygiene". - /// Identifiers with same string value become same if they came from the same "modern" macro - /// (e.g., `macro` item, but not `macro_rules` item) and stay different if they came from - /// different "modern" macros. - /// Technically, this operation strips all non-opaque marks from ident's syntactic context. - pub fn modern(self) -> Ident { - Ident::new(self.name, self.span.modern()) - } - - /// "Normalize" ident for use in comparisons using "local variable hygiene". - /// Identifiers with same string value become same if they came from the same non-transparent - /// macro (e.g., `macro` or `macro_rules!` items) and stay different if they came from different - /// non-transparent macros. - /// Technically, this operation strips all transparent marks from ident's syntactic context. - pub fn modern_and_legacy(self) -> Ident { - Ident::new(self.name, self.span.modern_and_legacy()) - } - - /// Convert the name to a `SymbolStr`. This is a slowish operation because - /// it requires locking the symbol interner. - pub fn as_str(self) -> SymbolStr { - self.name.as_str() - } -} - -impl PartialEq for Ident { - fn eq(&self, rhs: &Self) -> bool { - self.name == rhs.name && self.span.ctxt() == rhs.span.ctxt() - } -} - -impl Hash for Ident { - fn hash(&self, state: &mut H) { - self.name.hash(state); - self.span.ctxt().hash(state); - } -} - -impl fmt::Debug for Ident { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.is_raw_guess() { - write!(f, "r#")?; - } - write!(f, "{}{:?}", self.name, self.span.ctxt()) - } -} - -impl fmt::Display for Ident { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.is_raw_guess() { - write!(f, "r#")?; - } - fmt::Display::fmt(&self.name, f) - } -} - -impl UseSpecializedEncodable for Ident { - fn default_encode(&self, s: &mut S) -> Result<(), S::Error> { - s.emit_struct("Ident", 2, |s| { - s.emit_struct_field("name", 0, |s| self.name.encode(s))?; - s.emit_struct_field("span", 1, |s| self.span.encode(s)) - }) - } -} - -impl UseSpecializedDecodable for Ident { - fn default_decode(d: &mut D) -> Result { - d.read_struct("Ident", 2, |d| { - Ok(Ident { - name: d.read_struct_field("name", 0, Decodable::decode)?, - span: d.read_struct_field("span", 1, Decodable::decode)?, - }) - }) - } -} - -/// An interned string. -/// -/// Internally, a `Symbol` is implemented as an index, and all operations -/// (including hashing, equality, and ordering) operate on that index. The use -/// of `rustc_index::newtype_index!` means that `Option` only takes up 4 bytes, -/// because `rustc_index::newtype_index!` reserves the last 256 values for tagging purposes. -/// -/// Note that `Symbol` cannot directly be a `rustc_index::newtype_index!` because it -/// implements `fmt::Debug`, `Encodable`, and `Decodable` in special ways. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Symbol(SymbolIndex); - -rustc_index::newtype_index! { - pub struct SymbolIndex { .. } -} - -impl Symbol { - const fn new(n: u32) -> Self { - Symbol(SymbolIndex::from_u32_const(n)) - } - - /// Maps a string to its interned representation. - pub fn intern(string: &str) -> Self { - with_interner(|interner| interner.intern(string)) - } - - /// Access the symbol's chars. This is a slowish operation because it - /// requires locking the symbol interner. - pub fn with R, R>(self, f: F) -> R { - with_interner(|interner| f(interner.get(self))) - } - - /// Convert to a `SymbolStr`. This is a slowish operation because it - /// requires locking the symbol interner. - pub fn as_str(self) -> SymbolStr { - with_interner(|interner| unsafe { - SymbolStr { string: std::mem::transmute::<&str, &str>(interner.get(self)) } - }) - } - - pub fn as_u32(self) -> u32 { - self.0.as_u32() - } -} - -impl fmt::Debug for Symbol { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.with(|str| fmt::Debug::fmt(&str, f)) - } -} - -impl fmt::Display for Symbol { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.with(|str| fmt::Display::fmt(&str, f)) - } -} - -impl Encodable for Symbol { - fn encode(&self, s: &mut S) -> Result<(), S::Error> { - self.with(|string| s.emit_str(string)) - } -} - -impl Decodable for Symbol { - fn decode(d: &mut D) -> Result { - Ok(Symbol::intern(&d.read_str()?)) - } -} - -impl HashStable for Symbol { - #[inline] - fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { - self.as_str().hash_stable(hcx, hasher); - } -} - -impl ToStableHashKey for Symbol { - type KeyType = SymbolStr; - - #[inline] - fn to_stable_hash_key(&self, _: &CTX) -> SymbolStr { - self.as_str() - } -} - -// The `&'static str`s in this type actually point into the arena. -#[derive(Default)] -pub struct Interner { - arena: DroplessArena, - names: FxHashMap<&'static str, Symbol>, - strings: Vec<&'static str>, -} - -impl Interner { - fn prefill(init: &[&'static str]) -> Self { - Interner { - strings: init.into(), - names: init.iter().copied().zip((0..).map(Symbol::new)).collect(), - ..Default::default() - } - } - - pub fn intern(&mut self, string: &str) -> Symbol { - if let Some(&name) = self.names.get(string) { - return name; - } - - let name = Symbol::new(self.strings.len() as u32); - - // `from_utf8_unchecked` is safe since we just allocated a `&str` which is known to be - // UTF-8. - let string: &str = - unsafe { str::from_utf8_unchecked(self.arena.alloc_slice(string.as_bytes())) }; - // It is safe to extend the arena allocation to `'static` because we only access - // these while the arena is still alive. - let string: &'static str = unsafe { &*(string as *const str) }; - self.strings.push(string); - self.names.insert(string, name); - name - } - - // Get the symbol as a string. `Symbol::as_str()` should be used in - // preference to this function. - pub fn get(&self, symbol: Symbol) -> &str { - self.strings[symbol.0.as_usize()] - } -} - -// This module has a very short name because it's used a lot. -pub mod kw { - use super::Symbol; - keywords!(); -} - -// This module has a very short name because it's used a lot. -pub mod sym { - use super::Symbol; - use std::convert::TryInto; - - symbols!(); - - // Get the symbol for an integer. The first few non-negative integers each - // have a static symbol and therefore are fast. - pub fn integer + Copy + ToString>(n: N) -> Symbol { - if let Result::Ok(idx) = n.try_into() { - if let Option::Some(&sym) = digits_array.get(idx) { - return sym; - } - } - Symbol::intern(&n.to_string()) - } -} - -impl Symbol { - fn is_used_keyword_2018(self) -> bool { - self >= kw::Async && self <= kw::Dyn - } - - fn is_unused_keyword_2018(self) -> bool { - self == kw::Try - } - - /// Used for sanity checking rustdoc keyword sections. - pub fn is_doc_keyword(self) -> bool { - self <= kw::Union - } - - /// A keyword or reserved identifier that can be used as a path segment. - pub fn is_path_segment_keyword(self) -> bool { - self == kw::Super - || self == kw::SelfLower - || self == kw::SelfUpper - || self == kw::Crate - || self == kw::PathRoot - || self == kw::DollarCrate - } - - /// Returns `true` if the symbol is `true` or `false`. - pub fn is_bool_lit(self) -> bool { - self == kw::True || self == kw::False - } - - /// This symbol can be a raw identifier. - pub fn can_be_raw(self) -> bool { - self != kw::Invalid && self != kw::Underscore && !self.is_path_segment_keyword() - } -} - -impl Ident { - // Returns `true` for reserved identifiers used internally for elided lifetimes, - // unnamed method parameters, crate root module, error recovery etc. - pub fn is_special(self) -> bool { - self.name <= kw::Underscore - } - - /// Returns `true` if the token is a keyword used in the language. - pub fn is_used_keyword(self) -> bool { - // Note: `span.edition()` is relatively expensive, don't call it unless necessary. - self.name >= kw::As && self.name <= kw::While - || self.name.is_used_keyword_2018() && self.span.rust_2018() - } - - /// Returns `true` if the token is a keyword reserved for possible future use. - pub fn is_unused_keyword(self) -> bool { - // Note: `span.edition()` is relatively expensive, don't call it unless necessary. - self.name >= kw::Abstract && self.name <= kw::Yield - || self.name.is_unused_keyword_2018() && self.span.rust_2018() - } - - /// Returns `true` if the token is either a special identifier or a keyword. - pub fn is_reserved(self) -> bool { - self.is_special() || self.is_used_keyword() || self.is_unused_keyword() - } - - /// A keyword or reserved identifier that can be used as a path segment. - pub fn is_path_segment_keyword(self) -> bool { - self.name.is_path_segment_keyword() - } - - /// We see this identifier in a normal identifier position, like variable name or a type. - /// How was it written originally? Did it use the raw form? Let's try to guess. - pub fn is_raw_guess(self) -> bool { - self.name.can_be_raw() && self.is_reserved() - } -} - -#[inline] -fn with_interner T>(f: F) -> T { - GLOBALS.with(|globals| f(&mut *globals.symbol_interner.lock())) -} - -/// An alternative to `Symbol`, useful when the chars within the symbol need to -/// be accessed. It deliberately has limited functionality and should only be -/// used for temporary values. -/// -/// Because the interner outlives any thread which uses this type, we can -/// safely treat `string` which points to interner data, as an immortal string, -/// as long as this type never crosses between threads. -// -// FIXME: ensure that the interner outlives any thread which uses `SymbolStr`, -// by creating a new thread right after constructing the interner. -#[derive(Clone, Eq, PartialOrd, Ord)] -pub struct SymbolStr { - string: &'static str, -} - -// This impl allows a `SymbolStr` to be directly equated with a `String` or -// `&str`. -impl> std::cmp::PartialEq for SymbolStr { - fn eq(&self, other: &T) -> bool { - self.string == other.deref() - } -} - -impl !Send for SymbolStr {} -impl !Sync for SymbolStr {} - -/// This impl means that if `ss` is a `SymbolStr`: -/// - `*ss` is a `str`; -/// - `&*ss` is a `&str`; -/// - `&ss as &str` is a `&str`, which means that `&ss` can be passed to a -/// function expecting a `&str`. -impl std::ops::Deref for SymbolStr { - type Target = str; - #[inline] - fn deref(&self) -> &str { - self.string - } -} - -impl fmt::Debug for SymbolStr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(self.string, f) - } -} - -impl fmt::Display for SymbolStr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self.string, f) - } -} - -impl HashStable for SymbolStr { - #[inline] - fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { - self.string.hash_stable(hcx, hasher) - } -} - -impl ToStableHashKey for SymbolStr { - type KeyType = SymbolStr; - - #[inline] - fn to_stable_hash_key(&self, _: &CTX) -> SymbolStr { - self.clone() - } -} diff --git a/src/libsyntax_pos/symbol/tests.rs b/src/libsyntax_pos/symbol/tests.rs deleted file mode 100644 index f74b9a0cd1d..00000000000 --- a/src/libsyntax_pos/symbol/tests.rs +++ /dev/null @@ -1,25 +0,0 @@ -use super::*; - -use crate::{edition, Globals}; - -#[test] -fn interner_tests() { - let mut i: Interner = Interner::default(); - // first one is zero: - assert_eq!(i.intern("dog"), Symbol::new(0)); - // re-use gets the same entry: - assert_eq!(i.intern("dog"), Symbol::new(0)); - // different string gets a different #: - assert_eq!(i.intern("cat"), Symbol::new(1)); - assert_eq!(i.intern("cat"), Symbol::new(1)); - // dog is still at zero - assert_eq!(i.intern("dog"), Symbol::new(0)); -} - -#[test] -fn without_first_quote_test() { - GLOBALS.set(&Globals::new(edition::DEFAULT_EDITION), || { - let i = Ident::from_str("'break"); - assert_eq!(i.without_first_quote().name, kw::Break); - }); -} diff --git a/src/libsyntax_pos/tests.rs b/src/libsyntax_pos/tests.rs deleted file mode 100644 index 3c8eb8bcd31..00000000000 --- a/src/libsyntax_pos/tests.rs +++ /dev/null @@ -1,40 +0,0 @@ -use super::*; - -#[test] -fn test_lookup_line() { - let lines = &[BytePos(3), BytePos(17), BytePos(28)]; - - assert_eq!(lookup_line(lines, BytePos(0)), -1); - assert_eq!(lookup_line(lines, BytePos(3)), 0); - assert_eq!(lookup_line(lines, BytePos(4)), 0); - - assert_eq!(lookup_line(lines, BytePos(16)), 0); - assert_eq!(lookup_line(lines, BytePos(17)), 1); - assert_eq!(lookup_line(lines, BytePos(18)), 1); - - assert_eq!(lookup_line(lines, BytePos(28)), 2); - assert_eq!(lookup_line(lines, BytePos(29)), 2); -} - -#[test] -fn test_normalize_newlines() { - fn check(before: &str, after: &str, expected_positions: &[u32]) { - let mut actual = before.to_string(); - let mut actual_positions = vec![]; - normalize_newlines(&mut actual, &mut actual_positions); - let actual_positions: Vec<_> = actual_positions.into_iter().map(|nc| nc.pos.0).collect(); - assert_eq!(actual.as_str(), after); - assert_eq!(actual_positions, expected_positions); - } - check("", "", &[]); - check("\n", "\n", &[]); - check("\r", "\r", &[]); - check("\r\r", "\r\r", &[]); - check("\r\n", "\n", &[1]); - check("hello world", "hello world", &[]); - check("hello\nworld", "hello\nworld", &[]); - check("hello\r\nworld", "hello\nworld", &[6]); - check("\r\nhello\r\nworld\r\n", "\nhello\nworld\n", &[1, 7, 13]); - check("\r\r\n", "\r\n", &[2]); - check("hello\rworld", "hello\rworld", &[]); -} -- cgit 1.4.1-3-g733a5