Auto merge of #63823 - petrochenkov:noapply2, r=matthewjasper

Audit uses of `apply_mark` in built-in macros + Remove default macro transparencies

Every use of `apply_mark` in a built-in or procedural macro is supposed to look like this
```rust
location.with_ctxt(SyntaxContext::root().apply_mark(ecx.current_expansion.id))
```
where `SyntaxContext::root()` means that the built-in/procedural macro is defined directly, rather than expanded from some other macro.

However, few people understood what `apply_mark` does, so we had a lot of copy-pasted uses of it looking e.g. like
```rust
span = span.apply_mark(ecx.current_expansion.id);
```
, which doesn't really make sense for procedural macros, but at the same time is not too harmful, if the macros use the traditional `macro_rules` hygiene.

So, to fight this, we stop using `apply_mark` directly in built-in macro implementations, and follow the example of regular proc macros instead and use analogues of `Span::def_site()` and `Span::call_site()`, which are much more intuitive and less error-prone.
- `ecx.with_def_site_ctxt(span)` takes the `span`'s location and combines it with a def-site context.
- `ecx.with_call_site_ctxt(span)` takes the `span`'s location and combines it with a call-site context.

Even if called multiple times (which sometimes happens due to some historical messiness of the built-in macro code) these functions will produce the same result, unlike `apply_mark` which will grow  the mark chain further in this case.

---

After `apply_mark`s in built-in macros are eliminated, the remaining `apply_mark`s are very few in number, so we can start passing the previously implicit `Transparency` argument to them explicitly, thus eliminating the need in `default_transparency` fields in hygiene structures and `#[rustc_macro_transparency]` annotations on built-in macros.

So, the task of making built-in macros opaque can now be formulated as "eliminate `with_legacy_ctxt` in favor of `with_def_site_ctxt`" rather than "replace `#[rustc_macro_transparency = "semitransparent"]` with `#[rustc_macro_transparency = "opaque"]`".

r? @matthewjasper

15 files changed, 23 insertions, 24 deletions

diff --git a/src/libsyntax_ext/asm.rs b/src/libsyntax_ext/asm.rs
index 644a44f1989..28f907441d8 100644
--- a/src/libsyntax_ext/asm.rs
+++ b/src/libsyntax_ext/asm.rs
@@ -63,7 +63,7 @@ pub fn expand_asm<'cx>(cx: &'cx mut ExtCtxt<'_>,
     MacEager::expr(P(ast::Expr {
         id: ast::DUMMY_NODE_ID,
         node: ast::ExprKind::InlineAsm(P(inline_asm)),
-        span: sp.with_ctxt(cx.backtrace()),
+        span: cx.with_legacy_ctxt(sp),
         attrs: ThinVec::new(),
     }))
 }
diff --git a/src/libsyntax_ext/assert.rs b/src/libsyntax_ext/assert.rs
index 6301283460a..84583d0e5ec 100644
--- a/src/libsyntax_ext/assert.rs
+++ b/src/libsyntax_ext/assert.rs
@@ -23,7 +23,7 @@ pub fn expand_assert<'cx>(
         }
     };
 
-    let sp = sp.apply_mark(cx.current_expansion.id);
+    let sp = cx.with_legacy_ctxt(sp);
     let panic_call = Mac {
         path: Path::from_ident(Ident::new(sym::panic, sp)),
         tts: custom_message.unwrap_or_else(|| {
diff --git a/src/libsyntax_ext/cfg.rs b/src/libsyntax_ext/cfg.rs
index 0e52c1af908..21cee8ae1cb 100644
--- a/src/libsyntax_ext/cfg.rs
+++ b/src/libsyntax_ext/cfg.rs
@@ -16,7 +16,7 @@ pub fn expand_cfg(
     sp: Span,
     tts: &[tokenstream::TokenTree],
 ) -> Box<dyn base::MacResult + 'static> {
-    let sp = sp.apply_mark(cx.current_expansion.id);
+    let sp = cx.with_legacy_ctxt(sp);
 
     match parse_cfg(cx, sp, tts) {
         Ok(cfg) => {
diff --git a/src/libsyntax_ext/concat.rs b/src/libsyntax_ext/concat.rs
index 4cd17531a45..ffa5154ca0c 100644
--- a/src/libsyntax_ext/concat.rs
+++ b/src/libsyntax_ext/concat.rs
@@ -59,6 +59,6 @@ pub fn expand_syntax_ext(
     } else if has_errors {
         return DummyResult::any(sp);
     }
-    let sp = sp.apply_mark(cx.current_expansion.id);
+    let sp = cx.with_legacy_ctxt(sp);
     base::MacEager::expr(cx.expr_str(sp, Symbol::intern(&accumulator)))
 }
diff --git a/src/libsyntax_ext/concat_idents.rs b/src/libsyntax_ext/concat_idents.rs
index 8184fc44267..96677072d1b 100644
--- a/src/libsyntax_ext/concat_idents.rs
+++ b/src/libsyntax_ext/concat_idents.rs
@@ -39,7 +39,7 @@ pub fn expand_syntax_ext<'cx>(cx: &'cx mut ExtCtxt<'_>,
         }
     }
 
-    let ident = ast::Ident::new(Symbol::intern(&res_str), sp.apply_mark(cx.current_expansion.id));
+    let ident = ast::Ident::new(Symbol::intern(&res_str), cx.with_legacy_ctxt(sp));
 
     struct ConcatIdentsResult { ident: ast::Ident }
 
diff --git a/src/libsyntax_ext/deriving/clone.rs b/src/libsyntax_ext/deriving/clone.rs
index d030ea4a56e..4dd0ecfebef 100644
--- a/src/libsyntax_ext/deriving/clone.rs
+++ b/src/libsyntax_ext/deriving/clone.rs
@@ -112,7 +112,7 @@ fn cs_clone_shallow(name: &str,
                         ty: P<ast::Ty>, span: Span, helper_name: &str) {
         // Generate statement `let _: helper_name<ty>;`,
         // set the expn ID so we can use the unstable struct.
-        let span = span.with_ctxt(cx.backtrace());
+        let span = cx.with_def_site_ctxt(span);
         let assert_path = cx.path_all(span, true,
                                         cx.std_path(&[sym::clone, Symbol::intern(helper_name)]),
                                         vec![GenericArg::Type(ty)], vec![]);
diff --git a/src/libsyntax_ext/deriving/cmp/eq.rs b/src/libsyntax_ext/deriving/cmp/eq.rs
index 54027c600b4..32ab47969ad 100644
--- a/src/libsyntax_ext/deriving/cmp/eq.rs
+++ b/src/libsyntax_ext/deriving/cmp/eq.rs
@@ -53,7 +53,7 @@ fn cs_total_eq_assert(cx: &mut ExtCtxt<'_>,
                         ty: P<ast::Ty>, span: Span, helper_name: &str) {
         // Generate statement `let _: helper_name<ty>;`,
         // set the expn ID so we can use the unstable struct.
-        let span = span.with_ctxt(cx.backtrace());
+        let span = cx.with_def_site_ctxt(span);
         let assert_path = cx.path_all(span, true,
                                         cx.std_path(&[sym::cmp, Symbol::intern(helper_name)]),
                                         vec![GenericArg::Type(ty)], vec![]);
diff --git a/src/libsyntax_ext/deriving/debug.rs b/src/libsyntax_ext/deriving/debug.rs
index 44153541048..781645a574e 100644
--- a/src/libsyntax_ext/deriving/debug.rs
+++ b/src/libsyntax_ext/deriving/debug.rs
@@ -60,7 +60,7 @@ fn show_substructure(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>
     };
 
     // We want to make sure we have the ctxt set so that we can use unstable methods
-    let span = span.with_ctxt(cx.backtrace());
+    let span = cx.with_def_site_ctxt(span);
     let name = cx.expr_lit(span, ast::LitKind::Str(ident.name, ast::StrStyle::Cooked));
     let builder = Ident::from_str_and_span("debug_trait_builder", span);
     let builder_expr = cx.expr_ident(span, builder.clone());
diff --git a/src/libsyntax_ext/deriving/generic/ty.rs b/src/libsyntax_ext/deriving/generic/ty.rs
index 7fcf036fc81..cb1c7b21fee 100644
--- a/src/libsyntax_ext/deriving/generic/ty.rs
+++ b/src/libsyntax_ext/deriving/generic/ty.rs
@@ -85,7 +85,7 @@ impl<'a> Path<'a> {
             PathKind::Global => cx.path_all(span, true, idents, params, Vec::new()),
             PathKind::Local => cx.path_all(span, false, idents, params, Vec::new()),
             PathKind::Std => {
-                let def_site = DUMMY_SP.apply_mark(cx.current_expansion.id);
+                let def_site = cx.with_def_site_ctxt(DUMMY_SP);
                 idents.insert(0, Ident::new(kw::DollarCrate, def_site));
                 cx.path_all(span, false, idents, params, Vec::new())
             }
diff --git a/src/libsyntax_ext/deriving/mod.rs b/src/libsyntax_ext/deriving/mod.rs
index da68eea0c50..60b6eba7a4b 100644
--- a/src/libsyntax_ext/deriving/mod.rs
+++ b/src/libsyntax_ext/deriving/mod.rs
@@ -48,6 +48,9 @@ impl MultiItemModifier for BuiltinDerive {
               meta_item: &MetaItem,
               item: Annotatable)
               -> Vec<Annotatable> {
+        // FIXME: Built-in derives often forget to give spans contexts,
+        // so we are doing it here in a centralized way.
+        let span = ecx.with_def_site_ctxt(span);
         let mut items = Vec::new();
         (self.0)(ecx, span, meta_item, &item, &mut |a| items.push(a));
         items
@@ -60,7 +63,7 @@ fn call_intrinsic(cx: &ExtCtxt<'_>,
                   intrinsic: &str,
                   args: Vec<P<ast::Expr>>)
                   -> P<ast::Expr> {
-    let span = span.with_ctxt(cx.backtrace());
+    let span = cx.with_def_site_ctxt(span);
     let path = cx.std_path(&[sym::intrinsics, Symbol::intern(intrinsic)]);
     let call = cx.expr_call_global(span, path, args);
 
diff --git a/src/libsyntax_ext/env.rs b/src/libsyntax_ext/env.rs
index 9834130fa23..6343d218de8 100644
--- a/src/libsyntax_ext/env.rs
+++ b/src/libsyntax_ext/env.rs
@@ -20,7 +20,7 @@ pub fn expand_option_env<'cx>(cx: &'cx mut ExtCtxt<'_>,
         Some(v) => v,
     };
 
-    let sp = sp.apply_mark(cx.current_expansion.id);
+    let sp = cx.with_legacy_ctxt(sp);
     let e = match env::var(&*var.as_str()) {
         Err(..) => {
             let lt = cx.lifetime(sp, Ident::with_dummy_span(kw::StaticLifetime));
diff --git a/src/libsyntax_ext/format.rs b/src/libsyntax_ext/format.rs
index 83764205a19..47394c02b41 100644
--- a/src/libsyntax_ext/format.rs
+++ b/src/libsyntax_ext/format.rs
@@ -12,7 +12,7 @@ use syntax::parse::token;
 use syntax::ptr::P;
 use syntax::symbol::{Symbol, sym};
 use syntax::tokenstream;
-use syntax_pos::{MultiSpan, Span, DUMMY_SP};
+use syntax_pos::{MultiSpan, Span};
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use std::borrow::Cow;
@@ -666,8 +666,7 @@ impl<'a, 'b> Context<'a, 'b> {
         // passed to this function.
         for (i, e) in self.args.into_iter().enumerate() {
             let name = names_pos[i];
-            let span =
-                DUMMY_SP.with_ctxt(e.span.ctxt().apply_mark(self.ecx.current_expansion.id));
+            let span = self.ecx.with_def_site_ctxt(e.span);
             pats.push(self.ecx.pat_ident(span, name));
             for ref arg_ty in self.arg_unique_types[i].iter() {
                 locals.push(Context::format_arg(self.ecx, self.macsp, e.span, arg_ty, name));
@@ -745,7 +744,7 @@ impl<'a, 'b> Context<'a, 'b> {
         ty: &ArgumentType,
         arg: ast::Ident,
     ) -> P<ast::Expr> {
-        sp = sp.apply_mark(ecx.current_expansion.id);
+        sp = ecx.with_def_site_ctxt(sp);
         let arg = ecx.expr_ident(sp, arg);
         let trait_ = match *ty {
             Placeholder(ref tyname) => {
@@ -798,7 +797,7 @@ fn expand_format_args_impl<'cx>(
     tts: &[tokenstream::TokenTree],
     nl: bool,
 ) -> Box<dyn base::MacResult + 'cx> {
-    sp = sp.apply_mark(ecx.current_expansion.id);
+    sp = ecx.with_def_site_ctxt(sp);
     match parse_args(ecx, sp, tts) {
         Ok((efmt, args, names)) => {
             MacEager::expr(expand_preparsed_format_args(ecx, sp, efmt, args, names, nl))
@@ -842,7 +841,7 @@ pub fn expand_preparsed_format_args(
     let arg_unique_types: Vec<_> = (0..args.len()).map(|_| Vec::new()).collect();
 
     let mut macsp = ecx.call_site();
-    macsp = macsp.with_ctxt(ecx.backtrace());
+    macsp = ecx.with_def_site_ctxt(macsp);
 
     let msg = "format argument must be a string literal";
     let fmt_sp = efmt.span;
diff --git a/src/libsyntax_ext/global_allocator.rs b/src/libsyntax_ext/global_allocator.rs
index d2121abe3b4..97b8087ad15 100644
--- a/src/libsyntax_ext/global_allocator.rs
+++ b/src/libsyntax_ext/global_allocator.rs
@@ -3,7 +3,6 @@ use syntax::ast::{self, Arg, Attribute, Expr, FnHeader, Generics, Ident};
 use syntax::attr::check_builtin_macro_attribute;
 use syntax::ext::allocator::{AllocatorKind, AllocatorMethod, AllocatorTy, ALLOCATOR_METHODS};
 use syntax::ext::base::{Annotatable, ExtCtxt};
-use syntax::ext::hygiene::SyntaxContext;
 use syntax::ptr::P;
 use syntax::symbol::{kw, sym, Symbol};
 use syntax_pos::Span;
@@ -29,7 +28,7 @@ pub fn expand(
     };
 
     // Generate a bunch of new items using the AllocFnFactory
-    let span = item.span.with_ctxt(SyntaxContext::root().apply_mark(ecx.current_expansion.id));
+    let span = ecx.with_legacy_ctxt(item.span);
     let f = AllocFnFactory {
         span,
         kind: AllocatorKind::Global,
diff --git a/src/libsyntax_ext/global_asm.rs b/src/libsyntax_ext/global_asm.rs
index 73ebeaec454..a8b61593db7 100644
--- a/src/libsyntax_ext/global_asm.rs
+++ b/src/libsyntax_ext/global_asm.rs
@@ -30,7 +30,7 @@ pub fn expand_global_asm<'cx>(cx: &'cx mut ExtCtxt<'_>,
                 id: ast::DUMMY_NODE_ID,
                 node: ast::ItemKind::GlobalAsm(P(global_asm)),
                 vis: respan(sp.shrink_to_lo(), ast::VisibilityKind::Inherited),
-                span: sp.with_ctxt(cx.backtrace()),
+                span: cx.with_legacy_ctxt(sp),
                 tokens: None,
             })])
         }
diff --git a/src/libsyntax_ext/test.rs b/src/libsyntax_ext/test.rs
index 08582e714cc..5fd87d3a0e5 100644
--- a/src/libsyntax_ext/test.rs
+++ b/src/libsyntax_ext/test.rs
@@ -4,7 +4,6 @@
 use syntax::ast;
 use syntax::attr::{self, check_builtin_macro_attribute};
 use syntax::ext::base::*;
-use syntax::ext::hygiene::SyntaxContext;
 use syntax::print::pprust;
 use syntax::source_map::respan;
 use syntax::symbol::{Symbol, sym};
@@ -29,7 +28,7 @@ pub fn expand_test_case(
 
     if !ecx.ecfg.should_test { return vec![]; }
 
-    let sp = attr_sp.with_ctxt(SyntaxContext::root().apply_mark(ecx.current_expansion.id));
+    let sp = ecx.with_legacy_ctxt(attr_sp);
     let mut item = anno_item.expect_item();
     item = item.map(|mut item| {
         item.vis = respan(item.vis.span, ast::VisibilityKind::Public);
@@ -93,8 +92,7 @@ pub fn expand_test_or_bench(
         return vec![Annotatable::Item(item)];
     }
 
-    let ctxt = SyntaxContext::root().apply_mark(cx.current_expansion.id);
-    let (sp, attr_sp) = (item.span.with_ctxt(ctxt), attr_sp.with_ctxt(ctxt));
+    let (sp, attr_sp) = (cx.with_legacy_ctxt(item.span), cx.with_legacy_ctxt(attr_sp));
 
     // Gensym "test" so we can extern crate without conflicting with any local names
     let test_id = cx.ident_of("test").gensym();