Rollup merge of #87404 - rylev:artifact-size-profiling, r=wesleywiser

Add support for artifact size profiling

This adds support for profiling artifact file sizes (incremental compilation artifacts and query cache to begin with).

Eventually we want to track this in perf.rlo so we can ensure that file sizes do not change dramatically on each pull request.

This relies on support in measureme: https://github.com/rust-lang/measureme/pull/169. Once that lands we can update this PR to not point to a git dependency.

This was worked on together with `@michaelwoerister.`

r? `@wesleywiser`

8 files changed, 76 insertions, 15 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 7ea3a6fd9c3..aaa709e50f6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2144,6 +2144,20 @@ dependencies = [
 ]
 
 [[package]]
+name = "measureme"
+version = "10.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd460fad6e55ca82fa0cd9dab0d315294188fd9ec6efbf4105e5635d4872ef9c"
+dependencies = [
+ "log",
+ "memmap2",
+ "parking_lot",
+ "perf-event-open-sys",
+ "rustc-hash",
+ "smallvec",
+]
+
+[[package]]
 name = "memchr"
 version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2247,7 +2261,7 @@ dependencies = [
  "hex 0.4.2",
  "libc",
  "log",
- "measureme",
+ "measureme 9.1.2",
  "rand 0.8.4",
  "rustc-workspace-hack",
  "rustc_version 0.4.0",
@@ -3235,7 +3249,7 @@ dependencies = [
  "indexmap",
  "jobserver",
  "libc",
- "measureme",
+ "measureme 9.1.2",
  "memmap2",
  "parking_lot",
  "rustc-ap-rustc_graphviz",
@@ -3674,7 +3688,7 @@ dependencies = [
  "bitflags",
  "cstr",
  "libc",
- "measureme",
+ "measureme 10.0.0",
  "rustc-demangle",
  "rustc_arena",
  "rustc_ast",
@@ -3767,7 +3781,7 @@ dependencies = [
  "indexmap",
  "jobserver",
  "libc",
- "measureme",
+ "measureme 10.0.0",
  "memmap2",
  "parking_lot",
  "rustc-hash",
@@ -4292,7 +4306,7 @@ dependencies = [
 name = "rustc_query_impl"
 version = "0.0.0"
 dependencies = [
- "measureme",
+ "measureme 10.0.0",
  "rustc-rayon-core",
  "rustc_ast",
  "rustc_data_structures",
diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml
index a6a553b31a3..5f3f5334475 100644
--- a/compiler/rustc_codegen_llvm/Cargo.toml
+++ b/compiler/rustc_codegen_llvm/Cargo.toml
@@ -11,7 +11,7 @@ doctest = false
 bitflags = "1.0"
 cstr = "0.2"
 libc = "0.2"
-measureme = "9.1.0"
+measureme = "10.0.0"
 snap = "1"
 tracing = "0.1"
 rustc_middle = { path = "../rustc_middle" }
diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
index 49962570129..e3395df3590 100644
--- a/compiler/rustc_data_structures/Cargo.toml
+++ b/compiler/rustc_data_structures/Cargo.toml
@@ -23,7 +23,7 @@ rustc-hash = "1.1.0"
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
 rustc_index = { path = "../rustc_index", package = "rustc_index" }
 bitflags = "1.2.1"
-measureme = "9.1.0"
+measureme = "10.0.0"
 libc = "0.2"
 stacker = "0.1.14"
 tempfile = "3.2"
diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs
index 0bbd0eda0c6..c21939209fc 100644
--- a/compiler/rustc_data_structures/src/profiling.rs
+++ b/compiler/rustc_data_structures/src/profiling.rs
@@ -110,12 +110,14 @@ bitflags::bitflags! {
         const FUNCTION_ARGS       = 1 << 6;
         const LLVM                = 1 << 7;
         const INCR_RESULT_HASHING = 1 << 8;
+        const ARTIFACT_SIZES = 1 << 9;
 
         const DEFAULT = Self::GENERIC_ACTIVITIES.bits |
                         Self::QUERY_PROVIDERS.bits |
                         Self::QUERY_BLOCKED.bits |
                         Self::INCR_CACHE_LOADS.bits |
-                        Self::INCR_RESULT_HASHING.bits;
+                        Self::INCR_RESULT_HASHING.bits |
+                        Self::ARTIFACT_SIZES.bits;
 
         const ARGS = Self::QUERY_KEYS.bits | Self::FUNCTION_ARGS.bits;
     }
@@ -136,6 +138,7 @@ const EVENT_FILTERS_BY_NAME: &[(&str, EventFilter)] = &[
     ("args", EventFilter::ARGS),
     ("llvm", EventFilter::LLVM),
     ("incr-result-hashing", EventFilter::INCR_RESULT_HASHING),
+    ("artifact-sizes", EventFilter::ARTIFACT_SIZES),
 ];
 
 /// Something that uniquely identifies a query invocation.
@@ -285,6 +288,33 @@ impl SelfProfilerRef {
         })
     }
 
+    /// Record the size of an artifact that the compiler produces
+    ///
+    /// `artifact_kind` is the class of artifact (e.g., query_cache, object_file, etc.)
+    /// `artifact_name` is an identifier to the specific artifact being stored (usually a filename)
+    #[inline(always)]
+    pub fn artifact_size<A>(&self, artifact_kind: &str, artifact_name: A, size: u64)
+    where
+        A: Borrow<str> + Into<String>,
+    {
+        drop(self.exec(EventFilter::ARTIFACT_SIZES, |profiler| {
+            let builder = EventIdBuilder::new(&profiler.profiler);
+            let event_label = profiler.get_or_alloc_cached_string(artifact_kind);
+            let event_arg = profiler.get_or_alloc_cached_string(artifact_name);
+            let event_id = builder.from_label_and_arg(event_label, event_arg);
+            let thread_id = get_thread_id();
+
+            profiler.profiler.record_integer_event(
+                profiler.artifact_size_event_kind,
+                event_id,
+                thread_id,
+                size,
+            );
+
+            TimingGuard::none()
+        }))
+    }
+
     #[inline(always)]
     pub fn generic_activity_with_args(
         &self,
@@ -372,7 +402,7 @@ impl SelfProfilerRef {
     ) {
         drop(self.exec(event_filter, |profiler| {
             let event_id = StringId::new_virtual(query_invocation_id.0);
-            let thread_id = std::thread::current().id().as_u64().get() as u32;
+            let thread_id = get_thread_id();
 
             profiler.profiler.record_instant_event(
                 event_kind(profiler),
@@ -425,6 +455,7 @@ pub struct SelfProfiler {
     incremental_result_hashing_event_kind: StringId,
     query_blocked_event_kind: StringId,
     query_cache_hit_event_kind: StringId,
+    artifact_size_event_kind: StringId,
 }
 
 impl SelfProfiler {
@@ -447,6 +478,7 @@ impl SelfProfiler {
             profiler.alloc_string("IncrementalResultHashing");
         let query_blocked_event_kind = profiler.alloc_string("QueryBlocked");
         let query_cache_hit_event_kind = profiler.alloc_string("QueryCacheHit");
+        let artifact_size_event_kind = profiler.alloc_string("ArtifactSize");
 
         let mut event_filter_mask = EventFilter::empty();
 
@@ -491,6 +523,7 @@ impl SelfProfiler {
             incremental_result_hashing_event_kind,
             query_blocked_event_kind,
             query_cache_hit_event_kind,
+            artifact_size_event_kind,
         })
     }
 
@@ -561,7 +594,7 @@ impl<'a> TimingGuard<'a> {
         event_kind: StringId,
         event_id: EventId,
     ) -> TimingGuard<'a> {
-        let thread_id = std::thread::current().id().as_u64().get() as u32;
+        let thread_id = get_thread_id();
         let raw_profiler = &profiler.profiler;
         let timing_guard =
             raw_profiler.start_recording_interval_event(event_kind, event_id, thread_id);
@@ -655,6 +688,10 @@ pub fn duration_to_secs_str(dur: std::time::Duration) -> String {
     format!("{:.3}", dur.as_secs_f64())
 }
 
+fn get_thread_id() -> u32 {
+    std::thread::current().id().as_u64().get() as u32
+}
+
 // Memory reporting
 cfg_if! {
     if #[cfg(windows)] {
diff --git a/compiler/rustc_incremental/src/persist/file_format.rs b/compiler/rustc_incremental/src/persist/file_format.rs
index 572a4fc6971..392c5bdc15a 100644
--- a/compiler/rustc_incremental/src/persist/file_format.rs
+++ b/compiler/rustc_incremental/src/persist/file_format.rs
@@ -95,6 +95,12 @@ where
         return;
     }
 
+    sess.prof.artifact_size(
+        &name.replace(' ', "_"),
+        path_buf.file_name().unwrap().to_string_lossy(),
+        encoder.position() as u64,
+    );
+
     debug!("save: data written to disk successfully");
 }
 
diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml
index 89df3d4674b..81458156389 100644
--- a/compiler/rustc_query_impl/Cargo.toml
+++ b/compiler/rustc_query_impl/Cargo.toml
@@ -7,7 +7,7 @@ edition = "2021"
 doctest = false
 
 [dependencies]
-measureme = "9.0.0"
+measureme = "10.0.0"
 rustc-rayon-core = "0.3.1"
 tracing = "0.1"
 rustc_ast = { path = "../rustc_ast" }
diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs
index f5f67fcd0a0..47197a1e492 100644
--- a/compiler/rustc_query_system/src/dep_graph/serialized.rs
+++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs
@@ -222,7 +222,7 @@ impl<K: DepKind> EncoderState<K> {
         index
     }
 
-    fn finish(self) -> FileEncodeResult {
+    fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult {
         let Self { mut encoder, total_node_count, total_edge_count, result, stats: _ } = self;
         let () = result?;
 
@@ -235,7 +235,11 @@ impl<K: DepKind> EncoderState<K> {
         IntEncodedWithFixedSize(edge_count).encode(&mut encoder)?;
         debug!("position: {:?}", encoder.position());
         // Drop the encoder so that nothing is written after the counts.
-        encoder.flush()
+        let result = encoder.flush();
+        // FIXME(rylev): we hardcode the dep graph file name so we don't need a dependency on
+        // rustc_incremental just for that.
+        profiler.artifact_size("dep_graph", "dep-graph.bin", encoder.position() as u64);
+        result
     }
 }
 
@@ -332,6 +336,6 @@ impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
 
     pub fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult {
         let _prof_timer = profiler.generic_activity("incr_comp_encode_dep_graph");
-        self.status.into_inner().finish()
+        self.status.into_inner().finish(profiler)
     }
 }
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index b3d36b396c5..059b5dc5175 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -1283,7 +1283,7 @@ options! {
         "specify the events recorded by the self profiler;
         for example: `-Z self-profile-events=default,query-keys`
         all options: none, all, default, generic-activity, query-provider, query-cache-hit
-                     query-blocked, incr-cache-load, incr-result-hashing, query-keys, function-args, args, llvm"),
+                     query-blocked, incr-cache-load, incr-result-hashing, query-keys, function-args, args, llvm, artifact-sizes"),
     share_generics: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "make the current crate share its generic instantiations"),
     show_span: Option<String> = (None, parse_opt_string, [TRACKED],