Auto merge of #97137 - Kobzol:ci-llvm-pgo-pid, r=Mark-Simulacrum

Add PID to LLVM PGO profile path

This is a continuation of https://github.com/rust-lang/rust/pull/97110, which adds PID to the filename pattern of LLVM profiles. It also adds some metrics to the pgo.sh script, so that we can observe how many profiles there are and how large are they.

r? `@lqd`

2 files changed, 39 insertions, 10 deletions

diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs
index a41079333b1..79f2338b7ab 100644
--- a/src/bootstrap/native.rs
+++ b/src/bootstrap/native.rs
@@ -300,6 +300,9 @@ impl Step for Llvm {
 
         if builder.config.llvm_profile_generate {
             cfg.define("LLVM_BUILD_INSTRUMENTED", "IR");
+            if let Ok(llvm_profile_dir) = std::env::var("LLVM_PROFILE_DIR") {
+                cfg.define("LLVM_PROFILE_DATA_DIR", llvm_profile_dir);
+            }
             cfg.define("LLVM_BUILD_RUNTIME", "No");
         }
         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
diff --git a/src/ci/pgo.sh b/src/ci/pgo.sh
index 9b727947f6a..9de970c9c2a 100755
--- a/src/ci/pgo.sh
+++ b/src/ci/pgo.sh
@@ -39,12 +39,19 @@ gather_profiles () {
 
 rm -rf /tmp/rustc-pgo
 
+# This path has to be absolute
+LLVM_PROFILE_DIRECTORY_ROOT=/tmp/llvm-pgo
+
 # We collect LLVM profiling information and rustc profiling information in
 # separate phases. This increases build time -- though not by a huge amount --
 # but prevents any problems from arising due to different profiling runtimes
 # being simultaneously linked in.
-
-python3 ../x.py build --target=$PGO_HOST --host=$PGO_HOST \
+# LLVM IR PGO does not respect LLVM_PROFILE_FILE, so we have to set the profiling file
+# path through our custom environment variable. We include the PID in the directory path
+# to avoid updates to profile files being lost because of race conditions.
+LLVM_PROFILE_DIR=${LLVM_PROFILE_DIRECTORY_ROOT}/prof-%p python3 ../x.py build \
+    --target=$PGO_HOST \
+    --host=$PGO_HOST \
     --stage 2 library/std \
     --llvm-profile-generate
 
@@ -64,11 +71,18 @@ RUSTC_BOOTSTRAP=1 \
 gather_profiles "Debug,Opt" "Full" \
 "syn-1.0.89,cargo-0.60.0,serde-1.0.136,ripgrep-13.0.0,regex-1.5.5,clap-3.1.6,hyper-0.14.18"
 
+LLVM_PROFILE_MERGED_FILE=/tmp/llvm-pgo.profdata
+
 # Merge the profile data we gathered for LLVM
 # Note that this uses the profdata from the clang we used to build LLVM,
 # which likely has a different version than our in-tree clang.
-/rustroot/bin/llvm-profdata \
-    merge -o /tmp/llvm-pgo.profdata ./build/$PGO_HOST/llvm/build/profiles
+/rustroot/bin/llvm-profdata merge -o ${LLVM_PROFILE_MERGED_FILE} ${LLVM_PROFILE_DIRECTORY_ROOT}
+
+echo "LLVM PGO statistics"
+du -sh ${LLVM_PROFILE_MERGED_FILE}
+du -sh ${LLVM_PROFILE_DIRECTORY_ROOT}
+echo "Profile file count"
+find ${LLVM_PROFILE_DIRECTORY_ROOT} -type f | wc -l
 
 # Rustbuild currently doesn't support rebuilding LLVM when PGO options
 # change (or any other llvm-related options); so just clear out the relevant
@@ -77,9 +91,12 @@ rm -r ./build/$PGO_HOST/llvm ./build/$PGO_HOST/lld
 
 # Okay, LLVM profiling is done, switch to rustc PGO.
 
+# The path has to be absolute
+RUSTC_PROFILE_DIRECTORY_ROOT=/tmp/rustc-pgo
+
 python3 ../x.py build --target=$PGO_HOST --host=$PGO_HOST \
     --stage 2 library/std \
-    --rust-profile-generate=/tmp/rustc-pgo
+    --rust-profile-generate=${RUSTC_PROFILE_DIRECTORY_ROOT}
 
 # Here we're profiling the `rustc` frontend, so we also include `Check`.
 # The benchmark set includes various stress tests that put the frontend under pressure.
@@ -87,12 +104,21 @@ python3 ../x.py build --target=$PGO_HOST --host=$PGO_HOST \
 # rustc invocation ends. Empirically, this can result in some profiling data being lost.
 # That's why we override the profile path to include the PID. This will produce many more profiling
 # files, but the resulting profile will produce a slightly faster rustc binary.
-LLVM_PROFILE_FILE=/tmp/rustc-pgo/default_%m_%p.profraw gather_profiles "Check,Debug,Opt" "All" \
-"externs,ctfe-stress-5,cargo-0.60.0,token-stream-stress,match-stress,tuple-stress,diesel-1.4.8,bitmaps-3.1.0"
+LLVM_PROFILE_FILE=${RUSTC_PROFILE_DIRECTORY_ROOT}/default_%m_%p.profraw gather_profiles \
+  "Check,Debug,Opt" "All" \
+  "externs,ctfe-stress-5,cargo-0.60.0,token-stream-stress,match-stress,tuple-stress,diesel-1.4.8,bitmaps-3.1.0"
+
+RUSTC_PROFILE_MERGED_FILE=/tmp/rustc-pgo.profdata
 
 # Merge the profile data we gathered
 ./build/$PGO_HOST/llvm/bin/llvm-profdata \
-    merge -o /tmp/rustc-pgo.profdata /tmp/rustc-pgo
+    merge -o ${RUSTC_PROFILE_MERGED_FILE} ${RUSTC_PROFILE_DIRECTORY_ROOT}
+
+echo "Rustc PGO statistics"
+du -sh ${RUSTC_PROFILE_MERGED_FILE}
+du -sh ${RUSTC_PROFILE_DIRECTORY_ROOT}
+echo "Profile file count"
+find ${RUSTC_PROFILE_DIRECTORY_ROOT} -type f | wc -l
 
 # Rustbuild currently doesn't support rebuilding LLVM when PGO options
 # change (or any other llvm-related options); so just clear out the relevant
@@ -102,5 +128,5 @@ rm -r ./build/$PGO_HOST/llvm ./build/$PGO_HOST/lld
 # This produces the actual final set of artifacts, using both the LLVM and rustc
 # collected profiling data.
 $@ \
-    --rust-profile-use=/tmp/rustc-pgo.profdata \
-    --llvm-profile-use=/tmp/llvm-pgo.profdata
+    --rust-profile-use=${RUSTC_PROFILE_MERGED_FILE} \
+    --llvm-profile-use=${LLVM_PROFILE_MERGED_FILE}