Rollup merge of #115117 - pnkfelix:detect-and-report-nix-shell, r=albertlarsan68

Detect and report nix shell

Better diagnostics for people using nix subshell on non-NixOS.

1. Turned patch-binaries-for-nix from a boolean into a ternary flag: true, false, and unset.

2. When patch-binaries-for-nix is unset, we continue with the existing NixOS detection heuristic (look for nixos in /etc/os-release, if present), but if we are not atop NixOS, then issue a note if we see the IN_NIX_SHELL environment variable telling the user to consider setting patch-binaries-for-nix explicitly.

Fix #115073

3 files changed, 31 insertions, 9 deletions

diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py
index c5c70f2e18a..9dbc87c337c 100644
--- a/src/bootstrap/bootstrap.py
+++ b/src/bootstrap/bootstrap.py
@@ -623,7 +623,7 @@ class RustBuild(object):
 
     def should_fix_bins_and_dylibs(self):
         """Whether or not `fix_bin_or_dylib` needs to be run; can only be True
-        on NixOS.
+        on NixOS or if config.toml has `build.patch-binaries-for-nix` set.
         """
         if self._should_fix_bins_and_dylibs is not None:
             return self._should_fix_bins_and_dylibs
@@ -643,18 +643,31 @@ class RustBuild(object):
             if ostype != "Linux":
                 return False
 
-            # If the user has asked binaries to be patched for Nix, then
-            # don't check for NixOS.
+            # If the user has explicitly indicated whether binaries should be
+            # patched for Nix, then don't check for NixOS.
             if self.get_toml("patch-binaries-for-nix", "build") == "true":
                 return True
+            if self.get_toml("patch-binaries-for-nix", "build") == "false":
+                return False
 
             # Use `/etc/os-release` instead of `/etc/NIXOS`.
             # The latter one does not exist on NixOS when using tmpfs as root.
             try:
                 with open("/etc/os-release", "r") as f:
-                    return any(ln.strip() in ("ID=nixos", "ID='nixos'", 'ID="nixos"') for ln in f)
+                    is_nixos = any(ln.strip() in ("ID=nixos", "ID='nixos'", 'ID="nixos"')
+                                   for ln in f)
             except FileNotFoundError:
-                return False
+                is_nixos = False
+
+            # If not on NixOS, then warn if user seems to be atop Nix shell
+            if not is_nixos:
+                in_nix_shell = os.getenv('IN_NIX_SHELL')
+                if in_nix_shell:
+                    print("The IN_NIX_SHELL environment variable is `{}`;".format(in_nix_shell),
+                          "you may need to set `patch-binaries-for-nix=true` in config.toml",
+                          file=sys.stderr)
+
+            return is_nixos
 
         answer = self._should_fix_bins_and_dylibs = get_answer()
         if answer:
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index 762e66ac7cc..e5fdac3ceda 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -137,7 +137,7 @@ pub struct Config {
     pub json_output: bool,
     pub test_compare_mode: bool,
     pub color: Color,
-    pub patch_binaries_for_nix: bool,
+    pub patch_binaries_for_nix: Option<bool>,
     pub stage0_metadata: Stage0Metadata,
 
     pub stdout_is_tty: bool,
@@ -1339,7 +1339,7 @@ impl Config {
         set(&mut config.local_rebuild, build.local_rebuild);
         set(&mut config.print_step_timings, build.print_step_timings);
         set(&mut config.print_step_rusage, build.print_step_rusage);
-        set(&mut config.patch_binaries_for_nix, build.patch_binaries_for_nix);
+        config.patch_binaries_for_nix = build.patch_binaries_for_nix;
 
         config.verbose = cmp::max(config.verbose, flags.verbose as usize);
 
diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs
index 52162bf42ea..17c308e915b 100644
--- a/src/bootstrap/download.rs
+++ b/src/bootstrap/download.rs
@@ -91,8 +91,8 @@ impl Config {
             // NOTE: this intentionally comes after the Linux check:
             // - patchelf only works with ELF files, so no need to run it on Mac or Windows
             // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
-            if self.patch_binaries_for_nix {
-                return true;
+            if let Some(explicit_value) = self.patch_binaries_for_nix {
+                return explicit_value;
             }
 
             // Use `/etc/os-release` instead of `/etc/NIXOS`.
@@ -105,6 +105,15 @@ impl Config {
                     matches!(l.trim(), "ID=nixos" | "ID='nixos'" | "ID=\"nixos\"")
                 }),
             };
+            if !is_nixos {
+                let in_nix_shell = env::var("IN_NIX_SHELL");
+                if let Ok(in_nix_shell) = in_nix_shell {
+                    eprintln!(
+                        "The IN_NIX_SHELL environment variable is `{in_nix_shell}`; \
+                         you may need to set `patch-binaries-for-nix=true` in config.toml"
+                    );
+                }
+            }
             is_nixos
         });
         if val {