Rollup merge of #53229 - varkor:rlimits_min, r=nikomatsakis

Make sure rlimit is only ever increased

`libc::setrlimit` will fail if we try to set the rlimit to a value lower than it is currently, so make sure we're never trying to do this. Fixes #52801.

2 files changed, 12 insertions, 8 deletions

diff --git a/src/librustc_driver/lib.rs b/src/librustc_driver/lib.rs
index 81267f7293c..d25d57a004b 100644
--- a/src/librustc_driver/lib.rs
+++ b/src/librustc_driver/lib.rs
@@ -1512,7 +1512,7 @@ pub fn in_named_rustc_thread<F, R>(name: String, f: F) -> Result<R, Box<dyn Any
             true
         } else if rlim.rlim_max < STACK_SIZE as libc::rlim_t {
             true
-        } else {
+        } else if rlim.rlim_cur < STACK_SIZE as libc::rlim_t {
             std::rt::deinit_stack_guard();
             rlim.rlim_cur = STACK_SIZE as libc::rlim_t;
             if libc::setrlimit(libc::RLIMIT_STACK, &mut rlim) != 0 {
@@ -1524,6 +1524,8 @@ pub fn in_named_rustc_thread<F, R>(name: String, f: F) -> Result<R, Box<dyn Any
                 std::rt::update_stack_guard();
                 false
             }
+        } else {
+            false
         }
     };
 
diff --git a/src/tools/compiletest/src/raise_fd_limit.rs b/src/tools/compiletest/src/raise_fd_limit.rs
index 220082799a8..d1071231530 100644
--- a/src/tools/compiletest/src/raise_fd_limit.rs
+++ b/src/tools/compiletest/src/raise_fd_limit.rs
@@ -57,14 +57,16 @@ pub unsafe fn raise_fd_limit() {
         panic!("raise_fd_limit: error calling getrlimit: {}", err);
     }
 
-    // Bump the soft limit to the smaller of kern.maxfilesperproc and the hard
-    // limit
-    rlim.rlim_cur = cmp::min(maxfiles as libc::rlim_t, rlim.rlim_max);
+    // Make sure we're only ever going to increase the rlimit.
+    if rlim.rlim_cur < maxfiles as libc::rlim_t {
+        // Bump the soft limit to the smaller of kern.maxfilesperproc and the hard limit.
+        rlim.rlim_cur = cmp::min(maxfiles as libc::rlim_t, rlim.rlim_max);
 
-    // Set our newly-increased resource limit
-    if libc::setrlimit(libc::RLIMIT_NOFILE, &rlim) != 0 {
-        let err = io::Error::last_os_error();
-        panic!("raise_fd_limit: error calling setrlimit: {}", err);
+        // Set our newly-increased resource limit.
+        if libc::setrlimit(libc::RLIMIT_NOFILE, &rlim) != 0 {
+            let err = io::Error::last_os_error();
+            panic!("raise_fd_limit: error calling setrlimit: {}", err);
+        }
     }
 }