From bcca03cce7da9dc4125aa34943041cb65e0fd4bb Mon Sep 17 00:00:00 2001
From: Ivan Molodetskikh <yalterz@gmail.com>
Date: Thu, 23 May 2024 09:54:24 +0400
Subject: Increase RLIMIT_NOFILE to maximum

Fixes Xwayland + RustRover crashing.

See similar changes:
* https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/2235
* https://github.com/swaywm/sway/pull/6629
---
 src/main.rs           |  5 ++++-
 src/utils/spawning.rs | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 54 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/main.rs b/src/main.rs
index b5576e89..fb399f3a 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -18,7 +18,8 @@ use niri::dbus;
 use niri::ipc::client::handle_msg;
 use niri::niri::State;
 use niri::utils::spawning::{
-    spawn, CHILD_ENV, REMOVE_ENV_RUST_BACKTRACE, REMOVE_ENV_RUST_LIB_BACKTRACE,
+    spawn, store_and_increase_nofile_rlimit, CHILD_ENV, REMOVE_ENV_RUST_BACKTRACE,
+    REMOVE_ENV_RUST_LIB_BACKTRACE,
 };
 use niri::utils::watcher::Watcher;
 use niri::utils::{cause_panic, version, IS_SYSTEMD_SERVICE};
@@ -173,6 +174,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let spawn_at_startup = mem::take(&mut config.spawn_at_startup);
     *CHILD_ENV.write().unwrap() = mem::take(&mut config.environment);
 
+    store_and_increase_nofile_rlimit();
+
     // Create the compositor.
     let mut event_loop = EventLoop::try_new().unwrap();
     let display = Display::new().unwrap();
diff --git a/src/utils/spawning.rs b/src/utils/spawning.rs
index ad23a784..9a274ff3 100644
--- a/src/utils/spawning.rs
+++ b/src/utils/spawning.rs
@@ -2,10 +2,11 @@ use std::ffi::OsStr;
 use std::os::unix::process::CommandExt;
 use std::path::Path;
 use std::process::{Child, Command, Stdio};
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::RwLock;
 use std::{io, thread};
 
+use libc::{getrlimit, rlimit, setrlimit, RLIMIT_NOFILE};
 use niri_config::Environment;
 
 use crate::utils::expand_home;
@@ -14,6 +15,50 @@ pub static REMOVE_ENV_RUST_BACKTRACE: AtomicBool = AtomicBool::new(false);
 pub static REMOVE_ENV_RUST_LIB_BACKTRACE: AtomicBool = AtomicBool::new(false);
 pub static CHILD_ENV: RwLock<Environment> = RwLock::new(Environment(Vec::new()));
 
+static ORIGINAL_NOFILE_RLIMIT_CUR: AtomicU64 = AtomicU64::new(0);
+static ORIGINAL_NOFILE_RLIMIT_MAX: AtomicU64 = AtomicU64::new(0);
+
+/// Increases the nofile rlimit to the maximum and stores the original value.
+pub fn store_and_increase_nofile_rlimit() {
+    let mut rlim = rlimit {
+        rlim_cur: 0,
+        rlim_max: 0,
+    };
+    if unsafe { getrlimit(RLIMIT_NOFILE, &mut rlim) } != 0 {
+        let err = io::Error::last_os_error();
+        warn!("error getting nofile rlimit: {err:?}");
+        return;
+    }
+
+    ORIGINAL_NOFILE_RLIMIT_CUR.store(rlim.rlim_cur, Ordering::SeqCst);
+    ORIGINAL_NOFILE_RLIMIT_MAX.store(rlim.rlim_max, Ordering::SeqCst);
+
+    trace!(
+        "changing nofile rlimit from {} to {}",
+        rlim.rlim_cur,
+        rlim.rlim_max
+    );
+    rlim.rlim_cur = rlim.rlim_max;
+
+    if unsafe { setrlimit(RLIMIT_NOFILE, &rlim) } != 0 {
+        let err = io::Error::last_os_error();
+        warn!("error setting nofile rlimit: {err:?}");
+    }
+}
+
+/// Restores the original nofile rlimit.
+pub fn restore_nofile_rlimit() {
+    let rlim_cur = ORIGINAL_NOFILE_RLIMIT_CUR.load(Ordering::SeqCst);
+    let rlim_max = ORIGINAL_NOFILE_RLIMIT_MAX.load(Ordering::SeqCst);
+
+    if rlim_cur == 0 {
+        return;
+    }
+
+    let rlim = rlimit { rlim_cur, rlim_max };
+    unsafe { setrlimit(RLIMIT_NOFILE, &rlim) };
+}
+
 /// Spawns the command to run independently of the compositor.
 pub fn spawn<T: AsRef<OsStr> + Send + 'static>(command: Vec<T>) {
     let _span = tracy_client::span!();
@@ -103,6 +148,8 @@ fn do_spawn(command: &OsStr, mut process: Command) -> Option<Child> {
                 _ => libc::_exit(0),
             }
 
+            restore_nofile_rlimit();
+
             Ok(())
         });
     }
@@ -214,6 +261,8 @@ mod systemd {
                     }
                 }
 
+                restore_nofile_rlimit();
+
                 Ok(())
             });
         }
-- 
cgit