feat(guest): raise spawned-process RLIMIT_NOFILE with a sane default and env override

.github/workflows/kvm-test.yaml

@@ -375,6 +375,17 @@ jobs:
           cp target/x86_64-unknown-linux-musl/release/sandbox-agent /tmp/mitos-test/agent
           ls -lh /tmp/mitos-test/agent
 
+      - name: Guest agent RLIMIT_NOFILE tests (issue #786)
+        run: |
+          # Run the sys::rlimit suite on this Linux host: the pure parse/resolve
+          # unit tests plus the Linux-gated integration test that spawns a child
+          # and asserts its soft RLIMIT_NOFILE is actually raised. Scoped to the
+          # module so it stays deterministic and does not require the booted-VM
+          # or Python-kernel fixtures the other guest tests need.
+          source "$HOME/.cargo/env" 2>/dev/null || true
+          cd guest/agent-rs
+          cargo test --lib sys::rlimit
+
       # OCI image -> rootfs pipeline proof (issue #10). This is the gate that
       # proves Engine.CreateTemplate builds a bootable template FROM A REAL OCI
       # IMAGE, not a hand-built rootfs: pull busybox:stable, flatten its layers,

docs/templates.md

@@ -157,8 +157,42 @@ E2B-style fast-cached-build behavior. The key computation and the skip decision
 are pure and unit-tested on any host; the actual layer reuse on a live boot is
 KVM gated and asserted in the Firecracker suite.
 
+## Guest process resource limits
+
+The guest agent is PID 1 inside the microVM. A bare Linux init inherits a low
+soft `RLIMIT_NOFILE` (commonly 1024 open files), and every process the agent
+spawns for `exec`, PTY sessions, `run_code`, and serving workloads would inherit
+that same low limit. Data libraries that open many files at import time (pandas,
+openpyxl, scikit) hit `EMFILE` ("too many open files") under 1024, so a plain
+`import pandas` could die inside a sandbox even when the host-side caps (husk pod
+cgroup, per-sandbox stream caps) are nowhere near saturated. Those host caps do
+not govern in-guest per-process rlimits; only `setrlimit` inside the guest does.
+
+The agent therefore raises the soft `RLIMIT_NOFILE` of every process it spawns to
+a sane default of 65536, clamped to the inherited hard limit and never lowered
+below what the process already has. The raise is installed via a `pre_exec` hook
+that runs in the forked child before `execve`, so it applies to the whole child
+tree.
+
+To override the default, set the `MITOS_RLIMIT_NOFILE` environment variable in
+the guest agent's environment to a decimal file count of at least 64. It is read
+once per spawn from the agent's own environment, so it applies uniformly to every
+spawned process. A value below 64, or an unparseable value, is ignored with a
+warning and the default applies; the floor stops a `0` or tiny typo from silently
+crippling every spawned child. An operator may set a value below the default (but
+at or above the floor) deliberately; the agent never raises the hard limit (that
+needs `CAP_SYS_RESOURCE` and is out of scope).
+
+The default and the override resolution are unit-tested on any host; the
+end-to-end proof (a spawned guest process observing the raised limit and opening
+more than 1024 files) is KVM gated and runs in the Firecracker suite.
+
 ## Open follow-ups
 
+- A first-class `template.spec` field (for example `resources.rlimits.nofile`)
+  plumbed through CreateTemplate and Fork into the guest agent's
+  `MITOS_RLIMIT_NOFILE` boot environment, validated at the boundary, so operators
+  set the limit declaratively instead of via the raw environment variable.
 - `go:embed` the guest agent into the forkd binary so no external `--agent-bin`
   path is needed.
 - OCI layer caching tied to the CAS store so repeated pool builds do not

guest/agent-rs/src/kernel/driver.rs

@@ -414,20 +414,31 @@ impl KernelManager {
             ));
         }
 
-        let mut cmd = tokio::process::Command::new(&self.cfg.python);
-        cmd.arg(driver_path);
-        cmd.stdin(Stdio::piped());
-        cmd.stdout(Stdio::piped());
+        // Built as a std::process::Command so the RLIMIT_NOFILE pre_exec hook can
+        // be registered (all unsafe stays in sys/), then converted to a tokio
+        // Command for kill_on_drop and async spawn.
+        let mut std_cmd = std::process::Command::new(&self.cfg.python);
+        std_cmd.arg(driver_path);
+        std_cmd.stdin(Stdio::piped());
+        std_cmd.stdout(Stdio::piped());
         // Route driver stderr to this process's stderr (ipykernel debug noise).
-        cmd.stderr(Stdio::inherit());
-        // Kill the driver when its Child handle is dropped (agent shutdown).
-        cmd.kill_on_drop(true);
+        std_cmd.stderr(Stdio::inherit());
 
         // Set working directory to /workspace when it exists (mirrors Go).
         if std::path::Path::new("/workspace").is_dir() {
-            cmd.current_dir("/workspace");
+            std_cmd.current_dir("/workspace");
         }
 
+        // Raise the kernel's soft RLIMIT_NOFILE to a sane default (issue #786) so
+        // `import pandas` / openpyxl / scikit do not die with EMFILE under the
+        // 1024-fd limit a bare init inherits. See sys/rlimit.rs for the SAFETY
+        // contract of the pre_exec hook.
+        crate::sys::rlimit::apply_nofile_limit(&mut std_cmd);
+
+        let mut cmd = tokio::process::Command::from(std_cmd);
+        // Kill the driver when its Child handle is dropped (agent shutdown).
+        cmd.kill_on_drop(true);
+
         let mut child = cmd.spawn().map_err(|e| {
             format!(
                 "kernel unavailable: start kernel driver: {e}; rebuild the base image \

guest/agent-rs/src/service/exec.rs

@@ -34,6 +34,7 @@
 
 use std::collections::HashMap;
 use std::os::fd::AsRawFd;
+use std::os::unix::process::CommandExt as _;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
@@ -220,7 +221,11 @@ async fn exec_shell_nostdin(
     let configured = env.snapshot().await;
     let merged_env = build_merged_env(&configured, &req_env);
 
-    let mut child = tokio::process::Command::new("/bin/sh")
+    // Built as a std::process::Command so the RLIMIT_NOFILE pre_exec hook can be
+    // registered (all unsafe stays in sys/), then converted to a tokio Command
+    // to spawn. process_group(0) is equivalent to Setpgid=true.
+    let mut std_cmd = std::process::Command::new("/bin/sh");
+    std_cmd
         .arg("-c")
         .arg(&command)
         .current_dir(&cwd)
@@ -230,7 +235,10 @@ async fn exec_shell_nostdin(
         .stdin(std::process::Stdio::null())
         .stdout(std::process::Stdio::piped())
         .stderr(std::process::Stdio::piped())
-        .process_group(0)
+        .process_group(0);
+    // Raise the child's soft RLIMIT_NOFILE to a sane default (issue #786).
+    crate::sys::rlimit::apply_nofile_limit(&mut std_cmd);
+    let mut child = tokio::process::Command::from(std_cmd)
         .spawn()
         .map_err(|e| {
             let _ = tx.try_send(Ok(exit_frame(1, 0.0, format!("start: {e}"))));
@@ -373,7 +381,10 @@ async fn exec_shell(
 
     // Spawn /bin/sh -c <command> in its own process group so SIGKILL kills
     // the whole child tree. process_group(0) is equivalent to Setpgid=true.
-    let mut child = tokio::process::Command::new("/bin/sh")
+    // Built as a std::process::Command so the RLIMIT_NOFILE pre_exec hook can be
+    // registered (all unsafe stays in sys/), then converted to a tokio Command.
+    let mut std_cmd = std::process::Command::new("/bin/sh");
+    std_cmd
         .arg("-c")
         .arg(&open.command)
         .current_dir(&cwd)
@@ -383,7 +394,11 @@ async fn exec_shell(
         .stdin(std::process::Stdio::piped())
         .stdout(std::process::Stdio::piped())
         .stderr(std::process::Stdio::piped())
-        .process_group(0)
+        .process_group(0);
+    // Raise the child's soft RLIMIT_NOFILE to a sane default (issue #786) so a
+    // command like `python -c 'import pandas'` is not killed by EMFILE.
+    crate::sys::rlimit::apply_nofile_limit(&mut std_cmd);
+    let mut child = tokio::process::Command::from(std_cmd)
         .spawn()
         .map_err(|e| {
             // Best-effort: send a spawn-failure exit frame. Ignore if channel full.
@@ -659,6 +674,11 @@ async fn exec_pty(
     pty_sys::apply_pty_session_leader(&mut std_cmd)
         .map_err(|e| Status::internal(format!("exec: apply_pty_session_leader: {e}")))?;
 
+    // Raise the child's soft RLIMIT_NOFILE to a sane default (issue #786) so an
+    // interactive shell and anything it launches can open more than the 1024
+    // files a bare init inherits. See sys/rlimit.rs for the SAFETY contract.
+    crate::sys::rlimit::apply_nofile_limit(&mut std_cmd);
+
     // Convert the master to an AsyncFd for epoll-driven IO. tokio::fs::File
     // uses spawn_blocking and does not work correctly with O_NONBLOCK on a PTY
     // master character device: it would return EAGAIN immediately on reads

guest/agent-rs/src/service/workload.rs

@@ -63,6 +63,11 @@ pub fn spawn_detached(command: &str, env: &[(String, String)], cwd: &str) -> std
     for (k, v) in env {
         cmd.env(k, v);
     }
+    // Raise the workload's soft RLIMIT_NOFILE to a sane default (issue #786) so a
+    // long-running serving app inherits the same file-descriptor headroom as an
+    // exec child. Registers a second pre_exec hook; hooks stack in registration
+    // order and this one only touches getrlimit/setrlimit (see sys/rlimit.rs).
+    crate::sys::rlimit::apply_nofile_limit(&mut cmd);
     // SAFETY: setsid(2) is async-signal-safe and has no preconditions; the
     // pre-exec closure runs in the forked child before exec and performs only
     // setsid plus an error read, no allocation. Detaching the child into a new

guest/agent-rs/src/sys/mod.rs

@@ -24,6 +24,7 @@ pub mod kill;
 pub mod netlink;
 pub mod pty;
 pub mod mount;
+pub mod rlimit;
 pub mod signal;
 pub mod vsock;