From 2ac3c2abbf3d5a964f6cb50ab536bff426b28c8e Mon Sep 17 00:00:00 2001 From: erichanwang Date: Thu, 23 Jul 2026 00:03:56 -0500 Subject: [PATCH 1/6] Add a container primitive built on namespaces and cgroups v2 neuros-container run puts a command in a real cgroup v2 leaf and, when privilege allows, a fresh mount/UTS/PID/IPC namespace set, without runc, containerd, or libcontainer underneath. Resource limits are plain cgroup files (memory.max, pids.max, cpu.weight) written by hand. The interesting part turned out to be cgroup placement, not the limit values: a cgroup already holding member processes can't enable subtree_control for children (the "no internal process" rule), and an ordinary interactive shell's own cgroup is exactly such a cgroup. The tool walks up to the nearest ancestor that already delegates the wanted controller and nests the new leaf there instead, which is what lets this run without root from a normal terminal. Namespace isolation still needs root or an unprivileged user namespace; Ubuntu 24.04's default AppArmor policy blocks the latter for unconfined processes, so on a stock install this degrades to cgroup-only limiting and says so on stderr rather than pretending to sandbox anything. tests/test_container.py exercises this for real: a 16M memory.max cgroup holding a process that touches 200MB of bytearray keeps memory.current at the 16M ceiling, and a pids.max=4 cgroup stops a 20-iteration fork loop after 3 children. --- README.md | 38 +++ .../usr/local/bin/neuros-container | 303 ++++++++++++++++++ tests/test_container.py | 134 ++++++++ 3 files changed, 475 insertions(+) create mode 100755 config/includes.chroot/usr/local/bin/neuros-container create mode 100644 tests/test_container.py diff --git a/README.md b/README.md index cab0dbe..0ce1670 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,37 @@ tools for `read_file`, `list_directory`, `run_command`, `ask_llm`, `get_system_info`, and `git_status`. `run_command` is checked against shell-metacharacter injection; see `tests/test_mcp.py`. +### Container primitive (`neuros-container`) + +A from-scratch container runner built directly on `unshare(2)` and +cgroup v2, no runc/containerd/libcontainer: + +```sh +neuros-container run --mem 256M --pids 64 --hostname box -- bash +neuros-container list +``` + +Resource limits are real cgroup v2 accounting: `--mem` sets +`memory.max` on a fresh leaf cgroup, `--pids` sets `pids.max`, and +`--cpu` sets `cpu.weight` when the controller is delegated. Because a +cgroup that already holds member processes can't enable +`subtree_control` for children (cgroup v2's "no internal process" +rule), the tool walks up from its own cgroup to the nearest ancestor +that already delegates the wanted controller, so it works from an +ordinary interactive shell without root. Verified on this machine: +a `--mem 16M` cgroup holding a process that touches 200MB of +`bytearray` keeps `memory.current` at the 16MB ceiling instead of +growing past it, and a `--pids 4` cgroup stops a 20-iteration fork +loop after 3 children (see `tests/test_container.py`). + +Namespace isolation (mount, UTS, PID, IPC) needs either root or an +unprivileged user namespace; on Ubuntu 24.04+ the latter is blocked by +default for unconfined processes +(`kernel.apparmor_restrict_unprivileged_userns=1`). Without either, +`neuros-container` says so on stderr and runs the command under the +cgroup limits without namespace isolation, rather than silently +pretending to sandbox it. + ### Code completion VS Code ships with Continue.dev pre-installed, pointed at local Ollama. @@ -174,6 +205,7 @@ NeurOS/ │ │ │ ├── neuros-tray # system tray applet │ │ │ ├── neuros-model # model manager CLI │ │ │ ├── neuros-mcp # MCP server +│ │ │ ├── neuros-container # namespace + cgroup container runner │ │ │ ├── neuros-welcome # first-boot welcome screen │ │ │ └── ... # 70+ additional neuros-* utilities │ │ ├── etc/systemd/system/ @@ -228,6 +260,11 @@ Past MVP: Nothing is transmitted anywhere but the local Ollama prompt. - GUI chat application: a local browser-based chat UI exists (`neuros-chat`), not a native Tauri app. +- Container primitive: done. `neuros-container` runs a command under a + real cgroup v2 leaf (memory/pids/cpu limits) and, given root or an + unprivileged user namespace, Linux namespace isolation, without + runc/containerd. See `tests/test_container.py` for the measured + memory-cap and pids-cap enforcement. - Still open: voice input and output, a fine-tuning pipeline, and ARM/CUDA builds. @@ -243,6 +280,7 @@ python3 tests/test_nn.py python3 tests/test_autofix.py python3 tests/test_model.py python3 tests/test_mcp.py +python3 tests/test_container.py ./validate-build.sh ``` diff --git a/config/includes.chroot/usr/local/bin/neuros-container b/config/includes.chroot/usr/local/bin/neuros-container new file mode 100755 index 0000000..155e4eb --- /dev/null +++ b/config/includes.chroot/usr/local/bin/neuros-container @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +""" +neuros-container - a container primitive built from Linux namespaces and +cgroups v2, no runc/containerd/libcontainer involved. + +Isolation comes from three unshare(2) namespace groups (mount, UTS, PID, +IPC, and network) plus an optional chroot into a rootfs directory. +Resource limits come from a real cgroup v2 leaf: memory.max, pids.max, +and cpu.max when the cpu controller is delegated. + +Usage: + neuros-container run [--mem SIZE] [--pids N] [--cpu WEIGHT] + [--hostname NAME] [--rootfs DIR] -- CMD [ARGS...] + neuros-container list + +Namespace isolation needs either root, or (best effort, and blocked by +default on Ubuntu 24.04+ via AppArmor's unprivileged-userns restriction) +an unprivileged user namespace. Without either, the command still runs +under the cgroup limits but without namespace isolation, and this tool +says so rather than pretending otherwise. +""" + +import argparse +import os +import re +import sys +import time + +CGROUP_ROOT = "/sys/fs/cgroup" + + +def die(msg): + print(f"neuros-container: {msg}", file=sys.stderr) + sys.exit(1) + + +def parse_size(text): + """Parse '128M', '1G', '512K', a bare byte count, or 'max' into cgroup + file syntax (a byte count string, or the literal 'max').""" + if text is None: + return None + text = text.strip() + if text.lower() == "max": + return "max" + m = re.fullmatch(r"(\d+)([KMG]?)", text, re.IGNORECASE) + if not m: + die(f"invalid size '{text}' (expected e.g. 256M, 1G, or a byte count)") + n = int(m.group(1)) + mult = {"": 1, "K": 1024, "M": 1024**2, "G": 1024**3}[m.group(2).upper()] + return str(n * mult) + + +def own_cgroup_path(): + """Absolute filesystem path of the cgroup this process currently lives + in, from /proc/self/cgroup (cgroup v2 always has a single '0::' line).""" + with open("/proc/self/cgroup") as f: + for line in f: + _, _, rel = line.strip().partition("::") + if rel: + return CGROUP_ROOT + rel + return CGROUP_ROOT + + +def find_delegating_ancestor(wanted): + """Cgroup v2's 'no internal process' rule means a cgroup that already + holds member processes (a systemd scope or service, typically) cannot + itself enable subtree_control for a new child. The interactive shell + this tool is invoked from usually lives in exactly such a cgroup, so + walk up to the nearest ancestor that already delegates one of the + wanted controllers (a slice, ordinarily) and nest the new leaf there + as a sibling instead.""" + path = own_cgroup_path() + wanted = set(wanted) + while True: + try: + with open(os.path.join(path, "cgroup.subtree_control")) as f: + already = set(f.read().split()) + except OSError: + already = set() + if already & wanted: + return path + parent = os.path.dirname(path) + if parent == path: + return path + path = parent + + +def enable_controllers(parent, wanted): + """Best-effort: ask the parent cgroup to hand the given controllers + down to its children. Already-delegated controllers are a no-op; + controllers we have no permission to enable are dropped with a + warning rather than aborting the run.""" + available = set() + try: + with open(os.path.join(parent, "cgroup.controllers")) as f: + available = set(f.read().split()) + except OSError: + pass + try: + with open(os.path.join(parent, "cgroup.subtree_control")) as f: + already = set(f.read().split()) + except OSError: + already = set() + + ready = set() + missing = [c for c in wanted if c in available and c not in already] + if missing: + try: + with open(os.path.join(parent, "cgroup.subtree_control"), "w") as f: + f.write(" ".join(f"+{c}" for c in missing)) + already |= set(missing) + except OSError: + pass + for c in wanted: + if c in available and c in already: + ready.add(c) + elif c in wanted: + print(f"neuros-container: cgroup controller '{c}' not delegated here, " + f"skipping that limit", file=sys.stderr) + return ready + + +def make_cgroup(name, mem, pids, cpu_weight): + wanted = [c for c, v in (("memory", mem), ("pids", pids), ("cpu", cpu_weight)) if v] + parent = find_delegating_ancestor(wanted) + ready = enable_controllers(parent, wanted) + + cg = os.path.join(parent, name) + os.makedirs(cg, exist_ok=True) + + if mem and "memory" in ready: + with open(os.path.join(cg, "memory.max"), "w") as f: + f.write(mem) + if pids and "pids" in ready: + with open(os.path.join(cg, "pids.max"), "w") as f: + f.write(pids) + if cpu_weight and "cpu" in ready: + with open(os.path.join(cg, "cpu.weight"), "w") as f: + f.write(str(cpu_weight)) + return cg + + +def join_cgroup(cg, pid): + with open(os.path.join(cg, "cgroup.procs"), "w") as f: + f.write(str(pid)) + + +def cgroup_report(cg): + report = {} + for key, fname in (("memory_peak", "memory.peak"), ("pids_current", "pids.current")): + path = os.path.join(cg, fname) + if os.path.exists(path): + with open(path) as f: + report[key] = f.read().strip() + return report + + +def try_unprivileged_userns(): + """Best-effort unshare(CLONE_NEWUSER) with a 1:1 uid/gid map, so the + calling user appears as root inside the new namespace. Returns True if + it worked. On Ubuntu 24.04+ this is blocked by default for unconfined + processes (see /proc/sys/kernel/apparmor_restrict_unprivileged_userns); + EPERM there is expected, not a bug.""" + try: + os.unshare(os.CLONE_NEWUSER) + uid, gid = os.getuid(), os.getgid() + with open("/proc/self/setgroups", "w") as f: + f.write("deny\n") + with open("/proc/self/uid_map", "w") as f: + f.write(f"0 {uid} 1\n") + with open("/proc/self/gid_map", "w") as f: + f.write(f"0 {gid} 1\n") + except OSError: + return False + return True + + +def enter_namespaces(): + """Returns 'full' if mount/uts/pid/ipc namespaces were entered, 'none' + if we ran with no isolation at all (insufficient privilege).""" + have_caps = os.geteuid() == 0 + if not have_caps: + have_caps = try_unprivileged_userns() + + if not have_caps: + print("neuros-container: no root and unprivileged user namespaces " + "are unavailable; running without namespace isolation " + "(cgroup limits above still apply)", file=sys.stderr) + return "none" + + try: + os.unshare(os.CLONE_NEWNS | os.CLONE_NEWUTS | os.CLONE_NEWPID | os.CLONE_NEWIPC) + except OSError as e: + print(f"neuros-container: got a user namespace but not the rest ({e}); " + "running without namespace isolation (cgroup limits above still apply)", + file=sys.stderr) + return "none" + # hostname and rootfs are applied in the PID-1 child after the second + # fork below, since the UTS/mount namespaces are entered here but this + # process itself stays in the old PID namespace. + return "full" + + +def run_container(args): + name = f"neuros-{os.getpid()}" + mem = parse_size(args.mem) + pids = args.pids + cg = make_cgroup(name, mem, pids, args.cpu) + + pid = os.fork() + if pid == 0: + join_cgroup(cg, os.getpid()) + mode = enter_namespaces() + + if mode == "full": + # A second fork is required for CLONE_NEWPID: unshare() only + # affects children created after the call, so this process + # stays in the old PID namespace and the next fork's child + # becomes PID 1 in the new one. + inner = os.fork() + if inner == 0: + if args.hostname: + import socket + socket.sethostname(args.hostname) + if args.rootfs: + os.chroot(args.rootfs) + os.chdir("/") + try: + os.mount(b"proc", b"/proc", b"proc", 0, b"") + except (AttributeError, OSError): + # os.mount is only available on Python 3.13+; fall back + # to mount(8), which is present on every target host. + os.system("mount -t proc proc /proc") + os.execvp(args.cmd[0], args.cmd) + else: + _, status = os.waitpid(inner, 0) + os._exit(os.waitstatus_to_exitcode(status)) + else: + os.execvp(args.cmd[0], args.cmd) + os._exit(1) + + _, status = os.waitpid(pid, 0) + report = cgroup_report(cg) + try: + os.rmdir(cg) + except OSError: + pass + if report: + peak = report.get("memory_peak") + if peak: + print(f"neuros-container: peak memory {int(peak) / 1024 / 1024:.1f} MiB", file=sys.stderr) + return os.waitstatus_to_exitcode(status) + + +def list_cgroups(): + parent = own_cgroup_path() + found = False + for entry in sorted(os.listdir(parent)): + if entry.startswith("neuros-"): + found = True + path = os.path.join(parent, entry) + procs_path = os.path.join(path, "cgroup.procs") + n = 0 + if os.path.exists(procs_path): + with open(procs_path) as f: + n = len(f.read().split()) + print(f"{entry} ({n} process{'es' if n != 1 else ''})") + if not found: + print("no active neuros-container cgroups") + + +def main(): + parser = argparse.ArgumentParser(prog="neuros-container") + sub = parser.add_subparsers(dest="command") + + run = sub.add_parser("run", help="run a command in a cgroup-limited, namespace-isolated child") + run.add_argument("--mem", help="memory limit, e.g. 256M, 1G") + run.add_argument("--pids", help="max number of processes/threads") + run.add_argument("--cpu", type=int, help="cpu.weight (1-10000, default cgroup weight is 100)") + run.add_argument("--hostname", help="hostname to set inside the UTS namespace") + run.add_argument("--rootfs", help="directory to chroot into before exec") + run.add_argument("cmd", nargs=argparse.REMAINDER, help="-- command and arguments to run") + + sub.add_parser("list", help="list active neuros-container cgroups") + + args = parser.parse_args() + + if args.command == "run": + cmd = args.cmd + if cmd and cmd[0] == "--": + cmd = cmd[1:] + if not cmd: + die("no command given (usage: neuros-container run [opts] -- CMD [ARGS...])") + args.cmd = cmd + sys.exit(run_container(args)) + elif args.command == "list": + list_cgroups() + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/tests/test_container.py b/tests/test_container.py new file mode 100644 index 0000000..d3b9109 --- /dev/null +++ b/tests/test_container.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +""" +test_container.py — Unit and integration tests for neuros-container. + +The integration tests exercise real cgroup v2 enforcement (they create, +populate, and tear down an actual cgroup on the machine running the +suite) but skip cleanly if cgroup v2 delegation isn't available, the +same way the rest of this repo skips checks that need a full host. +""" + +import os +import subprocess +import sys +import unittest +import importlib.util +from importlib.machinery import SourceFileLoader + +CONTAINER_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", + "config", "includes.chroot", "usr", "local", "bin", "neuros-container" +) + + +def load_neuros_container(): + loader = SourceFileLoader("neuros_container", CONTAINER_PATH) + spec = importlib.util.spec_from_loader("neuros_container", loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +def cgroups_available(): + try: + nc = load_neuros_container() + cg = nc.make_cgroup("neuros-selftest-probe", "16777216", None, None) + os.rmdir(cg) + return True + except OSError: + return False + + +class TestSizeParsing(unittest.TestCase): + def setUp(self): + self.nc = load_neuros_container() + + def test_plain_bytes(self): + self.assertEqual(self.nc.parse_size("1000"), "1000") + + def test_kilobytes(self): + self.assertEqual(self.nc.parse_size("4K"), str(4 * 1024)) + + def test_megabytes(self): + self.assertEqual(self.nc.parse_size("256M"), str(256 * 1024 ** 2)) + + def test_gigabytes(self): + self.assertEqual(self.nc.parse_size("1G"), str(1024 ** 3)) + + def test_max_literal(self): + self.assertEqual(self.nc.parse_size("max"), "max") + + def test_none_passthrough(self): + self.assertIsNone(self.nc.parse_size(None)) + + def test_rejects_garbage(self): + with self.assertRaises(SystemExit): + self.nc.parse_size("not-a-size") + + +@unittest.skipUnless(cgroups_available(), "cgroup v2 delegation not available in this sandbox") +class TestCgroupEnforcement(unittest.TestCase): + def setUp(self): + self.nc = load_neuros_container() + + def test_memory_max_caps_actual_usage(self): + """A process that tries to touch 200MB inside a 16M memory.max + cgroup must not exceed that limit, per memory.current.""" + cg = self.nc.make_cgroup("neuros-selftest-mem", "16777216", None, None) + try: + proc = subprocess.Popen([ + sys.executable, "-c", + "import time; time.sleep(0.1)\n" + "b = bytearray(200 * 1024 * 1024)\n" + "for i in range(0, len(b), 4096):\n" + " b[i] = 1\n" + "time.sleep(0.2)\n", + ]) + self.nc.join_cgroup(cg, proc.pid) + proc.wait(timeout=10) + with open(os.path.join(cg, "memory.peak")) as f: + peak = int(f.read().strip()) + self.assertLessEqual(peak, 16 * 1024 * 1024 * 1.05) + finally: + try: + os.rmdir(cg) + except OSError: + pass + + def test_pids_max_blocks_extra_forks(self): + """pids.max=4 must stop a forking loop once 4 members are in the + cgroup (the harness process here counts as one of the four).""" + cg = self.nc.make_cgroup("neuros-selftest-pids", None, "4", None) + try: + script = ( + "import subprocess, sys, time\n" + "time.sleep(0.1)\n" + "forked = 0\n" + "procs = []\n" + "try:\n" + " for _ in range(20):\n" + " procs.append(subprocess.Popen(['sleep', '0.3']))\n" + " forked += 1\n" + "except OSError:\n" + " pass\n" + "for p in procs:\n" + " p.wait()\n" + "print(forked)\n" + ) + proc = subprocess.Popen( + [sys.executable, "-c", script], + stdout=subprocess.PIPE, text=True, + ) + self.nc.join_cgroup(cg, proc.pid) + out, _ = proc.communicate(timeout=10) + forked = int(out.strip()) + self.assertLess(forked, 20) + finally: + try: + os.rmdir(cg) + except OSError: + pass + + +if __name__ == "__main__": + unittest.main() From 2e4080a707db2ba20b45a4ffbb7f6a9b9a750c8d Mon Sep 17 00:00:00 2001 From: erichanwang Date: Tue, 28 Jul 2026 17:13:23 -0500 Subject: [PATCH 2/6] Pin build inputs, fix a broken bootstrap, and guard the air gap lb config was missing --mode ubuntu, so live-build defaulted to Debian's mirrors even with --distribution noble set. Debian has no noble suite, so a build from a clean checkout failed at bootstrap - a more basic problem than the missing version pins. Also fixes also-utils -> alsa-utils, which is not a real package name and would fail regardless. Nothing was pinned, so "reproducible" was not true: oh-my-zsh tracked master, GNOME extensions used mutable tags, and the package list had no versions. Pin oh-my-zsh to a commit, the Ollama installer to the v0.32.5 tag SHA, GNOME tarballs to commit SHAs, and VS Code plus Continue.dev to explicit versions. For the ~60 apt packages, point build-time mirrors at a snapshot.ubuntu.com timestamp rather than pinning each one, which is brittle when the archive rotates. --mirror-binary deliberately stays on the live archive so installed systems still receive security updates. This gives pinned-input reproducibility - same commit, same software versions. Not bit-identical ISOs: timestamps, file ordering, and initramfs generation are unaddressed, and the README now says so rather than implying more. Also still unpinned: ollama pull mistral resolves a floating tag, which would need a ~4GB pull to verify against a manifest digest. neuros-mcp spoke HTTP+JSON-RPC only, which no MCP client actually uses. Add a stdio transport, verified against the official mcp Python SDK - initialize, list_tools, and call_tool over a real handshake, not an assumed one. Add scripts/check-airgap.sh, which fails on any external URL outside a reviewed allowlist, and wire it into CI along with test_mcp.py - both existed but were never actually run. The build-iso job now runs a real lb config and resolves the package list against the pinned snapshot instead of only checking that files exist. --- .github/workflows/ci.yml | 44 +- README.md | 429 ++++++++++++++++-- build.sh | 26 +- .../live/0100-install-ollama.hook.chroot | 10 +- .../live/0200-install-vscode.hook.chroot | 10 +- .../live/0500-configure-system.hook.chroot | 5 +- .../0600-install-gnome-extensions.hook.chroot | 24 +- .../includes.chroot/usr/local/bin/neuros-mcp | 88 +++- config/package-lists/neuros.list.chroot | 2 +- scripts/check-airgap.sh | 78 ++++ tests/test_mcp.py | 34 ++ 11 files changed, 682 insertions(+), 68 deletions(-) create mode 100755 scripts/check-airgap.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37f977f..0ce58c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,12 @@ jobs: run: bash validate-build.sh - name: Run unit tests - run: python3 tests/test_nn.py -v + run: | + python3 tests/test_nn.py -v + python3 tests/test_mcp.py -v + + - name: Air-gap check (no new outbound network calls) + run: bash scripts/check-airgap.sh - name: Check shell script syntax run: | @@ -72,7 +77,7 @@ jobs: done build-iso: - name: Build ISO (Structure Check) + name: Build ISO (Structure + lb config + package resolution) runs-on: ubuntu-24.04 needs: validate steps: @@ -91,7 +96,40 @@ jobs: test -d config/package-lists && echo "OK: package lists present" test -f build.sh && echo "OK: build.sh present" test -f validate-build.sh && echo "OK: validate-build.sh present" - echo "Structure OK — full ISO build requires 20GB disk (CI runners limited)" + echo "Structure OK — a full ISO build (debootstrap + chroot + squashfs)" + echo "needs 20GB+ disk and privileged mounts CI runners don't have." + + - name: Run lb config with the pinned snapshot mirror + run: | + SNAPSHOT_TS=$(grep -oP '(?<=SNAPSHOT_TS=")[^"]+' build.sh) + SNAPSHOT_MIRROR="https://snapshot.ubuntu.com/ubuntu/${SNAPSHOT_TS}" + echo "Pinned snapshot: $SNAPSHOT_MIRROR" + lb config \ + --mode ubuntu \ + --distribution noble \ + --archive-areas "main restricted universe multiverse" \ + --debian-installer none \ + --memtest none \ + --binary-images iso-hybrid \ + --mirror-bootstrap "$SNAPSHOT_MIRROR" \ + --mirror-chroot "$SNAPSHOT_MIRROR" \ + --mirror-chroot-security "$SNAPSHOT_MIRROR" + echo "OK: lb config succeeded against the pinned snapshot" + + - name: Resolve neuros.list.chroot against the pinned snapshot + run: | + SNAPSHOT_TS=$(grep -oP '(?<=SNAPSHOT_TS=")[^"]+' build.sh) + SNAPSHOT_MIRROR="https://snapshot.ubuntu.com/ubuntu/${SNAPSHOT_TS}" + sudo tee /etc/apt/sources.list.d/neuros-snapshot.list > /dev/null <.json` describing the +inner pid and cgroup path; `cleanup` reaps those that have already +exited, refusing to remove cgroups that still hold live members. +`list` walks the cgroup subtree recursively (nested `neuros-*` leaves +inside parent `neuros-*` leaves are also found) and supports a +`--json` output mode. + +All model and config parsing in `nn`, `neuros-model`, and the rest +of the `neuros-*` tools delegates to the shared `neuroslib.py`, which +reads `~/.config/neuros/llm.conf` as an INI document — the +`[llm]` section owns `model`/`host`/`port`, the `[context]` section +owns the opt-in system-context flags, and additional sections round- +trip through unchanged. + +### Untrusted code sandbox (`neuros-sandbox`) + +A safe-runner wrapper around `neuros-container`, tuned for running +scripts produced by an LLM agent or pulled from an untrusted source. +The hardcoded defaults are the point of the tool: + +```sh +echo 'print("hello agent")' | neuros-sandbox run --json +neuros-sandbox run --mem 1G --timeout 60 ./smoke.py +neuros-sandbox run --bundle ./fixture.tar.gz --dry-run # show argv +neuros-sandbox run --json ./post.sh # envelope +``` + +Every safe-mode invocation always passes `--net`, `--read-only`, +`--mem 256M`, `--pids 64`, `--cpu-quota 50000` (50% of one CPU), +and `--cap-drop 'CAP_(NET_RAW|SYS_ADMIN|SYS_PTRACE|SETUID|SETPCAP| +MAC_ADMIN|DAC_OVERRIDE|LINUX_IMMUTABLE|SYS_CHROOT|SYS_RAWIO| +SYS_RESOURCE)'` to the underlying `neuros-container run`. The +cap-drop baseline is the load-bearing piece of the default safety +posture and persists under `--unsafe` as well; only `--cap-drop-keep +REGEX` (active only under `--unsafe`) replaces it for untrusted +work that genuinely needs broader capabilities. Override regexes +are pre-validated with `re.compile` so a malformed pattern fails +at the wrapper, not deep inside the kernel-bridge ctypes call when +the primitive would otherwise reject it. The host environment is +scrubbed to a minimal `PATH` so `ANTHROPIC_API_KEY`, +`HOME`, and other secrets don't leak into the worker's process env. +Scripts are always piped to `python3 -u -` via stdin — no temp +files on the host, no script path visible via `/proc//cmdline`. +A wall-clock watchdog (`--timeout`, default 30 s) SIGKILLs the +container on expiry and the wrapper exits 124 with `timeout_hit: +true` in the JSON envelope. `--json` emits one line of structured +output (`exit_code`, `stdout`, `stderr`, `wall_clock_ms`, +`timeout_hit`, `peak_mem_estimate`), suitable for orchestrator +consumption; in default human mode stdout/stderr pass through and a +single `[neuros-sandbox] exit=… wall_ms=… timeout_hit=…` trailer is +written to stderr. `peak_mem_estimate` is best-effort: the primitive +auto-cleans the cgroup on non-detached exit, so the `memory.peak` +file is often gone by the time we read it. For forensic-grade memory +accounting, run with `--unsafe --cap-drop-keep ''` plus an explicit +`--detach` and read `memory.peak` from `/` before invoking +`neuros-container cleanup `. + +`--unsafe` opts in to disabling `--net` and `--read-only` (with +`--cap-drop-keep REGEX` for replacing — not relaxing — the set of +dropped capabilities). `--bundle TAR` extracts a tarball into a +fresh tmp directory and uses it as the container's rootfs; +absolute paths and `..` traversal inside the tar are rejected at +extraction time. Tests in `tests/test_sandbox.py` mock +`subprocess.run` so no kernel cgroup delegation is required to +verify the argv shape, env scrubbing, JSON envelope, timeout +behavior, and bundle cleanup. + +### Regression benchmark harness (`neuros-bench`) + +A small driver that shells out to `neuros-sandbox run --json` with +a registry of canned workloads (CPU-bound recursion, memory growth, +regex compile, fork pressure, ctypes-bridge syscalls, file IO, JSON +parse, string ops), parses each envelope, and produces a metrics +JSON file that CI can diff against a stored baseline: + +```sh +neuros-bench list # show the workload suite +neuros-bench batch --out baseline.json # capture a baseline run +neuros-bench run mem_grow --out single.json # run one workload +neuros-bench compare baseline.json candidate.json + # exit 0 within tolerance, exit 1 on a row beyond threshold +``` + +The compare subcommand emits a grep-friendly table on stdout +(`workload`, `baseline_ms`, `candidate_ms`, `wall_pct`, `base_mem`, +`cand_mem`, `mem_pct`, `status=OK|REGRESSION|NEW|DROPPED`) and a +single verdict line on stderr. Tolerance defaults are 15% wall and +25% peak-mem; tighten via `--tolerance-wall-pct` / `--tolerance-mem-pct` +in CI. Each workload runs under `--timeout 60` (configurable) with +the sandbox's default `--mem 256M` and `--pids 64` so an unbounded +regression in any one workload can't stall the batch.Tests in `tests/test_bench.py` mock `subprocess.run` and let the workspace +run a regression check with no kernel/cgroup dependency. + +### Runbook verifier (`neuros-runbook`) + +A small assertion-driven runner that drives `neuros-sandbox` from a +JSON runbook file. Each step has a `script`, optional per-step +`limits` (mem/pids/timeout/cpu_quota), and an `expect` block that can +assert `exit_code`, regex-match `stdout`/`stderr`, or forbid +`timeout_hit=true`. Step verdicts roll up into a JSON envelope (with +`--json`) or a grep-friendly table on stdout (default). Exit 0 if +every step passes, 1 on any failure; `--only-violations` filters the +human table to just the failed rows. + +```sh +neuros-runbook run ./runbooks/smoke.json +neuros-runbook run --only-violations ./runbooks/prod-checks.json +neuros-runbook run --json ./runbooks/regression.json | jq '.steps[] | select(.passed==false)' +``` + +Example runbook: + +```json +[ + {"name": "imports-stdlib", + "script": "import sys; print(sys.version_info[:2])", + "expect": {"exit_code": 0, "stdout_matches": "^\\(3, [0-9]+\\)$"}}, + {"name": "network-blocked", + "script": "import socket; socket.gethostbyname('example.com'); print('UP')", + "limits": {"timeout": 5}, + "expect": {"exit_code": "any", "timeout_forbidden": false}}, + {"name": "no-tracebacks", + "script": "import sys; sys.exit(0)", + "expect": {"exit_code": 0, "stderr_matches": "^$"}} +] +``` + +The runbook parser pre-validates each `expect.*_matches` regex at +load time so a malformed pattern fails loudly when the runbook is +loaded, not at the assertion site. Tests in `tests/test_runbook.py` +mock `subprocess.run` for full coverage without kernel/cgroup +dependency. + +### Declarative security policy (`neuros-policy`) + +Hardened defaults are good until you need to *prove* which defaults +applied to a given run. `neuros-policy` is the place that +information lives: a JSON manifest naming a profile (`strict`, +`moderate`, `permissive`), declaring per-profile capability drops, +naming memory / PID / CPU-quota / timeout bounds, and pinning the +allowlist of env vars that may pass through to the container. The +three subcommands compose like this: + +* `neuros-policy validate ./policy.json` — schema + bounds check. + Bounds are conservative: mem 16K-1G, pids 1-4096, cpu_quota + 1000-1_000_000 (microseconds / 100ms CFS period), timeout + 1-3600s. Profile names must match `[a-z][a-z0-9_-]*`, cap names + `^CAP_[A-Z_]+$`, env-allowlist names `^[A-Z][A-Z0-9_]*$`. Exits + 0 on success, 1 on any violation, 2 on malformed input. +* `neuros-policy check ./policy.json --envelope ./envelope.json` — + parse a `neuros-sandbox --json` envelope and reconcile its + `wall_clock_ms`, `peak_mem_estimate`, and `timeout_hit` against + the policy's bounds. Exits 0 when the run conformed, 1 on a + violation (printed to stderr in the form `[severity] rule: + observed X, expected Y`), 2 on bad input. +* `neuros-policy transpile ./policy.json --profile strict` — emit + the `neuros-sandbox --unsafe ...` argv fragment that realises + the policy. A profile with an empty cap list transpiles to + `--cap-drop '^_NEVER_MATCH_$'` so nothing is dropped, which is + almost always a bug — but if you really mean it, the surface + is unambiguous. + +Sample manifest bundled in `tests/test_policy.py` (`_good_policy`). +Add `--json` to `validate` / `check` to emit a single-line envelope +that composes with the bench + runbook JSON surfaces downstream. + +### Offline envelope diagnostics (`neuros-replay`) + +When a `neuros-sandbox --json` envelope looks wrong, the typical +follow-up is either "how wrong?" (diff against an earlier good +run) or "what does this mean?" (humanize the metrics). Both are +now a single tool, no shell-out to `python3 -c`: + +* `neuros-replay diff ` — per-field regression verdict. + Tolerances default to 10% wall-clock / 20% peak-mem (tighter + than bench's 15%/25% because this is a single observation, not + a batch aggregate). Each violation surfaces as a single grep- + friendly rule string on stderr; `--json` emits a structured + envelope with `ok`, `wall_clock_pct_delta`, `peak_mem_pct_ + delta`, `rule_violations`, and the tolerances that were + applied. Exit 0 on conformant, 1 on any rule violation, 2 on + bad input. +* `neuros-replay explain ` — sinkable one-line summary + like `exit=0 wall=0ms peak=n/a timeout=False stdout_bytes=0 + stderr_bytes=0`. The exact numbers depend on the envelope, + but the field order is fixed. Add `--json` for the structured + equivalent with `neuros_replay_version`. Designed for CI grep. +* `neuros-replay extract --stream stdout|stderr|both` + — dump the embedded `stdout` / `stderr` payloads so standard + host tools (`grep`, `jq`, `less`) read them without JSON + escaping. The `both` stream mode inserts a single + `===STDERR===` separator line so a downstream parser can tell + the two streams apart without re-mapping field names. + +`neuros-replay` is purely offline: it consumes envelopes the +other tools emit and never re-runs a workload. For replay-with- +execute, use `neuros-bench run --json` and pipe the +envelope into `neuros-replay`. Tests in `tests/test_replay.py` +exercise every branch without subprocess shell-out. + +### Envelope-agnostic verifier (`neuros-verify`) + +All five emitter tools (sandbox, bench, runbook, policy, replay) +produce a JSON envelope, but their shapes diverge. `neuros-verify` +is the single tool that reads *any* of them, auto-detects the +shape, and emits a normalized `verdict=PASS|FAIL` line so a CI +gate only needs one consumer in its pipeline: + +* Kind detection is by version key first (`neuros_bench_version` + / `neuros_policy_version` / `neuros_replay_version`) and by + structural fingerprint second (`exit_code`+`wall_clock_ms`+ + `timeout_hit` => sandbox; `runbook_path`+`steps` => runbook). + First match wins; unrecognized is treated as `FAIL` with the + top-level keys listed for triage. +* Per-kind rules: + - Sandbox envelope needs `--policy ` — the policy's + `check_envelope_against_policy` is loaded via runtime + `compile()+exec()` of the on-disk neuros-policy script + (`mod.__file__` is set explicitly so the runtime loader can + find the manifest). A defensive `hasattr` check turns a + silent AttributeError later into a loud failure now if the + policy tool ever renames its public verify names. + - Bench metrics without `--bench-baseline` degrades to a + crash check (every run's `exit_code == 0`). With + `--bench-baseline`, each per-workload run is diffed against + the baseline using the same tolerances as + `neuros-bench compare` (15% wall / 25% mem). + - Policy and replay verdicts are passed through `env["ok"]`; + the downstream tool's `errors` / `rule_violations` lists are + carried as the violations this tool reports. + - Runbook envelopes get a structural smoke check (every step + has `name` + `ok`). Step-level assertion semantics live in + `neuros-runbook` itself. +* Multiple input files are **AND-merged**: any single failure + fails the overall run. +* Text output mode prints a per-file `verdict=… source=… + file=… violations=N` line (deterministic field order) plus + a final `source=aggregate` line. `--quiet` suppresses per- + file lines; only the aggregate remains. `--json` emits a + single-line envelope with `{ok, aggregate, parts[], + neuros_verify_version}` shaped like the other tools' machine + output. + +Exit codes: `0` all conformant; `1` any rule violation or +unrecognized envelope; `2` bad input (missing file, malformed +JSON, root not an object). + +Tests in `tests/test_verify.py` cover all five kinds, the +unknown-envelope path, every dispatch surface (text, `--json`, +`--quiet`, multi-file AND-merge, bench-with-baseline), and the +runtime sandbox+policy integration end-to-end via a tempdir- +mounted manifest. + ### Code completion VS Code ships with Continue.dev pre-installed, pointed at local Ollama. @@ -188,6 +476,83 @@ qemu-system-x86_64 -m 8192 -smp 4 -cdrom live-image-amd64.hybrid.iso \ -boot d -vga virtio -display sdl ``` +### Reproducibility + +"Reproducible" here means **pinned-inputs reproducibility**: every build +input that could otherwise drift (package versions, third-party +installers, GitHub release tarballs) is pinned to an explicit version or +commit, so the same commit of this repo installs the same software on +every rebuild. It does **not** mean bit-identical ISO output — timestamps +embedded by `mksquashfs`/`xorriso`, filesystem inode ordering, and +initramfs generation are not currently pinned or normalized, and would +need to be (via `SOURCE_DATE_EPOCH`, sorted file ordering, etc.) to make +that stronger claim. That's a separate, larger effort from pinning what +gets installed, and hasn't been done here. + +Along the way, `build.sh`'s `lb config` call was also missing +`--mode ubuntu`. Without it, live-build defaults to Debian's own mirrors +for an Ubuntu suite name, and `lb config`/debootstrap would fail on a +genuinely clean checkout since Debian's archive has no `noble` suite. +That's fixed alongside the pinning below, verified with `lb config` +exiting 0 against `--mode ubuntu` in a clean container. + +What's pinned, and how: + +- **~60 apt packages** (`config/package-lists/neuros.list.chroot`): no + per-package `=version` pins. Instead, `build.sh` points `lb config` at + a fixed Ubuntu archive snapshot (`SNAPSHOT_TS` in `build.sh`, via + [snapshot.ubuntu.com](https://snapshot.ubuntu.com)) for the bootstrap + and chroot mirrors, so `apt` resolves the exact same package versions + on every build regardless of what's since changed in the live Ubuntu + archive. This was chosen over pinning every package to `=version` + because a snapshot pins the whole dependency graph at once and doesn't + need updating package-by-package as the archive rotates; Canonical + commits to keeping snapshots available for at least 2 years. The + shipped ISO's own `/etc/apt/sources.list` (`--mirror-binary`) is + deliberately left on the regular live Ubuntu mirrors, not the + snapshot, so a running NeurOS system keeps getting real security + updates after install. +- **oh-my-zsh** (`0500-configure-system.hook.chroot`): pinned to a + specific commit SHA instead of the `master` branch tip. +- **VS Code and the Continue.dev extension** + (`0200-install-vscode.hook.chroot`): pinned to a specific `code` + package version (Microsoft's apt repo keeps a long version history, so + this doesn't go stale) and a specific Continue.dev marketplace version. +- **GNOME extensions** (`0600-install-gnome-extensions.hook.chroot`): + blur-my-shell and caffeine are downloaded from a GitHub archive URL + pinned to a commit SHA (not a tag name, which can be moved) for the + same release each build. +- **Ollama installer** (`0100-install-ollama.hook.chroot`): the + installer script is fetched from a pinned commit (the `v0.32.5` tag) + instead of the floating `ollama.com/install.sh`, and `OLLAMA_VERSION` + pins the installed binary itself. + +While verifying package resolution, `config/package-lists/neuros.list.chroot` +was also found to list `also-utils`, which isn't a real package (the +intended package is `alsa-utils`); this would have failed `apt-get +install` regardless of pinning, so it's fixed alongside this work. + +**Known gap, not fixed here:** `ollama pull mistral` still pulls +whatever the `mistral` tag in Ollama's library currently resolves to — +that tag can move to a different quantization over time. Pinning it to a +manifest digest (`mistral@sha256:...`) is possible in principle, but +verifying the digest reference actually works requires an `ollama pull` +of the full ~4GB model, which this environment intentionally does not +do. Left as a documented gap rather than shipped unverified. + +**What was verified, and how:** `lb config` (with `--mode ubuntu` and the +pinned snapshot mirrors) was run in a clean `ubuntu:24.04` Docker +container and exits 0. `apt-get install --dry-run` against the pinned +snapshot for every package in `neuros.list.chroot` was run in the same +container and resolves cleanly (exit 0, no dry-run conflicts). All +pinned commit SHAs/tags/versions above were confirmed to resolve via the +GitHub API and package repositories at the time of pinning. What was +**not** verified: an actual `lb build` (needs 20GB+ disk and privileged +chroot/mount this environment doesn't have), and the `ollama pull` / +GNOME extension downloads were not executed end-to-end inside a real +chroot (only their URLs were confirmed to resolve and serve the expected +content). + ## Project structure ``` @@ -204,10 +569,10 @@ NeurOS/ │ │ │ ├── nn # terminal assistant CLI │ │ │ ├── neuros-tray # system tray applet │ │ │ ├── neuros-model # model manager CLI -│ │ │ ├── neuros-mcp # MCP server -│ │ │ ├── neuros-container # namespace + cgroup container runner -│ │ │ ├── neuros-welcome # first-boot welcome screen -│ │ │ └── ... # 70+ additional neuros-* utilities +│ │ │ ├── neuros-mcp # MCP server │ │ │ ├── neuros-container # namespace + cgroup container runner + │ │ │ ├── neuros-sandbox # safe-runner wrapper for untrusted scripts + │ │ │ ├── neuros-welcome # first-boot welcome screen + │ │ │ └── ... # 70+ additional neuros-* utilities │ │ ├── etc/systemd/system/ │ │ │ └── neuros-llm.service │ │ ├── etc/polkit-1/rules.d/ @@ -247,8 +612,13 @@ integration, and privacy hardening. Past MVP: - MCP server: done. `neuros-mcp` implements `initialize`/`tools-list`/ - `tools-call`/`resources` over HTTP and JSON-RPC, verified end-to-end - against a live Ollama instance. + `tools-call`/`resources` over both HTTP/JSON-RPC and the MCP stdio + transport (`--stdio`), verified end-to-end against a live Ollama + instance and, for stdio, against the official `mcp` Python SDK + client. Only 6 tools are exposed through MCP function-calling; the + other ~80 `neuros-*` CLIs are standalone tools a user (or another + agent via `run_command`) invokes directly, not entries in an LLM + tool-call table. - Model switcher: done, CLI and tray. `neuros-model` implements `list/pull/remove/switch/info/search/benchmark/compare`, and the system tray now has a "Switch Model" submenu that shells out to @@ -281,6 +651,7 @@ python3 tests/test_autofix.py python3 tests/test_model.py python3 tests/test_mcp.py python3 tests/test_container.py +python3 tests/test_sandbox.py ./validate-build.sh ``` diff --git a/build.sh b/build.sh index e405841..8f42ad5 100755 --- a/build.sh +++ b/build.sh @@ -14,6 +14,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BUILD_LOG="$SCRIPT_DIR/build.log" ISO_NAME="live-image-amd64.hybrid.iso" +# Ubuntu archive snapshot used to build the chroot, pinned to a fixed date +# so `apt` resolves the same package versions on every build instead of +# whatever happens to be current in the live archive that day. See +# https://snapshot.ubuntu.com/ for details on the service. Bump this +# deliberately (and re-run validate-build.sh) when the package set needs +# to move forward; it will not drift on its own. +SNAPSHOT_TS="20260701T000000Z" +SNAPSHOT_MIRROR="https://snapshot.ubuntu.com/ubuntu/${SNAPSHOT_TS}" + # Colors RED='\033[0;31m' GREEN='\033[0;32m' @@ -85,12 +94,27 @@ configure_live_build() { # Initialize live-build config if not already done if [[ ! -f "auto/config" ]]; then + # --mirror-bootstrap/--mirror-chroot(-security) pin the package + # versions used to build the chroot to the snapshot above, for + # reproducibility. --mirror-binary is intentionally left at the + # live-build default (the regular Ubuntu mirrors) so the ISO + # ships with normal, current sources.list entries -- installed + # systems should get real ongoing updates, not be frozen to a + # months-old snapshot forever. + # --mode ubuntu is required for live-build to default to Ubuntu's + # mirrors/keyring for an Ubuntu suite; without it, lb config + # defaults to Debian's own archive, which has no "noble" suite + # and would break debootstrap on a truly clean checkout. lb config \ + --mode ubuntu \ --distribution noble \ --archive-areas "main restricted universe multiverse" \ --debian-installer none \ --memtest none \ - --binary-images iso-hybrid + --binary-images iso-hybrid \ + --mirror-bootstrap "$SNAPSHOT_MIRROR" \ + --mirror-chroot "$SNAPSHOT_MIRROR" \ + --mirror-chroot-security "$SNAPSHOT_MIRROR" success "live-build configured." else warn "live-build config already exists. Using existing configuration." diff --git a/config/hooks/live/0100-install-ollama.hook.chroot b/config/hooks/live/0100-install-ollama.hook.chroot index a1f7f56..d300cc8 100755 --- a/config/hooks/live/0100-install-ollama.hook.chroot +++ b/config/hooks/live/0100-install-ollama.hook.chroot @@ -5,8 +5,14 @@ set -e echo "[NeurOS] Installing Ollama..." -# Install Ollama -curl -fsSL https://ollama.com/install.sh | sh +# Install Ollama. The installer is fetched from a pinned commit (the +# scripts/install.sh at the v0.32.5 release tag) instead of the floating +# ollama.com/install.sh, and OLLAMA_VERSION pins the binary itself -- +# otherwise both the installer logic and the installed version can drift +# between builds. +OLLAMA_INSTALL_SHA="eec8e0b9458b8a01be0c216a9cc53eefde24ef50" # v0.32.5 +export OLLAMA_VERSION="0.32.5" +curl -fsSL "https://raw.githubusercontent.com/ollama/ollama/${OLLAMA_INSTALL_SHA}/scripts/install.sh" | sh # Pre-pull Mistral 7B model (quantized, ~4GB) echo "[NeurOS] Pulling Mistral 7B model (this will take a while)..." diff --git a/config/hooks/live/0200-install-vscode.hook.chroot b/config/hooks/live/0200-install-vscode.hook.chroot index 90e22e4..1e3c8c4 100755 --- a/config/hooks/live/0200-install-vscode.hook.chroot +++ b/config/hooks/live/0200-install-vscode.hook.chroot @@ -10,10 +10,14 @@ wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /u echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/code stable main" > /etc/apt/sources.list.d/vscode.list apt-get update -apt-get install -y code +# Pinned to a specific build so the package doesn't drift between builds; +# the Microsoft repo keeps a long version history, so this doesn't go +# stale the way a single-version mirror snapshot would. Bump deliberately. +apt-get install -y code=1.130.0-1784734578 -# Install Continue.dev extension (pre-configured for local Ollama) +# Install Continue.dev extension (pre-configured for local Ollama), pinned +# to a specific version for the same reason. echo "[NeurOS] Installing Continue.dev extension..." -code --install-extension Continue.continue --user-data-dir /etc/skel/.vscode +code --install-extension Continue.continue@2.1.0 --user-data-dir /etc/skel/.vscode echo "[NeurOS] VS Code installation complete." diff --git a/config/hooks/live/0500-configure-system.hook.chroot b/config/hooks/live/0500-configure-system.hook.chroot index 806ebfe..4783dec 100755 --- a/config/hooks/live/0500-configure-system.hook.chroot +++ b/config/hooks/live/0500-configure-system.hook.chroot @@ -18,7 +18,10 @@ chsh -s /usr/bin/zsh root 2>/dev/null || true if [ ! -d /etc/skel/.oh-my-zsh ]; then echo "[NeurOS] Installing oh-my-zsh..." export ZSH=/etc/skel/.oh-my-zsh - sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended --keep-zshrc 2>/dev/null || true + # Pinned to a specific commit on master rather than the branch tip, so + # the installer script can't change out from under the build. + OMZ_SHA="7ea697fd8138550ddf7262456d412f0dcd1cbf84" + sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/${OMZ_SHA}/tools/install.sh)" "" --unattended --keep-zshrc 2>/dev/null || true fi # Ensure nn CLI is executable diff --git a/config/hooks/live/0600-install-gnome-extensions.hook.chroot b/config/hooks/live/0600-install-gnome-extensions.hook.chroot index b74253d..970b398 100755 --- a/config/hooks/live/0600-install-gnome-extensions.hook.chroot +++ b/config/hooks/live/0600-install-gnome-extensions.hook.chroot @@ -40,15 +40,17 @@ if [ ! -d "$EXT_DIR" ]; then echo "[NeurOS] Downloading blur-my-shell extension..." mkdir -p "$EXT_DIR" - # Download latest release from GitHub - BLUR_VERSION="v60" # GNOME 46 compatible - BLUR_URL="https://github.com/aunetx/blur-my-shell/archive/refs/tags/${BLUR_VERSION}.tar.gz" + # Pinned to the commit for the v60 tag (GNOME 46 compatible). A commit + # SHA is used instead of the tag name so the archive can't change out + # from under us if the tag is ever moved upstream. + BLUR_SHA="5ac26249baa9dea04498b86312c52528254971f8" # v60 + BLUR_URL="https://github.com/aunetx/blur-my-shell/archive/${BLUR_SHA}.tar.gz" if wget -q "$BLUR_URL" -O /tmp/blur-my-shell.tar.gz 2>/dev/null; then tar -xzf /tmp/blur-my-shell.tar.gz -C /tmp/ - if [ -d "/tmp/blur-my-shell-${BLUR_VERSION#v}" ]; then - cp -r /tmp/blur-my-shell-${BLUR_VERSION#v}/* "$EXT_DIR/" 2>/dev/null || true - rm -rf /tmp/blur-my-shell-${BLUR_VERSION#v} + if [ -d "/tmp/blur-my-shell-${BLUR_SHA}" ]; then + cp -r /tmp/blur-my-shell-${BLUR_SHA}/* "$EXT_DIR/" 2>/dev/null || true + rm -rf /tmp/blur-my-shell-${BLUR_SHA} fi rm -f /tmp/blur-my-shell.tar.gz echo "[NeurOS] blur-my-shell installed." @@ -63,12 +65,14 @@ if [ ! -d "$CAFFEINE_DIR" ]; then echo "[NeurOS] Downloading caffeine extension..." mkdir -p "$CAFFEINE_DIR" - CAFFEINE_URL="https://github.com/eonpatapon/gnome-shell-extension-caffeine/archive/refs/tags/v50.tar.gz" + # Pinned to the commit for the v50 tag, same rationale as blur-my-shell above. + CAFFEINE_SHA="d8dbae0958a13bab9df0479497786066a269b859" # v50 + CAFFEINE_URL="https://github.com/eonpatapon/gnome-shell-extension-caffeine/archive/${CAFFEINE_SHA}.tar.gz" if wget -q "$CAFFEINE_URL" -O /tmp/caffeine.tar.gz 2>/dev/null; then tar -xzf /tmp/caffeine.tar.gz -C /tmp/ - if [ -d "/tmp/gnome-shell-extension-caffeine-50" ]; then - cp -r /tmp/gnome-shell-extension-caffeine-50/* "$CAFFEINE_DIR/" 2>/dev/null || true - rm -rf /tmp/gnome-shell-extension-caffeine-50 + if [ -d "/tmp/gnome-shell-extension-caffeine-${CAFFEINE_SHA}" ]; then + cp -r /tmp/gnome-shell-extension-caffeine-${CAFFEINE_SHA}/* "$CAFFEINE_DIR/" 2>/dev/null || true + rm -rf /tmp/gnome-shell-extension-caffeine-${CAFFEINE_SHA} fi rm -f /tmp/caffeine.tar.gz echo "[NeurOS] caffeine extension installed." diff --git a/config/includes.chroot/usr/local/bin/neuros-mcp b/config/includes.chroot/usr/local/bin/neuros-mcp index 8d938c2..f440643 100755 --- a/config/includes.chroot/usr/local/bin/neuros-mcp +++ b/config/includes.chroot/usr/local/bin/neuros-mcp @@ -36,25 +36,13 @@ class MCPServer(BaseHTTPRequestHandler): self.send_error(400, "Invalid JSON") return - method = request.get("method", "") - params = request.get("params", {}) - req_id = request.get("id") - - if method == "initialize": - result = self.handle_initialize(params) - elif method == "tools/list": - result = self.handle_list_tools() - elif method == "tools/call": - result = self.handle_call_tool(params) - elif method == "resources/list": - result = self.handle_list_resources() - elif method == "resources/read": - result = self.handle_read_resource(params) + response = dispatch_request(self, request) + if response is not None: + self.send_json(response) else: - self.send_json({"jsonrpc": "2.0", "error": {"code": -32601, "message": "Method not found"}, "id": req_id}) - return - - self.send_json({"jsonrpc": "2.0", "result": result, "id": req_id}) + # Notification (no "id") -- MCP expects no response body. + self.send_response(204) + self.end_headers() def send_json(self, data): self.send_response(200) @@ -341,6 +329,64 @@ class MCPServer(BaseHTTPRequestHandler): pass +def dispatch_request(handler, request): + """Route one JSON-RPC request to the matching handler method. + + Shared by both the HTTP transport (do_POST) and the stdio transport + (run_stdio), so tool/resource logic is only implemented once. + Returns None for notifications (no "id"), which must not get a reply. + """ + method = request.get("method", "") + params = request.get("params", {}) + req_id = request.get("id") + + if method == "notifications/initialized": + return None + + if method == "initialize": + result = handler.handle_initialize(params) + elif method == "tools/list": + result = handler.handle_list_tools() + elif method == "tools/call": + result = handler.handle_call_tool(params) + elif method == "resources/list": + result = handler.handle_list_resources() + elif method == "resources/read": + result = handler.handle_read_resource(params) + else: + return {"jsonrpc": "2.0", "error": {"code": -32601, "message": "Method not found"}, "id": req_id} + + return {"jsonrpc": "2.0", "result": result, "id": req_id} + + +def run_stdio(): + """MCP stdio transport: newline-delimited JSON-RPC on stdin/stdout. + + Per the MCP spec, the server must write ONLY JSON-RPC messages to + stdout (one per line) and may use stderr freely for logging. This is + the transport real MCP clients (Claude Code, Claude Desktop, etc.) + use to launch a local server as a subprocess. + """ + handler = MCPServer.__new__(MCPServer) + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + request = json.loads(line) + except json.JSONDecodeError: + print(json.dumps({ + "jsonrpc": "2.0", + "error": {"code": -32700, "message": "Parse error"}, + "id": None + }), flush=True) + continue + + response = dispatch_request(handler, request) + if response is not None: + print(json.dumps(response), flush=True) + + def check_ollama(): try: import urllib.request @@ -356,8 +402,14 @@ def main(): parser = argparse.ArgumentParser(description="NeurOS MCP Server") parser.add_argument("--port", type=int, default=DEFAULT_PORT, help=f"Port (default: {DEFAULT_PORT})") parser.add_argument("--host", default="127.0.0.1", help="Host (default: 127.0.0.1)") + parser.add_argument("--stdio", action="store_true", + help="Speak MCP over stdio instead of HTTP (for MCP clients that launch the server as a subprocess)") args = parser.parse_args() + if args.stdio: + run_stdio() + return + server = HTTPServer((args.host, args.port), MCPServer) print(f""" diff --git a/config/package-lists/neuros.list.chroot b/config/package-lists/neuros.list.chroot index 93fa823..9609dd6 100644 --- a/config/package-lists/neuros.list.chroot +++ b/config/package-lists/neuros.list.chroot @@ -29,7 +29,7 @@ gir1.2-gtk-3.0 gir1.2-appindicator3-0.1 # Audio (for voice/speech) -also-utils +alsa-utils espeak-ng # Screenshot & OCR diff --git a/scripts/check-airgap.sh b/scripts/check-airgap.sh new file mode 100755 index 0000000..388568b --- /dev/null +++ b/scripts/check-airgap.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# check-airgap.sh — Fail CI if a shipped runtime tool gains a new outbound +# network call to a host other than localhost/the local Ollama endpoint. +# +# Scope: config/includes.chroot/usr/local/bin/* — the scripts that run on a +# booted NeurOS system. Build-time hooks (config/hooks/live/*) legitimately +# fetch packages from the internet while building the ISO, same as any Linux +# distro's package manager; they are not part of the running system's +# air-gap guarantee and are intentionally excluded here. +# +# Any literal http(s):// URL whose host is not localhost/127.0.0.1/0.0.0.0 +# and is not on the allowlist below fails the check. This catches new +# outbound calls; it does not try to catch every possible obfuscation. +# +# Usage: ./scripts/check-airgap.sh + +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +BIN_DIR="config/includes.chroot/usr/local/bin" + +# host -> file it's allowed in (one pair per line: "file:host") +# Each entry here is a known, reviewed, user-invoked exception to the +# air-gap guarantee. Adding a new host here is a deliberate decision — +# it should show up in code review. +ALLOWLIST=" +neuros-music:www.youtube.com +neuros-network:google.com +neuros-speak:github.com +" + +FOUND=0 + +while IFS= read -r -d '' f; do + name="$(basename "$f")" + while IFS=: read -r line url; do + [ -z "${url:-}" ] && continue + host="$(echo "$url" | sed -E 's#https?://##; s#[/:].*##')" + case "$host" in + localhost|127.0.0.1|0.0.0.0|"") continue ;; + *.w3.org|w3.org) continue ;; # SVG XML namespace URI, not fetched + esac + # Skip f-string interpolated hosts (e.g. http://{config[host]}) -- these + # resolve to the local Ollama endpoint at runtime, not a literal host. + case "$host" in + \{*) continue ;; + esac + # Skip placeholder text like "https://..." in --help docstrings + case "$url" in + *"..."*) continue ;; + esac + + allowed=0 + while IFS=: read -r al_file al_host; do + [ -z "${al_file:-}" ] && continue + if [ "$name" = "$al_file" ] && [ "$host" = "$al_host" ]; then + allowed=1 + break + fi + done <<< "$ALLOWLIST" + + if [ "$allowed" -eq 0 ]; then + echo "AIR-GAP VIOLATION: $name:$line calls external host '$host' ($url)" + FOUND=1 + fi + done < <(grep -noE "https?://[^\"'\` \\)]+" "$f" 2>/dev/null || true) +done < <(find "$BIN_DIR" -maxdepth 1 -type f -not -name '*.pyc' -print0) + +if [ "$FOUND" -ne 0 ]; then + echo "" + echo "One or more shipped tools call an external host that isn't on the" + echo "air-gap allowlist in scripts/check-airgap.sh. If this is intentional" + echo "(a new opt-in, user-invoked feature), add it to the ALLOWLIST with a" + echo "one-line justification in the same PR. Otherwise, remove the call." + exit 1 +fi + +echo "Air-gap check passed: no unreviewed outbound network calls found in $BIN_DIR." diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 6f0e64a..7456595 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -69,5 +69,39 @@ def test_call_tool_unknown_name(self): self.assertIn("error", result) +class TestDispatchRequest(unittest.TestCase): + """dispatch_request is the shared router behind both the HTTP transport + (do_POST) and the stdio transport (run_stdio) -- exercise it directly + against real MCP JSON-RPC shapes, since that's what a real client sends.""" + + def setUp(self): + self.mcp = load_neuros_mcp() + self.handler = self.mcp.MCPServer.__new__(self.mcp.MCPServer) + + def test_initialize_returns_protocol_version_and_id(self): + response = self.mcp.dispatch_request(self.handler, { + "jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": "2024-11-05", "capabilities": {}, + "clientInfo": {"name": "test-client", "version": "0.1"}} + }) + self.assertEqual(response["id"], 1) + self.assertEqual(response["result"]["protocolVersion"], "2024-11-05") + + def test_notification_gets_no_response(self): + # Per the MCP spec, requests without an "id" are notifications and + # must not receive a reply. + response = self.mcp.dispatch_request( + self.handler, {"jsonrpc": "2.0", "method": "notifications/initialized"} + ) + self.assertIsNone(response) + + def test_unknown_method_returns_json_rpc_error(self): + response = self.mcp.dispatch_request( + self.handler, {"jsonrpc": "2.0", "id": 5, "method": "no/such/method"} + ) + self.assertEqual(response["error"]["code"], -32601) + self.assertEqual(response["id"], 5) + + if __name__ == "__main__": unittest.main() From 45d511d3b1784d0e93f77ed86e42e3989e8fe51d Mon Sep 17 00:00:00 2001 From: erichanwang Date: Thu, 30 Jul 2026 15:42:48 -0500 Subject: [PATCH 3/6] Fix neuros-verify --json/--quiet suppression and the tests that mis-specified them --json was meant to be an alternative single-line output mode (per the tool's own docstring) but cmd_verify printed the grep-friendly text lines unconditionally, so --json emitted text followed by JSON and callers parsing that stream as pure JSON would fail. --quiet was also silencing the aggregate line, not just the per-file lines, so a quiet run gave no overall PASS/FAIL at all. Fixed cmd_verify to gate text output on both flags correctly. Two of the four failing tests were themselves wrong: test_multi_file_and_merge asserted the literal substring "verdict=FAIL\n", which the aggregate line's documented format (verdict=... source=aggregate ...) can never produce since more fields always follow on the same line -- checked the last line's prefix instead. test_missing_script_raises assumed setting only NEUROS_POLICY_SCRIPT to a bad path was enough to force a lookup failure, but the resolver's next candidate (sibling neuros-policy next to neuros-verify) is a real file in this checkout, so it always succeeded -- now mocks os.path.isfile so every candidate misses, matching what the test actually wants to exercise. Also close the air-gap guard's biggest blind spot: check-airgap.sh only greps for literal http(s):// URLs, so a shipped tool that imports a third-party HTTP client (requests/httpx) and builds its target URL at runtime is completely invisible to it, even though that's the easiest way to add a real outbound call. Added an import-statement check for those two libraries; verified zero false positives against the current tree (no shipped tool uses them, unlike stdlib urllib which is legitimately used everywhere for local Ollama calls) and that it does catch an injected `import requests`. --- .../usr/local/bin/neuros-verify | 581 +++++++++++++++++ scripts/check-airgap.sh | 20 + tests/test_verify.py | 586 ++++++++++++++++++ 3 files changed, 1187 insertions(+) create mode 100755 config/includes.chroot/usr/local/bin/neuros-verify create mode 100755 tests/test_verify.py diff --git a/config/includes.chroot/usr/local/bin/neuros-verify b/config/includes.chroot/usr/local/bin/neuros-verify new file mode 100755 index 0000000..a05c078 --- /dev/null +++ b/config/includes.chroot/usr/local/bin/neuros-verify @@ -0,0 +1,581 @@ +#!/usr/bin/env python3 +"""neuros-verify: consume-anything verifier for the neuros toolchain. + +The toolchain's emitters (sandbox, bench, runbook, policy, replay) +all produce JSON envelopes with slightly different shapes. +`neuros-verify` is the single consumer that reads any of them, +identifies the kind by version key first then by structural +fingerprint, and emits a normalized PASS/FAIL verdict line on +stdout. Multiple files are AND'd so any single failure fails the +overall run. + +This tool does not introduce new envelope shapes; it routes what +already exists. The single edge is `--policy ` for raw +sandbox envelopes: the policy's `check_envelope_against_policy` +is loaded via runtime `compile()+exec()` against the on-disk +neuros-policy script so this binary stays a thin client of the +verifier logic instead of re-implementing it. + +Usage: + + neuros-verify [--policy POLICY] [--bench-baseline BASELINE] + [--json] [--quiet] + [FILE ... | -] + +Without --json the output is grep-friendly: + + verdict=PASS source=sandbox file=./env.json violations=0 ... + verdict=PASS source=aggregate files=3 passes=3 failures=0 + +With --json a single-line envelope with {ok, aggregate, parts[]}: + + {"ok": true, "aggregate": {...}, "parts": [...], + "neuros_verify_version": "0.1"} + +Exit codes: 0 all conformant; 1 any rule violation or unrecognized +envelope; 2 bad input (missing file, malformed JSON, root not an +object). + +Note: bench envelopes without --bench-baseline are evaluated as +"no run crashed" (every run's exit_code is 0). That is the safe +default — performance regression checks live in `neuros-bench +compare` and `neuros-replay diff`. Pass `--bench-baseline` here +only when you want a single-line verdict side-by-side with the +other consumer verdicts. +""" +import argparse +import json +import os +import sys +import types + +# --- Constants ---------------------------------------------------------- + +#: Tool version baked into emitted envelopes. +NEUROS_VERIFY_VERSION = "0.1" + +#: Order in which we search for the neuros-policy SCRIPT when we +#: need its ``_load_policy`` / ``validate_policy`` / +#: ``check_envelope_against_policy`` helpers. The SCRIPT and the +#: MANIFEST are distinct: --policy is the manifest, and the +#: script loader auto-discovers via this candidate list. On a +#: system install the second-to-last entry hits; in dev mode +#: the first entry (sibling of neuros-verify) keeps it working +#: without an install step. +#: lazily-built candidate list of neuros-policy script paths; +#: built per call inside _policy_script_candidates() so the +#: env override (NEUROS_POLICY_SCRIPT) is read at call time, +#: not at module-import time. +#: Default tolerances for the bench-with-baseline path. Same as +#: neuros-bench compare's defaults; documented here so a routing +#: decision made in this tool matches the policy of the underlying +#: bench workload. +BENCH_TOL_WALL_PCT = 15.0 +BENCH_TOL_MEM_PCT = 25.0 + +#: Detected envelope kinds, in resolution priority order. First match +#: wins; structural fingerprint fallback comes after the explicit +#: version-key checks. +_KIND_VERSION_KEYS = ( + ("neuros_bench_version", "bench"), + ("neuros_policy_version", "policy"), + ("neuros_replay_version", "replay"), +) + + +# --- Exceptions -------------------------------------------------------- + + +class VerifyError(Exception): + """Raised on user-fixable misconfiguration (missing file, malformed + JSON, wrong root type). Surfaced as exit code 2.""" + + +# --- Helpers ------------------------------------------------------------ + + +def _die(msg, code=2): + print(msg, file=sys.stderr) + sys.exit(code) + + +def _load_envelope(path): + """Read one JSON envelope from ``path`` ('-' for stdin).""" + if path == "-": + try: + text = sys.stdin.read() + except OSError as e: + raise VerifyError(f"cannot read stdin: {e}") + else: + try: + with open(path, "r", encoding="utf-8") as f: + text = f.read() + except OSError as e: + raise VerifyError(f"cannot read envelope {path!r}: {e}") + try: + obj = json.loads(text) + except json.JSONDecodeError as e: + raise VerifyError(f"envelope {path!r} is not valid JSON: {e}") + if not isinstance(obj, dict): + raise VerifyError( + f"envelope must be a JSON object (got {type(obj).__name__})") + return obj + + +def _detect_kind(env): + """Return one of {'bench', 'policy', 'replay', 'sandbox', + 'unknown'}. Version keys take priority; structural fingerprint + fallback for raw sandbox envelopes; everything else is unknown. + """ + for key, kind in _KIND_VERSION_KEYS: + if key in env: + return kind + # Runbook is detected by structural fingerprint of its top-level + # {runbook_path, steps} record. If the runbook tool doesn't yet + # emit that shape, fall through to sandbox/unknown. + if "runbook_path" in env and "steps" in env and isinstance( + env.get("steps"), list): + return "runbook" + if ("exit_code" in env + and "wall_clock_ms" in env + and "timeout_hit" in env): + return "sandbox" + return "unknown" + + +def _pct_delta(a, b): + """Round-to-2dp percent change; ``None`` if either is missing or + the denominator is zero. Mirrors neuros-replay's helper so a + verdict from this tool and a verdict from replay agree on + numeric shape. + """ + if a is None or b is None: + return None + try: + a_f = float(a); b_f = float(b) + except (TypeError, ValueError): + return None + if a_f == 0: + return None + return round((b_f - a_f) / a_f * 100.0, 2) + + +def _load_policy_module(path): + """``compile()+exec()`` the policy script into a types.ModuleType + so we can call ``check_envelope_against_policy`` without importing + it as a real package. Mirrors the test_sandbox / test_policy + loader. Returns the module. + """ + with open(path, "r", encoding="utf-8") as f: + src = f.read() + if src.startswith("#!"): + src = src.split("\n", 1)[1] + code = compile(src, path, "exec") + mod = types.ModuleType("neuros_policy_loader") + mod.__file__ = path + exec(code, mod.__dict__) + return mod + + +# --- Per-kind verifier dispatch ---------------------------------------- + + +def _verify_sandbox(env, policy_module, manifest_path): + """Route a sandbox envelope through policy's + check_envelope_against_policy in one pass: load the manifest + from ``manifest_path`` (the JSON file the CLI's --policy + points at), call ``validate_policy`` (refuses to check + against an invalid manifest), then call + ``check_envelope_against_policy``. Each tuple is rendered as + a grep-friendly violation string. + """ + # Defensive ward: this tool calls three names on the policy + # module via runtime compile()+exec(). If neuros-policy renames + # any of these in a future release we want a loud failure here + # rather than an opaque AttributeError deep in the call chain. + expected = ("_load_policy", "validate_policy", + "check_envelope_against_policy") + missing = [n for n in expected if not hasattr(policy_module, n)] + if missing: + public_names = sorted( + k for k in policy_module.__dict__ + if not k.startswith('_')) + return False, [ + f"policy module missing expected names: {missing} " + f"(got: {public_names})" + ] + policy_obj = policy_module._load_policy(manifest_path) + errs, cleaned = policy_module.validate_policy(policy_obj) + if errs: + return False, [ + f"policy has {len(errs)} violation(s); refusing to " + f"check envelope" + ] + violations_t = policy_module.check_envelope_against_policy( + policy_obj, env, cleaned) + return (not violations_t), [ + f"[{sev}] {rule}: observed {obs!r}, expected {exp!r}" + for (rule, obs, exp, sev) in violations_t + ] + + +#: Module-level cache for the loaded policy script. Loading is +#: expensive (compile()+exec of a ~20 KB source), and neuros-verify +#: can process many files in a single invocation when given a +#: glob — recompiling per file would be wasted work. +_CACHED_POLICY_SCRIPT_MOD = None +_CACHED_POLICY_SCRIPT_PATH = None + + +def _policy_script_candidates(): + """Build the script candidate list lazily so the env override + (NEUROS_POLICY_SCRIPT) is read at call time, not at + module-import time. + """ + here = os.path.dirname(os.path.abspath(__file__)) + return ( + os.environ.get("NEUROS_POLICY_SCRIPT"), + os.path.join(here, "neuros-policy"), + "/usr/local/bin/neuros-policy", + ) + + +def _resolve_policy_script(): + """Locate the neuros-policy SCRIPT and load it as a module. + Searches the candidate list in order: ENV override, sibling + of neuros-verify (dev mode), then the system install path. + Caches the loaded module so subsequent calls reuse it. + Raises VerifyError on a missing script. + """ + global _CACHED_POLICY_SCRIPT_MOD, _CACHED_POLICY_SCRIPT_PATH + if _CACHED_POLICY_SCRIPT_MOD is not None: + return _CACHED_POLICY_SCRIPT_MOD + cands = _policy_script_candidates() + for cand in cands: + if not cand: + continue + if os.path.isfile(cand): + _CACHED_POLICY_SCRIPT_PATH = cand + _CACHED_POLICY_SCRIPT_MOD = _load_policy_module(cand) + return _CACHED_POLICY_SCRIPT_MOD + raise VerifyError( + f"cannot locate neuros-policy script (looked at: " + f"{[c for c in cands if c]})") + + +def _verify_bench(env, baseline_path): + """Per-run envelope within tolerances vs a baseline JSON. Each + per-run violation is its own string; the overall verdict is + AND of all per-run results. + """ + if baseline_path is None: + # Without baseline we degrade to crash check: every run's + # exit_code must be 0. + runs = env.get("runs") or [] + if not isinstance(runs, list): + return False, ["bench envelope has no 'runs' list"] + violations = [ + f"run[{i}] exit_code={r.get('exit_code')!r}: non-zero " + f"without baseline" + for i, r in enumerate(runs) + if r.get("exit_code") not in (0, None) + ] + return (not violations), violations + try: + with open(baseline_path, "r") as f: + baseline = json.load(f) + except (OSError, json.JSONDecodeError) as e: + return False, [f"cannot read --bench-baseline: {e}"] + if not isinstance(baseline, dict) or "runs" not in baseline: + return False, [ + f"baseline {baseline_path!r} is not a bench metrics.json" + ] + base_runs = {r.get("name"): r + for r in (baseline.get("runs") or []) + if isinstance(r, dict)} + cand_runs = env.get("runs") or [] + if not isinstance(cand_runs, list): + return False, ["bench envelope has no 'runs' list"] + violations = [] + for r in cand_runs: + if not isinstance(r, dict): + continue + name = r.get("name") + b = base_runs.get(name) + if b is None: + # New workload in candidate is not a regression. + continue + wpct = _pct_delta(b.get("wall_clock_ms"), + r.get("wall_clock_ms")) + if wpct is not None and abs(wpct) > BENCH_TOL_WALL_PCT: + violations.append( + f"run[{name}] wall_clock_pct_delta {wpct}% exceeds " + f"{BENCH_TOL_WALL_PCT}%") + mpct = _pct_delta(b.get("peak_mem_estimate"), + r.get("peak_mem_estimate")) + if (mpct is not None + and b.get("peak_mem_estimate") is not None + and r.get("peak_mem_estimate") is not None + and abs(mpct) > BENCH_TOL_MEM_PCT): + violations.append( + f"run[{name}] peak_mem_pct_delta {mpct}% exceeds " + f"{BENCH_TOL_MEM_PCT}%") + if r.get("exit_code") != b.get("exit_code"): + violations.append( + f"run[{name}] exit_code differs " + f"({b.get('exit_code')!r} vs {r.get('exit_code')!r})") + if r.get("timeout_hit") != b.get("timeout_hit"): + violations.append( + f"run[{name}] timeout_hit differs " + f"({b.get('timeout_hit')!r} vs " + f"{r.get('timeout_hit')!r})") + return (not violations), violations + + +def _verify_policy_verdict(env): + """A policy envelope with an `ok` field; result = not ok.""" + ok = env.get("ok") + if ok is True: + return True, [] + if ok is False: + violations = [str(e) for e in env.get("errors", []) + if isinstance(e, str)] + return False, violations or ["policy verdict ok=False"] + return False, [f"policy envelope has unexpected ok value {ok!r}"] + + +def _verify_replay_verdict(env): + """A replay envelope with an `ok` field; result = not ok.""" + ok = env.get("ok") + if ok is True: + return True, [] + if ok is False: + violations = [str(v) for v in env.get("rule_violations", []) + if isinstance(v, str)] + return False, violations or ["replay verdict ok=False"] + return False, [f"replay envelope has unexpected ok value {ok!r}"] + + +def _verify_runbook(env): + """Per-step structural smoke check. The full assertion-pass + semantics live in the runbook tool itself; here we only verify + the envelope shape (runbook_path present, steps is a list, each + step has a name + ok field). + """ + if "runbook_path" not in env or "steps" not in env: + return False, ["runbook envelope missing runbook_path or steps"] + steps = env.get("steps") + if not isinstance(steps, list): + return False, ["runbook envelope steps is not a list"] + violations = [] + for i, s in enumerate(steps): + if not isinstance(s, dict): + violations.append(f"steps[{i}] is not an object") + continue + if "name" not in s or "ok" not in s: + violations.append( + f"steps[{i}] missing name or ok") + return (not violations), violations + + +def _verify_unknown(env): + """An unrecognized envelope is treated as a violation. We + surface the detected top-level keys so the caller can decide + whether the JSON came from a wrong tool. + """ + return False, [ + f"unrecognized envelope (top-level keys: " + f"{', '.join(sorted(env.keys()))})" + ] + + +# --- Runbook envelope helper ------------------------------------------ + + +def run_verify_for_file(path, args): + """Drive verification for a single file. Returns + ``(kind, ok, violations, metrics_for_line)``. + """ + env = _load_envelope(path) + kind = _detect_kind(env) + if kind == "sandbox": + if args.policy is None: + return (kind, False, [ + "sandbox envelope requires --policy "], {}) + try: + policy_mod = _resolve_policy_script() + except VerifyError as e: + return (kind, False, [f"cannot load policy: {e}"], {}) + ok, viols = _verify_sandbox(env, policy_mod, args.policy) + return (kind, ok, viols, _sandbox_metrics(env)) + if kind == "bench": + ok, viols = _verify_bench(env, args.bench_baseline) + return (kind, ok, viols, _bench_metrics(env)) + if kind == "policy": + ok, viols = _verify_policy_verdict(env) + return (kind, ok, viols, _verdict_metrics(env)) + if kind == "replay": + ok, viols = _verify_replay_verdict(env) + return (kind, ok, viols, _verdict_metrics(env)) + if kind == "runbook": + ok, viols = _verify_runbook(env) + return (kind, ok, viols, {}) + return (kind, False, _verify_unknown(env)[1], {}) + + +def _sandbox_metrics(env): + return { + "exit_code": env.get("exit_code"), + "wall_clock_ms": env.get("wall_clock_ms"), + "peak_mem_estimate": env.get("peak_mem_estimate"), + "timeout_hit": env.get("timeout_hit", False), + } + + +def _bench_metrics(env): + runs = env.get("runs") or [] + return { + "workloads": len(runs) + if isinstance(runs, list) else 0, + "total_wall_clock_ms": env.get("total_wall_clock_ms"), + "neuros_bench_version": env.get("neuros_bench_version"), + } + + +def _verdict_metrics(env): + return { + "rule_violations": len(env.get("rule_violations") + or env.get("errors") or []), + "ok": env.get("ok"), + } + + +# --- Output formatters ------------------------------------------------- + + +def _format_text_line(verdict, kind, file_path, violations, metrics): + base = (f"verdict={'PASS' if verdict else 'FAIL'} " + f"source={kind} file={file_path} " + f"violations={len(violations)}") + extras = [] + for k in ("exit_code", "wall_clock_ms", "peak_mem_estimate", + "timeout_hit", "workloads", "total_wall_clock_ms", + "rule_violations", "ok"): + v = metrics.get(k) + if v is not None: + extras.append(f"{k}={v}") + if extras: + base += " " + " ".join(extras) + return base + + +def _format_aggregate_text(passes, failures): + return (f"verdict={'PASS' if failures == 0 else 'FAIL'} " + f"source=aggregate files_checked={passes + failures} " + f"passes={passes} failures={failures}") + + +# --- Top-level dispatch ----------------------------------------------- + + +def cmd_verify(args): + paths = args.files or ["-"] + parts = [] + passes = failures = 0 + # --json is a single-line-envelope output mode: it replaces the + # grep-friendly text lines rather than adding to them. --quiet + # only suppresses the per-file text lines; the aggregate line + # still prints so a quiet run still reports overall PASS/FAIL. + show_text = not args.json + show_per_file = show_text and not args.quiet + for p in paths: + kind, ok, viols, metrics = run_verify_for_file(p, args) + if show_per_file: + print(_format_text_line(ok, kind, p, viols, metrics), + flush=True) + parts.append({ + "file": p, + "source_kind": kind, + "ok": ok, + "violations": viols, + "metrics": metrics, + }) + if ok: + passes += 1 + else: + failures += 1 + if show_text: + print(_format_aggregate_text(passes, failures), + flush=True) + if args.json: + _emit_json_envelope({ + "ok": failures == 0, + "aggregate": { + "files_checked": passes + failures, + "passes": passes, + "failures": failures, + }, + "parts": parts, + "neuros_verify_version": NEUROS_VERIFY_VERSION, + }) + return 0 if failures == 0 else 1 + + +def _emit_json_envelope(payload): + """Print a single-line envelope mirroring the shape of the + other tools' --json modes. + """ + print(json.dumps(payload, separators=(",", ":"), + ensure_ascii=False)) + + +def _build_parser(): + p = argparse.ArgumentParser( + prog="neuros-verify", + description=( + "Consume-anything verifier for the neuros toolchain " + "JSON envelopes. Reads sandbox / bench / policy / " + "replay envelopes and emits a normalized PASS/FAIL " + "verdict line. Multiple files are AND'd." + ), + epilog=( + "Examples:\n" + " neuros-verify ./envelope.json --policy ./policy.json\n" + " neuros-verify ./baseline.json ./candidate.json " + "--bench-baseline ./baseline.json\n" + " neuros-verify ./env1.json ./env2.json --json" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("--policy", + help="For raw sandbox envelopes, run the " + "named policy manifest's check. Loaded " + "via runtime compile()+exec().") + p.add_argument("--bench-baseline", + help="For bench metrics envelopes, diff each " + "per-workload run against this baseline " + "(tolerances match `neuros-bench compare`).") + p.add_argument("--json", action="store_true", + help="Emit a single-line JSON envelope instead " + "of text") + p.add_argument("--quiet", action="store_true", + help="Suppress per-file lines; only emit the " + "aggregate (last) line") + p.add_argument("files", nargs="*", + help="Envelope files to verify ('-' for stdin)") + return p + + +def main(argv=None): + args = _build_parser().parse_args(argv) + if not args.files: + # Default to stdin so the tool is usable in pipes. + args.files = ["-"] + try: + return cmd_verify(args) + except VerifyError as e: + _die(f"neuros-verify: {e}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-airgap.sh b/scripts/check-airgap.sh index 388568b..e6d69d3 100755 --- a/scripts/check-airgap.sh +++ b/scripts/check-airgap.sh @@ -31,6 +31,26 @@ neuros-speak:github.com FOUND=0 +# Literal-URL scanning (below) misses outbound calls made through a +# third-party HTTP client whose target URL is built at runtime (env +# var, f-string, config value) rather than typed as a literal in the +# file. stdlib urllib is used throughout these tools to talk to the +# local Ollama endpoint, so importing it is not itself a signal. +# `requests`/`httpx` are not stdlib, are not used anywhere in this +# tree today, and exist only to make arbitrary HTTP calls ergonomic - +# so any import of one is a hard fail here regardless of URL literals. +while IFS= read -r -d '' f; do + name="$(basename "$f")" + match="$(grep -nE '^\s*(import|from)\s+(requests|httpx)\b' "$f" 2>/dev/null || true)" + if [ -n "$match" ]; then + echo "AIR-GAP VIOLATION: $name imports a non-stdlib HTTP client not used " \ + "anywhere else in this tree (can make outbound calls with a " \ + "runtime-built URL that the literal-URL scan below can't see):" + echo "$match" | sed "s#^# $name:#" + FOUND=1 + fi +done < <(find "$BIN_DIR" -maxdepth 1 -type f -not -name '*.pyc' -print0) + while IFS= read -r -d '' f; do name="$(basename "$f")" while IFS=: read -r line url; do diff --git a/tests/test_verify.py b/tests/test_verify.py new file mode 100755 index 0000000..a31b304 --- /dev/null +++ b/tests/test_verify.py @@ -0,0 +1,586 @@ +"""Tests for neuros-verify (consume-anything verifier). + +Runs purely against the production script via compile()+exec() so we +do not need it on $PATH. No subprocess shell-out is exercised here +on purpose: every interesting invariant (kind detection, per-kind +verification, aggregate merge, --json shape) is reachable from +Python directly. Sandbox+policy integration is tested by writing a +small on-disk policy manifest in a tempdir and pointing verify at +it via --policy. +""" +import compileall +import io +import json +import os +import subprocess +import sys +import tempfile +import types +import unittest +from unittest import mock + +VERIFY_PATH = "config/includes.chroot/usr/local/bin/neuros-verify" +POLICY_PATH = "config/includes.chroot/usr/local/bin/neuros-policy" + + +def _load_module(): + with open(VERIFY_PATH, "r", encoding="utf-8") as f: + src = f.read() + if src.startswith("#!"): + src = src.split("\n", 1)[1] + code = compile(src, VERIFY_PATH, "exec") + mod = types.ModuleType("neuros_verify_under_test") + # Set __file__ BEFORE exec so the production module's + # top-level references to __file__ resolve correctly + # (e.g. the policy script resolver's candidate list). + mod.__file__ = VERIFY_PATH + exec(code, mod.__dict__) + return mod + + +def _write_json(obj): + """Dump ``obj`` to a temp .json file and return its path string. + """ + fd, p = tempfile.mkstemp(prefix="neuros-verify-", suffix=".json") + with os.fdopen(fd, "w") as f: + json.dump(obj, f) + return p + + +def _good_policy(): + """A conformant policy manifest for sandbox+policy integration + tests. Same shape as test_policy._good_policy. + """ + return { + "name": "neuros-default", + "version": "1.0.0", + "defaults": { + "mem": "256M", + "pids": 64, + "cpu_quota": 50000, + "timeout": 30, + }, + "profiles": { + "strict": ["CAP_NET_RAW", "CAP_SYS_ADMIN"], + "moderate": ["CAP_NET_RAW"], + "permissive": [], + }, + "net": "private", + "readonly": True, + "env_allowlist": ["PATH"], + "syscalls": None, + } + + +def _write_policy(): + return _write_json(_good_policy()) + + +def _ok_envelope(**overrides): + base = { + "exit_code": 0, "stdout": "", "stderr": "", + "wall_clock_ms": 1000, "timeout_hit": False, + "peak_mem_estimate": 64 * 1024 * 1024, + } + base.update(overrides) + return base + + +def _bench_metrics(runs=None): + return { + "runs": runs if runs is not None else [ + { + "name": "cpu_tight", "wall_clock_ms": 1100, + "exit_code": 0, "stdout": "", "stderr": "", + "timeout_hit": False, + "peak_mem_estimate": 32 * 1024 * 1024, + }, + { + "name": "json_parse", "wall_clock_ms": 900, + "exit_code": 0, "stdout": "", "stderr": "", + "timeout_hit": False, + "peak_mem_estimate": 18 * 1024 * 1024, + }, + ], + "total_wall_clock_ms": 2000, + "config": {}, + "neuros_bench_version": "0.1", + } + + +class TestCompile(unittest.TestCase): + def test_compiles_clean(self): + self.assertTrue(compileall.compile_file(VERIFY_PATH, + quiet=1, + force=True)) + + +class TestDetectKind(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_bench_by_version_key(self): + self.assertEqual(self.vf._detect_kind(_bench_metrics()), + "bench") + + def test_policy_by_version_key(self): + env = {"ok": True, "errors": [], "name": "x", + "version": "1.0.0", "neuros_policy_version": "0.1"} + self.assertEqual(self.vf._detect_kind(env), "policy") + + def test_replay_by_version_key(self): + env = {"ok": True, "neuros_replay_version": "0.1", + "rule_violations": []} + self.assertEqual(self.vf._detect_kind(env), "replay") + + def test_runbook_by_structure(self): + env = {"runbook_path": "/tmp/rb.json", + "steps": [{"name": "x", "ok": True}]} + self.assertEqual(self.vf._detect_kind(env), "runbook") + + def test_sandbox_by_structure(self): + self.assertEqual(self.vf._detect_kind(_ok_envelope()), + "sandbox") + + def test_runbook_priority_over_sandbox(self): + # If both structural patterns match, runbook wins because + # it's checked first (has the more specific runbook_path). + env = {"runbook_path": "/x", "steps": [], + "exit_code": 0, "wall_clock_ms": 0, + "timeout_hit": False} + self.assertEqual(self.vf._detect_kind(env), "runbook") + + def test_unknown(self): + self.assertEqual(self.vf._detect_kind({"foo": "bar"}), + "unknown") + + +class TestLoadEnvelope(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_missing_file(self): + with self.assertRaises(self.vf.VerifyError) as cm: + self.vf._load_envelope("/nonexistent/env.json") + self.assertIn("cannot read", str(cm.exception)) + + def test_malformed_json(self): + fd, p = tempfile.mkstemp(prefix="nv-bad-", suffix=".json") + try: + with os.fdopen(fd, "w") as f: + f.write("not json {{{") + with self.assertRaises(self.vf.VerifyError) as cm: + self.vf._load_envelope(p) + self.assertIn("not valid JSON", str(cm.exception)) + finally: + os.unlink(p) + + def test_root_must_be_object(self): + path = _write_json([1, 2, 3]) + try: + with self.assertRaises(self.vf.VerifyError): + self.vf._load_envelope(path) + finally: + os.unlink(path) + + +class TestPctDelta(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_normal(self): + self.assertEqual(self.vf._pct_delta(1000, 1100), 10.0) + self.assertEqual(self.vf._pct_delta(1000, 900), -10.0) + + def test_zero_div(self): + self.assertIsNone(self.vf._pct_delta(0, 100)) + + def test_missing(self): + self.assertIsNone(self.vf._pct_delta(None, 100)) + self.assertIsNone(self.vf._pct_delta(100, None)) + + +class TestVerifyPolicyVerdict(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_ok_true(self): + ok, viols = self.vf._verify_policy_verdict( + {"ok": True, "errors": []}) + self.assertTrue(ok) + self.assertEqual(viols, []) + + def test_ok_false_with_errors(self): + ok, viols = self.vf._verify_policy_verdict( + {"ok": False, "errors": ["bad mem"]}) + self.assertFalse(ok) + self.assertIn("bad mem", viols) + + def test_ok_false_without_errors(self): + ok, viols = self.vf._verify_policy_verdict({"ok": False}) + self.assertFalse(ok) + self.assertEqual(len(viols), 1) + + def test_ok_other_type(self): + ok, viols = self.vf._verify_policy_verdict({"ok": "yes"}) + self.assertFalse(ok) + + +class TestVerifyReplayVerdict(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_ok_true(self): + ok, viols = self.vf._verify_replay_verdict( + {"ok": True, "rule_violations": []}) + self.assertTrue(ok) + + def test_ok_false_with_rule_violations(self): + ok, viols = self.vf._verify_replay_verdict({ + "ok": False, + "rule_violations": ["wall_clock_pct_delta 50%"]}) + self.assertFalse(ok) + self.assertEqual(viols, + ["wall_clock_pct_delta 50%"]) + + +class TestVerifyBench(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_no_runs(self): + ok, _ = self.vf._verify_bench({"runs": []}, None) + self.assertTrue(ok) + + def test_runs_not_a_list(self): + ok, viols = self.vf._verify_bench({"runs": "nope"}, None) + self.assertFalse(ok) + + def test_runs_with_one_nonzero_exit(self): + ok, viols = self.vf._verify_bench( + _bench_metrics([{"name": "x", "exit_code": 1, + "wall_clock_ms": 0}]), + None) + self.assertFalse(ok) + self.assertTrue(any("exit_code=1" in v for v in viols)) + + def test_with_baseline_missing_file(self): + ok, viols = self.vf._verify_bench( + _bench_metrics(), "/nonexistent/baseline.json") + self.assertFalse(ok) + + def test_with_baseline_wall_clock_regression(self): + base = _bench_metrics() + cand = _bench_metrics() + cand["runs"][0]["wall_clock_ms"] = 5000 # +354% + b = _write_json(base) + try: + ok, viols = self.vf._verify_bench(cand, b) + self.assertFalse(ok) + self.assertTrue(any("wall_clock_pct_delta" in v + for v in viols)) + finally: + os.unlink(b) + + def test_with_baseline_conformant(self): + b = _write_json(_bench_metrics()) + try: + ok, viols = self.vf._verify_bench(_bench_metrics(), b) + self.assertTrue(ok) + self.assertEqual(viols, []) + finally: + os.unlink(b) + + def test_with_baseline_new_run_in_candidate_not_violation(self): + base = _bench_metrics(runs=[{ + "name": "cpu_tight", "wall_clock_ms": 1100, + "exit_code": 0, "timeout_hit": False, + "peak_mem_estimate": 32 * 1024 * 1024}]) + cand = _bench_metrics() + # Candidate has an extra run (json_parse) not in baseline. + b = _write_json(base) + try: + ok, viols = self.vf._verify_bench(cand, b) + # The existing cpu_tight run is conformant and the new + # json_parse run is not a regression (no baseline entry). + self.assertTrue(ok) + finally: + os.unlink(b) + + +class TestVerifyRunbook(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_conformant(self): + env = {"runbook_path": "/tmp/rb.json", + "steps": [{"name": "a", "ok": True}, + {"name": "b", "ok": True}]} + ok, viols = self.vf._verify_runbook(env) + self.assertTrue(ok) + + def test_missing_path(self): + ok, viols = self.vf._verify_runbook({"steps": []}) + self.assertFalse(ok) + + def test_step_not_object(self): + env = {"runbook_path": "/x", + "steps": ["stringy", {"name": "y", "ok": True}]} + ok, viols = self.vf._verify_runbook(env) + self.assertFalse(ok) + self.assertTrue(any("not an object" in v for v in viols)) + + def test_step_missing_name(self): + env = {"runbook_path": "/x", + "steps": [{"ok": True}]} + ok, viols = self.vf._verify_runbook(env) + self.assertFalse(ok) + + +class TestVerifySandbox(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_conformant_envelope(self): + # Load the policy MODULE (Python source) and pass a + # separate MANIFEST path to _verify_sandbox. The SCRIPT + # is reusable across manifests; the manifest is per-run. + with open(POLICY_PATH, "r") as f: + psrc = f.read() + if psrc.startswith("#!"): + psrc = psrc.split("\n", 1)[1] + pcode = compile(psrc, POLICY_PATH, "exec") + pmod = types.ModuleType("policy_loader") + pmod.__file__ = POLICY_PATH + exec(pcode, pmod.__dict__) + + manifest_path = _write_policy() + try: + ok, viols = self.vf._verify_sandbox( + _ok_envelope(), pmod, manifest_path) + self.assertTrue(ok) + finally: + os.unlink(manifest_path) + + def test_violation_envelope(self): + with open(POLICY_PATH, "r") as f: + psrc = f.read() + if psrc.startswith("#!"): + psrc = psrc.split("\n", 1)[1] + pcode = compile(psrc, POLICY_PATH, "exec") + pmod = types.ModuleType("policy_loader") + pmod.__file__ = POLICY_PATH + exec(pcode, pmod.__dict__) + + manifest_path = _write_policy() + try: + # timeout_hit=True is always a violation. + ok, viols = self.vf._verify_sandbox( + _ok_envelope(timeout_hit=True), + pmod, manifest_path) + self.assertFalse(ok) + self.assertTrue(any("timeout_hit" in v + for v in viols)) + finally: + os.unlink(manifest_path) + + +class TestVerifyUnknown(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_unrecognized_envelope(self): + ok, viols = self.vf._verify_unknown({"foo": "bar"}) + self.assertFalse(ok) + self.assertTrue(any("unrecognized" in v for v in viols)) + + +class TestMainDispatch(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_text_line_shape(self): + path = _write_json({"ok": True, "errors": [], + "neuros_policy_version": "0.1"}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.vf.main([path]) + self.assertEqual(rc, 0) + out = buf.getvalue() + self.assertIn("verdict=PASS", out) + self.assertIn("source=policy", out) + self.assertIn("violations=0", out) + self.assertIn("source=aggregate", out) + finally: + os.unlink(path) + + def test_text_line_for_sandbox_without_policy_fails(self): + path = _write_json(_ok_envelope()) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.vf.main([path]) + self.assertEqual(rc, 1) + self.assertIn("verdict=FAIL", buf.getvalue()) + self.assertIn("source=sandbox", buf.getvalue()) + self.assertIn("violations=1", buf.getvalue()) + finally: + os.unlink(path) + + def test_text_line_for_sandbox_with_policy_passes(self): + path = _write_json(_ok_envelope()) + policy_path = _write_policy() + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.vf.main([path, "--policy", policy_path]) + self.assertEqual(rc, 0) + self.assertIn("verdict=PASS", buf.getvalue()) + self.assertIn("source=sandbox", buf.getvalue()) + finally: + os.unlink(path) + os.unlink(policy_path) + + def test_unknown_envelope_verdict_fail(self): + path = _write_json({"foo": 1}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.vf.main([path]) + self.assertEqual(rc, 1) + self.assertIn("verdict=FAIL", buf.getvalue()) + self.assertIn("source=unknown", buf.getvalue()) + finally: + os.unlink(path) + + def test_json_envelope_shape(self): + path = _write_json({"ok": True, "errors": [], + "neuros_policy_version": "0.1"}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.vf.main([path, "--json"]) + self.assertEqual(rc, 0) + obj = json.loads(buf.getvalue().strip()) + self.assertTrue(obj["ok"]) + self.assertEqual(obj["aggregate"]["passes"], 1) + self.assertEqual(obj["aggregate"]["failures"], 0) + self.assertEqual(len(obj["parts"]), 1) + self.assertEqual(obj["parts"][0]["source_kind"], "policy") + self.assertIn("neuros_verify_version", obj) + finally: + os.unlink(path) + + def test_quiet_suppresses_per_file_lines(self): + path = _write_json({"ok": True, "errors": [], + "neuros_policy_version": "0.1"}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.vf.main([path, "--quiet"]) + self.assertEqual(rc, 0) + out = buf.getvalue() + # No per-file 'source=policy' line. + self.assertNotIn("source=policy", out) + self.assertIn("source=aggregate", out) + finally: + os.unlink(path) + + def test_multi_file_and_merge(self): + good = _write_json({"ok": True, "errors": [], + "neuros_policy_version": "0.1"}) + bad = _write_json({"ok": False, "errors": ["x"], + "neuros_policy_version": "0.1"}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.vf.main([good, bad]) + # One file passes, one fails → overall FAIL. + self.assertEqual(rc, 1) + out = buf.getvalue() + self.assertIn("passes=1", out) + self.assertIn("failures=1", out) + # The aggregate (last) line must report the overall FAIL + # verdict, not just one of the per-file lines. + last_line = out.strip().splitlines()[-1] + self.assertTrue(last_line.startswith("verdict=FAIL ")) + self.assertIn("source=aggregate", last_line) + finally: + os.unlink(good); os.unlink(bad) + + def test_missing_file_returns_two(self): + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + with self.assertRaises(SystemExit): + self.vf.main(["/nonexistent/env.json"]) + + def test_bench_with_baseline(self): + base = _bench_metrics() + cand = _bench_metrics() + cand_path = _write_json(cand) + base_path = _write_json(base) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.vf.main([cand_path, + "--bench-baseline", base_path]) + self.assertEqual(rc, 0) + self.assertIn("source=bench", buf.getvalue()) + self.assertIn("violations=0", buf.getvalue()) + finally: + os.unlink(cand_path); os.unlink(base_path) + + +class TestResolvePolicyScript(unittest.TestCase): + """Defensive: the resolver raises VerifyError with a clear + message when NO candidate path exists. In a dev checkout the + sibling-of-neuros-verify candidate is a real file, so the env + override alone isn't enough to force a miss -- os.path.isfile + is patched to make every candidate look absent. + """ + + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + @classmethod + def tearDownClass(cls): + # Clear the module-level cache so subsequent test classes + # re-resolve with the live env (the cache is shared + # singleton state that we don't want to leak across + # test classes). + cls.vf._CACHED_POLICY_SCRIPT_MOD = None + cls.vf._CACHED_POLICY_SCRIPT_PATH = None + + def test_missing_script_raises(self): + with mock.patch.dict(os.environ, + {"NEUROS_POLICY_SCRIPT": + "/nonexistent/script"}, + clear=False), \ + mock.patch("os.path.isfile", return_value=False): + self.vf._CACHED_POLICY_SCRIPT_MOD = None + self.vf._CACHED_POLICY_SCRIPT_PATH = None + with self.assertRaises(self.vf.VerifyError) as cm: + self.vf._resolve_policy_script() + msg = str(cm.exception) + self.assertIn("cannot locate", msg) + self.assertIn("/nonexistent/script", msg) + + +if __name__ == "__main__": + unittest.main() From 09a5c3606c5f1845cde652f24c22775cbe107376 Mon Sep 17 00:00:00 2001 From: erichanwang Date: Thu, 30 Jul 2026 15:48:41 -0500 Subject: [PATCH 4/6] Wire all test files into CI and close remaining air-gap guard gaps All 13 tests/test_*.py files pass individually and together (383 tests, 0 failures via unittest discover), so CI now runs discover instead of listing test_nn.py/test_mcp.py by name -- every current and future test file gates merges without another wiring PR. check-airgap.sh gaps closed: - Scans were -maxdepth 1, invisible to anything in a subdirectory of the tools path. Now recursive (excluding __pycache__). - neuros-network's socket.create_connection(("1.1.1.1", 53)) and its nslookup subprocess call contain no http(s):// literal, so the existing URL scan couldn't see them. Added a raw-socket/DNS-tool scan and allowlisted the two known-intentional neuros-network calls explicitly (1.1.1.1, google.com) instead of leaving them undetected. - Investigated removing the neuros-speak:github.com allowlist entry (per the stated goal that it only covers a URL printed in a help message, never fetched) -- confirmed removing it currently breaks the check, since the literal URL scan has no existing skip for "URL that only appears in a print statement". Left in place rather than silently reintroducing a false positive. Verified zero false positives across all 86 shipped tools after each change, and that the new detections still catch injected violations (raw socket + nslookup to non-allowlisted hosts, in a nested subdir). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016towgcp2NVGxUBbB9Tmpv4 --- .github/workflows/ci.yml | 4 +--- scripts/check-airgap.sh | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ce58c8..d78fafa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,9 +24,7 @@ jobs: run: bash validate-build.sh - name: Run unit tests - run: | - python3 tests/test_nn.py -v - python3 tests/test_mcp.py -v + run: python3 -m unittest discover -s tests -p 'test_*.py' -v - name: Air-gap check (no new outbound network calls) run: bash scripts/check-airgap.sh diff --git a/scripts/check-airgap.sh b/scripts/check-airgap.sh index e6d69d3..d140d30 100755 --- a/scripts/check-airgap.sh +++ b/scripts/check-airgap.sh @@ -26,6 +26,7 @@ BIN_DIR="config/includes.chroot/usr/local/bin" ALLOWLIST=" neuros-music:www.youtube.com neuros-network:google.com +neuros-network:1.1.1.1 neuros-speak:github.com " @@ -49,7 +50,37 @@ while IFS= read -r -d '' f; do echo "$match" | sed "s#^# $name:#" FOUND=1 fi -done < <(find "$BIN_DIR" -maxdepth 1 -type f -not -name '*.pyc' -print0) +done < <(find "$BIN_DIR" -type f -not -name '*.pyc' -not -path '*/__pycache__/*' -print0) + +# Raw-socket / DNS-tool scanning: catches outbound calls that contain no +# http(s):// literal at all, e.g. socket.create_connection(("1.1.1.1", 53)) +# or shelling out to nslookup/dig/host with a literal target host. The +# URL scan below can't see either of these. +while IFS= read -r -d '' f; do + name="$(basename "$f")" + while IFS=: read -r line host; do + [ -z "${host:-}" ] && continue + case "$host" in + localhost|127.0.0.1|0.0.0.0) continue ;; + -*|+*) continue ;; # a CLI flag (e.g. dig's "+short"), not a hostname + esac + + allowed=0 + while IFS=: read -r al_file al_host; do + [ -z "${al_file:-}" ] && continue + if [ "$name" = "$al_file" ] && [ "$host" = "$al_host" ]; then + allowed=1 + break + fi + done <<< "$ALLOWLIST" + + if [ "$allowed" -eq 0 ]; then + echo "AIR-GAP VIOLATION: $name:$line makes a raw network call to non-local host '$host'" + FOUND=1 + fi + done < <( { grep -noP "socket\.(create_connection|connect)\(\(\s*[\"']\K[^\"']+" "$f" 2>/dev/null || true + grep -noP "\[\s*[\"'](nslookup|dig|host)[\"']\s*,\s*[\"']\K[^\"']+" "$f" 2>/dev/null || true; } ) +done < <(find "$BIN_DIR" -type f -not -name '*.pyc' -not -path '*/__pycache__/*' -print0) while IFS= read -r -d '' f; do name="$(basename "$f")" @@ -84,7 +115,7 @@ while IFS= read -r -d '' f; do FOUND=1 fi done < <(grep -noE "https?://[^\"'\` \\)]+" "$f" 2>/dev/null || true) -done < <(find "$BIN_DIR" -maxdepth 1 -type f -not -name '*.pyc' -print0) +done < <(find "$BIN_DIR" -type f -not -name '*.pyc' -not -path '*/__pycache__/*' -print0) if [ "$FOUND" -ne 0 ]; then echo "" From ff051c965f21b447591d393aafc8b294267f2063 Mon Sep 17 00:00:00 2001 From: erichanwang Date: Wed, 5 Aug 2026 16:23:12 -0500 Subject: [PATCH 5/6] Pin the remaining floating build inputs: Docker base image, CI actions, pip A prior commit (2e4080a) pinned oh-my-zsh, the Ollama installer, GNOME extension tarballs, VS Code/Continue.dev, and the apt snapshot mirror -- verified all of those still resolve. This closes what a full grep for curl/wget/git-clone/pip/npm/Dockerfile fetches turned up beyond that: - Dockerfile: FROM ubuntu:24.04 pinned to its image digest. The bare tag is mutable -- Canonical rebuilds it in place for security patches -- so the Dockerized build container's own starting filesystem was still floating. `pip3 install requests` pinned to ==2.34.2. - .github/workflows/ci.yml: actions/checkout@v4 (x4) and docker/setup-buildx-action@v3 pinned to the commit SHA each tag currently resolves to, so CI can't silently start running different action code after an upstream tag move. `ollama pull mistral` is still unpinned, documented in README. New finding: the manifest digest is cheap to look up (registry.ollama.ai's manifest endpoint returns <1KB, not the 4GB blob) -- the original assumption that pinning required a full pull was wrong. But Ollama's pull path has no documented digest-pin syntax the way `docker pull` does, so there's still no way to verify a pinned reference would work without the 4GB pull this environment intentionally avoids. Added a comment at the call site with the exact finding for whoever picks this up with a real Ollama environment. config/package-lists/neuros.list.chroot intentionally left alone -- already covered by the snapshot.ubuntu.com pin in build.sh, and pinning ~60 individual package versions on top of that would trade a working snapshot pin for a build that breaks the moment one exact version rotates off a mirror. Verified: bash -n on the edited hook and all other hook.chroot files, `docker build --check` (confirms Dockerfile syntax and that the pinned digest resolves against the registry), full validate-build.sh (249/249 pass), and scripts/check-airgap.sh. All new SHAs/versions/digests confirmed live via GitHub API, PyPI, and `docker buildx imagetools inspect` before committing. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 10 +++---- Dockerfile | 10 +++++-- README.md | 30 +++++++++++++++---- .../live/0100-install-ollama.hook.chroot | 11 ++++++- 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d78fafa..4f8fe81 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: name: Validate Build Configuration runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Make scripts executable run: | @@ -41,7 +41,7 @@ jobs: name: Lint & Format Check runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Check all Python files compile run: | @@ -79,7 +79,7 @@ jobs: runs-on: ubuntu-24.04 needs: validate steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Install dependencies run: | @@ -134,10 +134,10 @@ jobs: runs-on: ubuntu-24.04 needs: validate steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Build Docker image run: docker build -t neuros-build . 2>&1 || echo "Docker build skipped (non-critical in CI)" diff --git a/Dockerfile b/Dockerfile index d5205a2..f97875c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,10 @@ # Or use with docker-compose: # docker compose up -FROM ubuntu:24.04 +# Pinned to a digest instead of the floating 24.04 tag, which Canonical +# rebuilds in place for security updates -- the digest is immutable, so the +# build container's starting filesystem can't drift between builds. +FROM ubuntu:24.04@sha256:561618e2c15bf2397621dd04f96926663a3b5616c189cf7e38db7e82f5c538ea ENV DEBIAN_FRONTEND=noninteractive ENV TZ=UTC @@ -48,8 +51,9 @@ COPY build.sh /opt/neuros/ COPY Makefile /opt/neuros/ COPY validate-build.sh /opt/neuros/ -# Install Python dependencies -RUN pip3 install --break-system-packages requests || true +# Install Python dependencies (pinned so this doesn't silently pick up a +# newer requests on rebuild) +RUN pip3 install --break-system-packages requests==2.34.2 || true # Set up neuros tools in container RUN mkdir -p /usr/local/bin && \ diff --git a/README.md b/README.md index 8e603d6..1b88034 100644 --- a/README.md +++ b/README.md @@ -526,6 +526,14 @@ What's pinned, and how: installer script is fetched from a pinned commit (the `v0.32.5` tag) instead of the floating `ollama.com/install.sh`, and `OLLAMA_VERSION` pins the installed binary itself. +- **Docker build container** (`Dockerfile`): `FROM ubuntu:24.04` pinned + to that tag's image digest instead of the mutable tag, which Canonical + rebuilds in place for security patches; `pip3 install requests` pinned + to `==2.34.2`. +- **CI workflow** (`.github/workflows/ci.yml`): `actions/checkout@v4` + and `docker/setup-buildx-action@v3` pinned to the commit SHA each tag + currently resolves to, so a maintainer moving the upstream `v4`/`v3` + tag to a new patch release can't silently change what CI runs. While verifying package resolution, `config/package-lists/neuros.list.chroot` was also found to list `also-utils`, which isn't a real package (the @@ -534,11 +542,16 @@ install` regardless of pinning, so it's fixed alongside this work. **Known gap, not fixed here:** `ollama pull mistral` still pulls whatever the `mistral` tag in Ollama's library currently resolves to — -that tag can move to a different quantization over time. Pinning it to a -manifest digest (`mistral@sha256:...`) is possible in principle, but -verifying the digest reference actually works requires an `ollama pull` -of the full ~4GB model, which this environment intentionally does not -do. Left as a documented gap rather than shipped unverified. +that tag can move to a different quantization over time. The manifest +digest turns out to be cheap to look up (`GET +https://registry.ollama.ai/v2/library/mistral/manifests/latest` returns +a <1KB JSON manifest, not the model blob), so that part of the original +assumption was wrong. What's still blocking a pin: Ollama's `pull` has +no documented `name@sha256:digest` reference syntax the way `docker +pull` does, so there's no verified way to make `ollama pull` consume +that digest without an actual ~4GB pull to confirm the CLI accepts it — +which this environment intentionally does not do. Left as a documented +gap rather than shipped unverified. **What was verified, and how:** `lb config` (with `--mode ubuntu` and the pinned snapshot mirrors) was run in a clean `ubuntu:24.04` Docker @@ -546,7 +559,12 @@ container and exits 0. `apt-get install --dry-run` against the pinned snapshot for every package in `neuros.list.chroot` was run in the same container and resolves cleanly (exit 0, no dry-run conflicts). All pinned commit SHAs/tags/versions above were confirmed to resolve via the -GitHub API and package repositories at the time of pinning. What was +GitHub API and package repositories at the time of pinning, including +the Docker base image digest (`docker buildx imagetools inspect +ubuntu:24.04`), the `requests` version (PyPI), and the two GitHub +Actions SHAs (`actions/checkout`, `docker/setup-buildx-action`, both +confirmed against the GitHub API to match the `v4`/`v3` tags' current +resolution). What was **not** verified: an actual `lb build` (needs 20GB+ disk and privileged chroot/mount this environment doesn't have), and the `ollama pull` / GNOME extension downloads were not executed end-to-end inside a real diff --git a/config/hooks/live/0100-install-ollama.hook.chroot b/config/hooks/live/0100-install-ollama.hook.chroot index d300cc8..2d446cf 100755 --- a/config/hooks/live/0100-install-ollama.hook.chroot +++ b/config/hooks/live/0100-install-ollama.hook.chroot @@ -14,7 +14,16 @@ OLLAMA_INSTALL_SHA="eec8e0b9458b8a01be0c216a9cc53eefde24ef50" # v0.32.5 export OLLAMA_VERSION="0.32.5" curl -fsSL "https://raw.githubusercontent.com/ollama/ollama/${OLLAMA_INSTALL_SHA}/scripts/install.sh" | sh -# Pre-pull Mistral 7B model (quantized, ~4GB) +# Pre-pull Mistral 7B model (quantized, ~4GB). This still floats: "mistral" +# resolves to whatever the Ollama library's latest tag points to. Unlike +# assumed, the manifest digest IS cheaply queryable without pulling the +# full model -- GET https://registry.ollama.ai/v2/library/mistral/manifests/latest +# returns a small (<1KB) JSON manifest, not the model blob. But Ollama's +# own pull path has no documented `name@sha256:digest` pin syntax (unlike +# `docker pull`/`ollama.com`'s per-tag digest display), so committing a +# digest here is unverified without an actual `ollama pull` of the ~4GB +# model against that reference, which this environment intentionally does +# not do. Left floating; see README's Reproducibility section. echo "[NeurOS] Pulling Mistral 7B model (this will take a while)..." ollama pull mistral From a8ab4059649c577d23653162dda1ea3a0e2c04b5 Mon Sep 17 00:00:00 2001 From: erichanwang <150875139+erichanwang@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:22:30 -0500 Subject: [PATCH 6/6] Add policy, replay, runbook, sandbox primitives with tests Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01W6ShnQA7UMNDtUL9DiBNnN --- .../usr/local/bin/neuros-bench | 706 ++++++++++------- .../usr/local/bin/neuros-container | 695 +++++++++++++++-- .../usr/local/bin/neuros-model | 50 +- .../usr/local/bin/neuros-policy | 577 ++++++++++++++ .../usr/local/bin/neuros-replay | 386 +++++++++ .../usr/local/bin/neuros-runbook | 283 +++++++ .../usr/local/bin/neuros-sandbox | 406 ++++++++++ .../usr/local/bin/neuroslib.py | 159 +++- config/includes.chroot/usr/local/bin/nn | 103 +-- tests/test_bench.py | 338 ++++++++ tests/test_container.py | 733 +++++++++++++++++- tests/test_model.py | 8 +- tests/test_neuroslib.py | 215 +++++ tests/test_nn.py | 32 +- tests/test_policy.py | 398 ++++++++++ tests/test_replay.py | 435 +++++++++++ tests/test_runbook.py | 437 +++++++++++ tests/test_sandbox.py | 550 +++++++++++++ 18 files changed, 6049 insertions(+), 462 deletions(-) create mode 100755 config/includes.chroot/usr/local/bin/neuros-policy create mode 100755 config/includes.chroot/usr/local/bin/neuros-replay create mode 100755 config/includes.chroot/usr/local/bin/neuros-runbook create mode 100755 config/includes.chroot/usr/local/bin/neuros-sandbox create mode 100755 tests/test_bench.py create mode 100644 tests/test_neuroslib.py create mode 100755 tests/test_policy.py create mode 100755 tests/test_replay.py create mode 100755 tests/test_runbook.py create mode 100755 tests/test_sandbox.py diff --git a/config/includes.chroot/usr/local/bin/neuros-bench b/config/includes.chroot/usr/local/bin/neuros-bench index 3600074..b03ab81 100755 --- a/config/includes.chroot/usr/local/bin/neuros-bench +++ b/config/includes.chroot/usr/local/bin/neuros-bench @@ -1,282 +1,458 @@ #!/usr/bin/env python3 +"""neuros-bench: regression benchmark harness for neuros-sandbox. + +The tool is intentionally small. It shells out to `neuros-sandbox run +--json` and treats each per-workload JSON envelope as the unit of +metric. That decouples it from any specific cgroup kernel delegation, +so the bench is fully testable on a developer laptop. + +Subcommands: + + neuros-bench list # print built-in workloads + neuros-bench run # one workload -> metrics.json + neuros-bench batch [--out PATH] # all workloads -> metrics.json + neuros-bench compare # diff two .json files + +The built-in workload suite is intentionally small (8 named entries) +but covers the spread of behaviors the sandbox is meant to isolate: +CPU-bound recursion, memory growth, regex compile, fork pressure, +ctype-bridge syscall count, file IO, JSON parse, and string ops. +The workloads each render to a deterministic Python string that the +wrapper pipes into `neuros-sandbox run -`; no temp files. + +Design choices worth knowing: + +* `--per-workload-timeout` defaults to 60s. A single runaway workload + in a CPU loop used to be able to block the entire batch because + the wrapper only SIGKILLs on its OWN watchdog; this matches the + shape that neuros-sandbox exposes. +* `--tolerance-wall-pct` and `--tolerance-mem-pct` are the regression + bounds for `compare`. Defaults are 15% / 25%; CI can be tighter via + configuration. +* The compare output is grep-friendly text on stdout, NOT JSON. + CIs reading the diff benefit from `+12%` rows so a human can + triage a regression fast. """ -neuros-bench — NeurOS Benchmark Suite -Comprehensive AI and system benchmarks with scoring and comparisons. - -Usage: - neuros-bench Run full benchmark suite - neuros-bench --ai AI/LLM benchmarks only - neuros-bench --system System benchmarks only - neuros-bench --compare Compare against previous runs - neuros-bench --quick Quick 30-second benchmark -""" - -import sys -import os +import argparse import json -import time +import os import subprocess -import argparse -import urllib.request -import urllib.error -from datetime import datetime - -BENCH_DIR = os.path.expanduser("~/.local/share/neuros/benchmarks") -RESULTS_FILE = os.path.join(BENCH_DIR, "results.json") - - -def ensure_dirs(): - os.makedirs(BENCH_DIR, exist_ok=True) - - -def load_results(): - if os.path.exists(RESULTS_FILE): - with open(RESULTS_FILE) as f: - return json.load(f) - return [] - - -def save_results(results): - with open(RESULTS_FILE, "w") as f: - json.dump(results, f, indent=2) - - -def bench_cpu(): - start = time.time() - ops = 0 - for _ in range(5000000): - ops += 1 - elapsed = time.time() - start - return {"ops": ops, "time_seconds": round(elapsed, 3), - "score": round(ops / elapsed / 1000000, 1)} - - -def bench_memory(): - sizes = [1000, 10000, 100000, 1000000] - results = [] - for size in sizes: - start = time.time() - arr = list(range(size)) - arr.reverse() - arr.sort() - elapsed = time.time() - start - results.append({"elements": size, "ms": round(elapsed * 1000, 1)}) - return results - - -def bench_disk(mb=100): +import sys +import time +import uuid + +# --- Constants ----------------------------------------------------------- + +#: The version baked into the metrics.json schema. Bump when the +#: shape changes in a non-additive way (new required key, removed +#: key, etc.). +NEUROS_BENCH_VERSION = "0.1" + +#: Default per-workload timeout in seconds. +DEFAULT_TIMEOUT = 60 + +#: Default tolerances (percent) for the `compare` subcommand. +DEFAULT_TOLERANCE_WALL_PCT = 15.0 +DEFAULT_TOLERANCE_MEM_PCT = 25.0 + + +# --- Workload registry --------------------------------------------------- + +def _w_cpu_tight(): + # Recursive fib — tight CPU usage, low memory. + def fib(n): return n if n < 2 else fib(n - 1) + fib(n - 2) + return fib(28) + +def _w_json_parse(): + # Generate a synthetic 5KB JSON string, parse-then-dump 5000 times. + import json as _json + payload = _json.dumps([{"i": i, "sq": i * i} for i in range(40)]) + for _ in range(5000): + _json.loads(payload) + return 0 + +def _w_regex_compile(): + import re as _re + pats = [r"^CAP_[A-Z_]+$", r"[0-9a-f]{8}", r"\b\w+\b", r"foo|bar|baz", + r"https?://[^\s]+", r"\d{4}-\d{2}-\d{2}", r"^[a-z]+$", + r"\s+", r"(\d)\1+", r"^[A-Z][a-z]*$"] + for _ in range(200): + for p in pats: + _re.compile(p) + return 0 + +def _w_subproc_spawn(): + # Spawn 8 subprocesses that run a no-op and return. + for _ in range(8): + subprocess.run([sys.executable, "-c", "pass"], + capture_output=True, check=False) + return 0 + +def _w_mem_grow(): + # Allocate a bytearray that doubles until it reaches ~50 MiB. + buf = bytearray() + target = 50 * 1024 * 1024 + while len(buf) < target: + buf.extend(b"\x00" * min(len(buf) or 1, target - len(buf))) + return len(buf) + +def _w_file_io(): + # Write+read 100 x 4KB files in /tmp; cleanup after. import tempfile - import os as os_mod - data = b"x" * (1024 * 1024) - with tempfile.NamedTemporaryFile(delete=False) as f: - start = time.time() - for _ in range(mb): - f.write(data) - f.flush() - write_time = time.time() - start - tmp = f.name - with open(tmp, "rb") as f: - start = time.time() - for _ in range(mb): - f.read(1024 * 1024) - read_time = time.time() - start - os_mod.unlink(tmp) - return { - "write_mbps": round(mb / write_time, 1), - "read_mbps": round(mb / read_time, 1) + paths = [] + for i in range(100): + fd, p = tempfile.mkstemp(prefix="neuros-bench-", suffix=".bin") + os.write(fd, b"\x00" * 4096) + os.close(fd) + paths.append(p) + for p in paths: + with open(p, "rb") as f: + f.read() + for p in paths: + try: + os.unlink(p) + except OSError: + pass + return len(paths) + +def _w_ctypes_call(): + import ctypes + libc = ctypes.CDLL("libc.so.6", use_errno=True) + for _ in range(500): + libc.getpid() + return 0 + +def _w_string_ops(): + # Heavy string interpolation/format, no I/O. + parts = ["alpha", "beta", "gamma", "delta", "epsilon"] + out = [] + for i in range(100_000): + out.append(f"{i:d}-{parts[i % len(parts)]}-{i * i}") + return len("".join(out)) + + +#: Workload registry: name -> (callable, baseline_ms_estimate). The +#: baseline_ms_estimate is informational only (used in `list` and +#: compare output); runtime numbers depend on the host. +WORKLOADS = { + "cpu_tight": (_w_cpu_tight, 1100), + "json_parse": (_w_json_parse, 900), + "regex_compile": (_w_regex_compile, 600), + "subproc_spawn": (_w_subproc_spawn, 1500), + "mem_grow": (_w_mem_grow, 350), + "file_io": (_w_file_io, 1800), + "ctypes_call": (_w_ctypes_call, 700), + "string_ops": (_w_string_ops, 1200), +} + + +# --- Subprocess envelope parsing ------------------------------------------ + +def _run_workload(name, timeout_s, mem_cap, pids_cap): + """Shell out to neuros-sandbox with the workload's Python source as + a piped stdin, parse the JSON envelope, and return a dict matching + the metrics schema. Raises SandboxError (we use a generic alias) + on subprocess or envelope failures.""" + fn, _ = WORKLOADS[name] + # The workload callable is executed INSIDE the sandbox via a fresh + # python3 interpreter; here we render the function source as a + # Python expression that ends with `return 0` on the outside of + # the workspace. Wrap in a try/except that prints traceback to + # stderr and exits 1, so the metrics envelope reflects the + # failure mode rather than silencing it. + src_lines = [ + "import sys, traceback", + "try:", + ] + for line in _render_fn(fn): + src_lines.append(" " + line) + src_lines.append("except Exception:") + src_lines.append(" traceback.print_exc()") + src_lines.append(" sys.exit(1)") + # Pull the trailing expression out: the last line of a workload + # is the return value. The wrapping `r = ...` captures it but we + # don't need to read it; sys.exit code on uncaught Exception + # surfaces as the envelope's exit_code. + body = "\n".join(src_lines) + argv = ["neuros-sandbox", "run", + "--timeout", str(timeout_s), + "--mem", mem_cap, + "--pids", str(pids_cap), + "--name", f"neuros-bench-{uuid.uuid4().hex[:8]}", + "--json", "--"] + cp = subprocess.run( + argv, + input=body.encode(), + env={"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, + capture_output=True, + check=False, + timeout=timeout_s + 10, # small slack beyond the inner watchdog + ) + stdout_text = cp.stdout.decode("utf-8", "replace").strip() + if not stdout_text: + return { + "name": name, + "wall_clock_ms": 0, + "exit_code": cp.returncode, + "stdout": "", + "stderr": cp.stderr.decode("utf-8", "replace"), + "timeout_hit": False, + "peak_mem_estimate": None, + } + try: + env_obj = json.loads(stdout_text.splitlines()[-1]) + except json.JSONDecodeError: + env_obj = {"exit_code": cp.returncode, "stderr": stdout_text} + env_obj.setdefault("name", name) + env_obj.setdefault("wall_clock_ms", 0) + env_obj.setdefault("stdout", "") + env_obj.setdefault("stderr", "") + env_obj.setdefault("timeout_hit", False) + env_obj.setdefault("peak_mem_estimate", None) + return env_obj + + +def _render_fn(fn): + """Pull the source of a function via inspect.getsource and unindent + it so it can be embedded inside a try/except block at the same + indent as the outer frame.""" + import inspect + import textwrap + raw = textwrap.dedent(inspect.getsource(fn)) + # Drop the leading `def _w_xxx():` and unindent the body so that + # everything below the def is indented by one level (matching + # the `try:` we wrap it in). + lines = raw.splitlines() + if lines and lines[0].lstrip().startswith("def "): + # drop def line; body follows + body = lines[1:] + else: + body = lines + # Trim leading blank line if present + while body and not body[0].strip(): + body.pop(0) + return body + + +# --- Metrics aggregation -------------------------------------------------- + +def cmd_run(args): + fn, _ = WORKLOADS.get(args.workload, (None, None)) + if fn is None: + _die(f"unknown workload {args.workload!r}; " + f"see `neuros-bench list`") + res = _run_workload(args.workload, args.timeout, args.mem, args.pids) + out = { + "runs": [res], + "total_wall_clock_ms": res["wall_clock_ms"], + "config": _config_block(args), + "neuros_bench_version": NEUROS_BENCH_VERSION, + } + _emit_metrics(out, args.out) + + +def cmd_batch(args): + runs = [] + t0 = time.monotonic() + for name in WORKLOADS: + runs.append(_run_workload(name, args.timeout, args.mem, args.pids)) + total = int((time.monotonic() - t0) * 1000) + out = { + "runs": runs, + "total_wall_clock_ms": total, + "config": _config_block(args), + "neuros_bench_version": NEUROS_BENCH_VERSION, } + _emit_metrics(out, args.out) -def bench_ai(): - """Benchmark Ollama inference speed.""" - config = {} - config_path = os.path.expanduser("~/.config/neuros/llm.conf") - if os.path.exists(config_path): - with open(config_path) as f: - for line in f: - line = line.strip() - if "=" in line and not line.startswith("["): - k, v = line.split("=", 1) - config[k.strip()] = v.strip().strip('"').strip("'") - model = config.get("model", "mistral") - ollama_url = config.get("ollama_url", "http://localhost:11434") - - metrics = {} - tests = [ - ("tiny", "Hello", 10), - ("short", "Write a haiku about coding", 50), - ("medium", "Explain binary search in one paragraph", 150), - ] +def cmd_list(args): + print("available workloads:") + widest = max(len(name) for name in WORKLOADS) + 2 + for name, (_, baseline_ms) in WORKLOADS.items(): + print(f" {name.ljust(widest)} baseline_ms_est={baseline_ms}") + return 0 - for name, prompt, expect_tokens in tests: - try: - start = time.time() - req = urllib.request.Request( - f"{ollama_url}/api/generate", - data=json.dumps({ - "model": model, "prompt": prompt, "stream": False, - "options": {"num_predict": expect_tokens} - }).encode(), - headers={"Content-Type": "application/json"} - ) - resp = urllib.request.urlopen(req, timeout=60) - data = json.loads(resp.read().decode()) - elapsed = time.time() - start - actual_tokens = data.get("eval_count", 0) - tps = actual_tokens / elapsed if elapsed > 0 else 0 - metrics[name] = { - "time_seconds": round(elapsed, 3), - "tokens": actual_tokens, - "tokens_per_sec": round(tps, 1) - } - except Exception as e: - metrics[name] = {"error": str(e)} - - return metrics - - -def bench_compression(): - """Benchmark compression speed.""" - import tempfile - import os as os_mod - data = b"Hello world! " * 100000 - with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f: - f.write(data) - tmp = f.name - start = time.time() - subprocess.run(["gzip", "-f", tmp], check=True) - compress_time = time.time() - start - start = time.time() - subprocess.run(["gunzip", "-f", tmp + ".gz"], check=True) - decompress_time = time.time() - start - os_mod.unlink(tmp) - del data - # Approximate compression ratio for text + +def _config_block(args): return { - "compress_ms": round(compress_time * 1000, 1), - "decompress_ms": round(decompress_time * 1000, 1), - "ratio": 5.5 + "per_workload_timeout_seconds": args.timeout, + "mem_cap": args.mem, + "pids_cap": args.pids, } -def run_full_benchmark(): - print("=== NeurOS Full System Benchmark ===") - result = {"timestamp": datetime.now().isoformat(), "scores": {}} - total_score = 0 - - print("\n[CPU] Integer operations...") - cpu = bench_cpu() - result["cpu"] = cpu - score_cpu = min(cpu["score"] * 20, 100) - result["scores"]["cpu"] = round(score_cpu, 1) - total_score += score_cpu - print(f" Score: {score_cpu:.1f}/100 ({cpu['score']:.1f} M ops/sec)") - - print("\n[Disk] Read/Write throughput...") - disk = bench_disk(50) - result["disk"] = disk - score_disk = min((disk["read_mbps"] + disk["write_mbps"]) / 2, 100) - result["scores"]["disk"] = round(score_disk, 1) - total_score += score_disk - print(f" Score: {score_disk:.1f}/100 (R: {disk['read_mbps']} W: {disk['write_mbps']} MB/s)") - - print("\n[Memory] Array operations...") - mem = bench_memory() - result["memory"] = mem - score_mem = max(0, min(100, 100 - (mem[-1]["ms"] / 10))) - result["scores"]["memory"] = round(score_mem, 1) - total_score += score_mem - print(f" Score: {score_mem:.1f}/100 ({mem[-1]['ms']}ms for 1M elements)") - - print("\n[AI] LLM inference (Ollama)...") - ai = bench_ai() - result["ai"] = ai - score_ai = 0 - if "medium" in ai and "tokens_per_sec" in ai["medium"]: - score_ai = min(ai["medium"]["tokens_per_sec"] * 2, 100) - elif "short" in ai and "tokens_per_sec" in ai["short"]: - score_ai = min(ai["short"]["tokens_per_sec"] * 2, 100) - result["scores"]["ai"] = round(score_ai, 1) - total_score += score_ai - if "medium" in ai and "tokens_per_sec" in ai["medium"]: - print(f" Score: {score_ai:.1f}/100 ({ai['medium']['tokens_per_sec']:.1f} tok/sec)") - else: - print(f" Score: {score_ai:.1f}/100") - - result["total_score"] = round(total_score / 4, 1) - print(f"\n=== OVERALL SCORE: {result['total_score']:.1f}/100 ===") - - results = load_results() - results.append(result) - if len(results) > 20: - results = results[-20:] - save_results(results) - - # Compare with best - if len(results) > 1: - best = max(r["total_score"] for r in results[:-1]) - delta = result["total_score"] - best - if delta >= 0: - print(f" New personal best! (+{delta:.1f})") - else: - print(f" Previous best: {best:.1f} ({delta:.1f} below)") - - return result - - -def run_quick(): - cpu = bench_cpu() - disk = bench_disk(10) - ai = bench_ai() - print(f"CPU: {cpu['score']:.1f} M ops/sec") - print(f"Disk: R {disk['read_mbps']:.1f} / W {disk['write_mbps']:.1f} MB/s") - if "short" in ai and "tokens_per_sec" in ai["short"]: - print(f"AI: {ai['short']['tokens_per_sec']:.1f} tok/sec") - - -def show_history(): - results = load_results() - if not results: - print("No benchmarks run yet.") +def _emit_metrics(obj, out_path): + text = json.dumps(obj, indent=2) + if out_path is None or out_path == "-": + print(text) return - print(f"{'Date':<18} {'CPU':<8} {'Disk':<8} {'Mem':<8} {'AI':<8} {'Total':<8}") - print("-" * 65) - for r in results: - ts = r["timestamp"][:16].replace("T", " ") - scores = r.get("scores", {}) - cpu = scores.get("cpu", 0) - disk = scores.get("disk", 0) - mem = scores.get("memory", 0) - ai = scores.get("ai", 0) - total = r.get("total_score", 0) - print(f"{ts:<18} {cpu:<8.1f} {disk:<8.1f} {mem:<8.1f} {ai:<8.1f} {total:<8.1f}") - - -def main(): - ensure_dirs() - parser = argparse.ArgumentParser(description="NeurOS Benchmark Suite") - parser.add_argument("--ai", "-a", action="store_true", help="AI benchmarks") - parser.add_argument("--system", "-s", action="store_true", help="System benchmarks") - parser.add_argument("--compare", "-c", action="store_true", help="Compare history") - parser.add_argument("--quick", "-q", action="store_true", help="Quick benchmark") - - args = parser.parse_args() - - if args.compare: - show_history() - elif args.quick: - run_quick() - elif args.ai: - ai = bench_ai() - print(json.dumps(ai, indent=2)) - elif args.system: - cpu = bench_cpu() - mem = bench_memory() - print(f"CPU: {cpu['score']:.1f} M ops/sec") - print(f"Memory (1M elements): {mem[-1]['ms']}ms") - else: - run_full_benchmark() + with open(out_path, "w") as f: + f.write(text) + print(f"wrote metrics to {out_path}", file=sys.stderr) + + +def cmd_compare(args): + """Diff two metrics.json files. Print a grep-friendly table on + stdout. Exit 1 if any workload's wall_ms regressed beyond + --tolerance-wall-pct OR peak_mem regressed beyond + --tolerance-mem-pct.""" + baseline = _load_metrics(args.baseline) + candidate = _load_metrics(args.candidate) + base_runs = {r["name"]: r for r in baseline["runs"]} + cand_runs = {r["name"]: r for r in candidate["runs"]} + + header = ( + f"{'workload':<16} {'baseline_ms':>11} {'candidate_ms':>13} " + f"{'wall_pct':>9} {'base_mem':>11} {'cand_mem':>11} " + f"{'mem_pct':>9} status" + ) + print(header) + any_regression = False + for name in sorted(set(base_runs) | set(cand_runs)): + b = base_runs.get(name) + c = cand_runs.get(name) + b_ms = b.get("wall_clock_ms", 0) if b else 0 + c_ms = c.get("wall_clock_ms", 0) if c else 0 + b_mem = b.get("peak_mem_estimate") if b else None + c_mem = c.get("peak_mem_estimate") if c else None + + wall_pct = _pct_delta(b_ms, c_ms) + mem_pct = _pct_delta_optional(b_mem, c_mem) + status = "OK" + # Wall regression only counted when both sides have a sample. + if b and c and wall_pct is not None: + if wall_pct > args.tolerance_wall_pct: + status = "REGRESSION" + any_regression = True + if status == "OK" and b and c and mem_pct is not None: + if mem_pct > args.tolerance_mem_pct: + status = "REGRESSION" + any_regression = True + if b is None: + status = "NEW" + if c is None: + status = "DROPPED" + any_regression = True + + print( + f"{name:<16} " + f"{b_ms:>11d} {c_ms:>13d} " + f"{_fmt_pct(wall_pct):>9} " + f"{_fmt_mem(b_mem):>11} {_fmt_mem(c_mem):>11} " + f"{_fmt_pct(mem_pct):>9} {status}" + ) + print() + if any_regression: + print("RESULT: regression(s) detected (exit 1)", file=sys.stderr) + return 1 + print("RESULT: within tolerance (exit 0)", file=sys.stderr) + return 0 + + +def _pct_delta(base, cand): + if not base: + return None + return (cand - base) * 100.0 / base + + +def _pct_delta_optional(b_mem, c_mem): + if b_mem is None or c_mem is None: + return None + if not b_mem: + return None + return (c_mem - b_mem) * 100.0 / b_mem + + +def _fmt_pct(pct): + if pct is None: + return "-" + return f"{pct:+.1f}%" + + +def _fmt_mem(m): + if m is None: + return "-" + return f"{m}" + + +def _load_metrics(path): + with open(path) as f: + return json.load(f) + + +# --- Dispatch ------------------------------------------------------------ + +def _die(msg, code=2): + print(f"neuros-bench: {msg}", file=sys.stderr) + sys.exit(code) + + +def _build_parser(): + p = argparse.ArgumentParser( + prog="neuros-bench", + description=( + "Regression benchmark harness that drives neuros-sandbox with " + "canned workloads, parses JSON envelopes, and reports metric " + "deltas. No kernel delegation required at the bench level " + "(each workload runs through the wrapper)." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, + help=f"per-workload timeout in seconds " + f"(default: {DEFAULT_TIMEOUT}).") + p.add_argument("--mem", default="256M", + help="mem cap passed through to neuros-sandbox " + "(default: 256M).") + p.add_argument("--pids", type=int, default=64, + help="pids cap passed through to neuros-sandbox " + "(default: 64).") + + sub = p.add_subparsers(dest="cmd", required=True) + p_list = sub.add_parser("list", help="list the built-in workload suite") + p_run = sub.add_parser("run", help="run a single workload") + p_run.add_argument("workload", help="workload name to run") + p_run.add_argument("--out", "-o", + help="write metrics JSON to this file " + "(default: stdout).") + + p_batch = sub.add_parser("batch", help="run every built-in workload") + p_batch.add_argument("--out", "-o", + help="write metrics JSON to this file " + "(default: stdout).") + + p_cmp = sub.add_parser("compare", + help="diff two metrics files; " + "exit 1 on regression") + p_cmp.add_argument("baseline") + p_cmp.add_argument("candidate") + p_cmp.add_argument("--tolerance-wall-pct", type=float, + default=DEFAULT_TOLERANCE_WALL_PCT) + p_cmp.add_argument("--tolerance-mem-pct", type=float, + default=DEFAULT_TOLERANCE_MEM_PCT) + + return p + + +def main(argv=None): + parser = _build_parser() + args = parser.parse_args(argv) + if args.cmd == "list": + rc = cmd_list(args) + # cmd_list prints to stdout (the workload list); keep the + # exit code explicit so callers (CI, shells) get a clean 0 + # rather than Python's None-through-sysexit mishmash. + return rc if rc is not None else 0 + if args.cmd == "run": + return cmd_run(args) + if args.cmd == "batch": + return cmd_batch(args) + if args.cmd == "compare": + return cmd_compare(args) + parser.print_help() + return 2 if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/config/includes.chroot/usr/local/bin/neuros-container b/config/includes.chroot/usr/local/bin/neuros-container index 155e4eb..c493bd3 100755 --- a/config/includes.chroot/usr/local/bin/neuros-container +++ b/config/includes.chroot/usr/local/bin/neuros-container @@ -3,30 +3,59 @@ neuros-container - a container primitive built from Linux namespaces and cgroups v2, no runc/containerd/libcontainer involved. -Isolation comes from three unshare(2) namespace groups (mount, UTS, PID, -IPC, and network) plus an optional chroot into a rootfs directory. +Isolation comes from Linux unshare(2) namespaces: mount, UTS, PID, IPC, +optionally network, plus an optional chroot into a rootfs directory. Resource limits come from a real cgroup v2 leaf: memory.max, pids.max, -and cpu.max when the cpu controller is delegated. +cpu.weight, and cpu.max. Usage: neuros-container run [--mem SIZE] [--pids N] [--cpu WEIGHT] - [--hostname NAME] [--rootfs DIR] -- CMD [ARGS...] - neuros-container list + [--cpu-quota MAX/PERIOD] [--hostname NAME] + [--rootfs DIR] [--workdir DIR] [--user UID:GID] + [--read-only] [--net] [--detach] [--name NAME] + -- CMD [ARGS...] + neuros-container list [--json] + neuros-container logs (placeholder, no-op for now) Namespace isolation needs either root, or (best effort, and blocked by default on Ubuntu 24.04+ via AppArmor's unprivileged-userns restriction) an unprivileged user namespace. Without either, the command still runs under the cgroup limits but without namespace isolation, and this tool says so rather than pretending otherwise. + +Detached runs leave the child running, write a small JSON state file +under /run/neuros-container/, and exit 0. The cgroup is left in place +so a reattach / cleanup pass can find it. """ import argparse +import ctypes +import ctypes.util +import json import os import re import sys -import time CGROUP_ROOT = "/sys/fs/cgroup" +STATE_DIR = "/run/neuros-container" + +PR_CAPBSET_DROP = 23 # ; reduces the bounding set, which + # execve inherits, so caps stay dropped across exec. + +# Capability names by index, entries 0..31. The kernel may support +# more (read /proc/sys/kernel/cap_last_cap at runtime); names past +# index 31 are kernel-version-dependent and we surface them with +# their numeric index in the matched-pre-exec warning so the user +# can override the pattern. +_CAP_NAMES_0_31 = [ + "CHOWN", "DAC_OVERRIDE", "DAC_READ_SEARCH", "FOWNER", "FSETID", + "KILL", "SETGID", "SETUID", "SETPCAP", "LINUX_IMMUTABLE", + "NET_BIND_SERVICE", "NET_BROADCAST", "NET_ADMIN", "NET_RAW", + "IPC_LOCK", "IPC_OWNER", "SYS_MODULE", "SYS_RAWIO", "SYS_CHROOT", + "SYS_PTRACE", "SYS_PACCT", "SYS_ADMIN", "SYS_BOOT", "SYS_NICE", + "SYS_RESOURCE", "SYS_TIME", "SYS_TTY_CONFIG", "MKNOD", "LEASE", + "AUDIT_WRITE", "AUDIT_CONTROL", "SETFCAP", +] # pad to length 32 def die(msg): @@ -50,6 +79,113 @@ def parse_size(text): return str(n * mult) +def parse_cpu_quota(text): + """Parse '50000/100000' or '50000 100000' into a (max_us, period_us) + string tuple suitable for the cpu.max cgroup file. Also accepts + the literal ``max``, which returns ``("max", "max")`` to clear any + existing quota. Rejects zero periods, negative max_us, and one- + component input.""" + if not text: + return None + text = text.strip() + if text.lower() == "max": + return ("max", "max") + parts = re.split(r"[\s/]+", text, maxsplit=1) + if len(parts) != 2 or not all(parts): + die(f"invalid --cpu-quota '{text}' (expected e.g. '50000/100000' for 50% of one CPU, or 'max')") + try: + max_us, period_us = int(parts[0]), int(parts[1]) + except ValueError: + die(f"invalid --cpu-quota '{text}' (expected integer microseconds)") + if max_us < 0 or period_us <= 0: + die(f"invalid --cpu-quota '{text}' (max_us >= 0 and period_us > 0)") + return (str(max_us), str(period_us)) + + +def parse_user(text): + """Parse 'UID:GID' (a single integer maps uid==gid).""" + if text is None: + return None + text = text.strip() + m = re.match(r"^(\d+)(?::(\d+))?$", text) + if not m: + die(f"invalid --user '{text}' (expected UID:GID, e.g. 1000:1000, or 0)") + return (int(m.group(1)), int(m.group(2) or m.group(1))) + + +def parse_env_file(path): + """Read KEY=VALUE pairs from a file (one per line; ``#`` lines and + blank lines are ignored; ``export FOO=bar`` is accepted via the + leading ``export `` keyword so traditional dotenv-like sources + work). Returns the list of validated ``K=V`` strings. The file + is opened with ``errors="replace"`` so non-UTF-8 bytes don't + kill the run. Validation errors carry the file path so the user + knows whether a malformed entry came from ``--env`` or + ``--env-from-file /etc/...``.""" + if path is None: + return [] + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + lines = f.read().splitlines() + except OSError as e: + die(f"--env-from-file {path!r}: cannot read ({e})") + cleaned = [] + for raw in lines: + stripped = raw.strip() + if not stripped or stripped.startswith("#"): + continue + if stripped.startswith("export "): + stripped = stripped[len("export "):].lstrip() + cleaned.append(stripped) + # Inlined parse_env validation but with path-prefixed errors so + # the user can tell the file from a CLI repeat. + for line in cleaned: + if "=" not in line: + die(f"--env-from-file {path!r}: missing '=' on line " + f"{line!r}") + k = line.partition("=")[0].strip() + if not k: + die(f"--env-from-file {path!r}: blank key on line {line!r}") + if "\x00" in line: + die(f"--env-from-file {path!r}: NUL byte on line {line!r}") + return cleaned + + +def parse_env(env_list): + """Validate a list of 'KEY=VALUE' strings (the result of + ``argparse action='append'`` on ``--env K=V``). Returns the list + unchanged on success. Blank keys, missing '=', or keys containing + NULs are rejected — the latter because ``os.execve`` truncates + at NUL silently. ``partition("=")`` already severs at the first + '=' so a second '=' check on the key is unnecessary.""" + if not env_list: + return [] + clean = [] + for raw in env_list: + if "=" not in raw: + die(f"invalid --env '{raw}' (expected KEY=VALUE)") + key, _, value = raw.partition("=") + key = key.strip() + if not key: + die(f"invalid --env '{raw}' (blank key)") + if "\x00" in key or "\x00" in value: + die(f"invalid --env '{raw}' (NUL byte rejected)") + clean.append(f"{key}={value}") + return clean + + +def parse_cap_drop(pattern): + """Compile a capability-name regex (e.g. 'CAP_(NET_RAW|SYS_ADMIN)') + for later matching against a known list. Returns the compiled + pattern, or None when no --cap-drop was given.""" + if pattern is None: + return None + try: + return re.compile(pattern) + except re.error as e: + die(f"invalid --cap-drop regex '{pattern}': {e}") + + def own_cgroup_path(): """Absolute filesystem path of the cgroup this process currently lives in, from /proc/self/cgroup (cgroup v2 always has a single '0::' line).""" @@ -120,14 +256,33 @@ def enable_controllers(parent, wanted): return ready -def make_cgroup(name, mem, pids, cpu_weight): - wanted = [c for c, v in (("memory", mem), ("pids", pids), ("cpu", cpu_weight)) if v] +def make_cgroup(name, mem, pids, cpu_weight, cpu_quota=None): + """Create a fresh leaf cgroup with the given limits. + + ``cpu_quota`` is a tuple ``(max_us_str, period_us_str)`` written to + cpu.max, or ``("max", "max")`` to clear the quota; ``None`` leaves + cpu.max untouched. ``cpu_weight`` is written to cpu.weight. + """ + wanted = [c for c, v in ( + ("memory", mem), ("pids", pids), ("cpu", cpu_weight or cpu_quota) + ) if v] parent = find_delegating_ancestor(wanted) ready = enable_controllers(parent, wanted) cg = os.path.join(parent, name) os.makedirs(cg, exist_ok=True) + # If a limit was requested but no controller is delegated all the + # way down to this leaf, the write below would silently no-op. + # Surface that loudly — it's the single most common cause of "I + # passed --mem and the container ate 4 GB" reports. + if wanted and not ready: + print(f"neuros-container: WARNING no requested controllers were " + f"delegated here; the cgroup at {cg} will not enforce " + f"{', '.join(wanted)}. Likely cause: the cgroup tree " + f"needs an ancestor with subtrees=+{wanted[0]} enabled.", + file=sys.stderr) + if mem and "memory" in ready: with open(os.path.join(cg, "memory.max"), "w") as f: f.write(mem) @@ -137,6 +292,10 @@ def make_cgroup(name, mem, pids, cpu_weight): if cpu_weight and "cpu" in ready: with open(os.path.join(cg, "cpu.weight"), "w") as f: f.write(str(cpu_weight)) + if cpu_quota and "cpu" in ready: + max_us, period_us = cpu_quota + with open(os.path.join(cg, "cpu.max"), "w") as f: + f.write(f"{max_us} {period_us}") return cg @@ -146,24 +305,27 @@ def join_cgroup(cg, pid): def cgroup_report(cg): + """Read leaf-cgroup accounting back into a dict. If the cgroup has + already been rmdir'd (a previous run cleaned up), the result is an + empty dict and the caller falls through silently.""" report = {} - for key, fname in (("memory_peak", "memory.peak"), ("pids_current", "pids.current")): - path = os.path.join(cg, fname) - if os.path.exists(path): - with open(path) as f: + for key, fname in ( + ("memory_peak", "memory.peak"), ("pids_current", "pids.current") + ): + try: + with open(os.path.join(cg, fname)) as f: report[key] = f.read().strip() + except OSError: + pass return report -def try_unprivileged_userns(): - """Best-effort unshare(CLONE_NEWUSER) with a 1:1 uid/gid map, so the - calling user appears as root inside the new namespace. Returns True if - it worked. On Ubuntu 24.04+ this is blocked by default for unconfined - processes (see /proc/sys/kernel/apparmor_restrict_unprivileged_userns); - EPERM there is expected, not a bug.""" +def try_unprivileged_userns(uid, gid): + """Enter an unprivileged user namespace with an explicit uid/gid map. + Returns True if the call succeeded. On Ubuntu 24.04+ this is blocked + by default for unconfined processes; EPERM there is expected.""" try: os.unshare(os.CLONE_NEWUSER) - uid, gid = os.getuid(), os.getgid() with open("/proc/self/setgroups", "w") as f: f.write("deny\n") with open("/proc/self/uid_map", "w") as f: @@ -175,48 +337,206 @@ def try_unprivileged_userns(): return True -def enter_namespaces(): - """Returns 'full' if mount/uts/pid/ipc namespaces were entered, 'none' - if we ran with no isolation at all (insufficient privilege).""" - have_caps = os.geteuid() == 0 - if not have_caps: - have_caps = try_unprivileged_userns() +def enter_namespaces(net=False, user=None): + """Enter mount/uts/pid/ipc and (optionally) net namespaces. + + Returns a tuple ``(mount, uts, pid, ipc, net, userns)`` listing which + namespaces actually got entered. The caller decides what to do when + something failed (we already print a single combined warning below). + + Inside the new namespace this process is root only after + ``try_unprivileged_userns`` maps the caller into uid 0; without + either root or a successful userns the namespaces list is all + False and the caller is told it ran unisolated. + """ + uid, gid = user if user is not None else (os.getuid(), os.getgid()) + if os.geteuid() == 0: + userns = False + have_caps = True + else: + # If --user targeted a uid/gid that doesn't match the caller's + # real ids, the kernel will reject the 1:1 map with EINVAL and + # the user will see the generic "no userns" warning without + # understanding why. Surface the obvious case up front so the + # cause is visible even before the unshare attempt. + if user is not None and os.getuid() != uid: + print(f"neuros-container: --user {uid}:{gid} was requested " + f"but the caller is uid {os.getuid()}:{os.getgid()}; " + f"unprivileged user namespaces can only map a " + f"single caller uid/gid. Drop --user or run as root " + f"to avoid this.", file=sys.stderr) + have_caps = try_unprivileged_userns(uid, gid) + userns = have_caps if not have_caps: print("neuros-container: no root and unprivileged user namespaces " "are unavailable; running without namespace isolation " "(cgroup limits above still apply)", file=sys.stderr) - return "none" + return (False, False, False, False, False, False) + flags = os.CLONE_NEWNS | os.CLONE_NEWUTS | os.CLONE_NEWPID | os.CLONE_NEWIPC + if net: + flags |= os.CLONE_NEWNET + gained_net = net try: - os.unshare(os.CLONE_NEWNS | os.CLONE_NEWUTS | os.CLONE_NEWPID | os.CLONE_NEWIPC) + os.unshare(flags) except OSError as e: - print(f"neuros-container: got a user namespace but not the rest ({e}); " - "running without namespace isolation (cgroup limits above still apply)", + # Some kernels refuse CLONE_NEWNET without privileges even + # after a userns; degrade gracefully instead of failing. + if net and (flags & ~os.CLONE_NEWNET): + try: + os.unshare(flags & ~os.CLONE_NEWNET) + gained_net = False + print(f"neuros-container: could not create network " + f"namespace ({e}); continuing without it", + file=sys.stderr) + except OSError as e2: + print(f"neuros-container: namespace setup failed ({e2}); " + "running without namespace isolation (cgroup limits " + "above still apply)", file=sys.stderr) + return (False, False, False, False, False, userns) + else: + print(f"neuros-container: namespace setup failed ({e}); " + "running without namespace isolation (cgroup limits " + "above still apply)", file=sys.stderr) + return (False, False, False, False, False, userns) + + return (True, True, True, True, gained_net, userns) + + +def write_state(state_path, name, inner_pid, cg): + """Persist {name, pid, cgroup} under STATE_DIR so detached runs + can be reattached without polling /proc.""" + os.makedirs(os.path.dirname(state_path), exist_ok=True) + with open(state_path, "w") as f: + json.dump({"name": name, "pid": inner_pid, "cgroup": cg}, f) + + +def mount_proc_best_effort(): + """Mount a fresh /proc inside the new PID namespace. Falls back to + the mount(8) binary on Python <3.13, and to no-op when /proc isn't + writable (e.g. chroot without it).""" + try: + os.mount("proc", "/proc", "proc", 0, "") + return + except (AttributeError, OSError): + pass + try: + os.system("mount -t proc proc /proc 2>/dev/null") + except Exception: + pass + + +def remount_readonly_best_effort(): + """Try to remount / read-only inside the container; if the mount + namespace isn't fully set up (e.g. we're in the unprivileged + fallback path) the remount will fail. Capture and surface the + non-zero returncode so the user can see whether the RO bit + actually took effect rather than silently believing it did.""" + rc = os.system("mount -o remount,ro / 2>/dev/null") + if rc != 0: + print(f"neuros-container: warning: 'mount -o remount,ro /' " + f"exited with status {rc}; the container's rootfs may " + f"not actually be read-only", file=sys.stderr) + + +def drop_capabilities_best_effort(pattern): + """Drop every capability whose name matches ``pattern`` from the + current process's bounding set via ``prctl(PR_CAPBSET_DROP)``. + + The bounding set is inherited across ``execve`` so the workload + sees the dropped state without a wrapper shell. We rely on the + user namespace giving us ``CAP_SETPCAP`` (which it does on Linux + 3.12+ by default); outside a userns, prctl will silently EPERM + and we surface that. + + Implementation notes: + * The capability name table is hardcoded for indices 0..31. + The kernel reports its maximum supported cap via + /proc/sys/kernel/cap_last_cap; we walk up to that index. + * For caps above index 31 we fall back to ``CAP_`` naming + when no human-readable name is known. + * The drop is best-effort: every successful drop is silent + on stdout; every failure is reported on stderr. + """ + if pattern is None: + return + try: + libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True) + except OSError: + print("neuros-container: --cap-drop requested but libc " + "could not be loaded; skipping capability drop.", + file=sys.stderr) + return + try: + with open("/proc/sys/kernel/cap_last_cap") as f: + last = int(f.read().strip()) + except (OSError, ValueError): + last = len(_CAP_NAMES_0_31) - 1 + + dropped, failed_to_drop = [], [] + for idx in range(last + 1): + if idx < len(_CAP_NAMES_0_31): + name = f"CAP_{_CAP_NAMES_0_31[idx]}" + else: + name = f"CAP_{idx}" # kernel-version-specific names + if not pattern.search(name): + continue + rc = libc.prctl(PR_CAPBSET_DROP, idx, 0, 0, 0) + if rc == 0: + dropped.append(name) + else: + err = ctypes.get_errno() + failed_to_drop.append((name, err)) + + if not dropped and not failed_to_drop: + print(f"neuros-container: --cap-drop pattern matched 0 of " + f"{last + 1} capabilities; nothing was dropped. " + f"Check the regex (e.g. 'CAP_NET_RAW', " + f"'^CAP_(SYS|NET)_' for a prefix).", + file=sys.stderr) + for name, err in failed_to_drop: + print(f"neuros-container: failed to drop {name} from bounding " + f"set (errno={err}); the user namespace likely lacks " + f"CAP_SETPCAP. Run as root, or inside a user namespace " + f"with CAP_SETPCAP inherited.", file=sys.stderr) - return "none" - # hostname and rootfs are applied in the PID-1 child after the second - # fork below, since the UTS/mount namespaces are entered here but this - # process itself stays in the old PID namespace. - return "full" def run_container(args): - name = f"neuros-{os.getpid()}" + """Top-level container-run path. Created cgroup is cleaned up on + exit when --detach is not in effect, and a state file is written + when it is.""" + name = args.name or f"neuros-{os.getpid()}" mem = parse_size(args.mem) pids = args.pids - cg = make_cgroup(name, mem, pids, args.cpu) + cg = make_cgroup(name, mem, pids, args.cpu, args.cpu_quota) + + state_path = None + if args.detach: + os.makedirs(STATE_DIR, exist_ok=True) + state_path = os.path.join(STATE_DIR, f"{name}.json") pid = os.fork() if pid == 0: - join_cgroup(cg, os.getpid()) - mode = enter_namespaces() - - if mode == "full": - # A second fork is required for CLONE_NEWPID: unshare() only - # affects children created after the call, so this process - # stays in the old PID namespace and the next fork's child - # becomes PID 1 in the new one. + try: + join_cgroup(cg, os.getpid()) + ns = enter_namespaces(net=args.net, user=args.user) + mount_ns, _uts, _pid, _ipc, net_ns, _userns = ns + + if not mount_ns: + # No mount namespace: drop straight into exec. + try: + os.execvp(args.cmd[0], args.cmd) + except OSError as e: + print(f"neuros-container: exec {args.cmd[0]} failed: {e}", + file=sys.stderr) + os._exit(127) + return + + # CLONE_NEWPID only affects post-unshare children, so a + # second fork is needed for the user's command to actually + # be PID 1 in the new PID namespace. inner = os.fork() if inner == 0: if args.hostname: @@ -225,48 +545,219 @@ def run_container(args): if args.rootfs: os.chroot(args.rootfs) os.chdir("/") + if args.read_only: + remount_readonly_best_effort() + if args.user is not None and os.geteuid() == 0: + target_uid, target_gid = args.user + try: + os.setgid(target_gid) + os.setuid(target_uid) + except (PermissionError, OSError) as e: + print(f"neuros-container: setuid/setgid failed: {e}", + file=sys.stderr) + drop_capabilities_best_effort(args.cap_drop) + if args.workdir: + try: + os.chdir(args.workdir) + except OSError as e: + print(f"neuros-container: cannot chdir to " + f"{args.workdir}: {e}", file=sys.stderr) + # Inject --env KEY=VALUE entries last so they take + # precedence over anything inherited from the parent + # shell (the user's intent for --env is usually "force + # this value", not "default if unset"). + if args.env: + for entry in args.env: + k, _, v = entry.partition("=") + os.environ[k] = v + mount_proc_best_effort() try: - os.mount(b"proc", b"/proc", b"proc", 0, b"") - except (AttributeError, OSError): - # os.mount is only available on Python 3.13+; fall back - # to mount(8), which is present on every target host. - os.system("mount -t proc proc /proc") - os.execvp(args.cmd[0], args.cmd) + os.execvp(args.cmd[0], args.cmd) + except OSError as e: + print(f"neuros-container: exec {args.cmd[0]} failed: {e}", + file=sys.stderr) + os._exit(127) else: + if state_path: + write_state(state_path, name, inner, cg) + if args.detach: + print(f"neuros-container: detached, inner pid {inner}, " + f"state {state_path}", file=sys.stderr) + os._exit(0) _, status = os.waitpid(inner, 0) os._exit(os.waitstatus_to_exitcode(status)) - else: - os.execvp(args.cmd[0], args.cmd) - os._exit(1) + except Exception as e: + print(f"neuros-container: child setup failed: {e}", + file=sys.stderr) + os._exit(1) _, status = os.waitpid(pid, 0) + + # Foreground (non-detached) runs: the cgroup was scoped to this PID, + # so by the time the parent wait()s, the leaf is empty and rmdirable. + # We rely on memory.peak being readable while the cgroup is still on + # disk; cgroup_report handles the already-removed case silently. report = cgroup_report(cg) + if not args.detach: + try: + os.rmdir(cg) + except OSError: + rmdir_failed = True + if report.get("memory_peak"): + peak_mib = int(report["memory_peak"]) / 1024 / 1024 + print(f"neuros-container: peak memory {peak_mib:.1f} MiB", + file=sys.stderr) + return os.waitstatus_to_exitcode(status) + + +def cleanup_container(name): + """Best-effort teardown of a single detached container. + + Reads the state file under STATE_DIR, attempts to rmdir the cgroup + (only succeeds if empty; populated cgroups are left alone and the + caller is told), then unlinks the state file. Returns 0 on success, + 1 if the cgroup was non-empty so a manual inspect is warranted.""" + state_path = os.path.join(STATE_DIR, f"{name}.json") + if not os.path.exists(state_path): + print(f"neuros-container: no state file for '{name}' " + f"(looked at {state_path})", file=sys.stderr) + return 1 + try: + with open(state_path) as f: + state = json.load(f) + cg = state.get("cgroup", "") + except (OSError, json.JSONDecodeError) as e: + print(f"neuros-container: cannot read state file for '{name}': " + f"{e}; unlinking", file=sys.stderr) + try: + os.unlink(state_path) + except OSError: + pass + return 1 + + still_running = 0 + procs_path = os.path.join(cg, "cgroup.procs") + try: + with open(procs_path) as f: + still_running = len(f.read().split()) + except OSError: + # A missing or unreadable cgroup.procs is treated as zero + # live members so partially-constructed leaves (and the + # synthetic tmpdir cgroups the test suite uses) can still be + # reaped. Without this branch, the tests would need to either + # leave a procs file behind (which blocks rmdir with ENOTEMPTY) + # or skip these leaves entirely. + still_running = 0 + + if still_running: + print(f"neuros-container: '{name}' still has {still_running} " + f"processes in {cg}; refusing to remove. Kill them first " + f"(the pid is in {state_path}).", file=sys.stderr) + return 1 try: os.rmdir(cg) + print(f"neuros-container: removed cgroup {cg}", file=sys.stderr) + except OSError as e: + print(f"neuros-container: could not rmdir {cg} ({e}); leaving " + f"the state file in place", file=sys.stderr) + return 1 + try: + os.unlink(state_path) except OSError: pass - if report: - peak = report.get("memory_peak") - if peak: - print(f"neuros-container: peak memory {int(peak) / 1024 / 1024:.1f} MiB", file=sys.stderr) - return os.waitstatus_to_exitcode(status) - - -def list_cgroups(): - parent = own_cgroup_path() - found = False - for entry in sorted(os.listdir(parent)): - if entry.startswith("neuros-"): - found = True - path = os.path.join(parent, entry) - procs_path = os.path.join(path, "cgroup.procs") - n = 0 - if os.path.exists(procs_path): - with open(procs_path) as f: - n = len(f.read().split()) - print(f"{entry} ({n} process{'es' if n != 1 else ''})") - if not found: + return 0 + + +def cleanup_all(): + """Cleanup every detached container. Each one that has live + members is skipped (with a stderr line) so a partially-clean + state surfaces.""" + if not os.path.isdir(STATE_DIR): + print("no detached neuros-container state to clean up") + return 0 + rc = 0 + for entry in sorted(os.listdir(STATE_DIR)): + if not entry.endswith(".json"): + continue + name = entry[:-len(".json")] + rc |= cleanup_container(name) + return rc + + +def _list_neuros_leaves(root): + """Recursively enumerate every ``neuros-*`` leaf cgroup under + ``root``. Returns a list of dicts; hoisted out of ``list_cgroups`` + so the test suite can drive it against synthetic cgroup trees + without spinning up a live delegation.""" + out = [] + try: + entries = sorted(os.listdir(root)) + except OSError: + return out + for entry in entries: + path = os.path.join(root, entry) + if not entry.startswith("neuros-"): + out.extend(_list_neuros_leaves(path)) + continue + rel = path[len(CGROUP_ROOT):] if path.startswith(CGROUP_ROOT) else path + try: + with open(os.path.join(path, "cgroup.procs")) as f: + procs = len(f.read().split()) + except OSError: + procs = 0 + limits = {} + for key, fname in ( + ("memory_max", "memory.max"), + ("pids_max", "pids.max"), + ("cpu_weight", "cpu.weight"), + ("cpu_max", "cpu.max"), + ): + try: + with open(os.path.join(path, fname)) as f: + limits[key] = f.read().strip() + except OSError: + limits[key] = None + peak = None + try: + with open(os.path.join(path, "memory.peak")) as f: + raw = f.read().strip() + if raw.isdigit(): + peak = int(raw) + except OSError: + peak = None + out.append({ + "name": entry, "path": rel, "procs": procs, + "limits": limits, "memory_peak": peak, + }) + return out + + +def list_cgroups(json_output=False): + """Walk the cgroup subtree under the delegating ancestor we found + earlier (since nested neuros-* leaves inside a parent neuros-* leaf + are also valid) and emit a per-leaf report. JSON mode emits a flat + array of records so scripts can pipe it through jq.""" + rows = _list_neuros_leaves(own_cgroup_path()) + if json_output: + print(json.dumps(rows, indent=2)) + return + if not rows: print("no active neuros-container cgroups") + return + for r in rows: + suffix = "es" if r["procs"] != 1 else "" + print(f"{r['name']} ({r['procs']} process{suffix})") + for label, fname in ( + ("memory.max", "memory_max"), + ("pids.max", "pids_max"), + ("cpu.weight", "cpu_weight"), + ("cpu.max", "cpu_max"), + ): + val = r["limits"].get(fname) + if val: + print(f" {label} = {val}") + if r["memory_peak"] is not None: + print(f" memory.peak = {r['memory_peak'] / 1024 / 1024:.1f} MiB") def main(): @@ -277,11 +768,41 @@ def main(): run.add_argument("--mem", help="memory limit, e.g. 256M, 1G") run.add_argument("--pids", help="max number of processes/threads") run.add_argument("--cpu", type=int, help="cpu.weight (1-10000, default cgroup weight is 100)") + run.add_argument("--cpu-quota", metavar="MAX/PERIOD", + help="cpu.max quota, e.g. '50000/100000' = 50%% of one CPU") run.add_argument("--hostname", help="hostname to set inside the UTS namespace") run.add_argument("--rootfs", help="directory to chroot into before exec") - run.add_argument("cmd", nargs=argparse.REMAINDER, help="-- command and arguments to run") - - sub.add_parser("list", help="list active neuros-container cgroups") + run.add_argument("--workdir", help="working directory inside the container (after chroot)") + run.add_argument("--user", metavar="UID:GID", + help="explicit UID:GID to set inside the container (root only)") + run.add_argument("--read-only", action="store_true", + help="remount / as read-only inside the container") + run.add_argument("--net", action="store_true", + help="enter a new network namespace") + run.add_argument("--env", action="append", metavar="KEY=VALUE", + help="set an environment variable inside the container (repeatable)") + run.add_argument("--env-from-file", metavar="PATH", + help="load KEY=VALUE pairs (one per line, # comments, optional 'export ' prefix) from PATH") + run.add_argument("--cap-drop", metavar="REGEX", + help="drop capabilities matching this regex from the bounding set (libcap2-bin required)") + run.add_argument("--detach", action="store_true", + help="don't wait; leave the child running and write /run/neuros-container/.json") + run.add_argument("--name", help="cgroup name (default: 'neuros-')") + run.add_argument("cmd", nargs=argparse.REMAINDER, + help="-- command and arguments to run") + + list_p = sub.add_parser("list", help="list active neuros-container cgroups (recursive)") + list_p.add_argument("--json", action="store_true", + help="emit structured JSON instead of human-readable output") + + sub.add_parser("logs", help="(placeholder) tail a detached container's state file") + + cleanup_p = sub.add_parser( + "cleanup", help="remove a detached container's cgroup + state file") + cleanup_p.add_argument("name", nargs="?", + help="container name to clean up (omit with --all)") + cleanup_p.add_argument("--all", action="store_true", + help="clean up every detached container") args = parser.parse_args() @@ -292,9 +813,27 @@ def main(): if not cmd: die("no command given (usage: neuros-container run [opts] -- CMD [ARGS...])") args.cmd = cmd + args.cpu_quota = parse_cpu_quota(args.cpu_quota) + args.user = parse_user(args.user) + # --env-from-file is merged with --env so file values land + # first (older defaults) and explicit --env K=V values + # override them on conflict, mirroring docker --env-file. + env_from_file = parse_env_file(args.env_from_file) + args.env = parse_env((env_from_file or []) + (args.env or [])) + args.cap_drop = parse_cap_drop(args.cap_drop) sys.exit(run_container(args)) elif args.command == "list": - list_cgroups() + list_cgroups(json_output=args.json) + elif args.command == "logs": + print("neuros-container logs: not implemented yet " + "(the state file lives at /run/neuros-container/.json)", + file=sys.stderr) + elif args.command == "cleanup": + if args.all: + sys.exit(cleanup_all()) + if not args.name: + die("cleanup needs either a container NAME or --all") + sys.exit(cleanup_container(args.name)) else: parser.print_help() diff --git a/config/includes.chroot/usr/local/bin/neuros-model b/config/includes.chroot/usr/local/bin/neuros-model index d1811b7..d812462 100755 --- a/config/includes.chroot/usr/local/bin/neuros-model +++ b/config/includes.chroot/usr/local/bin/neuros-model @@ -22,8 +22,18 @@ import time import argparse import textwrap -OLLAMA_URL = "http://localhost:11434/api" -CONFIG_PATH = os.path.expanduser("~/.config/neuros/llm.conf") +# Pull the shared config helpers from neuroslib rather than rolling our +# own key/value loop; keep ``CONFIG_PATH`` here for back-compat with +# callers and the test suite that patches it. +import importlib.util +_LIB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "neuroslib.py") +_spec = importlib.util.spec_from_file_location("neuroslib", _LIB_PATH) +neuroslib = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(neuroslib) + +OLLAMA_URL = f"http://{neuroslib.DEFAULT_HOST}:{neuroslib.DEFAULT_PORT}/api" +CONFIG_PATH = neuroslib.LLM_CONF +DEFAULT_MODEL = neuroslib.DEFAULT_MODEL def api_request(endpoint, method="GET", data=None, timeout=10): """Make a request to Ollama API.""" @@ -115,28 +125,9 @@ def switch_model(name): if name not in model_names: print(f"Warning: '{name}' is not installed. Pull it first with: neuros-model pull {name}") - os.makedirs(os.path.dirname(CONFIG_PATH), exist_ok=True) - - # Read existing config - lines = [] - found = False - try: - with open(CONFIG_PATH) as f: - for line in f: - if '=' in line and not line.startswith('[') and 'model' in line.split('=')[0]: - lines.append(f'model = "{name}"\n') - found = True - else: - lines.append(line) - except FileNotFoundError: - pass - - if not found: - lines.append(f'model = "{name}"\n') - - with open(CONFIG_PATH, 'w') as f: - f.writelines(lines) - + # Delegate the actual file write to neuroslib so the [context] + # section and other keys survive untouched. + neuroslib.set_default_model(name, CONFIG_PATH) print(f"✅ Default model set to '{name}'.") def model_info(name): @@ -267,16 +258,7 @@ def compare_models(models, prompt="Write a haiku about Linux."): def get_current_model(): """Read current default model from config.""" - try: - with open(CONFIG_PATH) as f: - for line in f: - if '=' in line and not line.startswith('['): - key, val = line.split('=', 1) - if key.strip() == "model": - return val.strip().strip('"').strip("'") - except Exception: - pass - return "mistral" + return neuroslib.get_default_model(CONFIG_PATH) def format_size(size_bytes): """Format size in human-readable format.""" diff --git a/config/includes.chroot/usr/local/bin/neuros-policy b/config/includes.chroot/usr/local/bin/neuros-policy new file mode 100755 index 0000000..1e843c0 --- /dev/null +++ b/config/includes.chroot/usr/local/bin/neuros-policy @@ -0,0 +1,577 @@ +#!/usr/bin/env python3 +"""neuros-policy: declarative security policy for neuros-sandbox. + +The wrapper `neuros-sandbox` is hardened by default, but it accepts +every limit and the cap-drop regex as a CLI flag. That makes a flat +safety posture hard to express as code, hard to version, and easy +to drift: a script that hard-coded --mem 256M in May has no way to +say "I really meant the team's 2024-Q4 policy, whatever it was +called then". + +`neuros-policy` is the place that manifest lives: + + { + "name": "neuros-default", + "version": "1.0.0", + "defaults": { + "mem": "256M", + "pids": 64, + "cpu_quota": 50000, + "timeout": 30 + }, + "profiles": { + "strict": ["CAP_NET_RAW", "CAP_SYS_ADMIN", "CAP_SYS_PTRACE"], + "moderate": ["CAP_NET_RAW", "CAP_SYS_ADMIN"], + "permissive": [] + }, + "net": "private", + "readonly": true, + "env_allowlist": ["PATH", "LANG", "LC_ALL"], + "syscalls": null + } + +Subcommands: + + neuros-policy validate + Schema + bounds + regex-shape check. Exit 0 on OK, 1 on + any violation (each violation is reported on stderr), 2 on + bad input (missing file, malformed JSON, wrong root type). + + neuros-policy check --envelope + Parse a neuros-sandbox JSON envelope and reconcile its + wall_clock_ms / peak_mem_estimate / exit_code against the + policy. Exit 0 if the run conformed, 1 on a violation, 2 + on bad input. + + neuros-policy transpile [--profile NAME] + Emit a copy-pastable argv slice suitable for + `neuros-sandbox --unsafe ...`: + + --mem 256M --pids 64 --cpu-quota 50000 \ + --timeout 30 --cap-drop CAP_(NET_RAW|SYS_ADMIN) + + The wrapper appends ``--`` followed by its own entry + (``python3 -u -``) once it parses the user argv, so the + transpile output is intentionally flag-only. An empty + profile (permissive) transpiles to + ``--cap-drop '^_NEVER_MATCH_$'`` so nothing is dropped. + + --json makes validate / check emit a one-line JSON envelope + on stdout (for piping into verify-style tooling). transpile + always prints text because its output IS the command line. +""" +import argparse +import json +import os +import re +import sys + +# --- Constants ---------------------------------------------------------- + +#: Lower bound on memory defaults (bytes). 16 KiB is what a busy +#: interpreter imports plus a small buffer. +MEM_MIN_BYTES = 16 * 1024 + +#: Upper bound on memory defaults (bytes). 1 GiB matches what the +#: bench + runbook paths already exercise; bigger requests should +#: be split into multiple containers. +MEM_MAX_BYTES = 1024 * 1024 * 1024 + +#: cgroup v2's pids.max is "max" for unbounded, but in policy we +#: cap it because the default policy is what kills an LLM agent +#: fork-bomb, not the host kernel. +PIDS_MIN, PIDS_MAX = 1, 4096 + +#: cpu-quota is microseconds / 100 ms CFS period. 1000 = 1% of one +#: CPU. 1_000_000 = 10× one CPU (would saturate any single workload). +CPU_QUOTA_MIN, CPU_QUOTA_MAX = 1000, 1_000_000 + +#: Wall-clock cap. The bench harness times out at 60s; we leave +#: 60× headroom so a 1-hour benchmark is still expressible. +TIMEOUT_MIN, TIMEOUT_MAX = 1, 3600 + +#: Sentinel regex emitted when the caller picked a profile with an +#: EMPTY cap list (i.e. "drop nothing"). Matches no CAP_* name in +#: the kernel's table, since caps start at ``CAP_CHOWN``. +NEVER_MATCH_CAP_REGEX = r"^_NEVER_MATCH_$" + +#: Regexes used during validate. +_CAP_NAME_RE = re.compile(r"^CAP_[A-Z_]+$") +_PROFILE_NAME_RE = re.compile(r"^[a-z][a-z0-9_-]*$") +_ENV_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*$") +_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$") +_NET_VALUES = ("private", "host", "isolated") + +#: Tool version baked into emitted envelopes. Bump on a non-additive +#: schema change. +NEUROS_POLICY_VERSION = "0.1" + + +# --- Exceptions -------------------------------------------------------- + + +class PolicyError(Exception): + """Raised on any user-fixable misconfiguration (bad field, bad + regex, out-of-range bound, malformed manifest). Surfaced as exit + 1 by `validate` / `check`, and 2 by `transpile` if the profile + itself is malformed.""" + + +# --- Helpers ------------------------------------------------------------ + + +def _die(msg, code=2): + print(msg, file=sys.stderr) + sys.exit(code) + + +def _parse_size_bytes(text): + """Parse ``'128M'``, ``'1G'``, ``'512K'``, a plain byte count, + or ``'max'`` into an int (or None when ``text == 'max'``). + + Raises ``PolicyError`` on bad input. + """ + if text is None: + return None + text = text.strip() + if not text: + raise PolicyError("size is empty") + if text.lower() == "max": + return None + m = re.fullmatch(r"(\d+)([KMG]?)", text, re.IGNORECASE) + if not m: + raise PolicyError( + f"invalid size {text!r} (expected e.g. 256M, 1G, or a byte count)") + n = int(m.group(1)) + mult = {"": 1, "K": 1024, "M": 1024 ** 2, "G": 1024 ** 3}[m.group(2).upper()] + if n <= 0: + raise PolicyError(f"size {text!r} must be > 0 bytes") + return n * mult + + +def _is_int(text): + """True if ``text`` is a positive integer in canonical string form.""" + return isinstance(text, int) and not isinstance(text, bool) and text >= 0 + + +def _load_policy(path): + """Read JSON from ``path`` (or '-' for stdin). Raises ``PolicyError`` + on missing file or malformed JSON. The returned object is the + raw dict; the caller is expected to run ``validate_policy`` on it. + """ + if path == "-": + try: + text = sys.stdin.read() + except OSError as e: + raise PolicyError(f"cannot read stdin: {e}") + else: + try: + with open(path, "r", encoding="utf-8") as f: + text = f.read() + except OSError as e: + raise PolicyError(f"cannot read policy file {path!r}: {e}") + try: + obj = json.loads(text) + except json.JSONDecodeError as e: + raise PolicyError(f"policy file {path!r} is not valid JSON: {e}") + if not isinstance(obj, dict): + raise PolicyError( + f"policy root must be a JSON object (got {type(obj).__name__})") + return obj + + +def _emit_json_envelope(payload): + """Single-line JSON record, mirrors the shape of the sandbox + envelope so downstream tooling can ingest both interchangeably. + """ + print(json.dumps(payload, separators=(",", ":"), ensure_ascii=False)) + + +# --- Validation --------------------------------------------------------- + + +def validate_policy(policy): + """Validate an already-parsed policy dict. Returns a list of + :class:`PolicyError` instances (empty list on success); never + raises. Errors are meant to be printed by the CLI layer so the + user sees ALL of them, not just the first one. + """ + errors = [] + if not isinstance(policy, dict): + # The loader already guarantees this; defense in depth. + return [PolicyError(f"policy root must be a JSON object " + f"(got {type(policy).__name__})")] + + name = policy.get("name") + if not isinstance(name, str) or not name.strip(): + errors.append(PolicyError( + f"field 'name' must be a non-empty string (got {name!r})")) + + version = policy.get("version", "") + if not isinstance(version, str) or not _VERSION_RE.fullmatch(version or ""): + errors.append(PolicyError( + f"field 'version' must match MAJOR.MINOR.PATCH (got {version!r})")) + + # defaults block -------------------------------------------------- + defaults = policy.get("defaults", {}) + if not isinstance(defaults, dict): + errors.append(PolicyError( + f"field 'defaults' must be a JSON object (got " + f"{type(defaults).__name__})")) + defaults = {} + + mem = defaults.get("mem") + if mem is not None: + try: + n = _parse_size_bytes(mem) + if n is not None and not (MEM_MIN_BYTES <= n <= MEM_MAX_BYTES): + errors.append(PolicyError( + f"defaults.mem {mem!r} parses to {n} bytes; " + f"must be in [{MEM_MIN_BYTES}, {MEM_MAX_BYTES}]")) + except PolicyError as e: + errors.append(PolicyError(f"defaults.mem: {e}")) + + pids = defaults.get("pids") + if pids is not None and (not _is_int(pids) or + not (PIDS_MIN <= pids <= PIDS_MAX)): + errors.append(PolicyError( + f"defaults.pids {pids!r} must be an integer in " + f"[{PIDS_MIN}, {PIDS_MAX}]")) + + cpu_quota = defaults.get("cpu_quota") + if cpu_quota is not None and (not _is_int(cpu_quota) or + not (CPU_QUOTA_MIN <= cpu_quota <= CPU_QUOTA_MAX)): + errors.append(PolicyError( + f"defaults.cpu_quota {cpu_quota!r} must be an integer in " + f"[{CPU_QUOTA_MIN}, {CPU_QUOTA_MAX}]")) + + timeout = defaults.get("timeout") + if timeout is not None and (not _is_int(timeout) or + not (TIMEOUT_MIN <= timeout <= TIMEOUT_MAX)): + errors.append(PolicyError( + f"defaults.timeout {timeout!r} must be an integer in " + f"[{TIMEOUT_MIN}, {TIMEOUT_MAX}]")) + + # profiles block -------------------------------------------------- + profiles = policy.get("profiles", {}) + if not isinstance(profiles, dict): + errors.append(PolicyError( + f"field 'profiles' must be a JSON object (got " + f"{type(profiles).__name__})")) + profiles = {} + + seen_caps_per_profile = {} + for pname, caps in profiles.items(): + if not _PROFILE_NAME_RE.fullmatch(pname or ""): + errors.append(PolicyError( + f"profile name {pname!r} must match " + f"{_PROFILE_NAME_RE.pattern!r}")) + if not isinstance(caps, list) or not all(isinstance(c, str) for c in caps): + errors.append(PolicyError( + f"profile {pname!r}.caps must be a list of strings")) + continue + # A profile may be empty (drops nothing on purpose). + cleaned = [] + for c in caps: + if not _CAP_NAME_RE.fullmatch(c): + errors.append(PolicyError( + f"profile {pname!r} contains non-cap name {c!r}; " + f"must match {_CAP_NAME_RE.pattern!r}")) + else: + cleaned.append(c) + # De-dup + sort so transpile produces a canonical regex. + seen_caps_per_profile[pname] = sorted(set(cleaned)) + + # net ------------------------------------------------------------- + net = policy.get("net") + if net is not None and net not in _NET_VALUES: + errors.append(PolicyError( + f"field 'net' {net!r} must be one of {_NET_VALUES}")) + + # readonly -------------------------------------------------------- + ro = policy.get("readonly") + if ro is not None and not isinstance(ro, bool): + errors.append(PolicyError( + f"field 'readonly' must be a boolean (got {type(ro).__name__})")) + + # env_allowlist --------------------------------------------------- + env = policy.get("env_allowlist", []) + if not isinstance(env, list): + errors.append(PolicyError( + f"field 'env_allowlist' must be a list (got {type(env).__name__})")) + else: + for v in env: + if not isinstance(v, str) or not _ENV_NAME_RE.fullmatch(v): + errors.append(PolicyError( + f"env_allowlist entry {v!r} must match " + f"{_ENV_NAME_RE.pattern!r}")) + + # syscalls (optional) -------------------------------------------- + syscalls = policy.get("syscalls") + if syscalls is not None and not isinstance(syscalls, dict): + errors.append(PolicyError( + f"field 'syscalls' must be a JSON object or null " + f"(got {type(syscalls).__name__})")) + + # Attach the deduped caps back so the caller can build a + # canonical transpile without re-running the regex match. + return errors, seen_caps_per_profile + + +# --- Transpile ---------------------------------------------------------- + + +def transpile_argv(policy, profile_name, cleaned_caps): + """Build the argv slice that, when pasted after + ``neuros-sandbox --unsafe``, realises the policy + chosen + profile. The profile's caps are converted into a single + ``--cap-drop`` regex: ``CAP_(X|Y|Z)`` with sorted-unique + entries, or the never-match sentinel when empty. + """ + args = [] + defaults = policy.get("defaults", {}) + if "mem" in defaults and defaults["mem"] is not None: + args += ["--mem", defaults["mem"]] + if "pids" in defaults and defaults["pids"] is not None: + args += ["--pids", str(defaults["pids"])] + if "cpu_quota" in defaults and defaults["cpu_quota"] is not None: + args += ["--cpu-quota", str(defaults["cpu_quota"])] + if "timeout" in defaults and defaults["timeout"] is not None: + args += ["--timeout", str(defaults["timeout"])] + + if profile_name is not None: + if profile_name not in policy.get("profiles", {}): + raise PolicyError( + f"profile {profile_name!r} is not declared in policy " + f"(known: {sorted(policy.get('profiles', {}).keys())})") + caps = cleaned_caps.get(profile_name, []) + if caps: + args += ["--cap-drop", "CAP_(" + "|".join(caps) + ")"] + else: + args += ["--cap-drop", NEVER_MATCH_CAP_REGEX] + return args + + +# --- Check (envelope compliance) ---------------------------------------- + + +def check_envelope_against_policy(policy, envelope, cleaned_caps): + """Reconcile a neuros-sandbox JSON envelope against the policy. + Returns a list of violations (empty == conformant). Each violation + is a (rule, observed, expected, severity) tuple so the JSON + envelope has the same shape as the bench + runbook outputs. + """ + violations = [] + if not isinstance(envelope, dict): + return [("malformed", type(envelope).__name__, + "object", "error")] + defaults = policy.get("defaults", {}) + + # wall_clock is the most common regression vector: a workload + # that used to finish in 200 ms now takes 30 s is rare but real. + wall = envelope.get("wall_clock_ms") + timeout = defaults.get("timeout") + if wall is not None and timeout is not None: + wall_s = wall / 1000.0 + if wall_s > timeout: + violations.append( + ("wall_clock", round(wall_s, 3), timeout, "error")) + + # peak memory: only check when the primitive actually filled in + # the field (it can be None on hosts without cgroup delegation). + peak = envelope.get("peak_mem_estimate") + mem = defaults.get("mem") + if peak is not None and mem is not None: + try: + mem_bytes = _parse_size_bytes(mem) + if mem_bytes is not None and peak > mem_bytes: + violations.append( + ("peak_mem", peak, mem_bytes, "error")) + except PolicyError: + # mem field already validated upstream; skip quietly. + pass + + # timeout_hit bool: a `true` here means the wrapper had to SIGKILL + # the workload, which is a policy violation regardless of bounds. + if envelope.get("timeout_hit"): + violations.append( + ("timeout_hit", True, False, "error")) + + return violations + + +# --- Subcommand dispatch ------------------------------------------------ + + +def cmd_validate(args): + try: + policy = _load_policy(args.policy) + except PolicyError as e: + if args.json: + _emit_json_envelope({ + "ok": False, "errors": [str(e)], + "neuros_policy_version": NEUROS_POLICY_VERSION}) + else: + print(f"neuros-policy: {e}", file=sys.stderr) + return 2 + + errs, _cleaned = validate_policy(policy) + if args.json: + _emit_json_envelope({ + "ok": not errs, + "errors": [str(e) for e in errs], + "name": policy.get("name"), + "version": policy.get("version"), + "neuros_policy_version": NEUROS_POLICY_VERSION, + }) + else: + if errs: + print(f"neuros-policy: {len(errs)} violation(s):", + file=sys.stderr) + for e in errs: + print(f" - {e}", file=sys.stderr) + else: + print(f"neuros-policy: ok ({policy.get('name')!r} " + f"v{policy.get('version')})") + return 1 if errs else 0 + + +def cmd_check(args): + try: + policy = _load_policy(args.policy) + except PolicyError as e: + print(f"neuros-policy: {e}", file=sys.stderr) + return 2 + errs, cleaned = validate_policy(policy) + if errs: + # We refuse to check against a policy that itself is invalid. + if args.json: + _emit_json_envelope({ + "ok": False, + "policy_errors": [str(e) for e in errs], + "neuros_policy_version": NEUROS_POLICY_VERSION, + }) + else: + print(f"neuros-policy: policy has {len(errs)} violation(s); " + f"refusing to check envelope", file=sys.stderr) + for e in errs: + print(f" - {e}", file=sys.stderr) + return 2 + + try: + envelope = _load_policy(args.envelope) + except PolicyError as e: + print(f"neuros-policy: {e}", file=sys.stderr) + return 2 + + violations = check_envelope_against_policy(policy, envelope, cleaned) + if args.json: + _emit_json_envelope({ + "ok": not violations, + "violations": [ + {"rule": r, "observed": o, "expected": x, "severity": s} + for (r, o, x, s) in violations + ], + "policy_name": policy.get("name"), + "policy_version": policy.get("version"), + "neuros_policy_version": NEUROS_POLICY_VERSION, + }) + else: + if violations: + print(f"neuros-policy: {len(violations)} violation(s):", + file=sys.stderr) + for r, o, x, s in violations: + print(f" - [{s}] {r}: observed {o!r}, " + f"expected {x!r}", file=sys.stderr) + else: + exit_code = envelope.get("exit_code") + print(f"neuros-policy: ok " + f"(exit_code={exit_code}, wall_clock_ms=" + f"{envelope.get('wall_clock_ms')}, " + f"peak_mem={envelope.get('peak_mem_estimate')})") + return 1 if violations else 0 + + +def cmd_transpile(args): + try: + policy = _load_policy(args.policy) + except PolicyError as e: + print(f"neuros-policy: {e}", file=sys.stderr) + return 2 + errs, cleaned = validate_policy(policy) + if errs: + print(f"neuros-policy: policy has {len(errs)} violation(s); " + f"refusing to transpile", file=sys.stderr) + for e in errs: + print(f" - {e}", file=sys.stderr) + return 1 + try: + argv = transpile_argv(policy, args.profile, cleaned) + except PolicyError as e: + print(f"neuros-policy: {e}", file=sys.stderr) + return 1 + print(" ".join(argv)) + return 0 + + +# --- Argparse ------------------------------------------------------------ + + +def _build_parser(): + p = argparse.ArgumentParser( + prog="neuros-policy", + description=( + "Declarative security policy for neuros-sandbox. Validate, " + "check envelope compliance, or transpile to a sandbox argv " + "fragment." + ), + epilog=( + "Examples:\n" + " neuros-policy validate ./policy.json\n" + " neuros-policy check ./policy.json --envelope ./envelope.json\n" + " neuros-policy transpile ./policy.json --profile strict\n" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = p.add_subparsers(dest="cmd", required=True) + + val = sub.add_parser("validate", + help="schema + bounds check on a policy file") + val.add_argument("policy", + help="Path to a JSON policy file ('-' for stdin)") + val.add_argument("--json", action="store_true", + help="Emit a one-line JSON envelope instead of text") + + chk = sub.add_parser("check", + help="Check a neuros-sandbox JSON envelope " + "against a policy") + chk.add_argument("policy", help="Path to a JSON policy file") + chk.add_argument("--envelope", required=True, + help="Path to a JSON envelope file ('-' for stdin)") + chk.add_argument("--json", action="store_true", + help="Emit a one-line JSON envelope instead of text") + + tr = sub.add_parser("transpile", + help="Emit the neuros-sandbox argv fragment " + "realising this policy") + tr.add_argument("policy", help="Path to a JSON policy file") + tr.add_argument("--profile", default=None, + help="Profile name to enable (default: none)") + return p + + +def main(argv=None): + parser = _build_parser() + args = parser.parse_args(argv) + if args.cmd == "validate": + return cmd_validate(args) + if args.cmd == "check": + return cmd_check(args) + if args.cmd == "transpile": + return cmd_transpile(args) + parser.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/config/includes.chroot/usr/local/bin/neuros-replay b/config/includes.chroot/usr/local/bin/neuros-replay new file mode 100755 index 0000000..fc734eb --- /dev/null +++ b/config/includes.chroot/usr/local/bin/neuros-replay @@ -0,0 +1,386 @@ +#!/usr/bin/env python3 +"""neuros-replay: offline diagnostics over neuros-sandbox envelopes. + +The `neuros-sandbox` wrapper emits a single-line JSON envelope per +run (`exit_code`, `wall_clock_ms`, `peak_mem_estimate`, `timeout_hit`, +`stdout`, `stderr`). When a run looks wrong, the question is usually +"how wrong?". The bench harness compares batch aggregates but won't +help on a single observation. `neuros-replay` is the offline-matching +tool: it takes one or two envelopes and emits diagnostics without +re-executing the workload. + +Subcommands: + + neuros-replay diff + Compare two envelopes, emit a verdict envelope. Tolerances + default to 10% wall, 20% peak-mem (tighter than bench's + 15%/25% because diff operates on single observations, not + aggregates). Override with --tolerance-wall-pct and + --tolerance-mem-pct. Exit 0 on conformant, 1 on any rule + violation, 2 on bad input. + + neuros-replay explain + Emit a single sinkable line onto stdout (the "CI diaper"):: + + exit=0 wall=1542ms peak=14.2MiB timeout=false + stdout_bytes=0 stderr_bytes=132 + + Use `--json` for a structured equivalent. + + neuros-replay extract [--stream stdout|stderr|both] + Dump the embedded `stdout` and/or `stderr` payloads to stdout + so standard host tools (`grep`, `jq`, `less`) can read them + without escaping JSON strings. With both streams selected + the output gets a one-line ``===STDERR===`` separator so a + human or downstream tool can tell them apart. + + Common flags: + --json emit a single-line JSON envelope + on stdout (machines, not the diaper) +""" +import argparse +import json +import os +import sys + +# --- Constants ---------------------------------------------------------- + +#: Default tolerance (percent) on wall_clock_ms in `diff`. Single +#: observations on a shared kernel are noisier than bench's batch +#: aggregates, so 10% is the floor; users can tighten via +#: --tolerance-wall-pct. +DEFAULT_TOLERANCE_WALL_PCT = 10.0 + +#: Default tolerance for peak_mem_estimate in `diff`. Memory pressure +#: jitter on Linux is much higher than CPU jitter, hence 20%. +DEFAULT_TOLERANCE_MEM_PCT = 20.0 + +#: Tool version baked into emitted envelopes. +NEUROS_REPLAY_VERSION = "0.1" + + +# --- Exceptions -------------------------------------------------------- + + +class ReplayError(Exception): + """Raised on user-fixable misconfiguration (bad envelope, missing + field, ambiguous stream selector). Surfaced as exit code 2.""" + + +# --- Helpers ------------------------------------------------------------ + + +def _die(msg, code=2): + print(msg, file=sys.stderr) + sys.exit(code) + + +def _load_envelope(path): + """Read one JSON envelope from ``path`` ('-' for stdin). The + loader enforces 'object' at the top level so we can fail loudly + on user error instead of AttributeError-ing into TypeError. + """ + if path == "-": + try: + text = sys.stdin.read() + except OSError as e: + raise ReplayError(f"cannot read stdin: {e}") + else: + try: + with open(path, "r", encoding="utf-8") as f: + text = f.read() + except OSError as e: + raise ReplayError(f"cannot read envelope {path!r}: {e}") + try: + obj = json.loads(text) + except json.JSONDecodeError as e: + raise ReplayError(f"envelope {path!r} is not valid JSON: {e}") + if not isinstance(obj, dict): + raise ReplayError( + f"envelope must be a JSON object (got {type(obj).__name__})") + return obj + + +def _emit_json(payload): + print(json.dumps(payload, separators=(",", ":"), ensure_ascii=False)) + + +def _pct_delta(a, b): + """Return ``(b - a) / a * 100`` rounded to 2 dp, or ``None`` when + either operand is missing/zero (zero is a real workload + observation but divides to infinity; we surface it as None rather + than blowing up the diff with inf). + """ + if a is None or b is None: + return None + try: + a_f = float(a) + b_f = float(b) + except (TypeError, ValueError): + return None + if a_f == 0: + return None + return round((b_f - a_f) / a_f * 100.0, 2) + + +# --- diff -------------------------------------------------------------- + + +def _envelope_for_diff(env): + """Trim a sandbox envelope to the fields we score on. Anything + else (stdout/stderr) is irrelevant to a regression verdict and + would just bloat the output envelope. + " + """ + return { + "exit_code": env.get("exit_code"), + "wall_clock_ms": env.get("wall_clock_ms"), + "peak_mem_estimate": env.get("peak_mem_estimate"), + "timeout_hit": env.get("timeout_hit"), + } + + +def diff_envelopes(a, b, tol_wall_pct, tol_mem_pct): + """Compare two envelopes. Returns a dict ready for --json + output. Rule violations list human-readable strings so the + diaper (text mode) is informative without --json. + """ + violations = [] + aa, bb = _envelope_for_diff(a), _envelope_for_diff(b) + + # Exit code must match. + if aa.get("exit_code") != bb.get("exit_code"): + violations.append( + f"exit_code differs ({aa.get('exit_code')} vs " + f"{bb.get('exit_code')})") + + # Wall clock pct delta within --tolerance-wall-pct. + wall_pct = _pct_delta(aa.get("wall_clock_ms"), + bb.get("wall_clock_ms")) + if wall_pct is not None and abs(wall_pct) > tol_wall_pct: + violations.append( + f"wall_clock_pct_delta ({wall_pct}%) exceeds tolerance " + f"({tol_wall_pct}%)") + + # Peak memory pct delta within --tolerance-mem-pct. Skip when + # either side is None (the field was absent, common on hosts + # without cgroup delegation). + mem_pct = _pct_delta(aa.get("peak_mem_estimate"), + bb.get("peak_mem_estimate")) + if (mem_pct is not None + and aa.get("peak_mem_estimate") is not None + and bb.get("peak_mem_estimate") is not None + and abs(mem_pct) > tol_mem_pct): + violations.append( + f"peak_mem_pct_delta ({mem_pct}%) exceeds tolerance " + f"({tol_mem_pct}%)") + + # timeout_hit must match. + if aa.get("timeout_hit") != bb.get("timeout_hit"): + violations.append( + f"timeout_hit differs ({aa.get('timeout_hit')} vs " + f"{bb.get('timeout_hit')})") + + return { + "ok": not violations, + "exit_code_a": aa.get("exit_code"), + "exit_code_b": bb.get("exit_code"), + "wall_clock_pct_delta": wall_pct, + "peak_mem_pct_delta": mem_pct, + "rule_violations": violations, + "tolerance_wall_pct": tol_wall_pct, + "tolerance_mem_pct": tol_mem_pct, + "neuros_replay_version": NEUROS_REPLAY_VERSION, + } + + +def cmd_diff(args): + try: + a = _load_envelope(args.a) + b = _load_envelope(args.b) + except ReplayError as e: + print(f"neuros-replay: {e}", file=sys.stderr) + return 2 + verdict = diff_envelopes(a, b, args.tolerance_wall_pct, + args.tolerance_mem_pct) + if args.json: + _emit_json(verdict) + else: + if verdict["ok"]: + print(f"neuros-replay: diff OK " + f"(wall_delta={verdict['wall_clock_pct_delta']}%, " + f"mem_delta={verdict['peak_mem_pct_delta']}%)") + else: + print(f"neuros-replay: {len(verdict['rule_violations'])} " + f"violation(s):", file=sys.stderr) + for v in verdict["rule_violations"]: + print(f" - {v}", file=sys.stderr) + return 0 if verdict["ok"] else 1 + + +# --- explain ------------------------------------------------------------ + + +def _format_bytes(n): + """Humanize a byte count into 'MiB' style. None → 'n/a'. + """ + if n is None: + return "n/a" + try: + v = float(n) + except (TypeError, ValueError): + return "n/a" + if v >= 1024 * 1024: + return f"{v / 1024 / 1024:.1f}MiB" + if v >= 1024: + return f"{v / 1024:.1f}KiB" + return f"{int(v)}B" + + +def explain_envelope(env): + """Build the single-line explanation. The shape is grep-friendly: + cx field names without separators so it's easy to pipe through + awk/sed downstream. + """ + return { + "exit_code": env.get("exit_code", "n/a"), + "wall_clock_ms": env.get("wall_clock_ms", "n/a"), + "peak_mem_estimate": env.get("peak_mem_estimate", "n/a"), + "timeout_hit": env.get("timeout_hit", False), + "stdout_bytes": len(env.get("stdout", "") or ""), + "stderr_bytes": len(env.get("stderr", "") or ""), + } + + +def _explain_line(info): + return (f"exit={info['exit_code']} " + f"wall={info['wall_clock_ms']}ms " + f"peak={_format_bytes(info['peak_mem_estimate'])} " + f"timeout={info['timeout_hit']} " + f"stdout_bytes={info['stdout_bytes']} " + f"stderr_bytes={info['stderr_bytes']}") + + +def cmd_explain(args): + try: + env = _load_envelope(args.envelope) + except ReplayError as e: + print(f"neuros-replay: {e}", file=sys.stderr) + return 2 + info = explain_envelope(env) + if args.json: + info["neuros_replay_version"] = NEUROS_REPLAY_VERSION + _emit_json(info) + else: + print(_explain_line(info)) + return 0 + + +# --- extract ------------------------------------------------------------ + + +_STREAMS = ("stdout", "stderr", "both") + + +def cmd_extract(args): + try: + env = _load_envelope(args.envelope) + except ReplayError as e: + print(f"neuros-replay: {e}", file=sys.stderr) + return 2 + if args.stream not in _STREAMS: + print(f"neuros-replay: --stream must be one of {_STREAMS} " + f"(got {args.stream!r})", file=sys.stderr) + return 2 + out = env.get("stdout", "") or "" + err = env.get("stderr", "") or "" + if args.stream == "stdout": + sys.stdout.write(out) + elif args.stream == "stderr": + sys.stdout.write(err) + else: + sys.stdout.write(out) + if out and not out.endswith("\n"): + sys.stdout.write("\n") + sys.stdout.write("===STDERR===\n") + sys.stdout.write(err) + if err and not err.endswith("\n"): + sys.stdout.write("\n") + return 0 + + +# --- Argparse ----------------------------------------------------------- + + +def _build_parser(): + p = argparse.ArgumentParser( + prog="neuros-replay", + description=( + "Offline diagnostics over neuros-sandbox JSON envelopes. " + "Diff two envelopes, humanize one, or extract embedded " + "stdout/stderr." + ), + epilog=( + "Examples:\n" + " neuros-replay diff ./baseline.json ./candidate.json\n" + " neuros-replay explain ./envelope.json\n" + " neuros-replay extract ./envelope.json --stream stderr" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = p.add_subparsers(dest="cmd", required=True) + + diff_p = sub.add_parser("diff", + help="Compare two envelopes with a " + "regression verdict") + diff_p.add_argument("a", help="Path to baseline envelope " + "JSON ('-' for stdin)") + diff_p.add_argument("b", help="Path to candidate envelope " + "JSON ('-' for stdin)") + diff_p.add_argument("--tolerance-wall-pct", type=float, + default=DEFAULT_TOLERANCE_WALL_PCT, + help=f"Tolerance on wall-clock pct delta " + f"(default: {DEFAULT_TOLERANCE_WALL_PCT})") + diff_p.add_argument("--tolerance-mem-pct", type=float, + default=DEFAULT_TOLERANCE_MEM_PCT, + help=f"Tolerance on peak-mem pct delta " + f"(default: {DEFAULT_TOLERANCE_MEM_PCT})") + diff_p.add_argument("--json", action="store_true", + help="Emit a one-line JSON envelope instead " + "of text") + + expl_p = sub.add_parser("explain", + help="Humanize one envelope to a " + "grep-friendly summary line") + expl_p.add_argument("envelope", + help="Path to envelope JSON ('-' for stdin)") + expl_p.add_argument("--json", action="store_true", + help="Emit a one-line JSON envelope instead " + "of text") + + ext_p = sub.add_parser("extract", + help="Dump embedded stdout/stderr from " + "an envelope") + ext_p.add_argument("envelope", + help="Path to envelope JSON ('-' for stdin)") + ext_p.add_argument("--stream", choices=_STREAMS, default="stdout", + help="Which stream to emit " + "(default: stdout)") + return p + + +def main(argv=None): + parser = _build_parser() + args = parser.parse_args(argv) + if args.cmd == "diff": + return cmd_diff(args) + if args.cmd == "explain": + return cmd_explain(args) + if args.cmd == "extract": + return cmd_extract(args) + parser.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/config/includes.chroot/usr/local/bin/neuros-runbook b/config/includes.chroot/usr/local/bin/neuros-runbook new file mode 100755 index 0000000..730ccf1 --- /dev/null +++ b/config/includes.chroot/usr/local/bin/neuros-runbook @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +"""neuros-runbook: assertion-bearing runner for neuros-sandbox. + +The runbook format is JSON (one array of steps). Each step has: + + { + "name": "", + "script": "", + "limits": {"mem": "...", "pids": N, "timeout": N, "cpu_quota": N}, + "expect": { + "exit_code": N, + "stdout_matches": "", + "stderr_matches": "", + "timeout_forbidden": bool + } + } + +Output: per-step verdict on stdout (human-readable table) plus a +JSON envelope when --json is set (one record per step). Exit 0 if +every step passes, exit 1 otherwise. + +With --rest=violations-only, the human table only shows failed +steps; --rest=all shows every step. Default = all. + +The wrapper shells out to neuros-sandbox exactly like the bench +does, so they share the same envelope schema. That means runbook +verdicts can be diffed against production captures to detect when an +OS upgrade silently changes what a step's stdout looks like. +""" +import argparse +import json +import os +import re +import subprocess +import sys +import uuid + +# --- Defaults ---------------------------------------------------------- + +DEFAULT_TIMEOUT = 30 +DEFAULT_MEM = "256M" +DEFAULT_PIDS = 64 +DEFAULT_CPU_QUOTA = 50000 # 50% of one CPU +NEUROS_RUNBOOK_VERSION = "0.1" + + +# --- Runbook loading ---------------------------------------------------- + +class RunbookError(Exception): + """Raised on a malformed runbook the user can fix (not a kernel fault).""" + + +def load_runbook(path): + """Read and validate a JSON runbook file. + + Returns a list of step dicts. Each step has: + - name (str, required) + - script (str, required, non-empty) + - limits (dict, optional): per-step incisor for mem/pids/timeout/cpu_quota + - expect (dict, optional): exit_code (int or "any"), stdout_matches, + stderr_matches (regex str), timeout_forbidden + + Raises RunbookError on any structural problem with a message + that's safe to surface on stderr. + """ + try: + with open(path) as f: + raw = json.load(f) + except (OSError, json.JSONDecodeError) as e: + raise RunbookError(f"cannot read runbook {path!r}: {e}") + if not isinstance(raw, list): + raise RunbookError( + f"runbook {path!r} must be a JSON array of step objects") + seen_names = set() + for i, step in enumerate(raw): + if not isinstance(step, dict): + raise RunbookError(f"step {i}: must be an object") + name = step.get("name") + if not isinstance(name, str) or not name: + raise RunbookError(f"step {i}: 'name' must be a non-empty string") + if name in seen_names: + raise RunbookError( + f"step {i}: duplicate name {name!r}") + seen_names.add(name) + script = step.get("script") + if not isinstance(script, str) or not script: + raise RunbookError( + f"step {name!r}: 'script' must be a non-empty string") + limits = step.get("limits", {}) + if not isinstance(limits, dict): + raise RunbookError( + f"step {name!r}: 'limits' must be an object") + expect = step.get("expect", {}) + if not isinstance(expect, dict): + raise RunbookError( + f"step {name!r}: 'expect' must be an object") + # Pre-compile regexes so a malformed pattern fails the parser + # rather than the runtime assertion. Failures here are + # reported as RunbookError with the step name for grep-ability. + for key in ("stdout_matches", "stderr_matches"): + pat = expect.get(key) + if pat is not None: + if not isinstance(pat, str): + raise RunbookError( + f"step {name!r}: expect.{key} must be a string regex") + try: + re.compile(pat) + except re.error as e: + raise RunbookError( + f"step {name!r}: expect.{key} regex invalid: {e}") + if "exit_code" in expect: + ec = expect["exit_code"] + if ec != "any" and not isinstance(ec, int): + raise RunbookError( + f"step {name!r}: expect.exit_code must be an int or 'any'") + return raw + + +# --- Single-step execution ---------------------------------------------- + +def _run_step(step, env): + """Shell out to neuros-sandbox and return the parsed envelope plus + per-step assertion results.""" + name = step["name"] + script = step["script"] + limits = step.get("limits", {}) + argv = ["neuros-sandbox", "run", + "--timeout", str(limits.get("timeout", DEFAULT_TIMEOUT)), + "--mem", limits.get("mem", DEFAULT_MEM), + "--pids", str(limits.get("pids", DEFAULT_PIDS)), + "--name", f"neuros-runbook-{uuid.uuid4().hex[:8]}", + "--json", "--"] + cp = subprocess.run( + argv, + input=script.encode(), + env=env, + capture_output=True, + check=False, + timeout=limits.get("timeout", DEFAULT_TIMEOUT) + 10, + ) + stdout_text = cp.stdout.decode("utf-8", "replace").strip() + env_obj = {"exit_code": cp.returncode} + if stdout_text: + try: + env_obj = json.loads(stdout_text.splitlines()[-1]) + except json.JSONDecodeError: + env_obj.setdefault("stdout", stdout_text) + env_obj.setdefault("stdout", "") + env_obj.setdefault("stderr", + cp.stderr.decode("utf-8", "replace")) + env_obj.setdefault("wall_clock_ms", 0) + env_obj.setdefault("timeout_hit", False) + env_obj.setdefault("peak_mem_estimate", None) + + # Apply assertions. + expect = step.get("expect", {}) + failures = [] + if "exit_code" in expect: + ec = expect["exit_code"] + if ec != "any" and env_obj["exit_code"] != ec: + failures.append( + f"exit_code was {env_obj['exit_code']}, expected {ec}") + for key, text in (("stdout_matches", env_obj["stdout"]), + ("stderr_matches", env_obj["stderr"])): + pat = expect.get(key) + if pat is None: + continue + if not re.search(pat, text): + failures.append(f"{key} did not match /{pat}/") + if expect.get("timeout_forbidden") and env_obj["timeout_hit"]: + failures.append("timeout_hit=true but timeout_forbidden=true") + + return env_obj, failures + + +# --- Aggregation -------------------------------------------------------- + +def _run_runbook(path, env, only_violations=False): + steps = load_runbook(path) + record = {"version": NEUROS_RUNBOOK_VERSION, + "runbook_path": path, + "steps": []} + any_fail = False + for step in steps: + env_obj, failures = _run_step(step, env) + passed = not failures + if not passed: + any_fail = True + record["steps"].append({ + "name": step["name"], + "passed": passed, + "exit_code": env_obj["exit_code"], + "wall_clock_ms": env_obj["wall_clock_ms"], + "timeout_hit": env_obj["timeout_hit"], + "peak_mem_estimate": env_obj["peak_mem_estimate"], + "failures": failures, + }) + if only_violations: + visible = [s for s in record["steps"] if not s["passed"]] + else: + visible = record["steps"] + return record, visible, any_fail + + +# --- Output formatting -------------------------------------------------- + +def _print_table(records): + name_w = max(12, max((len(r["name"]) for r in records), default=12)) + print(f"{'name'.ljust(name_w)} {'status':<11} " + f"{'exit':>5} {'wall_ms':>9} {'peak_mem':>11} failures") + for r in records: + status = "PASS" if r["passed"] else "FAIL" + peak = "-" if r["peak_mem_estimate"] is None else str(r["peak_mem_estimate"]) + fail_summary = "" + if r["failures"]: + fail_summary = "; ".join(r["failures"]) + print( + f"{r['name'].ljust(name_w)} {status:<11} " + f"{r['exit_code']:>5} {r['wall_clock_ms']:>9} {peak:>11} " + f"{fail_summary}" + ) + + +# --- Subcommand dispatch ----------------------------------------------- + +def cmd_run(args): + env = { + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + } + try: + record, visible, any_fail = _run_runbook( + args.runbook, env, + only_violations=args.only_violations) + except RunbookError as e: + _die(str(e)) + if args.json: + print(json.dumps(record, indent=2)) + else: + if not visible: + print("(no violations)") + else: + _print_table(visible) + return 1 if any_fail else 0 + + +def _die(msg, code=2): + print(f"neuros-runbook: {msg}", file=sys.stderr) + sys.exit(code) + + +def _build_parser(): + p = argparse.ArgumentParser( + prog="neuros-runbook", + description=( + "Run a JSON runbook of neuros-sandbox steps with assertion " + "checks per step. Exit 0 if all steps pass, 1 on any failure." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = p.add_subparsers(dest="cmd", required=True) + p_run = sub.add_parser("run", help="run a runbook from a JSON file") + p_run.add_argument("runbook", help="path to the runbook JSON file") + p_run.add_argument( + "--only-violations", action="store_true", + help="only print rows that failed (default: all).") + p_run.add_argument( + "--json", action="store_true", + help="emit a JSON envelope instead of a human-readable table") + return p + + +def main(argv=None): + parser = _build_parser() + args = parser.parse_args(argv) + if args.cmd == "run": + rc = cmd_run(args) + return rc if rc is not None else 0 + parser.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/config/includes.chroot/usr/local/bin/neuros-sandbox b/config/includes.chroot/usr/local/bin/neuros-sandbox new file mode 100755 index 0000000..7ce2c60 --- /dev/null +++ b/config/includes.chroot/usr/local/bin/neuros-sandbox @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +"""neuros-sandbox: safe-runner wrapper around the `neuros-container` +primitive, tuned for **untrusted code execution** (e.g. scripts produced +by an LLM agent). The hardcoded defaults are the point of the tool: + + * `--net` : private network namespace (loopback only) + * `--read-only` : rootfs remounted RO + * `--mem 256M` : cgroup memory cap (override w/ --mem) + * `--pids 64` : cgroup PID cap (override w/ --pids) + * `--cpu-quota 50000` : 50% of one CPU (override w/ --cpu or + --cpu-quota) + * `--cap-drop ` : drop a baseline set of dangerous caps + * host env scrubbed : only PATH survives; no ANTHROPIC_API_KEY + or HOME leaks + +`--unsafe` opts in to relaxing the network/read-only/cap-drop trio, +which is the only path that should reach the host for untrusted work. + +Stdin-hand-off: the script body is always read into memory and piped +to `python3 -u -` inside the container. Nothing touches a tempfile on +the host; nothing leaks via /proc//cmdline because the path is +literal `-`. + +A watchdog enforces `--timeout` with SIGKILL on expiry. The output +mode is either `--human` (pass-through, like a normal command) or +`--json` (one line of structured envelope on stdout). + +Examples: + + echo 'print("hello agent")' | neuros-sandbox run --json + neuros-sandbox run --mem 1G --timeout 60 ./scripts/smoke.py + neuros-sandbox run --bundle ./fixture.tar.gz --json pre.sh +""" +import argparse +import glob +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +import uuid + +# --- Defaults -------------------------------------------------------------- + +#: cgroup v2 memory cap (bytes-string accepted by memory.max) +DEFAULT_MEM = "256M" + +#: cgroup v2 pids.max value +DEFAULT_PIDS = 64 + +#: CPU quota in microseconds per 100ms period (50000 = 50%) +DEFAULT_CPU_QUOTA = 50000 + +#: Capabilities dropped by default; regex syntax shared with the +#: primitive's --cap-drop. +DEFAULT_CAP_DROP = ( + r"CAP_(NET_RAW|SYS_ADMIN|SYS_PTRACE|SETUID|SETPCAP|" + r"MAC_ADMIN|DAC_OVERRIDE|LINUX_IMMUTABLE|SYS_CHROOT|" + r"SYS_RAWIO|SYS_RESOURCE)" +) + +#: Pre-compiled DEFAULT_CAP_DROP pattern so _build_argv doesn't +#: re-run re.compile on every wrapper-invocation. The constant is +#: itself validated (re.error at import time if malformed) so a +#: future edit to DEFAULT_CAP_DROP that breaks the regex fails +#: loudly at module load, not at kernel-bridge exec time. +_COMPILED_DEFAULT_CAP_DROP = re.compile(DEFAULT_CAP_DROP) + +#: Hard wall-clock timeout in seconds +DEFAULT_TIMEOUT = 30 + +#: Minimal env that gets passed through to the container. Deliberately +#: tiny; we do NOT inherit the host env. This is the difference between +#: "container" and "sandbox". +MINIMAL_ENV = { + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +} + +#: Name prefix for state files / forensic logs +NAME_PREFIX = "neuros-sbx-" + + +# --- Exceptions ------------------------------------------------------------ + + +class SandboxError(Exception): + """Raised on misconfiguration the user can fix (not kernel faults).""" + + +# --- Helpers --------------------------------------------------------------- + + +def _die(msg, code=2): + print(msg, file=sys.stderr) + sys.exit(code) + + +def _parse_percent(text): + """Parse '50%' or '0.5' or '50000us' (loose) into a CPU quota in + microseconds per 100ms CFS period.""" + s = text.strip() + if s.endswith("%"): + return int(float(s[:-1].strip()) * 1000) + if s.endswith("us"): + return int(s[:-2].strip()) + f = float(s) + if 0 < f <= 1.0: + return int(f * 100000) + return int(f) + + +def _build_argv(args, script_target, rootfs_override=None, name_override=None): + """Translate sandbox options into a neuros-container argv list. + + Order matters because `neuros-container` uses argparse with + `--name`/`--rootfs`/etc as dest flags and the run subcommand + triggers a positional dispatcher. + """ + argv = ["neuros-container", "run"] + argv += ["--name", name_override or args.name + or f"{NAME_PREFIX}{uuid.uuid4().hex[:10]}"] + argv += ["--rootfs", rootfs_override or args.rootfs or "/"] + if not args.unsafe: + argv += ["--net", "--read-only"] + argv += ["--mem", args.mem] + if args.pids is not None and args.pids > 0: + argv += ["--pids", str(args.pids)] + if args.cpu_quota is not None and args.cpu_quota > 0: + argv += ["--cpu-quota", str(args.cpu_quota)] + # The cap-drop baseline is the most load-bearing piece of the + # default safety posture, so it persists under --unsafe. The + # --cap-drop-keep knob is the explicit override path; only active + # in --unsafe mode (in safe mode the user just sets --cap-drop). + # Pre-validate the override regex so an invalid pattern fails + # at the wrapper layer (SandboxError), not deep inside the + # kernel-bridge ctypes call when the primitive rejects it. + if args.unsafe and args.cap_drop_keep is not None: + if not args.cap_drop_keep.strip(): + raise SandboxError( + "--cap-drop-keep must be a non-empty regex (got " + f"{args.cap_drop_keep!r}); omit --cap-drop-keep " + "to use the default") + try: + re.compile(args.cap_drop_keep) + except re.error as e: + raise SandboxError( + f"--cap-drop-keep {args.cap_drop_keep!r} is not a " + f"valid regex: {e}") + argv += ["--cap-drop", args.cap_drop_keep] + else: + argv += ["--cap-drop", args.cap_drop] + if args.user: + argv += ["--user", args.user] + # The script body is always piped via stdin; we use `python3 -u -` + # as a stable, executable-without-tmpfile entry point. When the + # caller passed a literal path, we pass it through argv; when the + # caller feeds stdin (script_target="-"), the "-" entry is already + # in argv so we must NOT append another "-". + argv += ["--", "python3", "-u", "-"] + if script_target != "-": + argv.append(script_target) + return argv + + +def _load_script(script_path): + """Return script bytes. `None` means 'read stdin'.""" + if script_path is None: + data = sys.stdin.buffer.read() + else: + try: + with open(script_path, "rb") as f: + data = f.read() + except OSError as e: + raise SandboxError(f"cannot read script: {script_path}: {e}") + if not data: + raise SandboxError("empty script (no bytes piped or file empty)") + return data + + +def _extract_bundle(tar_path): + """Unpack a tar.gz into a fresh tmp dir; return its path.""" + target = tempfile.mkdtemp(prefix="neuros-sbx-bundle-") + try: + with tarfile.open(tar_path, "r:*") as tar: + # Defensive: refuse absolute paths and `..` traversal + for member in tar.getmembers(): + p = os.path.normpath(member.name) + if p.startswith("/") or p.startswith(".."): + raise SandboxError( + f"unsafe bundle entry: {member.name!r}") + tar.extractall(target) + except (tarfile.TarError, OSError) as e: + shutil.rmtree(target, ignore_errors=True) + raise SandboxError(f"bundle extract failed: {e}") + return target + + +# --- JSON envelope --------------------------------------------------------- + + +def _envelope(rc, stdout, stderr, wall_ms, timeout_hit, peak_mem=None): + """Build a single-line JSON record describing the run.""" + return json.dumps({ + "exit_code": rc, + "stdout": stdout.decode("utf-8", "replace") if stdout else "", + "stderr": stderr.decode("utf-8", "replace") if stderr else "", + "wall_clock_ms": int(wall_ms), + "timeout_hit": timeout_hit, + "peak_mem_estimate": peak_mem, + }, separators=(",", ":"), ensure_ascii=False) + + +# --- cgroup peak-memory polling -------------------------------------------- + +#: Standard cgroup v2 mount root on modern Ubuntu; we keep this as a +#: list because some container hosts remount under /sys/fs/cgroup/. +_CGROUP_V2_ROOTS = ("/sys/fs/cgroup",) + + +def _read_peak_memory(name): + """Best-effort read of memory.peak for a leaf cgroup whose name + contains ``name``. Returns ``None`` on any failure (file missing, + permission denied, parse error). Best-effort only: the primitive + auto-cleans the cgroup on non-detached exit, so the peak file is + often already gone by the time we read. For forensic accuracy run + the container with `--detach` and read the file before calling + `neuros-container cleanup`.""" + suffix = f"neuros-sbx-{name}" if not name.startswith("neuros-sbx-") else name + for root in _CGROUP_V2_ROOTS: + # match either a leaf named the suffix, or a path that ends + # with / somewhere under the root + candidates = glob.glob(os.path.join(root, "**", suffix), + recursive=True) + if not candidates: + # also try a direct path under the root for the common + # case where the cgroup was created in the default tree + direct = os.path.join(root, suffix) + if os.path.isdir(direct): + candidates = [direct] + # When multiple leaves match (a re-runner of the same wrapper + # name), prefer the most recently modified cgroup: that is the + # one whose kernel stats describe the just-finished run. + candidates.sort(key=lambda p: os.path.getmtime(p), reverse=True) + for path in candidates: + peak = os.path.join(path, "memory.peak") + try: + with open(peak) as f: + val = f.read().strip() + return int(val) if val.isdigit() else None + except (OSError, ValueError): + continue + return None + + +# --- Top-level run --------------------------------------------------------- + + +def cmd_run(args): + """Execute the `run` subcommand end-to-end.""" + rootfs_override = None + cleanup_paths = [] + try: + if args.bundle: + rootfs_override = _extract_bundle(args.bundle) + cleanup_paths.append(rootfs_override) + # Build argv BEFORE reading the script so --dry-run can be + # exercised without a real script file on disk. + argv = _build_argv(args, + script_target=args.script or "-", + rootfs_override=rootfs_override) + if args.dry_run: + print(" ".join(argv)) + return 0 + script_bytes = _load_script(args.script) + # Pick the final container name now (the same way _build_argv + # would have chosen it) so the post-run cgroup poll has a + # stable target. + container_name = (args.name + or f"{NAME_PREFIX}{uuid.uuid4().hex[:10]}") + t0 = time.monotonic() + timeout_hit = False + peak_mem = None + try: + cp = subprocess.run( + argv, + env=MINIMAL_ENV, + input=script_bytes, + cwd="/", + capture_output=True, + check=False, + timeout=args.timeout, + ) + except subprocess.TimeoutExpired: + timeout_hit = True + cp = subprocess.CompletedProcess( + argv, 124, b"", b"") + wall_ms = (time.monotonic() - t0) * 1000.0 + # Best-effort peak-memory read; safe in environments without + # cgroup delegation (returns None silently). + peak_mem = _read_peak_memory(container_name) + if args.json: + print(_envelope( + rc=cp.returncode, + stdout=cp.stdout, + stderr=cp.stderr, + wall_ms=wall_ms, + timeout_hit=timeout_hit, + peak_mem=peak_mem, + )) + else: + sys.stdout.buffer.write(cp.stdout or b"") + sys.stderr.buffer.write(cp.stderr or b"") + sys.stderr.write( + f"[neuros-sandbox] exit={cp.returncode} " + f"wall_ms={wall_ms:.0f} timeout_hit={timeout_hit} " + f"peak_mem_estimate={peak_mem}\n" + ) + return 124 if timeout_hit else (cp.returncode or 0) + finally: + for p in cleanup_paths: + shutil.rmtree(p, ignore_errors=True) + + +# --- Argparse -------------------------------------------------------------- + + +def _build_parser(): + p = argparse.ArgumentParser( + prog="neuros-sandbox", + description=( + "Safe-runner for untrusted scripts via neuros-container. " + "Hardened by default; --unsafe relocks the blast radius." + ), + epilog=( + "Examples:\n" + " echo 'print(1+1)' | neuros-sandbox run --json\n" + " neuros-sandbox run --mem 1G --timeout 60 ./smoke.py\n" + " neuros-sandbox run --bundle ./fixture.tar.gz --dry-run\n" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = p.add_subparsers(dest="cmd", required=True) + run = sub.add_parser("run", help="Run a script in a sandboxed container") + + run.add_argument("script", nargs="?", + help="Script file; stdin is read if omitted.") + run.add_argument("--mem", default=DEFAULT_MEM, + help=f"Memory cap (default: {DEFAULT_MEM}); " + "e.g. 256M, 1G.") + run.add_argument("--pids", type=int, default=DEFAULT_PIDS, + help=f"PID cap (default: {DEFAULT_PIDS}).") + run.add_argument("--cpu", + help=f"CPU %% quota; maps to --cpu-quota " + f"(default: {DEFAULT_CPU_QUOTA/1000:.0f}%%).") + run.add_argument("--cpu-quota", type=int, default=DEFAULT_CPU_QUOTA, + help=f"CPU quota µs/100ms (default: " + f"{DEFAULT_CPU_QUOTA}).") + run.add_argument("--cap-drop", default=DEFAULT_CAP_DROP, + help="Regex of Linux capabilities to drop " + "(default: a hardened baseline).") + run.add_argument("--cap-drop-keep", + help="Under --unsafe, replace (not relax) the " + "default cap-drop baseline with this " + "regex (re.compile-validated up front). " + "Ignored in safe mode.") + run.add_argument("--rootfs", + help="Rootfs path inside the container " + "(default: /).") + run.add_argument("--name", + help="Container name; default = auto-random.") + run.add_argument("--user", + help="UID:GID inside the container.") + run.add_argument("--bundle", + help="Tar.gz to extract into the container's " + "rootfs (use with --rootfs pointing inside).") + run.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, + help=f"Wall-clock cap in seconds " + f"(default: {DEFAULT_TIMEOUT}).") + run.add_argument("--unsafe", action="store_true", + help="Disable --net / --read-only / default caps.") + run.add_argument("--json", action="store_true", + help="Emit a single JSON envelope on stdout.") + run.add_argument("--dry-run", action="store_true", + help="Print the neuros-container argv and exit.") + return p + + +def main(argv=None): + parser = _build_parser() + args = parser.parse_args(argv) + if args.cmd == "run": + # Resolve --cpu into --cpu_quota if both weren't set explicitly + if args.cpu is not None: + args.cpu_quota = _parse_percent(args.cpu) + try: + return cmd_run(args) + except SandboxError as e: + _die(f"neuros-sandbox: {e}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/config/includes.chroot/usr/local/bin/neuroslib.py b/config/includes.chroot/usr/local/bin/neuroslib.py index 459eae3..044c16c 100755 --- a/config/includes.chroot/usr/local/bin/neuroslib.py +++ b/config/includes.chroot/usr/local/bin/neuroslib.py @@ -5,12 +5,14 @@ LLM querying, encrypted storage, and file operations. Usage (in other neuros-* tools): - from neuroslib import load_config, query_llm, load_db, save_db + from neuroslib import load_config, get_default_model, get_context_config, + query_llm, load_db, save_db This eliminates code duplication across 70+ Neuros tools. """ import base64 +import configparser import hashlib import json import os @@ -24,25 +26,156 @@ CONFIG_DIR = os.path.expanduser("~/.config/neuros") LLM_CONF = os.path.join(CONFIG_DIR, "llm.conf") +# === Defaults === +DEFAULT_MODEL = "mistral" +DEFAULT_HOST = "localhost" +DEFAULT_PORT = "11434" + +# Opt-in context sources. Every source defaults to False (off). +DEFAULT_CONTEXT_SOURCES = frozenset({"window_title", "clipboard", "recent_files"}) + +# Truthy string values for boolean coercion in llm.conf. +_TRUTHY = frozenset({"true", "1", "yes", "on"}) + # ═══════════════════════════════════════════════════════════════════════ # Config Loading # ═══════════════════════════════════════════════════════════════════════ -def load_config(): - """Load llm.conf settings. Returns dict with model, host, port.""" - cfg = {"model": "mistral", "host": "localhost", "port": "11434"} +def _parse_bool(text): + if text is None: + return False + return text.strip().lower() in _TRUTHY + + +def _strip_quotes(text): + """Drop one pair of surrounding matching single/double quotes + from a configparser value. configparser preserves quotation verbatim; + the previous naive key=value loop stripped quotes so existing + llm.conf files (most of which write ``model = "mistral"``) still + round-trip identically under the new parser.""" + if text is None: + return None + text = text.strip() + if len(text) >= 2 and text[0] == text[-1] and text[0] in ('"', "'"): + return text[1:-1].strip() + return text + + +def _read_parser(path): + """Return a configparser.RawConfigParser with interpolation off + (we use '=' in model names like 'qwen2.5:7b' that would otherwise + confuse interpolators). Returns a defaults-only parser if the + file is missing, unreadable, or empty. + + The previous naive parser happily consumed flat ``key = value`` + files written without a section header. configparser does not — + it raises ``MissingSectionHeaderError``. To preserve back-compat + with existing llm.conf files in the wild, we detect the no-section + case and wrap the content in a synthetic ``[llm]`` block before + handing it to configparser.""" + parser = configparser.RawConfigParser( + interpolation=None, + allow_no_value=True, + inline_comment_prefixes=("#",), + ) + # Default fallbacks; overridden by values present in the file. + parser.add_section("llm") + parser.set("llm", "model", DEFAULT_MODEL) + parser.set("llm", "host", DEFAULT_HOST) + parser.set("llm", "port", DEFAULT_PORT) + + if path and os.path.exists(path): + try: + with open(path, "r", encoding="utf-8") as f: + raw = f.read() + except (OSError, UnicodeDecodeError): + return parser + try: + if raw.lstrip().startswith("["): + parser.read_string(raw) + else: + parser.read_string("[llm]\n" + raw) + except configparser.Error: + return parser + return parser + + +def load_config(path=None): + """Load llm.conf as a flat dict with section-prefixed keys (e.g. + 'llm.model', 'context.window_title'). When ``path`` is None the + default ``LLM_CONF`` is read. A missing or unreadable file returns + defaults — callers don't need to guard. Quoted string values are + unquoted on the way out so files written by the old naive parser + (``model = "mistral"``) still round-trip identically under + configparser, which preserves quotes verbatim.""" + parser = _read_parser(path or LLM_CONF) + flat = { + "llm.model": _strip_quotes(parser.get("llm", "model", fallback=DEFAULT_MODEL)), + "llm.host": _strip_quotes(parser.get("llm", "host", fallback=DEFAULT_HOST)), + "llm.port": _strip_quotes(parser.get("llm", "port", fallback=DEFAULT_PORT)), + } + # Always include every known context source at False so callers + # don't need to guard for missing keys; absent [context] section + # is a valid configuration state, not a malformed one. + for key in DEFAULT_CONTEXT_SOURCES: + flat[f"context.{key}"] = _parse_bool( + parser.get("context", key, fallback=None) + ) + for section in parser.sections(): + if section in ("llm", "context"): + continue + for key, value in parser.items(section): + flat[f"{section}.{key}"] = _strip_quotes(value) + return flat + + +def get_default_model(path=None): + """Return the configured default model, falling back to DEFAULT_MODEL.""" + cfg = load_config(path) + model = cfg.get("llm.model", DEFAULT_MODEL).strip().strip('"').strip("'") + return model or DEFAULT_MODEL + + +def get_context_config(path=None): + """Return the dict of opt-in system-context sources, all False by + default. See the [context] section of llm.conf.""" + cfg = load_config(path) + return { + source: bool(cfg.get(f"context.{source}", False)) + for source in DEFAULT_CONTEXT_SOURCES + } + + +def set_default_model(name, path=None): + """Persist a new default model in llm.conf under [llm]. Creates the + file and parent directories if needed. Existing lines in [llm] are + preserved; only the ``model`` key is overwritten. The write goes + through a ``.tmp`` + ``os.replace`` so a crash mid-write leaves + either the previous file or the new file, never a half-written + one — and the ``.tmp`` is cleaned up on failure.""" + config_path = path or LLM_CONF + parser = _read_parser(config_path) + if not parser.has_section("llm"): + parser.add_section("llm") + parser.set("llm", "model", name) + + target_dir = os.path.dirname(config_path) or CONFIG_DIR + os.makedirs(target_dir, exist_ok=True) + tmp = config_path + ".tmp" try: - if os.path.exists(LLM_CONF): - with open(LLM_CONF) as f: - for line in f: - line = line.strip() - if "=" in line and not line.startswith("#"): - k, v = line.split("=", 1) - cfg[k.strip()] = v.strip() + with open(tmp, "w", encoding="utf-8") as f: + parser.write(f) + os.replace(tmp, config_path) except Exception: - pass - return cfg + # ``parser.write(f)`` raising mid-write would leave the tmp + # file behind; unlink it so the user's config dir doesn't + # accumulate orphan .tmp files. + try: + os.unlink(tmp) + except OSError: + pass + raise # ═══════════════════════════════════════════════════════════════════════ diff --git a/config/includes.chroot/usr/local/bin/nn b/config/includes.chroot/usr/local/bin/nn index 4606bf3..6af6365 100755 --- a/config/includes.chroot/usr/local/bin/nn +++ b/config/includes.chroot/usr/local/bin/nn @@ -8,52 +8,42 @@ Usage: """ import sys, os, subprocess, json -try: - import requests -except ImportError: - print("requests not installed. Run: pip3 install requests") - sys.exit(1) +import urllib.request +import urllib.error + +# Shared library lives next to this script in /usr/local/bin; importing +# it by path keeps ``nn`` robust to whatever the on-disk PYTHONPATH is +# (the build installs them side by side). +import importlib.util +_LIB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "neuroslib.py") +_spec = importlib.util.spec_from_file_location("neuroslib", _LIB_PATH) +neuroslib = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(neuroslib) + +OLLAMA_URL = f"http://{neuroslib.DEFAULT_HOST}:{neuroslib.DEFAULT_PORT}/api/generate" +CONFIG_PATH = neuroslib.LLM_CONF +DEFAULT_MODEL = "mistral" # mirrors neuroslib.DEFAULT_MODEL; kept as a literal so static assertions can grep for it. +# Fails loudly if the canonical value ever drifts away from this +# mirror, instead of silently producing stale container images. +assert DEFAULT_MODEL == neuroslib.DEFAULT_MODEL, ( + f"nn.DEFAULT_MODEL {DEFAULT_MODEL!r} drifted from " + f"neuroslib.DEFAULT_MODEL {neuroslib.DEFAULT_MODEL!r} — keep them in sync." +) -OLLAMA_URL = "http://localhost:11434/api/generate" -CONFIG_PATH = os.path.expanduser("~/.config/neuros/llm.conf") -DEFAULT_MODEL = "mistral" def get_model(): - try: - with open(CONFIG_PATH) as f: - for line in f: - line = line.strip() - # Parse both flat key=value and TOML-style key = "value" - if '=' in line and not line.startswith('['): - key, val = line.split('=', 1) - if key.strip() == "model": - return val.strip().strip('"').strip("'") - except: - pass - return DEFAULT_MODEL + """Return the configured default model. Reads llm.conf at + CONFIG_PATH; falls back to ``neuroslib.DEFAULT_MODEL`` if unset or + unreadable. ``CONFIG_PATH`` is kept as a module-level redirect so + existing callers (and the test suite's patches) still bind here.""" + return neuroslib.get_default_model(CONFIG_PATH) + def get_context_config(): """Read opt-in system-context settings from the [context] section of llm.conf. Every source defaults to False (off) unless the config explicitly turns it on, including when the section is missing entirely.""" - settings = {"window_title": False, "clipboard": False, "recent_files": False} - section = None - try: - with open(CONFIG_PATH) as f: - for line in f: - line = line.strip() - if line.startswith('['): - section = line.strip('[]').strip() - continue - if section == "context" and '=' in line: - key, val = line.split('=', 1) - key = key.strip() - val = val.strip().strip('"').strip("'").lower() - if key in settings: - settings[key] = val in ("true", "1", "yes", "on") - except: - pass - return settings + return neuroslib.get_context_config(CONFIG_PATH) def get_active_window_title(): """Return the title of the focused window via xdotool, if installed. @@ -152,23 +142,34 @@ def read_file_safe(path): return f"(Could not read file: {e})" def stream_response(prompt, model): + """Stream a single prompt to Ollama. Uses urllib so nn needs no + third-party deps (the live-build target pulls only stdlib). The + previous implementation required the ``requests`` package and + crashed hard if it wasn't installed.""" + body = json.dumps({"model": model, "prompt": prompt, "stream": True}).encode() + req = urllib.request.Request( + OLLAMA_URL, data=body, + headers={"Content-Type": "application/json"}, + ) try: - resp = requests.post(OLLAMA_URL, json={ - "model": model, - "prompt": prompt, - "stream": True - }, stream=True, timeout=120) - resp.raise_for_status() - for line in resp.iter_lines(): - if line: - data = json.loads(line) + with urllib.request.urlopen(req, timeout=120) as resp: + for line in resp: + if not line: + continue + try: + data = json.loads(line) + except json.JSONDecodeError: + continue print(data.get("response", ""), end="", flush=True) if data.get("done"): print() - break - except requests.exceptions.ConnectionError: - print("Error: NeurOS LLM daemon is not running.") - print("Start it with: sudo systemctl start neuros-llm") + return + except urllib.error.URLError as e: + if isinstance(e.reason, ConnectionRefusedError): + print("Error: NeurOS LLM daemon is not running.") + print("Start it with: sudo systemctl start neuros-llm") + else: + print(f"Error: {e}") except Exception as e: print(f"Error: {e}") diff --git a/tests/test_bench.py b/tests/test_bench.py new file mode 100755 index 0000000..4b5666d --- /dev/null +++ b/tests/test_bench.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""Tests for neuros-bench regression benchmark harness. + +These tests do not need real container kernel delegation because +they mock subprocess.run and the inner _run_workload helper. They +assert: + + * argparse dispatch routes list / run / batch / compare correctly + * --dry-run-equivalent paths emit metrics JSON shaped per the contract + * compare-mode rejects regressions beyond --tolerance-wall-pct / mem + * builtin workload registry contains the documented 8 entries + * _pct_delta handles None + zero-base gracefully + * _render_fn strips the leading `def _w_xxx():` line and re-indents +""" +import argparse +import importlib.util +import io +import json +import os +import sys +import tempfile +import types +import unittest +from unittest import mock + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) +_BENCH_PATH = os.path.join( + ROOT, "config", "includes.chroot", "usr", "local", "bin", "neuros-bench") + + +def _load_neuros_bench(): + """Load `neuros-bench` (no .py suffix in its install path) by + compiling its source and exec-ing in a fresh module namespace. + Same pattern as `_load_neuros_sandbox` in test_sandbox.py.""" + with open(_BENCH_PATH) as f: + source = f.read() + code = compile(source, _BENCH_PATH, "exec") + module = sys.modules.get("neuros_bench") or types.ModuleType("neuros_bench") + module.__file__ = _BENCH_PATH + sys.modules["neuros_bench"] = module + exec(code, module.__dict__) + return module + + +nb = _load_neuros_bench() + + +class TestWorkloadRegistry(unittest.TestCase): + """Locks the workload suite membership so an accidental rename or + drop is a visible test failure rather than a silent regression.""" + + EXPECTED = frozenset({ + "cpu_tight", "json_parse", "regex_compile", "subproc_spawn", + "mem_grow", "file_io", "ctypes_call", "string_ops", + }) + + def test_expected_workloads_present(self): + self.assertEqual(set(nb.WORKLOADS), self.EXPECTED) + + def test_each_workload_has_callable_and_baseline(self): + for name, (fn, baseline) in nb.WORKLOADS.items(): + self.assertTrue(callable(fn), + f"workload {name} fn is not callable") + self.assertIsInstance(baseline, int) + self.assertGreater(baseline, 0, + f"workload {name} baseline_ms must be > 0") + + def test_registered_workloads_distinct(self): + # Each fn is a top-level function with a distinct identity. + seen = set() + for name, (fn, _) in nb.WORKLOADS.items(): + self.assertNotIn(id(fn), seen, + f"workload {name} shares identity with " + "another workload") + seen.add(id(fn)) + + +class TestRenderFn(unittest.TestCase): + """Locks the contract: _render_fn strips the def line and yields + a re-indented body so the inner try/except sees uniform indent.""" + + def test_renders_function_body_without_def_line(self): + rendered = nb._render_fn(nb._w_cpu_tight) + text = "\n".join(rendered) + self.assertNotIn("def _w_cpu_tight", text) + self.assertIn("def fib", text) # nested function preserved + self.assertIn("fib(28)", text) # the trailing call preserved + + def test_each_builtin_renders(self): + for name, (fn, _) in nb.WORKLOADS.items(): + rendered = nb._render_fn(fn) + self.assertTrue(rendered, + f"workload {name} rendered to empty list") + + +class TestPctDelta(unittest.TestCase): + """Locks the percent-delta helpers' edge cases.""" + + def test_zero_base_returns_none(self): + self.assertIsNone(nb._pct_delta(0, 100)) + + def test_negative_delta(self): + # Improvement: candidate is faster than baseline. + self.assertAlmostEqual(nb._pct_delta(100, 80), -20.0) + + def test_positive_delta(self): + # Regression: candidate is slower. + self.assertAlmostEqual(nb._pct_delta(100, 130), 30.0) + + def test_optional_handles_none(self): + self.assertIsNone(nb._pct_delta_optional(None, 100)) + self.assertIsNone(nb._pct_delta_optional(100, None)) + + def test_optional_handles_zero_base(self): + self.assertIsNone(nb._pct_delta_optional(0, 100)) + + +class TestCompareFunction(unittest.TestCase): + """The compare subcommand is the load-bearing piece for CI: exit + 1 if any workload's wall or mem regresses beyond tolerance.""" + + def _metrics(self, runs): + return { + "runs": runs, + "total_wall_clock_ms": 12345, + "config": {}, + "neuros_bench_version": nb.NEUROS_BENCH_VERSION, + } + + def _run_row(self, name, wall_ms, peak=None): + return { + "name": name, "wall_clock_ms": wall_ms, + "exit_code": 0, "stdout": "", "stderr": "", + "timeout_hit": False, "peak_mem_estimate": peak, + } + + def _write_metrics(self, runs): + f = tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") + json.dump(self._metrics(runs), f) + f.close() + return f.name + + def test_no_regression_exits_zero(self): + a = self._write_metrics([ + self._run_row("cpu_tight", 1000, peak=10_000_000) + ]) + b = self._write_metrics([ + self._run_row("cpu_tight", 1050, peak=10_500_000) + ]) + ns = mock.MagicMock() + ns.baseline = a + ns.candidate = b + ns.tolerance_wall_pct = 15.0 + ns.tolerance_mem_pct = 25.0 + # Capture both stdout and stderr; compare-mode writes the + # table on stdout and the verdict on stderr. + buf_out, buf_err = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdout", buf_out), \ + mock.patch.object(sys, "stderr", buf_err): + rc = nb.cmd_compare(ns) + self.assertEqual(rc, 0, buf_out.getvalue()) + self.assertIn("OK", buf_out.getvalue()) + self.assertNotIn("REGRESSION", buf_out.getvalue()) + os.unlink(a) + os.unlink(b) + + def test_wall_regression_exits_one(self): + a = self._write_metrics([ + self._run_row("cpu_tight", 1000, peak=10_000_000) + ]) + b = self._write_metrics([ + self._run_row("cpu_tight", 1300, peak=10_500_000) + ]) + ns = mock.MagicMock() + ns.baseline = a + ns.candidate = b + ns.tolerance_wall_pct = 15.0 + ns.tolerance_mem_pct = 25.0 + buf_out, buf_err = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdout", buf_out), \ + mock.patch.object(sys, "stderr", buf_err): + rc = nb.cmd_compare(ns) + self.assertEqual(rc, 1) + self.assertIn("REGRESSION", buf_out.getvalue()) + os.unlink(a) + os.unlink(b) + + def test_new_workload_added_in_candidate_is_not_regression(self): + """A NEW workload appears only in candidate, not in baseline. + This is an addition, NOT a regression — CI should pass with + exit 0 and the row should be marked NEW (informational).""" + a = self._write_metrics([ + self._run_row("cpu_tight", 1000, peak=10_000_000) + ]) + b = self._write_metrics([ + self._run_row("cpu_tight", 1000, peak=10_000_000), + self._run_row("json_parse", 800, peak=10_000_000), + ]) + ns = mock.MagicMock() + ns.baseline = a + ns.candidate = b + ns.tolerance_wall_pct = 15.0 + ns.tolerance_mem_pct = 25.0 + buf_out, buf_err = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdout", buf_out), \ + mock.patch.object(sys, "stderr", buf_err): + rc = nb.cmd_compare(ns) + self.assertEqual(rc, 0, buf_out.getvalue()) + self.assertIn("NEW", buf_out.getvalue()) + os.unlink(a) + os.unlink(b) + + def test_dropped_workload_in_candidate_is_regression(self): + """A workload present in baseline but absent in candidate is a + DROPPED workload — CI should exit 1.""" + a = self._write_metrics([ + self._run_row("cpu_tight", 1000, peak=10_000_000), + self._run_row("string_ops", 500, peak=10_000_000), + ]) + b = self._write_metrics([ + self._run_row("cpu_tight", 1000, peak=10_000_000), + ]) + ns = mock.MagicMock() + ns.baseline = a + ns.candidate = b + ns.tolerance_wall_pct = 15.0 + ns.tolerance_mem_pct = 25.0 + buf_out, buf_err = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdout", buf_out), \ + mock.patch.object(sys, "stderr", buf_err): + rc = nb.cmd_compare(ns) + self.assertEqual(rc, 1) + self.assertIn("DROPPED", buf_out.getvalue()) + os.unlink(a) + os.unlink(b) + + def test_mem_regression_exits_one(self): + a = self._write_metrics([ + self._run_row("mem_grow", 300, peak=10_000_000) + ]) + b = self._write_metrics([ + self._run_row("mem_grow", 305, peak=15_000_000) + ]) + ns = mock.MagicMock() + ns.baseline = a + ns.candidate = b + ns.tolerance_wall_pct = 15.0 + ns.tolerance_mem_pct = 25.0 + buf_out, buf_err = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdout", buf_out), \ + mock.patch.object(sys, "stderr", buf_err): + rc = nb.cmd_compare(ns) + self.assertEqual(rc, 1) + self.assertIn("REGRESSION", buf_out.getvalue()) + os.unlink(a) + os.unlink(b) + + +class TestRunWorkloadMocked(unittest.TestCase): + """Drive _run_workload with a mocked subprocess.run so we exercise + the JSON-envelope-parsing path without invoking the real + neuros-sandbox binary.""" + + def _fake_completed(self, stdout_text, returncode=0, stderr=b""): + cp = mock.Mock(returncode=returncode, stdout=stdout_text.encode()) + cp.stderr = stderr + return cp + + def test_parses_top_line_envelope(self): + env = {"exit_code": 0, "wall_clock_ms": 250, "timeout_hit": False, + "peak_mem_estimate": 31457280} + cp = self._fake_completed(json.dumps(env)) + with mock.patch.object(nb.subprocess, "run", return_value=cp), \ + mock.patch.object(nb, "_load_script", + create=True, return_value=b"x"): + out = nb._run_workload("cpu_tight", 60, "256M", 64) + self.assertEqual(out["name"], "cpu_tight") + self.assertEqual(out["wall_clock_ms"], 250) + self.assertEqual(out["peak_mem_estimate"], 31457280) + + def test_no_envelope_returns_fallback_record(self): + cp = self._fake_completed("") + with mock.patch.object(nb.subprocess, "run", return_value=cp): + out = nb._run_workload("cpu_tight", 60, "256M", 64) + self.assertEqual(out["name"], "cpu_tight") + self.assertEqual(out["wall_clock_ms"], 0) + self.assertIsNone(out["peak_mem_estimate"]) + + +class TestMainDispatch(unittest.TestCase): + """Argparse routes subcommands correctly.""" + + def test_unknown_subcommand_exits(self): + with self.assertRaises(SystemExit): + nb.main(["nope"]) + + def test_list_dispatches(self): + # argparse print_help writes "usage: ..." on stdout before our + # cmd_list handler runs when an unknown option trips argparse; + # for the `list` subcommand alone, main should return 0. + with mock.patch.object(sys, "stdout", io.StringIO()), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = nb.main(["list"]) + self.assertEqual(rc, 0) + + def test_list_emits_cpu_tight_workload(self): + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + nb.main(["list"]) + out = buf.getvalue() + self.assertIn("available workloads", out) + self.assertIn("cpu_tight", out) + + def test_run_unknown_workload_dies(self): + with mock.patch.object(sys, "stderr", io.StringIO()), \ + self.assertRaises(SystemExit) as cm: + nb.main(["run", "does-not-exist"]) + self.assertEqual(cm.exception.code, 2) + + +class TestConfigBlock(unittest.TestCase): + """Locks the schema of the metrics.json 'config' sub-record.""" + + def test_keys_present(self): + ns = mock.MagicMock() + ns.timeout = 60 + ns.mem = "256M" + ns.pids = 64 + cfg = nb._config_block(ns) + self.assertEqual(cfg["per_workload_timeout_seconds"], 60) + self.assertEqual(cfg["mem_cap"], "256M") + self.assertEqual(cfg["pids_cap"], 64) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_container.py b/tests/test_container.py index d3b9109..f466f68 100644 --- a/tests/test_container.py +++ b/tests/test_container.py @@ -8,12 +8,17 @@ same way the rest of this repo skips checks that need a full host. """ +import io +import json as _json import os +import shutil import subprocess import sys +import tempfile import unittest import importlib.util from importlib.machinery import SourceFileLoader +from unittest.mock import patch, MagicMock CONTAINER_PATH = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", @@ -30,11 +35,24 @@ def load_neuros_container(): def cgroups_available(): + """Stronger probe than just "make_cgroup returns a path": also + read back the limit file we set to confirm the write actually took + effect. A bare-EACCES mkdir leaves the leaf visible but the + limit files never get populated, which silently produces + containers that ignore `--mem`. Detect that here so the detach + integration test isn't misclassified as runnable.""" try: nc = load_neuros_container() cg = nc.make_cgroup("neuros-selftest-probe", "16777216", None, None) - os.rmdir(cg) - return True + # Read back and confirm the controller write was real. + with open(os.path.join(cg, "memory.max")) as f: + content = f.read().strip() + ok = content == "16777216" + try: + os.rmdir(cg) + except OSError: + pass + return ok except OSError: return False @@ -66,11 +84,722 @@ def test_rejects_garbage(self): self.nc.parse_size("not-a-size") +class TestCpuQuotaParsing(unittest.TestCase): + def setUp(self): + self.nc = load_neuros_container() + + def test_slash_separator(self): + self.assertEqual(self.nc.parse_cpu_quota("50000/100000"), ("50000", "100000")) + + def test_space_separator(self): + self.assertEqual(self.nc.parse_cpu_quota("25000 100000"), ("25000", "100000")) + + def test_max_literal(self): + self.assertEqual(self.nc.parse_cpu_quota("max"), ("max", "max")) + + def test_none_passthrough(self): + self.assertIsNone(self.nc.parse_cpu_quota(None)) + + def test_rejects_garbage(self): + with self.assertRaises(SystemExit): + self.nc.parse_cpu_quota("not-a-quota") + + def test_rejects_zero_period(self): + """period_us must be > 0; cpu.max rejects zero periods.""" + with self.assertRaises(SystemExit): + self.nc.parse_cpu_quota("50000/0") + + def test_rejects_negative_max(self): + with self.assertRaises(SystemExit): + self.nc.parse_cpu_quota("-1/100000") + + def test_rejects_one_part(self): + with self.assertRaises(SystemExit): + self.nc.parse_cpu_quota("50000") + + +class TestUserParsing(unittest.TestCase): + def setUp(self): + self.nc = load_neuros_container() + + def test_uid_only_collapses_to_same_gid(self): + self.assertEqual(self.nc.parse_user("1000"), (1000, 1000)) + + def test_uid_gid_pair(self): + self.assertEqual(self.nc.parse_user("1000:1001"), (1000, 1001)) + + def test_root(self): + self.assertEqual(self.nc.parse_user("0:0"), (0, 0)) + + def test_none_passthrough(self): + self.assertIsNone(self.nc.parse_user(None)) + + def test_rejects_garbage(self): + with self.assertRaises(SystemExit): + self.nc.parse_user("not-a-user") + + def test_rejects_name_style(self): + with self.assertRaises(SystemExit): + self.nc.parse_user("alice") + + +class TestEnterNamespacesBitmask(unittest.TestCase): + """The 6-tuple contract from enter_namespaces() must report the + right combination of True/False for each feasibility path. + + Implementation note: ``load_neuros_container()`` returns a fresh + ``module`` object on each call (``importlib.util.module_from_spec`` + is not cached when invoked outside of an actual ``import`` statement), + so decorator-level ``@patch.object(load_neuros_container(), ...)`` + targets a module that ``self.nc`` no longer points at. Patching + ``self.nc`` directly inside each test sidesteps that.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_root_with_net_gets_full_mask(self): + with patch("os.geteuid", return_value=0), \ + patch.object(self.nc, "try_unprivileged_userns", + return_value=True), \ + patch("os.unshare"): + ns = self.nc.enter_namespaces(net=True, user=None) + self.assertEqual(ns, (True, True, True, True, True, False)) + + def test_root_without_net_has_net_false(self): + with patch("os.geteuid", return_value=0), \ + patch("os.unshare"): + ns = self.nc.enter_namespaces(net=False, user=None) + self.assertEqual(ns, (True, True, True, True, False, False)) + + def test_no_privs_yields_all_false(self): + with patch("os.geteuid", return_value=1000), \ + patch.object(self.nc, "try_unprivileged_userns", + return_value=False): + ns = self.nc.enter_namespaces(net=False, user=None) + self.assertEqual(ns, (False, False, False, False, False, False)) + + def test_unprivileged_with_userns_gets_full_mask(self): + with patch("os.geteuid", return_value=1000), \ + patch.object(self.nc, "try_unprivileged_userns", + return_value=True), \ + patch("os.unshare"): + ns = self.nc.enter_namespaces(net=False, user=(1000, 1000)) + self.assertEqual(ns, (True, True, True, True, False, True)) + + def test_unshare_failure_returns_all_false(self): + with patch("os.geteuid", return_value=0), \ + patch("os.unshare", side_effect=OSError("eperm")): + ns = self.nc.enter_namespaces(net=False, user=None) + self.assertEqual(ns, (False, False, False, False, False, False)) + + +class TestListRecursion(unittest.TestCase): + """list_cgroups descends through non-neuros- intermediate + directories; the integration tests use the live cgroup tree under + the delegating ancestor, and we exercise that on a synthetic tree + here.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_finds_nested_leaves_via_synthetic_tree(self): + """Build a tiny tree under a tmpdir and verify the recursive + walker reaches ``neuros-*`` leaves at multiple depths.""" + root = tempfile.mkdtemp() + try: + deep = os.path.join(root, "intermediate", "neuros-deep-1") + os.makedirs(deep) + # A procs file is what the walker reads; empty means + # the leaf claims to have 0 members, which is fine here. + open(os.path.join(deep, "cgroup.procs"), "w").close() + rows = self.nc._list_neuros_leaves(root) + names = [r["name"] for r in rows] + self.assertIn("neuros-deep-1", names) + # If two leaves sit at different depths, both are picked up. + sibling = os.path.join(root, "neuros-shallow") + os.makedirs(sibling) + open(os.path.join(sibling, "cgroup.procs"), "w").close() + names = [r["name"] for r in self.nc._list_neuros_leaves(root)] + self.assertEqual(set(names), {"neuros-deep-1", "neuros-shallow"}) + finally: + shutil.rmtree(root, ignore_errors=True) + + def test_json_output_is_valid_json_array(self): + """list_cgroups(--json) must produce a parseable JSON array, + empty or populated, on stdout. Patch own_cgroup_path to an + empty tmpdir so no live cgroup tree is required.""" + root = tempfile.mkdtemp() + try: + with patch.object(self.nc, "own_cgroup_path", + return_value=root), \ + patch("sys.stdout", new_callable=io.StringIO) as buf: + self.nc.list_cgroups(json_output=True) + payload = _json.loads(buf.getvalue().strip()) + self.assertIsInstance(payload, list) + finally: + shutil.rmtree(root, ignore_errors=True) + + def test_human_output_reports_zero_processes(self): + """list_cgroups() with no JSON flag on an empty tree prints a + single human-readable line instead of the JSON array.""" + root = tempfile.mkdtemp() + try: + with patch.object(self.nc, "own_cgroup_path", + return_value=root), \ + patch("sys.stdout", new_callable=io.StringIO) as buf: + self.nc.list_cgroups(json_output=False) + self.assertIn("no active neuros-container cgroups", buf.getvalue()) + finally: + shutil.rmtree(root, ignore_errors=True) + + +class TestCpuQuotaMaxRoundTrip(unittest.TestCase): + """parse_cpu_quota('max') must round-trip through make_cgroup so + that ``cpu.max = 'max max'`` is written and recoverable.""" + + def setUp(self): + self.nc = load_neuros_container() + + @unittest.skipUnless( + __import__("os").path.isdir("/sys/fs/cgroup"), + "needs a cgroup v2 mount", + ) + def test_max_writes_max_max_to_cpu_max(self): + cg = self.nc.make_cgroup("neuros-selftest-max", None, None, None, + cpu_quota=("max", "max")) + try: + with open(os.path.join(cg, "cpu.max")) as f: + value = f.read().strip() + # Treat "max max" as the accepted unlimited form. + self.assertIn(value, ("max max", "max 100000")) + finally: + try: + os.rmdir(cg) + except OSError: + pass + + +class TestCleanupDetached(unittest.TestCase): + """Unit coverage for ``cleanup_container`` / ``cleanup_all`` — + drives them against a synthetic state file and a fake (empty) + cgroup path so the test doesn't need a real detached run, root, + or a delegated cgroup tree.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_cleanup_removes_empty_cgroup_and_state(self): + root = tempfile.mkdtemp() + try: + cg_dir = os.path.join(root, "neuros-cgood") + os.makedirs(cg_dir) + # The "empty cgroup" path means cleanup_container's probe + # for cgroup.procs falls through the OSError branch (which + # is what an actually-empty leaf looks like on most + # hosts). Don't write a procs file: rmdir requires the + # directory to be fully empty, including any leftover + # cgroup.procs the kernel might not have created. + with patch.object(self.nc, "STATE_DIR", root): + state_path = os.path.join(self.nc.STATE_DIR, + "neuros-cgood.json") + with open(state_path, "w") as f: + _json.dump({"name": "neuros-cgood", + "pid": 99999, + "cgroup": cg_dir}, f) + rc = self.nc.cleanup_container("neuros-cgood") + self.assertEqual(rc, 0) + self.assertFalse(os.path.exists(cg_dir)) + self.assertFalse(os.path.exists(state_path)) + finally: + shutil.rmtree(root, ignore_errors=True) + + def test_cleanup_refuses_when_cgroup_still_has_procs(self): + root = tempfile.mkdtemp() + try: + cg_dir = os.path.join(root, "neuros-cbusy") + os.makedirs(cg_dir) + with open(os.path.join(cg_dir, "cgroup.procs"), "w") as f: + f.write("1234\n5678\n") # two fake live members + state_path = os.path.join(root, "neuros-cbusy.json") + with open(state_path, "w") as f: + _json.dump({"name": "neuros-cbusy", + "pid": 1234, + "cgroup": cg_dir}, f) + with patch.object(self.nc, "STATE_DIR", root): + rc = self.nc.cleanup_container("neuros-cbusy") + self.assertEqual(rc, 1) + self.assertTrue(os.path.exists(cg_dir)) + self.assertTrue(os.path.exists(state_path)) + finally: + shutil.rmtree(root, ignore_errors=True) + + def test_cleanup_missing_state_exits_nonzero(self): + empty = tempfile.mkdtemp() + try: + with patch.object(self.nc, "STATE_DIR", empty): + rc = self.nc.cleanup_container("neuros-nope") + self.assertEqual(rc, 1) + finally: + shutil.rmtree(empty, ignore_errors=True) + + def test_cleanup_all_aggregates_exit_code(self): + root = tempfile.mkdtemp() + try: + # ``a`` is an empty cgroup (no procs file at all -> cleanup + # proceeds to rmdir and succeeds). ``b`` has a populated + # cgroup.procs -> cleanup refuses. cleanup_all should OR + # the exit codes and fail overall. + cg_a = os.path.join(root, "neuros-a") + os.makedirs(cg_a) + with open(os.path.join(root, "neuros-a.json"), "w") as f: + _json.dump({"name": "neuros-a", + "pid": 1, "cgroup": cg_a}, f) + cg_b = os.path.join(root, "neuros-b") + os.makedirs(cg_b) + with open(os.path.join(cg_b, "cgroup.procs"), "w") as f: + f.write("1234\n5678\n") # two fake live members + with open(os.path.join(root, "neuros-b.json"), "w") as f: + _json.dump({"name": "neuros-b", + "pid": 1234, "cgroup": cg_b}, f) + with patch.object(self.nc, "STATE_DIR", root): + rc = self.nc.cleanup_all() + # "a" is cleaned (its cgroup dir is gone), "b" remains. + self.assertEqual(rc, 1) + self.assertFalse(os.path.exists(os.path.join(root, "neuros-a"))) + self.assertTrue(os.path.exists(os.path.join(root, "neuros-b"))) + finally: + shutil.rmtree(root, ignore_errors=True) + + +class TestRemountReadonlySurface(unittest.TestCase): + """The --read-only path used to drop the mount(8) returncode on + the floor; it now warns on non-zero exit.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_surfaces_warning_on_nonzero_returncode(self): + with patch("os.system", return_value=32), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.remount_readonly_best_effort() + self.assertIn("exited with status 32", err.getvalue()) + + def test_silent_on_zero_returncode(self): + with patch("os.system", return_value=0), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.remount_readonly_best_effort() + self.assertEqual(err.getvalue(), "") + + +class TestUserMismatchHint(unittest.TestCase): + """``--user UID:GID`` requested with a target that doesn't match + the caller's real uid should print a hint at the top of the + userns setup so the user understands the upcoming fall-back.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_prints_hint_when_user_differs_from_caller(self): + # Caller uid=1000 tries to map 2000:2000. ``try_unprivileged_userns`` + # is mocked to return False so we hit the “no userns" path. + with patch("os.geteuid", return_value=1000), \ + patch("os.getuid", return_value=1000), \ + patch("os.getgid", return_value=1000), \ + patch.object(self.nc, "try_unprivileged_userns", + return_value=False), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.enter_namespaces(net=False, user=(2000, 2000)) + self.assertIn("--user 2000:2000 was requested", err.getvalue()) + + def test_no_hint_when_user_matches_caller(self): + with patch("os.geteuid", return_value=1000), \ + patch("os.getuid", return_value=1000), \ + patch("os.getgid", return_value=1000), \ + patch.object(self.nc, "try_unprivileged_userns", + return_value=False), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.enter_namespaces(net=False, user=(1000, 1000)) + self.assertNotIn("--user ... was requested", err.getvalue()) + + def test_no_hint_when_no_user_arg(self): + with patch("os.geteuid", return_value=1000), \ + patch.object(self.nc, "try_unprivileged_userns", + return_value=False), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.enter_namespaces(net=False, user=None) + self.assertNotIn("--user", err.getvalue()) + + +class TestEnvFileParsing(unittest.TestCase): + """``--env-from-file PATH`` reads a dotenv-style file, strips + blanks/comments/exports, and feeds the rest through ``parse_env``. + The 5 cases cover happy path, blank/comment skipping, ``export`` + prefix tolerance, ``None`` passthrough, and missing-file exit.""" + + def setUp(self): + self.nc = load_neuros_container() + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def _write(self, name, body): + path = os.path.join(self.tmp, name) + with open(path, "w") as f: + f.write(body) + return path + + def test_env_file_parses_key_value_pairs(self): + path = self._write("env", "FOO=bar\nBAZ=qux\n") + self.assertEqual(self.nc.parse_env_file(path), + ["FOO=bar", "BAZ=qux"]) + + def test_env_file_skips_blanks_and_comments(self): + path = self._write("env", "# header\n\nFOO=bar\n# trailing\n") + self.assertEqual(self.nc.parse_env_file(path), ["FOO=bar"]) + + def test_env_file_accepts_export_prefix(self): + path = self._write("env", "export FOO=bar\nBAZ=qux\n") + self.assertEqual(self.nc.parse_env_file(path), + ["FOO=bar", "BAZ=qux"]) + + def test_env_file_none_passthrough(self): + self.assertEqual(self.nc.parse_env_file(None), []) + + def test_env_file_missing_path_exits(self): + with self.assertRaises(SystemExit), \ + patch("sys.stderr", new_callable=io.StringIO): + self.nc.parse_env_file(os.path.join(self.tmp, "nope")) + + +class TestDetachIntegration(unittest.TestCase): + """Drive the detached-container lifecycle end-to-end without + depending on the kernel's cgroup delegation state at the moment + of the test. ``--detach`` writes a JSON state file via + ``write_state`` and leaves the cgroup in place; ``cleanup`` reaps + empty leaves. We exercise that contract against a tmpdir + "state dir" and a tmpdir "cgroup dir", so the assertions don't + depend on whether the kernel delegation path is actually open + right now (which can disagree between the probe and a subprocess + in some sandboxes). + + This test is synthetic-only. A real e2e form should live in + ``tests/test_container_integration.py`` once a CI with + kernel-cgroup delegation is available — that form should + subprocess-run ``neuros-container run --detach …`` end-to-end. + The synthetic form here is portable and doesn't depend on the + kernel's cgroup delegation state at the moment of the test. + """ + + def setUp(self): + self.nc = load_neuros_container() + + def test_full_lifecycle_write_state_then_cleanup(self): + root = tempfile.mkdtemp() + try: + state_dir = root + cg_dir = os.path.join(root, "neuros-detach") + os.makedirs(cg_dir) + name = "neuros-detach-selftest" + + # Phase 1: write the state file via the helper the + # detached child would have written. + state_path = os.path.join(state_dir, f"{name}.json") + self.nc.write_state(state_path=state_path, name=name, + inner_pid=99999, cg=cg_dir) + self.assertTrue(os.path.exists(state_path)) + + # Phase 2: cleanup_container should reap the empty + # cgroup and unlink the state file. + with patch.object(self.nc, "STATE_DIR", state_dir): + rc = self.nc.cleanup_container(name) + self.assertEqual(rc, 0) + self.assertFalse(os.path.exists(cg_dir)) + self.assertFalse(os.path.exists(state_path)) + finally: + shutil.rmtree(root, ignore_errors=True) + + def test_state_shape_matches_documented_contract(self): + """The state JSON must include name/pid/cgroup so a reattach + tool can find the right processes.""" + root = tempfile.mkdtemp() + try: + # No real cgroup.procs on this synthetic path is the + # important invariant: cleanup_container treats a missing + # cgroup.procs as "zero live members" via its OSError + # branch, so the test relies on that branch silently + # succeeding. if a future reader adds a stray procs file + # here, full_lifecycle_write_state_then_cleanup will start + # failing with "still has N processes" — by design. + name = "neuros-shape-test" + path = os.path.join(root, f"{name}.json") + self.nc.write_state(state_path=path, name=name, + inner_pid=12345, + cg="/sys/fs/cgroup/neuros-x") + with open(path) as f: + state = _json.load(f) + self.assertEqual(set(state.keys()), + {"name", "pid", "cgroup"}) + self.assertEqual(state["name"], name) + self.assertEqual(state["pid"], 12345) + self.assertEqual(state["cgroup"], "/sys/fs/cgroup/neuros-x") + finally: + shutil.rmtree(root, ignore_errors=True) + + +class TestUtilitySmoke(unittest.TestCase): + """Five high-traffic ``neuros-*`` utilities that previously had + zero coverage. The assertions are deliberately structural so they + survive across schema changes in the tools themselves but still + catch a broken script (missing shebang, syntax error, ``--help`` + not behaving like argparse).""" + + UTILITIES = ( + "neuros-firewall", + "neuros-network", + "neuros-backup", + "neuros-monitor", + "neuros-cron", + ) + BIN_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", + "config", "includes.chroot", "usr", "local", "bin", + ) + + def _path(self, name): + full = os.path.join(self.BIN_DIR, name) + if not os.path.exists(full): + self.skipTest(f"{name} not present on disk") + return full + + def test_each_utility_has_python3_shebang(self): + for name in self.UTILITIES: + with open(self._path(name)) as f: + first = f.readline() + self.assertIn("python3", first, + msg=f"{name} missing python3 shebang") + + def _load(self, name): + loader = SourceFileLoader(name, self._path(name)) + spec = importlib.util.spec_from_loader(name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + def test_each_utility_imports_without_syntax_error(self): + for name in self.UTILITIES: + module = self._load(name) + self.assertIsNotNone(module) + + def test_each_utility_reports_help(self): + """Argparse-based CLIs print help on ``--help`` and exit 0. + A tool that crashes on ``--help`` means a new opt was added + without a test, and live callers will run into the same + regression on first invocation. The assertion is strict: + argparse prints ``usage:`` to stdout, so we require it.""" + for name in self.UTILITIES: + proc = subprocess.run( + [sys.executable, self._path(name), "--help"], + capture_output=True, text=True, timeout=10, + ) + combined = proc.stdout + proc.stderr + self.assertIn( + "usage:", combined, + msg=f"{name} --help did not print argparse usage " + f"(rc={proc.returncode}, output={combined!r})", + ) + + +class TestEnvAndCapDropParsing(unittest.TestCase): + """``--env`` and ``--cap-drop`` round-trip through their parsers + without touching the kernel.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_env_accepts_well_formed_pairs(self): + out = self.nc.parse_env(["FOO=bar", "BAZ=qux quux"]) + self.assertEqual(out, ["FOO=bar", "BAZ=qux quux"]) + + def test_env_empty_returns_empty(self): + self.assertEqual(self.nc.parse_env(None), []) + self.assertEqual(self.nc.parse_env([]), []) + + def test_env_rejects_missing_equals(self): + with self.assertRaises(SystemExit): + self.nc.parse_env(["NO_EQUALS"]) + + def test_env_rejects_blank_key(self): + with self.assertRaises(SystemExit): + self.nc.parse_env(["=value"]) + + def test_env_rejects_nul_bytes(self): + with self.assertRaises(SystemExit), \ + patch("sys.stderr", new_callable=io.StringIO): + self.nc.parse_env(["K=\x00v"]) + + def test_cap_drop_returns_none_when_unset(self): + self.assertIsNone(self.nc.parse_cap_drop(None)) + + def test_cap_drop_compiles_regex(self): + pattern = self.nc.parse_cap_drop("CAP_(NET_RAW|SYS_ADMIN)") + self.assertTrue(pattern.search("CAP_NET_RAW")) + self.assertTrue(pattern.search("CAP_SYS_ADMIN")) + self.assertFalse(pattern.search("CAP_CHOWN")) + + def test_cap_drop_rejects_invalid_regex(self): + with self.assertRaises(SystemExit): + self.nc.parse_cap_drop("(unclosed") + + +class TestEnvInjection(unittest.TestCase): + """The grandchild of run_container inserts --env entries into + ``os.environ`` just before exec. We don't fork a real process; + instead, we directly call the same injection block against the + real ``os.environ`` so the assertion is on what an execvp would + inherit.""" + + def setUp(self): + self.nc = load_neuros_container() + self.old_environ = dict(os.environ) + + def tearDown(self): + os.environ.clear() + os.environ.update(self.old_environ) + + def test_env_entries_override_inherited(self): + # Mirror exactly what run_container does after the parse step. + env = self.nc.parse_env(["NEUROS_TEST_K=v1"]) + os.environ["NEUROS_TEST_K"] = "inherited" + for entry in env: + k, _, v = entry.partition("=") + os.environ[k] = v + self.assertEqual(os.environ["NEUROS_TEST_K"], "v1") + + +class TestDropCapabilitiesHelper(unittest.TestCase): + """``drop_capabilities_best_effort`` walks the running kernel's + cap list and prctl(PR_CAPBSET_DROP)'s every match. We stub + ctypes.CDLL so no real prctl syscall runs; the helper is exercised + end-to-end against mock libcs that return either success or + EPERM-set errno.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_silent_when_pattern_is_none(self): + with patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.drop_capabilities_best_effort(None) + self.assertEqual(err.getvalue(), "") + + def test_no_match_warns_loudly(self): + """Pattern matches nothing — the helper must surface a clear + stderr line, otherwise the user thinks their regex dropped + caps when it didn't.""" + libc = MagicMock() + libc.prctl = MagicMock(return_value=0) + pattern = self.nc.parse_cap_drop("CAP_BOGUS_DISABLE_THIS") + with patch.object(self.nc, "_CAP_NAMES_0_31", + ["CHOWN", "DAC_OVERRIDE"]), \ + patch("ctypes.CDLL", return_value=libc), \ + patch("ctypes.get_errno", return_value=0), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.drop_capabilities_best_effort(pattern) + self.assertIn("matched 0 of", err.getvalue()) + + def test_libc_load_failure_warns(self): + libc = MagicMock() + libc.prctl = MagicMock(return_value=0) + pattern = self.nc.parse_cap_drop("CAP_NET_RAW") + with patch("ctypes.CDLL", side_effect=OSError("no libc")), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.drop_capabilities_best_effort(pattern) + self.assertIn("libc", err.getvalue()) + + def test_prctl_eperm_warns_per_cap(self): + """A failure on one specific cap reports the errno per-cap so + users can tell which cap couldn't be dropped (often CAP_SETPCAP + on a userns without the right capability).""" + libc = MagicMock() + # Fail when prctl is called for PR_CAPBSET_DROP on cap 0 + # (CAP_CHOWN); any other cap succeeds. + def fake_prctl(op, capidx, *_): + if op == 23 and capidx == 0: # PR_CAPBSET_DROP=23 + return -1 + return 0 + libc.prctl = fake_prctl + pattern = self.nc.parse_cap_drop("CAP_CHOWN") + with patch("ctypes.CDLL", return_value=libc), \ + patch("ctypes.get_errno", return_value=1), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.drop_capabilities_best_effort(pattern) + text = err.getvalue() + self.assertIn("CAP_CHOWN", text) + self.assertIn("errno=1", text) + self.assertIn("CAP_SETPCAP", text) + + +class TestZeroControllersWarning(unittest.TestCase): + """If a limit is requested but no controller is delegated all the + way down, make_cgroup should print a stderr warning telling the + user their --mem/--pids/--cpu did nothing.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_warning_on_zero_ready_controllers(self): + # Force enable_controllers to return an empty set, then make + # sure the warning surfaces. The cgroup is still created in a + # tmpdir so the rmdir at the end works. We also patch + # find_delegating_ancestor — the walker otherwise climbs all + # the way up to ``/`` looking for a delegating ancestor and + # returns it, makedirs-ing at the filesystem root. + root = tempfile.mkdtemp() + try: + with patch.object(self.nc, "find_delegating_ancestor", + return_value=root), \ + patch.object(self.nc, "enable_controllers", + return_value=set()), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.make_cgroup("neuros-zero", "1000", None, None) + self.assertIn("WARNING", err.getvalue()) + finally: + shutil.rmtree(root, ignore_errors=True) + + @unittest.skipUnless(cgroups_available(), "cgroup v2 delegation not available in this sandbox") class TestCgroupEnforcement(unittest.TestCase): def setUp(self): self.nc = load_neuros_container() + def test_cpu_max_quota_writes_through(self): + """make_cgroup(..., cpu_quota=("50000","100000")) must produce a + cgroup whose cpu.max file is exactly '50000 100000'. Tests the + new --cpu-quota code path without needing to schedule a CPU- + bound workload.""" + cg = self.nc.make_cgroup("neuros-selftest-cpuq", None, None, None, + cpu_quota=("50000", "100000")) + try: + with open(os.path.join(cg, "cpu.max")) as f: + self.assertEqual(f.read().strip(), "50000 100000") + finally: + try: + os.rmdir(cg) + except OSError: + pass + + def test_cpu_weight_still_writes_cpu_weight(self): + """The previous weight-only path stays backwards compatible.""" + cg = self.nc.make_cgroup("neuros-selftest-cpuw", None, None, 250) + try: + with open(os.path.join(cg, "cpu.weight")) as f: + self.assertEqual(f.read().strip(), "250") + finally: + try: + os.rmdir(cg) + except OSError: + pass + def test_memory_max_caps_actual_usage(self): """A process that tries to touch 200MB inside a 16M memory.max cgroup must not exceed that limit, per memory.current.""" diff --git a/tests/test_model.py b/tests/test_model.py index 268754d..e0bb744 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -48,9 +48,8 @@ def test_switch_creates_config_when_missing(self): with patch.object(self.m, "CONFIG_PATH", self.config_path), \ patch.object(self.m, "api_request", return_value=None): self.m.switch_model("llama3") - with open(self.config_path) as f: - content = f.read() - self.assertIn('model = "llama3"', content) + with patch.object(self.m, "CONFIG_PATH", self.config_path): + self.assertEqual(self.m.get_current_model(), "llama3") def test_switch_replaces_existing_model_line(self): with open(self.config_path, "w") as f: @@ -58,9 +57,10 @@ def test_switch_replaces_existing_model_line(self): with patch.object(self.m, "CONFIG_PATH", self.config_path), \ patch.object(self.m, "api_request", return_value=None): self.m.switch_model("codellama") + with patch.object(self.m, "CONFIG_PATH", self.config_path): + self.assertEqual(self.m.get_current_model(), "codellama") with open(self.config_path) as f: content = f.read() - self.assertIn('model = "codellama"', content) self.assertNotIn("mistral", content) self.assertIn("context_window = 4096", content) diff --git a/tests/test_neuroslib.py b/tests/test_neuroslib.py new file mode 100644 index 0000000..29f6968 --- /dev/null +++ b/tests/test_neuroslib.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +""" +test_neuroslib.py — Unit tests for the shared neuroslib module. + +Covers section-aware config parsing (replacing the per-tool +key=value loops in nn and neuros-model), JSON DB atomic writes, and +the canonical helpers get_default_model/get_context_config/ +set_default_model. Runs in a tempdir so it doesn't touch the user's +real ~/.config/neuros. +""" + +import importlib.util +import json +import os +import sys +import tempfile +import unittest +from importlib.machinery import SourceFileLoader +from unittest.mock import patch + +LIB_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", + "config", "includes.chroot", "usr", "local", "bin", "neuroslib.py", +) + + +def load_lib(): + loader = SourceFileLoader("neuroslib", LIB_PATH) + spec = importlib.util.spec_from_loader("neuroslib", loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +class TestLoadConfig(unittest.TestCase): + """The new configparser-backed loader must understand [section] + headers, comments, and quote stripping that the prior naive loop + couldn't handle cleanly.""" + + def setUp(self): + self.lib = load_lib() + self.tmp = tempfile.mkdtemp() + + def test_returns_defaults_for_missing_file(self): + cfg = self.lib.load_config(os.path.join(self.tmp, "missing.conf")) + self.assertEqual(cfg["llm.model"], "mistral") + self.assertEqual(cfg["llm.host"], "localhost") + self.assertEqual(cfg["llm.port"], "11434") + self.assertFalse(cfg["context.window_title"]) + self.assertFalse(cfg["context.clipboard"]) + self.assertFalse(cfg["context.recent_files"]) + + def test_parses_flat_llm_section(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = "codellama"\nhost = example\nport = 9999\n') + cfg = self.lib.load_config(path) + self.assertEqual(cfg["llm.model"], "codellama") + self.assertEqual(cfg["llm.host"], "example") + self.assertEqual(cfg["llm.port"], "9999") + + def test_parses_context_section_with_truthy(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[context]\nwindow_title = true\nclipboard = 1\nrecent_files = yes\n') + cfg = self.lib.load_config(path) + self.assertTrue(cfg["context.window_title"]) + self.assertTrue(cfg["context.clipboard"]) + self.assertTrue(cfg["context.recent_files"]) + + def test_unknown_truthy_strings_default_off(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[context]\nwindow_title = maybe\n') + cfg = self.lib.load_config(path) + self.assertFalse(cfg["context.window_title"]) + + def test_full_line_comments_are_ignored(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('# this is a comment\n[llm]\n# another\nmodel = "mistral"\n') + cfg = self.lib.load_config(path) + self.assertEqual(cfg["llm.model"], "mistral") + + def test_interpolation_in_value_is_not_resolved(self): + """Model names with '=' in them (e.g. custom tags like + 'code=base') must round-trip verbatim, which configparser + interpolation would otherwise mangle.""" + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = "custom=tag"\n') + cfg = self.lib.load_config(path) + self.assertEqual(cfg["llm.model"], "custom=tag") + + +class TestGetDefaultModel(unittest.TestCase): + def setUp(self): + self.lib = load_lib() + self.tmp = tempfile.mkdtemp() + + def test_returns_fallback_on_missing_config(self): + path = os.path.join(self.tmp, "nope.conf") + self.assertEqual(self.lib.get_default_model(path), "mistral") + + def test_strips_quotes_in_value(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = "qwen2.5:7b"\n') + self.assertEqual(self.lib.get_default_model(path), "qwen2.5:7b") + + def test_blank_model_falls_back(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = " "\n') + self.assertEqual(self.lib.get_default_model(path), "mistral") + + +class TestGetContextConfig(unittest.TestCase): + def setUp(self): + self.lib = load_lib() + self.tmp = tempfile.mkdtemp() + + def test_all_off_when_file_missing(self): + cfg = self.lib.get_context_config(os.path.join(self.tmp, "nope.conf")) + self.assertEqual(cfg, { + "window_title": False, "clipboard": False, "recent_files": False, + }) + + def test_only_explicit_sources_enabled(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[context]\nwindow_title = true\n') + cfg = self.lib.get_context_config(path) + self.assertTrue(cfg["window_title"]) + self.assertFalse(cfg["clipboard"]) + self.assertFalse(cfg["recent_files"]) + + +class TestSetDefaultModel(unittest.TestCase): + def setUp(self): + self.lib = load_lib() + self.tmp = tempfile.mkdtemp() + + def test_creates_file_and_directory(self): + path = os.path.join(self.tmp, "deep", "dir", "llm.conf") + self.lib.set_default_model("llama3", path) + self.assertTrue(os.path.exists(path)) + with open(path) as f: + data = f.read() + self.assertIn('model = llama3', data) + + def test_preserves_existing_keys(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = "mistral"\ncontext_window = 4096\n') + self.lib.set_default_model("codellama", path) + with open(path) as f: + data = f.read() + self.assertIn('model = codellama', data) + self.assertIn('context_window = 4096', data) + self.assertNotIn('mistral', data) + + def test_preserves_context_section(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = "mistral"\n[context]\nclipboard = true\n') + self.lib.set_default_model("qwen", path) + with open(path) as f: + data = f.read() + self.assertIn("[context]", data) + self.assertIn("clipboard = true", data) + self.assertIn('model = qwen', data) + + def test_overwrites_model_only(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = "old"\ncontext_window = 8192\n' + '[context]\nwindow_title = true\n') + self.lib.set_default_model("new", path) + with open(path) as f: + data = f.read() + # Host/port are NOT touched, so the configurable values survive. + self.assertIn("context_window = 8192", data) + self.assertIn("window_title = true", data) + + +class TestLoadDbSaveDb(unittest.TestCase): + def setUp(self): + self.lib = load_lib() + self.tmp = tempfile.mkdtemp() + + def test_save_then_load_round_trips(self): + path = os.path.join(self.tmp, "state.json") + self.lib.save_db(path, {"a": 1, "b": [2, 3, 4]}) + self.assertEqual(self.lib.load_db(path), {"a": 1, "b": [2, 3, 4]}) + + def test_load_db_returns_default_when_missing(self): + self.assertEqual( + self.lib.load_db(os.path.join(self.tmp, "nope.json"), + default={"x": 9}), + {"x": 9}, + ) + + def test_save_db_writes_pretty_indented_json(self): + path = os.path.join(self.tmp, "pretty.json") + self.lib.save_db(path, {"k": "v"}) + with open(path) as f: + text = f.read() + self.assertIn("\n ", text) # two-space indent present + j = json.loads(text) + self.assertEqual(j, {"k": "v"}) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_nn.py b/tests/test_nn.py index 5f86daf..9183c8e 100644 --- a/tests/test_nn.py +++ b/tests/test_nn.py @@ -35,31 +35,33 @@ def load_nn(): return module class TestNNCore(unittest.TestCase): - """Test core nn CLI functionality.""" + """Test core nn CLI functionality. The previous implementation + hard-coded ``localhost:11434`` and ``~/.config/neuros/llm.conf`` as + source-level literals; the new nn derives them from the shared + neuroslib constants, so these tests assert that the resolved + values at runtime still point at the expected host/path.""" - def test_config_path_in_source(self): - """Test that nn source contains CONFIG_PATH with correct path.""" + def test_resolved_config_path_points_at_neuros_llm_conf(self): nn_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), '..', 'config', 'includes.chroot', 'usr', 'local', 'bin', 'nn' ) - with open(nn_path) as f: - content = f.read() - self.assertIn(".config/neuros/llm.conf", content) - self.assertIn("CONFIG_PATH", content) + loader = SourceFileLoader("nn_module", nn_path) + spec = importlib.util.spec_from_loader("nn_module", loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + self.assertTrue(module.CONFIG_PATH.endswith(".config/neuros/llm.conf")) - def test_ollama_url_correct(self): - """Test that OLLAMA_URL points to correct localhost port.""" - # Read the nn file and find OLLAMA_URL + def test_resolved_ollama_url_points_at_localhost_11434(self): nn_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), '..', 'config', 'includes.chroot', 'usr', 'local', 'bin', 'nn' ) - with open(nn_path) as f: - content = f.read() - - self.assertIn("localhost:11434", content) - self.assertIn("OLLAMA_URL", content) + loader = SourceFileLoader("nn_module", nn_path) + spec = importlib.util.spec_from_loader("nn_module", loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + self.assertIn("localhost:11434", module.OLLAMA_URL) def test_default_model_is_mistral(self): """Test that default model is mistral.""" diff --git a/tests/test_policy.py b/tests/test_policy.py new file mode 100755 index 0000000..5cc34f1 --- /dev/null +++ b/tests/test_policy.py @@ -0,0 +1,398 @@ +"""Tests for neuros-policy (loader pattern mirrors test_sandbox.py). + +Runs purely against the production script via compile()+exec() so we +do not need it on $PATH. No subprocess enforcement is exercised here +on purpose: every interesting invariant (bounds, regex shape, profile +emission, envelope reconciliation) is reachable from Python directly. +""" +import compileall +import io +import json +import os +import subprocess +import sys +import tempfile +import types +import unittest +from unittest import mock + +POLICY_PATH = "config/includes.chroot/usr/local/bin/neuros-policy" + + +def _load_module(): + with open(POLICY_PATH, "r", encoding="utf-8") as f: + src = f.read() + # Strip the shebang line for exec(); everything else is plain + # Python 3 and re-exports cleanly when given a real module + # namespace (so `cls.pl.validate_policy` style attribute access + # works; a plain dict would fail with AttributeError on lookup). + if src.startswith("#!"): + src = src.split("\n", 1)[1] + code = compile(src, POLICY_PATH, "exec") + mod = types.ModuleType("neuros_policy_under_test") + exec(code, mod.__dict__) + return mod + + +class TestCompile(unittest.TestCase): + def test_compiles_clean(self): + # The wrapper imports no third-party modules, so py_compile + # is a sufficient static check before we even try exec(). + self.assertTrue(compileall.compile_file(POLICY_PATH, + quiet=1, + force=True)) + + +def _write_policy(d): + """Dump ``d`` to a temp .json file and return its path string + (not the wrapper, so the production loader can open() it).""" + fd, p = tempfile.mkstemp(prefix="neuros-policy-", suffix=".json") + with os.fdopen(fd, "w") as f: + json.dump(d, f) + return p + + +def _good_policy(**overrides): + base = { + "name": "neuros-default", + "version": "1.0.0", + "defaults": { + "mem": "256M", + "pids": 64, + "cpu_quota": 50000, + "timeout": 30, + }, + "profiles": { + "strict": ["CAP_NET_RAW", "CAP_SYS_ADMIN", "CAP_SYS_PTRACE"], + "moderate": ["CAP_NET_RAW", "CAP_SYS_ADMIN"], + "permissive": [], + }, + "net": "private", + "readonly": True, + "env_allowlist": ["PATH", "LANG"], + "syscalls": None, + } + base.update(overrides) + return base + + +class TestValidate(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.pl = _load_module() + + def test_clean_policy_passes(self): + path = _write_policy(_good_policy()) + try: + errs, cleaned = self.pl.validate_policy(_good_policy()) + self.assertEqual(errs, []) + self.assertEqual(cleaned["strict"], + sorted({"CAP_NET_RAW", "CAP_SYS_ADMIN", + "CAP_SYS_PTRACE"})) + finally: + os.unlink(path) + + def test_root_must_be_dict(self): + path = _write_policy(["not", "a", "dict"]) + try: + with self.assertRaises(self.pl.PolicyError) as cm: + self.pl._load_policy(path) + self.assertIn("JSON object", str(cm.exception)) + finally: + os.unlink(path) + + def test_missing_name_violates(self): + p = _good_policy(); del p["name"] + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("name" in str(e) for e in errs)) + + def test_blank_name_violates(self): + errs, _ = self.pl.validate_policy(_good_policy(name=" ")) + self.assertTrue(any("name" in str(e) for e in errs)) + + def test_bad_version_violates(self): + errs, _ = self.pl.validate_policy(_good_policy(version="v1")) + self.assertTrue(any("version" in str(e) for e in errs)) + + def test_mem_below_minimum_violates(self): + p = _good_policy() + p["defaults"]["mem"] = "8K" + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.mem" in str(e) for e in errs)) + + def test_mem_above_maximum_violates(self): + p = _good_policy() + p["defaults"]["mem"] = "2G" + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.mem" in str(e) for e in errs)) + + def test_mem_unparseable_violates(self): + p = _good_policy() + p["defaults"]["mem"] = "lots" + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.mem" in str(e) for e in errs)) + + def test_pids_out_of_range_violates(self): + p = _good_policy() + p["defaults"]["pids"] = 9999 + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.pids" in str(e) for e in errs)) + + def test_pids_zero_violates(self): + p = _good_policy() + p["defaults"]["pids"] = 0 + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.pids" in str(e) for e in errs)) + + def test_cpu_quota_out_of_range_violates(self): + p = _good_policy() + p["defaults"]["cpu_quota"] = 500 + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.cpu_quota" in str(e) for e in errs)) + + def test_timeout_too_long_violates(self): + p = _good_policy() + p["defaults"]["timeout"] = 7200 + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.timeout" in str(e) for e in errs)) + + def test_profile_name_uppercase_violates(self): + p = _good_policy() + p["profiles"]["Strict"] = [] + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("profile name" in str(e).lower() or + "Strict" in str(e) for e in errs)) + + def test_bad_cap_name_violates(self): + p = _good_policy() + p["profiles"]["strict"] = ["net_raw", "CAP_SYS_ADMIN"] + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("non-cap name" in str(e) for e in errs)) + + def test_empty_profile_is_allowed(self): + p = _good_policy() + p["profiles"]["permissive"] = [] + errs, _ = self.pl.validate_policy(p) + self.assertEqual(errs, []) + + def test_net_must_be_known(self): + p = _good_policy() + p["net"] = "loopback" + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("net" in str(e) for e in errs)) + + def test_bad_env_name_violates(self): + p = _good_policy() + p["env_allowlist"] = ["path", "LANG"] + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("env_allowlist" in str(e) for e in errs)) + + def test_readonly_must_be_bool(self): + p = _good_policy() + p["readonly"] = "yes" + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("readonly" in str(e) for e in errs)) + + def test_caps_deduped_and_sorted(self): + p = _good_policy() + p["profiles"]["strict"] = ["CAP_SYS_ADMIN", "CAP_NET_RAW", + "CAP_SYS_ADMIN"] + _, cleaned = self.pl.validate_policy(p) + self.assertEqual(cleaned["strict"], + ["CAP_NET_RAW", "CAP_SYS_ADMIN"]) + + +class TestTranspile(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.pl = _load_module() + + def test_strict_profile_builds_cap_regex(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + argv = self.pl.transpile_argv(p, "strict", cleaned) + # Default order is: --mem, --pids, --cpu-quota, --timeout, + # --cap-drop. Assert presence + cap regex shape. + self.assertIn("--cap-drop", argv) + cap_idx = argv.index("--cap-drop") + # Sorted alphabetically per validate_policy. + self.assertEqual(argv[cap_idx + 1], + "CAP_(CAP_NET_RAW|CAP_SYS_ADMIN|CAP_SYS_PTRACE)") + + def test_permissive_profile_emits_never_match(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + argv = self.pl.transpile_argv(p, "permissive", cleaned) + self.assertEqual(argv[argv.index("--cap-drop") + 1], + self.pl.NEVER_MATCH_CAP_REGEX) + + def test_unknown_profile_raises(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + with self.assertRaises(self.pl.PolicyError): + self.pl.transpile_argv(p, "unknown", cleaned) + + def test_no_profile_omits_cap_drop(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + argv = self.pl.transpile_argv(p, None, cleaned) + self.assertNotIn("--cap-drop", argv) + + +class TestCheck(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.pl = _load_module() + + def test_conformant_envelope_passes(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + env = { + "exit_code": 0, "stdout": "", "stderr": "", + "wall_clock_ms": 1500, "timeout_hit": False, + "peak_mem_estimate": 50 * 1024 * 1024, + } + violations = self.pl.check_envelope_against_policy(p, env, cleaned) + self.assertEqual(violations, []) + + def test_wall_clock_over_violates(self): + p = _good_policy() + p["defaults"]["timeout"] = 5 + _, cleaned = self.pl.validate_policy(p) + env = {"wall_clock_ms": 12000, "timeout_hit": False, + "peak_mem_estimate": None} + violations = self.pl.check_envelope_against_policy(p, env, cleaned) + self.assertTrue(any(v[0] == "wall_clock" for v in violations)) + + def test_peak_mem_over_violates(self): + p = _good_policy() + p["defaults"]["mem"] = "128M" + _, cleaned = self.pl.validate_policy(p) + env = {"wall_clock_ms": 1000, "timeout_hit": False, + "peak_mem_estimate": 200 * 1024 * 1024} + violations = self.pl.check_envelope_against_policy(p, env, cleaned) + self.assertTrue(any(v[0] == "peak_mem" for v in violations)) + + def test_timeout_hit_always_violates(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + env = {"wall_clock_ms": 1000, "timeout_hit": True} + violations = self.pl.check_envelope_against_policy(p, env, cleaned) + self.assertTrue(any(v[0] == "timeout_hit" for v in violations)) + + def test_malformed_envelope_violates(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + violations = self.pl.check_envelope_against_policy(p, "string", + cleaned) + self.assertTrue(any(v[0] == "malformed" for v in violations)) + + +class TestMainDispatch(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.pl = _load_module() + + def _ok_envelope(self): + return json.dumps({ + "exit_code": 0, "stdout": "", "stderr": "", + "wall_clock_ms": 1000, "timeout_hit": False, + "peak_mem_estimate": 1024 * 1024, + }) + + def test_validate_dispatches(self): + path = _write_policy(_good_policy()) + try: + with mock.patch.object(sys, "stdout", io.StringIO()): + rc = self.pl.main(["validate", path]) + self.assertEqual(rc, 0) + finally: + os.unlink(path) + + def test_validate_text_reports_violations(self): + bad = _good_policy() + bad["defaults"]["mem"] = "lots" + path = _write_policy(bad) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.pl.main(["validate", path]) + self.assertEqual(rc, 1) + self.assertIn("violation", buf.getvalue()) + finally: + os.unlink(path) + + def test_validate_json_envelope_shape(self): + path = _write_policy(_good_policy()) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.pl.main(["validate", "--json", path]) + self.assertEqual(rc, 0) + line = buf.getvalue().strip() + obj = json.loads(line) + self.assertTrue(obj["ok"]) + self.assertEqual(obj["errors"], []) + self.assertEqual(obj["name"], "neuros-default") + finally: + os.unlink(path) + + def test_check_passes(self): + p_path = _write_policy(_good_policy()) + e_path = _write_policy(json.loads(self._ok_envelope())) + try: + with mock.patch.object(sys, "stdout", io.StringIO()): + rc = self.pl.main(["check", p_path, "--envelope", e_path]) + self.assertEqual(rc, 0) + finally: + os.unlink(p_path); os.unlink(e_path) + + def test_check_fails_on_violation(self): + bad_env = json.loads(self._ok_envelope()) + bad_env["wall_clock_ms"] = 999_999 # exceeds 30s default + p_path = _write_policy(_good_policy()) + e_path = _write_policy(bad_env) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.pl.main(["check", p_path, "--envelope", e_path]) + self.assertEqual(rc, 1) + finally: + os.unlink(p_path); os.unlink(e_path) + + def test_transpile_emits_argv_text(self): + path = _write_policy(_good_policy()) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.pl.main(["transpile", path, + "--profile", "strict"]) + self.assertEqual(rc, 0) + txt = buf.getvalue().strip() + self.assertIn("--mem 256M", txt) + self.assertIn("--pids 64", txt) + self.assertIn("--cpu-quota 50000", txt) + self.assertIn("--cap-drop CAP_(", txt) + finally: + os.unlink(path) + + def test_transpile_refuses_invalid_policy(self): + bad = _good_policy() + bad["name"] = "" + path = _write_policy(bad) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.pl.main(["transpile", path]) + self.assertEqual(rc, 1) + finally: + os.unlink(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_replay.py b/tests/test_replay.py new file mode 100755 index 0000000..5d51527 --- /dev/null +++ b/tests/test_replay.py @@ -0,0 +1,435 @@ +"""Tests for neuros-replay (offline diagnostics over envelopes). + +Runs purely against the production script via compile()+exec() so we +do not need it on $PATH. No subprocess shell-out is exercised here on +purpose: every interesting invariant (bounds of pct_delta, missing +fields, tolerance overrides, JSON envelope shape) is reachable from +Python directly. The `extract` subcommand writes to stdout which we +capture via sys.stdout mocks. +""" +import compileall +import io +import json +import os +import subprocess +import sys +import tempfile +import types +import unittest +from unittest import mock + +REPLAY_PATH = "config/includes.chroot/usr/local/bin/neuros-replay" + + +def _load_module(): + with open(REPLAY_PATH, "r", encoding="utf-8") as f: + src = f.read() + if src.startswith("#!"): + src = src.split("\n", 1)[1] + code = compile(src, REPLAY_PATH, "exec") + mod = types.ModuleType("neuros_replay_under_test") + exec(code, mod.__dict__) + return mod + + +def _envelope(**overrides): + """A conformant sandbox envelope with all fields present. + Picked exit_code=0, wall_clock_ms=1000, peak_mem=64M so the + tests can mutate one field at a time without making the envelope + regress accidentally. + """ + base = { + "exit_code": 0, + "stdout": "", + "stderr": "", + "wall_clock_ms": 1000, + "timeout_hit": False, + "peak_mem_estimate": 64 * 1024 * 1024, + } + base.update(overrides) + return base + + +def _write_envelope(env): + """Dump ``env`` to a temp .json file and return its path string.""" + fd, p = tempfile.mkstemp(prefix="neuros-replay-", suffix=".json") + with os.fdopen(fd, "w") as f: + json.dump(env, f) + return p + + +class TestCompile(unittest.TestCase): + def test_compiles_clean(self): + self.assertTrue(compileall.compile_file(REPLAY_PATH, + quiet=1, + force=True)) + + +class TestPctDelta(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rp = _load_module() + + def test_normal_positive_delta(self): + # b = 1100, a = 1000 → +10% + self.assertEqual(self.rp._pct_delta(1000, 1100), 10.0) + + def test_normal_negative_delta(self): + self.assertEqual(self.rp._pct_delta(1000, 900), -10.0) + + def test_zero_returns_none(self): + # Zero in the denominator would explode to +inf; surface as None. + self.assertIsNone(self.rp._pct_delta(0, 100)) + + def test_missing_returns_none(self): + self.assertIsNone(self.rp._pct_delta(None, 100)) + self.assertIsNone(self.rp._pct_delta(100, None)) + self.assertIsNone(self.rp._pct_delta(None, None)) + + def test_non_numeric_returns_none(self): + self.assertIsNone(self.rp._pct_delta("foo", 100)) + self.assertIsNone(self.rp._pct_delta(100, "bar")) + + def test_rounded_to_two_decimals(self): + # (103 - 100) / 100 * 100 = 3.0 + self.assertEqual(self.rp._pct_delta(100, 103), 3.0) + + +class TestFormatBytes(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rp = _load_module() + + def test_none_returns_na(self): + self.assertEqual(self.rp._format_bytes(None), "n/a") + + def test_bytes(self): + self.assertEqual(self.rp._format_bytes(512), "512B") + + def test_kib(self): + self.assertEqual(self.rp._format_bytes(2048), "2.0KiB") + + def test_mib(self): + self.assertEqual(self.rp._format_bytes(64 * 1024 * 1024), + "64.0MiB") + + +class TestDiff(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rp = _load_module() + + def test_identical_envelopes_pass(self): + env = _envelope() + verdict = self.rp.diff_envelopes(env, env, 10.0, 20.0) + self.assertTrue(verdict["ok"]) + self.assertEqual(verdict["rule_violations"], []) + self.assertEqual(verdict["wall_clock_pct_delta"], 0.0) + + def test_wall_clock_within_tolerance_passes(self): + # +5% is within default 10% wall-clock tolerance. + verdict = self.rp.diff_envelopes( + _envelope(wall_clock_ms=1000), + _envelope(wall_clock_ms=1050), + 10.0, 20.0) + self.assertTrue(verdict["ok"]) + + def test_wall_clock_over_tolerance_fails(self): + # +50% is well outside default 10%. + verdict = self.rp.diff_envelopes( + _envelope(wall_clock_ms=1000), + _envelope(wall_clock_ms=1500), + 10.0, 20.0) + self.assertFalse(verdict["ok"]) + self.assertTrue(any("wall_clock_pct_delta" in v + for v in verdict["rule_violations"])) + + def test_peak_mem_over_tolerance_fails(self): + # +50% memory. + verdict = self.rp.diff_envelopes( + _envelope(peak_mem_estimate=64 * 1024 * 1024), + _envelope(peak_mem_estimate=96 * 1024 * 1024), + 10.0, 20.0) + self.assertFalse(verdict["ok"]) + self.assertTrue(any("peak_mem_pct_delta" in v + for v in verdict["rule_violations"])) + + def test_peak_mem_null_skips_check(self): + verdict = self.rp.diff_envelopes( + _envelope(peak_mem_estimate=None), + _envelope(peak_mem_estimate=None), + 10.0, 20.0) + self.assertTrue(verdict["ok"]) + self.assertIsNone(verdict["peak_mem_pct_delta"]) + + def test_exit_code_mismatch_fails(self): + verdict = self.rp.diff_envelopes( + _envelope(exit_code=0), + _envelope(exit_code=1), + 10.0, 20.0) + self.assertFalse(verdict["ok"]) + self.assertTrue(any("exit_code" in v + for v in verdict["rule_violations"])) + + def test_timeout_hit_mismatch_fails(self): + verdict = self.rp.diff_envelopes( + _envelope(timeout_hit=False), + _envelope(timeout_hit=True), + 10.0, 20.0) + self.assertFalse(verdict["ok"]) + self.assertTrue(any("timeout_hit" in v + for v in verdict["rule_violations"])) + + def test_zero_wall_clock_handled(self): + # a has wall_clock_ms=0 → pct_delta is None → no violation + verdict = self.rp.diff_envelopes( + _envelope(wall_clock_ms=0), + _envelope(wall_clock_ms=500), + 10.0, 20.0) + self.assertTrue(verdict["ok"]) + self.assertIsNone(verdict["wall_clock_pct_delta"]) + + def test_tight_tolerance_overrides_default(self): + # Tolerances set to 1% — even +5% regresses. + verdict = self.rp.diff_envelopes( + _envelope(wall_clock_ms=1000), + _envelope(wall_clock_ms=1050), + 1.0, 1.0) + self.assertFalse(verdict["ok"]) + + +class TestExplain(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rp = _load_module() + + def test_explain_envelope_includes_all_keys(self): + info = self.rp.explain_envelope(_envelope()) + for k in ("exit_code", "wall_clock_ms", "peak_mem_estimate", + "timeout_hit", "stdout_bytes", "stderr_bytes"): + self.assertIn(k, info) + + def test_explain_line_is_single_space_separated(self): + info = self.rp.explain_envelope(_envelope()) + line = self.rp._explain_line(info) + # Starts with the canonical key=value structure + self.assertTrue(line.startswith("exit=")) + self.assertIn("wall=", line) + self.assertIn("peak=64.0MiB", line) + self.assertIn("timeout=False", line) + + def test_missing_fields_render_na(self): + env = {"exit_code": 0} # no wall / peak / etc. + info = self.rp.explain_envelope(env) + self.assertEqual(info["wall_clock_ms"], "n/a") + self.assertEqual(info["peak_mem_estimate"], "n/a") + line = self.rp._explain_line(info) + self.assertIn("wall=n/a", line) + self.assertIn("peak=n/a", line) + + +class TestLoadEnvelope(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rp = _load_module() + + def test_missing_file(self): + with self.assertRaises(self.rp.ReplayError) as cm: + self.rp._load_envelope("/nonexistent/envelope.json") + self.assertIn("cannot read", str(cm.exception)) + + def test_malformed_json(self): + path = _write_envelope_str("not valid json {{{") + try: + with self.assertRaises(self.rp.ReplayError): + self.rp._load_envelope(path) + finally: + os.unlink(path) + + def test_root_must_be_object(self): + path = _write_envelope_str("[1, 2, 3]") + try: + with self.assertRaises(self.rp.ReplayError): + self.rp._load_envelope(path) + finally: + os.unlink(path) + + +def _write_envelope_str(text): + fd, p = tempfile.mkstemp(prefix="neuros-replay-bad-", suffix=".json") + with os.fdopen(fd, "w") as f: + f.write(text) + return p + + +class TestMainDispatch(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rp = _load_module() + + def _ok(self): + return _envelope() + + def test_diff_returns_zero_for_identical(self): + a = b = _write_envelope(self._ok()) + try: + with mock.patch.object(sys, "stdout", io.StringIO()): + rc = self.rp.main(["diff", a, b]) + self.assertEqual(rc, 0) + finally: + os.unlink(a) + + def test_diff_returns_one_for_regression(self): + a = _write_envelope(self._ok()) + b = _write_envelope(self._ok() | {"wall_clock_ms": 9999}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.rp.main(["diff", a, b]) + self.assertEqual(rc, 1) + self.assertIn("violation", buf.getvalue()) + finally: + os.unlink(a); os.unlink(b) + + def test_diff_json_envelope_shape(self): + # +5% on wall-clock is within the default 10% tolerance. + a = _write_envelope(self._ok()) + b = _write_envelope(self._ok() | {"wall_clock_ms": 1050}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["diff", "--json", a, b]) + self.assertEqual(rc, 0) + obj = json.loads(buf.getvalue().strip()) + self.assertTrue(obj["ok"]) + self.assertEqual(obj["wall_clock_pct_delta"], 5.0) + self.assertEqual(obj["peak_mem_pct_delta"], 0.0) + finally: + os.unlink(a); os.unlink(b) + + def test_diff_bad_input_returns_two(self): + with mock.patch.object(sys, "stdout", io.StringIO()), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["diff", "/nonexistent/a", + "/nonexistent/b"]) + self.assertEqual(rc, 2) + + def test_explain_prints_line(self): + path = _write_envelope(self._ok()) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.rp.main(["explain", path]) + self.assertEqual(rc, 0) + self.assertIn("exit=0", buf.getvalue()) + self.assertIn("wall=1000ms", buf.getvalue()) + finally: + os.unlink(path) + + def test_explain_json_envelope_shape(self): + path = _write_envelope(self._ok()) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.rp.main(["explain", "--json", path]) + self.assertEqual(rc, 0) + obj = json.loads(buf.getvalue().strip()) + self.assertEqual(obj["exit_code"], 0) + self.assertEqual(obj["wall_clock_ms"], 1000) + self.assertIn("neuros_replay_version", obj) + finally: + os.unlink(path) + + def test_extract_stdout(self): + env = self._ok() | {"stdout": "hello\n"} + path = _write_envelope(env) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["extract", path, + "--stream", "stdout"]) + self.assertEqual(rc, 0) + self.assertIn("hello", buf.getvalue()) + self.assertNotIn("===STDERR===", buf.getvalue()) + finally: + os.unlink(path) + + def test_extract_stderr(self): + env = self._ok() | {"stderr": "boom\n"} + path = _write_envelope(env) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["extract", path, + "--stream", "stderr"]) + self.assertEqual(rc, 0) + self.assertIn("boom", buf.getvalue()) + finally: + os.unlink(path) + + def test_extract_both_has_separator(self): + env = self._ok() | {"stdout": "x", "stderr": "y"} + path = _write_envelope(env) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["extract", path, + "--stream", "both"]) + self.assertEqual(rc, 0) + self.assertIn("x", buf.getvalue()) + self.assertIn("===STDERR===", buf.getvalue()) + self.assertIn("y", buf.getvalue()) + finally: + os.unlink(path) + + def test_extract_both_when_stdout_empty(self): + # When one stream is empty, --stream both still emits + # the ===STDERR=== separator so a downstream parser + # can split the streams without field-name re-mapping. + env = self._ok() | {"stdout": "", "stderr": "boom\n"} + path = _write_envelope(env) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["extract", path, + "--stream", "both"]) + self.assertEqual(rc, 0) + self.assertIn("===STDERR===", buf.getvalue()) + self.assertIn("boom", buf.getvalue()) + finally: + os.unlink(path) + + def test_extract_both_when_stderr_empty(self): + # Symmetric case to test_extract_both_when_stdout_empty: + # if stderr is empty, --stream both still emits the + # ===STDERR=== separator before the (empty) stderr + # payload, so a downstream parser can rely on the + # boundary marker in either stream condition. + env = self._ok() | {"stdout": "hello\n", "stderr": ""} + path = _write_envelope(env) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["extract", path, + "--stream", "both"]) + self.assertEqual(rc, 0) + self.assertIn("hello", buf.getvalue()) + self.assertIn("===STDERR===", buf.getvalue()) + finally: + os.unlink(path) + def test_extract_nonexistent_returns_two(self): + with mock.patch.object(sys, "stdout", io.StringIO()), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["extract", "/nonexistent/env.json"]) + self.assertEqual(rc, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runbook.py b/tests/test_runbook.py new file mode 100755 index 0000000..c0564a0 --- /dev/null +++ b/tests/test_runbook.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python3 +"""Tests for neuros-runbook runbook verifier. + +These tests do not need real container kernel delegation because they +mock subprocess.run (the call into neuros-sandbox) and the inner +_run_step helper. They assert: + + * Runbook parser accepts well-formed JSON arrays of steps. + * Runbook parser rejects name collisions, non-string scripts, malformed + expect.regex, etc., with a RunbookError that names the bad step. + * _run_step surfaces stdout/stderr/exit_code from the JSON envelope. + * Per-step assertion logic (exit_code, stdout_matches, timeout_forbidden) + surfaces failures as a list. + * The cmd_run dispatcher exits 0 when all steps pass, 1 on any failure. + * --only-violations filters table output. + * --json emits an envelope whose schema matches the documented contract. +""" +import importlib.util +import io +import json +import os +import sys +import tempfile +import types +import unittest +from unittest import mock + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) +_RUNBOOK_PATH = os.path.join( + ROOT, "config", "includes.chroot", "usr", "local", "bin", "neuros-runbook") + + +def _load_neuros_runbook(): + """Load `neuros-runbook` (no .py suffix) by compiling + exec in a + fresh module namespace, mirroring the loader pattern used in + tests/test_sandbox.py and tests/test_bench.py.""" + with open(_RUNBOOK_PATH) as f: + source = f.read() + code = compile(source, _RUNBOOK_PATH, "exec") + module = sys.modules.get("neuros_runbook") or types.ModuleType( + "neuros_runbook") + module.__file__ = _RUNBOOK_PATH + sys.modules["neuros_runbook"] = module + exec(code, module.__dict__) + return module + + +rb = _load_neuros_runbook() + + +class TestRunbookLoader(unittest.TestCase): + """Locks the runbook JSON validation contract.""" + + def _write(self, content): + f = tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") + json.dump(content, f) + f.close() + return f.name + + def test_well_formed_runbook_loads(self): + path = self._write([ + {"name": "smoke", "script": "print(1)\n"}, + {"name": "fail", "script": "raise Exception\n", + "expect": {"exit_code": 1}}, + {"name": "regex", "script": "print('hello')\n", + "expect": {"stdout_matches": "hel+"}}, + ]) + try: + steps = rb.load_runbook(path) + self.assertEqual(len(steps), 3) + self.assertEqual(steps[0]["name"], "smoke") + self.assertEqual(steps[1]["expect"]["exit_code"], 1) + finally: + os.unlink(path) + + def test_duplicate_name_rejected(self): + path = self._write([ + {"name": "x", "script": "pass\n"}, + {"name": "x", "script": "pass\n"}, + ]) + try: + with self.assertRaises(rb.RunbookError) as cm: + rb.load_runbook(path) + self.assertIn("duplicate name", str(cm.exception)) + finally: + os.unlink(path) + + def test_empty_script_rejected(self): + path = self._write([{"name": "x", "script": ""}]) + try: + with self.assertRaises(rb.RunbookError): + rb.load_runbook(path) + finally: + os.unlink(path) + + def test_non_array_top_level_rejected(self): + f = tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") + f.write(json.dumps({"not": "an array"})) + f.close() + try: + with self.assertRaises(rb.RunbookError) as cm: + rb.load_runbook(f.name) + self.assertIn("must be a JSON array", str(cm.exception)) + finally: + os.unlink(f.name) + + def test_invalid_regex_rejected_at_parse_time(self): + path = self._write([ + {"name": "bad", "script": "pass\n", + "expect": {"stdout_matches": "[unclosed"}}, + ]) + try: + with self.assertRaises(rb.RunbookError) as cm: + rb.load_runbook(path) + self.assertIn("regex invalid", str(cm.exception)) + self.assertIn("bad", str(cm.exception)) + finally: + os.unlink(path) + + def test_empty_name_rejected(self): + path = self._write([{"name": "", "script": "pass\n"}]) + try: + with self.assertRaises(rb.RunbookError): + rb.load_runbook(path) + finally: + os.unlink(path) + + def test_step_must_be_object(self): + path = self._write(["not a dict"]) + try: + with self.assertRaises(rb.RunbookError): + rb.load_runbook(path) + finally: + os.unlink(path) + + +class TestRunStepAndAssertions(unittest.TestCase): + """Drive _run_step with a mocked subprocess.run so the JSON-envelope + parsing + assertion logic both get exercised.""" + + def _fake_cp(self, stdout_text, returncode=0): + cp = mock.Mock() + cp.stdout = stdout_text.encode() + cp.stderr = b"" + cp.returncode = returncode + return cp + + def test_exit_code_match(self): + step = {"name": "x", "script": "print(1)", + "expect": {"exit_code": 0}} + cp = self._fake_cp(json.dumps({ + "exit_code": 0, "stdout": "1\n", + "wall_clock_ms": 1, "timeout_hit": False, + "peak_mem_estimate": 1024})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + env_obj, failures = rb._run_step(step, + env={"PATH": "/bin"}) + self.assertEqual(env_obj["exit_code"], 0) + self.assertEqual(failures, []) + + def test_exit_code_mismatch(self): + step = {"name": "x", "script": "raise Exception", + "expect": {"exit_code": 0}} + cp = self._fake_cp(json.dumps({ + "exit_code": 1, "stdout": "", + "wall_clock_ms": 1, "timeout_hit": False, + "peak_mem_estimate": None})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + env_obj, failures = rb._run_step(step, + env={"PATH": "/bin"}) + self.assertTrue(any("exit_code" in f for f in failures)) + + def test_exit_code_any(self): + step = {"name": "x", "script": "anything", + "expect": {"exit_code": "any"}} + cp = self._fake_cp(json.dumps({ + "exit_code": 137, "stdout": "", + "wall_clock_ms": 1, "timeout_hit": False, + "peak_mem_estimate": 100})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + env_obj, failures = rb._run_step(step, + env={"PATH": "/bin"}) + self.assertEqual(failures, []) + + def test_stdout_pattern_mismatch(self): + step = {"name": "x", "script": "pass", + "expect": {"stdout_matches": "needle"}} + cp = self._fake_cp(json.dumps({ + "exit_code": 0, "stdout": "haystack\n", + "wall_clock_ms": 1, "timeout_hit": False, + "peak_mem_estimate": None})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + _, failures = rb._run_step(step, env={"PATH": "/bin"}) + self.assertTrue(any("stdout_matches" in f for f in failures)) + + def test_stderr_pattern_match(self): + step = {"name": "x", "script": "raise Exception", + "expect": {"stderr_matches": "Traceback"}} + cp = self._fake_cp(json.dumps({ + "exit_code": 1, "stdout": "", + "stderr": "Traceback (most recent call last):\n", + "wall_clock_ms": 1, "timeout_hit": False, + "peak_mem_estimate": None})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + _, failures = rb._run_step(step, env={"PATH": "/bin"}) + self.assertEqual(failures, []) + + def test_timeout_forbidden(self): + step = {"name": "x", "script": "while True: pass", + "expect": {"timeout_forbidden": True}} + cp = self._fake_cp(json.dumps({ + "exit_code": 124, "stdout": "", + "wall_clock_ms": 5000, "timeout_hit": True, + "peak_mem_estimate": None})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + _, failures = rb._run_step(step, env={"PATH": "/bin"}) + self.assertTrue(any("timeout" in f for f in failures)) + + def test_all_assertions_pass(self): + step = {"name": "x", "script": "print('hello')", + "expect": {"exit_code": 0, + "stdout_matches": "^hello$", + "stderr_matches": ".*"}} + cp = self._fake_cp(json.dumps({ + "exit_code": 0, "stdout": "hello\n", + "stderr": "", + "wall_clock_ms": 2, "timeout_hit": False, + "peak_mem_estimate": 4096})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + env_obj, failures = rb._run_step(step, + env={"PATH": "/bin"}) + self.assertEqual(env_obj["peak_mem_estimate"], 4096) + self.assertEqual(failures, []) + + +class TestCmdRunDispatch(unittest.TestCase): + """Top-level cmd_run aggregates step results into the documented + JSON envelope and pickes the right exit code.""" + + def _write(self, content): + f = tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") + json.dump(content, f) + f.close() + return f.name + + def _ok_envelope(self, exit_code=0, timeout_hit=False, + peak=None, wall_ms=10, stdout="", stderr=""): + return json.dumps({ + "exit_code": exit_code, "stdout": stdout, "stderr": stderr, + "wall_clock_ms": wall_ms, "timeout_hit": timeout_hit, + "peak_mem_estimate": peak}) + + def test_all_pass_returns_zero(self): + path = self._write([ + {"name": "a", "script": "pass", "expect": {"exit_code": 0}}, + {"name": "b", "script": "pass", "expect": {"exit_code": 0}}, + ]) + try: + ns = mock.MagicMock() + ns.runbook = path + ns.json = False + ns.only_violations = False + cp = mock.Mock(returncode=0, + stdout=self._ok_envelope().encode(), + stderr=b"") + with mock.patch.object(rb.subprocess, "run", return_value=cp), \ + mock.patch.object(sys, "stdout", io.StringIO()), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = rb.cmd_run(ns) + self.assertEqual(rc, 0) + finally: + os.unlink(path) + + def test_any_fail_returns_one(self): + """Two-step runbook where step b fails its exit_code + assertion. cmd_run should aggregate and exit 1.""" + path = self._write([ + {"name": "a", "script": "pass", "expect": {"exit_code": 0}}, + {"name": "b", "script": "raise", + "expect": {"exit_code": 0}}, + ]) + try: + ns = mock.MagicMock() + ns.runbook = path + ns.json = False + ns.only_violations = False + + # Stream two envelopes: step a OK, step b fails. + queue = iter([ + self._ok_envelope().encode(), + self._ok_envelope(exit_code=1, stderr="boom").encode(), + ]) + + def fake_run(*a, **kw): + cp = mock.Mock() + cp.stdout = next(queue) + cp.stderr = b"" + cp.returncode = 0 + return cp + + with (mock.patch.object(rb.subprocess, "run", + side_effect=fake_run), + mock.patch.object(sys, "stdout", io.StringIO()), + mock.patch.object(sys, "stderr", io.StringIO())): + rc = rb.cmd_run(ns) + self.assertEqual(rc, 1) + finally: + os.unlink(path) + + def test_only_violations_table_filters_pass(self): + path = self._write([ + {"name": "pass-step", "script": "pass"}, + {"name": "fail-step", "script": "raise", + "expect": {"exit_code": 0}}, + ]) + try: + ns = mock.MagicMock() + ns.runbook = path + ns.json = False + ns.only_violations = True + + ok_env = self._ok_envelope() + fail_env = self._ok_envelope(exit_code=1, stderr="boom") + queue = iter([ok_env.encode(), fail_env.encode()]) + + def fake_run(*a, **kw): + cp = mock.Mock() + cp.stdout = next(queue) + cp.stderr = b"" + cp.returncode = 0 + return cp + + with mock.patch.object(rb.subprocess, "run", + side_effect=fake_run): + buf_out = io.StringIO() + with mock.patch.object(sys, "stdout", buf_out), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = rb.cmd_run(ns) + self.assertEqual(rc, 1) + # Only fail-step should appear in the table + self.assertIn("fail-step", buf_out.getvalue()) + self.assertNotIn("pass-step", buf_out.getvalue()) + finally: + os.unlink(path) + + def test_json_envelope_shape(self): + path = self._write([ + {"name": "audit-step", "script": "pass", + "expect": {"exit_code": 0}}, + ]) + try: + ns = mock.MagicMock() + ns.runbook = path + ns.json = True + ns.only_violations = False + queue = iter([self._ok_envelope( + exit_code=0, peak=4096, wall_ms=2).encode()]) + + def fake_run(*a, **kw): + cp = mock.Mock() + cp.stdout = next(queue) + cp.stderr = b"" + cp.returncode = 0 + return cp + + with mock.patch.object(rb.subprocess, "run", + side_effect=fake_run): + buf_out = io.StringIO() + with mock.patch.object(sys, "stdout", buf_out): + rc = rb.cmd_run(ns) + self.assertEqual(rc, 0) + env = json.loads(buf_out.getvalue()) + self.assertIn("steps", env) + self.assertEqual(len(env["steps"]), 1) + step = env["steps"][0] + self.assertEqual(step["name"], "audit-step") + self.assertTrue(step["passed"]) + self.assertEqual(step["peak_mem_estimate"], 4096) + self.assertIn("version", env) + self.assertEqual(env["runbook_path"], path) + finally: + os.unlink(path) + + def test_runbook_error_dies_two(self): + path = "/nonexistent/runbook.json" + ns = mock.MagicMock() + ns.runbook = path + ns.json = False + ns.only_violations = False + with mock.patch.object(sys, "stderr", io.StringIO()): + with self.assertRaises(SystemExit) as cm: + rb.cmd_run(ns) + self.assertEqual(cm.exception.code, 2) + + +class TestMainDispatch(unittest.TestCase): + """Argparse routes correctly; exit code semantics.""" + + def test_no_subcommand_exits_two(self): + with self.assertRaises(SystemExit): + rb.main([]) + + def test_run_returns_zero_for_clean_run(self): + path = tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") + json.dump([{"name": "x", "script": "pass", + "expect": {"exit_code": 0}}], path) + path.close() + try: + queue = iter([self._envelope_ok().encode()]) + + def fake_run(*a, **kw): + cp = mock.Mock() + cp.stdout = next(queue) + cp.stderr = b"" + cp.returncode = 0 + return cp + + with (mock.patch.object(rb.subprocess, "run", + side_effect=fake_run), + mock.patch.object(sys, "stdout", io.StringIO()), + mock.patch.object(sys, "stderr", io.StringIO())): + rc = rb.main(["run", path.name]) + self.assertEqual(rc, 0) + finally: + os.unlink(path.name) + + @staticmethod + def _envelope_ok(exit_code=0): + return json.dumps({ + "exit_code": exit_code, "stdout": "", "stderr": "", + "wall_clock_ms": 1, "timeout_hit": False, + "peak_mem_estimate": 0}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py new file mode 100755 index 0000000..437b2b3 --- /dev/null +++ b/tests/test_sandbox.py @@ -0,0 +1,550 @@ +#!/usr/bin/env python3 +"""Tests for neuros-sandbox safe-runner wrapper. + +These tests do not need real kernel/cgroup delegation because they +mock `subprocess.run` and the filesystem paths that the wrapper +constructs. They assert: + + * Default safe flag set is always passed to neuros-container + * --unsafe relaxes --net/--read-only/cap-drop and lets + --cap-drop-keep apply its own regex + * Host env is scrubbed to PATH only (no API keys leak) + * Watchdog timeout surfaces as exit 124 with timeout_hit=True + * JSON envelope is single-line and JSON-parseable + * Bundle extraction rejects absolute/traversal paths and cleans up + * Stdin is read when no script path is given + * Script file read errors surface as SandboxError +""" +import argparse +import io +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import types +import unittest +from unittest import mock + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) +_SANDBOX_PATH = os.path.join( + ROOT, "config", "includes.chroot", "usr", "local", "bin", "neuros-sandbox") + + +def _load_neuros_sandbox(): + """Load ``neuros-sandbox`` (no .py suffix in its install path) by + compiling its source and exec-ing in a fresh module namespace. + This is the only stable way to import a hyphen-named script. + + The loader caches in ``sys.modules`` indefinitely. After editing + the production source file, call ``importlib.reload(sb)`` (or + ``importlib.reload(sys.modules["neuros_sandbox"])``) to pick up + the change without restarting the test process. + """ + with open(_SANDBOX_PATH) as f: + source = f.read() + code = compile(source, _SANDBOX_PATH, "exec") + module = sys.modules.get("neuros_sandbox") or types.ModuleType( + "neuros_sandbox") + module.__file__ = _SANDBOX_PATH + sys.modules["neuros_sandbox"] = module + exec(code, module.__dict__) + return module + + +class _FakeStdStream: + """Mimic sys.stdout/sys.stderr's contract so the production + `sys.stdout.buffer.write(bytes)` AND `sys.stderr.write(str)` calls + both work. Use it via `with mock.patch.object(sys, "stdout", stream):`. + + Reads from .buffer accumulate bytes; reads from .write accumulate + text. The combined payload is exposed as .value(). + """ + + def __init__(self): + self._buffer = io.BytesIO() + self._text = [] + + @property + def buffer(self): + return self._buffer + + def write(self, data): + if isinstance(data, bytes): + self._buffer.write(data) + else: + self._text.append(data) + + def flush(self): + pass + + def value(self): + return self._buffer.getvalue() + "".join(self._text).encode( + "utf-8", errors="replace") + + +sb = _load_neuros_sandbox() + + +class TestParsePercent(unittest.TestCase): + def test_percent_string(self): + self.assertEqual(sb._parse_percent("50%"), 50000) + self.assertEqual(sb._parse_percent("25%"), 25000) + self.assertEqual(sb._parse_percent("100%"), 100000) + + def test_fraction(self): + self.assertEqual(sb._parse_percent("0.25"), 25000) + self.assertEqual(sb._parse_percent("1.0"), 100000) + + def test_microseconds(self): + self.assertEqual(sb._parse_percent("50000us"), 50000) + self.assertEqual(sb._parse_percent("100000us"), 100000) + + +class TestBuildArgv(unittest.TestCase): + def _args(self, **overrides): + # MUST use argparse.Namespace (not mock.Mock): Mock evaluates + # as truthy regardless of the kwarg passed, which silently + # disables the safety-net flag set in _build_argv. The defaults + # dict pattern lets overrides cleanly replace named defaults + # without producing argparse.Namespace(TypeError: multiple values). + defaults = dict( + unsafe=False, + mem=sb.DEFAULT_MEM, + pids=sb.DEFAULT_PIDS, + cpu_quota=sb.DEFAULT_CPU_QUOTA, + cap_drop=sb.DEFAULT_CAP_DROP, + cap_drop_keep=None, + rootfs=None, + name=None, + user=None, + ) + defaults.update(overrides) + return argparse.Namespace(**defaults) + + def test_safe_mode_default_flags_present(self): + argv = sb._build_argv(self._args(), script_target="-") + self.assertEqual(argv[0], "neuros-container") + self.assertEqual(argv[1], "run") + # safety nets are on + self.assertIn("--net", argv) + self.assertIn("--read-only", argv) + self.assertIn("--cap-drop", argv) + # Lock: the value following --cap-drop must be exactly the + # baseline DEFAULT_CAP_DROP, not the wrapper's blank string or + # any silent re-default. + self.assertEqual( + argv[argv.index("--cap-drop") + 1], sb.DEFAULT_CAP_DROP) + # defaults flow through + self.assertIn("--mem", argv) + self.assertEqual(argv[argv.index("--mem") + 1], sb.DEFAULT_MEM) + self.assertIn("--pids", argv) + self.assertEqual(argv[argv.index("--pids") + 1], str(sb.DEFAULT_PIDS)) + self.assertIn("--cpu-quota", argv) + self.assertEqual( + argv[argv.index("--cpu-quota") + 1], str(sb.DEFAULT_CPU_QUOTA)) + # name is generated when not given + name = argv[argv.index("--name") + 1] + self.assertTrue(name.startswith(sb.NAME_PREFIX)) + # entry point is the stdin python interpreter + self.assertEqual(argv[-3:], ["python3", "-u", "-"]) + + def test_unsafe_mode_relaxes_safe_flags(self): + argv = sb._build_argv(self._args(unsafe=True), script_target="-") + self.assertNotIn("--net", argv) + self.assertNotIn("--read-only", argv) + # The default cap-drop is still passed under --unsafe unless + # --cap-drop-keep replaces it; here we don't override. + self.assertIn("--cap-drop", argv) + # value-level invariant: even in unsafe mode the baseline + # DEFAULT_CAP_DROP is what lands in the primitive's --cap-drop + # slot, not any silent relaxation. + self.assertEqual( + argv[argv.index("--cap-drop") + 1], sb.DEFAULT_CAP_DROP) + + def test_unsafe_with_cap_drop_keep_replaces(self): + argv = sb._build_argv(self._args( + unsafe=True, cap_drop_keep=r"CAP_NET_RAW" + ), script_target="-") + self.assertNotIn("--net", argv) + self.assertNotIn("--read-only", argv) + self.assertEqual( + argv[argv.index("--cap-drop") + 1], r"CAP_NET_RAW") + + def test_rootfs_and_user_appear(self): + argv = sb._build_argv(self._args( + rootfs="/srv/rootfs", user="1000:1000" + ), script_target="/tmp/script.py") + self.assertEqual(argv[argv.index("--rootfs") + 1], "/srv/rootfs") + self.assertEqual(argv[argv.index("--user") + 1], "1000:1000") + # entry point ends with the script path + self.assertEqual(argv[-1], "/tmp/script.py") + + def test_name_override_passthrough(self): + argv = sb._build_argv( + self._args(name="explicit-name"), script_target="-") + self.assertEqual(argv[argv.index("--name") + 1], "explicit-name") + + def test_safe_mode_ignores_cap_drop_keep(self): + """--cap-drop-keep is documented as unsafe-mode-only. Pass it + under safe mode and assert (a) the value following --cap-drop + is still DEFAULT_CAP_DROP, and (b) the literal override value + the user passed in is NOT substituted into argv. Locks the + policy so a future refactor that adds a fast path doesn't + accidentally honor --cap-drop-keep in safe mode.""" + cap_keep = r"CAP_NET_RAW" + argv = sb._build_argv(self._args( + cap_drop_keep=cap_keep + ), script_target="-") + self.assertEqual( + argv[argv.index("--cap-drop") + 1], sb.DEFAULT_CAP_DROP) + # The literal override value must not appear in the produced + # argv (only DEFAULT_CAP_DROP should). This locks the policy + # at value-granularity rather than flag-name-granularity, + # which would be a tautology since _build_argv never emits + # --cap-drop-keep as a separate flag. + self.assertNotIn(cap_keep, argv) + + def test_cap_drop_keep_invalid_regex_raises_sandbox_error(self): + """Lock the contract that an invalid --cap-drop-keep regex + fails at the wrapper layer (SandboxError) before the primitive + is even invoked. Keeps the failure surface out of the + kernel-bridge ctypes call. + """ + with self.assertRaises(sb.SandboxError): + sb._build_argv(self._args( + unsafe=True, cap_drop_keep=r"[unclosed" + ), script_target="-") + + def test_cap_drop_keep_empty_string_raises_sandbox_error(self): + """Lock the contract that --cap-drop-keep '' under --unsafe is + an explicit user error, NOT a silent fallback to the default + cap-drop regex. Empty string was previously silenced by the + truthiness check; the is-not-None check exposes it. + """ + with self.assertRaises(sb.SandboxError): + sb._build_argv(self._args( + unsafe=True, cap_drop_keep="" + ), script_target="-") + + def test_cap_drop_keep_whitespace_only_raises_sandbox_error(self): + """Lock the contract that --cap-drop-keep ' ' (whitespace + only) is rejected just like the empty string — otherwise + re.compile(' ') silently produces a valid-but-useless regex + that matches literal whitespace exclusively. + """ + with self.assertRaises(sb.SandboxError): + sb._build_argv(self._args( + unsafe=True, cap_drop_keep=" \t\n" + ), script_target="-") + + +class TestLoadScript(unittest.TestCase): + def test_file_path(self): + with tempfile.NamedTemporaryFile("wb", delete=False) as f: + f.write(b"print('hello')\n") + path = f.name + try: + self.assertEqual(sb._load_script(path), b"print('hello')\n") + finally: + os.unlink(path) + + def test_missing_file_raises_sandbox_error(self): + with self.assertRaises(sb.SandboxError): + sb._load_script("/nonexistent/path/script.py") + + def test_empty_string_path_raises(self): + with tempfile.NamedTemporaryFile("wb", delete=False) as f: + f.write(b"") + path = f.name + try: + with self.assertRaises(sb.SandboxError): + sb._load_script(path) + finally: + os.unlink(path) + + +class TestStdInLoadScript(unittest.TestCase): + def test_stdin_when_path_is_none(self): + with mock.patch.object(sys, "stdin") as mock_stdin: + mock_stdin.buffer = io.BytesIO(b"from_stdin") + self.assertEqual(sb._load_script(None), b"from_stdin") + + +class TestExtractBundle(unittest.TestCase): + def _make_tar(self, members): + f = tempfile.NamedTemporaryFile("wb", delete=False, suffix=".tar") + with tarfile.open(fileobj=f, mode="w") as tar: + for name, content in members.items(): + data = content.encode("utf-8") if isinstance(content, str) else content + info = tarfile.TarInfo(name=name) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return f.name + + def test_extracts_simple_bundle_and_returns_path(self): + path = self._make_tar({"a.txt": "alpha", "b/c.txt": "beta"}) + try: + out = sb._extract_bundle(path) + try: + self.assertTrue(os.path.isdir(out)) + self.assertEqual( + open(os.path.join(out, "a.txt")).read(), "alpha") + self.assertEqual( + open(os.path.join(out, "b", "c.txt")).read(), "beta") + finally: + shutil.rmtree(out, ignore_errors=True) + finally: + os.unlink(path) + + def test_rejects_absolute_path_entry(self): + path = self._make_tar({"/abs.txt": "x"}) + try: + with self.assertRaises(sb.SandboxError): + sb._extract_bundle(path) + finally: + os.unlink(path) + + def test_rejects_traversal_entry(self): + path = self._make_tar({"../escape.txt": "x"}) + try: + with self.assertRaises(sb.SandboxError): + sb._extract_bundle(path) + finally: + os.unlink(path) + + +class TestRunInSandbox(unittest.TestCase): + """Drives cmd_run end-to-end with a mocked subprocess.run.""" + + def _args(self, **overrides): + defaults = dict( + cmd="run", + script="script.py", + mem="256M", + pids=sb.DEFAULT_PIDS, + cpu=None, + cpu_quota=sb.DEFAULT_CPU_QUOTA, + cap_drop=sb.DEFAULT_CAP_DROP, + cap_drop_keep=None, + rootfs=None, + name="test-name", + user=None, + bundle=None, + timeout=5, + unsafe=False, + json=False, + dry_run=False, + ) + defaults.update(overrides) + return argparse.Namespace(**defaults) + + @mock.patch.object(sb, "_load_script", return_value=b"script-bytes") + @mock.patch.object(sb, "_read_peak_memory", return_value=None) + @mock.patch.object(sb.subprocess, "run") + def test_human_mode_passes_through_stdout_and_stderr( + self, m_run, _mp, _ml): + cp = mock.Mock(returncode=0, stdout=b"hello\n", stderr=b"warn\n") + m_run.return_value = cp + # FakeStdStream exposes both .buffer (BytesIO) for binary + # writes AND .write(str) for the trailer line, so the + # production path through sys.stdout.buffer + sys.stderr.write + # both work cleanly. + s_out = _FakeStdStream() + s_err = _FakeStdStream() + with mock.patch.object(sys, "stdout", s_out), \ + mock.patch.object(sys, "stderr", s_err): + rc = sb.cmd_run(self._args()) + self.assertEqual(rc, 0) + out_payload = s_out.value() + err_payload = s_err.value() + self.assertIn(b"hello", out_payload) + self.assertIn(b"warn", err_payload) + self.assertIn(b"[neuros-sandbox]", err_payload) + # Env MUST be scrubbed to MINIMAL_ENV + positional, kwargs = m_run.call_args + self.assertEqual(kwargs["env"], sb.MINIMAL_ENV) + # input must contain the script bytes + self.assertEqual(kwargs["input"], b"script-bytes") + # Wall-clock mgmt passes timeout through + self.assertEqual(kwargs["timeout"], 5) + + @mock.patch.object(sb, "_load_script", return_value=b"script-bytes") + @mock.patch.object(sb, "_read_peak_memory", return_value=None) + @mock.patch.object(sb.subprocess, "run") + def test_json_mode_emits_single_line_envelope( + self, m_run, _mp, _ml): + cp = mock.Mock(returncode=2, stdout=b"out", stderr=b"err") + m_run.return_value = cp + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = sb.cmd_run(self._args(json=True)) + self.assertEqual(rc, 2) + lines = [l for l in buf.getvalue().splitlines() if l] + self.assertEqual(len(lines), 1) + env = json.loads(lines[0]) + self.assertEqual(env["exit_code"], 2) + self.assertEqual(env["stdout"], "out") + self.assertEqual(env["stderr"], "err") + self.assertFalse(env["timeout_hit"]) + self.assertIn("wall_clock_ms", env) + # peak_mem_estimate is propagated when non-None + self.assertIsNone(env["peak_mem_estimate"]) + + @mock.patch.object(sb, "_load_script", return_value=b"script-bytes") + @mock.patch.object(sb, "_read_peak_memory", return_value=8388608) + @mock.patch.object(sb.subprocess, "run") + def test_json_envelope_propagates_peak_mem( + self, m_run, _mp, _ml): + m_run.return_value = mock.Mock( + returncode=0, stdout=b"", stderr=b"") + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + sb.cmd_run(self._args(json=True)) + env = json.loads(buf.getvalue().strip()) + self.assertEqual(env["peak_mem_estimate"], 8388608) + + @mock.patch.object(sb, "_load_script", return_value=b"x") + @mock.patch.object(sb, "_read_peak_memory", return_value=None) + @mock.patch.object(sb.subprocess, "run") + def test_timeout_returns_124_and_marks_envelope( + self, m_run, _mp, _ml): + m_run.side_effect = subprocess.TimeoutExpired(cmd=["x"], timeout=5) + buf = io.StringIO() # --json mode uses print() which is text + with mock.patch.object(sys, "stdout", buf): + rc = sb.cmd_run(self._args(json=True)) + self.assertEqual(rc, 124) + env = json.loads(buf.getvalue().strip()) + self.assertTrue(env["timeout_hit"]) + self.assertEqual(env["exit_code"], 124) + # Lock the contract: cmd_run forwards the same argv to + # subprocess.run that callers can correlate with the + # constructed CompletedProcess in the timeout path. Production + # calls subprocess.run(ARGV_LIST, ...), so the ARGV_LIST is + # captured as the single first positional argument to run: + # call_args.args == ([ARGV_LIST],) — and ARGV_LIST[0] is then + # "neuros-container". + run_args, _ = m_run.call_args + self.assertEqual(len(run_args), 1, "subprocess.run got a single argv list") + self.assertEqual(run_args[0][0], "neuros-container") + + @mock.patch.object(sb, "_load_script", return_value=b"x") + @mock.patch.object(sb.subprocess, "run") + def test_dry_run_prints_argv_and_does_not_spawn( + self, m_run, _ml): + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = sb.cmd_run(self._args(dry_run=True)) + self.assertEqual(rc, 0) + self.assertIn("neuros-container", buf.getvalue()) + self.assertIn("--net", buf.getvalue()) + self.assertIn("--read-only", buf.getvalue()) + # No actual subprocess call was made + m_run.assert_not_called() + + @mock.patch.object(sb.subprocess, "run") + def test_bundle_extracts_and_rmtree_cleanup_is_called( + self, m_run): + """Verify the wrapper explicitly rmtree's the bundle tmpdir we + created (no implicit leak).""" + captured_roots = [] + real_rmtree = sb.shutil.rmtree + + def track_rmtree(path, *a, **kw): + captured_roots.append(path) + real_rmtree(path, *a, **kw) + + with tempfile.TemporaryDirectory() as tmp: + tar_path = os.path.join(tmp, "bundle.tar") + with tarfile.open(tar_path, "w") as tar: + data = b"echo hi\n" + info = tarfile.TarInfo("script.sh") + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + with mock.patch.object(sb, "_load_script", + return_value=b"echo hi"), \ + mock.patch.object(sb.shutil, "rmtree", + side_effect=track_rmtree): + cp = mock.Mock(returncode=0, stdout=b"out", stderr=b"") + m_run.return_value = cp + s_err = _FakeStdStream() + with mock.patch.object(sys, "stdout", _FakeStdStream()), \ + mock.patch.object(sys, "stderr", s_err): + rc = sb.cmd_run(self._args(bundle=tar_path)) + self.assertEqual(rc, 0) + # The wrapper created at least one tmpdir for the bundle, + # and rmtree was called on it. + self.assertTrue(captured_roots, "rmtree was never called") + self.assertTrue(any( + "neuros-sbx-bundle-" in p for p in captured_roots)) + + +class TestReadPeakMemory(unittest.TestCase): + """Best-effort memory.peak polling exercised with tmpdir fake files.""" + + def test_returns_int_when_file_exists(self): + with tempfile.TemporaryDirectory() as td: + # Create the cgroup-tree convention neuros-container uses + # by carving out the leaf dir directly under /sys/fs/cgroup. + leaf = os.path.join(td, "neuros-sbx-foo") + os.makedirs(leaf) + with open(os.path.join(leaf, "memory.peak"), "w") as f: + f.write("1234567\n") + with mock.patch.object(sb, "_CGROUP_V2_ROOTS", (td,)): + self.assertEqual(sb._read_peak_memory("foo"), 1234567) + + def test_returns_none_when_file_missing(self): + with tempfile.TemporaryDirectory() as td: + with mock.patch.object(sb, "_CGROUP_V2_ROOTS", (td,)): + self.assertIsNone(sb._read_peak_memory("missing")) + + def test_returns_none_on_unparseable_value(self): + with tempfile.TemporaryDirectory() as td: + leaf = os.path.join(td, "neuros-sbx-foo") + os.makedirs(leaf) + with open(os.path.join(leaf, "memory.peak"), "w") as f: + f.write("not-a-number\n") + with mock.patch.object(sb, "_CGROUP_V2_ROOTS", (td,)): + self.assertIsNone(sb._read_peak_memory("foo")) + + def test_mtime_tie_breaks_by_shorter_path(self): + """Two leaves with identical mtime: the shorter (closer-to-root) + path wins via the (mtime, -len(p)) sort key.""" + with tempfile.TemporaryDirectory() as td: + shallow = os.path.join(td, "neuros-sbx-foo") + deep = os.path.join(td, "slice", "neuros", "neuros-sbx-foo") + for d in (shallow, deep): + os.makedirs(d, exist_ok=True) + # Both files exist; force identical mtime. + fixed = 1_700_000_000.0 + for leaf in (shallow, deep): + with open(os.path.join(leaf, "memory.peak"), "w") as f: + f.write("42\n") + os.utime(leaf, (fixed, fixed)) + with mock.patch.object(sb, "_CGROUP_V2_ROOTS", (td,)): + # Shorter path should win, returning 42. + self.assertEqual(sb._read_peak_memory("foo"), 42) + + +class TestMainDispatch(unittest.TestCase): + def test_unknown_cmd_exits(self): + with self.assertRaises(SystemExit): + sb.main(["nope"]) + + def test_cpu_flag_translates_to_quota(self): + argv = ["run", "script.py", "--cpu", "25%", "--dry-run"] + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = sb.main(argv) + self.assertEqual(rc, 0) + out = buf.getvalue() + # 25% of 100000 = 25000 + self.assertIn("--cpu-quota 25000", out) + + +if __name__ == "__main__": + unittest.main()