diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37f977f..4f8fe81 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: name: Validate Build Configuration runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Make scripts executable run: | @@ -24,7 +24,10 @@ jobs: run: bash validate-build.sh - name: Run unit tests - run: python3 tests/test_nn.py -v + run: python3 -m unittest discover -s tests -p 'test_*.py' -v + + - name: Air-gap check (no new outbound network calls) + run: bash scripts/check-airgap.sh - name: Check shell script syntax run: | @@ -38,7 +41,7 @@ jobs: name: Lint & Format Check runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Check all Python files compile run: | @@ -72,11 +75,11 @@ jobs: done build-iso: - name: Build ISO (Structure Check) + name: Build ISO (Structure + lb config + package resolution) runs-on: ubuntu-24.04 needs: validate steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Install dependencies run: | @@ -91,17 +94,50 @@ jobs: test -d config/package-lists && echo "OK: package lists present" test -f build.sh && echo "OK: build.sh present" test -f validate-build.sh && echo "OK: validate-build.sh present" - echo "Structure OK — full ISO build requires 20GB disk (CI runners limited)" + echo "Structure OK — a full ISO build (debootstrap + chroot + squashfs)" + echo "needs 20GB+ disk and privileged mounts CI runners don't have." + + - name: Run lb config with the pinned snapshot mirror + run: | + SNAPSHOT_TS=$(grep -oP '(?<=SNAPSHOT_TS=")[^"]+' build.sh) + SNAPSHOT_MIRROR="https://snapshot.ubuntu.com/ubuntu/${SNAPSHOT_TS}" + echo "Pinned snapshot: $SNAPSHOT_MIRROR" + lb config \ + --mode ubuntu \ + --distribution noble \ + --archive-areas "main restricted universe multiverse" \ + --debian-installer none \ + --memtest none \ + --binary-images iso-hybrid \ + --mirror-bootstrap "$SNAPSHOT_MIRROR" \ + --mirror-chroot "$SNAPSHOT_MIRROR" \ + --mirror-chroot-security "$SNAPSHOT_MIRROR" + echo "OK: lb config succeeded against the pinned snapshot" + + - name: Resolve neuros.list.chroot against the pinned snapshot + run: | + SNAPSHOT_TS=$(grep -oP '(?<=SNAPSHOT_TS=")[^"]+' build.sh) + SNAPSHOT_MIRROR="https://snapshot.ubuntu.com/ubuntu/${SNAPSHOT_TS}" + sudo tee /etc/apt/sources.list.d/neuros-snapshot.list > /dev/null <&1 || echo "Docker build skipped (non-critical in CI)" diff --git a/Dockerfile b/Dockerfile index d5205a2..f97875c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,10 @@ # Or use with docker-compose: # docker compose up -FROM ubuntu:24.04 +# Pinned to a digest instead of the floating 24.04 tag, which Canonical +# rebuilds in place for security updates -- the digest is immutable, so the +# build container's starting filesystem can't drift between builds. +FROM ubuntu:24.04@sha256:561618e2c15bf2397621dd04f96926663a3b5616c189cf7e38db7e82f5c538ea ENV DEBIAN_FRONTEND=noninteractive ENV TZ=UTC @@ -48,8 +51,9 @@ COPY build.sh /opt/neuros/ COPY Makefile /opt/neuros/ COPY validate-build.sh /opt/neuros/ -# Install Python dependencies -RUN pip3 install --break-system-packages requests || true +# Install Python dependencies (pinned so this doesn't silently pick up a +# newer requests on rebuild) +RUN pip3 install --break-system-packages requests==2.34.2 || true # Set up neuros tools in container RUN mkdir -p /usr/local/bin && \ diff --git a/README.md b/README.md index cab0dbe..1b88034 100644 --- a/README.md +++ b/README.md @@ -68,11 +68,330 @@ a thin GUI over this CLI (see above). ### MCP server (`neuros-mcp`) -Implements the Model Context Protocol over HTTP and JSON-RPC: -`initialize`, `tools/list`, `tools/call`, and `resources/list`, with -tools for `read_file`, `list_directory`, `run_command`, `ask_llm`, -`get_system_info`, and `git_status`. `run_command` is checked against -shell-metacharacter injection; see `tests/test_mcp.py`. +Implements the Model Context Protocol over two transports: HTTP/JSON-RPC +(`neuros-mcp`) and the spec's stdio transport (`neuros-mcp --stdio`), +which is what real MCP clients (Claude Code, Claude Desktop, etc.) use +to launch a local server as a subprocess. Both transports share one +dispatcher (`dispatch_request`) for `initialize`, `notifications/ +initialized`, `tools/list`, `tools/call`, `resources/list`, and +`resources/read`, with tools for `read_file`, `list_directory`, +`run_command`, `ask_llm`, `get_system_info`, and `git_status`. +`run_command` is checked against shell-metacharacter injection. The +stdio transport was verified against the official `mcp` Python SDK +client (`ClientSession` over `stdio_client`): initialize handshake, +`tools/list`, and `tools/call` all round-trip correctly. See +`tests/test_mcp.py` for the dispatcher-level tests. + +### Container primitive (`neuros-container`) + +A from-scratch container runner built directly on `unshare(2)` and +cgroup v2, no runc/containerd/libcontainer: + +```sh neuros-container run --mem 256M --pids 64 --hostname box -- bash + neuros-container run --cpu-quota 50000/100000 --read-only --net -- bash + neuros-container run --user 1000:1000 --workdir /tmp --detach --name job -- sleep 60 + neuros-container run --env DB=postgres --env DEBUG=1 -- bash + neuros-container run --env-from-file ./prod.env -- bash + neuros-container run --cap-drop 'CAP_(NET_RAW|SYS_PTRACE|SYS_ADMIN)' -- bash + neuros-container list --json + neuros-container cleanup --all # remove empty detached cgroups + state +``` + +Resource limits are real cgroup v2 accounting: `--mem` sets +`memory.max` on a fresh leaf cgroup, `--pids` sets `pids.max`, +`--cpu` sets `cpu.weight`, and `--cpu-quota "MAX/PERIOD"` writes the +PETF v2 cpu.max quota (e.g. `--cpu-quota 50000/100000` caps a workload +to half of one CPU). Because a cgroup that already holds member +processes can't enable `subtree_control` for children (cgroup v2's +"no internal process" rule), the tool walks up from its own cgroup to +the nearest ancestor that already delegates the wanted controller, so +it works from an ordinary interactive shell without root. If a +controller was requested and none were delegated all the way down, +the tool prints a single stderr warning rather than silently creating +an empty cgroup. + +Verified on this machine: a `--mem 16M` cgroup holding a process that +touches 200MB of `bytearray` keeps `memory.current` at the 16MB +ceiling instead of growing past it, and a `--pids 4` cgroup stops a +20-iteration fork loop after 3 children (see +`tests/test_container.py`). + +Namespace isolation (mount, UTS, PID, IPC, optionally network) needs +either root or an unprivileged user namespace; on Ubuntu 24.04+ the +latter is blocked by default for unconfined processes +(`kernel.apparmor_restrict_unprivileged_userns=1`). Without either, +`neuros-container` says so on stderr and runs the command under the +cgroup limits without namespace isolation, rather than silently +pretending to sandbox it. + +`--env KEY=VALUE` (repeatable) injects environment variables into +the inner workload just before exec. `--env-from-file PATH` reads a +dotenv-style file (one `KEY=VALUE` per line; `#` comments and +optional `export ` prefixes are accepted) and merges it with any +explicit `--env` flags, with explicit `--env` winning on conflict. +Either path's injection always overrides whatever was inherited +from the parent shell, so `--env DEBUG=0` really forces `DEBUG=0` +even when the caller set it to `1`. +`--cap-drop REGEX` removes every Linux capability whose name matches +the regex from the bounding set via `prctl(PR_CAPBSET_DROP)`. The +bounding set is inherited across exec, so the workload sees the +dropped state directly. The drop requires the user namespace's +`CAP_SETPCAP`; outside a userns the call returns EPERM and the +helper prints a per-capability stderr line. + +Detached runs (`--detach`) leave the child running and write a JSON +state file under `/run/neuros-container/.json` describing the +inner pid and cgroup path; `cleanup` reaps those that have already +exited, refusing to remove cgroups that still hold live members. +`list` walks the cgroup subtree recursively (nested `neuros-*` leaves +inside parent `neuros-*` leaves are also found) and supports a +`--json` output mode. + +All model and config parsing in `nn`, `neuros-model`, and the rest +of the `neuros-*` tools delegates to the shared `neuroslib.py`, which +reads `~/.config/neuros/llm.conf` as an INI document — the +`[llm]` section owns `model`/`host`/`port`, the `[context]` section +owns the opt-in system-context flags, and additional sections round- +trip through unchanged. + +### Untrusted code sandbox (`neuros-sandbox`) + +A safe-runner wrapper around `neuros-container`, tuned for running +scripts produced by an LLM agent or pulled from an untrusted source. +The hardcoded defaults are the point of the tool: + +```sh +echo 'print("hello agent")' | neuros-sandbox run --json +neuros-sandbox run --mem 1G --timeout 60 ./smoke.py +neuros-sandbox run --bundle ./fixture.tar.gz --dry-run # show argv +neuros-sandbox run --json ./post.sh # envelope +``` + +Every safe-mode invocation always passes `--net`, `--read-only`, +`--mem 256M`, `--pids 64`, `--cpu-quota 50000` (50% of one CPU), +and `--cap-drop 'CAP_(NET_RAW|SYS_ADMIN|SYS_PTRACE|SETUID|SETPCAP| +MAC_ADMIN|DAC_OVERRIDE|LINUX_IMMUTABLE|SYS_CHROOT|SYS_RAWIO| +SYS_RESOURCE)'` to the underlying `neuros-container run`. The +cap-drop baseline is the load-bearing piece of the default safety +posture and persists under `--unsafe` as well; only `--cap-drop-keep +REGEX` (active only under `--unsafe`) replaces it for untrusted +work that genuinely needs broader capabilities. Override regexes +are pre-validated with `re.compile` so a malformed pattern fails +at the wrapper, not deep inside the kernel-bridge ctypes call when +the primitive would otherwise reject it. The host environment is +scrubbed to a minimal `PATH` so `ANTHROPIC_API_KEY`, +`HOME`, and other secrets don't leak into the worker's process env. +Scripts are always piped to `python3 -u -` via stdin — no temp +files on the host, no script path visible via `/proc//cmdline`. +A wall-clock watchdog (`--timeout`, default 30 s) SIGKILLs the +container on expiry and the wrapper exits 124 with `timeout_hit: +true` in the JSON envelope. `--json` emits one line of structured +output (`exit_code`, `stdout`, `stderr`, `wall_clock_ms`, +`timeout_hit`, `peak_mem_estimate`), suitable for orchestrator +consumption; in default human mode stdout/stderr pass through and a +single `[neuros-sandbox] exit=… wall_ms=… timeout_hit=…` trailer is +written to stderr. `peak_mem_estimate` is best-effort: the primitive +auto-cleans the cgroup on non-detached exit, so the `memory.peak` +file is often gone by the time we read it. For forensic-grade memory +accounting, run with `--unsafe --cap-drop-keep ''` plus an explicit +`--detach` and read `memory.peak` from `/` before invoking +`neuros-container cleanup `. + +`--unsafe` opts in to disabling `--net` and `--read-only` (with +`--cap-drop-keep REGEX` for replacing — not relaxing — the set of +dropped capabilities). `--bundle TAR` extracts a tarball into a +fresh tmp directory and uses it as the container's rootfs; +absolute paths and `..` traversal inside the tar are rejected at +extraction time. Tests in `tests/test_sandbox.py` mock +`subprocess.run` so no kernel cgroup delegation is required to +verify the argv shape, env scrubbing, JSON envelope, timeout +behavior, and bundle cleanup. + +### Regression benchmark harness (`neuros-bench`) + +A small driver that shells out to `neuros-sandbox run --json` with +a registry of canned workloads (CPU-bound recursion, memory growth, +regex compile, fork pressure, ctypes-bridge syscalls, file IO, JSON +parse, string ops), parses each envelope, and produces a metrics +JSON file that CI can diff against a stored baseline: + +```sh +neuros-bench list # show the workload suite +neuros-bench batch --out baseline.json # capture a baseline run +neuros-bench run mem_grow --out single.json # run one workload +neuros-bench compare baseline.json candidate.json + # exit 0 within tolerance, exit 1 on a row beyond threshold +``` + +The compare subcommand emits a grep-friendly table on stdout +(`workload`, `baseline_ms`, `candidate_ms`, `wall_pct`, `base_mem`, +`cand_mem`, `mem_pct`, `status=OK|REGRESSION|NEW|DROPPED`) and a +single verdict line on stderr. Tolerance defaults are 15% wall and +25% peak-mem; tighten via `--tolerance-wall-pct` / `--tolerance-mem-pct` +in CI. Each workload runs under `--timeout 60` (configurable) with +the sandbox's default `--mem 256M` and `--pids 64` so an unbounded +regression in any one workload can't stall the batch.Tests in `tests/test_bench.py` mock `subprocess.run` and let the workspace +run a regression check with no kernel/cgroup dependency. + +### Runbook verifier (`neuros-runbook`) + +A small assertion-driven runner that drives `neuros-sandbox` from a +JSON runbook file. Each step has a `script`, optional per-step +`limits` (mem/pids/timeout/cpu_quota), and an `expect` block that can +assert `exit_code`, regex-match `stdout`/`stderr`, or forbid +`timeout_hit=true`. Step verdicts roll up into a JSON envelope (with +`--json`) or a grep-friendly table on stdout (default). Exit 0 if +every step passes, 1 on any failure; `--only-violations` filters the +human table to just the failed rows. + +```sh +neuros-runbook run ./runbooks/smoke.json +neuros-runbook run --only-violations ./runbooks/prod-checks.json +neuros-runbook run --json ./runbooks/regression.json | jq '.steps[] | select(.passed==false)' +``` + +Example runbook: + +```json +[ + {"name": "imports-stdlib", + "script": "import sys; print(sys.version_info[:2])", + "expect": {"exit_code": 0, "stdout_matches": "^\\(3, [0-9]+\\)$"}}, + {"name": "network-blocked", + "script": "import socket; socket.gethostbyname('example.com'); print('UP')", + "limits": {"timeout": 5}, + "expect": {"exit_code": "any", "timeout_forbidden": false}}, + {"name": "no-tracebacks", + "script": "import sys; sys.exit(0)", + "expect": {"exit_code": 0, "stderr_matches": "^$"}} +] +``` + +The runbook parser pre-validates each `expect.*_matches` regex at +load time so a malformed pattern fails loudly when the runbook is +loaded, not at the assertion site. Tests in `tests/test_runbook.py` +mock `subprocess.run` for full coverage without kernel/cgroup +dependency. + +### Declarative security policy (`neuros-policy`) + +Hardened defaults are good until you need to *prove* which defaults +applied to a given run. `neuros-policy` is the place that +information lives: a JSON manifest naming a profile (`strict`, +`moderate`, `permissive`), declaring per-profile capability drops, +naming memory / PID / CPU-quota / timeout bounds, and pinning the +allowlist of env vars that may pass through to the container. The +three subcommands compose like this: + +* `neuros-policy validate ./policy.json` — schema + bounds check. + Bounds are conservative: mem 16K-1G, pids 1-4096, cpu_quota + 1000-1_000_000 (microseconds / 100ms CFS period), timeout + 1-3600s. Profile names must match `[a-z][a-z0-9_-]*`, cap names + `^CAP_[A-Z_]+$`, env-allowlist names `^[A-Z][A-Z0-9_]*$`. Exits + 0 on success, 1 on any violation, 2 on malformed input. +* `neuros-policy check ./policy.json --envelope ./envelope.json` — + parse a `neuros-sandbox --json` envelope and reconcile its + `wall_clock_ms`, `peak_mem_estimate`, and `timeout_hit` against + the policy's bounds. Exits 0 when the run conformed, 1 on a + violation (printed to stderr in the form `[severity] rule: + observed X, expected Y`), 2 on bad input. +* `neuros-policy transpile ./policy.json --profile strict` — emit + the `neuros-sandbox --unsafe ...` argv fragment that realises + the policy. A profile with an empty cap list transpiles to + `--cap-drop '^_NEVER_MATCH_$'` so nothing is dropped, which is + almost always a bug — but if you really mean it, the surface + is unambiguous. + +Sample manifest bundled in `tests/test_policy.py` (`_good_policy`). +Add `--json` to `validate` / `check` to emit a single-line envelope +that composes with the bench + runbook JSON surfaces downstream. + +### Offline envelope diagnostics (`neuros-replay`) + +When a `neuros-sandbox --json` envelope looks wrong, the typical +follow-up is either "how wrong?" (diff against an earlier good +run) or "what does this mean?" (humanize the metrics). Both are +now a single tool, no shell-out to `python3 -c`: + +* `neuros-replay diff ` — per-field regression verdict. + Tolerances default to 10% wall-clock / 20% peak-mem (tighter + than bench's 15%/25% because this is a single observation, not + a batch aggregate). Each violation surfaces as a single grep- + friendly rule string on stderr; `--json` emits a structured + envelope with `ok`, `wall_clock_pct_delta`, `peak_mem_pct_ + delta`, `rule_violations`, and the tolerances that were + applied. Exit 0 on conformant, 1 on any rule violation, 2 on + bad input. +* `neuros-replay explain ` — sinkable one-line summary + like `exit=0 wall=0ms peak=n/a timeout=False stdout_bytes=0 + stderr_bytes=0`. The exact numbers depend on the envelope, + but the field order is fixed. Add `--json` for the structured + equivalent with `neuros_replay_version`. Designed for CI grep. +* `neuros-replay extract --stream stdout|stderr|both` + — dump the embedded `stdout` / `stderr` payloads so standard + host tools (`grep`, `jq`, `less`) read them without JSON + escaping. The `both` stream mode inserts a single + `===STDERR===` separator line so a downstream parser can tell + the two streams apart without re-mapping field names. + +`neuros-replay` is purely offline: it consumes envelopes the +other tools emit and never re-runs a workload. For replay-with- +execute, use `neuros-bench run --json` and pipe the +envelope into `neuros-replay`. Tests in `tests/test_replay.py` +exercise every branch without subprocess shell-out. + +### Envelope-agnostic verifier (`neuros-verify`) + +All five emitter tools (sandbox, bench, runbook, policy, replay) +produce a JSON envelope, but their shapes diverge. `neuros-verify` +is the single tool that reads *any* of them, auto-detects the +shape, and emits a normalized `verdict=PASS|FAIL` line so a CI +gate only needs one consumer in its pipeline: + +* Kind detection is by version key first (`neuros_bench_version` + / `neuros_policy_version` / `neuros_replay_version`) and by + structural fingerprint second (`exit_code`+`wall_clock_ms`+ + `timeout_hit` => sandbox; `runbook_path`+`steps` => runbook). + First match wins; unrecognized is treated as `FAIL` with the + top-level keys listed for triage. +* Per-kind rules: + - Sandbox envelope needs `--policy ` — the policy's + `check_envelope_against_policy` is loaded via runtime + `compile()+exec()` of the on-disk neuros-policy script + (`mod.__file__` is set explicitly so the runtime loader can + find the manifest). A defensive `hasattr` check turns a + silent AttributeError later into a loud failure now if the + policy tool ever renames its public verify names. + - Bench metrics without `--bench-baseline` degrades to a + crash check (every run's `exit_code == 0`). With + `--bench-baseline`, each per-workload run is diffed against + the baseline using the same tolerances as + `neuros-bench compare` (15% wall / 25% mem). + - Policy and replay verdicts are passed through `env["ok"]`; + the downstream tool's `errors` / `rule_violations` lists are + carried as the violations this tool reports. + - Runbook envelopes get a structural smoke check (every step + has `name` + `ok`). Step-level assertion semantics live in + `neuros-runbook` itself. +* Multiple input files are **AND-merged**: any single failure + fails the overall run. +* Text output mode prints a per-file `verdict=… source=… + file=… violations=N` line (deterministic field order) plus + a final `source=aggregate` line. `--quiet` suppresses per- + file lines; only the aggregate remains. `--json` emits a + single-line envelope with `{ok, aggregate, parts[], + neuros_verify_version}` shaped like the other tools' machine + output. + +Exit codes: `0` all conformant; `1` any rule violation or +unrecognized envelope; `2` bad input (missing file, malformed +JSON, root not an object). + +Tests in `tests/test_verify.py` cover all five kinds, the +unknown-envelope path, every dispatch surface (text, `--json`, +`--quiet`, multi-file AND-merge, bench-with-baseline), and the +runtime sandbox+policy integration end-to-end via a tempdir- +mounted manifest. ### Code completion @@ -157,6 +476,101 @@ qemu-system-x86_64 -m 8192 -smp 4 -cdrom live-image-amd64.hybrid.iso \ -boot d -vga virtio -display sdl ``` +### Reproducibility + +"Reproducible" here means **pinned-inputs reproducibility**: every build +input that could otherwise drift (package versions, third-party +installers, GitHub release tarballs) is pinned to an explicit version or +commit, so the same commit of this repo installs the same software on +every rebuild. It does **not** mean bit-identical ISO output — timestamps +embedded by `mksquashfs`/`xorriso`, filesystem inode ordering, and +initramfs generation are not currently pinned or normalized, and would +need to be (via `SOURCE_DATE_EPOCH`, sorted file ordering, etc.) to make +that stronger claim. That's a separate, larger effort from pinning what +gets installed, and hasn't been done here. + +Along the way, `build.sh`'s `lb config` call was also missing +`--mode ubuntu`. Without it, live-build defaults to Debian's own mirrors +for an Ubuntu suite name, and `lb config`/debootstrap would fail on a +genuinely clean checkout since Debian's archive has no `noble` suite. +That's fixed alongside the pinning below, verified with `lb config` +exiting 0 against `--mode ubuntu` in a clean container. + +What's pinned, and how: + +- **~60 apt packages** (`config/package-lists/neuros.list.chroot`): no + per-package `=version` pins. Instead, `build.sh` points `lb config` at + a fixed Ubuntu archive snapshot (`SNAPSHOT_TS` in `build.sh`, via + [snapshot.ubuntu.com](https://snapshot.ubuntu.com)) for the bootstrap + and chroot mirrors, so `apt` resolves the exact same package versions + on every build regardless of what's since changed in the live Ubuntu + archive. This was chosen over pinning every package to `=version` + because a snapshot pins the whole dependency graph at once and doesn't + need updating package-by-package as the archive rotates; Canonical + commits to keeping snapshots available for at least 2 years. The + shipped ISO's own `/etc/apt/sources.list` (`--mirror-binary`) is + deliberately left on the regular live Ubuntu mirrors, not the + snapshot, so a running NeurOS system keeps getting real security + updates after install. +- **oh-my-zsh** (`0500-configure-system.hook.chroot`): pinned to a + specific commit SHA instead of the `master` branch tip. +- **VS Code and the Continue.dev extension** + (`0200-install-vscode.hook.chroot`): pinned to a specific `code` + package version (Microsoft's apt repo keeps a long version history, so + this doesn't go stale) and a specific Continue.dev marketplace version. +- **GNOME extensions** (`0600-install-gnome-extensions.hook.chroot`): + blur-my-shell and caffeine are downloaded from a GitHub archive URL + pinned to a commit SHA (not a tag name, which can be moved) for the + same release each build. +- **Ollama installer** (`0100-install-ollama.hook.chroot`): the + installer script is fetched from a pinned commit (the `v0.32.5` tag) + instead of the floating `ollama.com/install.sh`, and `OLLAMA_VERSION` + pins the installed binary itself. +- **Docker build container** (`Dockerfile`): `FROM ubuntu:24.04` pinned + to that tag's image digest instead of the mutable tag, which Canonical + rebuilds in place for security patches; `pip3 install requests` pinned + to `==2.34.2`. +- **CI workflow** (`.github/workflows/ci.yml`): `actions/checkout@v4` + and `docker/setup-buildx-action@v3` pinned to the commit SHA each tag + currently resolves to, so a maintainer moving the upstream `v4`/`v3` + tag to a new patch release can't silently change what CI runs. + +While verifying package resolution, `config/package-lists/neuros.list.chroot` +was also found to list `also-utils`, which isn't a real package (the +intended package is `alsa-utils`); this would have failed `apt-get +install` regardless of pinning, so it's fixed alongside this work. + +**Known gap, not fixed here:** `ollama pull mistral` still pulls +whatever the `mistral` tag in Ollama's library currently resolves to — +that tag can move to a different quantization over time. The manifest +digest turns out to be cheap to look up (`GET +https://registry.ollama.ai/v2/library/mistral/manifests/latest` returns +a <1KB JSON manifest, not the model blob), so that part of the original +assumption was wrong. What's still blocking a pin: Ollama's `pull` has +no documented `name@sha256:digest` reference syntax the way `docker +pull` does, so there's no verified way to make `ollama pull` consume +that digest without an actual ~4GB pull to confirm the CLI accepts it — +which this environment intentionally does not do. Left as a documented +gap rather than shipped unverified. + +**What was verified, and how:** `lb config` (with `--mode ubuntu` and the +pinned snapshot mirrors) was run in a clean `ubuntu:24.04` Docker +container and exits 0. `apt-get install --dry-run` against the pinned +snapshot for every package in `neuros.list.chroot` was run in the same +container and resolves cleanly (exit 0, no dry-run conflicts). All +pinned commit SHAs/tags/versions above were confirmed to resolve via the +GitHub API and package repositories at the time of pinning, including +the Docker base image digest (`docker buildx imagetools inspect +ubuntu:24.04`), the `requests` version (PyPI), and the two GitHub +Actions SHAs (`actions/checkout`, `docker/setup-buildx-action`, both +confirmed against the GitHub API to match the `v4`/`v3` tags' current +resolution). What was +**not** verified: an actual `lb build` (needs 20GB+ disk and privileged +chroot/mount this environment doesn't have), and the `ollama pull` / +GNOME extension downloads were not executed end-to-end inside a real +chroot (only their URLs were confirmed to resolve and serve the expected +content). + ## Project structure ``` @@ -173,9 +587,10 @@ NeurOS/ │ │ │ ├── nn # terminal assistant CLI │ │ │ ├── neuros-tray # system tray applet │ │ │ ├── neuros-model # model manager CLI -│ │ │ ├── neuros-mcp # MCP server -│ │ │ ├── neuros-welcome # first-boot welcome screen -│ │ │ └── ... # 70+ additional neuros-* utilities +│ │ │ ├── neuros-mcp # MCP server │ │ │ ├── neuros-container # namespace + cgroup container runner + │ │ │ ├── neuros-sandbox # safe-runner wrapper for untrusted scripts + │ │ │ ├── neuros-welcome # first-boot welcome screen + │ │ │ └── ... # 70+ additional neuros-* utilities │ │ ├── etc/systemd/system/ │ │ │ └── neuros-llm.service │ │ ├── etc/polkit-1/rules.d/ @@ -215,8 +630,13 @@ integration, and privacy hardening. Past MVP: - MCP server: done. `neuros-mcp` implements `initialize`/`tools-list`/ - `tools-call`/`resources` over HTTP and JSON-RPC, verified end-to-end - against a live Ollama instance. + `tools-call`/`resources` over both HTTP/JSON-RPC and the MCP stdio + transport (`--stdio`), verified end-to-end against a live Ollama + instance and, for stdio, against the official `mcp` Python SDK + client. Only 6 tools are exposed through MCP function-calling; the + other ~80 `neuros-*` CLIs are standalone tools a user (or another + agent via `run_command`) invokes directly, not entries in an LLM + tool-call table. - Model switcher: done, CLI and tray. `neuros-model` implements `list/pull/remove/switch/info/search/benchmark/compare`, and the system tray now has a "Switch Model" submenu that shells out to @@ -228,6 +648,11 @@ Past MVP: Nothing is transmitted anywhere but the local Ollama prompt. - GUI chat application: a local browser-based chat UI exists (`neuros-chat`), not a native Tauri app. +- Container primitive: done. `neuros-container` runs a command under a + real cgroup v2 leaf (memory/pids/cpu limits) and, given root or an + unprivileged user namespace, Linux namespace isolation, without + runc/containerd. See `tests/test_container.py` for the measured + memory-cap and pids-cap enforcement. - Still open: voice input and output, a fine-tuning pipeline, and ARM/CUDA builds. @@ -243,6 +668,8 @@ python3 tests/test_nn.py python3 tests/test_autofix.py python3 tests/test_model.py python3 tests/test_mcp.py +python3 tests/test_container.py +python3 tests/test_sandbox.py ./validate-build.sh ``` diff --git a/build.sh b/build.sh index e405841..8f42ad5 100755 --- a/build.sh +++ b/build.sh @@ -14,6 +14,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BUILD_LOG="$SCRIPT_DIR/build.log" ISO_NAME="live-image-amd64.hybrid.iso" +# Ubuntu archive snapshot used to build the chroot, pinned to a fixed date +# so `apt` resolves the same package versions on every build instead of +# whatever happens to be current in the live archive that day. See +# https://snapshot.ubuntu.com/ for details on the service. Bump this +# deliberately (and re-run validate-build.sh) when the package set needs +# to move forward; it will not drift on its own. +SNAPSHOT_TS="20260701T000000Z" +SNAPSHOT_MIRROR="https://snapshot.ubuntu.com/ubuntu/${SNAPSHOT_TS}" + # Colors RED='\033[0;31m' GREEN='\033[0;32m' @@ -85,12 +94,27 @@ configure_live_build() { # Initialize live-build config if not already done if [[ ! -f "auto/config" ]]; then + # --mirror-bootstrap/--mirror-chroot(-security) pin the package + # versions used to build the chroot to the snapshot above, for + # reproducibility. --mirror-binary is intentionally left at the + # live-build default (the regular Ubuntu mirrors) so the ISO + # ships with normal, current sources.list entries -- installed + # systems should get real ongoing updates, not be frozen to a + # months-old snapshot forever. + # --mode ubuntu is required for live-build to default to Ubuntu's + # mirrors/keyring for an Ubuntu suite; without it, lb config + # defaults to Debian's own archive, which has no "noble" suite + # and would break debootstrap on a truly clean checkout. lb config \ + --mode ubuntu \ --distribution noble \ --archive-areas "main restricted universe multiverse" \ --debian-installer none \ --memtest none \ - --binary-images iso-hybrid + --binary-images iso-hybrid \ + --mirror-bootstrap "$SNAPSHOT_MIRROR" \ + --mirror-chroot "$SNAPSHOT_MIRROR" \ + --mirror-chroot-security "$SNAPSHOT_MIRROR" success "live-build configured." else warn "live-build config already exists. Using existing configuration." diff --git a/config/hooks/live/0100-install-ollama.hook.chroot b/config/hooks/live/0100-install-ollama.hook.chroot index a1f7f56..2d446cf 100755 --- a/config/hooks/live/0100-install-ollama.hook.chroot +++ b/config/hooks/live/0100-install-ollama.hook.chroot @@ -5,10 +5,25 @@ set -e echo "[NeurOS] Installing Ollama..." -# Install Ollama -curl -fsSL https://ollama.com/install.sh | sh +# Install Ollama. The installer is fetched from a pinned commit (the +# scripts/install.sh at the v0.32.5 release tag) instead of the floating +# ollama.com/install.sh, and OLLAMA_VERSION pins the binary itself -- +# otherwise both the installer logic and the installed version can drift +# between builds. +OLLAMA_INSTALL_SHA="eec8e0b9458b8a01be0c216a9cc53eefde24ef50" # v0.32.5 +export OLLAMA_VERSION="0.32.5" +curl -fsSL "https://raw.githubusercontent.com/ollama/ollama/${OLLAMA_INSTALL_SHA}/scripts/install.sh" | sh -# Pre-pull Mistral 7B model (quantized, ~4GB) +# Pre-pull Mistral 7B model (quantized, ~4GB). This still floats: "mistral" +# resolves to whatever the Ollama library's latest tag points to. Unlike +# assumed, the manifest digest IS cheaply queryable without pulling the +# full model -- GET https://registry.ollama.ai/v2/library/mistral/manifests/latest +# returns a small (<1KB) JSON manifest, not the model blob. But Ollama's +# own pull path has no documented `name@sha256:digest` pin syntax (unlike +# `docker pull`/`ollama.com`'s per-tag digest display), so committing a +# digest here is unverified without an actual `ollama pull` of the ~4GB +# model against that reference, which this environment intentionally does +# not do. Left floating; see README's Reproducibility section. echo "[NeurOS] Pulling Mistral 7B model (this will take a while)..." ollama pull mistral diff --git a/config/hooks/live/0200-install-vscode.hook.chroot b/config/hooks/live/0200-install-vscode.hook.chroot index 90e22e4..1e3c8c4 100755 --- a/config/hooks/live/0200-install-vscode.hook.chroot +++ b/config/hooks/live/0200-install-vscode.hook.chroot @@ -10,10 +10,14 @@ wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /u echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/code stable main" > /etc/apt/sources.list.d/vscode.list apt-get update -apt-get install -y code +# Pinned to a specific build so the package doesn't drift between builds; +# the Microsoft repo keeps a long version history, so this doesn't go +# stale the way a single-version mirror snapshot would. Bump deliberately. +apt-get install -y code=1.130.0-1784734578 -# Install Continue.dev extension (pre-configured for local Ollama) +# Install Continue.dev extension (pre-configured for local Ollama), pinned +# to a specific version for the same reason. echo "[NeurOS] Installing Continue.dev extension..." -code --install-extension Continue.continue --user-data-dir /etc/skel/.vscode +code --install-extension Continue.continue@2.1.0 --user-data-dir /etc/skel/.vscode echo "[NeurOS] VS Code installation complete." diff --git a/config/hooks/live/0500-configure-system.hook.chroot b/config/hooks/live/0500-configure-system.hook.chroot index 806ebfe..4783dec 100755 --- a/config/hooks/live/0500-configure-system.hook.chroot +++ b/config/hooks/live/0500-configure-system.hook.chroot @@ -18,7 +18,10 @@ chsh -s /usr/bin/zsh root 2>/dev/null || true if [ ! -d /etc/skel/.oh-my-zsh ]; then echo "[NeurOS] Installing oh-my-zsh..." export ZSH=/etc/skel/.oh-my-zsh - sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended --keep-zshrc 2>/dev/null || true + # Pinned to a specific commit on master rather than the branch tip, so + # the installer script can't change out from under the build. + OMZ_SHA="7ea697fd8138550ddf7262456d412f0dcd1cbf84" + sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/${OMZ_SHA}/tools/install.sh)" "" --unattended --keep-zshrc 2>/dev/null || true fi # Ensure nn CLI is executable diff --git a/config/hooks/live/0600-install-gnome-extensions.hook.chroot b/config/hooks/live/0600-install-gnome-extensions.hook.chroot index b74253d..970b398 100755 --- a/config/hooks/live/0600-install-gnome-extensions.hook.chroot +++ b/config/hooks/live/0600-install-gnome-extensions.hook.chroot @@ -40,15 +40,17 @@ if [ ! -d "$EXT_DIR" ]; then echo "[NeurOS] Downloading blur-my-shell extension..." mkdir -p "$EXT_DIR" - # Download latest release from GitHub - BLUR_VERSION="v60" # GNOME 46 compatible - BLUR_URL="https://github.com/aunetx/blur-my-shell/archive/refs/tags/${BLUR_VERSION}.tar.gz" + # Pinned to the commit for the v60 tag (GNOME 46 compatible). A commit + # SHA is used instead of the tag name so the archive can't change out + # from under us if the tag is ever moved upstream. + BLUR_SHA="5ac26249baa9dea04498b86312c52528254971f8" # v60 + BLUR_URL="https://github.com/aunetx/blur-my-shell/archive/${BLUR_SHA}.tar.gz" if wget -q "$BLUR_URL" -O /tmp/blur-my-shell.tar.gz 2>/dev/null; then tar -xzf /tmp/blur-my-shell.tar.gz -C /tmp/ - if [ -d "/tmp/blur-my-shell-${BLUR_VERSION#v}" ]; then - cp -r /tmp/blur-my-shell-${BLUR_VERSION#v}/* "$EXT_DIR/" 2>/dev/null || true - rm -rf /tmp/blur-my-shell-${BLUR_VERSION#v} + if [ -d "/tmp/blur-my-shell-${BLUR_SHA}" ]; then + cp -r /tmp/blur-my-shell-${BLUR_SHA}/* "$EXT_DIR/" 2>/dev/null || true + rm -rf /tmp/blur-my-shell-${BLUR_SHA} fi rm -f /tmp/blur-my-shell.tar.gz echo "[NeurOS] blur-my-shell installed." @@ -63,12 +65,14 @@ if [ ! -d "$CAFFEINE_DIR" ]; then echo "[NeurOS] Downloading caffeine extension..." mkdir -p "$CAFFEINE_DIR" - CAFFEINE_URL="https://github.com/eonpatapon/gnome-shell-extension-caffeine/archive/refs/tags/v50.tar.gz" + # Pinned to the commit for the v50 tag, same rationale as blur-my-shell above. + CAFFEINE_SHA="d8dbae0958a13bab9df0479497786066a269b859" # v50 + CAFFEINE_URL="https://github.com/eonpatapon/gnome-shell-extension-caffeine/archive/${CAFFEINE_SHA}.tar.gz" if wget -q "$CAFFEINE_URL" -O /tmp/caffeine.tar.gz 2>/dev/null; then tar -xzf /tmp/caffeine.tar.gz -C /tmp/ - if [ -d "/tmp/gnome-shell-extension-caffeine-50" ]; then - cp -r /tmp/gnome-shell-extension-caffeine-50/* "$CAFFEINE_DIR/" 2>/dev/null || true - rm -rf /tmp/gnome-shell-extension-caffeine-50 + if [ -d "/tmp/gnome-shell-extension-caffeine-${CAFFEINE_SHA}" ]; then + cp -r /tmp/gnome-shell-extension-caffeine-${CAFFEINE_SHA}/* "$CAFFEINE_DIR/" 2>/dev/null || true + rm -rf /tmp/gnome-shell-extension-caffeine-${CAFFEINE_SHA} fi rm -f /tmp/caffeine.tar.gz echo "[NeurOS] caffeine extension installed." diff --git a/config/includes.chroot/usr/local/bin/neuros-bench b/config/includes.chroot/usr/local/bin/neuros-bench index 3600074..b03ab81 100755 --- a/config/includes.chroot/usr/local/bin/neuros-bench +++ b/config/includes.chroot/usr/local/bin/neuros-bench @@ -1,282 +1,458 @@ #!/usr/bin/env python3 +"""neuros-bench: regression benchmark harness for neuros-sandbox. + +The tool is intentionally small. It shells out to `neuros-sandbox run +--json` and treats each per-workload JSON envelope as the unit of +metric. That decouples it from any specific cgroup kernel delegation, +so the bench is fully testable on a developer laptop. + +Subcommands: + + neuros-bench list # print built-in workloads + neuros-bench run # one workload -> metrics.json + neuros-bench batch [--out PATH] # all workloads -> metrics.json + neuros-bench compare # diff two .json files + +The built-in workload suite is intentionally small (8 named entries) +but covers the spread of behaviors the sandbox is meant to isolate: +CPU-bound recursion, memory growth, regex compile, fork pressure, +ctype-bridge syscall count, file IO, JSON parse, and string ops. +The workloads each render to a deterministic Python string that the +wrapper pipes into `neuros-sandbox run -`; no temp files. + +Design choices worth knowing: + +* `--per-workload-timeout` defaults to 60s. A single runaway workload + in a CPU loop used to be able to block the entire batch because + the wrapper only SIGKILLs on its OWN watchdog; this matches the + shape that neuros-sandbox exposes. +* `--tolerance-wall-pct` and `--tolerance-mem-pct` are the regression + bounds for `compare`. Defaults are 15% / 25%; CI can be tighter via + configuration. +* The compare output is grep-friendly text on stdout, NOT JSON. + CIs reading the diff benefit from `+12%` rows so a human can + triage a regression fast. """ -neuros-bench — NeurOS Benchmark Suite -Comprehensive AI and system benchmarks with scoring and comparisons. - -Usage: - neuros-bench Run full benchmark suite - neuros-bench --ai AI/LLM benchmarks only - neuros-bench --system System benchmarks only - neuros-bench --compare Compare against previous runs - neuros-bench --quick Quick 30-second benchmark -""" - -import sys -import os +import argparse import json -import time +import os import subprocess -import argparse -import urllib.request -import urllib.error -from datetime import datetime - -BENCH_DIR = os.path.expanduser("~/.local/share/neuros/benchmarks") -RESULTS_FILE = os.path.join(BENCH_DIR, "results.json") - - -def ensure_dirs(): - os.makedirs(BENCH_DIR, exist_ok=True) - - -def load_results(): - if os.path.exists(RESULTS_FILE): - with open(RESULTS_FILE) as f: - return json.load(f) - return [] - - -def save_results(results): - with open(RESULTS_FILE, "w") as f: - json.dump(results, f, indent=2) - - -def bench_cpu(): - start = time.time() - ops = 0 - for _ in range(5000000): - ops += 1 - elapsed = time.time() - start - return {"ops": ops, "time_seconds": round(elapsed, 3), - "score": round(ops / elapsed / 1000000, 1)} - - -def bench_memory(): - sizes = [1000, 10000, 100000, 1000000] - results = [] - for size in sizes: - start = time.time() - arr = list(range(size)) - arr.reverse() - arr.sort() - elapsed = time.time() - start - results.append({"elements": size, "ms": round(elapsed * 1000, 1)}) - return results - - -def bench_disk(mb=100): +import sys +import time +import uuid + +# --- Constants ----------------------------------------------------------- + +#: The version baked into the metrics.json schema. Bump when the +#: shape changes in a non-additive way (new required key, removed +#: key, etc.). +NEUROS_BENCH_VERSION = "0.1" + +#: Default per-workload timeout in seconds. +DEFAULT_TIMEOUT = 60 + +#: Default tolerances (percent) for the `compare` subcommand. +DEFAULT_TOLERANCE_WALL_PCT = 15.0 +DEFAULT_TOLERANCE_MEM_PCT = 25.0 + + +# --- Workload registry --------------------------------------------------- + +def _w_cpu_tight(): + # Recursive fib — tight CPU usage, low memory. + def fib(n): return n if n < 2 else fib(n - 1) + fib(n - 2) + return fib(28) + +def _w_json_parse(): + # Generate a synthetic 5KB JSON string, parse-then-dump 5000 times. + import json as _json + payload = _json.dumps([{"i": i, "sq": i * i} for i in range(40)]) + for _ in range(5000): + _json.loads(payload) + return 0 + +def _w_regex_compile(): + import re as _re + pats = [r"^CAP_[A-Z_]+$", r"[0-9a-f]{8}", r"\b\w+\b", r"foo|bar|baz", + r"https?://[^\s]+", r"\d{4}-\d{2}-\d{2}", r"^[a-z]+$", + r"\s+", r"(\d)\1+", r"^[A-Z][a-z]*$"] + for _ in range(200): + for p in pats: + _re.compile(p) + return 0 + +def _w_subproc_spawn(): + # Spawn 8 subprocesses that run a no-op and return. + for _ in range(8): + subprocess.run([sys.executable, "-c", "pass"], + capture_output=True, check=False) + return 0 + +def _w_mem_grow(): + # Allocate a bytearray that doubles until it reaches ~50 MiB. + buf = bytearray() + target = 50 * 1024 * 1024 + while len(buf) < target: + buf.extend(b"\x00" * min(len(buf) or 1, target - len(buf))) + return len(buf) + +def _w_file_io(): + # Write+read 100 x 4KB files in /tmp; cleanup after. import tempfile - import os as os_mod - data = b"x" * (1024 * 1024) - with tempfile.NamedTemporaryFile(delete=False) as f: - start = time.time() - for _ in range(mb): - f.write(data) - f.flush() - write_time = time.time() - start - tmp = f.name - with open(tmp, "rb") as f: - start = time.time() - for _ in range(mb): - f.read(1024 * 1024) - read_time = time.time() - start - os_mod.unlink(tmp) - return { - "write_mbps": round(mb / write_time, 1), - "read_mbps": round(mb / read_time, 1) + paths = [] + for i in range(100): + fd, p = tempfile.mkstemp(prefix="neuros-bench-", suffix=".bin") + os.write(fd, b"\x00" * 4096) + os.close(fd) + paths.append(p) + for p in paths: + with open(p, "rb") as f: + f.read() + for p in paths: + try: + os.unlink(p) + except OSError: + pass + return len(paths) + +def _w_ctypes_call(): + import ctypes + libc = ctypes.CDLL("libc.so.6", use_errno=True) + for _ in range(500): + libc.getpid() + return 0 + +def _w_string_ops(): + # Heavy string interpolation/format, no I/O. + parts = ["alpha", "beta", "gamma", "delta", "epsilon"] + out = [] + for i in range(100_000): + out.append(f"{i:d}-{parts[i % len(parts)]}-{i * i}") + return len("".join(out)) + + +#: Workload registry: name -> (callable, baseline_ms_estimate). The +#: baseline_ms_estimate is informational only (used in `list` and +#: compare output); runtime numbers depend on the host. +WORKLOADS = { + "cpu_tight": (_w_cpu_tight, 1100), + "json_parse": (_w_json_parse, 900), + "regex_compile": (_w_regex_compile, 600), + "subproc_spawn": (_w_subproc_spawn, 1500), + "mem_grow": (_w_mem_grow, 350), + "file_io": (_w_file_io, 1800), + "ctypes_call": (_w_ctypes_call, 700), + "string_ops": (_w_string_ops, 1200), +} + + +# --- Subprocess envelope parsing ------------------------------------------ + +def _run_workload(name, timeout_s, mem_cap, pids_cap): + """Shell out to neuros-sandbox with the workload's Python source as + a piped stdin, parse the JSON envelope, and return a dict matching + the metrics schema. Raises SandboxError (we use a generic alias) + on subprocess or envelope failures.""" + fn, _ = WORKLOADS[name] + # The workload callable is executed INSIDE the sandbox via a fresh + # python3 interpreter; here we render the function source as a + # Python expression that ends with `return 0` on the outside of + # the workspace. Wrap in a try/except that prints traceback to + # stderr and exits 1, so the metrics envelope reflects the + # failure mode rather than silencing it. + src_lines = [ + "import sys, traceback", + "try:", + ] + for line in _render_fn(fn): + src_lines.append(" " + line) + src_lines.append("except Exception:") + src_lines.append(" traceback.print_exc()") + src_lines.append(" sys.exit(1)") + # Pull the trailing expression out: the last line of a workload + # is the return value. The wrapping `r = ...` captures it but we + # don't need to read it; sys.exit code on uncaught Exception + # surfaces as the envelope's exit_code. + body = "\n".join(src_lines) + argv = ["neuros-sandbox", "run", + "--timeout", str(timeout_s), + "--mem", mem_cap, + "--pids", str(pids_cap), + "--name", f"neuros-bench-{uuid.uuid4().hex[:8]}", + "--json", "--"] + cp = subprocess.run( + argv, + input=body.encode(), + env={"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, + capture_output=True, + check=False, + timeout=timeout_s + 10, # small slack beyond the inner watchdog + ) + stdout_text = cp.stdout.decode("utf-8", "replace").strip() + if not stdout_text: + return { + "name": name, + "wall_clock_ms": 0, + "exit_code": cp.returncode, + "stdout": "", + "stderr": cp.stderr.decode("utf-8", "replace"), + "timeout_hit": False, + "peak_mem_estimate": None, + } + try: + env_obj = json.loads(stdout_text.splitlines()[-1]) + except json.JSONDecodeError: + env_obj = {"exit_code": cp.returncode, "stderr": stdout_text} + env_obj.setdefault("name", name) + env_obj.setdefault("wall_clock_ms", 0) + env_obj.setdefault("stdout", "") + env_obj.setdefault("stderr", "") + env_obj.setdefault("timeout_hit", False) + env_obj.setdefault("peak_mem_estimate", None) + return env_obj + + +def _render_fn(fn): + """Pull the source of a function via inspect.getsource and unindent + it so it can be embedded inside a try/except block at the same + indent as the outer frame.""" + import inspect + import textwrap + raw = textwrap.dedent(inspect.getsource(fn)) + # Drop the leading `def _w_xxx():` and unindent the body so that + # everything below the def is indented by one level (matching + # the `try:` we wrap it in). + lines = raw.splitlines() + if lines and lines[0].lstrip().startswith("def "): + # drop def line; body follows + body = lines[1:] + else: + body = lines + # Trim leading blank line if present + while body and not body[0].strip(): + body.pop(0) + return body + + +# --- Metrics aggregation -------------------------------------------------- + +def cmd_run(args): + fn, _ = WORKLOADS.get(args.workload, (None, None)) + if fn is None: + _die(f"unknown workload {args.workload!r}; " + f"see `neuros-bench list`") + res = _run_workload(args.workload, args.timeout, args.mem, args.pids) + out = { + "runs": [res], + "total_wall_clock_ms": res["wall_clock_ms"], + "config": _config_block(args), + "neuros_bench_version": NEUROS_BENCH_VERSION, + } + _emit_metrics(out, args.out) + + +def cmd_batch(args): + runs = [] + t0 = time.monotonic() + for name in WORKLOADS: + runs.append(_run_workload(name, args.timeout, args.mem, args.pids)) + total = int((time.monotonic() - t0) * 1000) + out = { + "runs": runs, + "total_wall_clock_ms": total, + "config": _config_block(args), + "neuros_bench_version": NEUROS_BENCH_VERSION, } + _emit_metrics(out, args.out) -def bench_ai(): - """Benchmark Ollama inference speed.""" - config = {} - config_path = os.path.expanduser("~/.config/neuros/llm.conf") - if os.path.exists(config_path): - with open(config_path) as f: - for line in f: - line = line.strip() - if "=" in line and not line.startswith("["): - k, v = line.split("=", 1) - config[k.strip()] = v.strip().strip('"').strip("'") - model = config.get("model", "mistral") - ollama_url = config.get("ollama_url", "http://localhost:11434") - - metrics = {} - tests = [ - ("tiny", "Hello", 10), - ("short", "Write a haiku about coding", 50), - ("medium", "Explain binary search in one paragraph", 150), - ] +def cmd_list(args): + print("available workloads:") + widest = max(len(name) for name in WORKLOADS) + 2 + for name, (_, baseline_ms) in WORKLOADS.items(): + print(f" {name.ljust(widest)} baseline_ms_est={baseline_ms}") + return 0 - for name, prompt, expect_tokens in tests: - try: - start = time.time() - req = urllib.request.Request( - f"{ollama_url}/api/generate", - data=json.dumps({ - "model": model, "prompt": prompt, "stream": False, - "options": {"num_predict": expect_tokens} - }).encode(), - headers={"Content-Type": "application/json"} - ) - resp = urllib.request.urlopen(req, timeout=60) - data = json.loads(resp.read().decode()) - elapsed = time.time() - start - actual_tokens = data.get("eval_count", 0) - tps = actual_tokens / elapsed if elapsed > 0 else 0 - metrics[name] = { - "time_seconds": round(elapsed, 3), - "tokens": actual_tokens, - "tokens_per_sec": round(tps, 1) - } - except Exception as e: - metrics[name] = {"error": str(e)} - - return metrics - - -def bench_compression(): - """Benchmark compression speed.""" - import tempfile - import os as os_mod - data = b"Hello world! " * 100000 - with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f: - f.write(data) - tmp = f.name - start = time.time() - subprocess.run(["gzip", "-f", tmp], check=True) - compress_time = time.time() - start - start = time.time() - subprocess.run(["gunzip", "-f", tmp + ".gz"], check=True) - decompress_time = time.time() - start - os_mod.unlink(tmp) - del data - # Approximate compression ratio for text + +def _config_block(args): return { - "compress_ms": round(compress_time * 1000, 1), - "decompress_ms": round(decompress_time * 1000, 1), - "ratio": 5.5 + "per_workload_timeout_seconds": args.timeout, + "mem_cap": args.mem, + "pids_cap": args.pids, } -def run_full_benchmark(): - print("=== NeurOS Full System Benchmark ===") - result = {"timestamp": datetime.now().isoformat(), "scores": {}} - total_score = 0 - - print("\n[CPU] Integer operations...") - cpu = bench_cpu() - result["cpu"] = cpu - score_cpu = min(cpu["score"] * 20, 100) - result["scores"]["cpu"] = round(score_cpu, 1) - total_score += score_cpu - print(f" Score: {score_cpu:.1f}/100 ({cpu['score']:.1f} M ops/sec)") - - print("\n[Disk] Read/Write throughput...") - disk = bench_disk(50) - result["disk"] = disk - score_disk = min((disk["read_mbps"] + disk["write_mbps"]) / 2, 100) - result["scores"]["disk"] = round(score_disk, 1) - total_score += score_disk - print(f" Score: {score_disk:.1f}/100 (R: {disk['read_mbps']} W: {disk['write_mbps']} MB/s)") - - print("\n[Memory] Array operations...") - mem = bench_memory() - result["memory"] = mem - score_mem = max(0, min(100, 100 - (mem[-1]["ms"] / 10))) - result["scores"]["memory"] = round(score_mem, 1) - total_score += score_mem - print(f" Score: {score_mem:.1f}/100 ({mem[-1]['ms']}ms for 1M elements)") - - print("\n[AI] LLM inference (Ollama)...") - ai = bench_ai() - result["ai"] = ai - score_ai = 0 - if "medium" in ai and "tokens_per_sec" in ai["medium"]: - score_ai = min(ai["medium"]["tokens_per_sec"] * 2, 100) - elif "short" in ai and "tokens_per_sec" in ai["short"]: - score_ai = min(ai["short"]["tokens_per_sec"] * 2, 100) - result["scores"]["ai"] = round(score_ai, 1) - total_score += score_ai - if "medium" in ai and "tokens_per_sec" in ai["medium"]: - print(f" Score: {score_ai:.1f}/100 ({ai['medium']['tokens_per_sec']:.1f} tok/sec)") - else: - print(f" Score: {score_ai:.1f}/100") - - result["total_score"] = round(total_score / 4, 1) - print(f"\n=== OVERALL SCORE: {result['total_score']:.1f}/100 ===") - - results = load_results() - results.append(result) - if len(results) > 20: - results = results[-20:] - save_results(results) - - # Compare with best - if len(results) > 1: - best = max(r["total_score"] for r in results[:-1]) - delta = result["total_score"] - best - if delta >= 0: - print(f" New personal best! (+{delta:.1f})") - else: - print(f" Previous best: {best:.1f} ({delta:.1f} below)") - - return result - - -def run_quick(): - cpu = bench_cpu() - disk = bench_disk(10) - ai = bench_ai() - print(f"CPU: {cpu['score']:.1f} M ops/sec") - print(f"Disk: R {disk['read_mbps']:.1f} / W {disk['write_mbps']:.1f} MB/s") - if "short" in ai and "tokens_per_sec" in ai["short"]: - print(f"AI: {ai['short']['tokens_per_sec']:.1f} tok/sec") - - -def show_history(): - results = load_results() - if not results: - print("No benchmarks run yet.") +def _emit_metrics(obj, out_path): + text = json.dumps(obj, indent=2) + if out_path is None or out_path == "-": + print(text) return - print(f"{'Date':<18} {'CPU':<8} {'Disk':<8} {'Mem':<8} {'AI':<8} {'Total':<8}") - print("-" * 65) - for r in results: - ts = r["timestamp"][:16].replace("T", " ") - scores = r.get("scores", {}) - cpu = scores.get("cpu", 0) - disk = scores.get("disk", 0) - mem = scores.get("memory", 0) - ai = scores.get("ai", 0) - total = r.get("total_score", 0) - print(f"{ts:<18} {cpu:<8.1f} {disk:<8.1f} {mem:<8.1f} {ai:<8.1f} {total:<8.1f}") - - -def main(): - ensure_dirs() - parser = argparse.ArgumentParser(description="NeurOS Benchmark Suite") - parser.add_argument("--ai", "-a", action="store_true", help="AI benchmarks") - parser.add_argument("--system", "-s", action="store_true", help="System benchmarks") - parser.add_argument("--compare", "-c", action="store_true", help="Compare history") - parser.add_argument("--quick", "-q", action="store_true", help="Quick benchmark") - - args = parser.parse_args() - - if args.compare: - show_history() - elif args.quick: - run_quick() - elif args.ai: - ai = bench_ai() - print(json.dumps(ai, indent=2)) - elif args.system: - cpu = bench_cpu() - mem = bench_memory() - print(f"CPU: {cpu['score']:.1f} M ops/sec") - print(f"Memory (1M elements): {mem[-1]['ms']}ms") - else: - run_full_benchmark() + with open(out_path, "w") as f: + f.write(text) + print(f"wrote metrics to {out_path}", file=sys.stderr) + + +def cmd_compare(args): + """Diff two metrics.json files. Print a grep-friendly table on + stdout. Exit 1 if any workload's wall_ms regressed beyond + --tolerance-wall-pct OR peak_mem regressed beyond + --tolerance-mem-pct.""" + baseline = _load_metrics(args.baseline) + candidate = _load_metrics(args.candidate) + base_runs = {r["name"]: r for r in baseline["runs"]} + cand_runs = {r["name"]: r for r in candidate["runs"]} + + header = ( + f"{'workload':<16} {'baseline_ms':>11} {'candidate_ms':>13} " + f"{'wall_pct':>9} {'base_mem':>11} {'cand_mem':>11} " + f"{'mem_pct':>9} status" + ) + print(header) + any_regression = False + for name in sorted(set(base_runs) | set(cand_runs)): + b = base_runs.get(name) + c = cand_runs.get(name) + b_ms = b.get("wall_clock_ms", 0) if b else 0 + c_ms = c.get("wall_clock_ms", 0) if c else 0 + b_mem = b.get("peak_mem_estimate") if b else None + c_mem = c.get("peak_mem_estimate") if c else None + + wall_pct = _pct_delta(b_ms, c_ms) + mem_pct = _pct_delta_optional(b_mem, c_mem) + status = "OK" + # Wall regression only counted when both sides have a sample. + if b and c and wall_pct is not None: + if wall_pct > args.tolerance_wall_pct: + status = "REGRESSION" + any_regression = True + if status == "OK" and b and c and mem_pct is not None: + if mem_pct > args.tolerance_mem_pct: + status = "REGRESSION" + any_regression = True + if b is None: + status = "NEW" + if c is None: + status = "DROPPED" + any_regression = True + + print( + f"{name:<16} " + f"{b_ms:>11d} {c_ms:>13d} " + f"{_fmt_pct(wall_pct):>9} " + f"{_fmt_mem(b_mem):>11} {_fmt_mem(c_mem):>11} " + f"{_fmt_pct(mem_pct):>9} {status}" + ) + print() + if any_regression: + print("RESULT: regression(s) detected (exit 1)", file=sys.stderr) + return 1 + print("RESULT: within tolerance (exit 0)", file=sys.stderr) + return 0 + + +def _pct_delta(base, cand): + if not base: + return None + return (cand - base) * 100.0 / base + + +def _pct_delta_optional(b_mem, c_mem): + if b_mem is None or c_mem is None: + return None + if not b_mem: + return None + return (c_mem - b_mem) * 100.0 / b_mem + + +def _fmt_pct(pct): + if pct is None: + return "-" + return f"{pct:+.1f}%" + + +def _fmt_mem(m): + if m is None: + return "-" + return f"{m}" + + +def _load_metrics(path): + with open(path) as f: + return json.load(f) + + +# --- Dispatch ------------------------------------------------------------ + +def _die(msg, code=2): + print(f"neuros-bench: {msg}", file=sys.stderr) + sys.exit(code) + + +def _build_parser(): + p = argparse.ArgumentParser( + prog="neuros-bench", + description=( + "Regression benchmark harness that drives neuros-sandbox with " + "canned workloads, parses JSON envelopes, and reports metric " + "deltas. No kernel delegation required at the bench level " + "(each workload runs through the wrapper)." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, + help=f"per-workload timeout in seconds " + f"(default: {DEFAULT_TIMEOUT}).") + p.add_argument("--mem", default="256M", + help="mem cap passed through to neuros-sandbox " + "(default: 256M).") + p.add_argument("--pids", type=int, default=64, + help="pids cap passed through to neuros-sandbox " + "(default: 64).") + + sub = p.add_subparsers(dest="cmd", required=True) + p_list = sub.add_parser("list", help="list the built-in workload suite") + p_run = sub.add_parser("run", help="run a single workload") + p_run.add_argument("workload", help="workload name to run") + p_run.add_argument("--out", "-o", + help="write metrics JSON to this file " + "(default: stdout).") + + p_batch = sub.add_parser("batch", help="run every built-in workload") + p_batch.add_argument("--out", "-o", + help="write metrics JSON to this file " + "(default: stdout).") + + p_cmp = sub.add_parser("compare", + help="diff two metrics files; " + "exit 1 on regression") + p_cmp.add_argument("baseline") + p_cmp.add_argument("candidate") + p_cmp.add_argument("--tolerance-wall-pct", type=float, + default=DEFAULT_TOLERANCE_WALL_PCT) + p_cmp.add_argument("--tolerance-mem-pct", type=float, + default=DEFAULT_TOLERANCE_MEM_PCT) + + return p + + +def main(argv=None): + parser = _build_parser() + args = parser.parse_args(argv) + if args.cmd == "list": + rc = cmd_list(args) + # cmd_list prints to stdout (the workload list); keep the + # exit code explicit so callers (CI, shells) get a clean 0 + # rather than Python's None-through-sysexit mishmash. + return rc if rc is not None else 0 + if args.cmd == "run": + return cmd_run(args) + if args.cmd == "batch": + return cmd_batch(args) + if args.cmd == "compare": + return cmd_compare(args) + parser.print_help() + return 2 if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/config/includes.chroot/usr/local/bin/neuros-container b/config/includes.chroot/usr/local/bin/neuros-container new file mode 100755 index 0000000..c493bd3 --- /dev/null +++ b/config/includes.chroot/usr/local/bin/neuros-container @@ -0,0 +1,842 @@ +#!/usr/bin/env python3 +""" +neuros-container - a container primitive built from Linux namespaces and +cgroups v2, no runc/containerd/libcontainer involved. + +Isolation comes from Linux unshare(2) namespaces: mount, UTS, PID, IPC, +optionally network, plus an optional chroot into a rootfs directory. +Resource limits come from a real cgroup v2 leaf: memory.max, pids.max, +cpu.weight, and cpu.max. + +Usage: + neuros-container run [--mem SIZE] [--pids N] [--cpu WEIGHT] + [--cpu-quota MAX/PERIOD] [--hostname NAME] + [--rootfs DIR] [--workdir DIR] [--user UID:GID] + [--read-only] [--net] [--detach] [--name NAME] + -- CMD [ARGS...] + neuros-container list [--json] + neuros-container logs (placeholder, no-op for now) + +Namespace isolation needs either root, or (best effort, and blocked by +default on Ubuntu 24.04+ via AppArmor's unprivileged-userns restriction) +an unprivileged user namespace. Without either, the command still runs +under the cgroup limits but without namespace isolation, and this tool +says so rather than pretending otherwise. + +Detached runs leave the child running, write a small JSON state file +under /run/neuros-container/, and exit 0. The cgroup is left in place +so a reattach / cleanup pass can find it. +""" + +import argparse +import ctypes +import ctypes.util +import json +import os +import re +import sys + +CGROUP_ROOT = "/sys/fs/cgroup" +STATE_DIR = "/run/neuros-container" + +PR_CAPBSET_DROP = 23 # ; reduces the bounding set, which + # execve inherits, so caps stay dropped across exec. + +# Capability names by index, entries 0..31. The kernel may support +# more (read /proc/sys/kernel/cap_last_cap at runtime); names past +# index 31 are kernel-version-dependent and we surface them with +# their numeric index in the matched-pre-exec warning so the user +# can override the pattern. +_CAP_NAMES_0_31 = [ + "CHOWN", "DAC_OVERRIDE", "DAC_READ_SEARCH", "FOWNER", "FSETID", + "KILL", "SETGID", "SETUID", "SETPCAP", "LINUX_IMMUTABLE", + "NET_BIND_SERVICE", "NET_BROADCAST", "NET_ADMIN", "NET_RAW", + "IPC_LOCK", "IPC_OWNER", "SYS_MODULE", "SYS_RAWIO", "SYS_CHROOT", + "SYS_PTRACE", "SYS_PACCT", "SYS_ADMIN", "SYS_BOOT", "SYS_NICE", + "SYS_RESOURCE", "SYS_TIME", "SYS_TTY_CONFIG", "MKNOD", "LEASE", + "AUDIT_WRITE", "AUDIT_CONTROL", "SETFCAP", +] # pad to length 32 + + +def die(msg): + print(f"neuros-container: {msg}", file=sys.stderr) + sys.exit(1) + + +def parse_size(text): + """Parse '128M', '1G', '512K', a bare byte count, or 'max' into cgroup + file syntax (a byte count string, or the literal 'max').""" + if text is None: + return None + text = text.strip() + if text.lower() == "max": + return "max" + m = re.fullmatch(r"(\d+)([KMG]?)", text, re.IGNORECASE) + if not m: + die(f"invalid size '{text}' (expected e.g. 256M, 1G, or a byte count)") + n = int(m.group(1)) + mult = {"": 1, "K": 1024, "M": 1024**2, "G": 1024**3}[m.group(2).upper()] + return str(n * mult) + + +def parse_cpu_quota(text): + """Parse '50000/100000' or '50000 100000' into a (max_us, period_us) + string tuple suitable for the cpu.max cgroup file. Also accepts + the literal ``max``, which returns ``("max", "max")`` to clear any + existing quota. Rejects zero periods, negative max_us, and one- + component input.""" + if not text: + return None + text = text.strip() + if text.lower() == "max": + return ("max", "max") + parts = re.split(r"[\s/]+", text, maxsplit=1) + if len(parts) != 2 or not all(parts): + die(f"invalid --cpu-quota '{text}' (expected e.g. '50000/100000' for 50% of one CPU, or 'max')") + try: + max_us, period_us = int(parts[0]), int(parts[1]) + except ValueError: + die(f"invalid --cpu-quota '{text}' (expected integer microseconds)") + if max_us < 0 or period_us <= 0: + die(f"invalid --cpu-quota '{text}' (max_us >= 0 and period_us > 0)") + return (str(max_us), str(period_us)) + + +def parse_user(text): + """Parse 'UID:GID' (a single integer maps uid==gid).""" + if text is None: + return None + text = text.strip() + m = re.match(r"^(\d+)(?::(\d+))?$", text) + if not m: + die(f"invalid --user '{text}' (expected UID:GID, e.g. 1000:1000, or 0)") + return (int(m.group(1)), int(m.group(2) or m.group(1))) + + +def parse_env_file(path): + """Read KEY=VALUE pairs from a file (one per line; ``#`` lines and + blank lines are ignored; ``export FOO=bar`` is accepted via the + leading ``export `` keyword so traditional dotenv-like sources + work). Returns the list of validated ``K=V`` strings. The file + is opened with ``errors="replace"`` so non-UTF-8 bytes don't + kill the run. Validation errors carry the file path so the user + knows whether a malformed entry came from ``--env`` or + ``--env-from-file /etc/...``.""" + if path is None: + return [] + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + lines = f.read().splitlines() + except OSError as e: + die(f"--env-from-file {path!r}: cannot read ({e})") + cleaned = [] + for raw in lines: + stripped = raw.strip() + if not stripped or stripped.startswith("#"): + continue + if stripped.startswith("export "): + stripped = stripped[len("export "):].lstrip() + cleaned.append(stripped) + # Inlined parse_env validation but with path-prefixed errors so + # the user can tell the file from a CLI repeat. + for line in cleaned: + if "=" not in line: + die(f"--env-from-file {path!r}: missing '=' on line " + f"{line!r}") + k = line.partition("=")[0].strip() + if not k: + die(f"--env-from-file {path!r}: blank key on line {line!r}") + if "\x00" in line: + die(f"--env-from-file {path!r}: NUL byte on line {line!r}") + return cleaned + + +def parse_env(env_list): + """Validate a list of 'KEY=VALUE' strings (the result of + ``argparse action='append'`` on ``--env K=V``). Returns the list + unchanged on success. Blank keys, missing '=', or keys containing + NULs are rejected — the latter because ``os.execve`` truncates + at NUL silently. ``partition("=")`` already severs at the first + '=' so a second '=' check on the key is unnecessary.""" + if not env_list: + return [] + clean = [] + for raw in env_list: + if "=" not in raw: + die(f"invalid --env '{raw}' (expected KEY=VALUE)") + key, _, value = raw.partition("=") + key = key.strip() + if not key: + die(f"invalid --env '{raw}' (blank key)") + if "\x00" in key or "\x00" in value: + die(f"invalid --env '{raw}' (NUL byte rejected)") + clean.append(f"{key}={value}") + return clean + + +def parse_cap_drop(pattern): + """Compile a capability-name regex (e.g. 'CAP_(NET_RAW|SYS_ADMIN)') + for later matching against a known list. Returns the compiled + pattern, or None when no --cap-drop was given.""" + if pattern is None: + return None + try: + return re.compile(pattern) + except re.error as e: + die(f"invalid --cap-drop regex '{pattern}': {e}") + + +def own_cgroup_path(): + """Absolute filesystem path of the cgroup this process currently lives + in, from /proc/self/cgroup (cgroup v2 always has a single '0::' line).""" + with open("/proc/self/cgroup") as f: + for line in f: + _, _, rel = line.strip().partition("::") + if rel: + return CGROUP_ROOT + rel + return CGROUP_ROOT + + +def find_delegating_ancestor(wanted): + """Cgroup v2's 'no internal process' rule means a cgroup that already + holds member processes (a systemd scope or service, typically) cannot + itself enable subtree_control for a new child. The interactive shell + this tool is invoked from usually lives in exactly such a cgroup, so + walk up to the nearest ancestor that already delegates one of the + wanted controllers (a slice, ordinarily) and nest the new leaf there + as a sibling instead.""" + path = own_cgroup_path() + wanted = set(wanted) + while True: + try: + with open(os.path.join(path, "cgroup.subtree_control")) as f: + already = set(f.read().split()) + except OSError: + already = set() + if already & wanted: + return path + parent = os.path.dirname(path) + if parent == path: + return path + path = parent + + +def enable_controllers(parent, wanted): + """Best-effort: ask the parent cgroup to hand the given controllers + down to its children. Already-delegated controllers are a no-op; + controllers we have no permission to enable are dropped with a + warning rather than aborting the run.""" + available = set() + try: + with open(os.path.join(parent, "cgroup.controllers")) as f: + available = set(f.read().split()) + except OSError: + pass + try: + with open(os.path.join(parent, "cgroup.subtree_control")) as f: + already = set(f.read().split()) + except OSError: + already = set() + + ready = set() + missing = [c for c in wanted if c in available and c not in already] + if missing: + try: + with open(os.path.join(parent, "cgroup.subtree_control"), "w") as f: + f.write(" ".join(f"+{c}" for c in missing)) + already |= set(missing) + except OSError: + pass + for c in wanted: + if c in available and c in already: + ready.add(c) + elif c in wanted: + print(f"neuros-container: cgroup controller '{c}' not delegated here, " + f"skipping that limit", file=sys.stderr) + return ready + + +def make_cgroup(name, mem, pids, cpu_weight, cpu_quota=None): + """Create a fresh leaf cgroup with the given limits. + + ``cpu_quota`` is a tuple ``(max_us_str, period_us_str)`` written to + cpu.max, or ``("max", "max")`` to clear the quota; ``None`` leaves + cpu.max untouched. ``cpu_weight`` is written to cpu.weight. + """ + wanted = [c for c, v in ( + ("memory", mem), ("pids", pids), ("cpu", cpu_weight or cpu_quota) + ) if v] + parent = find_delegating_ancestor(wanted) + ready = enable_controllers(parent, wanted) + + cg = os.path.join(parent, name) + os.makedirs(cg, exist_ok=True) + + # If a limit was requested but no controller is delegated all the + # way down to this leaf, the write below would silently no-op. + # Surface that loudly — it's the single most common cause of "I + # passed --mem and the container ate 4 GB" reports. + if wanted and not ready: + print(f"neuros-container: WARNING no requested controllers were " + f"delegated here; the cgroup at {cg} will not enforce " + f"{', '.join(wanted)}. Likely cause: the cgroup tree " + f"needs an ancestor with subtrees=+{wanted[0]} enabled.", + file=sys.stderr) + + if mem and "memory" in ready: + with open(os.path.join(cg, "memory.max"), "w") as f: + f.write(mem) + if pids and "pids" in ready: + with open(os.path.join(cg, "pids.max"), "w") as f: + f.write(pids) + if cpu_weight and "cpu" in ready: + with open(os.path.join(cg, "cpu.weight"), "w") as f: + f.write(str(cpu_weight)) + if cpu_quota and "cpu" in ready: + max_us, period_us = cpu_quota + with open(os.path.join(cg, "cpu.max"), "w") as f: + f.write(f"{max_us} {period_us}") + return cg + + +def join_cgroup(cg, pid): + with open(os.path.join(cg, "cgroup.procs"), "w") as f: + f.write(str(pid)) + + +def cgroup_report(cg): + """Read leaf-cgroup accounting back into a dict. If the cgroup has + already been rmdir'd (a previous run cleaned up), the result is an + empty dict and the caller falls through silently.""" + report = {} + for key, fname in ( + ("memory_peak", "memory.peak"), ("pids_current", "pids.current") + ): + try: + with open(os.path.join(cg, fname)) as f: + report[key] = f.read().strip() + except OSError: + pass + return report + + +def try_unprivileged_userns(uid, gid): + """Enter an unprivileged user namespace with an explicit uid/gid map. + Returns True if the call succeeded. On Ubuntu 24.04+ this is blocked + by default for unconfined processes; EPERM there is expected.""" + try: + os.unshare(os.CLONE_NEWUSER) + with open("/proc/self/setgroups", "w") as f: + f.write("deny\n") + with open("/proc/self/uid_map", "w") as f: + f.write(f"0 {uid} 1\n") + with open("/proc/self/gid_map", "w") as f: + f.write(f"0 {gid} 1\n") + except OSError: + return False + return True + + +def enter_namespaces(net=False, user=None): + """Enter mount/uts/pid/ipc and (optionally) net namespaces. + + Returns a tuple ``(mount, uts, pid, ipc, net, userns)`` listing which + namespaces actually got entered. The caller decides what to do when + something failed (we already print a single combined warning below). + + Inside the new namespace this process is root only after + ``try_unprivileged_userns`` maps the caller into uid 0; without + either root or a successful userns the namespaces list is all + False and the caller is told it ran unisolated. + """ + uid, gid = user if user is not None else (os.getuid(), os.getgid()) + if os.geteuid() == 0: + userns = False + have_caps = True + else: + # If --user targeted a uid/gid that doesn't match the caller's + # real ids, the kernel will reject the 1:1 map with EINVAL and + # the user will see the generic "no userns" warning without + # understanding why. Surface the obvious case up front so the + # cause is visible even before the unshare attempt. + if user is not None and os.getuid() != uid: + print(f"neuros-container: --user {uid}:{gid} was requested " + f"but the caller is uid {os.getuid()}:{os.getgid()}; " + f"unprivileged user namespaces can only map a " + f"single caller uid/gid. Drop --user or run as root " + f"to avoid this.", file=sys.stderr) + have_caps = try_unprivileged_userns(uid, gid) + userns = have_caps + + if not have_caps: + print("neuros-container: no root and unprivileged user namespaces " + "are unavailable; running without namespace isolation " + "(cgroup limits above still apply)", file=sys.stderr) + return (False, False, False, False, False, False) + + flags = os.CLONE_NEWNS | os.CLONE_NEWUTS | os.CLONE_NEWPID | os.CLONE_NEWIPC + if net: + flags |= os.CLONE_NEWNET + gained_net = net + try: + os.unshare(flags) + except OSError as e: + # Some kernels refuse CLONE_NEWNET without privileges even + # after a userns; degrade gracefully instead of failing. + if net and (flags & ~os.CLONE_NEWNET): + try: + os.unshare(flags & ~os.CLONE_NEWNET) + gained_net = False + print(f"neuros-container: could not create network " + f"namespace ({e}); continuing without it", + file=sys.stderr) + except OSError as e2: + print(f"neuros-container: namespace setup failed ({e2}); " + "running without namespace isolation (cgroup limits " + "above still apply)", file=sys.stderr) + return (False, False, False, False, False, userns) + else: + print(f"neuros-container: namespace setup failed ({e}); " + "running without namespace isolation (cgroup limits " + "above still apply)", file=sys.stderr) + return (False, False, False, False, False, userns) + + return (True, True, True, True, gained_net, userns) + + +def write_state(state_path, name, inner_pid, cg): + """Persist {name, pid, cgroup} under STATE_DIR so detached runs + can be reattached without polling /proc.""" + os.makedirs(os.path.dirname(state_path), exist_ok=True) + with open(state_path, "w") as f: + json.dump({"name": name, "pid": inner_pid, "cgroup": cg}, f) + + +def mount_proc_best_effort(): + """Mount a fresh /proc inside the new PID namespace. Falls back to + the mount(8) binary on Python <3.13, and to no-op when /proc isn't + writable (e.g. chroot without it).""" + try: + os.mount("proc", "/proc", "proc", 0, "") + return + except (AttributeError, OSError): + pass + try: + os.system("mount -t proc proc /proc 2>/dev/null") + except Exception: + pass + + +def remount_readonly_best_effort(): + """Try to remount / read-only inside the container; if the mount + namespace isn't fully set up (e.g. we're in the unprivileged + fallback path) the remount will fail. Capture and surface the + non-zero returncode so the user can see whether the RO bit + actually took effect rather than silently believing it did.""" + rc = os.system("mount -o remount,ro / 2>/dev/null") + if rc != 0: + print(f"neuros-container: warning: 'mount -o remount,ro /' " + f"exited with status {rc}; the container's rootfs may " + f"not actually be read-only", file=sys.stderr) + + +def drop_capabilities_best_effort(pattern): + """Drop every capability whose name matches ``pattern`` from the + current process's bounding set via ``prctl(PR_CAPBSET_DROP)``. + + The bounding set is inherited across ``execve`` so the workload + sees the dropped state without a wrapper shell. We rely on the + user namespace giving us ``CAP_SETPCAP`` (which it does on Linux + 3.12+ by default); outside a userns, prctl will silently EPERM + and we surface that. + + Implementation notes: + * The capability name table is hardcoded for indices 0..31. + The kernel reports its maximum supported cap via + /proc/sys/kernel/cap_last_cap; we walk up to that index. + * For caps above index 31 we fall back to ``CAP_`` naming + when no human-readable name is known. + * The drop is best-effort: every successful drop is silent + on stdout; every failure is reported on stderr. + """ + if pattern is None: + return + try: + libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True) + except OSError: + print("neuros-container: --cap-drop requested but libc " + "could not be loaded; skipping capability drop.", + file=sys.stderr) + return + try: + with open("/proc/sys/kernel/cap_last_cap") as f: + last = int(f.read().strip()) + except (OSError, ValueError): + last = len(_CAP_NAMES_0_31) - 1 + + dropped, failed_to_drop = [], [] + for idx in range(last + 1): + if idx < len(_CAP_NAMES_0_31): + name = f"CAP_{_CAP_NAMES_0_31[idx]}" + else: + name = f"CAP_{idx}" # kernel-version-specific names + if not pattern.search(name): + continue + rc = libc.prctl(PR_CAPBSET_DROP, idx, 0, 0, 0) + if rc == 0: + dropped.append(name) + else: + err = ctypes.get_errno() + failed_to_drop.append((name, err)) + + if not dropped and not failed_to_drop: + print(f"neuros-container: --cap-drop pattern matched 0 of " + f"{last + 1} capabilities; nothing was dropped. " + f"Check the regex (e.g. 'CAP_NET_RAW', " + f"'^CAP_(SYS|NET)_' for a prefix).", + file=sys.stderr) + for name, err in failed_to_drop: + print(f"neuros-container: failed to drop {name} from bounding " + f"set (errno={err}); the user namespace likely lacks " + f"CAP_SETPCAP. Run as root, or inside a user namespace " + f"with CAP_SETPCAP inherited.", + file=sys.stderr) + + +def run_container(args): + """Top-level container-run path. Created cgroup is cleaned up on + exit when --detach is not in effect, and a state file is written + when it is.""" + name = args.name or f"neuros-{os.getpid()}" + mem = parse_size(args.mem) + pids = args.pids + cg = make_cgroup(name, mem, pids, args.cpu, args.cpu_quota) + + state_path = None + if args.detach: + os.makedirs(STATE_DIR, exist_ok=True) + state_path = os.path.join(STATE_DIR, f"{name}.json") + + pid = os.fork() + if pid == 0: + try: + join_cgroup(cg, os.getpid()) + ns = enter_namespaces(net=args.net, user=args.user) + mount_ns, _uts, _pid, _ipc, net_ns, _userns = ns + + if not mount_ns: + # No mount namespace: drop straight into exec. + try: + os.execvp(args.cmd[0], args.cmd) + except OSError as e: + print(f"neuros-container: exec {args.cmd[0]} failed: {e}", + file=sys.stderr) + os._exit(127) + return + + # CLONE_NEWPID only affects post-unshare children, so a + # second fork is needed for the user's command to actually + # be PID 1 in the new PID namespace. + inner = os.fork() + if inner == 0: + if args.hostname: + import socket + socket.sethostname(args.hostname) + if args.rootfs: + os.chroot(args.rootfs) + os.chdir("/") + if args.read_only: + remount_readonly_best_effort() + if args.user is not None and os.geteuid() == 0: + target_uid, target_gid = args.user + try: + os.setgid(target_gid) + os.setuid(target_uid) + except (PermissionError, OSError) as e: + print(f"neuros-container: setuid/setgid failed: {e}", + file=sys.stderr) + drop_capabilities_best_effort(args.cap_drop) + if args.workdir: + try: + os.chdir(args.workdir) + except OSError as e: + print(f"neuros-container: cannot chdir to " + f"{args.workdir}: {e}", file=sys.stderr) + # Inject --env KEY=VALUE entries last so they take + # precedence over anything inherited from the parent + # shell (the user's intent for --env is usually "force + # this value", not "default if unset"). + if args.env: + for entry in args.env: + k, _, v = entry.partition("=") + os.environ[k] = v + mount_proc_best_effort() + try: + os.execvp(args.cmd[0], args.cmd) + except OSError as e: + print(f"neuros-container: exec {args.cmd[0]} failed: {e}", + file=sys.stderr) + os._exit(127) + else: + if state_path: + write_state(state_path, name, inner, cg) + if args.detach: + print(f"neuros-container: detached, inner pid {inner}, " + f"state {state_path}", file=sys.stderr) + os._exit(0) + _, status = os.waitpid(inner, 0) + os._exit(os.waitstatus_to_exitcode(status)) + except Exception as e: + print(f"neuros-container: child setup failed: {e}", + file=sys.stderr) + os._exit(1) + + _, status = os.waitpid(pid, 0) + + # Foreground (non-detached) runs: the cgroup was scoped to this PID, + # so by the time the parent wait()s, the leaf is empty and rmdirable. + # We rely on memory.peak being readable while the cgroup is still on + # disk; cgroup_report handles the already-removed case silently. + report = cgroup_report(cg) + if not args.detach: + try: + os.rmdir(cg) + except OSError: + rmdir_failed = True + if report.get("memory_peak"): + peak_mib = int(report["memory_peak"]) / 1024 / 1024 + print(f"neuros-container: peak memory {peak_mib:.1f} MiB", + file=sys.stderr) + return os.waitstatus_to_exitcode(status) + + +def cleanup_container(name): + """Best-effort teardown of a single detached container. + + Reads the state file under STATE_DIR, attempts to rmdir the cgroup + (only succeeds if empty; populated cgroups are left alone and the + caller is told), then unlinks the state file. Returns 0 on success, + 1 if the cgroup was non-empty so a manual inspect is warranted.""" + state_path = os.path.join(STATE_DIR, f"{name}.json") + if not os.path.exists(state_path): + print(f"neuros-container: no state file for '{name}' " + f"(looked at {state_path})", file=sys.stderr) + return 1 + try: + with open(state_path) as f: + state = json.load(f) + cg = state.get("cgroup", "") + except (OSError, json.JSONDecodeError) as e: + print(f"neuros-container: cannot read state file for '{name}': " + f"{e}; unlinking", file=sys.stderr) + try: + os.unlink(state_path) + except OSError: + pass + return 1 + + still_running = 0 + procs_path = os.path.join(cg, "cgroup.procs") + try: + with open(procs_path) as f: + still_running = len(f.read().split()) + except OSError: + # A missing or unreadable cgroup.procs is treated as zero + # live members so partially-constructed leaves (and the + # synthetic tmpdir cgroups the test suite uses) can still be + # reaped. Without this branch, the tests would need to either + # leave a procs file behind (which blocks rmdir with ENOTEMPTY) + # or skip these leaves entirely. + still_running = 0 + + if still_running: + print(f"neuros-container: '{name}' still has {still_running} " + f"processes in {cg}; refusing to remove. Kill them first " + f"(the pid is in {state_path}).", file=sys.stderr) + return 1 + try: + os.rmdir(cg) + print(f"neuros-container: removed cgroup {cg}", file=sys.stderr) + except OSError as e: + print(f"neuros-container: could not rmdir {cg} ({e}); leaving " + f"the state file in place", file=sys.stderr) + return 1 + try: + os.unlink(state_path) + except OSError: + pass + return 0 + + +def cleanup_all(): + """Cleanup every detached container. Each one that has live + members is skipped (with a stderr line) so a partially-clean + state surfaces.""" + if not os.path.isdir(STATE_DIR): + print("no detached neuros-container state to clean up") + return 0 + rc = 0 + for entry in sorted(os.listdir(STATE_DIR)): + if not entry.endswith(".json"): + continue + name = entry[:-len(".json")] + rc |= cleanup_container(name) + return rc + + +def _list_neuros_leaves(root): + """Recursively enumerate every ``neuros-*`` leaf cgroup under + ``root``. Returns a list of dicts; hoisted out of ``list_cgroups`` + so the test suite can drive it against synthetic cgroup trees + without spinning up a live delegation.""" + out = [] + try: + entries = sorted(os.listdir(root)) + except OSError: + return out + for entry in entries: + path = os.path.join(root, entry) + if not entry.startswith("neuros-"): + out.extend(_list_neuros_leaves(path)) + continue + rel = path[len(CGROUP_ROOT):] if path.startswith(CGROUP_ROOT) else path + try: + with open(os.path.join(path, "cgroup.procs")) as f: + procs = len(f.read().split()) + except OSError: + procs = 0 + limits = {} + for key, fname in ( + ("memory_max", "memory.max"), + ("pids_max", "pids.max"), + ("cpu_weight", "cpu.weight"), + ("cpu_max", "cpu.max"), + ): + try: + with open(os.path.join(path, fname)) as f: + limits[key] = f.read().strip() + except OSError: + limits[key] = None + peak = None + try: + with open(os.path.join(path, "memory.peak")) as f: + raw = f.read().strip() + if raw.isdigit(): + peak = int(raw) + except OSError: + peak = None + out.append({ + "name": entry, "path": rel, "procs": procs, + "limits": limits, "memory_peak": peak, + }) + return out + + +def list_cgroups(json_output=False): + """Walk the cgroup subtree under the delegating ancestor we found + earlier (since nested neuros-* leaves inside a parent neuros-* leaf + are also valid) and emit a per-leaf report. JSON mode emits a flat + array of records so scripts can pipe it through jq.""" + rows = _list_neuros_leaves(own_cgroup_path()) + if json_output: + print(json.dumps(rows, indent=2)) + return + if not rows: + print("no active neuros-container cgroups") + return + for r in rows: + suffix = "es" if r["procs"] != 1 else "" + print(f"{r['name']} ({r['procs']} process{suffix})") + for label, fname in ( + ("memory.max", "memory_max"), + ("pids.max", "pids_max"), + ("cpu.weight", "cpu_weight"), + ("cpu.max", "cpu_max"), + ): + val = r["limits"].get(fname) + if val: + print(f" {label} = {val}") + if r["memory_peak"] is not None: + print(f" memory.peak = {r['memory_peak'] / 1024 / 1024:.1f} MiB") + + +def main(): + parser = argparse.ArgumentParser(prog="neuros-container") + sub = parser.add_subparsers(dest="command") + + run = sub.add_parser("run", help="run a command in a cgroup-limited, namespace-isolated child") + run.add_argument("--mem", help="memory limit, e.g. 256M, 1G") + run.add_argument("--pids", help="max number of processes/threads") + run.add_argument("--cpu", type=int, help="cpu.weight (1-10000, default cgroup weight is 100)") + run.add_argument("--cpu-quota", metavar="MAX/PERIOD", + help="cpu.max quota, e.g. '50000/100000' = 50%% of one CPU") + run.add_argument("--hostname", help="hostname to set inside the UTS namespace") + run.add_argument("--rootfs", help="directory to chroot into before exec") + run.add_argument("--workdir", help="working directory inside the container (after chroot)") + run.add_argument("--user", metavar="UID:GID", + help="explicit UID:GID to set inside the container (root only)") + run.add_argument("--read-only", action="store_true", + help="remount / as read-only inside the container") + run.add_argument("--net", action="store_true", + help="enter a new network namespace") + run.add_argument("--env", action="append", metavar="KEY=VALUE", + help="set an environment variable inside the container (repeatable)") + run.add_argument("--env-from-file", metavar="PATH", + help="load KEY=VALUE pairs (one per line, # comments, optional 'export ' prefix) from PATH") + run.add_argument("--cap-drop", metavar="REGEX", + help="drop capabilities matching this regex from the bounding set (libcap2-bin required)") + run.add_argument("--detach", action="store_true", + help="don't wait; leave the child running and write /run/neuros-container/.json") + run.add_argument("--name", help="cgroup name (default: 'neuros-')") + run.add_argument("cmd", nargs=argparse.REMAINDER, + help="-- command and arguments to run") + + list_p = sub.add_parser("list", help="list active neuros-container cgroups (recursive)") + list_p.add_argument("--json", action="store_true", + help="emit structured JSON instead of human-readable output") + + sub.add_parser("logs", help="(placeholder) tail a detached container's state file") + + cleanup_p = sub.add_parser( + "cleanup", help="remove a detached container's cgroup + state file") + cleanup_p.add_argument("name", nargs="?", + help="container name to clean up (omit with --all)") + cleanup_p.add_argument("--all", action="store_true", + help="clean up every detached container") + + args = parser.parse_args() + + if args.command == "run": + cmd = args.cmd + if cmd and cmd[0] == "--": + cmd = cmd[1:] + if not cmd: + die("no command given (usage: neuros-container run [opts] -- CMD [ARGS...])") + args.cmd = cmd + args.cpu_quota = parse_cpu_quota(args.cpu_quota) + args.user = parse_user(args.user) + # --env-from-file is merged with --env so file values land + # first (older defaults) and explicit --env K=V values + # override them on conflict, mirroring docker --env-file. + env_from_file = parse_env_file(args.env_from_file) + args.env = parse_env((env_from_file or []) + (args.env or [])) + args.cap_drop = parse_cap_drop(args.cap_drop) + sys.exit(run_container(args)) + elif args.command == "list": + list_cgroups(json_output=args.json) + elif args.command == "logs": + print("neuros-container logs: not implemented yet " + "(the state file lives at /run/neuros-container/.json)", + file=sys.stderr) + elif args.command == "cleanup": + if args.all: + sys.exit(cleanup_all()) + if not args.name: + die("cleanup needs either a container NAME or --all") + sys.exit(cleanup_container(args.name)) + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/config/includes.chroot/usr/local/bin/neuros-mcp b/config/includes.chroot/usr/local/bin/neuros-mcp index 8d938c2..f440643 100755 --- a/config/includes.chroot/usr/local/bin/neuros-mcp +++ b/config/includes.chroot/usr/local/bin/neuros-mcp @@ -36,25 +36,13 @@ class MCPServer(BaseHTTPRequestHandler): self.send_error(400, "Invalid JSON") return - method = request.get("method", "") - params = request.get("params", {}) - req_id = request.get("id") - - if method == "initialize": - result = self.handle_initialize(params) - elif method == "tools/list": - result = self.handle_list_tools() - elif method == "tools/call": - result = self.handle_call_tool(params) - elif method == "resources/list": - result = self.handle_list_resources() - elif method == "resources/read": - result = self.handle_read_resource(params) + response = dispatch_request(self, request) + if response is not None: + self.send_json(response) else: - self.send_json({"jsonrpc": "2.0", "error": {"code": -32601, "message": "Method not found"}, "id": req_id}) - return - - self.send_json({"jsonrpc": "2.0", "result": result, "id": req_id}) + # Notification (no "id") -- MCP expects no response body. + self.send_response(204) + self.end_headers() def send_json(self, data): self.send_response(200) @@ -341,6 +329,64 @@ class MCPServer(BaseHTTPRequestHandler): pass +def dispatch_request(handler, request): + """Route one JSON-RPC request to the matching handler method. + + Shared by both the HTTP transport (do_POST) and the stdio transport + (run_stdio), so tool/resource logic is only implemented once. + Returns None for notifications (no "id"), which must not get a reply. + """ + method = request.get("method", "") + params = request.get("params", {}) + req_id = request.get("id") + + if method == "notifications/initialized": + return None + + if method == "initialize": + result = handler.handle_initialize(params) + elif method == "tools/list": + result = handler.handle_list_tools() + elif method == "tools/call": + result = handler.handle_call_tool(params) + elif method == "resources/list": + result = handler.handle_list_resources() + elif method == "resources/read": + result = handler.handle_read_resource(params) + else: + return {"jsonrpc": "2.0", "error": {"code": -32601, "message": "Method not found"}, "id": req_id} + + return {"jsonrpc": "2.0", "result": result, "id": req_id} + + +def run_stdio(): + """MCP stdio transport: newline-delimited JSON-RPC on stdin/stdout. + + Per the MCP spec, the server must write ONLY JSON-RPC messages to + stdout (one per line) and may use stderr freely for logging. This is + the transport real MCP clients (Claude Code, Claude Desktop, etc.) + use to launch a local server as a subprocess. + """ + handler = MCPServer.__new__(MCPServer) + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + request = json.loads(line) + except json.JSONDecodeError: + print(json.dumps({ + "jsonrpc": "2.0", + "error": {"code": -32700, "message": "Parse error"}, + "id": None + }), flush=True) + continue + + response = dispatch_request(handler, request) + if response is not None: + print(json.dumps(response), flush=True) + + def check_ollama(): try: import urllib.request @@ -356,8 +402,14 @@ def main(): parser = argparse.ArgumentParser(description="NeurOS MCP Server") parser.add_argument("--port", type=int, default=DEFAULT_PORT, help=f"Port (default: {DEFAULT_PORT})") parser.add_argument("--host", default="127.0.0.1", help="Host (default: 127.0.0.1)") + parser.add_argument("--stdio", action="store_true", + help="Speak MCP over stdio instead of HTTP (for MCP clients that launch the server as a subprocess)") args = parser.parse_args() + if args.stdio: + run_stdio() + return + server = HTTPServer((args.host, args.port), MCPServer) print(f""" diff --git a/config/includes.chroot/usr/local/bin/neuros-model b/config/includes.chroot/usr/local/bin/neuros-model index d1811b7..d812462 100755 --- a/config/includes.chroot/usr/local/bin/neuros-model +++ b/config/includes.chroot/usr/local/bin/neuros-model @@ -22,8 +22,18 @@ import time import argparse import textwrap -OLLAMA_URL = "http://localhost:11434/api" -CONFIG_PATH = os.path.expanduser("~/.config/neuros/llm.conf") +# Pull the shared config helpers from neuroslib rather than rolling our +# own key/value loop; keep ``CONFIG_PATH`` here for back-compat with +# callers and the test suite that patches it. +import importlib.util +_LIB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "neuroslib.py") +_spec = importlib.util.spec_from_file_location("neuroslib", _LIB_PATH) +neuroslib = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(neuroslib) + +OLLAMA_URL = f"http://{neuroslib.DEFAULT_HOST}:{neuroslib.DEFAULT_PORT}/api" +CONFIG_PATH = neuroslib.LLM_CONF +DEFAULT_MODEL = neuroslib.DEFAULT_MODEL def api_request(endpoint, method="GET", data=None, timeout=10): """Make a request to Ollama API.""" @@ -115,28 +125,9 @@ def switch_model(name): if name not in model_names: print(f"Warning: '{name}' is not installed. Pull it first with: neuros-model pull {name}") - os.makedirs(os.path.dirname(CONFIG_PATH), exist_ok=True) - - # Read existing config - lines = [] - found = False - try: - with open(CONFIG_PATH) as f: - for line in f: - if '=' in line and not line.startswith('[') and 'model' in line.split('=')[0]: - lines.append(f'model = "{name}"\n') - found = True - else: - lines.append(line) - except FileNotFoundError: - pass - - if not found: - lines.append(f'model = "{name}"\n') - - with open(CONFIG_PATH, 'w') as f: - f.writelines(lines) - + # Delegate the actual file write to neuroslib so the [context] + # section and other keys survive untouched. + neuroslib.set_default_model(name, CONFIG_PATH) print(f"✅ Default model set to '{name}'.") def model_info(name): @@ -267,16 +258,7 @@ def compare_models(models, prompt="Write a haiku about Linux."): def get_current_model(): """Read current default model from config.""" - try: - with open(CONFIG_PATH) as f: - for line in f: - if '=' in line and not line.startswith('['): - key, val = line.split('=', 1) - if key.strip() == "model": - return val.strip().strip('"').strip("'") - except Exception: - pass - return "mistral" + return neuroslib.get_default_model(CONFIG_PATH) def format_size(size_bytes): """Format size in human-readable format.""" diff --git a/config/includes.chroot/usr/local/bin/neuros-policy b/config/includes.chroot/usr/local/bin/neuros-policy new file mode 100755 index 0000000..1e843c0 --- /dev/null +++ b/config/includes.chroot/usr/local/bin/neuros-policy @@ -0,0 +1,577 @@ +#!/usr/bin/env python3 +"""neuros-policy: declarative security policy for neuros-sandbox. + +The wrapper `neuros-sandbox` is hardened by default, but it accepts +every limit and the cap-drop regex as a CLI flag. That makes a flat +safety posture hard to express as code, hard to version, and easy +to drift: a script that hard-coded --mem 256M in May has no way to +say "I really meant the team's 2024-Q4 policy, whatever it was +called then". + +`neuros-policy` is the place that manifest lives: + + { + "name": "neuros-default", + "version": "1.0.0", + "defaults": { + "mem": "256M", + "pids": 64, + "cpu_quota": 50000, + "timeout": 30 + }, + "profiles": { + "strict": ["CAP_NET_RAW", "CAP_SYS_ADMIN", "CAP_SYS_PTRACE"], + "moderate": ["CAP_NET_RAW", "CAP_SYS_ADMIN"], + "permissive": [] + }, + "net": "private", + "readonly": true, + "env_allowlist": ["PATH", "LANG", "LC_ALL"], + "syscalls": null + } + +Subcommands: + + neuros-policy validate + Schema + bounds + regex-shape check. Exit 0 on OK, 1 on + any violation (each violation is reported on stderr), 2 on + bad input (missing file, malformed JSON, wrong root type). + + neuros-policy check --envelope + Parse a neuros-sandbox JSON envelope and reconcile its + wall_clock_ms / peak_mem_estimate / exit_code against the + policy. Exit 0 if the run conformed, 1 on a violation, 2 + on bad input. + + neuros-policy transpile [--profile NAME] + Emit a copy-pastable argv slice suitable for + `neuros-sandbox --unsafe ...`: + + --mem 256M --pids 64 --cpu-quota 50000 \ + --timeout 30 --cap-drop CAP_(NET_RAW|SYS_ADMIN) + + The wrapper appends ``--`` followed by its own entry + (``python3 -u -``) once it parses the user argv, so the + transpile output is intentionally flag-only. An empty + profile (permissive) transpiles to + ``--cap-drop '^_NEVER_MATCH_$'`` so nothing is dropped. + + --json makes validate / check emit a one-line JSON envelope + on stdout (for piping into verify-style tooling). transpile + always prints text because its output IS the command line. +""" +import argparse +import json +import os +import re +import sys + +# --- Constants ---------------------------------------------------------- + +#: Lower bound on memory defaults (bytes). 16 KiB is what a busy +#: interpreter imports plus a small buffer. +MEM_MIN_BYTES = 16 * 1024 + +#: Upper bound on memory defaults (bytes). 1 GiB matches what the +#: bench + runbook paths already exercise; bigger requests should +#: be split into multiple containers. +MEM_MAX_BYTES = 1024 * 1024 * 1024 + +#: cgroup v2's pids.max is "max" for unbounded, but in policy we +#: cap it because the default policy is what kills an LLM agent +#: fork-bomb, not the host kernel. +PIDS_MIN, PIDS_MAX = 1, 4096 + +#: cpu-quota is microseconds / 100 ms CFS period. 1000 = 1% of one +#: CPU. 1_000_000 = 10× one CPU (would saturate any single workload). +CPU_QUOTA_MIN, CPU_QUOTA_MAX = 1000, 1_000_000 + +#: Wall-clock cap. The bench harness times out at 60s; we leave +#: 60× headroom so a 1-hour benchmark is still expressible. +TIMEOUT_MIN, TIMEOUT_MAX = 1, 3600 + +#: Sentinel regex emitted when the caller picked a profile with an +#: EMPTY cap list (i.e. "drop nothing"). Matches no CAP_* name in +#: the kernel's table, since caps start at ``CAP_CHOWN``. +NEVER_MATCH_CAP_REGEX = r"^_NEVER_MATCH_$" + +#: Regexes used during validate. +_CAP_NAME_RE = re.compile(r"^CAP_[A-Z_]+$") +_PROFILE_NAME_RE = re.compile(r"^[a-z][a-z0-9_-]*$") +_ENV_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*$") +_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$") +_NET_VALUES = ("private", "host", "isolated") + +#: Tool version baked into emitted envelopes. Bump on a non-additive +#: schema change. +NEUROS_POLICY_VERSION = "0.1" + + +# --- Exceptions -------------------------------------------------------- + + +class PolicyError(Exception): + """Raised on any user-fixable misconfiguration (bad field, bad + regex, out-of-range bound, malformed manifest). Surfaced as exit + 1 by `validate` / `check`, and 2 by `transpile` if the profile + itself is malformed.""" + + +# --- Helpers ------------------------------------------------------------ + + +def _die(msg, code=2): + print(msg, file=sys.stderr) + sys.exit(code) + + +def _parse_size_bytes(text): + """Parse ``'128M'``, ``'1G'``, ``'512K'``, a plain byte count, + or ``'max'`` into an int (or None when ``text == 'max'``). + + Raises ``PolicyError`` on bad input. + """ + if text is None: + return None + text = text.strip() + if not text: + raise PolicyError("size is empty") + if text.lower() == "max": + return None + m = re.fullmatch(r"(\d+)([KMG]?)", text, re.IGNORECASE) + if not m: + raise PolicyError( + f"invalid size {text!r} (expected e.g. 256M, 1G, or a byte count)") + n = int(m.group(1)) + mult = {"": 1, "K": 1024, "M": 1024 ** 2, "G": 1024 ** 3}[m.group(2).upper()] + if n <= 0: + raise PolicyError(f"size {text!r} must be > 0 bytes") + return n * mult + + +def _is_int(text): + """True if ``text`` is a positive integer in canonical string form.""" + return isinstance(text, int) and not isinstance(text, bool) and text >= 0 + + +def _load_policy(path): + """Read JSON from ``path`` (or '-' for stdin). Raises ``PolicyError`` + on missing file or malformed JSON. The returned object is the + raw dict; the caller is expected to run ``validate_policy`` on it. + """ + if path == "-": + try: + text = sys.stdin.read() + except OSError as e: + raise PolicyError(f"cannot read stdin: {e}") + else: + try: + with open(path, "r", encoding="utf-8") as f: + text = f.read() + except OSError as e: + raise PolicyError(f"cannot read policy file {path!r}: {e}") + try: + obj = json.loads(text) + except json.JSONDecodeError as e: + raise PolicyError(f"policy file {path!r} is not valid JSON: {e}") + if not isinstance(obj, dict): + raise PolicyError( + f"policy root must be a JSON object (got {type(obj).__name__})") + return obj + + +def _emit_json_envelope(payload): + """Single-line JSON record, mirrors the shape of the sandbox + envelope so downstream tooling can ingest both interchangeably. + """ + print(json.dumps(payload, separators=(",", ":"), ensure_ascii=False)) + + +# --- Validation --------------------------------------------------------- + + +def validate_policy(policy): + """Validate an already-parsed policy dict. Returns a list of + :class:`PolicyError` instances (empty list on success); never + raises. Errors are meant to be printed by the CLI layer so the + user sees ALL of them, not just the first one. + """ + errors = [] + if not isinstance(policy, dict): + # The loader already guarantees this; defense in depth. + return [PolicyError(f"policy root must be a JSON object " + f"(got {type(policy).__name__})")] + + name = policy.get("name") + if not isinstance(name, str) or not name.strip(): + errors.append(PolicyError( + f"field 'name' must be a non-empty string (got {name!r})")) + + version = policy.get("version", "") + if not isinstance(version, str) or not _VERSION_RE.fullmatch(version or ""): + errors.append(PolicyError( + f"field 'version' must match MAJOR.MINOR.PATCH (got {version!r})")) + + # defaults block -------------------------------------------------- + defaults = policy.get("defaults", {}) + if not isinstance(defaults, dict): + errors.append(PolicyError( + f"field 'defaults' must be a JSON object (got " + f"{type(defaults).__name__})")) + defaults = {} + + mem = defaults.get("mem") + if mem is not None: + try: + n = _parse_size_bytes(mem) + if n is not None and not (MEM_MIN_BYTES <= n <= MEM_MAX_BYTES): + errors.append(PolicyError( + f"defaults.mem {mem!r} parses to {n} bytes; " + f"must be in [{MEM_MIN_BYTES}, {MEM_MAX_BYTES}]")) + except PolicyError as e: + errors.append(PolicyError(f"defaults.mem: {e}")) + + pids = defaults.get("pids") + if pids is not None and (not _is_int(pids) or + not (PIDS_MIN <= pids <= PIDS_MAX)): + errors.append(PolicyError( + f"defaults.pids {pids!r} must be an integer in " + f"[{PIDS_MIN}, {PIDS_MAX}]")) + + cpu_quota = defaults.get("cpu_quota") + if cpu_quota is not None and (not _is_int(cpu_quota) or + not (CPU_QUOTA_MIN <= cpu_quota <= CPU_QUOTA_MAX)): + errors.append(PolicyError( + f"defaults.cpu_quota {cpu_quota!r} must be an integer in " + f"[{CPU_QUOTA_MIN}, {CPU_QUOTA_MAX}]")) + + timeout = defaults.get("timeout") + if timeout is not None and (not _is_int(timeout) or + not (TIMEOUT_MIN <= timeout <= TIMEOUT_MAX)): + errors.append(PolicyError( + f"defaults.timeout {timeout!r} must be an integer in " + f"[{TIMEOUT_MIN}, {TIMEOUT_MAX}]")) + + # profiles block -------------------------------------------------- + profiles = policy.get("profiles", {}) + if not isinstance(profiles, dict): + errors.append(PolicyError( + f"field 'profiles' must be a JSON object (got " + f"{type(profiles).__name__})")) + profiles = {} + + seen_caps_per_profile = {} + for pname, caps in profiles.items(): + if not _PROFILE_NAME_RE.fullmatch(pname or ""): + errors.append(PolicyError( + f"profile name {pname!r} must match " + f"{_PROFILE_NAME_RE.pattern!r}")) + if not isinstance(caps, list) or not all(isinstance(c, str) for c in caps): + errors.append(PolicyError( + f"profile {pname!r}.caps must be a list of strings")) + continue + # A profile may be empty (drops nothing on purpose). + cleaned = [] + for c in caps: + if not _CAP_NAME_RE.fullmatch(c): + errors.append(PolicyError( + f"profile {pname!r} contains non-cap name {c!r}; " + f"must match {_CAP_NAME_RE.pattern!r}")) + else: + cleaned.append(c) + # De-dup + sort so transpile produces a canonical regex. + seen_caps_per_profile[pname] = sorted(set(cleaned)) + + # net ------------------------------------------------------------- + net = policy.get("net") + if net is not None and net not in _NET_VALUES: + errors.append(PolicyError( + f"field 'net' {net!r} must be one of {_NET_VALUES}")) + + # readonly -------------------------------------------------------- + ro = policy.get("readonly") + if ro is not None and not isinstance(ro, bool): + errors.append(PolicyError( + f"field 'readonly' must be a boolean (got {type(ro).__name__})")) + + # env_allowlist --------------------------------------------------- + env = policy.get("env_allowlist", []) + if not isinstance(env, list): + errors.append(PolicyError( + f"field 'env_allowlist' must be a list (got {type(env).__name__})")) + else: + for v in env: + if not isinstance(v, str) or not _ENV_NAME_RE.fullmatch(v): + errors.append(PolicyError( + f"env_allowlist entry {v!r} must match " + f"{_ENV_NAME_RE.pattern!r}")) + + # syscalls (optional) -------------------------------------------- + syscalls = policy.get("syscalls") + if syscalls is not None and not isinstance(syscalls, dict): + errors.append(PolicyError( + f"field 'syscalls' must be a JSON object or null " + f"(got {type(syscalls).__name__})")) + + # Attach the deduped caps back so the caller can build a + # canonical transpile without re-running the regex match. + return errors, seen_caps_per_profile + + +# --- Transpile ---------------------------------------------------------- + + +def transpile_argv(policy, profile_name, cleaned_caps): + """Build the argv slice that, when pasted after + ``neuros-sandbox --unsafe``, realises the policy + chosen + profile. The profile's caps are converted into a single + ``--cap-drop`` regex: ``CAP_(X|Y|Z)`` with sorted-unique + entries, or the never-match sentinel when empty. + """ + args = [] + defaults = policy.get("defaults", {}) + if "mem" in defaults and defaults["mem"] is not None: + args += ["--mem", defaults["mem"]] + if "pids" in defaults and defaults["pids"] is not None: + args += ["--pids", str(defaults["pids"])] + if "cpu_quota" in defaults and defaults["cpu_quota"] is not None: + args += ["--cpu-quota", str(defaults["cpu_quota"])] + if "timeout" in defaults and defaults["timeout"] is not None: + args += ["--timeout", str(defaults["timeout"])] + + if profile_name is not None: + if profile_name not in policy.get("profiles", {}): + raise PolicyError( + f"profile {profile_name!r} is not declared in policy " + f"(known: {sorted(policy.get('profiles', {}).keys())})") + caps = cleaned_caps.get(profile_name, []) + if caps: + args += ["--cap-drop", "CAP_(" + "|".join(caps) + ")"] + else: + args += ["--cap-drop", NEVER_MATCH_CAP_REGEX] + return args + + +# --- Check (envelope compliance) ---------------------------------------- + + +def check_envelope_against_policy(policy, envelope, cleaned_caps): + """Reconcile a neuros-sandbox JSON envelope against the policy. + Returns a list of violations (empty == conformant). Each violation + is a (rule, observed, expected, severity) tuple so the JSON + envelope has the same shape as the bench + runbook outputs. + """ + violations = [] + if not isinstance(envelope, dict): + return [("malformed", type(envelope).__name__, + "object", "error")] + defaults = policy.get("defaults", {}) + + # wall_clock is the most common regression vector: a workload + # that used to finish in 200 ms now takes 30 s is rare but real. + wall = envelope.get("wall_clock_ms") + timeout = defaults.get("timeout") + if wall is not None and timeout is not None: + wall_s = wall / 1000.0 + if wall_s > timeout: + violations.append( + ("wall_clock", round(wall_s, 3), timeout, "error")) + + # peak memory: only check when the primitive actually filled in + # the field (it can be None on hosts without cgroup delegation). + peak = envelope.get("peak_mem_estimate") + mem = defaults.get("mem") + if peak is not None and mem is not None: + try: + mem_bytes = _parse_size_bytes(mem) + if mem_bytes is not None and peak > mem_bytes: + violations.append( + ("peak_mem", peak, mem_bytes, "error")) + except PolicyError: + # mem field already validated upstream; skip quietly. + pass + + # timeout_hit bool: a `true` here means the wrapper had to SIGKILL + # the workload, which is a policy violation regardless of bounds. + if envelope.get("timeout_hit"): + violations.append( + ("timeout_hit", True, False, "error")) + + return violations + + +# --- Subcommand dispatch ------------------------------------------------ + + +def cmd_validate(args): + try: + policy = _load_policy(args.policy) + except PolicyError as e: + if args.json: + _emit_json_envelope({ + "ok": False, "errors": [str(e)], + "neuros_policy_version": NEUROS_POLICY_VERSION}) + else: + print(f"neuros-policy: {e}", file=sys.stderr) + return 2 + + errs, _cleaned = validate_policy(policy) + if args.json: + _emit_json_envelope({ + "ok": not errs, + "errors": [str(e) for e in errs], + "name": policy.get("name"), + "version": policy.get("version"), + "neuros_policy_version": NEUROS_POLICY_VERSION, + }) + else: + if errs: + print(f"neuros-policy: {len(errs)} violation(s):", + file=sys.stderr) + for e in errs: + print(f" - {e}", file=sys.stderr) + else: + print(f"neuros-policy: ok ({policy.get('name')!r} " + f"v{policy.get('version')})") + return 1 if errs else 0 + + +def cmd_check(args): + try: + policy = _load_policy(args.policy) + except PolicyError as e: + print(f"neuros-policy: {e}", file=sys.stderr) + return 2 + errs, cleaned = validate_policy(policy) + if errs: + # We refuse to check against a policy that itself is invalid. + if args.json: + _emit_json_envelope({ + "ok": False, + "policy_errors": [str(e) for e in errs], + "neuros_policy_version": NEUROS_POLICY_VERSION, + }) + else: + print(f"neuros-policy: policy has {len(errs)} violation(s); " + f"refusing to check envelope", file=sys.stderr) + for e in errs: + print(f" - {e}", file=sys.stderr) + return 2 + + try: + envelope = _load_policy(args.envelope) + except PolicyError as e: + print(f"neuros-policy: {e}", file=sys.stderr) + return 2 + + violations = check_envelope_against_policy(policy, envelope, cleaned) + if args.json: + _emit_json_envelope({ + "ok": not violations, + "violations": [ + {"rule": r, "observed": o, "expected": x, "severity": s} + for (r, o, x, s) in violations + ], + "policy_name": policy.get("name"), + "policy_version": policy.get("version"), + "neuros_policy_version": NEUROS_POLICY_VERSION, + }) + else: + if violations: + print(f"neuros-policy: {len(violations)} violation(s):", + file=sys.stderr) + for r, o, x, s in violations: + print(f" - [{s}] {r}: observed {o!r}, " + f"expected {x!r}", file=sys.stderr) + else: + exit_code = envelope.get("exit_code") + print(f"neuros-policy: ok " + f"(exit_code={exit_code}, wall_clock_ms=" + f"{envelope.get('wall_clock_ms')}, " + f"peak_mem={envelope.get('peak_mem_estimate')})") + return 1 if violations else 0 + + +def cmd_transpile(args): + try: + policy = _load_policy(args.policy) + except PolicyError as e: + print(f"neuros-policy: {e}", file=sys.stderr) + return 2 + errs, cleaned = validate_policy(policy) + if errs: + print(f"neuros-policy: policy has {len(errs)} violation(s); " + f"refusing to transpile", file=sys.stderr) + for e in errs: + print(f" - {e}", file=sys.stderr) + return 1 + try: + argv = transpile_argv(policy, args.profile, cleaned) + except PolicyError as e: + print(f"neuros-policy: {e}", file=sys.stderr) + return 1 + print(" ".join(argv)) + return 0 + + +# --- Argparse ------------------------------------------------------------ + + +def _build_parser(): + p = argparse.ArgumentParser( + prog="neuros-policy", + description=( + "Declarative security policy for neuros-sandbox. Validate, " + "check envelope compliance, or transpile to a sandbox argv " + "fragment." + ), + epilog=( + "Examples:\n" + " neuros-policy validate ./policy.json\n" + " neuros-policy check ./policy.json --envelope ./envelope.json\n" + " neuros-policy transpile ./policy.json --profile strict\n" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = p.add_subparsers(dest="cmd", required=True) + + val = sub.add_parser("validate", + help="schema + bounds check on a policy file") + val.add_argument("policy", + help="Path to a JSON policy file ('-' for stdin)") + val.add_argument("--json", action="store_true", + help="Emit a one-line JSON envelope instead of text") + + chk = sub.add_parser("check", + help="Check a neuros-sandbox JSON envelope " + "against a policy") + chk.add_argument("policy", help="Path to a JSON policy file") + chk.add_argument("--envelope", required=True, + help="Path to a JSON envelope file ('-' for stdin)") + chk.add_argument("--json", action="store_true", + help="Emit a one-line JSON envelope instead of text") + + tr = sub.add_parser("transpile", + help="Emit the neuros-sandbox argv fragment " + "realising this policy") + tr.add_argument("policy", help="Path to a JSON policy file") + tr.add_argument("--profile", default=None, + help="Profile name to enable (default: none)") + return p + + +def main(argv=None): + parser = _build_parser() + args = parser.parse_args(argv) + if args.cmd == "validate": + return cmd_validate(args) + if args.cmd == "check": + return cmd_check(args) + if args.cmd == "transpile": + return cmd_transpile(args) + parser.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/config/includes.chroot/usr/local/bin/neuros-replay b/config/includes.chroot/usr/local/bin/neuros-replay new file mode 100755 index 0000000..fc734eb --- /dev/null +++ b/config/includes.chroot/usr/local/bin/neuros-replay @@ -0,0 +1,386 @@ +#!/usr/bin/env python3 +"""neuros-replay: offline diagnostics over neuros-sandbox envelopes. + +The `neuros-sandbox` wrapper emits a single-line JSON envelope per +run (`exit_code`, `wall_clock_ms`, `peak_mem_estimate`, `timeout_hit`, +`stdout`, `stderr`). When a run looks wrong, the question is usually +"how wrong?". The bench harness compares batch aggregates but won't +help on a single observation. `neuros-replay` is the offline-matching +tool: it takes one or two envelopes and emits diagnostics without +re-executing the workload. + +Subcommands: + + neuros-replay diff + Compare two envelopes, emit a verdict envelope. Tolerances + default to 10% wall, 20% peak-mem (tighter than bench's + 15%/25% because diff operates on single observations, not + aggregates). Override with --tolerance-wall-pct and + --tolerance-mem-pct. Exit 0 on conformant, 1 on any rule + violation, 2 on bad input. + + neuros-replay explain + Emit a single sinkable line onto stdout (the "CI diaper"):: + + exit=0 wall=1542ms peak=14.2MiB timeout=false + stdout_bytes=0 stderr_bytes=132 + + Use `--json` for a structured equivalent. + + neuros-replay extract [--stream stdout|stderr|both] + Dump the embedded `stdout` and/or `stderr` payloads to stdout + so standard host tools (`grep`, `jq`, `less`) can read them + without escaping JSON strings. With both streams selected + the output gets a one-line ``===STDERR===`` separator so a + human or downstream tool can tell them apart. + + Common flags: + --json emit a single-line JSON envelope + on stdout (machines, not the diaper) +""" +import argparse +import json +import os +import sys + +# --- Constants ---------------------------------------------------------- + +#: Default tolerance (percent) on wall_clock_ms in `diff`. Single +#: observations on a shared kernel are noisier than bench's batch +#: aggregates, so 10% is the floor; users can tighten via +#: --tolerance-wall-pct. +DEFAULT_TOLERANCE_WALL_PCT = 10.0 + +#: Default tolerance for peak_mem_estimate in `diff`. Memory pressure +#: jitter on Linux is much higher than CPU jitter, hence 20%. +DEFAULT_TOLERANCE_MEM_PCT = 20.0 + +#: Tool version baked into emitted envelopes. +NEUROS_REPLAY_VERSION = "0.1" + + +# --- Exceptions -------------------------------------------------------- + + +class ReplayError(Exception): + """Raised on user-fixable misconfiguration (bad envelope, missing + field, ambiguous stream selector). Surfaced as exit code 2.""" + + +# --- Helpers ------------------------------------------------------------ + + +def _die(msg, code=2): + print(msg, file=sys.stderr) + sys.exit(code) + + +def _load_envelope(path): + """Read one JSON envelope from ``path`` ('-' for stdin). The + loader enforces 'object' at the top level so we can fail loudly + on user error instead of AttributeError-ing into TypeError. + """ + if path == "-": + try: + text = sys.stdin.read() + except OSError as e: + raise ReplayError(f"cannot read stdin: {e}") + else: + try: + with open(path, "r", encoding="utf-8") as f: + text = f.read() + except OSError as e: + raise ReplayError(f"cannot read envelope {path!r}: {e}") + try: + obj = json.loads(text) + except json.JSONDecodeError as e: + raise ReplayError(f"envelope {path!r} is not valid JSON: {e}") + if not isinstance(obj, dict): + raise ReplayError( + f"envelope must be a JSON object (got {type(obj).__name__})") + return obj + + +def _emit_json(payload): + print(json.dumps(payload, separators=(",", ":"), ensure_ascii=False)) + + +def _pct_delta(a, b): + """Return ``(b - a) / a * 100`` rounded to 2 dp, or ``None`` when + either operand is missing/zero (zero is a real workload + observation but divides to infinity; we surface it as None rather + than blowing up the diff with inf). + """ + if a is None or b is None: + return None + try: + a_f = float(a) + b_f = float(b) + except (TypeError, ValueError): + return None + if a_f == 0: + return None + return round((b_f - a_f) / a_f * 100.0, 2) + + +# --- diff -------------------------------------------------------------- + + +def _envelope_for_diff(env): + """Trim a sandbox envelope to the fields we score on. Anything + else (stdout/stderr) is irrelevant to a regression verdict and + would just bloat the output envelope. + " + """ + return { + "exit_code": env.get("exit_code"), + "wall_clock_ms": env.get("wall_clock_ms"), + "peak_mem_estimate": env.get("peak_mem_estimate"), + "timeout_hit": env.get("timeout_hit"), + } + + +def diff_envelopes(a, b, tol_wall_pct, tol_mem_pct): + """Compare two envelopes. Returns a dict ready for --json + output. Rule violations list human-readable strings so the + diaper (text mode) is informative without --json. + """ + violations = [] + aa, bb = _envelope_for_diff(a), _envelope_for_diff(b) + + # Exit code must match. + if aa.get("exit_code") != bb.get("exit_code"): + violations.append( + f"exit_code differs ({aa.get('exit_code')} vs " + f"{bb.get('exit_code')})") + + # Wall clock pct delta within --tolerance-wall-pct. + wall_pct = _pct_delta(aa.get("wall_clock_ms"), + bb.get("wall_clock_ms")) + if wall_pct is not None and abs(wall_pct) > tol_wall_pct: + violations.append( + f"wall_clock_pct_delta ({wall_pct}%) exceeds tolerance " + f"({tol_wall_pct}%)") + + # Peak memory pct delta within --tolerance-mem-pct. Skip when + # either side is None (the field was absent, common on hosts + # without cgroup delegation). + mem_pct = _pct_delta(aa.get("peak_mem_estimate"), + bb.get("peak_mem_estimate")) + if (mem_pct is not None + and aa.get("peak_mem_estimate") is not None + and bb.get("peak_mem_estimate") is not None + and abs(mem_pct) > tol_mem_pct): + violations.append( + f"peak_mem_pct_delta ({mem_pct}%) exceeds tolerance " + f"({tol_mem_pct}%)") + + # timeout_hit must match. + if aa.get("timeout_hit") != bb.get("timeout_hit"): + violations.append( + f"timeout_hit differs ({aa.get('timeout_hit')} vs " + f"{bb.get('timeout_hit')})") + + return { + "ok": not violations, + "exit_code_a": aa.get("exit_code"), + "exit_code_b": bb.get("exit_code"), + "wall_clock_pct_delta": wall_pct, + "peak_mem_pct_delta": mem_pct, + "rule_violations": violations, + "tolerance_wall_pct": tol_wall_pct, + "tolerance_mem_pct": tol_mem_pct, + "neuros_replay_version": NEUROS_REPLAY_VERSION, + } + + +def cmd_diff(args): + try: + a = _load_envelope(args.a) + b = _load_envelope(args.b) + except ReplayError as e: + print(f"neuros-replay: {e}", file=sys.stderr) + return 2 + verdict = diff_envelopes(a, b, args.tolerance_wall_pct, + args.tolerance_mem_pct) + if args.json: + _emit_json(verdict) + else: + if verdict["ok"]: + print(f"neuros-replay: diff OK " + f"(wall_delta={verdict['wall_clock_pct_delta']}%, " + f"mem_delta={verdict['peak_mem_pct_delta']}%)") + else: + print(f"neuros-replay: {len(verdict['rule_violations'])} " + f"violation(s):", file=sys.stderr) + for v in verdict["rule_violations"]: + print(f" - {v}", file=sys.stderr) + return 0 if verdict["ok"] else 1 + + +# --- explain ------------------------------------------------------------ + + +def _format_bytes(n): + """Humanize a byte count into 'MiB' style. None → 'n/a'. + """ + if n is None: + return "n/a" + try: + v = float(n) + except (TypeError, ValueError): + return "n/a" + if v >= 1024 * 1024: + return f"{v / 1024 / 1024:.1f}MiB" + if v >= 1024: + return f"{v / 1024:.1f}KiB" + return f"{int(v)}B" + + +def explain_envelope(env): + """Build the single-line explanation. The shape is grep-friendly: + cx field names without separators so it's easy to pipe through + awk/sed downstream. + """ + return { + "exit_code": env.get("exit_code", "n/a"), + "wall_clock_ms": env.get("wall_clock_ms", "n/a"), + "peak_mem_estimate": env.get("peak_mem_estimate", "n/a"), + "timeout_hit": env.get("timeout_hit", False), + "stdout_bytes": len(env.get("stdout", "") or ""), + "stderr_bytes": len(env.get("stderr", "") or ""), + } + + +def _explain_line(info): + return (f"exit={info['exit_code']} " + f"wall={info['wall_clock_ms']}ms " + f"peak={_format_bytes(info['peak_mem_estimate'])} " + f"timeout={info['timeout_hit']} " + f"stdout_bytes={info['stdout_bytes']} " + f"stderr_bytes={info['stderr_bytes']}") + + +def cmd_explain(args): + try: + env = _load_envelope(args.envelope) + except ReplayError as e: + print(f"neuros-replay: {e}", file=sys.stderr) + return 2 + info = explain_envelope(env) + if args.json: + info["neuros_replay_version"] = NEUROS_REPLAY_VERSION + _emit_json(info) + else: + print(_explain_line(info)) + return 0 + + +# --- extract ------------------------------------------------------------ + + +_STREAMS = ("stdout", "stderr", "both") + + +def cmd_extract(args): + try: + env = _load_envelope(args.envelope) + except ReplayError as e: + print(f"neuros-replay: {e}", file=sys.stderr) + return 2 + if args.stream not in _STREAMS: + print(f"neuros-replay: --stream must be one of {_STREAMS} " + f"(got {args.stream!r})", file=sys.stderr) + return 2 + out = env.get("stdout", "") or "" + err = env.get("stderr", "") or "" + if args.stream == "stdout": + sys.stdout.write(out) + elif args.stream == "stderr": + sys.stdout.write(err) + else: + sys.stdout.write(out) + if out and not out.endswith("\n"): + sys.stdout.write("\n") + sys.stdout.write("===STDERR===\n") + sys.stdout.write(err) + if err and not err.endswith("\n"): + sys.stdout.write("\n") + return 0 + + +# --- Argparse ----------------------------------------------------------- + + +def _build_parser(): + p = argparse.ArgumentParser( + prog="neuros-replay", + description=( + "Offline diagnostics over neuros-sandbox JSON envelopes. " + "Diff two envelopes, humanize one, or extract embedded " + "stdout/stderr." + ), + epilog=( + "Examples:\n" + " neuros-replay diff ./baseline.json ./candidate.json\n" + " neuros-replay explain ./envelope.json\n" + " neuros-replay extract ./envelope.json --stream stderr" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = p.add_subparsers(dest="cmd", required=True) + + diff_p = sub.add_parser("diff", + help="Compare two envelopes with a " + "regression verdict") + diff_p.add_argument("a", help="Path to baseline envelope " + "JSON ('-' for stdin)") + diff_p.add_argument("b", help="Path to candidate envelope " + "JSON ('-' for stdin)") + diff_p.add_argument("--tolerance-wall-pct", type=float, + default=DEFAULT_TOLERANCE_WALL_PCT, + help=f"Tolerance on wall-clock pct delta " + f"(default: {DEFAULT_TOLERANCE_WALL_PCT})") + diff_p.add_argument("--tolerance-mem-pct", type=float, + default=DEFAULT_TOLERANCE_MEM_PCT, + help=f"Tolerance on peak-mem pct delta " + f"(default: {DEFAULT_TOLERANCE_MEM_PCT})") + diff_p.add_argument("--json", action="store_true", + help="Emit a one-line JSON envelope instead " + "of text") + + expl_p = sub.add_parser("explain", + help="Humanize one envelope to a " + "grep-friendly summary line") + expl_p.add_argument("envelope", + help="Path to envelope JSON ('-' for stdin)") + expl_p.add_argument("--json", action="store_true", + help="Emit a one-line JSON envelope instead " + "of text") + + ext_p = sub.add_parser("extract", + help="Dump embedded stdout/stderr from " + "an envelope") + ext_p.add_argument("envelope", + help="Path to envelope JSON ('-' for stdin)") + ext_p.add_argument("--stream", choices=_STREAMS, default="stdout", + help="Which stream to emit " + "(default: stdout)") + return p + + +def main(argv=None): + parser = _build_parser() + args = parser.parse_args(argv) + if args.cmd == "diff": + return cmd_diff(args) + if args.cmd == "explain": + return cmd_explain(args) + if args.cmd == "extract": + return cmd_extract(args) + parser.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/config/includes.chroot/usr/local/bin/neuros-runbook b/config/includes.chroot/usr/local/bin/neuros-runbook new file mode 100755 index 0000000..730ccf1 --- /dev/null +++ b/config/includes.chroot/usr/local/bin/neuros-runbook @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +"""neuros-runbook: assertion-bearing runner for neuros-sandbox. + +The runbook format is JSON (one array of steps). Each step has: + + { + "name": "", + "script": "", + "limits": {"mem": "...", "pids": N, "timeout": N, "cpu_quota": N}, + "expect": { + "exit_code": N, + "stdout_matches": "", + "stderr_matches": "", + "timeout_forbidden": bool + } + } + +Output: per-step verdict on stdout (human-readable table) plus a +JSON envelope when --json is set (one record per step). Exit 0 if +every step passes, exit 1 otherwise. + +With --rest=violations-only, the human table only shows failed +steps; --rest=all shows every step. Default = all. + +The wrapper shells out to neuros-sandbox exactly like the bench +does, so they share the same envelope schema. That means runbook +verdicts can be diffed against production captures to detect when an +OS upgrade silently changes what a step's stdout looks like. +""" +import argparse +import json +import os +import re +import subprocess +import sys +import uuid + +# --- Defaults ---------------------------------------------------------- + +DEFAULT_TIMEOUT = 30 +DEFAULT_MEM = "256M" +DEFAULT_PIDS = 64 +DEFAULT_CPU_QUOTA = 50000 # 50% of one CPU +NEUROS_RUNBOOK_VERSION = "0.1" + + +# --- Runbook loading ---------------------------------------------------- + +class RunbookError(Exception): + """Raised on a malformed runbook the user can fix (not a kernel fault).""" + + +def load_runbook(path): + """Read and validate a JSON runbook file. + + Returns a list of step dicts. Each step has: + - name (str, required) + - script (str, required, non-empty) + - limits (dict, optional): per-step incisor for mem/pids/timeout/cpu_quota + - expect (dict, optional): exit_code (int or "any"), stdout_matches, + stderr_matches (regex str), timeout_forbidden + + Raises RunbookError on any structural problem with a message + that's safe to surface on stderr. + """ + try: + with open(path) as f: + raw = json.load(f) + except (OSError, json.JSONDecodeError) as e: + raise RunbookError(f"cannot read runbook {path!r}: {e}") + if not isinstance(raw, list): + raise RunbookError( + f"runbook {path!r} must be a JSON array of step objects") + seen_names = set() + for i, step in enumerate(raw): + if not isinstance(step, dict): + raise RunbookError(f"step {i}: must be an object") + name = step.get("name") + if not isinstance(name, str) or not name: + raise RunbookError(f"step {i}: 'name' must be a non-empty string") + if name in seen_names: + raise RunbookError( + f"step {i}: duplicate name {name!r}") + seen_names.add(name) + script = step.get("script") + if not isinstance(script, str) or not script: + raise RunbookError( + f"step {name!r}: 'script' must be a non-empty string") + limits = step.get("limits", {}) + if not isinstance(limits, dict): + raise RunbookError( + f"step {name!r}: 'limits' must be an object") + expect = step.get("expect", {}) + if not isinstance(expect, dict): + raise RunbookError( + f"step {name!r}: 'expect' must be an object") + # Pre-compile regexes so a malformed pattern fails the parser + # rather than the runtime assertion. Failures here are + # reported as RunbookError with the step name for grep-ability. + for key in ("stdout_matches", "stderr_matches"): + pat = expect.get(key) + if pat is not None: + if not isinstance(pat, str): + raise RunbookError( + f"step {name!r}: expect.{key} must be a string regex") + try: + re.compile(pat) + except re.error as e: + raise RunbookError( + f"step {name!r}: expect.{key} regex invalid: {e}") + if "exit_code" in expect: + ec = expect["exit_code"] + if ec != "any" and not isinstance(ec, int): + raise RunbookError( + f"step {name!r}: expect.exit_code must be an int or 'any'") + return raw + + +# --- Single-step execution ---------------------------------------------- + +def _run_step(step, env): + """Shell out to neuros-sandbox and return the parsed envelope plus + per-step assertion results.""" + name = step["name"] + script = step["script"] + limits = step.get("limits", {}) + argv = ["neuros-sandbox", "run", + "--timeout", str(limits.get("timeout", DEFAULT_TIMEOUT)), + "--mem", limits.get("mem", DEFAULT_MEM), + "--pids", str(limits.get("pids", DEFAULT_PIDS)), + "--name", f"neuros-runbook-{uuid.uuid4().hex[:8]}", + "--json", "--"] + cp = subprocess.run( + argv, + input=script.encode(), + env=env, + capture_output=True, + check=False, + timeout=limits.get("timeout", DEFAULT_TIMEOUT) + 10, + ) + stdout_text = cp.stdout.decode("utf-8", "replace").strip() + env_obj = {"exit_code": cp.returncode} + if stdout_text: + try: + env_obj = json.loads(stdout_text.splitlines()[-1]) + except json.JSONDecodeError: + env_obj.setdefault("stdout", stdout_text) + env_obj.setdefault("stdout", "") + env_obj.setdefault("stderr", + cp.stderr.decode("utf-8", "replace")) + env_obj.setdefault("wall_clock_ms", 0) + env_obj.setdefault("timeout_hit", False) + env_obj.setdefault("peak_mem_estimate", None) + + # Apply assertions. + expect = step.get("expect", {}) + failures = [] + if "exit_code" in expect: + ec = expect["exit_code"] + if ec != "any" and env_obj["exit_code"] != ec: + failures.append( + f"exit_code was {env_obj['exit_code']}, expected {ec}") + for key, text in (("stdout_matches", env_obj["stdout"]), + ("stderr_matches", env_obj["stderr"])): + pat = expect.get(key) + if pat is None: + continue + if not re.search(pat, text): + failures.append(f"{key} did not match /{pat}/") + if expect.get("timeout_forbidden") and env_obj["timeout_hit"]: + failures.append("timeout_hit=true but timeout_forbidden=true") + + return env_obj, failures + + +# --- Aggregation -------------------------------------------------------- + +def _run_runbook(path, env, only_violations=False): + steps = load_runbook(path) + record = {"version": NEUROS_RUNBOOK_VERSION, + "runbook_path": path, + "steps": []} + any_fail = False + for step in steps: + env_obj, failures = _run_step(step, env) + passed = not failures + if not passed: + any_fail = True + record["steps"].append({ + "name": step["name"], + "passed": passed, + "exit_code": env_obj["exit_code"], + "wall_clock_ms": env_obj["wall_clock_ms"], + "timeout_hit": env_obj["timeout_hit"], + "peak_mem_estimate": env_obj["peak_mem_estimate"], + "failures": failures, + }) + if only_violations: + visible = [s for s in record["steps"] if not s["passed"]] + else: + visible = record["steps"] + return record, visible, any_fail + + +# --- Output formatting -------------------------------------------------- + +def _print_table(records): + name_w = max(12, max((len(r["name"]) for r in records), default=12)) + print(f"{'name'.ljust(name_w)} {'status':<11} " + f"{'exit':>5} {'wall_ms':>9} {'peak_mem':>11} failures") + for r in records: + status = "PASS" if r["passed"] else "FAIL" + peak = "-" if r["peak_mem_estimate"] is None else str(r["peak_mem_estimate"]) + fail_summary = "" + if r["failures"]: + fail_summary = "; ".join(r["failures"]) + print( + f"{r['name'].ljust(name_w)} {status:<11} " + f"{r['exit_code']:>5} {r['wall_clock_ms']:>9} {peak:>11} " + f"{fail_summary}" + ) + + +# --- Subcommand dispatch ----------------------------------------------- + +def cmd_run(args): + env = { + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + } + try: + record, visible, any_fail = _run_runbook( + args.runbook, env, + only_violations=args.only_violations) + except RunbookError as e: + _die(str(e)) + if args.json: + print(json.dumps(record, indent=2)) + else: + if not visible: + print("(no violations)") + else: + _print_table(visible) + return 1 if any_fail else 0 + + +def _die(msg, code=2): + print(f"neuros-runbook: {msg}", file=sys.stderr) + sys.exit(code) + + +def _build_parser(): + p = argparse.ArgumentParser( + prog="neuros-runbook", + description=( + "Run a JSON runbook of neuros-sandbox steps with assertion " + "checks per step. Exit 0 if all steps pass, 1 on any failure." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = p.add_subparsers(dest="cmd", required=True) + p_run = sub.add_parser("run", help="run a runbook from a JSON file") + p_run.add_argument("runbook", help="path to the runbook JSON file") + p_run.add_argument( + "--only-violations", action="store_true", + help="only print rows that failed (default: all).") + p_run.add_argument( + "--json", action="store_true", + help="emit a JSON envelope instead of a human-readable table") + return p + + +def main(argv=None): + parser = _build_parser() + args = parser.parse_args(argv) + if args.cmd == "run": + rc = cmd_run(args) + return rc if rc is not None else 0 + parser.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/config/includes.chroot/usr/local/bin/neuros-sandbox b/config/includes.chroot/usr/local/bin/neuros-sandbox new file mode 100755 index 0000000..7ce2c60 --- /dev/null +++ b/config/includes.chroot/usr/local/bin/neuros-sandbox @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +"""neuros-sandbox: safe-runner wrapper around the `neuros-container` +primitive, tuned for **untrusted code execution** (e.g. scripts produced +by an LLM agent). The hardcoded defaults are the point of the tool: + + * `--net` : private network namespace (loopback only) + * `--read-only` : rootfs remounted RO + * `--mem 256M` : cgroup memory cap (override w/ --mem) + * `--pids 64` : cgroup PID cap (override w/ --pids) + * `--cpu-quota 50000` : 50% of one CPU (override w/ --cpu or + --cpu-quota) + * `--cap-drop ` : drop a baseline set of dangerous caps + * host env scrubbed : only PATH survives; no ANTHROPIC_API_KEY + or HOME leaks + +`--unsafe` opts in to relaxing the network/read-only/cap-drop trio, +which is the only path that should reach the host for untrusted work. + +Stdin-hand-off: the script body is always read into memory and piped +to `python3 -u -` inside the container. Nothing touches a tempfile on +the host; nothing leaks via /proc//cmdline because the path is +literal `-`. + +A watchdog enforces `--timeout` with SIGKILL on expiry. The output +mode is either `--human` (pass-through, like a normal command) or +`--json` (one line of structured envelope on stdout). + +Examples: + + echo 'print("hello agent")' | neuros-sandbox run --json + neuros-sandbox run --mem 1G --timeout 60 ./scripts/smoke.py + neuros-sandbox run --bundle ./fixture.tar.gz --json pre.sh +""" +import argparse +import glob +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +import uuid + +# --- Defaults -------------------------------------------------------------- + +#: cgroup v2 memory cap (bytes-string accepted by memory.max) +DEFAULT_MEM = "256M" + +#: cgroup v2 pids.max value +DEFAULT_PIDS = 64 + +#: CPU quota in microseconds per 100ms period (50000 = 50%) +DEFAULT_CPU_QUOTA = 50000 + +#: Capabilities dropped by default; regex syntax shared with the +#: primitive's --cap-drop. +DEFAULT_CAP_DROP = ( + r"CAP_(NET_RAW|SYS_ADMIN|SYS_PTRACE|SETUID|SETPCAP|" + r"MAC_ADMIN|DAC_OVERRIDE|LINUX_IMMUTABLE|SYS_CHROOT|" + r"SYS_RAWIO|SYS_RESOURCE)" +) + +#: Pre-compiled DEFAULT_CAP_DROP pattern so _build_argv doesn't +#: re-run re.compile on every wrapper-invocation. The constant is +#: itself validated (re.error at import time if malformed) so a +#: future edit to DEFAULT_CAP_DROP that breaks the regex fails +#: loudly at module load, not at kernel-bridge exec time. +_COMPILED_DEFAULT_CAP_DROP = re.compile(DEFAULT_CAP_DROP) + +#: Hard wall-clock timeout in seconds +DEFAULT_TIMEOUT = 30 + +#: Minimal env that gets passed through to the container. Deliberately +#: tiny; we do NOT inherit the host env. This is the difference between +#: "container" and "sandbox". +MINIMAL_ENV = { + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +} + +#: Name prefix for state files / forensic logs +NAME_PREFIX = "neuros-sbx-" + + +# --- Exceptions ------------------------------------------------------------ + + +class SandboxError(Exception): + """Raised on misconfiguration the user can fix (not kernel faults).""" + + +# --- Helpers --------------------------------------------------------------- + + +def _die(msg, code=2): + print(msg, file=sys.stderr) + sys.exit(code) + + +def _parse_percent(text): + """Parse '50%' or '0.5' or '50000us' (loose) into a CPU quota in + microseconds per 100ms CFS period.""" + s = text.strip() + if s.endswith("%"): + return int(float(s[:-1].strip()) * 1000) + if s.endswith("us"): + return int(s[:-2].strip()) + f = float(s) + if 0 < f <= 1.0: + return int(f * 100000) + return int(f) + + +def _build_argv(args, script_target, rootfs_override=None, name_override=None): + """Translate sandbox options into a neuros-container argv list. + + Order matters because `neuros-container` uses argparse with + `--name`/`--rootfs`/etc as dest flags and the run subcommand + triggers a positional dispatcher. + """ + argv = ["neuros-container", "run"] + argv += ["--name", name_override or args.name + or f"{NAME_PREFIX}{uuid.uuid4().hex[:10]}"] + argv += ["--rootfs", rootfs_override or args.rootfs or "/"] + if not args.unsafe: + argv += ["--net", "--read-only"] + argv += ["--mem", args.mem] + if args.pids is not None and args.pids > 0: + argv += ["--pids", str(args.pids)] + if args.cpu_quota is not None and args.cpu_quota > 0: + argv += ["--cpu-quota", str(args.cpu_quota)] + # The cap-drop baseline is the most load-bearing piece of the + # default safety posture, so it persists under --unsafe. The + # --cap-drop-keep knob is the explicit override path; only active + # in --unsafe mode (in safe mode the user just sets --cap-drop). + # Pre-validate the override regex so an invalid pattern fails + # at the wrapper layer (SandboxError), not deep inside the + # kernel-bridge ctypes call when the primitive rejects it. + if args.unsafe and args.cap_drop_keep is not None: + if not args.cap_drop_keep.strip(): + raise SandboxError( + "--cap-drop-keep must be a non-empty regex (got " + f"{args.cap_drop_keep!r}); omit --cap-drop-keep " + "to use the default") + try: + re.compile(args.cap_drop_keep) + except re.error as e: + raise SandboxError( + f"--cap-drop-keep {args.cap_drop_keep!r} is not a " + f"valid regex: {e}") + argv += ["--cap-drop", args.cap_drop_keep] + else: + argv += ["--cap-drop", args.cap_drop] + if args.user: + argv += ["--user", args.user] + # The script body is always piped via stdin; we use `python3 -u -` + # as a stable, executable-without-tmpfile entry point. When the + # caller passed a literal path, we pass it through argv; when the + # caller feeds stdin (script_target="-"), the "-" entry is already + # in argv so we must NOT append another "-". + argv += ["--", "python3", "-u", "-"] + if script_target != "-": + argv.append(script_target) + return argv + + +def _load_script(script_path): + """Return script bytes. `None` means 'read stdin'.""" + if script_path is None: + data = sys.stdin.buffer.read() + else: + try: + with open(script_path, "rb") as f: + data = f.read() + except OSError as e: + raise SandboxError(f"cannot read script: {script_path}: {e}") + if not data: + raise SandboxError("empty script (no bytes piped or file empty)") + return data + + +def _extract_bundle(tar_path): + """Unpack a tar.gz into a fresh tmp dir; return its path.""" + target = tempfile.mkdtemp(prefix="neuros-sbx-bundle-") + try: + with tarfile.open(tar_path, "r:*") as tar: + # Defensive: refuse absolute paths and `..` traversal + for member in tar.getmembers(): + p = os.path.normpath(member.name) + if p.startswith("/") or p.startswith(".."): + raise SandboxError( + f"unsafe bundle entry: {member.name!r}") + tar.extractall(target) + except (tarfile.TarError, OSError) as e: + shutil.rmtree(target, ignore_errors=True) + raise SandboxError(f"bundle extract failed: {e}") + return target + + +# --- JSON envelope --------------------------------------------------------- + + +def _envelope(rc, stdout, stderr, wall_ms, timeout_hit, peak_mem=None): + """Build a single-line JSON record describing the run.""" + return json.dumps({ + "exit_code": rc, + "stdout": stdout.decode("utf-8", "replace") if stdout else "", + "stderr": stderr.decode("utf-8", "replace") if stderr else "", + "wall_clock_ms": int(wall_ms), + "timeout_hit": timeout_hit, + "peak_mem_estimate": peak_mem, + }, separators=(",", ":"), ensure_ascii=False) + + +# --- cgroup peak-memory polling -------------------------------------------- + +#: Standard cgroup v2 mount root on modern Ubuntu; we keep this as a +#: list because some container hosts remount under /sys/fs/cgroup/. +_CGROUP_V2_ROOTS = ("/sys/fs/cgroup",) + + +def _read_peak_memory(name): + """Best-effort read of memory.peak for a leaf cgroup whose name + contains ``name``. Returns ``None`` on any failure (file missing, + permission denied, parse error). Best-effort only: the primitive + auto-cleans the cgroup on non-detached exit, so the peak file is + often already gone by the time we read. For forensic accuracy run + the container with `--detach` and read the file before calling + `neuros-container cleanup`.""" + suffix = f"neuros-sbx-{name}" if not name.startswith("neuros-sbx-") else name + for root in _CGROUP_V2_ROOTS: + # match either a leaf named the suffix, or a path that ends + # with / somewhere under the root + candidates = glob.glob(os.path.join(root, "**", suffix), + recursive=True) + if not candidates: + # also try a direct path under the root for the common + # case where the cgroup was created in the default tree + direct = os.path.join(root, suffix) + if os.path.isdir(direct): + candidates = [direct] + # When multiple leaves match (a re-runner of the same wrapper + # name), prefer the most recently modified cgroup: that is the + # one whose kernel stats describe the just-finished run. + candidates.sort(key=lambda p: os.path.getmtime(p), reverse=True) + for path in candidates: + peak = os.path.join(path, "memory.peak") + try: + with open(peak) as f: + val = f.read().strip() + return int(val) if val.isdigit() else None + except (OSError, ValueError): + continue + return None + + +# --- Top-level run --------------------------------------------------------- + + +def cmd_run(args): + """Execute the `run` subcommand end-to-end.""" + rootfs_override = None + cleanup_paths = [] + try: + if args.bundle: + rootfs_override = _extract_bundle(args.bundle) + cleanup_paths.append(rootfs_override) + # Build argv BEFORE reading the script so --dry-run can be + # exercised without a real script file on disk. + argv = _build_argv(args, + script_target=args.script or "-", + rootfs_override=rootfs_override) + if args.dry_run: + print(" ".join(argv)) + return 0 + script_bytes = _load_script(args.script) + # Pick the final container name now (the same way _build_argv + # would have chosen it) so the post-run cgroup poll has a + # stable target. + container_name = (args.name + or f"{NAME_PREFIX}{uuid.uuid4().hex[:10]}") + t0 = time.monotonic() + timeout_hit = False + peak_mem = None + try: + cp = subprocess.run( + argv, + env=MINIMAL_ENV, + input=script_bytes, + cwd="/", + capture_output=True, + check=False, + timeout=args.timeout, + ) + except subprocess.TimeoutExpired: + timeout_hit = True + cp = subprocess.CompletedProcess( + argv, 124, b"", b"") + wall_ms = (time.monotonic() - t0) * 1000.0 + # Best-effort peak-memory read; safe in environments without + # cgroup delegation (returns None silently). + peak_mem = _read_peak_memory(container_name) + if args.json: + print(_envelope( + rc=cp.returncode, + stdout=cp.stdout, + stderr=cp.stderr, + wall_ms=wall_ms, + timeout_hit=timeout_hit, + peak_mem=peak_mem, + )) + else: + sys.stdout.buffer.write(cp.stdout or b"") + sys.stderr.buffer.write(cp.stderr or b"") + sys.stderr.write( + f"[neuros-sandbox] exit={cp.returncode} " + f"wall_ms={wall_ms:.0f} timeout_hit={timeout_hit} " + f"peak_mem_estimate={peak_mem}\n" + ) + return 124 if timeout_hit else (cp.returncode or 0) + finally: + for p in cleanup_paths: + shutil.rmtree(p, ignore_errors=True) + + +# --- Argparse -------------------------------------------------------------- + + +def _build_parser(): + p = argparse.ArgumentParser( + prog="neuros-sandbox", + description=( + "Safe-runner for untrusted scripts via neuros-container. " + "Hardened by default; --unsafe relocks the blast radius." + ), + epilog=( + "Examples:\n" + " echo 'print(1+1)' | neuros-sandbox run --json\n" + " neuros-sandbox run --mem 1G --timeout 60 ./smoke.py\n" + " neuros-sandbox run --bundle ./fixture.tar.gz --dry-run\n" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = p.add_subparsers(dest="cmd", required=True) + run = sub.add_parser("run", help="Run a script in a sandboxed container") + + run.add_argument("script", nargs="?", + help="Script file; stdin is read if omitted.") + run.add_argument("--mem", default=DEFAULT_MEM, + help=f"Memory cap (default: {DEFAULT_MEM}); " + "e.g. 256M, 1G.") + run.add_argument("--pids", type=int, default=DEFAULT_PIDS, + help=f"PID cap (default: {DEFAULT_PIDS}).") + run.add_argument("--cpu", + help=f"CPU %% quota; maps to --cpu-quota " + f"(default: {DEFAULT_CPU_QUOTA/1000:.0f}%%).") + run.add_argument("--cpu-quota", type=int, default=DEFAULT_CPU_QUOTA, + help=f"CPU quota µs/100ms (default: " + f"{DEFAULT_CPU_QUOTA}).") + run.add_argument("--cap-drop", default=DEFAULT_CAP_DROP, + help="Regex of Linux capabilities to drop " + "(default: a hardened baseline).") + run.add_argument("--cap-drop-keep", + help="Under --unsafe, replace (not relax) the " + "default cap-drop baseline with this " + "regex (re.compile-validated up front). " + "Ignored in safe mode.") + run.add_argument("--rootfs", + help="Rootfs path inside the container " + "(default: /).") + run.add_argument("--name", + help="Container name; default = auto-random.") + run.add_argument("--user", + help="UID:GID inside the container.") + run.add_argument("--bundle", + help="Tar.gz to extract into the container's " + "rootfs (use with --rootfs pointing inside).") + run.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, + help=f"Wall-clock cap in seconds " + f"(default: {DEFAULT_TIMEOUT}).") + run.add_argument("--unsafe", action="store_true", + help="Disable --net / --read-only / default caps.") + run.add_argument("--json", action="store_true", + help="Emit a single JSON envelope on stdout.") + run.add_argument("--dry-run", action="store_true", + help="Print the neuros-container argv and exit.") + return p + + +def main(argv=None): + parser = _build_parser() + args = parser.parse_args(argv) + if args.cmd == "run": + # Resolve --cpu into --cpu_quota if both weren't set explicitly + if args.cpu is not None: + args.cpu_quota = _parse_percent(args.cpu) + try: + return cmd_run(args) + except SandboxError as e: + _die(f"neuros-sandbox: {e}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/config/includes.chroot/usr/local/bin/neuros-verify b/config/includes.chroot/usr/local/bin/neuros-verify new file mode 100755 index 0000000..a05c078 --- /dev/null +++ b/config/includes.chroot/usr/local/bin/neuros-verify @@ -0,0 +1,581 @@ +#!/usr/bin/env python3 +"""neuros-verify: consume-anything verifier for the neuros toolchain. + +The toolchain's emitters (sandbox, bench, runbook, policy, replay) +all produce JSON envelopes with slightly different shapes. +`neuros-verify` is the single consumer that reads any of them, +identifies the kind by version key first then by structural +fingerprint, and emits a normalized PASS/FAIL verdict line on +stdout. Multiple files are AND'd so any single failure fails the +overall run. + +This tool does not introduce new envelope shapes; it routes what +already exists. The single edge is `--policy ` for raw +sandbox envelopes: the policy's `check_envelope_against_policy` +is loaded via runtime `compile()+exec()` against the on-disk +neuros-policy script so this binary stays a thin client of the +verifier logic instead of re-implementing it. + +Usage: + + neuros-verify [--policy POLICY] [--bench-baseline BASELINE] + [--json] [--quiet] + [FILE ... | -] + +Without --json the output is grep-friendly: + + verdict=PASS source=sandbox file=./env.json violations=0 ... + verdict=PASS source=aggregate files=3 passes=3 failures=0 + +With --json a single-line envelope with {ok, aggregate, parts[]}: + + {"ok": true, "aggregate": {...}, "parts": [...], + "neuros_verify_version": "0.1"} + +Exit codes: 0 all conformant; 1 any rule violation or unrecognized +envelope; 2 bad input (missing file, malformed JSON, root not an +object). + +Note: bench envelopes without --bench-baseline are evaluated as +"no run crashed" (every run's exit_code is 0). That is the safe +default — performance regression checks live in `neuros-bench +compare` and `neuros-replay diff`. Pass `--bench-baseline` here +only when you want a single-line verdict side-by-side with the +other consumer verdicts. +""" +import argparse +import json +import os +import sys +import types + +# --- Constants ---------------------------------------------------------- + +#: Tool version baked into emitted envelopes. +NEUROS_VERIFY_VERSION = "0.1" + +#: Order in which we search for the neuros-policy SCRIPT when we +#: need its ``_load_policy`` / ``validate_policy`` / +#: ``check_envelope_against_policy`` helpers. The SCRIPT and the +#: MANIFEST are distinct: --policy is the manifest, and the +#: script loader auto-discovers via this candidate list. On a +#: system install the second-to-last entry hits; in dev mode +#: the first entry (sibling of neuros-verify) keeps it working +#: without an install step. +#: lazily-built candidate list of neuros-policy script paths; +#: built per call inside _policy_script_candidates() so the +#: env override (NEUROS_POLICY_SCRIPT) is read at call time, +#: not at module-import time. +#: Default tolerances for the bench-with-baseline path. Same as +#: neuros-bench compare's defaults; documented here so a routing +#: decision made in this tool matches the policy of the underlying +#: bench workload. +BENCH_TOL_WALL_PCT = 15.0 +BENCH_TOL_MEM_PCT = 25.0 + +#: Detected envelope kinds, in resolution priority order. First match +#: wins; structural fingerprint fallback comes after the explicit +#: version-key checks. +_KIND_VERSION_KEYS = ( + ("neuros_bench_version", "bench"), + ("neuros_policy_version", "policy"), + ("neuros_replay_version", "replay"), +) + + +# --- Exceptions -------------------------------------------------------- + + +class VerifyError(Exception): + """Raised on user-fixable misconfiguration (missing file, malformed + JSON, wrong root type). Surfaced as exit code 2.""" + + +# --- Helpers ------------------------------------------------------------ + + +def _die(msg, code=2): + print(msg, file=sys.stderr) + sys.exit(code) + + +def _load_envelope(path): + """Read one JSON envelope from ``path`` ('-' for stdin).""" + if path == "-": + try: + text = sys.stdin.read() + except OSError as e: + raise VerifyError(f"cannot read stdin: {e}") + else: + try: + with open(path, "r", encoding="utf-8") as f: + text = f.read() + except OSError as e: + raise VerifyError(f"cannot read envelope {path!r}: {e}") + try: + obj = json.loads(text) + except json.JSONDecodeError as e: + raise VerifyError(f"envelope {path!r} is not valid JSON: {e}") + if not isinstance(obj, dict): + raise VerifyError( + f"envelope must be a JSON object (got {type(obj).__name__})") + return obj + + +def _detect_kind(env): + """Return one of {'bench', 'policy', 'replay', 'sandbox', + 'unknown'}. Version keys take priority; structural fingerprint + fallback for raw sandbox envelopes; everything else is unknown. + """ + for key, kind in _KIND_VERSION_KEYS: + if key in env: + return kind + # Runbook is detected by structural fingerprint of its top-level + # {runbook_path, steps} record. If the runbook tool doesn't yet + # emit that shape, fall through to sandbox/unknown. + if "runbook_path" in env and "steps" in env and isinstance( + env.get("steps"), list): + return "runbook" + if ("exit_code" in env + and "wall_clock_ms" in env + and "timeout_hit" in env): + return "sandbox" + return "unknown" + + +def _pct_delta(a, b): + """Round-to-2dp percent change; ``None`` if either is missing or + the denominator is zero. Mirrors neuros-replay's helper so a + verdict from this tool and a verdict from replay agree on + numeric shape. + """ + if a is None or b is None: + return None + try: + a_f = float(a); b_f = float(b) + except (TypeError, ValueError): + return None + if a_f == 0: + return None + return round((b_f - a_f) / a_f * 100.0, 2) + + +def _load_policy_module(path): + """``compile()+exec()`` the policy script into a types.ModuleType + so we can call ``check_envelope_against_policy`` without importing + it as a real package. Mirrors the test_sandbox / test_policy + loader. Returns the module. + """ + with open(path, "r", encoding="utf-8") as f: + src = f.read() + if src.startswith("#!"): + src = src.split("\n", 1)[1] + code = compile(src, path, "exec") + mod = types.ModuleType("neuros_policy_loader") + mod.__file__ = path + exec(code, mod.__dict__) + return mod + + +# --- Per-kind verifier dispatch ---------------------------------------- + + +def _verify_sandbox(env, policy_module, manifest_path): + """Route a sandbox envelope through policy's + check_envelope_against_policy in one pass: load the manifest + from ``manifest_path`` (the JSON file the CLI's --policy + points at), call ``validate_policy`` (refuses to check + against an invalid manifest), then call + ``check_envelope_against_policy``. Each tuple is rendered as + a grep-friendly violation string. + """ + # Defensive ward: this tool calls three names on the policy + # module via runtime compile()+exec(). If neuros-policy renames + # any of these in a future release we want a loud failure here + # rather than an opaque AttributeError deep in the call chain. + expected = ("_load_policy", "validate_policy", + "check_envelope_against_policy") + missing = [n for n in expected if not hasattr(policy_module, n)] + if missing: + public_names = sorted( + k for k in policy_module.__dict__ + if not k.startswith('_')) + return False, [ + f"policy module missing expected names: {missing} " + f"(got: {public_names})" + ] + policy_obj = policy_module._load_policy(manifest_path) + errs, cleaned = policy_module.validate_policy(policy_obj) + if errs: + return False, [ + f"policy has {len(errs)} violation(s); refusing to " + f"check envelope" + ] + violations_t = policy_module.check_envelope_against_policy( + policy_obj, env, cleaned) + return (not violations_t), [ + f"[{sev}] {rule}: observed {obs!r}, expected {exp!r}" + for (rule, obs, exp, sev) in violations_t + ] + + +#: Module-level cache for the loaded policy script. Loading is +#: expensive (compile()+exec of a ~20 KB source), and neuros-verify +#: can process many files in a single invocation when given a +#: glob — recompiling per file would be wasted work. +_CACHED_POLICY_SCRIPT_MOD = None +_CACHED_POLICY_SCRIPT_PATH = None + + +def _policy_script_candidates(): + """Build the script candidate list lazily so the env override + (NEUROS_POLICY_SCRIPT) is read at call time, not at + module-import time. + """ + here = os.path.dirname(os.path.abspath(__file__)) + return ( + os.environ.get("NEUROS_POLICY_SCRIPT"), + os.path.join(here, "neuros-policy"), + "/usr/local/bin/neuros-policy", + ) + + +def _resolve_policy_script(): + """Locate the neuros-policy SCRIPT and load it as a module. + Searches the candidate list in order: ENV override, sibling + of neuros-verify (dev mode), then the system install path. + Caches the loaded module so subsequent calls reuse it. + Raises VerifyError on a missing script. + """ + global _CACHED_POLICY_SCRIPT_MOD, _CACHED_POLICY_SCRIPT_PATH + if _CACHED_POLICY_SCRIPT_MOD is not None: + return _CACHED_POLICY_SCRIPT_MOD + cands = _policy_script_candidates() + for cand in cands: + if not cand: + continue + if os.path.isfile(cand): + _CACHED_POLICY_SCRIPT_PATH = cand + _CACHED_POLICY_SCRIPT_MOD = _load_policy_module(cand) + return _CACHED_POLICY_SCRIPT_MOD + raise VerifyError( + f"cannot locate neuros-policy script (looked at: " + f"{[c for c in cands if c]})") + + +def _verify_bench(env, baseline_path): + """Per-run envelope within tolerances vs a baseline JSON. Each + per-run violation is its own string; the overall verdict is + AND of all per-run results. + """ + if baseline_path is None: + # Without baseline we degrade to crash check: every run's + # exit_code must be 0. + runs = env.get("runs") or [] + if not isinstance(runs, list): + return False, ["bench envelope has no 'runs' list"] + violations = [ + f"run[{i}] exit_code={r.get('exit_code')!r}: non-zero " + f"without baseline" + for i, r in enumerate(runs) + if r.get("exit_code") not in (0, None) + ] + return (not violations), violations + try: + with open(baseline_path, "r") as f: + baseline = json.load(f) + except (OSError, json.JSONDecodeError) as e: + return False, [f"cannot read --bench-baseline: {e}"] + if not isinstance(baseline, dict) or "runs" not in baseline: + return False, [ + f"baseline {baseline_path!r} is not a bench metrics.json" + ] + base_runs = {r.get("name"): r + for r in (baseline.get("runs") or []) + if isinstance(r, dict)} + cand_runs = env.get("runs") or [] + if not isinstance(cand_runs, list): + return False, ["bench envelope has no 'runs' list"] + violations = [] + for r in cand_runs: + if not isinstance(r, dict): + continue + name = r.get("name") + b = base_runs.get(name) + if b is None: + # New workload in candidate is not a regression. + continue + wpct = _pct_delta(b.get("wall_clock_ms"), + r.get("wall_clock_ms")) + if wpct is not None and abs(wpct) > BENCH_TOL_WALL_PCT: + violations.append( + f"run[{name}] wall_clock_pct_delta {wpct}% exceeds " + f"{BENCH_TOL_WALL_PCT}%") + mpct = _pct_delta(b.get("peak_mem_estimate"), + r.get("peak_mem_estimate")) + if (mpct is not None + and b.get("peak_mem_estimate") is not None + and r.get("peak_mem_estimate") is not None + and abs(mpct) > BENCH_TOL_MEM_PCT): + violations.append( + f"run[{name}] peak_mem_pct_delta {mpct}% exceeds " + f"{BENCH_TOL_MEM_PCT}%") + if r.get("exit_code") != b.get("exit_code"): + violations.append( + f"run[{name}] exit_code differs " + f"({b.get('exit_code')!r} vs {r.get('exit_code')!r})") + if r.get("timeout_hit") != b.get("timeout_hit"): + violations.append( + f"run[{name}] timeout_hit differs " + f"({b.get('timeout_hit')!r} vs " + f"{r.get('timeout_hit')!r})") + return (not violations), violations + + +def _verify_policy_verdict(env): + """A policy envelope with an `ok` field; result = not ok.""" + ok = env.get("ok") + if ok is True: + return True, [] + if ok is False: + violations = [str(e) for e in env.get("errors", []) + if isinstance(e, str)] + return False, violations or ["policy verdict ok=False"] + return False, [f"policy envelope has unexpected ok value {ok!r}"] + + +def _verify_replay_verdict(env): + """A replay envelope with an `ok` field; result = not ok.""" + ok = env.get("ok") + if ok is True: + return True, [] + if ok is False: + violations = [str(v) for v in env.get("rule_violations", []) + if isinstance(v, str)] + return False, violations or ["replay verdict ok=False"] + return False, [f"replay envelope has unexpected ok value {ok!r}"] + + +def _verify_runbook(env): + """Per-step structural smoke check. The full assertion-pass + semantics live in the runbook tool itself; here we only verify + the envelope shape (runbook_path present, steps is a list, each + step has a name + ok field). + """ + if "runbook_path" not in env or "steps" not in env: + return False, ["runbook envelope missing runbook_path or steps"] + steps = env.get("steps") + if not isinstance(steps, list): + return False, ["runbook envelope steps is not a list"] + violations = [] + for i, s in enumerate(steps): + if not isinstance(s, dict): + violations.append(f"steps[{i}] is not an object") + continue + if "name" not in s or "ok" not in s: + violations.append( + f"steps[{i}] missing name or ok") + return (not violations), violations + + +def _verify_unknown(env): + """An unrecognized envelope is treated as a violation. We + surface the detected top-level keys so the caller can decide + whether the JSON came from a wrong tool. + """ + return False, [ + f"unrecognized envelope (top-level keys: " + f"{', '.join(sorted(env.keys()))})" + ] + + +# --- Runbook envelope helper ------------------------------------------ + + +def run_verify_for_file(path, args): + """Drive verification for a single file. Returns + ``(kind, ok, violations, metrics_for_line)``. + """ + env = _load_envelope(path) + kind = _detect_kind(env) + if kind == "sandbox": + if args.policy is None: + return (kind, False, [ + "sandbox envelope requires --policy "], {}) + try: + policy_mod = _resolve_policy_script() + except VerifyError as e: + return (kind, False, [f"cannot load policy: {e}"], {}) + ok, viols = _verify_sandbox(env, policy_mod, args.policy) + return (kind, ok, viols, _sandbox_metrics(env)) + if kind == "bench": + ok, viols = _verify_bench(env, args.bench_baseline) + return (kind, ok, viols, _bench_metrics(env)) + if kind == "policy": + ok, viols = _verify_policy_verdict(env) + return (kind, ok, viols, _verdict_metrics(env)) + if kind == "replay": + ok, viols = _verify_replay_verdict(env) + return (kind, ok, viols, _verdict_metrics(env)) + if kind == "runbook": + ok, viols = _verify_runbook(env) + return (kind, ok, viols, {}) + return (kind, False, _verify_unknown(env)[1], {}) + + +def _sandbox_metrics(env): + return { + "exit_code": env.get("exit_code"), + "wall_clock_ms": env.get("wall_clock_ms"), + "peak_mem_estimate": env.get("peak_mem_estimate"), + "timeout_hit": env.get("timeout_hit", False), + } + + +def _bench_metrics(env): + runs = env.get("runs") or [] + return { + "workloads": len(runs) + if isinstance(runs, list) else 0, + "total_wall_clock_ms": env.get("total_wall_clock_ms"), + "neuros_bench_version": env.get("neuros_bench_version"), + } + + +def _verdict_metrics(env): + return { + "rule_violations": len(env.get("rule_violations") + or env.get("errors") or []), + "ok": env.get("ok"), + } + + +# --- Output formatters ------------------------------------------------- + + +def _format_text_line(verdict, kind, file_path, violations, metrics): + base = (f"verdict={'PASS' if verdict else 'FAIL'} " + f"source={kind} file={file_path} " + f"violations={len(violations)}") + extras = [] + for k in ("exit_code", "wall_clock_ms", "peak_mem_estimate", + "timeout_hit", "workloads", "total_wall_clock_ms", + "rule_violations", "ok"): + v = metrics.get(k) + if v is not None: + extras.append(f"{k}={v}") + if extras: + base += " " + " ".join(extras) + return base + + +def _format_aggregate_text(passes, failures): + return (f"verdict={'PASS' if failures == 0 else 'FAIL'} " + f"source=aggregate files_checked={passes + failures} " + f"passes={passes} failures={failures}") + + +# --- Top-level dispatch ----------------------------------------------- + + +def cmd_verify(args): + paths = args.files or ["-"] + parts = [] + passes = failures = 0 + # --json is a single-line-envelope output mode: it replaces the + # grep-friendly text lines rather than adding to them. --quiet + # only suppresses the per-file text lines; the aggregate line + # still prints so a quiet run still reports overall PASS/FAIL. + show_text = not args.json + show_per_file = show_text and not args.quiet + for p in paths: + kind, ok, viols, metrics = run_verify_for_file(p, args) + if show_per_file: + print(_format_text_line(ok, kind, p, viols, metrics), + flush=True) + parts.append({ + "file": p, + "source_kind": kind, + "ok": ok, + "violations": viols, + "metrics": metrics, + }) + if ok: + passes += 1 + else: + failures += 1 + if show_text: + print(_format_aggregate_text(passes, failures), + flush=True) + if args.json: + _emit_json_envelope({ + "ok": failures == 0, + "aggregate": { + "files_checked": passes + failures, + "passes": passes, + "failures": failures, + }, + "parts": parts, + "neuros_verify_version": NEUROS_VERIFY_VERSION, + }) + return 0 if failures == 0 else 1 + + +def _emit_json_envelope(payload): + """Print a single-line envelope mirroring the shape of the + other tools' --json modes. + """ + print(json.dumps(payload, separators=(",", ":"), + ensure_ascii=False)) + + +def _build_parser(): + p = argparse.ArgumentParser( + prog="neuros-verify", + description=( + "Consume-anything verifier for the neuros toolchain " + "JSON envelopes. Reads sandbox / bench / policy / " + "replay envelopes and emits a normalized PASS/FAIL " + "verdict line. Multiple files are AND'd." + ), + epilog=( + "Examples:\n" + " neuros-verify ./envelope.json --policy ./policy.json\n" + " neuros-verify ./baseline.json ./candidate.json " + "--bench-baseline ./baseline.json\n" + " neuros-verify ./env1.json ./env2.json --json" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("--policy", + help="For raw sandbox envelopes, run the " + "named policy manifest's check. Loaded " + "via runtime compile()+exec().") + p.add_argument("--bench-baseline", + help="For bench metrics envelopes, diff each " + "per-workload run against this baseline " + "(tolerances match `neuros-bench compare`).") + p.add_argument("--json", action="store_true", + help="Emit a single-line JSON envelope instead " + "of text") + p.add_argument("--quiet", action="store_true", + help="Suppress per-file lines; only emit the " + "aggregate (last) line") + p.add_argument("files", nargs="*", + help="Envelope files to verify ('-' for stdin)") + return p + + +def main(argv=None): + args = _build_parser().parse_args(argv) + if not args.files: + # Default to stdin so the tool is usable in pipes. + args.files = ["-"] + try: + return cmd_verify(args) + except VerifyError as e: + _die(f"neuros-verify: {e}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/config/includes.chroot/usr/local/bin/neuroslib.py b/config/includes.chroot/usr/local/bin/neuroslib.py index 459eae3..044c16c 100755 --- a/config/includes.chroot/usr/local/bin/neuroslib.py +++ b/config/includes.chroot/usr/local/bin/neuroslib.py @@ -5,12 +5,14 @@ LLM querying, encrypted storage, and file operations. Usage (in other neuros-* tools): - from neuroslib import load_config, query_llm, load_db, save_db + from neuroslib import load_config, get_default_model, get_context_config, + query_llm, load_db, save_db This eliminates code duplication across 70+ Neuros tools. """ import base64 +import configparser import hashlib import json import os @@ -24,25 +26,156 @@ CONFIG_DIR = os.path.expanduser("~/.config/neuros") LLM_CONF = os.path.join(CONFIG_DIR, "llm.conf") +# === Defaults === +DEFAULT_MODEL = "mistral" +DEFAULT_HOST = "localhost" +DEFAULT_PORT = "11434" + +# Opt-in context sources. Every source defaults to False (off). +DEFAULT_CONTEXT_SOURCES = frozenset({"window_title", "clipboard", "recent_files"}) + +# Truthy string values for boolean coercion in llm.conf. +_TRUTHY = frozenset({"true", "1", "yes", "on"}) + # ═══════════════════════════════════════════════════════════════════════ # Config Loading # ═══════════════════════════════════════════════════════════════════════ -def load_config(): - """Load llm.conf settings. Returns dict with model, host, port.""" - cfg = {"model": "mistral", "host": "localhost", "port": "11434"} +def _parse_bool(text): + if text is None: + return False + return text.strip().lower() in _TRUTHY + + +def _strip_quotes(text): + """Drop one pair of surrounding matching single/double quotes + from a configparser value. configparser preserves quotation verbatim; + the previous naive key=value loop stripped quotes so existing + llm.conf files (most of which write ``model = "mistral"``) still + round-trip identically under the new parser.""" + if text is None: + return None + text = text.strip() + if len(text) >= 2 and text[0] == text[-1] and text[0] in ('"', "'"): + return text[1:-1].strip() + return text + + +def _read_parser(path): + """Return a configparser.RawConfigParser with interpolation off + (we use '=' in model names like 'qwen2.5:7b' that would otherwise + confuse interpolators). Returns a defaults-only parser if the + file is missing, unreadable, or empty. + + The previous naive parser happily consumed flat ``key = value`` + files written without a section header. configparser does not — + it raises ``MissingSectionHeaderError``. To preserve back-compat + with existing llm.conf files in the wild, we detect the no-section + case and wrap the content in a synthetic ``[llm]`` block before + handing it to configparser.""" + parser = configparser.RawConfigParser( + interpolation=None, + allow_no_value=True, + inline_comment_prefixes=("#",), + ) + # Default fallbacks; overridden by values present in the file. + parser.add_section("llm") + parser.set("llm", "model", DEFAULT_MODEL) + parser.set("llm", "host", DEFAULT_HOST) + parser.set("llm", "port", DEFAULT_PORT) + + if path and os.path.exists(path): + try: + with open(path, "r", encoding="utf-8") as f: + raw = f.read() + except (OSError, UnicodeDecodeError): + return parser + try: + if raw.lstrip().startswith("["): + parser.read_string(raw) + else: + parser.read_string("[llm]\n" + raw) + except configparser.Error: + return parser + return parser + + +def load_config(path=None): + """Load llm.conf as a flat dict with section-prefixed keys (e.g. + 'llm.model', 'context.window_title'). When ``path`` is None the + default ``LLM_CONF`` is read. A missing or unreadable file returns + defaults — callers don't need to guard. Quoted string values are + unquoted on the way out so files written by the old naive parser + (``model = "mistral"``) still round-trip identically under + configparser, which preserves quotes verbatim.""" + parser = _read_parser(path or LLM_CONF) + flat = { + "llm.model": _strip_quotes(parser.get("llm", "model", fallback=DEFAULT_MODEL)), + "llm.host": _strip_quotes(parser.get("llm", "host", fallback=DEFAULT_HOST)), + "llm.port": _strip_quotes(parser.get("llm", "port", fallback=DEFAULT_PORT)), + } + # Always include every known context source at False so callers + # don't need to guard for missing keys; absent [context] section + # is a valid configuration state, not a malformed one. + for key in DEFAULT_CONTEXT_SOURCES: + flat[f"context.{key}"] = _parse_bool( + parser.get("context", key, fallback=None) + ) + for section in parser.sections(): + if section in ("llm", "context"): + continue + for key, value in parser.items(section): + flat[f"{section}.{key}"] = _strip_quotes(value) + return flat + + +def get_default_model(path=None): + """Return the configured default model, falling back to DEFAULT_MODEL.""" + cfg = load_config(path) + model = cfg.get("llm.model", DEFAULT_MODEL).strip().strip('"').strip("'") + return model or DEFAULT_MODEL + + +def get_context_config(path=None): + """Return the dict of opt-in system-context sources, all False by + default. See the [context] section of llm.conf.""" + cfg = load_config(path) + return { + source: bool(cfg.get(f"context.{source}", False)) + for source in DEFAULT_CONTEXT_SOURCES + } + + +def set_default_model(name, path=None): + """Persist a new default model in llm.conf under [llm]. Creates the + file and parent directories if needed. Existing lines in [llm] are + preserved; only the ``model`` key is overwritten. The write goes + through a ``.tmp`` + ``os.replace`` so a crash mid-write leaves + either the previous file or the new file, never a half-written + one — and the ``.tmp`` is cleaned up on failure.""" + config_path = path or LLM_CONF + parser = _read_parser(config_path) + if not parser.has_section("llm"): + parser.add_section("llm") + parser.set("llm", "model", name) + + target_dir = os.path.dirname(config_path) or CONFIG_DIR + os.makedirs(target_dir, exist_ok=True) + tmp = config_path + ".tmp" try: - if os.path.exists(LLM_CONF): - with open(LLM_CONF) as f: - for line in f: - line = line.strip() - if "=" in line and not line.startswith("#"): - k, v = line.split("=", 1) - cfg[k.strip()] = v.strip() + with open(tmp, "w", encoding="utf-8") as f: + parser.write(f) + os.replace(tmp, config_path) except Exception: - pass - return cfg + # ``parser.write(f)`` raising mid-write would leave the tmp + # file behind; unlink it so the user's config dir doesn't + # accumulate orphan .tmp files. + try: + os.unlink(tmp) + except OSError: + pass + raise # ═══════════════════════════════════════════════════════════════════════ diff --git a/config/includes.chroot/usr/local/bin/nn b/config/includes.chroot/usr/local/bin/nn index 4606bf3..6af6365 100755 --- a/config/includes.chroot/usr/local/bin/nn +++ b/config/includes.chroot/usr/local/bin/nn @@ -8,52 +8,42 @@ Usage: """ import sys, os, subprocess, json -try: - import requests -except ImportError: - print("requests not installed. Run: pip3 install requests") - sys.exit(1) +import urllib.request +import urllib.error + +# Shared library lives next to this script in /usr/local/bin; importing +# it by path keeps ``nn`` robust to whatever the on-disk PYTHONPATH is +# (the build installs them side by side). +import importlib.util +_LIB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "neuroslib.py") +_spec = importlib.util.spec_from_file_location("neuroslib", _LIB_PATH) +neuroslib = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(neuroslib) + +OLLAMA_URL = f"http://{neuroslib.DEFAULT_HOST}:{neuroslib.DEFAULT_PORT}/api/generate" +CONFIG_PATH = neuroslib.LLM_CONF +DEFAULT_MODEL = "mistral" # mirrors neuroslib.DEFAULT_MODEL; kept as a literal so static assertions can grep for it. +# Fails loudly if the canonical value ever drifts away from this +# mirror, instead of silently producing stale container images. +assert DEFAULT_MODEL == neuroslib.DEFAULT_MODEL, ( + f"nn.DEFAULT_MODEL {DEFAULT_MODEL!r} drifted from " + f"neuroslib.DEFAULT_MODEL {neuroslib.DEFAULT_MODEL!r} — keep them in sync." +) -OLLAMA_URL = "http://localhost:11434/api/generate" -CONFIG_PATH = os.path.expanduser("~/.config/neuros/llm.conf") -DEFAULT_MODEL = "mistral" def get_model(): - try: - with open(CONFIG_PATH) as f: - for line in f: - line = line.strip() - # Parse both flat key=value and TOML-style key = "value" - if '=' in line and not line.startswith('['): - key, val = line.split('=', 1) - if key.strip() == "model": - return val.strip().strip('"').strip("'") - except: - pass - return DEFAULT_MODEL + """Return the configured default model. Reads llm.conf at + CONFIG_PATH; falls back to ``neuroslib.DEFAULT_MODEL`` if unset or + unreadable. ``CONFIG_PATH`` is kept as a module-level redirect so + existing callers (and the test suite's patches) still bind here.""" + return neuroslib.get_default_model(CONFIG_PATH) + def get_context_config(): """Read opt-in system-context settings from the [context] section of llm.conf. Every source defaults to False (off) unless the config explicitly turns it on, including when the section is missing entirely.""" - settings = {"window_title": False, "clipboard": False, "recent_files": False} - section = None - try: - with open(CONFIG_PATH) as f: - for line in f: - line = line.strip() - if line.startswith('['): - section = line.strip('[]').strip() - continue - if section == "context" and '=' in line: - key, val = line.split('=', 1) - key = key.strip() - val = val.strip().strip('"').strip("'").lower() - if key in settings: - settings[key] = val in ("true", "1", "yes", "on") - except: - pass - return settings + return neuroslib.get_context_config(CONFIG_PATH) def get_active_window_title(): """Return the title of the focused window via xdotool, if installed. @@ -152,23 +142,34 @@ def read_file_safe(path): return f"(Could not read file: {e})" def stream_response(prompt, model): + """Stream a single prompt to Ollama. Uses urllib so nn needs no + third-party deps (the live-build target pulls only stdlib). The + previous implementation required the ``requests`` package and + crashed hard if it wasn't installed.""" + body = json.dumps({"model": model, "prompt": prompt, "stream": True}).encode() + req = urllib.request.Request( + OLLAMA_URL, data=body, + headers={"Content-Type": "application/json"}, + ) try: - resp = requests.post(OLLAMA_URL, json={ - "model": model, - "prompt": prompt, - "stream": True - }, stream=True, timeout=120) - resp.raise_for_status() - for line in resp.iter_lines(): - if line: - data = json.loads(line) + with urllib.request.urlopen(req, timeout=120) as resp: + for line in resp: + if not line: + continue + try: + data = json.loads(line) + except json.JSONDecodeError: + continue print(data.get("response", ""), end="", flush=True) if data.get("done"): print() - break - except requests.exceptions.ConnectionError: - print("Error: NeurOS LLM daemon is not running.") - print("Start it with: sudo systemctl start neuros-llm") + return + except urllib.error.URLError as e: + if isinstance(e.reason, ConnectionRefusedError): + print("Error: NeurOS LLM daemon is not running.") + print("Start it with: sudo systemctl start neuros-llm") + else: + print(f"Error: {e}") except Exception as e: print(f"Error: {e}") diff --git a/config/package-lists/neuros.list.chroot b/config/package-lists/neuros.list.chroot index 93fa823..9609dd6 100644 --- a/config/package-lists/neuros.list.chroot +++ b/config/package-lists/neuros.list.chroot @@ -29,7 +29,7 @@ gir1.2-gtk-3.0 gir1.2-appindicator3-0.1 # Audio (for voice/speech) -also-utils +alsa-utils espeak-ng # Screenshot & OCR diff --git a/scripts/check-airgap.sh b/scripts/check-airgap.sh new file mode 100755 index 0000000..d140d30 --- /dev/null +++ b/scripts/check-airgap.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# check-airgap.sh — Fail CI if a shipped runtime tool gains a new outbound +# network call to a host other than localhost/the local Ollama endpoint. +# +# Scope: config/includes.chroot/usr/local/bin/* — the scripts that run on a +# booted NeurOS system. Build-time hooks (config/hooks/live/*) legitimately +# fetch packages from the internet while building the ISO, same as any Linux +# distro's package manager; they are not part of the running system's +# air-gap guarantee and are intentionally excluded here. +# +# Any literal http(s):// URL whose host is not localhost/127.0.0.1/0.0.0.0 +# and is not on the allowlist below fails the check. This catches new +# outbound calls; it does not try to catch every possible obfuscation. +# +# Usage: ./scripts/check-airgap.sh + +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +BIN_DIR="config/includes.chroot/usr/local/bin" + +# host -> file it's allowed in (one pair per line: "file:host") +# Each entry here is a known, reviewed, user-invoked exception to the +# air-gap guarantee. Adding a new host here is a deliberate decision — +# it should show up in code review. +ALLOWLIST=" +neuros-music:www.youtube.com +neuros-network:google.com +neuros-network:1.1.1.1 +neuros-speak:github.com +" + +FOUND=0 + +# Literal-URL scanning (below) misses outbound calls made through a +# third-party HTTP client whose target URL is built at runtime (env +# var, f-string, config value) rather than typed as a literal in the +# file. stdlib urllib is used throughout these tools to talk to the +# local Ollama endpoint, so importing it is not itself a signal. +# `requests`/`httpx` are not stdlib, are not used anywhere in this +# tree today, and exist only to make arbitrary HTTP calls ergonomic - +# so any import of one is a hard fail here regardless of URL literals. +while IFS= read -r -d '' f; do + name="$(basename "$f")" + match="$(grep -nE '^\s*(import|from)\s+(requests|httpx)\b' "$f" 2>/dev/null || true)" + if [ -n "$match" ]; then + echo "AIR-GAP VIOLATION: $name imports a non-stdlib HTTP client not used " \ + "anywhere else in this tree (can make outbound calls with a " \ + "runtime-built URL that the literal-URL scan below can't see):" + echo "$match" | sed "s#^# $name:#" + FOUND=1 + fi +done < <(find "$BIN_DIR" -type f -not -name '*.pyc' -not -path '*/__pycache__/*' -print0) + +# Raw-socket / DNS-tool scanning: catches outbound calls that contain no +# http(s):// literal at all, e.g. socket.create_connection(("1.1.1.1", 53)) +# or shelling out to nslookup/dig/host with a literal target host. The +# URL scan below can't see either of these. +while IFS= read -r -d '' f; do + name="$(basename "$f")" + while IFS=: read -r line host; do + [ -z "${host:-}" ] && continue + case "$host" in + localhost|127.0.0.1|0.0.0.0) continue ;; + -*|+*) continue ;; # a CLI flag (e.g. dig's "+short"), not a hostname + esac + + allowed=0 + while IFS=: read -r al_file al_host; do + [ -z "${al_file:-}" ] && continue + if [ "$name" = "$al_file" ] && [ "$host" = "$al_host" ]; then + allowed=1 + break + fi + done <<< "$ALLOWLIST" + + if [ "$allowed" -eq 0 ]; then + echo "AIR-GAP VIOLATION: $name:$line makes a raw network call to non-local host '$host'" + FOUND=1 + fi + done < <( { grep -noP "socket\.(create_connection|connect)\(\(\s*[\"']\K[^\"']+" "$f" 2>/dev/null || true + grep -noP "\[\s*[\"'](nslookup|dig|host)[\"']\s*,\s*[\"']\K[^\"']+" "$f" 2>/dev/null || true; } ) +done < <(find "$BIN_DIR" -type f -not -name '*.pyc' -not -path '*/__pycache__/*' -print0) + +while IFS= read -r -d '' f; do + name="$(basename "$f")" + while IFS=: read -r line url; do + [ -z "${url:-}" ] && continue + host="$(echo "$url" | sed -E 's#https?://##; s#[/:].*##')" + case "$host" in + localhost|127.0.0.1|0.0.0.0|"") continue ;; + *.w3.org|w3.org) continue ;; # SVG XML namespace URI, not fetched + esac + # Skip f-string interpolated hosts (e.g. http://{config[host]}) -- these + # resolve to the local Ollama endpoint at runtime, not a literal host. + case "$host" in + \{*) continue ;; + esac + # Skip placeholder text like "https://..." in --help docstrings + case "$url" in + *"..."*) continue ;; + esac + + allowed=0 + while IFS=: read -r al_file al_host; do + [ -z "${al_file:-}" ] && continue + if [ "$name" = "$al_file" ] && [ "$host" = "$al_host" ]; then + allowed=1 + break + fi + done <<< "$ALLOWLIST" + + if [ "$allowed" -eq 0 ]; then + echo "AIR-GAP VIOLATION: $name:$line calls external host '$host' ($url)" + FOUND=1 + fi + done < <(grep -noE "https?://[^\"'\` \\)]+" "$f" 2>/dev/null || true) +done < <(find "$BIN_DIR" -type f -not -name '*.pyc' -not -path '*/__pycache__/*' -print0) + +if [ "$FOUND" -ne 0 ]; then + echo "" + echo "One or more shipped tools call an external host that isn't on the" + echo "air-gap allowlist in scripts/check-airgap.sh. If this is intentional" + echo "(a new opt-in, user-invoked feature), add it to the ALLOWLIST with a" + echo "one-line justification in the same PR. Otherwise, remove the call." + exit 1 +fi + +echo "Air-gap check passed: no unreviewed outbound network calls found in $BIN_DIR." diff --git a/tests/test_bench.py b/tests/test_bench.py new file mode 100755 index 0000000..4b5666d --- /dev/null +++ b/tests/test_bench.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""Tests for neuros-bench regression benchmark harness. + +These tests do not need real container kernel delegation because +they mock subprocess.run and the inner _run_workload helper. They +assert: + + * argparse dispatch routes list / run / batch / compare correctly + * --dry-run-equivalent paths emit metrics JSON shaped per the contract + * compare-mode rejects regressions beyond --tolerance-wall-pct / mem + * builtin workload registry contains the documented 8 entries + * _pct_delta handles None + zero-base gracefully + * _render_fn strips the leading `def _w_xxx():` line and re-indents +""" +import argparse +import importlib.util +import io +import json +import os +import sys +import tempfile +import types +import unittest +from unittest import mock + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) +_BENCH_PATH = os.path.join( + ROOT, "config", "includes.chroot", "usr", "local", "bin", "neuros-bench") + + +def _load_neuros_bench(): + """Load `neuros-bench` (no .py suffix in its install path) by + compiling its source and exec-ing in a fresh module namespace. + Same pattern as `_load_neuros_sandbox` in test_sandbox.py.""" + with open(_BENCH_PATH) as f: + source = f.read() + code = compile(source, _BENCH_PATH, "exec") + module = sys.modules.get("neuros_bench") or types.ModuleType("neuros_bench") + module.__file__ = _BENCH_PATH + sys.modules["neuros_bench"] = module + exec(code, module.__dict__) + return module + + +nb = _load_neuros_bench() + + +class TestWorkloadRegistry(unittest.TestCase): + """Locks the workload suite membership so an accidental rename or + drop is a visible test failure rather than a silent regression.""" + + EXPECTED = frozenset({ + "cpu_tight", "json_parse", "regex_compile", "subproc_spawn", + "mem_grow", "file_io", "ctypes_call", "string_ops", + }) + + def test_expected_workloads_present(self): + self.assertEqual(set(nb.WORKLOADS), self.EXPECTED) + + def test_each_workload_has_callable_and_baseline(self): + for name, (fn, baseline) in nb.WORKLOADS.items(): + self.assertTrue(callable(fn), + f"workload {name} fn is not callable") + self.assertIsInstance(baseline, int) + self.assertGreater(baseline, 0, + f"workload {name} baseline_ms must be > 0") + + def test_registered_workloads_distinct(self): + # Each fn is a top-level function with a distinct identity. + seen = set() + for name, (fn, _) in nb.WORKLOADS.items(): + self.assertNotIn(id(fn), seen, + f"workload {name} shares identity with " + "another workload") + seen.add(id(fn)) + + +class TestRenderFn(unittest.TestCase): + """Locks the contract: _render_fn strips the def line and yields + a re-indented body so the inner try/except sees uniform indent.""" + + def test_renders_function_body_without_def_line(self): + rendered = nb._render_fn(nb._w_cpu_tight) + text = "\n".join(rendered) + self.assertNotIn("def _w_cpu_tight", text) + self.assertIn("def fib", text) # nested function preserved + self.assertIn("fib(28)", text) # the trailing call preserved + + def test_each_builtin_renders(self): + for name, (fn, _) in nb.WORKLOADS.items(): + rendered = nb._render_fn(fn) + self.assertTrue(rendered, + f"workload {name} rendered to empty list") + + +class TestPctDelta(unittest.TestCase): + """Locks the percent-delta helpers' edge cases.""" + + def test_zero_base_returns_none(self): + self.assertIsNone(nb._pct_delta(0, 100)) + + def test_negative_delta(self): + # Improvement: candidate is faster than baseline. + self.assertAlmostEqual(nb._pct_delta(100, 80), -20.0) + + def test_positive_delta(self): + # Regression: candidate is slower. + self.assertAlmostEqual(nb._pct_delta(100, 130), 30.0) + + def test_optional_handles_none(self): + self.assertIsNone(nb._pct_delta_optional(None, 100)) + self.assertIsNone(nb._pct_delta_optional(100, None)) + + def test_optional_handles_zero_base(self): + self.assertIsNone(nb._pct_delta_optional(0, 100)) + + +class TestCompareFunction(unittest.TestCase): + """The compare subcommand is the load-bearing piece for CI: exit + 1 if any workload's wall or mem regresses beyond tolerance.""" + + def _metrics(self, runs): + return { + "runs": runs, + "total_wall_clock_ms": 12345, + "config": {}, + "neuros_bench_version": nb.NEUROS_BENCH_VERSION, + } + + def _run_row(self, name, wall_ms, peak=None): + return { + "name": name, "wall_clock_ms": wall_ms, + "exit_code": 0, "stdout": "", "stderr": "", + "timeout_hit": False, "peak_mem_estimate": peak, + } + + def _write_metrics(self, runs): + f = tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") + json.dump(self._metrics(runs), f) + f.close() + return f.name + + def test_no_regression_exits_zero(self): + a = self._write_metrics([ + self._run_row("cpu_tight", 1000, peak=10_000_000) + ]) + b = self._write_metrics([ + self._run_row("cpu_tight", 1050, peak=10_500_000) + ]) + ns = mock.MagicMock() + ns.baseline = a + ns.candidate = b + ns.tolerance_wall_pct = 15.0 + ns.tolerance_mem_pct = 25.0 + # Capture both stdout and stderr; compare-mode writes the + # table on stdout and the verdict on stderr. + buf_out, buf_err = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdout", buf_out), \ + mock.patch.object(sys, "stderr", buf_err): + rc = nb.cmd_compare(ns) + self.assertEqual(rc, 0, buf_out.getvalue()) + self.assertIn("OK", buf_out.getvalue()) + self.assertNotIn("REGRESSION", buf_out.getvalue()) + os.unlink(a) + os.unlink(b) + + def test_wall_regression_exits_one(self): + a = self._write_metrics([ + self._run_row("cpu_tight", 1000, peak=10_000_000) + ]) + b = self._write_metrics([ + self._run_row("cpu_tight", 1300, peak=10_500_000) + ]) + ns = mock.MagicMock() + ns.baseline = a + ns.candidate = b + ns.tolerance_wall_pct = 15.0 + ns.tolerance_mem_pct = 25.0 + buf_out, buf_err = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdout", buf_out), \ + mock.patch.object(sys, "stderr", buf_err): + rc = nb.cmd_compare(ns) + self.assertEqual(rc, 1) + self.assertIn("REGRESSION", buf_out.getvalue()) + os.unlink(a) + os.unlink(b) + + def test_new_workload_added_in_candidate_is_not_regression(self): + """A NEW workload appears only in candidate, not in baseline. + This is an addition, NOT a regression — CI should pass with + exit 0 and the row should be marked NEW (informational).""" + a = self._write_metrics([ + self._run_row("cpu_tight", 1000, peak=10_000_000) + ]) + b = self._write_metrics([ + self._run_row("cpu_tight", 1000, peak=10_000_000), + self._run_row("json_parse", 800, peak=10_000_000), + ]) + ns = mock.MagicMock() + ns.baseline = a + ns.candidate = b + ns.tolerance_wall_pct = 15.0 + ns.tolerance_mem_pct = 25.0 + buf_out, buf_err = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdout", buf_out), \ + mock.patch.object(sys, "stderr", buf_err): + rc = nb.cmd_compare(ns) + self.assertEqual(rc, 0, buf_out.getvalue()) + self.assertIn("NEW", buf_out.getvalue()) + os.unlink(a) + os.unlink(b) + + def test_dropped_workload_in_candidate_is_regression(self): + """A workload present in baseline but absent in candidate is a + DROPPED workload — CI should exit 1.""" + a = self._write_metrics([ + self._run_row("cpu_tight", 1000, peak=10_000_000), + self._run_row("string_ops", 500, peak=10_000_000), + ]) + b = self._write_metrics([ + self._run_row("cpu_tight", 1000, peak=10_000_000), + ]) + ns = mock.MagicMock() + ns.baseline = a + ns.candidate = b + ns.tolerance_wall_pct = 15.0 + ns.tolerance_mem_pct = 25.0 + buf_out, buf_err = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdout", buf_out), \ + mock.patch.object(sys, "stderr", buf_err): + rc = nb.cmd_compare(ns) + self.assertEqual(rc, 1) + self.assertIn("DROPPED", buf_out.getvalue()) + os.unlink(a) + os.unlink(b) + + def test_mem_regression_exits_one(self): + a = self._write_metrics([ + self._run_row("mem_grow", 300, peak=10_000_000) + ]) + b = self._write_metrics([ + self._run_row("mem_grow", 305, peak=15_000_000) + ]) + ns = mock.MagicMock() + ns.baseline = a + ns.candidate = b + ns.tolerance_wall_pct = 15.0 + ns.tolerance_mem_pct = 25.0 + buf_out, buf_err = io.StringIO(), io.StringIO() + with mock.patch.object(sys, "stdout", buf_out), \ + mock.patch.object(sys, "stderr", buf_err): + rc = nb.cmd_compare(ns) + self.assertEqual(rc, 1) + self.assertIn("REGRESSION", buf_out.getvalue()) + os.unlink(a) + os.unlink(b) + + +class TestRunWorkloadMocked(unittest.TestCase): + """Drive _run_workload with a mocked subprocess.run so we exercise + the JSON-envelope-parsing path without invoking the real + neuros-sandbox binary.""" + + def _fake_completed(self, stdout_text, returncode=0, stderr=b""): + cp = mock.Mock(returncode=returncode, stdout=stdout_text.encode()) + cp.stderr = stderr + return cp + + def test_parses_top_line_envelope(self): + env = {"exit_code": 0, "wall_clock_ms": 250, "timeout_hit": False, + "peak_mem_estimate": 31457280} + cp = self._fake_completed(json.dumps(env)) + with mock.patch.object(nb.subprocess, "run", return_value=cp), \ + mock.patch.object(nb, "_load_script", + create=True, return_value=b"x"): + out = nb._run_workload("cpu_tight", 60, "256M", 64) + self.assertEqual(out["name"], "cpu_tight") + self.assertEqual(out["wall_clock_ms"], 250) + self.assertEqual(out["peak_mem_estimate"], 31457280) + + def test_no_envelope_returns_fallback_record(self): + cp = self._fake_completed("") + with mock.patch.object(nb.subprocess, "run", return_value=cp): + out = nb._run_workload("cpu_tight", 60, "256M", 64) + self.assertEqual(out["name"], "cpu_tight") + self.assertEqual(out["wall_clock_ms"], 0) + self.assertIsNone(out["peak_mem_estimate"]) + + +class TestMainDispatch(unittest.TestCase): + """Argparse routes subcommands correctly.""" + + def test_unknown_subcommand_exits(self): + with self.assertRaises(SystemExit): + nb.main(["nope"]) + + def test_list_dispatches(self): + # argparse print_help writes "usage: ..." on stdout before our + # cmd_list handler runs when an unknown option trips argparse; + # for the `list` subcommand alone, main should return 0. + with mock.patch.object(sys, "stdout", io.StringIO()), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = nb.main(["list"]) + self.assertEqual(rc, 0) + + def test_list_emits_cpu_tight_workload(self): + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + nb.main(["list"]) + out = buf.getvalue() + self.assertIn("available workloads", out) + self.assertIn("cpu_tight", out) + + def test_run_unknown_workload_dies(self): + with mock.patch.object(sys, "stderr", io.StringIO()), \ + self.assertRaises(SystemExit) as cm: + nb.main(["run", "does-not-exist"]) + self.assertEqual(cm.exception.code, 2) + + +class TestConfigBlock(unittest.TestCase): + """Locks the schema of the metrics.json 'config' sub-record.""" + + def test_keys_present(self): + ns = mock.MagicMock() + ns.timeout = 60 + ns.mem = "256M" + ns.pids = 64 + cfg = nb._config_block(ns) + self.assertEqual(cfg["per_workload_timeout_seconds"], 60) + self.assertEqual(cfg["mem_cap"], "256M") + self.assertEqual(cfg["pids_cap"], 64) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_container.py b/tests/test_container.py new file mode 100644 index 0000000..f466f68 --- /dev/null +++ b/tests/test_container.py @@ -0,0 +1,863 @@ +#!/usr/bin/env python3 +""" +test_container.py — Unit and integration tests for neuros-container. + +The integration tests exercise real cgroup v2 enforcement (they create, +populate, and tear down an actual cgroup on the machine running the +suite) but skip cleanly if cgroup v2 delegation isn't available, the +same way the rest of this repo skips checks that need a full host. +""" + +import io +import json as _json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +import importlib.util +from importlib.machinery import SourceFileLoader +from unittest.mock import patch, MagicMock + +CONTAINER_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", + "config", "includes.chroot", "usr", "local", "bin", "neuros-container" +) + + +def load_neuros_container(): + loader = SourceFileLoader("neuros_container", CONTAINER_PATH) + spec = importlib.util.spec_from_loader("neuros_container", loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +def cgroups_available(): + """Stronger probe than just "make_cgroup returns a path": also + read back the limit file we set to confirm the write actually took + effect. A bare-EACCES mkdir leaves the leaf visible but the + limit files never get populated, which silently produces + containers that ignore `--mem`. Detect that here so the detach + integration test isn't misclassified as runnable.""" + try: + nc = load_neuros_container() + cg = nc.make_cgroup("neuros-selftest-probe", "16777216", None, None) + # Read back and confirm the controller write was real. + with open(os.path.join(cg, "memory.max")) as f: + content = f.read().strip() + ok = content == "16777216" + try: + os.rmdir(cg) + except OSError: + pass + return ok + except OSError: + return False + + +class TestSizeParsing(unittest.TestCase): + def setUp(self): + self.nc = load_neuros_container() + + def test_plain_bytes(self): + self.assertEqual(self.nc.parse_size("1000"), "1000") + + def test_kilobytes(self): + self.assertEqual(self.nc.parse_size("4K"), str(4 * 1024)) + + def test_megabytes(self): + self.assertEqual(self.nc.parse_size("256M"), str(256 * 1024 ** 2)) + + def test_gigabytes(self): + self.assertEqual(self.nc.parse_size("1G"), str(1024 ** 3)) + + def test_max_literal(self): + self.assertEqual(self.nc.parse_size("max"), "max") + + def test_none_passthrough(self): + self.assertIsNone(self.nc.parse_size(None)) + + def test_rejects_garbage(self): + with self.assertRaises(SystemExit): + self.nc.parse_size("not-a-size") + + +class TestCpuQuotaParsing(unittest.TestCase): + def setUp(self): + self.nc = load_neuros_container() + + def test_slash_separator(self): + self.assertEqual(self.nc.parse_cpu_quota("50000/100000"), ("50000", "100000")) + + def test_space_separator(self): + self.assertEqual(self.nc.parse_cpu_quota("25000 100000"), ("25000", "100000")) + + def test_max_literal(self): + self.assertEqual(self.nc.parse_cpu_quota("max"), ("max", "max")) + + def test_none_passthrough(self): + self.assertIsNone(self.nc.parse_cpu_quota(None)) + + def test_rejects_garbage(self): + with self.assertRaises(SystemExit): + self.nc.parse_cpu_quota("not-a-quota") + + def test_rejects_zero_period(self): + """period_us must be > 0; cpu.max rejects zero periods.""" + with self.assertRaises(SystemExit): + self.nc.parse_cpu_quota("50000/0") + + def test_rejects_negative_max(self): + with self.assertRaises(SystemExit): + self.nc.parse_cpu_quota("-1/100000") + + def test_rejects_one_part(self): + with self.assertRaises(SystemExit): + self.nc.parse_cpu_quota("50000") + + +class TestUserParsing(unittest.TestCase): + def setUp(self): + self.nc = load_neuros_container() + + def test_uid_only_collapses_to_same_gid(self): + self.assertEqual(self.nc.parse_user("1000"), (1000, 1000)) + + def test_uid_gid_pair(self): + self.assertEqual(self.nc.parse_user("1000:1001"), (1000, 1001)) + + def test_root(self): + self.assertEqual(self.nc.parse_user("0:0"), (0, 0)) + + def test_none_passthrough(self): + self.assertIsNone(self.nc.parse_user(None)) + + def test_rejects_garbage(self): + with self.assertRaises(SystemExit): + self.nc.parse_user("not-a-user") + + def test_rejects_name_style(self): + with self.assertRaises(SystemExit): + self.nc.parse_user("alice") + + +class TestEnterNamespacesBitmask(unittest.TestCase): + """The 6-tuple contract from enter_namespaces() must report the + right combination of True/False for each feasibility path. + + Implementation note: ``load_neuros_container()`` returns a fresh + ``module`` object on each call (``importlib.util.module_from_spec`` + is not cached when invoked outside of an actual ``import`` statement), + so decorator-level ``@patch.object(load_neuros_container(), ...)`` + targets a module that ``self.nc`` no longer points at. Patching + ``self.nc`` directly inside each test sidesteps that.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_root_with_net_gets_full_mask(self): + with patch("os.geteuid", return_value=0), \ + patch.object(self.nc, "try_unprivileged_userns", + return_value=True), \ + patch("os.unshare"): + ns = self.nc.enter_namespaces(net=True, user=None) + self.assertEqual(ns, (True, True, True, True, True, False)) + + def test_root_without_net_has_net_false(self): + with patch("os.geteuid", return_value=0), \ + patch("os.unshare"): + ns = self.nc.enter_namespaces(net=False, user=None) + self.assertEqual(ns, (True, True, True, True, False, False)) + + def test_no_privs_yields_all_false(self): + with patch("os.geteuid", return_value=1000), \ + patch.object(self.nc, "try_unprivileged_userns", + return_value=False): + ns = self.nc.enter_namespaces(net=False, user=None) + self.assertEqual(ns, (False, False, False, False, False, False)) + + def test_unprivileged_with_userns_gets_full_mask(self): + with patch("os.geteuid", return_value=1000), \ + patch.object(self.nc, "try_unprivileged_userns", + return_value=True), \ + patch("os.unshare"): + ns = self.nc.enter_namespaces(net=False, user=(1000, 1000)) + self.assertEqual(ns, (True, True, True, True, False, True)) + + def test_unshare_failure_returns_all_false(self): + with patch("os.geteuid", return_value=0), \ + patch("os.unshare", side_effect=OSError("eperm")): + ns = self.nc.enter_namespaces(net=False, user=None) + self.assertEqual(ns, (False, False, False, False, False, False)) + + +class TestListRecursion(unittest.TestCase): + """list_cgroups descends through non-neuros- intermediate + directories; the integration tests use the live cgroup tree under + the delegating ancestor, and we exercise that on a synthetic tree + here.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_finds_nested_leaves_via_synthetic_tree(self): + """Build a tiny tree under a tmpdir and verify the recursive + walker reaches ``neuros-*`` leaves at multiple depths.""" + root = tempfile.mkdtemp() + try: + deep = os.path.join(root, "intermediate", "neuros-deep-1") + os.makedirs(deep) + # A procs file is what the walker reads; empty means + # the leaf claims to have 0 members, which is fine here. + open(os.path.join(deep, "cgroup.procs"), "w").close() + rows = self.nc._list_neuros_leaves(root) + names = [r["name"] for r in rows] + self.assertIn("neuros-deep-1", names) + # If two leaves sit at different depths, both are picked up. + sibling = os.path.join(root, "neuros-shallow") + os.makedirs(sibling) + open(os.path.join(sibling, "cgroup.procs"), "w").close() + names = [r["name"] for r in self.nc._list_neuros_leaves(root)] + self.assertEqual(set(names), {"neuros-deep-1", "neuros-shallow"}) + finally: + shutil.rmtree(root, ignore_errors=True) + + def test_json_output_is_valid_json_array(self): + """list_cgroups(--json) must produce a parseable JSON array, + empty or populated, on stdout. Patch own_cgroup_path to an + empty tmpdir so no live cgroup tree is required.""" + root = tempfile.mkdtemp() + try: + with patch.object(self.nc, "own_cgroup_path", + return_value=root), \ + patch("sys.stdout", new_callable=io.StringIO) as buf: + self.nc.list_cgroups(json_output=True) + payload = _json.loads(buf.getvalue().strip()) + self.assertIsInstance(payload, list) + finally: + shutil.rmtree(root, ignore_errors=True) + + def test_human_output_reports_zero_processes(self): + """list_cgroups() with no JSON flag on an empty tree prints a + single human-readable line instead of the JSON array.""" + root = tempfile.mkdtemp() + try: + with patch.object(self.nc, "own_cgroup_path", + return_value=root), \ + patch("sys.stdout", new_callable=io.StringIO) as buf: + self.nc.list_cgroups(json_output=False) + self.assertIn("no active neuros-container cgroups", buf.getvalue()) + finally: + shutil.rmtree(root, ignore_errors=True) + + +class TestCpuQuotaMaxRoundTrip(unittest.TestCase): + """parse_cpu_quota('max') must round-trip through make_cgroup so + that ``cpu.max = 'max max'`` is written and recoverable.""" + + def setUp(self): + self.nc = load_neuros_container() + + @unittest.skipUnless( + __import__("os").path.isdir("/sys/fs/cgroup"), + "needs a cgroup v2 mount", + ) + def test_max_writes_max_max_to_cpu_max(self): + cg = self.nc.make_cgroup("neuros-selftest-max", None, None, None, + cpu_quota=("max", "max")) + try: + with open(os.path.join(cg, "cpu.max")) as f: + value = f.read().strip() + # Treat "max max" as the accepted unlimited form. + self.assertIn(value, ("max max", "max 100000")) + finally: + try: + os.rmdir(cg) + except OSError: + pass + + +class TestCleanupDetached(unittest.TestCase): + """Unit coverage for ``cleanup_container`` / ``cleanup_all`` — + drives them against a synthetic state file and a fake (empty) + cgroup path so the test doesn't need a real detached run, root, + or a delegated cgroup tree.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_cleanup_removes_empty_cgroup_and_state(self): + root = tempfile.mkdtemp() + try: + cg_dir = os.path.join(root, "neuros-cgood") + os.makedirs(cg_dir) + # The "empty cgroup" path means cleanup_container's probe + # for cgroup.procs falls through the OSError branch (which + # is what an actually-empty leaf looks like on most + # hosts). Don't write a procs file: rmdir requires the + # directory to be fully empty, including any leftover + # cgroup.procs the kernel might not have created. + with patch.object(self.nc, "STATE_DIR", root): + state_path = os.path.join(self.nc.STATE_DIR, + "neuros-cgood.json") + with open(state_path, "w") as f: + _json.dump({"name": "neuros-cgood", + "pid": 99999, + "cgroup": cg_dir}, f) + rc = self.nc.cleanup_container("neuros-cgood") + self.assertEqual(rc, 0) + self.assertFalse(os.path.exists(cg_dir)) + self.assertFalse(os.path.exists(state_path)) + finally: + shutil.rmtree(root, ignore_errors=True) + + def test_cleanup_refuses_when_cgroup_still_has_procs(self): + root = tempfile.mkdtemp() + try: + cg_dir = os.path.join(root, "neuros-cbusy") + os.makedirs(cg_dir) + with open(os.path.join(cg_dir, "cgroup.procs"), "w") as f: + f.write("1234\n5678\n") # two fake live members + state_path = os.path.join(root, "neuros-cbusy.json") + with open(state_path, "w") as f: + _json.dump({"name": "neuros-cbusy", + "pid": 1234, + "cgroup": cg_dir}, f) + with patch.object(self.nc, "STATE_DIR", root): + rc = self.nc.cleanup_container("neuros-cbusy") + self.assertEqual(rc, 1) + self.assertTrue(os.path.exists(cg_dir)) + self.assertTrue(os.path.exists(state_path)) + finally: + shutil.rmtree(root, ignore_errors=True) + + def test_cleanup_missing_state_exits_nonzero(self): + empty = tempfile.mkdtemp() + try: + with patch.object(self.nc, "STATE_DIR", empty): + rc = self.nc.cleanup_container("neuros-nope") + self.assertEqual(rc, 1) + finally: + shutil.rmtree(empty, ignore_errors=True) + + def test_cleanup_all_aggregates_exit_code(self): + root = tempfile.mkdtemp() + try: + # ``a`` is an empty cgroup (no procs file at all -> cleanup + # proceeds to rmdir and succeeds). ``b`` has a populated + # cgroup.procs -> cleanup refuses. cleanup_all should OR + # the exit codes and fail overall. + cg_a = os.path.join(root, "neuros-a") + os.makedirs(cg_a) + with open(os.path.join(root, "neuros-a.json"), "w") as f: + _json.dump({"name": "neuros-a", + "pid": 1, "cgroup": cg_a}, f) + cg_b = os.path.join(root, "neuros-b") + os.makedirs(cg_b) + with open(os.path.join(cg_b, "cgroup.procs"), "w") as f: + f.write("1234\n5678\n") # two fake live members + with open(os.path.join(root, "neuros-b.json"), "w") as f: + _json.dump({"name": "neuros-b", + "pid": 1234, "cgroup": cg_b}, f) + with patch.object(self.nc, "STATE_DIR", root): + rc = self.nc.cleanup_all() + # "a" is cleaned (its cgroup dir is gone), "b" remains. + self.assertEqual(rc, 1) + self.assertFalse(os.path.exists(os.path.join(root, "neuros-a"))) + self.assertTrue(os.path.exists(os.path.join(root, "neuros-b"))) + finally: + shutil.rmtree(root, ignore_errors=True) + + +class TestRemountReadonlySurface(unittest.TestCase): + """The --read-only path used to drop the mount(8) returncode on + the floor; it now warns on non-zero exit.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_surfaces_warning_on_nonzero_returncode(self): + with patch("os.system", return_value=32), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.remount_readonly_best_effort() + self.assertIn("exited with status 32", err.getvalue()) + + def test_silent_on_zero_returncode(self): + with patch("os.system", return_value=0), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.remount_readonly_best_effort() + self.assertEqual(err.getvalue(), "") + + +class TestUserMismatchHint(unittest.TestCase): + """``--user UID:GID`` requested with a target that doesn't match + the caller's real uid should print a hint at the top of the + userns setup so the user understands the upcoming fall-back.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_prints_hint_when_user_differs_from_caller(self): + # Caller uid=1000 tries to map 2000:2000. ``try_unprivileged_userns`` + # is mocked to return False so we hit the “no userns" path. + with patch("os.geteuid", return_value=1000), \ + patch("os.getuid", return_value=1000), \ + patch("os.getgid", return_value=1000), \ + patch.object(self.nc, "try_unprivileged_userns", + return_value=False), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.enter_namespaces(net=False, user=(2000, 2000)) + self.assertIn("--user 2000:2000 was requested", err.getvalue()) + + def test_no_hint_when_user_matches_caller(self): + with patch("os.geteuid", return_value=1000), \ + patch("os.getuid", return_value=1000), \ + patch("os.getgid", return_value=1000), \ + patch.object(self.nc, "try_unprivileged_userns", + return_value=False), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.enter_namespaces(net=False, user=(1000, 1000)) + self.assertNotIn("--user ... was requested", err.getvalue()) + + def test_no_hint_when_no_user_arg(self): + with patch("os.geteuid", return_value=1000), \ + patch.object(self.nc, "try_unprivileged_userns", + return_value=False), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.enter_namespaces(net=False, user=None) + self.assertNotIn("--user", err.getvalue()) + + +class TestEnvFileParsing(unittest.TestCase): + """``--env-from-file PATH`` reads a dotenv-style file, strips + blanks/comments/exports, and feeds the rest through ``parse_env``. + The 5 cases cover happy path, blank/comment skipping, ``export`` + prefix tolerance, ``None`` passthrough, and missing-file exit.""" + + def setUp(self): + self.nc = load_neuros_container() + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def _write(self, name, body): + path = os.path.join(self.tmp, name) + with open(path, "w") as f: + f.write(body) + return path + + def test_env_file_parses_key_value_pairs(self): + path = self._write("env", "FOO=bar\nBAZ=qux\n") + self.assertEqual(self.nc.parse_env_file(path), + ["FOO=bar", "BAZ=qux"]) + + def test_env_file_skips_blanks_and_comments(self): + path = self._write("env", "# header\n\nFOO=bar\n# trailing\n") + self.assertEqual(self.nc.parse_env_file(path), ["FOO=bar"]) + + def test_env_file_accepts_export_prefix(self): + path = self._write("env", "export FOO=bar\nBAZ=qux\n") + self.assertEqual(self.nc.parse_env_file(path), + ["FOO=bar", "BAZ=qux"]) + + def test_env_file_none_passthrough(self): + self.assertEqual(self.nc.parse_env_file(None), []) + + def test_env_file_missing_path_exits(self): + with self.assertRaises(SystemExit), \ + patch("sys.stderr", new_callable=io.StringIO): + self.nc.parse_env_file(os.path.join(self.tmp, "nope")) + + +class TestDetachIntegration(unittest.TestCase): + """Drive the detached-container lifecycle end-to-end without + depending on the kernel's cgroup delegation state at the moment + of the test. ``--detach`` writes a JSON state file via + ``write_state`` and leaves the cgroup in place; ``cleanup`` reaps + empty leaves. We exercise that contract against a tmpdir + "state dir" and a tmpdir "cgroup dir", so the assertions don't + depend on whether the kernel delegation path is actually open + right now (which can disagree between the probe and a subprocess + in some sandboxes). + + This test is synthetic-only. A real e2e form should live in + ``tests/test_container_integration.py`` once a CI with + kernel-cgroup delegation is available — that form should + subprocess-run ``neuros-container run --detach …`` end-to-end. + The synthetic form here is portable and doesn't depend on the + kernel's cgroup delegation state at the moment of the test. + """ + + def setUp(self): + self.nc = load_neuros_container() + + def test_full_lifecycle_write_state_then_cleanup(self): + root = tempfile.mkdtemp() + try: + state_dir = root + cg_dir = os.path.join(root, "neuros-detach") + os.makedirs(cg_dir) + name = "neuros-detach-selftest" + + # Phase 1: write the state file via the helper the + # detached child would have written. + state_path = os.path.join(state_dir, f"{name}.json") + self.nc.write_state(state_path=state_path, name=name, + inner_pid=99999, cg=cg_dir) + self.assertTrue(os.path.exists(state_path)) + + # Phase 2: cleanup_container should reap the empty + # cgroup and unlink the state file. + with patch.object(self.nc, "STATE_DIR", state_dir): + rc = self.nc.cleanup_container(name) + self.assertEqual(rc, 0) + self.assertFalse(os.path.exists(cg_dir)) + self.assertFalse(os.path.exists(state_path)) + finally: + shutil.rmtree(root, ignore_errors=True) + + def test_state_shape_matches_documented_contract(self): + """The state JSON must include name/pid/cgroup so a reattach + tool can find the right processes.""" + root = tempfile.mkdtemp() + try: + # No real cgroup.procs on this synthetic path is the + # important invariant: cleanup_container treats a missing + # cgroup.procs as "zero live members" via its OSError + # branch, so the test relies on that branch silently + # succeeding. if a future reader adds a stray procs file + # here, full_lifecycle_write_state_then_cleanup will start + # failing with "still has N processes" — by design. + name = "neuros-shape-test" + path = os.path.join(root, f"{name}.json") + self.nc.write_state(state_path=path, name=name, + inner_pid=12345, + cg="/sys/fs/cgroup/neuros-x") + with open(path) as f: + state = _json.load(f) + self.assertEqual(set(state.keys()), + {"name", "pid", "cgroup"}) + self.assertEqual(state["name"], name) + self.assertEqual(state["pid"], 12345) + self.assertEqual(state["cgroup"], "/sys/fs/cgroup/neuros-x") + finally: + shutil.rmtree(root, ignore_errors=True) + + +class TestUtilitySmoke(unittest.TestCase): + """Five high-traffic ``neuros-*`` utilities that previously had + zero coverage. The assertions are deliberately structural so they + survive across schema changes in the tools themselves but still + catch a broken script (missing shebang, syntax error, ``--help`` + not behaving like argparse).""" + + UTILITIES = ( + "neuros-firewall", + "neuros-network", + "neuros-backup", + "neuros-monitor", + "neuros-cron", + ) + BIN_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", + "config", "includes.chroot", "usr", "local", "bin", + ) + + def _path(self, name): + full = os.path.join(self.BIN_DIR, name) + if not os.path.exists(full): + self.skipTest(f"{name} not present on disk") + return full + + def test_each_utility_has_python3_shebang(self): + for name in self.UTILITIES: + with open(self._path(name)) as f: + first = f.readline() + self.assertIn("python3", first, + msg=f"{name} missing python3 shebang") + + def _load(self, name): + loader = SourceFileLoader(name, self._path(name)) + spec = importlib.util.spec_from_loader(name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + def test_each_utility_imports_without_syntax_error(self): + for name in self.UTILITIES: + module = self._load(name) + self.assertIsNotNone(module) + + def test_each_utility_reports_help(self): + """Argparse-based CLIs print help on ``--help`` and exit 0. + A tool that crashes on ``--help`` means a new opt was added + without a test, and live callers will run into the same + regression on first invocation. The assertion is strict: + argparse prints ``usage:`` to stdout, so we require it.""" + for name in self.UTILITIES: + proc = subprocess.run( + [sys.executable, self._path(name), "--help"], + capture_output=True, text=True, timeout=10, + ) + combined = proc.stdout + proc.stderr + self.assertIn( + "usage:", combined, + msg=f"{name} --help did not print argparse usage " + f"(rc={proc.returncode}, output={combined!r})", + ) + + +class TestEnvAndCapDropParsing(unittest.TestCase): + """``--env`` and ``--cap-drop`` round-trip through their parsers + without touching the kernel.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_env_accepts_well_formed_pairs(self): + out = self.nc.parse_env(["FOO=bar", "BAZ=qux quux"]) + self.assertEqual(out, ["FOO=bar", "BAZ=qux quux"]) + + def test_env_empty_returns_empty(self): + self.assertEqual(self.nc.parse_env(None), []) + self.assertEqual(self.nc.parse_env([]), []) + + def test_env_rejects_missing_equals(self): + with self.assertRaises(SystemExit): + self.nc.parse_env(["NO_EQUALS"]) + + def test_env_rejects_blank_key(self): + with self.assertRaises(SystemExit): + self.nc.parse_env(["=value"]) + + def test_env_rejects_nul_bytes(self): + with self.assertRaises(SystemExit), \ + patch("sys.stderr", new_callable=io.StringIO): + self.nc.parse_env(["K=\x00v"]) + + def test_cap_drop_returns_none_when_unset(self): + self.assertIsNone(self.nc.parse_cap_drop(None)) + + def test_cap_drop_compiles_regex(self): + pattern = self.nc.parse_cap_drop("CAP_(NET_RAW|SYS_ADMIN)") + self.assertTrue(pattern.search("CAP_NET_RAW")) + self.assertTrue(pattern.search("CAP_SYS_ADMIN")) + self.assertFalse(pattern.search("CAP_CHOWN")) + + def test_cap_drop_rejects_invalid_regex(self): + with self.assertRaises(SystemExit): + self.nc.parse_cap_drop("(unclosed") + + +class TestEnvInjection(unittest.TestCase): + """The grandchild of run_container inserts --env entries into + ``os.environ`` just before exec. We don't fork a real process; + instead, we directly call the same injection block against the + real ``os.environ`` so the assertion is on what an execvp would + inherit.""" + + def setUp(self): + self.nc = load_neuros_container() + self.old_environ = dict(os.environ) + + def tearDown(self): + os.environ.clear() + os.environ.update(self.old_environ) + + def test_env_entries_override_inherited(self): + # Mirror exactly what run_container does after the parse step. + env = self.nc.parse_env(["NEUROS_TEST_K=v1"]) + os.environ["NEUROS_TEST_K"] = "inherited" + for entry in env: + k, _, v = entry.partition("=") + os.environ[k] = v + self.assertEqual(os.environ["NEUROS_TEST_K"], "v1") + + +class TestDropCapabilitiesHelper(unittest.TestCase): + """``drop_capabilities_best_effort`` walks the running kernel's + cap list and prctl(PR_CAPBSET_DROP)'s every match. We stub + ctypes.CDLL so no real prctl syscall runs; the helper is exercised + end-to-end against mock libcs that return either success or + EPERM-set errno.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_silent_when_pattern_is_none(self): + with patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.drop_capabilities_best_effort(None) + self.assertEqual(err.getvalue(), "") + + def test_no_match_warns_loudly(self): + """Pattern matches nothing — the helper must surface a clear + stderr line, otherwise the user thinks their regex dropped + caps when it didn't.""" + libc = MagicMock() + libc.prctl = MagicMock(return_value=0) + pattern = self.nc.parse_cap_drop("CAP_BOGUS_DISABLE_THIS") + with patch.object(self.nc, "_CAP_NAMES_0_31", + ["CHOWN", "DAC_OVERRIDE"]), \ + patch("ctypes.CDLL", return_value=libc), \ + patch("ctypes.get_errno", return_value=0), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.drop_capabilities_best_effort(pattern) + self.assertIn("matched 0 of", err.getvalue()) + + def test_libc_load_failure_warns(self): + libc = MagicMock() + libc.prctl = MagicMock(return_value=0) + pattern = self.nc.parse_cap_drop("CAP_NET_RAW") + with patch("ctypes.CDLL", side_effect=OSError("no libc")), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.drop_capabilities_best_effort(pattern) + self.assertIn("libc", err.getvalue()) + + def test_prctl_eperm_warns_per_cap(self): + """A failure on one specific cap reports the errno per-cap so + users can tell which cap couldn't be dropped (often CAP_SETPCAP + on a userns without the right capability).""" + libc = MagicMock() + # Fail when prctl is called for PR_CAPBSET_DROP on cap 0 + # (CAP_CHOWN); any other cap succeeds. + def fake_prctl(op, capidx, *_): + if op == 23 and capidx == 0: # PR_CAPBSET_DROP=23 + return -1 + return 0 + libc.prctl = fake_prctl + pattern = self.nc.parse_cap_drop("CAP_CHOWN") + with patch("ctypes.CDLL", return_value=libc), \ + patch("ctypes.get_errno", return_value=1), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.drop_capabilities_best_effort(pattern) + text = err.getvalue() + self.assertIn("CAP_CHOWN", text) + self.assertIn("errno=1", text) + self.assertIn("CAP_SETPCAP", text) + + +class TestZeroControllersWarning(unittest.TestCase): + """If a limit is requested but no controller is delegated all the + way down, make_cgroup should print a stderr warning telling the + user their --mem/--pids/--cpu did nothing.""" + + def setUp(self): + self.nc = load_neuros_container() + + def test_warning_on_zero_ready_controllers(self): + # Force enable_controllers to return an empty set, then make + # sure the warning surfaces. The cgroup is still created in a + # tmpdir so the rmdir at the end works. We also patch + # find_delegating_ancestor — the walker otherwise climbs all + # the way up to ``/`` looking for a delegating ancestor and + # returns it, makedirs-ing at the filesystem root. + root = tempfile.mkdtemp() + try: + with patch.object(self.nc, "find_delegating_ancestor", + return_value=root), \ + patch.object(self.nc, "enable_controllers", + return_value=set()), \ + patch("sys.stderr", new_callable=io.StringIO) as err: + self.nc.make_cgroup("neuros-zero", "1000", None, None) + self.assertIn("WARNING", err.getvalue()) + finally: + shutil.rmtree(root, ignore_errors=True) + + +@unittest.skipUnless(cgroups_available(), "cgroup v2 delegation not available in this sandbox") +class TestCgroupEnforcement(unittest.TestCase): + def setUp(self): + self.nc = load_neuros_container() + + def test_cpu_max_quota_writes_through(self): + """make_cgroup(..., cpu_quota=("50000","100000")) must produce a + cgroup whose cpu.max file is exactly '50000 100000'. Tests the + new --cpu-quota code path without needing to schedule a CPU- + bound workload.""" + cg = self.nc.make_cgroup("neuros-selftest-cpuq", None, None, None, + cpu_quota=("50000", "100000")) + try: + with open(os.path.join(cg, "cpu.max")) as f: + self.assertEqual(f.read().strip(), "50000 100000") + finally: + try: + os.rmdir(cg) + except OSError: + pass + + def test_cpu_weight_still_writes_cpu_weight(self): + """The previous weight-only path stays backwards compatible.""" + cg = self.nc.make_cgroup("neuros-selftest-cpuw", None, None, 250) + try: + with open(os.path.join(cg, "cpu.weight")) as f: + self.assertEqual(f.read().strip(), "250") + finally: + try: + os.rmdir(cg) + except OSError: + pass + + def test_memory_max_caps_actual_usage(self): + """A process that tries to touch 200MB inside a 16M memory.max + cgroup must not exceed that limit, per memory.current.""" + cg = self.nc.make_cgroup("neuros-selftest-mem", "16777216", None, None) + try: + proc = subprocess.Popen([ + sys.executable, "-c", + "import time; time.sleep(0.1)\n" + "b = bytearray(200 * 1024 * 1024)\n" + "for i in range(0, len(b), 4096):\n" + " b[i] = 1\n" + "time.sleep(0.2)\n", + ]) + self.nc.join_cgroup(cg, proc.pid) + proc.wait(timeout=10) + with open(os.path.join(cg, "memory.peak")) as f: + peak = int(f.read().strip()) + self.assertLessEqual(peak, 16 * 1024 * 1024 * 1.05) + finally: + try: + os.rmdir(cg) + except OSError: + pass + + def test_pids_max_blocks_extra_forks(self): + """pids.max=4 must stop a forking loop once 4 members are in the + cgroup (the harness process here counts as one of the four).""" + cg = self.nc.make_cgroup("neuros-selftest-pids", None, "4", None) + try: + script = ( + "import subprocess, sys, time\n" + "time.sleep(0.1)\n" + "forked = 0\n" + "procs = []\n" + "try:\n" + " for _ in range(20):\n" + " procs.append(subprocess.Popen(['sleep', '0.3']))\n" + " forked += 1\n" + "except OSError:\n" + " pass\n" + "for p in procs:\n" + " p.wait()\n" + "print(forked)\n" + ) + proc = subprocess.Popen( + [sys.executable, "-c", script], + stdout=subprocess.PIPE, text=True, + ) + self.nc.join_cgroup(cg, proc.pid) + out, _ = proc.communicate(timeout=10) + forked = int(out.strip()) + self.assertLess(forked, 20) + finally: + try: + os.rmdir(cg) + except OSError: + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 6f0e64a..7456595 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -69,5 +69,39 @@ def test_call_tool_unknown_name(self): self.assertIn("error", result) +class TestDispatchRequest(unittest.TestCase): + """dispatch_request is the shared router behind both the HTTP transport + (do_POST) and the stdio transport (run_stdio) -- exercise it directly + against real MCP JSON-RPC shapes, since that's what a real client sends.""" + + def setUp(self): + self.mcp = load_neuros_mcp() + self.handler = self.mcp.MCPServer.__new__(self.mcp.MCPServer) + + def test_initialize_returns_protocol_version_and_id(self): + response = self.mcp.dispatch_request(self.handler, { + "jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": "2024-11-05", "capabilities": {}, + "clientInfo": {"name": "test-client", "version": "0.1"}} + }) + self.assertEqual(response["id"], 1) + self.assertEqual(response["result"]["protocolVersion"], "2024-11-05") + + def test_notification_gets_no_response(self): + # Per the MCP spec, requests without an "id" are notifications and + # must not receive a reply. + response = self.mcp.dispatch_request( + self.handler, {"jsonrpc": "2.0", "method": "notifications/initialized"} + ) + self.assertIsNone(response) + + def test_unknown_method_returns_json_rpc_error(self): + response = self.mcp.dispatch_request( + self.handler, {"jsonrpc": "2.0", "id": 5, "method": "no/such/method"} + ) + self.assertEqual(response["error"]["code"], -32601) + self.assertEqual(response["id"], 5) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_model.py b/tests/test_model.py index 268754d..e0bb744 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -48,9 +48,8 @@ def test_switch_creates_config_when_missing(self): with patch.object(self.m, "CONFIG_PATH", self.config_path), \ patch.object(self.m, "api_request", return_value=None): self.m.switch_model("llama3") - with open(self.config_path) as f: - content = f.read() - self.assertIn('model = "llama3"', content) + with patch.object(self.m, "CONFIG_PATH", self.config_path): + self.assertEqual(self.m.get_current_model(), "llama3") def test_switch_replaces_existing_model_line(self): with open(self.config_path, "w") as f: @@ -58,9 +57,10 @@ def test_switch_replaces_existing_model_line(self): with patch.object(self.m, "CONFIG_PATH", self.config_path), \ patch.object(self.m, "api_request", return_value=None): self.m.switch_model("codellama") + with patch.object(self.m, "CONFIG_PATH", self.config_path): + self.assertEqual(self.m.get_current_model(), "codellama") with open(self.config_path) as f: content = f.read() - self.assertIn('model = "codellama"', content) self.assertNotIn("mistral", content) self.assertIn("context_window = 4096", content) diff --git a/tests/test_neuroslib.py b/tests/test_neuroslib.py new file mode 100644 index 0000000..29f6968 --- /dev/null +++ b/tests/test_neuroslib.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +""" +test_neuroslib.py — Unit tests for the shared neuroslib module. + +Covers section-aware config parsing (replacing the per-tool +key=value loops in nn and neuros-model), JSON DB atomic writes, and +the canonical helpers get_default_model/get_context_config/ +set_default_model. Runs in a tempdir so it doesn't touch the user's +real ~/.config/neuros. +""" + +import importlib.util +import json +import os +import sys +import tempfile +import unittest +from importlib.machinery import SourceFileLoader +from unittest.mock import patch + +LIB_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", + "config", "includes.chroot", "usr", "local", "bin", "neuroslib.py", +) + + +def load_lib(): + loader = SourceFileLoader("neuroslib", LIB_PATH) + spec = importlib.util.spec_from_loader("neuroslib", loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +class TestLoadConfig(unittest.TestCase): + """The new configparser-backed loader must understand [section] + headers, comments, and quote stripping that the prior naive loop + couldn't handle cleanly.""" + + def setUp(self): + self.lib = load_lib() + self.tmp = tempfile.mkdtemp() + + def test_returns_defaults_for_missing_file(self): + cfg = self.lib.load_config(os.path.join(self.tmp, "missing.conf")) + self.assertEqual(cfg["llm.model"], "mistral") + self.assertEqual(cfg["llm.host"], "localhost") + self.assertEqual(cfg["llm.port"], "11434") + self.assertFalse(cfg["context.window_title"]) + self.assertFalse(cfg["context.clipboard"]) + self.assertFalse(cfg["context.recent_files"]) + + def test_parses_flat_llm_section(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = "codellama"\nhost = example\nport = 9999\n') + cfg = self.lib.load_config(path) + self.assertEqual(cfg["llm.model"], "codellama") + self.assertEqual(cfg["llm.host"], "example") + self.assertEqual(cfg["llm.port"], "9999") + + def test_parses_context_section_with_truthy(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[context]\nwindow_title = true\nclipboard = 1\nrecent_files = yes\n') + cfg = self.lib.load_config(path) + self.assertTrue(cfg["context.window_title"]) + self.assertTrue(cfg["context.clipboard"]) + self.assertTrue(cfg["context.recent_files"]) + + def test_unknown_truthy_strings_default_off(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[context]\nwindow_title = maybe\n') + cfg = self.lib.load_config(path) + self.assertFalse(cfg["context.window_title"]) + + def test_full_line_comments_are_ignored(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('# this is a comment\n[llm]\n# another\nmodel = "mistral"\n') + cfg = self.lib.load_config(path) + self.assertEqual(cfg["llm.model"], "mistral") + + def test_interpolation_in_value_is_not_resolved(self): + """Model names with '=' in them (e.g. custom tags like + 'code=base') must round-trip verbatim, which configparser + interpolation would otherwise mangle.""" + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = "custom=tag"\n') + cfg = self.lib.load_config(path) + self.assertEqual(cfg["llm.model"], "custom=tag") + + +class TestGetDefaultModel(unittest.TestCase): + def setUp(self): + self.lib = load_lib() + self.tmp = tempfile.mkdtemp() + + def test_returns_fallback_on_missing_config(self): + path = os.path.join(self.tmp, "nope.conf") + self.assertEqual(self.lib.get_default_model(path), "mistral") + + def test_strips_quotes_in_value(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = "qwen2.5:7b"\n') + self.assertEqual(self.lib.get_default_model(path), "qwen2.5:7b") + + def test_blank_model_falls_back(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = " "\n') + self.assertEqual(self.lib.get_default_model(path), "mistral") + + +class TestGetContextConfig(unittest.TestCase): + def setUp(self): + self.lib = load_lib() + self.tmp = tempfile.mkdtemp() + + def test_all_off_when_file_missing(self): + cfg = self.lib.get_context_config(os.path.join(self.tmp, "nope.conf")) + self.assertEqual(cfg, { + "window_title": False, "clipboard": False, "recent_files": False, + }) + + def test_only_explicit_sources_enabled(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[context]\nwindow_title = true\n') + cfg = self.lib.get_context_config(path) + self.assertTrue(cfg["window_title"]) + self.assertFalse(cfg["clipboard"]) + self.assertFalse(cfg["recent_files"]) + + +class TestSetDefaultModel(unittest.TestCase): + def setUp(self): + self.lib = load_lib() + self.tmp = tempfile.mkdtemp() + + def test_creates_file_and_directory(self): + path = os.path.join(self.tmp, "deep", "dir", "llm.conf") + self.lib.set_default_model("llama3", path) + self.assertTrue(os.path.exists(path)) + with open(path) as f: + data = f.read() + self.assertIn('model = llama3', data) + + def test_preserves_existing_keys(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = "mistral"\ncontext_window = 4096\n') + self.lib.set_default_model("codellama", path) + with open(path) as f: + data = f.read() + self.assertIn('model = codellama', data) + self.assertIn('context_window = 4096', data) + self.assertNotIn('mistral', data) + + def test_preserves_context_section(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = "mistral"\n[context]\nclipboard = true\n') + self.lib.set_default_model("qwen", path) + with open(path) as f: + data = f.read() + self.assertIn("[context]", data) + self.assertIn("clipboard = true", data) + self.assertIn('model = qwen', data) + + def test_overwrites_model_only(self): + path = os.path.join(self.tmp, "llm.conf") + with open(path, "w") as f: + f.write('[llm]\nmodel = "old"\ncontext_window = 8192\n' + '[context]\nwindow_title = true\n') + self.lib.set_default_model("new", path) + with open(path) as f: + data = f.read() + # Host/port are NOT touched, so the configurable values survive. + self.assertIn("context_window = 8192", data) + self.assertIn("window_title = true", data) + + +class TestLoadDbSaveDb(unittest.TestCase): + def setUp(self): + self.lib = load_lib() + self.tmp = tempfile.mkdtemp() + + def test_save_then_load_round_trips(self): + path = os.path.join(self.tmp, "state.json") + self.lib.save_db(path, {"a": 1, "b": [2, 3, 4]}) + self.assertEqual(self.lib.load_db(path), {"a": 1, "b": [2, 3, 4]}) + + def test_load_db_returns_default_when_missing(self): + self.assertEqual( + self.lib.load_db(os.path.join(self.tmp, "nope.json"), + default={"x": 9}), + {"x": 9}, + ) + + def test_save_db_writes_pretty_indented_json(self): + path = os.path.join(self.tmp, "pretty.json") + self.lib.save_db(path, {"k": "v"}) + with open(path) as f: + text = f.read() + self.assertIn("\n ", text) # two-space indent present + j = json.loads(text) + self.assertEqual(j, {"k": "v"}) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_nn.py b/tests/test_nn.py index 5f86daf..9183c8e 100644 --- a/tests/test_nn.py +++ b/tests/test_nn.py @@ -35,31 +35,33 @@ def load_nn(): return module class TestNNCore(unittest.TestCase): - """Test core nn CLI functionality.""" + """Test core nn CLI functionality. The previous implementation + hard-coded ``localhost:11434`` and ``~/.config/neuros/llm.conf`` as + source-level literals; the new nn derives them from the shared + neuroslib constants, so these tests assert that the resolved + values at runtime still point at the expected host/path.""" - def test_config_path_in_source(self): - """Test that nn source contains CONFIG_PATH with correct path.""" + def test_resolved_config_path_points_at_neuros_llm_conf(self): nn_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), '..', 'config', 'includes.chroot', 'usr', 'local', 'bin', 'nn' ) - with open(nn_path) as f: - content = f.read() - self.assertIn(".config/neuros/llm.conf", content) - self.assertIn("CONFIG_PATH", content) + loader = SourceFileLoader("nn_module", nn_path) + spec = importlib.util.spec_from_loader("nn_module", loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + self.assertTrue(module.CONFIG_PATH.endswith(".config/neuros/llm.conf")) - def test_ollama_url_correct(self): - """Test that OLLAMA_URL points to correct localhost port.""" - # Read the nn file and find OLLAMA_URL + def test_resolved_ollama_url_points_at_localhost_11434(self): nn_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), '..', 'config', 'includes.chroot', 'usr', 'local', 'bin', 'nn' ) - with open(nn_path) as f: - content = f.read() - - self.assertIn("localhost:11434", content) - self.assertIn("OLLAMA_URL", content) + loader = SourceFileLoader("nn_module", nn_path) + spec = importlib.util.spec_from_loader("nn_module", loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + self.assertIn("localhost:11434", module.OLLAMA_URL) def test_default_model_is_mistral(self): """Test that default model is mistral.""" diff --git a/tests/test_policy.py b/tests/test_policy.py new file mode 100755 index 0000000..5cc34f1 --- /dev/null +++ b/tests/test_policy.py @@ -0,0 +1,398 @@ +"""Tests for neuros-policy (loader pattern mirrors test_sandbox.py). + +Runs purely against the production script via compile()+exec() so we +do not need it on $PATH. No subprocess enforcement is exercised here +on purpose: every interesting invariant (bounds, regex shape, profile +emission, envelope reconciliation) is reachable from Python directly. +""" +import compileall +import io +import json +import os +import subprocess +import sys +import tempfile +import types +import unittest +from unittest import mock + +POLICY_PATH = "config/includes.chroot/usr/local/bin/neuros-policy" + + +def _load_module(): + with open(POLICY_PATH, "r", encoding="utf-8") as f: + src = f.read() + # Strip the shebang line for exec(); everything else is plain + # Python 3 and re-exports cleanly when given a real module + # namespace (so `cls.pl.validate_policy` style attribute access + # works; a plain dict would fail with AttributeError on lookup). + if src.startswith("#!"): + src = src.split("\n", 1)[1] + code = compile(src, POLICY_PATH, "exec") + mod = types.ModuleType("neuros_policy_under_test") + exec(code, mod.__dict__) + return mod + + +class TestCompile(unittest.TestCase): + def test_compiles_clean(self): + # The wrapper imports no third-party modules, so py_compile + # is a sufficient static check before we even try exec(). + self.assertTrue(compileall.compile_file(POLICY_PATH, + quiet=1, + force=True)) + + +def _write_policy(d): + """Dump ``d`` to a temp .json file and return its path string + (not the wrapper, so the production loader can open() it).""" + fd, p = tempfile.mkstemp(prefix="neuros-policy-", suffix=".json") + with os.fdopen(fd, "w") as f: + json.dump(d, f) + return p + + +def _good_policy(**overrides): + base = { + "name": "neuros-default", + "version": "1.0.0", + "defaults": { + "mem": "256M", + "pids": 64, + "cpu_quota": 50000, + "timeout": 30, + }, + "profiles": { + "strict": ["CAP_NET_RAW", "CAP_SYS_ADMIN", "CAP_SYS_PTRACE"], + "moderate": ["CAP_NET_RAW", "CAP_SYS_ADMIN"], + "permissive": [], + }, + "net": "private", + "readonly": True, + "env_allowlist": ["PATH", "LANG"], + "syscalls": None, + } + base.update(overrides) + return base + + +class TestValidate(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.pl = _load_module() + + def test_clean_policy_passes(self): + path = _write_policy(_good_policy()) + try: + errs, cleaned = self.pl.validate_policy(_good_policy()) + self.assertEqual(errs, []) + self.assertEqual(cleaned["strict"], + sorted({"CAP_NET_RAW", "CAP_SYS_ADMIN", + "CAP_SYS_PTRACE"})) + finally: + os.unlink(path) + + def test_root_must_be_dict(self): + path = _write_policy(["not", "a", "dict"]) + try: + with self.assertRaises(self.pl.PolicyError) as cm: + self.pl._load_policy(path) + self.assertIn("JSON object", str(cm.exception)) + finally: + os.unlink(path) + + def test_missing_name_violates(self): + p = _good_policy(); del p["name"] + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("name" in str(e) for e in errs)) + + def test_blank_name_violates(self): + errs, _ = self.pl.validate_policy(_good_policy(name=" ")) + self.assertTrue(any("name" in str(e) for e in errs)) + + def test_bad_version_violates(self): + errs, _ = self.pl.validate_policy(_good_policy(version="v1")) + self.assertTrue(any("version" in str(e) for e in errs)) + + def test_mem_below_minimum_violates(self): + p = _good_policy() + p["defaults"]["mem"] = "8K" + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.mem" in str(e) for e in errs)) + + def test_mem_above_maximum_violates(self): + p = _good_policy() + p["defaults"]["mem"] = "2G" + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.mem" in str(e) for e in errs)) + + def test_mem_unparseable_violates(self): + p = _good_policy() + p["defaults"]["mem"] = "lots" + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.mem" in str(e) for e in errs)) + + def test_pids_out_of_range_violates(self): + p = _good_policy() + p["defaults"]["pids"] = 9999 + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.pids" in str(e) for e in errs)) + + def test_pids_zero_violates(self): + p = _good_policy() + p["defaults"]["pids"] = 0 + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.pids" in str(e) for e in errs)) + + def test_cpu_quota_out_of_range_violates(self): + p = _good_policy() + p["defaults"]["cpu_quota"] = 500 + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.cpu_quota" in str(e) for e in errs)) + + def test_timeout_too_long_violates(self): + p = _good_policy() + p["defaults"]["timeout"] = 7200 + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("defaults.timeout" in str(e) for e in errs)) + + def test_profile_name_uppercase_violates(self): + p = _good_policy() + p["profiles"]["Strict"] = [] + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("profile name" in str(e).lower() or + "Strict" in str(e) for e in errs)) + + def test_bad_cap_name_violates(self): + p = _good_policy() + p["profiles"]["strict"] = ["net_raw", "CAP_SYS_ADMIN"] + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("non-cap name" in str(e) for e in errs)) + + def test_empty_profile_is_allowed(self): + p = _good_policy() + p["profiles"]["permissive"] = [] + errs, _ = self.pl.validate_policy(p) + self.assertEqual(errs, []) + + def test_net_must_be_known(self): + p = _good_policy() + p["net"] = "loopback" + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("net" in str(e) for e in errs)) + + def test_bad_env_name_violates(self): + p = _good_policy() + p["env_allowlist"] = ["path", "LANG"] + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("env_allowlist" in str(e) for e in errs)) + + def test_readonly_must_be_bool(self): + p = _good_policy() + p["readonly"] = "yes" + errs, _ = self.pl.validate_policy(p) + self.assertTrue(any("readonly" in str(e) for e in errs)) + + def test_caps_deduped_and_sorted(self): + p = _good_policy() + p["profiles"]["strict"] = ["CAP_SYS_ADMIN", "CAP_NET_RAW", + "CAP_SYS_ADMIN"] + _, cleaned = self.pl.validate_policy(p) + self.assertEqual(cleaned["strict"], + ["CAP_NET_RAW", "CAP_SYS_ADMIN"]) + + +class TestTranspile(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.pl = _load_module() + + def test_strict_profile_builds_cap_regex(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + argv = self.pl.transpile_argv(p, "strict", cleaned) + # Default order is: --mem, --pids, --cpu-quota, --timeout, + # --cap-drop. Assert presence + cap regex shape. + self.assertIn("--cap-drop", argv) + cap_idx = argv.index("--cap-drop") + # Sorted alphabetically per validate_policy. + self.assertEqual(argv[cap_idx + 1], + "CAP_(CAP_NET_RAW|CAP_SYS_ADMIN|CAP_SYS_PTRACE)") + + def test_permissive_profile_emits_never_match(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + argv = self.pl.transpile_argv(p, "permissive", cleaned) + self.assertEqual(argv[argv.index("--cap-drop") + 1], + self.pl.NEVER_MATCH_CAP_REGEX) + + def test_unknown_profile_raises(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + with self.assertRaises(self.pl.PolicyError): + self.pl.transpile_argv(p, "unknown", cleaned) + + def test_no_profile_omits_cap_drop(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + argv = self.pl.transpile_argv(p, None, cleaned) + self.assertNotIn("--cap-drop", argv) + + +class TestCheck(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.pl = _load_module() + + def test_conformant_envelope_passes(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + env = { + "exit_code": 0, "stdout": "", "stderr": "", + "wall_clock_ms": 1500, "timeout_hit": False, + "peak_mem_estimate": 50 * 1024 * 1024, + } + violations = self.pl.check_envelope_against_policy(p, env, cleaned) + self.assertEqual(violations, []) + + def test_wall_clock_over_violates(self): + p = _good_policy() + p["defaults"]["timeout"] = 5 + _, cleaned = self.pl.validate_policy(p) + env = {"wall_clock_ms": 12000, "timeout_hit": False, + "peak_mem_estimate": None} + violations = self.pl.check_envelope_against_policy(p, env, cleaned) + self.assertTrue(any(v[0] == "wall_clock" for v in violations)) + + def test_peak_mem_over_violates(self): + p = _good_policy() + p["defaults"]["mem"] = "128M" + _, cleaned = self.pl.validate_policy(p) + env = {"wall_clock_ms": 1000, "timeout_hit": False, + "peak_mem_estimate": 200 * 1024 * 1024} + violations = self.pl.check_envelope_against_policy(p, env, cleaned) + self.assertTrue(any(v[0] == "peak_mem" for v in violations)) + + def test_timeout_hit_always_violates(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + env = {"wall_clock_ms": 1000, "timeout_hit": True} + violations = self.pl.check_envelope_against_policy(p, env, cleaned) + self.assertTrue(any(v[0] == "timeout_hit" for v in violations)) + + def test_malformed_envelope_violates(self): + p = _good_policy() + _, cleaned = self.pl.validate_policy(p) + violations = self.pl.check_envelope_against_policy(p, "string", + cleaned) + self.assertTrue(any(v[0] == "malformed" for v in violations)) + + +class TestMainDispatch(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.pl = _load_module() + + def _ok_envelope(self): + return json.dumps({ + "exit_code": 0, "stdout": "", "stderr": "", + "wall_clock_ms": 1000, "timeout_hit": False, + "peak_mem_estimate": 1024 * 1024, + }) + + def test_validate_dispatches(self): + path = _write_policy(_good_policy()) + try: + with mock.patch.object(sys, "stdout", io.StringIO()): + rc = self.pl.main(["validate", path]) + self.assertEqual(rc, 0) + finally: + os.unlink(path) + + def test_validate_text_reports_violations(self): + bad = _good_policy() + bad["defaults"]["mem"] = "lots" + path = _write_policy(bad) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.pl.main(["validate", path]) + self.assertEqual(rc, 1) + self.assertIn("violation", buf.getvalue()) + finally: + os.unlink(path) + + def test_validate_json_envelope_shape(self): + path = _write_policy(_good_policy()) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.pl.main(["validate", "--json", path]) + self.assertEqual(rc, 0) + line = buf.getvalue().strip() + obj = json.loads(line) + self.assertTrue(obj["ok"]) + self.assertEqual(obj["errors"], []) + self.assertEqual(obj["name"], "neuros-default") + finally: + os.unlink(path) + + def test_check_passes(self): + p_path = _write_policy(_good_policy()) + e_path = _write_policy(json.loads(self._ok_envelope())) + try: + with mock.patch.object(sys, "stdout", io.StringIO()): + rc = self.pl.main(["check", p_path, "--envelope", e_path]) + self.assertEqual(rc, 0) + finally: + os.unlink(p_path); os.unlink(e_path) + + def test_check_fails_on_violation(self): + bad_env = json.loads(self._ok_envelope()) + bad_env["wall_clock_ms"] = 999_999 # exceeds 30s default + p_path = _write_policy(_good_policy()) + e_path = _write_policy(bad_env) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.pl.main(["check", p_path, "--envelope", e_path]) + self.assertEqual(rc, 1) + finally: + os.unlink(p_path); os.unlink(e_path) + + def test_transpile_emits_argv_text(self): + path = _write_policy(_good_policy()) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.pl.main(["transpile", path, + "--profile", "strict"]) + self.assertEqual(rc, 0) + txt = buf.getvalue().strip() + self.assertIn("--mem 256M", txt) + self.assertIn("--pids 64", txt) + self.assertIn("--cpu-quota 50000", txt) + self.assertIn("--cap-drop CAP_(", txt) + finally: + os.unlink(path) + + def test_transpile_refuses_invalid_policy(self): + bad = _good_policy() + bad["name"] = "" + path = _write_policy(bad) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.pl.main(["transpile", path]) + self.assertEqual(rc, 1) + finally: + os.unlink(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_replay.py b/tests/test_replay.py new file mode 100755 index 0000000..5d51527 --- /dev/null +++ b/tests/test_replay.py @@ -0,0 +1,435 @@ +"""Tests for neuros-replay (offline diagnostics over envelopes). + +Runs purely against the production script via compile()+exec() so we +do not need it on $PATH. No subprocess shell-out is exercised here on +purpose: every interesting invariant (bounds of pct_delta, missing +fields, tolerance overrides, JSON envelope shape) is reachable from +Python directly. The `extract` subcommand writes to stdout which we +capture via sys.stdout mocks. +""" +import compileall +import io +import json +import os +import subprocess +import sys +import tempfile +import types +import unittest +from unittest import mock + +REPLAY_PATH = "config/includes.chroot/usr/local/bin/neuros-replay" + + +def _load_module(): + with open(REPLAY_PATH, "r", encoding="utf-8") as f: + src = f.read() + if src.startswith("#!"): + src = src.split("\n", 1)[1] + code = compile(src, REPLAY_PATH, "exec") + mod = types.ModuleType("neuros_replay_under_test") + exec(code, mod.__dict__) + return mod + + +def _envelope(**overrides): + """A conformant sandbox envelope with all fields present. + Picked exit_code=0, wall_clock_ms=1000, peak_mem=64M so the + tests can mutate one field at a time without making the envelope + regress accidentally. + """ + base = { + "exit_code": 0, + "stdout": "", + "stderr": "", + "wall_clock_ms": 1000, + "timeout_hit": False, + "peak_mem_estimate": 64 * 1024 * 1024, + } + base.update(overrides) + return base + + +def _write_envelope(env): + """Dump ``env`` to a temp .json file and return its path string.""" + fd, p = tempfile.mkstemp(prefix="neuros-replay-", suffix=".json") + with os.fdopen(fd, "w") as f: + json.dump(env, f) + return p + + +class TestCompile(unittest.TestCase): + def test_compiles_clean(self): + self.assertTrue(compileall.compile_file(REPLAY_PATH, + quiet=1, + force=True)) + + +class TestPctDelta(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rp = _load_module() + + def test_normal_positive_delta(self): + # b = 1100, a = 1000 → +10% + self.assertEqual(self.rp._pct_delta(1000, 1100), 10.0) + + def test_normal_negative_delta(self): + self.assertEqual(self.rp._pct_delta(1000, 900), -10.0) + + def test_zero_returns_none(self): + # Zero in the denominator would explode to +inf; surface as None. + self.assertIsNone(self.rp._pct_delta(0, 100)) + + def test_missing_returns_none(self): + self.assertIsNone(self.rp._pct_delta(None, 100)) + self.assertIsNone(self.rp._pct_delta(100, None)) + self.assertIsNone(self.rp._pct_delta(None, None)) + + def test_non_numeric_returns_none(self): + self.assertIsNone(self.rp._pct_delta("foo", 100)) + self.assertIsNone(self.rp._pct_delta(100, "bar")) + + def test_rounded_to_two_decimals(self): + # (103 - 100) / 100 * 100 = 3.0 + self.assertEqual(self.rp._pct_delta(100, 103), 3.0) + + +class TestFormatBytes(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rp = _load_module() + + def test_none_returns_na(self): + self.assertEqual(self.rp._format_bytes(None), "n/a") + + def test_bytes(self): + self.assertEqual(self.rp._format_bytes(512), "512B") + + def test_kib(self): + self.assertEqual(self.rp._format_bytes(2048), "2.0KiB") + + def test_mib(self): + self.assertEqual(self.rp._format_bytes(64 * 1024 * 1024), + "64.0MiB") + + +class TestDiff(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rp = _load_module() + + def test_identical_envelopes_pass(self): + env = _envelope() + verdict = self.rp.diff_envelopes(env, env, 10.0, 20.0) + self.assertTrue(verdict["ok"]) + self.assertEqual(verdict["rule_violations"], []) + self.assertEqual(verdict["wall_clock_pct_delta"], 0.0) + + def test_wall_clock_within_tolerance_passes(self): + # +5% is within default 10% wall-clock tolerance. + verdict = self.rp.diff_envelopes( + _envelope(wall_clock_ms=1000), + _envelope(wall_clock_ms=1050), + 10.0, 20.0) + self.assertTrue(verdict["ok"]) + + def test_wall_clock_over_tolerance_fails(self): + # +50% is well outside default 10%. + verdict = self.rp.diff_envelopes( + _envelope(wall_clock_ms=1000), + _envelope(wall_clock_ms=1500), + 10.0, 20.0) + self.assertFalse(verdict["ok"]) + self.assertTrue(any("wall_clock_pct_delta" in v + for v in verdict["rule_violations"])) + + def test_peak_mem_over_tolerance_fails(self): + # +50% memory. + verdict = self.rp.diff_envelopes( + _envelope(peak_mem_estimate=64 * 1024 * 1024), + _envelope(peak_mem_estimate=96 * 1024 * 1024), + 10.0, 20.0) + self.assertFalse(verdict["ok"]) + self.assertTrue(any("peak_mem_pct_delta" in v + for v in verdict["rule_violations"])) + + def test_peak_mem_null_skips_check(self): + verdict = self.rp.diff_envelopes( + _envelope(peak_mem_estimate=None), + _envelope(peak_mem_estimate=None), + 10.0, 20.0) + self.assertTrue(verdict["ok"]) + self.assertIsNone(verdict["peak_mem_pct_delta"]) + + def test_exit_code_mismatch_fails(self): + verdict = self.rp.diff_envelopes( + _envelope(exit_code=0), + _envelope(exit_code=1), + 10.0, 20.0) + self.assertFalse(verdict["ok"]) + self.assertTrue(any("exit_code" in v + for v in verdict["rule_violations"])) + + def test_timeout_hit_mismatch_fails(self): + verdict = self.rp.diff_envelopes( + _envelope(timeout_hit=False), + _envelope(timeout_hit=True), + 10.0, 20.0) + self.assertFalse(verdict["ok"]) + self.assertTrue(any("timeout_hit" in v + for v in verdict["rule_violations"])) + + def test_zero_wall_clock_handled(self): + # a has wall_clock_ms=0 → pct_delta is None → no violation + verdict = self.rp.diff_envelopes( + _envelope(wall_clock_ms=0), + _envelope(wall_clock_ms=500), + 10.0, 20.0) + self.assertTrue(verdict["ok"]) + self.assertIsNone(verdict["wall_clock_pct_delta"]) + + def test_tight_tolerance_overrides_default(self): + # Tolerances set to 1% — even +5% regresses. + verdict = self.rp.diff_envelopes( + _envelope(wall_clock_ms=1000), + _envelope(wall_clock_ms=1050), + 1.0, 1.0) + self.assertFalse(verdict["ok"]) + + +class TestExplain(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rp = _load_module() + + def test_explain_envelope_includes_all_keys(self): + info = self.rp.explain_envelope(_envelope()) + for k in ("exit_code", "wall_clock_ms", "peak_mem_estimate", + "timeout_hit", "stdout_bytes", "stderr_bytes"): + self.assertIn(k, info) + + def test_explain_line_is_single_space_separated(self): + info = self.rp.explain_envelope(_envelope()) + line = self.rp._explain_line(info) + # Starts with the canonical key=value structure + self.assertTrue(line.startswith("exit=")) + self.assertIn("wall=", line) + self.assertIn("peak=64.0MiB", line) + self.assertIn("timeout=False", line) + + def test_missing_fields_render_na(self): + env = {"exit_code": 0} # no wall / peak / etc. + info = self.rp.explain_envelope(env) + self.assertEqual(info["wall_clock_ms"], "n/a") + self.assertEqual(info["peak_mem_estimate"], "n/a") + line = self.rp._explain_line(info) + self.assertIn("wall=n/a", line) + self.assertIn("peak=n/a", line) + + +class TestLoadEnvelope(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rp = _load_module() + + def test_missing_file(self): + with self.assertRaises(self.rp.ReplayError) as cm: + self.rp._load_envelope("/nonexistent/envelope.json") + self.assertIn("cannot read", str(cm.exception)) + + def test_malformed_json(self): + path = _write_envelope_str("not valid json {{{") + try: + with self.assertRaises(self.rp.ReplayError): + self.rp._load_envelope(path) + finally: + os.unlink(path) + + def test_root_must_be_object(self): + path = _write_envelope_str("[1, 2, 3]") + try: + with self.assertRaises(self.rp.ReplayError): + self.rp._load_envelope(path) + finally: + os.unlink(path) + + +def _write_envelope_str(text): + fd, p = tempfile.mkstemp(prefix="neuros-replay-bad-", suffix=".json") + with os.fdopen(fd, "w") as f: + f.write(text) + return p + + +class TestMainDispatch(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rp = _load_module() + + def _ok(self): + return _envelope() + + def test_diff_returns_zero_for_identical(self): + a = b = _write_envelope(self._ok()) + try: + with mock.patch.object(sys, "stdout", io.StringIO()): + rc = self.rp.main(["diff", a, b]) + self.assertEqual(rc, 0) + finally: + os.unlink(a) + + def test_diff_returns_one_for_regression(self): + a = _write_envelope(self._ok()) + b = _write_envelope(self._ok() | {"wall_clock_ms": 9999}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.rp.main(["diff", a, b]) + self.assertEqual(rc, 1) + self.assertIn("violation", buf.getvalue()) + finally: + os.unlink(a); os.unlink(b) + + def test_diff_json_envelope_shape(self): + # +5% on wall-clock is within the default 10% tolerance. + a = _write_envelope(self._ok()) + b = _write_envelope(self._ok() | {"wall_clock_ms": 1050}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["diff", "--json", a, b]) + self.assertEqual(rc, 0) + obj = json.loads(buf.getvalue().strip()) + self.assertTrue(obj["ok"]) + self.assertEqual(obj["wall_clock_pct_delta"], 5.0) + self.assertEqual(obj["peak_mem_pct_delta"], 0.0) + finally: + os.unlink(a); os.unlink(b) + + def test_diff_bad_input_returns_two(self): + with mock.patch.object(sys, "stdout", io.StringIO()), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["diff", "/nonexistent/a", + "/nonexistent/b"]) + self.assertEqual(rc, 2) + + def test_explain_prints_line(self): + path = _write_envelope(self._ok()) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.rp.main(["explain", path]) + self.assertEqual(rc, 0) + self.assertIn("exit=0", buf.getvalue()) + self.assertIn("wall=1000ms", buf.getvalue()) + finally: + os.unlink(path) + + def test_explain_json_envelope_shape(self): + path = _write_envelope(self._ok()) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.rp.main(["explain", "--json", path]) + self.assertEqual(rc, 0) + obj = json.loads(buf.getvalue().strip()) + self.assertEqual(obj["exit_code"], 0) + self.assertEqual(obj["wall_clock_ms"], 1000) + self.assertIn("neuros_replay_version", obj) + finally: + os.unlink(path) + + def test_extract_stdout(self): + env = self._ok() | {"stdout": "hello\n"} + path = _write_envelope(env) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["extract", path, + "--stream", "stdout"]) + self.assertEqual(rc, 0) + self.assertIn("hello", buf.getvalue()) + self.assertNotIn("===STDERR===", buf.getvalue()) + finally: + os.unlink(path) + + def test_extract_stderr(self): + env = self._ok() | {"stderr": "boom\n"} + path = _write_envelope(env) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["extract", path, + "--stream", "stderr"]) + self.assertEqual(rc, 0) + self.assertIn("boom", buf.getvalue()) + finally: + os.unlink(path) + + def test_extract_both_has_separator(self): + env = self._ok() | {"stdout": "x", "stderr": "y"} + path = _write_envelope(env) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["extract", path, + "--stream", "both"]) + self.assertEqual(rc, 0) + self.assertIn("x", buf.getvalue()) + self.assertIn("===STDERR===", buf.getvalue()) + self.assertIn("y", buf.getvalue()) + finally: + os.unlink(path) + + def test_extract_both_when_stdout_empty(self): + # When one stream is empty, --stream both still emits + # the ===STDERR=== separator so a downstream parser + # can split the streams without field-name re-mapping. + env = self._ok() | {"stdout": "", "stderr": "boom\n"} + path = _write_envelope(env) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["extract", path, + "--stream", "both"]) + self.assertEqual(rc, 0) + self.assertIn("===STDERR===", buf.getvalue()) + self.assertIn("boom", buf.getvalue()) + finally: + os.unlink(path) + + def test_extract_both_when_stderr_empty(self): + # Symmetric case to test_extract_both_when_stdout_empty: + # if stderr is empty, --stream both still emits the + # ===STDERR=== separator before the (empty) stderr + # payload, so a downstream parser can rely on the + # boundary marker in either stream condition. + env = self._ok() | {"stdout": "hello\n", "stderr": ""} + path = _write_envelope(env) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["extract", path, + "--stream", "both"]) + self.assertEqual(rc, 0) + self.assertIn("hello", buf.getvalue()) + self.assertIn("===STDERR===", buf.getvalue()) + finally: + os.unlink(path) + def test_extract_nonexistent_returns_two(self): + with mock.patch.object(sys, "stdout", io.StringIO()), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = self.rp.main(["extract", "/nonexistent/env.json"]) + self.assertEqual(rc, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runbook.py b/tests/test_runbook.py new file mode 100755 index 0000000..c0564a0 --- /dev/null +++ b/tests/test_runbook.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python3 +"""Tests for neuros-runbook runbook verifier. + +These tests do not need real container kernel delegation because they +mock subprocess.run (the call into neuros-sandbox) and the inner +_run_step helper. They assert: + + * Runbook parser accepts well-formed JSON arrays of steps. + * Runbook parser rejects name collisions, non-string scripts, malformed + expect.regex, etc., with a RunbookError that names the bad step. + * _run_step surfaces stdout/stderr/exit_code from the JSON envelope. + * Per-step assertion logic (exit_code, stdout_matches, timeout_forbidden) + surfaces failures as a list. + * The cmd_run dispatcher exits 0 when all steps pass, 1 on any failure. + * --only-violations filters table output. + * --json emits an envelope whose schema matches the documented contract. +""" +import importlib.util +import io +import json +import os +import sys +import tempfile +import types +import unittest +from unittest import mock + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) +_RUNBOOK_PATH = os.path.join( + ROOT, "config", "includes.chroot", "usr", "local", "bin", "neuros-runbook") + + +def _load_neuros_runbook(): + """Load `neuros-runbook` (no .py suffix) by compiling + exec in a + fresh module namespace, mirroring the loader pattern used in + tests/test_sandbox.py and tests/test_bench.py.""" + with open(_RUNBOOK_PATH) as f: + source = f.read() + code = compile(source, _RUNBOOK_PATH, "exec") + module = sys.modules.get("neuros_runbook") or types.ModuleType( + "neuros_runbook") + module.__file__ = _RUNBOOK_PATH + sys.modules["neuros_runbook"] = module + exec(code, module.__dict__) + return module + + +rb = _load_neuros_runbook() + + +class TestRunbookLoader(unittest.TestCase): + """Locks the runbook JSON validation contract.""" + + def _write(self, content): + f = tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") + json.dump(content, f) + f.close() + return f.name + + def test_well_formed_runbook_loads(self): + path = self._write([ + {"name": "smoke", "script": "print(1)\n"}, + {"name": "fail", "script": "raise Exception\n", + "expect": {"exit_code": 1}}, + {"name": "regex", "script": "print('hello')\n", + "expect": {"stdout_matches": "hel+"}}, + ]) + try: + steps = rb.load_runbook(path) + self.assertEqual(len(steps), 3) + self.assertEqual(steps[0]["name"], "smoke") + self.assertEqual(steps[1]["expect"]["exit_code"], 1) + finally: + os.unlink(path) + + def test_duplicate_name_rejected(self): + path = self._write([ + {"name": "x", "script": "pass\n"}, + {"name": "x", "script": "pass\n"}, + ]) + try: + with self.assertRaises(rb.RunbookError) as cm: + rb.load_runbook(path) + self.assertIn("duplicate name", str(cm.exception)) + finally: + os.unlink(path) + + def test_empty_script_rejected(self): + path = self._write([{"name": "x", "script": ""}]) + try: + with self.assertRaises(rb.RunbookError): + rb.load_runbook(path) + finally: + os.unlink(path) + + def test_non_array_top_level_rejected(self): + f = tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") + f.write(json.dumps({"not": "an array"})) + f.close() + try: + with self.assertRaises(rb.RunbookError) as cm: + rb.load_runbook(f.name) + self.assertIn("must be a JSON array", str(cm.exception)) + finally: + os.unlink(f.name) + + def test_invalid_regex_rejected_at_parse_time(self): + path = self._write([ + {"name": "bad", "script": "pass\n", + "expect": {"stdout_matches": "[unclosed"}}, + ]) + try: + with self.assertRaises(rb.RunbookError) as cm: + rb.load_runbook(path) + self.assertIn("regex invalid", str(cm.exception)) + self.assertIn("bad", str(cm.exception)) + finally: + os.unlink(path) + + def test_empty_name_rejected(self): + path = self._write([{"name": "", "script": "pass\n"}]) + try: + with self.assertRaises(rb.RunbookError): + rb.load_runbook(path) + finally: + os.unlink(path) + + def test_step_must_be_object(self): + path = self._write(["not a dict"]) + try: + with self.assertRaises(rb.RunbookError): + rb.load_runbook(path) + finally: + os.unlink(path) + + +class TestRunStepAndAssertions(unittest.TestCase): + """Drive _run_step with a mocked subprocess.run so the JSON-envelope + parsing + assertion logic both get exercised.""" + + def _fake_cp(self, stdout_text, returncode=0): + cp = mock.Mock() + cp.stdout = stdout_text.encode() + cp.stderr = b"" + cp.returncode = returncode + return cp + + def test_exit_code_match(self): + step = {"name": "x", "script": "print(1)", + "expect": {"exit_code": 0}} + cp = self._fake_cp(json.dumps({ + "exit_code": 0, "stdout": "1\n", + "wall_clock_ms": 1, "timeout_hit": False, + "peak_mem_estimate": 1024})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + env_obj, failures = rb._run_step(step, + env={"PATH": "/bin"}) + self.assertEqual(env_obj["exit_code"], 0) + self.assertEqual(failures, []) + + def test_exit_code_mismatch(self): + step = {"name": "x", "script": "raise Exception", + "expect": {"exit_code": 0}} + cp = self._fake_cp(json.dumps({ + "exit_code": 1, "stdout": "", + "wall_clock_ms": 1, "timeout_hit": False, + "peak_mem_estimate": None})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + env_obj, failures = rb._run_step(step, + env={"PATH": "/bin"}) + self.assertTrue(any("exit_code" in f for f in failures)) + + def test_exit_code_any(self): + step = {"name": "x", "script": "anything", + "expect": {"exit_code": "any"}} + cp = self._fake_cp(json.dumps({ + "exit_code": 137, "stdout": "", + "wall_clock_ms": 1, "timeout_hit": False, + "peak_mem_estimate": 100})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + env_obj, failures = rb._run_step(step, + env={"PATH": "/bin"}) + self.assertEqual(failures, []) + + def test_stdout_pattern_mismatch(self): + step = {"name": "x", "script": "pass", + "expect": {"stdout_matches": "needle"}} + cp = self._fake_cp(json.dumps({ + "exit_code": 0, "stdout": "haystack\n", + "wall_clock_ms": 1, "timeout_hit": False, + "peak_mem_estimate": None})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + _, failures = rb._run_step(step, env={"PATH": "/bin"}) + self.assertTrue(any("stdout_matches" in f for f in failures)) + + def test_stderr_pattern_match(self): + step = {"name": "x", "script": "raise Exception", + "expect": {"stderr_matches": "Traceback"}} + cp = self._fake_cp(json.dumps({ + "exit_code": 1, "stdout": "", + "stderr": "Traceback (most recent call last):\n", + "wall_clock_ms": 1, "timeout_hit": False, + "peak_mem_estimate": None})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + _, failures = rb._run_step(step, env={"PATH": "/bin"}) + self.assertEqual(failures, []) + + def test_timeout_forbidden(self): + step = {"name": "x", "script": "while True: pass", + "expect": {"timeout_forbidden": True}} + cp = self._fake_cp(json.dumps({ + "exit_code": 124, "stdout": "", + "wall_clock_ms": 5000, "timeout_hit": True, + "peak_mem_estimate": None})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + _, failures = rb._run_step(step, env={"PATH": "/bin"}) + self.assertTrue(any("timeout" in f for f in failures)) + + def test_all_assertions_pass(self): + step = {"name": "x", "script": "print('hello')", + "expect": {"exit_code": 0, + "stdout_matches": "^hello$", + "stderr_matches": ".*"}} + cp = self._fake_cp(json.dumps({ + "exit_code": 0, "stdout": "hello\n", + "stderr": "", + "wall_clock_ms": 2, "timeout_hit": False, + "peak_mem_estimate": 4096})) + with mock.patch.object(rb.subprocess, "run", return_value=cp): + env_obj, failures = rb._run_step(step, + env={"PATH": "/bin"}) + self.assertEqual(env_obj["peak_mem_estimate"], 4096) + self.assertEqual(failures, []) + + +class TestCmdRunDispatch(unittest.TestCase): + """Top-level cmd_run aggregates step results into the documented + JSON envelope and pickes the right exit code.""" + + def _write(self, content): + f = tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") + json.dump(content, f) + f.close() + return f.name + + def _ok_envelope(self, exit_code=0, timeout_hit=False, + peak=None, wall_ms=10, stdout="", stderr=""): + return json.dumps({ + "exit_code": exit_code, "stdout": stdout, "stderr": stderr, + "wall_clock_ms": wall_ms, "timeout_hit": timeout_hit, + "peak_mem_estimate": peak}) + + def test_all_pass_returns_zero(self): + path = self._write([ + {"name": "a", "script": "pass", "expect": {"exit_code": 0}}, + {"name": "b", "script": "pass", "expect": {"exit_code": 0}}, + ]) + try: + ns = mock.MagicMock() + ns.runbook = path + ns.json = False + ns.only_violations = False + cp = mock.Mock(returncode=0, + stdout=self._ok_envelope().encode(), + stderr=b"") + with mock.patch.object(rb.subprocess, "run", return_value=cp), \ + mock.patch.object(sys, "stdout", io.StringIO()), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = rb.cmd_run(ns) + self.assertEqual(rc, 0) + finally: + os.unlink(path) + + def test_any_fail_returns_one(self): + """Two-step runbook where step b fails its exit_code + assertion. cmd_run should aggregate and exit 1.""" + path = self._write([ + {"name": "a", "script": "pass", "expect": {"exit_code": 0}}, + {"name": "b", "script": "raise", + "expect": {"exit_code": 0}}, + ]) + try: + ns = mock.MagicMock() + ns.runbook = path + ns.json = False + ns.only_violations = False + + # Stream two envelopes: step a OK, step b fails. + queue = iter([ + self._ok_envelope().encode(), + self._ok_envelope(exit_code=1, stderr="boom").encode(), + ]) + + def fake_run(*a, **kw): + cp = mock.Mock() + cp.stdout = next(queue) + cp.stderr = b"" + cp.returncode = 0 + return cp + + with (mock.patch.object(rb.subprocess, "run", + side_effect=fake_run), + mock.patch.object(sys, "stdout", io.StringIO()), + mock.patch.object(sys, "stderr", io.StringIO())): + rc = rb.cmd_run(ns) + self.assertEqual(rc, 1) + finally: + os.unlink(path) + + def test_only_violations_table_filters_pass(self): + path = self._write([ + {"name": "pass-step", "script": "pass"}, + {"name": "fail-step", "script": "raise", + "expect": {"exit_code": 0}}, + ]) + try: + ns = mock.MagicMock() + ns.runbook = path + ns.json = False + ns.only_violations = True + + ok_env = self._ok_envelope() + fail_env = self._ok_envelope(exit_code=1, stderr="boom") + queue = iter([ok_env.encode(), fail_env.encode()]) + + def fake_run(*a, **kw): + cp = mock.Mock() + cp.stdout = next(queue) + cp.stderr = b"" + cp.returncode = 0 + return cp + + with mock.patch.object(rb.subprocess, "run", + side_effect=fake_run): + buf_out = io.StringIO() + with mock.patch.object(sys, "stdout", buf_out), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = rb.cmd_run(ns) + self.assertEqual(rc, 1) + # Only fail-step should appear in the table + self.assertIn("fail-step", buf_out.getvalue()) + self.assertNotIn("pass-step", buf_out.getvalue()) + finally: + os.unlink(path) + + def test_json_envelope_shape(self): + path = self._write([ + {"name": "audit-step", "script": "pass", + "expect": {"exit_code": 0}}, + ]) + try: + ns = mock.MagicMock() + ns.runbook = path + ns.json = True + ns.only_violations = False + queue = iter([self._ok_envelope( + exit_code=0, peak=4096, wall_ms=2).encode()]) + + def fake_run(*a, **kw): + cp = mock.Mock() + cp.stdout = next(queue) + cp.stderr = b"" + cp.returncode = 0 + return cp + + with mock.patch.object(rb.subprocess, "run", + side_effect=fake_run): + buf_out = io.StringIO() + with mock.patch.object(sys, "stdout", buf_out): + rc = rb.cmd_run(ns) + self.assertEqual(rc, 0) + env = json.loads(buf_out.getvalue()) + self.assertIn("steps", env) + self.assertEqual(len(env["steps"]), 1) + step = env["steps"][0] + self.assertEqual(step["name"], "audit-step") + self.assertTrue(step["passed"]) + self.assertEqual(step["peak_mem_estimate"], 4096) + self.assertIn("version", env) + self.assertEqual(env["runbook_path"], path) + finally: + os.unlink(path) + + def test_runbook_error_dies_two(self): + path = "/nonexistent/runbook.json" + ns = mock.MagicMock() + ns.runbook = path + ns.json = False + ns.only_violations = False + with mock.patch.object(sys, "stderr", io.StringIO()): + with self.assertRaises(SystemExit) as cm: + rb.cmd_run(ns) + self.assertEqual(cm.exception.code, 2) + + +class TestMainDispatch(unittest.TestCase): + """Argparse routes correctly; exit code semantics.""" + + def test_no_subcommand_exits_two(self): + with self.assertRaises(SystemExit): + rb.main([]) + + def test_run_returns_zero_for_clean_run(self): + path = tempfile.NamedTemporaryFile("w", delete=False, suffix=".json") + json.dump([{"name": "x", "script": "pass", + "expect": {"exit_code": 0}}], path) + path.close() + try: + queue = iter([self._envelope_ok().encode()]) + + def fake_run(*a, **kw): + cp = mock.Mock() + cp.stdout = next(queue) + cp.stderr = b"" + cp.returncode = 0 + return cp + + with (mock.patch.object(rb.subprocess, "run", + side_effect=fake_run), + mock.patch.object(sys, "stdout", io.StringIO()), + mock.patch.object(sys, "stderr", io.StringIO())): + rc = rb.main(["run", path.name]) + self.assertEqual(rc, 0) + finally: + os.unlink(path.name) + + @staticmethod + def _envelope_ok(exit_code=0): + return json.dumps({ + "exit_code": exit_code, "stdout": "", "stderr": "", + "wall_clock_ms": 1, "timeout_hit": False, + "peak_mem_estimate": 0}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py new file mode 100755 index 0000000..437b2b3 --- /dev/null +++ b/tests/test_sandbox.py @@ -0,0 +1,550 @@ +#!/usr/bin/env python3 +"""Tests for neuros-sandbox safe-runner wrapper. + +These tests do not need real kernel/cgroup delegation because they +mock `subprocess.run` and the filesystem paths that the wrapper +constructs. They assert: + + * Default safe flag set is always passed to neuros-container + * --unsafe relaxes --net/--read-only/cap-drop and lets + --cap-drop-keep apply its own regex + * Host env is scrubbed to PATH only (no API keys leak) + * Watchdog timeout surfaces as exit 124 with timeout_hit=True + * JSON envelope is single-line and JSON-parseable + * Bundle extraction rejects absolute/traversal paths and cleans up + * Stdin is read when no script path is given + * Script file read errors surface as SandboxError +""" +import argparse +import io +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import types +import unittest +from unittest import mock + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) +_SANDBOX_PATH = os.path.join( + ROOT, "config", "includes.chroot", "usr", "local", "bin", "neuros-sandbox") + + +def _load_neuros_sandbox(): + """Load ``neuros-sandbox`` (no .py suffix in its install path) by + compiling its source and exec-ing in a fresh module namespace. + This is the only stable way to import a hyphen-named script. + + The loader caches in ``sys.modules`` indefinitely. After editing + the production source file, call ``importlib.reload(sb)`` (or + ``importlib.reload(sys.modules["neuros_sandbox"])``) to pick up + the change without restarting the test process. + """ + with open(_SANDBOX_PATH) as f: + source = f.read() + code = compile(source, _SANDBOX_PATH, "exec") + module = sys.modules.get("neuros_sandbox") or types.ModuleType( + "neuros_sandbox") + module.__file__ = _SANDBOX_PATH + sys.modules["neuros_sandbox"] = module + exec(code, module.__dict__) + return module + + +class _FakeStdStream: + """Mimic sys.stdout/sys.stderr's contract so the production + `sys.stdout.buffer.write(bytes)` AND `sys.stderr.write(str)` calls + both work. Use it via `with mock.patch.object(sys, "stdout", stream):`. + + Reads from .buffer accumulate bytes; reads from .write accumulate + text. The combined payload is exposed as .value(). + """ + + def __init__(self): + self._buffer = io.BytesIO() + self._text = [] + + @property + def buffer(self): + return self._buffer + + def write(self, data): + if isinstance(data, bytes): + self._buffer.write(data) + else: + self._text.append(data) + + def flush(self): + pass + + def value(self): + return self._buffer.getvalue() + "".join(self._text).encode( + "utf-8", errors="replace") + + +sb = _load_neuros_sandbox() + + +class TestParsePercent(unittest.TestCase): + def test_percent_string(self): + self.assertEqual(sb._parse_percent("50%"), 50000) + self.assertEqual(sb._parse_percent("25%"), 25000) + self.assertEqual(sb._parse_percent("100%"), 100000) + + def test_fraction(self): + self.assertEqual(sb._parse_percent("0.25"), 25000) + self.assertEqual(sb._parse_percent("1.0"), 100000) + + def test_microseconds(self): + self.assertEqual(sb._parse_percent("50000us"), 50000) + self.assertEqual(sb._parse_percent("100000us"), 100000) + + +class TestBuildArgv(unittest.TestCase): + def _args(self, **overrides): + # MUST use argparse.Namespace (not mock.Mock): Mock evaluates + # as truthy regardless of the kwarg passed, which silently + # disables the safety-net flag set in _build_argv. The defaults + # dict pattern lets overrides cleanly replace named defaults + # without producing argparse.Namespace(TypeError: multiple values). + defaults = dict( + unsafe=False, + mem=sb.DEFAULT_MEM, + pids=sb.DEFAULT_PIDS, + cpu_quota=sb.DEFAULT_CPU_QUOTA, + cap_drop=sb.DEFAULT_CAP_DROP, + cap_drop_keep=None, + rootfs=None, + name=None, + user=None, + ) + defaults.update(overrides) + return argparse.Namespace(**defaults) + + def test_safe_mode_default_flags_present(self): + argv = sb._build_argv(self._args(), script_target="-") + self.assertEqual(argv[0], "neuros-container") + self.assertEqual(argv[1], "run") + # safety nets are on + self.assertIn("--net", argv) + self.assertIn("--read-only", argv) + self.assertIn("--cap-drop", argv) + # Lock: the value following --cap-drop must be exactly the + # baseline DEFAULT_CAP_DROP, not the wrapper's blank string or + # any silent re-default. + self.assertEqual( + argv[argv.index("--cap-drop") + 1], sb.DEFAULT_CAP_DROP) + # defaults flow through + self.assertIn("--mem", argv) + self.assertEqual(argv[argv.index("--mem") + 1], sb.DEFAULT_MEM) + self.assertIn("--pids", argv) + self.assertEqual(argv[argv.index("--pids") + 1], str(sb.DEFAULT_PIDS)) + self.assertIn("--cpu-quota", argv) + self.assertEqual( + argv[argv.index("--cpu-quota") + 1], str(sb.DEFAULT_CPU_QUOTA)) + # name is generated when not given + name = argv[argv.index("--name") + 1] + self.assertTrue(name.startswith(sb.NAME_PREFIX)) + # entry point is the stdin python interpreter + self.assertEqual(argv[-3:], ["python3", "-u", "-"]) + + def test_unsafe_mode_relaxes_safe_flags(self): + argv = sb._build_argv(self._args(unsafe=True), script_target="-") + self.assertNotIn("--net", argv) + self.assertNotIn("--read-only", argv) + # The default cap-drop is still passed under --unsafe unless + # --cap-drop-keep replaces it; here we don't override. + self.assertIn("--cap-drop", argv) + # value-level invariant: even in unsafe mode the baseline + # DEFAULT_CAP_DROP is what lands in the primitive's --cap-drop + # slot, not any silent relaxation. + self.assertEqual( + argv[argv.index("--cap-drop") + 1], sb.DEFAULT_CAP_DROP) + + def test_unsafe_with_cap_drop_keep_replaces(self): + argv = sb._build_argv(self._args( + unsafe=True, cap_drop_keep=r"CAP_NET_RAW" + ), script_target="-") + self.assertNotIn("--net", argv) + self.assertNotIn("--read-only", argv) + self.assertEqual( + argv[argv.index("--cap-drop") + 1], r"CAP_NET_RAW") + + def test_rootfs_and_user_appear(self): + argv = sb._build_argv(self._args( + rootfs="/srv/rootfs", user="1000:1000" + ), script_target="/tmp/script.py") + self.assertEqual(argv[argv.index("--rootfs") + 1], "/srv/rootfs") + self.assertEqual(argv[argv.index("--user") + 1], "1000:1000") + # entry point ends with the script path + self.assertEqual(argv[-1], "/tmp/script.py") + + def test_name_override_passthrough(self): + argv = sb._build_argv( + self._args(name="explicit-name"), script_target="-") + self.assertEqual(argv[argv.index("--name") + 1], "explicit-name") + + def test_safe_mode_ignores_cap_drop_keep(self): + """--cap-drop-keep is documented as unsafe-mode-only. Pass it + under safe mode and assert (a) the value following --cap-drop + is still DEFAULT_CAP_DROP, and (b) the literal override value + the user passed in is NOT substituted into argv. Locks the + policy so a future refactor that adds a fast path doesn't + accidentally honor --cap-drop-keep in safe mode.""" + cap_keep = r"CAP_NET_RAW" + argv = sb._build_argv(self._args( + cap_drop_keep=cap_keep + ), script_target="-") + self.assertEqual( + argv[argv.index("--cap-drop") + 1], sb.DEFAULT_CAP_DROP) + # The literal override value must not appear in the produced + # argv (only DEFAULT_CAP_DROP should). This locks the policy + # at value-granularity rather than flag-name-granularity, + # which would be a tautology since _build_argv never emits + # --cap-drop-keep as a separate flag. + self.assertNotIn(cap_keep, argv) + + def test_cap_drop_keep_invalid_regex_raises_sandbox_error(self): + """Lock the contract that an invalid --cap-drop-keep regex + fails at the wrapper layer (SandboxError) before the primitive + is even invoked. Keeps the failure surface out of the + kernel-bridge ctypes call. + """ + with self.assertRaises(sb.SandboxError): + sb._build_argv(self._args( + unsafe=True, cap_drop_keep=r"[unclosed" + ), script_target="-") + + def test_cap_drop_keep_empty_string_raises_sandbox_error(self): + """Lock the contract that --cap-drop-keep '' under --unsafe is + an explicit user error, NOT a silent fallback to the default + cap-drop regex. Empty string was previously silenced by the + truthiness check; the is-not-None check exposes it. + """ + with self.assertRaises(sb.SandboxError): + sb._build_argv(self._args( + unsafe=True, cap_drop_keep="" + ), script_target="-") + + def test_cap_drop_keep_whitespace_only_raises_sandbox_error(self): + """Lock the contract that --cap-drop-keep ' ' (whitespace + only) is rejected just like the empty string — otherwise + re.compile(' ') silently produces a valid-but-useless regex + that matches literal whitespace exclusively. + """ + with self.assertRaises(sb.SandboxError): + sb._build_argv(self._args( + unsafe=True, cap_drop_keep=" \t\n" + ), script_target="-") + + +class TestLoadScript(unittest.TestCase): + def test_file_path(self): + with tempfile.NamedTemporaryFile("wb", delete=False) as f: + f.write(b"print('hello')\n") + path = f.name + try: + self.assertEqual(sb._load_script(path), b"print('hello')\n") + finally: + os.unlink(path) + + def test_missing_file_raises_sandbox_error(self): + with self.assertRaises(sb.SandboxError): + sb._load_script("/nonexistent/path/script.py") + + def test_empty_string_path_raises(self): + with tempfile.NamedTemporaryFile("wb", delete=False) as f: + f.write(b"") + path = f.name + try: + with self.assertRaises(sb.SandboxError): + sb._load_script(path) + finally: + os.unlink(path) + + +class TestStdInLoadScript(unittest.TestCase): + def test_stdin_when_path_is_none(self): + with mock.patch.object(sys, "stdin") as mock_stdin: + mock_stdin.buffer = io.BytesIO(b"from_stdin") + self.assertEqual(sb._load_script(None), b"from_stdin") + + +class TestExtractBundle(unittest.TestCase): + def _make_tar(self, members): + f = tempfile.NamedTemporaryFile("wb", delete=False, suffix=".tar") + with tarfile.open(fileobj=f, mode="w") as tar: + for name, content in members.items(): + data = content.encode("utf-8") if isinstance(content, str) else content + info = tarfile.TarInfo(name=name) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return f.name + + def test_extracts_simple_bundle_and_returns_path(self): + path = self._make_tar({"a.txt": "alpha", "b/c.txt": "beta"}) + try: + out = sb._extract_bundle(path) + try: + self.assertTrue(os.path.isdir(out)) + self.assertEqual( + open(os.path.join(out, "a.txt")).read(), "alpha") + self.assertEqual( + open(os.path.join(out, "b", "c.txt")).read(), "beta") + finally: + shutil.rmtree(out, ignore_errors=True) + finally: + os.unlink(path) + + def test_rejects_absolute_path_entry(self): + path = self._make_tar({"/abs.txt": "x"}) + try: + with self.assertRaises(sb.SandboxError): + sb._extract_bundle(path) + finally: + os.unlink(path) + + def test_rejects_traversal_entry(self): + path = self._make_tar({"../escape.txt": "x"}) + try: + with self.assertRaises(sb.SandboxError): + sb._extract_bundle(path) + finally: + os.unlink(path) + + +class TestRunInSandbox(unittest.TestCase): + """Drives cmd_run end-to-end with a mocked subprocess.run.""" + + def _args(self, **overrides): + defaults = dict( + cmd="run", + script="script.py", + mem="256M", + pids=sb.DEFAULT_PIDS, + cpu=None, + cpu_quota=sb.DEFAULT_CPU_QUOTA, + cap_drop=sb.DEFAULT_CAP_DROP, + cap_drop_keep=None, + rootfs=None, + name="test-name", + user=None, + bundle=None, + timeout=5, + unsafe=False, + json=False, + dry_run=False, + ) + defaults.update(overrides) + return argparse.Namespace(**defaults) + + @mock.patch.object(sb, "_load_script", return_value=b"script-bytes") + @mock.patch.object(sb, "_read_peak_memory", return_value=None) + @mock.patch.object(sb.subprocess, "run") + def test_human_mode_passes_through_stdout_and_stderr( + self, m_run, _mp, _ml): + cp = mock.Mock(returncode=0, stdout=b"hello\n", stderr=b"warn\n") + m_run.return_value = cp + # FakeStdStream exposes both .buffer (BytesIO) for binary + # writes AND .write(str) for the trailer line, so the + # production path through sys.stdout.buffer + sys.stderr.write + # both work cleanly. + s_out = _FakeStdStream() + s_err = _FakeStdStream() + with mock.patch.object(sys, "stdout", s_out), \ + mock.patch.object(sys, "stderr", s_err): + rc = sb.cmd_run(self._args()) + self.assertEqual(rc, 0) + out_payload = s_out.value() + err_payload = s_err.value() + self.assertIn(b"hello", out_payload) + self.assertIn(b"warn", err_payload) + self.assertIn(b"[neuros-sandbox]", err_payload) + # Env MUST be scrubbed to MINIMAL_ENV + positional, kwargs = m_run.call_args + self.assertEqual(kwargs["env"], sb.MINIMAL_ENV) + # input must contain the script bytes + self.assertEqual(kwargs["input"], b"script-bytes") + # Wall-clock mgmt passes timeout through + self.assertEqual(kwargs["timeout"], 5) + + @mock.patch.object(sb, "_load_script", return_value=b"script-bytes") + @mock.patch.object(sb, "_read_peak_memory", return_value=None) + @mock.patch.object(sb.subprocess, "run") + def test_json_mode_emits_single_line_envelope( + self, m_run, _mp, _ml): + cp = mock.Mock(returncode=2, stdout=b"out", stderr=b"err") + m_run.return_value = cp + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = sb.cmd_run(self._args(json=True)) + self.assertEqual(rc, 2) + lines = [l for l in buf.getvalue().splitlines() if l] + self.assertEqual(len(lines), 1) + env = json.loads(lines[0]) + self.assertEqual(env["exit_code"], 2) + self.assertEqual(env["stdout"], "out") + self.assertEqual(env["stderr"], "err") + self.assertFalse(env["timeout_hit"]) + self.assertIn("wall_clock_ms", env) + # peak_mem_estimate is propagated when non-None + self.assertIsNone(env["peak_mem_estimate"]) + + @mock.patch.object(sb, "_load_script", return_value=b"script-bytes") + @mock.patch.object(sb, "_read_peak_memory", return_value=8388608) + @mock.patch.object(sb.subprocess, "run") + def test_json_envelope_propagates_peak_mem( + self, m_run, _mp, _ml): + m_run.return_value = mock.Mock( + returncode=0, stdout=b"", stderr=b"") + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + sb.cmd_run(self._args(json=True)) + env = json.loads(buf.getvalue().strip()) + self.assertEqual(env["peak_mem_estimate"], 8388608) + + @mock.patch.object(sb, "_load_script", return_value=b"x") + @mock.patch.object(sb, "_read_peak_memory", return_value=None) + @mock.patch.object(sb.subprocess, "run") + def test_timeout_returns_124_and_marks_envelope( + self, m_run, _mp, _ml): + m_run.side_effect = subprocess.TimeoutExpired(cmd=["x"], timeout=5) + buf = io.StringIO() # --json mode uses print() which is text + with mock.patch.object(sys, "stdout", buf): + rc = sb.cmd_run(self._args(json=True)) + self.assertEqual(rc, 124) + env = json.loads(buf.getvalue().strip()) + self.assertTrue(env["timeout_hit"]) + self.assertEqual(env["exit_code"], 124) + # Lock the contract: cmd_run forwards the same argv to + # subprocess.run that callers can correlate with the + # constructed CompletedProcess in the timeout path. Production + # calls subprocess.run(ARGV_LIST, ...), so the ARGV_LIST is + # captured as the single first positional argument to run: + # call_args.args == ([ARGV_LIST],) — and ARGV_LIST[0] is then + # "neuros-container". + run_args, _ = m_run.call_args + self.assertEqual(len(run_args), 1, "subprocess.run got a single argv list") + self.assertEqual(run_args[0][0], "neuros-container") + + @mock.patch.object(sb, "_load_script", return_value=b"x") + @mock.patch.object(sb.subprocess, "run") + def test_dry_run_prints_argv_and_does_not_spawn( + self, m_run, _ml): + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = sb.cmd_run(self._args(dry_run=True)) + self.assertEqual(rc, 0) + self.assertIn("neuros-container", buf.getvalue()) + self.assertIn("--net", buf.getvalue()) + self.assertIn("--read-only", buf.getvalue()) + # No actual subprocess call was made + m_run.assert_not_called() + + @mock.patch.object(sb.subprocess, "run") + def test_bundle_extracts_and_rmtree_cleanup_is_called( + self, m_run): + """Verify the wrapper explicitly rmtree's the bundle tmpdir we + created (no implicit leak).""" + captured_roots = [] + real_rmtree = sb.shutil.rmtree + + def track_rmtree(path, *a, **kw): + captured_roots.append(path) + real_rmtree(path, *a, **kw) + + with tempfile.TemporaryDirectory() as tmp: + tar_path = os.path.join(tmp, "bundle.tar") + with tarfile.open(tar_path, "w") as tar: + data = b"echo hi\n" + info = tarfile.TarInfo("script.sh") + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + with mock.patch.object(sb, "_load_script", + return_value=b"echo hi"), \ + mock.patch.object(sb.shutil, "rmtree", + side_effect=track_rmtree): + cp = mock.Mock(returncode=0, stdout=b"out", stderr=b"") + m_run.return_value = cp + s_err = _FakeStdStream() + with mock.patch.object(sys, "stdout", _FakeStdStream()), \ + mock.patch.object(sys, "stderr", s_err): + rc = sb.cmd_run(self._args(bundle=tar_path)) + self.assertEqual(rc, 0) + # The wrapper created at least one tmpdir for the bundle, + # and rmtree was called on it. + self.assertTrue(captured_roots, "rmtree was never called") + self.assertTrue(any( + "neuros-sbx-bundle-" in p for p in captured_roots)) + + +class TestReadPeakMemory(unittest.TestCase): + """Best-effort memory.peak polling exercised with tmpdir fake files.""" + + def test_returns_int_when_file_exists(self): + with tempfile.TemporaryDirectory() as td: + # Create the cgroup-tree convention neuros-container uses + # by carving out the leaf dir directly under /sys/fs/cgroup. + leaf = os.path.join(td, "neuros-sbx-foo") + os.makedirs(leaf) + with open(os.path.join(leaf, "memory.peak"), "w") as f: + f.write("1234567\n") + with mock.patch.object(sb, "_CGROUP_V2_ROOTS", (td,)): + self.assertEqual(sb._read_peak_memory("foo"), 1234567) + + def test_returns_none_when_file_missing(self): + with tempfile.TemporaryDirectory() as td: + with mock.patch.object(sb, "_CGROUP_V2_ROOTS", (td,)): + self.assertIsNone(sb._read_peak_memory("missing")) + + def test_returns_none_on_unparseable_value(self): + with tempfile.TemporaryDirectory() as td: + leaf = os.path.join(td, "neuros-sbx-foo") + os.makedirs(leaf) + with open(os.path.join(leaf, "memory.peak"), "w") as f: + f.write("not-a-number\n") + with mock.patch.object(sb, "_CGROUP_V2_ROOTS", (td,)): + self.assertIsNone(sb._read_peak_memory("foo")) + + def test_mtime_tie_breaks_by_shorter_path(self): + """Two leaves with identical mtime: the shorter (closer-to-root) + path wins via the (mtime, -len(p)) sort key.""" + with tempfile.TemporaryDirectory() as td: + shallow = os.path.join(td, "neuros-sbx-foo") + deep = os.path.join(td, "slice", "neuros", "neuros-sbx-foo") + for d in (shallow, deep): + os.makedirs(d, exist_ok=True) + # Both files exist; force identical mtime. + fixed = 1_700_000_000.0 + for leaf in (shallow, deep): + with open(os.path.join(leaf, "memory.peak"), "w") as f: + f.write("42\n") + os.utime(leaf, (fixed, fixed)) + with mock.patch.object(sb, "_CGROUP_V2_ROOTS", (td,)): + # Shorter path should win, returning 42. + self.assertEqual(sb._read_peak_memory("foo"), 42) + + +class TestMainDispatch(unittest.TestCase): + def test_unknown_cmd_exits(self): + with self.assertRaises(SystemExit): + sb.main(["nope"]) + + def test_cpu_flag_translates_to_quota(self): + argv = ["run", "script.py", "--cpu", "25%", "--dry-run"] + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + rc = sb.main(argv) + self.assertEqual(rc, 0) + out = buf.getvalue() + # 25% of 100000 = 25000 + self.assertIn("--cpu-quota 25000", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_verify.py b/tests/test_verify.py new file mode 100755 index 0000000..a31b304 --- /dev/null +++ b/tests/test_verify.py @@ -0,0 +1,586 @@ +"""Tests for neuros-verify (consume-anything verifier). + +Runs purely against the production script via compile()+exec() so we +do not need it on $PATH. No subprocess shell-out is exercised here +on purpose: every interesting invariant (kind detection, per-kind +verification, aggregate merge, --json shape) is reachable from +Python directly. Sandbox+policy integration is tested by writing a +small on-disk policy manifest in a tempdir and pointing verify at +it via --policy. +""" +import compileall +import io +import json +import os +import subprocess +import sys +import tempfile +import types +import unittest +from unittest import mock + +VERIFY_PATH = "config/includes.chroot/usr/local/bin/neuros-verify" +POLICY_PATH = "config/includes.chroot/usr/local/bin/neuros-policy" + + +def _load_module(): + with open(VERIFY_PATH, "r", encoding="utf-8") as f: + src = f.read() + if src.startswith("#!"): + src = src.split("\n", 1)[1] + code = compile(src, VERIFY_PATH, "exec") + mod = types.ModuleType("neuros_verify_under_test") + # Set __file__ BEFORE exec so the production module's + # top-level references to __file__ resolve correctly + # (e.g. the policy script resolver's candidate list). + mod.__file__ = VERIFY_PATH + exec(code, mod.__dict__) + return mod + + +def _write_json(obj): + """Dump ``obj`` to a temp .json file and return its path string. + """ + fd, p = tempfile.mkstemp(prefix="neuros-verify-", suffix=".json") + with os.fdopen(fd, "w") as f: + json.dump(obj, f) + return p + + +def _good_policy(): + """A conformant policy manifest for sandbox+policy integration + tests. Same shape as test_policy._good_policy. + """ + return { + "name": "neuros-default", + "version": "1.0.0", + "defaults": { + "mem": "256M", + "pids": 64, + "cpu_quota": 50000, + "timeout": 30, + }, + "profiles": { + "strict": ["CAP_NET_RAW", "CAP_SYS_ADMIN"], + "moderate": ["CAP_NET_RAW"], + "permissive": [], + }, + "net": "private", + "readonly": True, + "env_allowlist": ["PATH"], + "syscalls": None, + } + + +def _write_policy(): + return _write_json(_good_policy()) + + +def _ok_envelope(**overrides): + base = { + "exit_code": 0, "stdout": "", "stderr": "", + "wall_clock_ms": 1000, "timeout_hit": False, + "peak_mem_estimate": 64 * 1024 * 1024, + } + base.update(overrides) + return base + + +def _bench_metrics(runs=None): + return { + "runs": runs if runs is not None else [ + { + "name": "cpu_tight", "wall_clock_ms": 1100, + "exit_code": 0, "stdout": "", "stderr": "", + "timeout_hit": False, + "peak_mem_estimate": 32 * 1024 * 1024, + }, + { + "name": "json_parse", "wall_clock_ms": 900, + "exit_code": 0, "stdout": "", "stderr": "", + "timeout_hit": False, + "peak_mem_estimate": 18 * 1024 * 1024, + }, + ], + "total_wall_clock_ms": 2000, + "config": {}, + "neuros_bench_version": "0.1", + } + + +class TestCompile(unittest.TestCase): + def test_compiles_clean(self): + self.assertTrue(compileall.compile_file(VERIFY_PATH, + quiet=1, + force=True)) + + +class TestDetectKind(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_bench_by_version_key(self): + self.assertEqual(self.vf._detect_kind(_bench_metrics()), + "bench") + + def test_policy_by_version_key(self): + env = {"ok": True, "errors": [], "name": "x", + "version": "1.0.0", "neuros_policy_version": "0.1"} + self.assertEqual(self.vf._detect_kind(env), "policy") + + def test_replay_by_version_key(self): + env = {"ok": True, "neuros_replay_version": "0.1", + "rule_violations": []} + self.assertEqual(self.vf._detect_kind(env), "replay") + + def test_runbook_by_structure(self): + env = {"runbook_path": "/tmp/rb.json", + "steps": [{"name": "x", "ok": True}]} + self.assertEqual(self.vf._detect_kind(env), "runbook") + + def test_sandbox_by_structure(self): + self.assertEqual(self.vf._detect_kind(_ok_envelope()), + "sandbox") + + def test_runbook_priority_over_sandbox(self): + # If both structural patterns match, runbook wins because + # it's checked first (has the more specific runbook_path). + env = {"runbook_path": "/x", "steps": [], + "exit_code": 0, "wall_clock_ms": 0, + "timeout_hit": False} + self.assertEqual(self.vf._detect_kind(env), "runbook") + + def test_unknown(self): + self.assertEqual(self.vf._detect_kind({"foo": "bar"}), + "unknown") + + +class TestLoadEnvelope(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_missing_file(self): + with self.assertRaises(self.vf.VerifyError) as cm: + self.vf._load_envelope("/nonexistent/env.json") + self.assertIn("cannot read", str(cm.exception)) + + def test_malformed_json(self): + fd, p = tempfile.mkstemp(prefix="nv-bad-", suffix=".json") + try: + with os.fdopen(fd, "w") as f: + f.write("not json {{{") + with self.assertRaises(self.vf.VerifyError) as cm: + self.vf._load_envelope(p) + self.assertIn("not valid JSON", str(cm.exception)) + finally: + os.unlink(p) + + def test_root_must_be_object(self): + path = _write_json([1, 2, 3]) + try: + with self.assertRaises(self.vf.VerifyError): + self.vf._load_envelope(path) + finally: + os.unlink(path) + + +class TestPctDelta(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_normal(self): + self.assertEqual(self.vf._pct_delta(1000, 1100), 10.0) + self.assertEqual(self.vf._pct_delta(1000, 900), -10.0) + + def test_zero_div(self): + self.assertIsNone(self.vf._pct_delta(0, 100)) + + def test_missing(self): + self.assertIsNone(self.vf._pct_delta(None, 100)) + self.assertIsNone(self.vf._pct_delta(100, None)) + + +class TestVerifyPolicyVerdict(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_ok_true(self): + ok, viols = self.vf._verify_policy_verdict( + {"ok": True, "errors": []}) + self.assertTrue(ok) + self.assertEqual(viols, []) + + def test_ok_false_with_errors(self): + ok, viols = self.vf._verify_policy_verdict( + {"ok": False, "errors": ["bad mem"]}) + self.assertFalse(ok) + self.assertIn("bad mem", viols) + + def test_ok_false_without_errors(self): + ok, viols = self.vf._verify_policy_verdict({"ok": False}) + self.assertFalse(ok) + self.assertEqual(len(viols), 1) + + def test_ok_other_type(self): + ok, viols = self.vf._verify_policy_verdict({"ok": "yes"}) + self.assertFalse(ok) + + +class TestVerifyReplayVerdict(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_ok_true(self): + ok, viols = self.vf._verify_replay_verdict( + {"ok": True, "rule_violations": []}) + self.assertTrue(ok) + + def test_ok_false_with_rule_violations(self): + ok, viols = self.vf._verify_replay_verdict({ + "ok": False, + "rule_violations": ["wall_clock_pct_delta 50%"]}) + self.assertFalse(ok) + self.assertEqual(viols, + ["wall_clock_pct_delta 50%"]) + + +class TestVerifyBench(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_no_runs(self): + ok, _ = self.vf._verify_bench({"runs": []}, None) + self.assertTrue(ok) + + def test_runs_not_a_list(self): + ok, viols = self.vf._verify_bench({"runs": "nope"}, None) + self.assertFalse(ok) + + def test_runs_with_one_nonzero_exit(self): + ok, viols = self.vf._verify_bench( + _bench_metrics([{"name": "x", "exit_code": 1, + "wall_clock_ms": 0}]), + None) + self.assertFalse(ok) + self.assertTrue(any("exit_code=1" in v for v in viols)) + + def test_with_baseline_missing_file(self): + ok, viols = self.vf._verify_bench( + _bench_metrics(), "/nonexistent/baseline.json") + self.assertFalse(ok) + + def test_with_baseline_wall_clock_regression(self): + base = _bench_metrics() + cand = _bench_metrics() + cand["runs"][0]["wall_clock_ms"] = 5000 # +354% + b = _write_json(base) + try: + ok, viols = self.vf._verify_bench(cand, b) + self.assertFalse(ok) + self.assertTrue(any("wall_clock_pct_delta" in v + for v in viols)) + finally: + os.unlink(b) + + def test_with_baseline_conformant(self): + b = _write_json(_bench_metrics()) + try: + ok, viols = self.vf._verify_bench(_bench_metrics(), b) + self.assertTrue(ok) + self.assertEqual(viols, []) + finally: + os.unlink(b) + + def test_with_baseline_new_run_in_candidate_not_violation(self): + base = _bench_metrics(runs=[{ + "name": "cpu_tight", "wall_clock_ms": 1100, + "exit_code": 0, "timeout_hit": False, + "peak_mem_estimate": 32 * 1024 * 1024}]) + cand = _bench_metrics() + # Candidate has an extra run (json_parse) not in baseline. + b = _write_json(base) + try: + ok, viols = self.vf._verify_bench(cand, b) + # The existing cpu_tight run is conformant and the new + # json_parse run is not a regression (no baseline entry). + self.assertTrue(ok) + finally: + os.unlink(b) + + +class TestVerifyRunbook(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_conformant(self): + env = {"runbook_path": "/tmp/rb.json", + "steps": [{"name": "a", "ok": True}, + {"name": "b", "ok": True}]} + ok, viols = self.vf._verify_runbook(env) + self.assertTrue(ok) + + def test_missing_path(self): + ok, viols = self.vf._verify_runbook({"steps": []}) + self.assertFalse(ok) + + def test_step_not_object(self): + env = {"runbook_path": "/x", + "steps": ["stringy", {"name": "y", "ok": True}]} + ok, viols = self.vf._verify_runbook(env) + self.assertFalse(ok) + self.assertTrue(any("not an object" in v for v in viols)) + + def test_step_missing_name(self): + env = {"runbook_path": "/x", + "steps": [{"ok": True}]} + ok, viols = self.vf._verify_runbook(env) + self.assertFalse(ok) + + +class TestVerifySandbox(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_conformant_envelope(self): + # Load the policy MODULE (Python source) and pass a + # separate MANIFEST path to _verify_sandbox. The SCRIPT + # is reusable across manifests; the manifest is per-run. + with open(POLICY_PATH, "r") as f: + psrc = f.read() + if psrc.startswith("#!"): + psrc = psrc.split("\n", 1)[1] + pcode = compile(psrc, POLICY_PATH, "exec") + pmod = types.ModuleType("policy_loader") + pmod.__file__ = POLICY_PATH + exec(pcode, pmod.__dict__) + + manifest_path = _write_policy() + try: + ok, viols = self.vf._verify_sandbox( + _ok_envelope(), pmod, manifest_path) + self.assertTrue(ok) + finally: + os.unlink(manifest_path) + + def test_violation_envelope(self): + with open(POLICY_PATH, "r") as f: + psrc = f.read() + if psrc.startswith("#!"): + psrc = psrc.split("\n", 1)[1] + pcode = compile(psrc, POLICY_PATH, "exec") + pmod = types.ModuleType("policy_loader") + pmod.__file__ = POLICY_PATH + exec(pcode, pmod.__dict__) + + manifest_path = _write_policy() + try: + # timeout_hit=True is always a violation. + ok, viols = self.vf._verify_sandbox( + _ok_envelope(timeout_hit=True), + pmod, manifest_path) + self.assertFalse(ok) + self.assertTrue(any("timeout_hit" in v + for v in viols)) + finally: + os.unlink(manifest_path) + + +class TestVerifyUnknown(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_unrecognized_envelope(self): + ok, viols = self.vf._verify_unknown({"foo": "bar"}) + self.assertFalse(ok) + self.assertTrue(any("unrecognized" in v for v in viols)) + + +class TestMainDispatch(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + def test_text_line_shape(self): + path = _write_json({"ok": True, "errors": [], + "neuros_policy_version": "0.1"}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.vf.main([path]) + self.assertEqual(rc, 0) + out = buf.getvalue() + self.assertIn("verdict=PASS", out) + self.assertIn("source=policy", out) + self.assertIn("violations=0", out) + self.assertIn("source=aggregate", out) + finally: + os.unlink(path) + + def test_text_line_for_sandbox_without_policy_fails(self): + path = _write_json(_ok_envelope()) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.vf.main([path]) + self.assertEqual(rc, 1) + self.assertIn("verdict=FAIL", buf.getvalue()) + self.assertIn("source=sandbox", buf.getvalue()) + self.assertIn("violations=1", buf.getvalue()) + finally: + os.unlink(path) + + def test_text_line_for_sandbox_with_policy_passes(self): + path = _write_json(_ok_envelope()) + policy_path = _write_policy() + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.vf.main([path, "--policy", policy_path]) + self.assertEqual(rc, 0) + self.assertIn("verdict=PASS", buf.getvalue()) + self.assertIn("source=sandbox", buf.getvalue()) + finally: + os.unlink(path) + os.unlink(policy_path) + + def test_unknown_envelope_verdict_fail(self): + path = _write_json({"foo": 1}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.vf.main([path]) + self.assertEqual(rc, 1) + self.assertIn("verdict=FAIL", buf.getvalue()) + self.assertIn("source=unknown", buf.getvalue()) + finally: + os.unlink(path) + + def test_json_envelope_shape(self): + path = _write_json({"ok": True, "errors": [], + "neuros_policy_version": "0.1"}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.vf.main([path, "--json"]) + self.assertEqual(rc, 0) + obj = json.loads(buf.getvalue().strip()) + self.assertTrue(obj["ok"]) + self.assertEqual(obj["aggregate"]["passes"], 1) + self.assertEqual(obj["aggregate"]["failures"], 0) + self.assertEqual(len(obj["parts"]), 1) + self.assertEqual(obj["parts"][0]["source_kind"], "policy") + self.assertIn("neuros_verify_version", obj) + finally: + os.unlink(path) + + def test_quiet_suppresses_per_file_lines(self): + path = _write_json({"ok": True, "errors": [], + "neuros_policy_version": "0.1"}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.vf.main([path, "--quiet"]) + self.assertEqual(rc, 0) + out = buf.getvalue() + # No per-file 'source=policy' line. + self.assertNotIn("source=policy", out) + self.assertIn("source=aggregate", out) + finally: + os.unlink(path) + + def test_multi_file_and_merge(self): + good = _write_json({"ok": True, "errors": [], + "neuros_policy_version": "0.1"}) + bad = _write_json({"ok": False, "errors": ["x"], + "neuros_policy_version": "0.1"}) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", buf): + rc = self.vf.main([good, bad]) + # One file passes, one fails → overall FAIL. + self.assertEqual(rc, 1) + out = buf.getvalue() + self.assertIn("passes=1", out) + self.assertIn("failures=1", out) + # The aggregate (last) line must report the overall FAIL + # verdict, not just one of the per-file lines. + last_line = out.strip().splitlines()[-1] + self.assertTrue(last_line.startswith("verdict=FAIL ")) + self.assertIn("source=aggregate", last_line) + finally: + os.unlink(good); os.unlink(bad) + + def test_missing_file_returns_two(self): + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf), \ + mock.patch.object(sys, "stderr", io.StringIO()): + with self.assertRaises(SystemExit): + self.vf.main(["/nonexistent/env.json"]) + + def test_bench_with_baseline(self): + base = _bench_metrics() + cand = _bench_metrics() + cand_path = _write_json(cand) + base_path = _write_json(base) + try: + buf = io.StringIO() + with mock.patch.object(sys, "stdout", buf): + rc = self.vf.main([cand_path, + "--bench-baseline", base_path]) + self.assertEqual(rc, 0) + self.assertIn("source=bench", buf.getvalue()) + self.assertIn("violations=0", buf.getvalue()) + finally: + os.unlink(cand_path); os.unlink(base_path) + + +class TestResolvePolicyScript(unittest.TestCase): + """Defensive: the resolver raises VerifyError with a clear + message when NO candidate path exists. In a dev checkout the + sibling-of-neuros-verify candidate is a real file, so the env + override alone isn't enough to force a miss -- os.path.isfile + is patched to make every candidate look absent. + """ + + @classmethod + def setUpClass(cls): + cls.vf = _load_module() + + @classmethod + def tearDownClass(cls): + # Clear the module-level cache so subsequent test classes + # re-resolve with the live env (the cache is shared + # singleton state that we don't want to leak across + # test classes). + cls.vf._CACHED_POLICY_SCRIPT_MOD = None + cls.vf._CACHED_POLICY_SCRIPT_PATH = None + + def test_missing_script_raises(self): + with mock.patch.dict(os.environ, + {"NEUROS_POLICY_SCRIPT": + "/nonexistent/script"}, + clear=False), \ + mock.patch("os.path.isfile", return_value=False): + self.vf._CACHED_POLICY_SCRIPT_MOD = None + self.vf._CACHED_POLICY_SCRIPT_PATH = None + with self.assertRaises(self.vf.VerifyError) as cm: + self.vf._resolve_policy_script() + msg = str(cm.exception) + self.assertIn("cannot locate", msg) + self.assertIn("/nonexistent/script", msg) + + +if __name__ == "__main__": + unittest.main()