From 057f85fe89c704ddc38d6452dcc5ee27c6ada404 Mon Sep 17 00:00:00 2001 From: "secopsai-blog-ops[bot]" <115749095+Techris93@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:03:15 +0300 Subject: [PATCH] feat: add guarded agent research completion --- README.md | 10 ++ docs/implementation-checkpoints.md | 16 +++ docs/research-automation.md | 45 +++++++-- secopsai/cli.py | 26 ++++- secopsai/codex_bridge.py | 12 +++ secopsai/codex_bridge_service.py | 16 ++- secopsai/intelligence.py | 22 ++++- secopsai/research_pipeline.py | 153 ++++++++++++++++++++++++++++- tests/test_intelligence.py | 14 +++ tests/test_research_pipeline.py | 115 ++++++++++++++++++++++ 10 files changed, 415 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 1e45570..6a131c6 100644 --- a/README.md +++ b/README.md @@ -427,6 +427,16 @@ Higher-risk actions stay queued for analyst approval and application: - threshold tuning - escalation closures such as `needs_review` or `tune_policy` +## Guarded Research Autonomy + +The Local Codex/OpenCodex bridge can complete repeatable research review work with the selected Kimi, Grok, Gemini, or Codex model. In `agent_review` mode it accepts bounded pipeline proposals, records an evidence-linked verdict, and reruns publication safety automatically: + +```bash +secopsai intelligence bridge service install --autonomy-mode agent_review +``` + +Core guardrails prevent local dependency absence from becoming a benign verdict, constrain verdict evidence to the active pipeline, and reduce conclusions that lack sufficient confidence or proof. Package execution, external sandbox submission, disclosure delivery, customer-control changes, and final publication remain human-approved actions. See [Research Automation](docs/research-automation.md). + ## Background Monitoring Example operational model: diff --git a/docs/implementation-checkpoints.md b/docs/implementation-checkpoints.md index 8d5564b..a83b1d6 100644 --- a/docs/implementation-checkpoints.md +++ b/docs/implementation-checkpoints.md @@ -1,5 +1,21 @@ # Implementation Checkpoints +## 108 Guarded Agent Research Completion + +Status: implementation complete; full verification and Mission Control release are pending. + +Added an optional `agent_review` mode for the Local Codex/OpenCodex bridge. When all bounded static-analysis jobs finish, Core accepts pipeline-scoped proposals, records an evidence-linked verdict, and reruns publication safety without requiring card-by-card operator acceptance. Verdict output is schema constrained and then independently guarded by Core: low-confidence work remains inconclusive, a credible verdict requires advisory-backed or sandbox evidence, unresolved contradictions and unsupported claims reduce certainty, and absence from a local repository can never establish package benignness or make ecosystem intelligence non-actionable. + +The model receives normalized evidence only. Verdict evidence is limited to active records created by the same investigation pipeline. Every decision records the actor and pipeline revision, and repeated completion is idempotent. The automation cannot execute packages, submit artifacts to an external sandbox, send disclosure, change customer controls, approve publication, or publish content. + +Operator surfaces: + +- `secopsai research pipeline agent-complete RPL-...` +- `secopsai intelligence bridge service install --autonomy-mode agent_review` +- Mission Control **Complete Agent Review** + +The remaining human gates are external sandbox submission, external disclosure delivery, and final publication approval. + ## 107 Hermes Agent v1.0.0 Integration Status: complete in production. diff --git a/docs/research-automation.md b/docs/research-automation.md index 4a07c4c..b72c983 100644 --- a/docs/research-automation.md +++ b/docs/research-automation.md @@ -12,13 +12,44 @@ The normal workflow no longer requires an evidence-bundle export, a file upload, 4. Core collects official-registry metadata and the package artifact using bounded static-intake controls. It records hashes, inspects the archive without execution, and performs a deterministic comparison when a trusted reference was supplied. 5. Core queues three durable Intelligence jobs. The installed Local Codex Bridge reads only minimized case and static-analysis context and writes structured proposals back to the same pipeline record. 6. Mission Control refreshes the running pipeline automatically. -7. Review every proposed fact, inference, unsupported claim, contradiction, missing-evidence group, recommended-action group, disclosure draft, and publication-risk group. Related model list items are deduplicated and grouped into one editable review card so the queue remains usable. Click **Accept** or **Reject**. -8. Accepted static evidence is attached with its pipeline and review provenance. Accepted model text becomes an immutable analyst-reviewed case note. Rejected proposals remain auditable and do not change canonical evidence. -9. Record the human verdict, approve any sandbox request, send disclosure, and approve publication through their existing separate gates. +7. In **agent review** mode, SecOpsAI accepts the bounded evidence proposals, records an evidence-linked agent verdict, and reruns publication safety automatically. The verdict is constrained by deterministic guardrails: local absence cannot prove benignness, `credible` requires advisory-backed or sandbox evidence, and low-confidence work remains `inconclusive`. +8. In **supervised** mode, review each proposal manually with **Accept** or **Reject**, then record the verdict yourself. +9. Review the resulting case. External sandbox submission, external disclosure delivery, and final publication approval remain separate human gates. If the bridge or collection step fails, click **Retry from checkpoint**. A new pipeline revision is created, stale proposals are superseded, and the previous revision remains auditable. If comparison was incomplete, enter a verified reference and click **Add reference and rerun analysis**. -The pipeline cannot classify a package as malicious, execute it, submit it to a sandbox, send external communication, approve publication, or publish an article. +The pipeline never executes package code, submits an artifact to an external sandbox, sends external communication, approves publication, or publishes an article. In agent-review mode it may record a bounded case verdict, but only against accepted, pipeline-specific evidence and with all guardrail decisions retained in the audit trail. + +## Agent-review mode + +Agent-review mode is the recommended high-automation setting for a local research workstation. It delegates the repeatable evidence review, verdict recommendation, analyst brief, and publication preflight to the selected Local Codex/OpenCodex model. Models remain fallible; SecOpsAI therefore validates their schema, evidence links, confidence, contradictions, and local-exposure reasoning before writing a verdict. + +Install or update the background bridge in agent-review mode: + +```bash +secopsai intelligence bridge service install --autonomy-mode agent_review +``` + +Complete an already-waiting pipeline from the CLI: + +```bash +secopsai research pipeline agent-complete RPL-XXXXXXXXXXXXXXXX +``` + +Mission Control exposes the same operation as **Complete Agent Review**. The action: + +- accepts bounded static evidence and model proposals; +- records a verdict with confidence, rationale, evidence identifiers, model provider, and pipeline revision; +- blocks benign or not-substantiated conclusions based only on missing local exposure; +- downgrades unsupported or contradictory claims; +- reruns publication safety; +- does not upload an artifact, contact a third party, or publish content. + +Use supervised mode when policy requires proposal-by-proposal human acceptance: + +```bash +secopsai intelligence bridge service install --autonomy-mode supervised +``` ## Granular recovery controls @@ -29,7 +60,7 @@ Use the individual actions only when diagnosing a step or deliberately running a 3. Use **Collect Metadata Preview** to confirm the official registry target. 4. Use **Run Safe Package Intake**. The artifact is fetched into local quarantine, hashed, and inspected in memory. No package manager or build command runs. 5. Review the job result and indicators. Use **Attach Verified Evidence** only after reviewing the preview. -6. Use **Generate Evidence Matrix** and record a human verdict with rationale and evidence IDs. +6. Use **Generate Evidence Matrix** and either click **Complete Agent Review** or record a human verdict with rationale and evidence IDs. 7. Use **Run Publication Safety Check** before drafting public content. 8. Use **Prepare Disclosure** to create a reviewable maintainer/registry message. Approval and sending are separate actions. 9. Use **Request Sandbox Approval** only when static evidence leaves an important runtime question. The default provider is manual-result-import; Core never executes packages locally. @@ -50,7 +81,7 @@ secopsai research workflow evidence-matrix RSC-XXXXXXXXXXXX secopsai research workflow publication-check RSC-XXXXXXXXXXXX ``` -Human-gated actions are explicit: +Human-gated external actions are explicit: ```bash secopsai research workflow verdict RSC-XXXXXXXXXXXX --verdict likely --confidence 70 --rationale "Explain the evidence and limitations." --evidence-id EVD-XXXXXXXXXXXX @@ -63,4 +94,4 @@ secopsai research workflow publication-approve RSC-XXXXXXXXXXXX --review-id PUB- Artifacts are bounded by size, redirect, archive-entry, expanded-content, and inspected-text limits. Absolute paths, traversal, links, devices, archive bombs, private-address resolutions, credential-bearing URLs, and unapproved redirects are rejected. Static indicators are evidence leads, not proof of maliciousness. Raw artifacts and raw registry responses are not sent to AI. -Dynamic analysis is not enabled by pretending the Core host is a sandbox. Configure a dedicated isolated provider or import a sanitized result manually. Disclosure sending and publication approval remain auditable human decisions. +Dynamic analysis is not enabled by pretending the Core host is a sandbox. Configure a dedicated isolated provider or import a sanitized result manually. External sandbox submission, disclosure sending, and publication approval remain auditable human decisions. A model may prepare the justification or draft, but it cannot perform the external action. diff --git a/secopsai/cli.py b/secopsai/cli.py index 2bfa4af..a2ed90b 100644 --- a/secopsai/cli.py +++ b/secopsai/cli.py @@ -177,6 +177,7 @@ set_sandbox_status, ) from secopsai.research_pipeline import ( + agent_complete_pipeline, get_pipeline as get_research_pipeline, list_pipelines as list_research_pipelines, resume_investigation_pipeline, @@ -1198,6 +1199,12 @@ def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: intelligence_bridge_service.add_argument("--db-path", default=None) intelligence_bridge_service.add_argument("--no-start", action="store_true") intelligence_bridge_service.add_argument("--tail", type=int, default=80) + intelligence_bridge_service.add_argument( + "--autonomy-mode", + choices=["supervised", "agent_review"], + default="supervised", + help="Automatically complete bounded research review when set to agent_review", + ) research = sub.add_parser("research", help="Generate source-backed research reports and preflight checks") research_sub = research.add_subparsers(dest="research_cmd", required=True) @@ -1615,6 +1622,13 @@ def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: pipeline_auto.add_argument("pipeline_id") pipeline_auto.add_argument("--actor", default="analyst") pipeline_auto.add_argument("--db-path", default=None) + pipeline_agent = research_pipeline_sub.add_parser( + "agent-complete", + help="Accept bounded proposals and record an evidence-linked agent verdict without external actions", + ) + pipeline_agent.add_argument("pipeline_id") + pipeline_agent.add_argument("--actor", default="secopsai-agent-autonomy") + pipeline_agent.add_argument("--db-path", default=None) workflow = research_sub.add_parser("workflow", help="Evidence, verdict, disclosure, publication, and sandbox gates") workflow_sub = workflow.add_subparsers(dest="research_workflow_cmd", required=True) @@ -2354,6 +2368,12 @@ def _run_research_automation_command(args: argparse.Namespace) -> int: actor=args.actor, db_path=args.db_path, ) + elif args.research_pipeline_cmd == "agent-complete": + payload = agent_complete_pipeline( + args.pipeline_id, + actor=args.actor, + db_path=args.db_path, + ) else: payload = review_pipeline_item( args.pipeline_id, @@ -3234,7 +3254,11 @@ def main(argv: Optional[List[str]] = None) -> int: ) elif args.intelligence_bridge_cmd == "service": if args.action == "install": - payload = install_codex_bridge_service(db_path=args.db_path, start=not args.no_start) + payload = install_codex_bridge_service( + db_path=args.db_path, + start=not args.no_start, + autonomy_mode=args.autonomy_mode, + ) else: payload = codex_bridge_service_action(args.action, tail=args.tail) else: diff --git a/secopsai/codex_bridge.py b/secopsai/codex_bridge.py index 6a98e36..7afdf92 100644 --- a/secopsai/codex_bridge.py +++ b/secopsai/codex_bridge.py @@ -714,6 +714,7 @@ def _normalize_bridge_result(payload: dict[str, Any]) -> dict[str, Any]: "evidence", "recommended_actions", "limitations", + "verdict_evidence_refs", ) for field_name in list_fields: values = result.get(field_name) @@ -747,6 +748,17 @@ def _normalize_bridge_result(payload: dict[str, Any]) -> dict[str, Any]: if not _present(result.get("limitations")): missing = result.get("missing_evidence") result["limitations"] = list(missing[:25]) if isinstance(missing, list) else [] + recommendation = str(result.get("verdict_recommendation") or "").strip().lower() + if recommendation not in {"credible", "likely", "inconclusive", "not_substantiated", "benign"}: + result["verdict_recommendation"] = "inconclusive" + try: + result["verdict_confidence"] = max(0, min(int(result.get("verdict_confidence") or 0), 100)) + except (TypeError, ValueError): + result["verdict_confidence"] = 0 + if not _present(result.get("verdict_rationale")): + result["verdict_rationale"] = "The model did not provide a bounded package verdict rationale." + if not isinstance(result.get("verdict_evidence_refs"), list): + result["verdict_evidence_refs"] = [] return result diff --git a/secopsai/codex_bridge_service.py b/secopsai/codex_bridge_service.py index b21b3ca..9b56208 100644 --- a/secopsai/codex_bridge_service.py +++ b/secopsai/codex_bridge_service.py @@ -24,14 +24,18 @@ def install_service( home: Path | None = None, platform_name: str | None = None, runner: RunCommand | None = None, + autonomy_mode: str = "supervised", ) -> dict[str, Any]: resolved_home = (home or Path.home()).expanduser().resolve() system = (platform_name or platform.system()).lower() run = runner or _run + autonomy_mode = str(autonomy_mode or "supervised").strip().lower() + if autonomy_mode not in {"supervised", "agent_review"}: + raise ValueError("autonomy mode must be supervised or agent_review") if system == "darwin": - result = _install_launchd(resolved_home, db_path, run, start) + result = _install_launchd(resolved_home, db_path, run, start, autonomy_mode) elif system == "linux": - result = _install_systemd(resolved_home, db_path, run, start) + result = _install_systemd(resolved_home, db_path, run, start, autonomy_mode) else: raise ValueError("automatic Codex bridge service installation supports macOS and Linux") return {"status": "installed", "service": LABEL, **result} @@ -60,7 +64,7 @@ def service_action( raise ValueError("Codex bridge service controls support macOS and Linux") -def _install_launchd(home: Path, db_path: str | None, run: RunCommand, start: bool) -> dict[str, Any]: +def _install_launchd(home: Path, db_path: str | None, run: RunCommand, start: bool, autonomy_mode: str) -> dict[str, Any]: launch_agents = home / "Library" / "LaunchAgents" logs = home / "Library" / "Logs" / "SecOpsAI" launch_agents.mkdir(parents=True, exist_ok=True) @@ -80,6 +84,7 @@ def _install_launchd(home: Path, db_path: str | None, run: RunCommand, start: bo environment = { "HOME": str(home), "PATH": os.environ.get("PATH", "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin"), + "SECOPSAI_RESEARCH_AUTONOMY_MODE": autonomy_mode, } if os.environ.get("CODEX_HOME"): environment["CODEX_HOME"] = os.environ["CODEX_HOME"] @@ -109,10 +114,11 @@ def _install_launchd(home: Path, db_path: str | None, run: RunCommand, start: bo "started": start, "logs": [str(logs / "codex-bridge.out.log"), str(logs / "codex-bridge.err.log")], "credentials_persisted": False, + "autonomy_mode": autonomy_mode, } -def _install_systemd(home: Path, db_path: str | None, run: RunCommand, start: bool) -> dict[str, Any]: +def _install_systemd(home: Path, db_path: str | None, run: RunCommand, start: bool, autonomy_mode: str) -> dict[str, Any]: unit_dir = home / ".config" / "systemd" / "user" logs = home / ".local" / "state" / "secopsai" unit_dir.mkdir(parents=True, exist_ok=True) @@ -138,6 +144,7 @@ def _install_systemd(home: Path, db_path: str | None, run: RunCommand, start: bo "Type=simple", f"WorkingDirectory={working_directory}", f"ExecStart={shlex.join(command)}", + f"Environment=SECOPSAI_RESEARCH_AUTONOMY_MODE={autonomy_mode}", "Restart=on-failure", "RestartSec=15", "NoNewPrivileges=true", @@ -160,6 +167,7 @@ def _install_systemd(home: Path, db_path: str | None, run: RunCommand, start: bo "started": start, "logs": [f"journalctl --user -u {SYSTEMD_UNIT}"], "credentials_persisted": False, + "autonomy_mode": autonomy_mode, } diff --git a/secopsai/intelligence.py b/secopsai/intelligence.py index 0fc94c1..08fd757 100644 --- a/secopsai/intelligence.py +++ b/secopsai/intelligence.py @@ -268,6 +268,9 @@ def _bridge_instructions(action: Action) -> str: "analyze_research_case": ( "Also return confirmed_facts, inferences, unsupported_claims, contradictions, and missing_evidence as arrays. " "A confirmed fact must cite supplied normalized evidence; otherwise classify it as an inference or unsupported claim. " + "Return verdict_recommendation as credible, likely, inconclusive, not_substantiated, or benign; " + "verdict_confidence as 0-100; verdict_rationale; and verdict_evidence_refs using only supplied evidence or pipeline step identifiers. " + "Never downgrade a package merely because local exposure was not observed. " ), "generate_analyst_brief": ( "Also return an article_outline array. Keep it suitable for a technical draft, not publication-ready copy. " @@ -280,7 +283,9 @@ def _bridge_instructions(action: Action) -> str: f"Perform the approved SecOpsAI action '{action.name}'. Use only the supplied normalized context. " "Do not claim that missing evidence was observed. Distinguish facts, inferences, and limitations. " "Do not execute commands, access files, browse, contact external parties, change product state, or approve publication. " - f"{action_guidance}Return concise JSON matching the required output schema. Human review is mandatory." + f"{action_guidance}For actions other than analyze_research_case, set verdict_recommendation to inconclusive, " + "verdict_confidence to 0, verdict_rationale to 'Verdict not assessed by this action', and verdict_evidence_refs to an empty array. " + "Return concise JSON matching the required output schema. Model output is evidence-bounded and subject to SecOpsAI guardrails." ) @@ -299,6 +304,10 @@ def bridge_output_schema() -> dict[str, Any]: "publication_risks", "article_outline", "disclosure_draft", + "verdict_recommendation", + "verdict_confidence", + "verdict_rationale", + "verdict_evidence_refs", ] return { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -319,6 +328,17 @@ def bridge_output_schema() -> dict[str, Any]: "publication_risks": {"type": "array", "maxItems": 50, "items": {"type": "string", "maxLength": 2000}}, "article_outline": {"type": "array", "maxItems": 30, "items": {"type": "string", "maxLength": 2000}}, "disclosure_draft": {"type": "string", "maxLength": 12000}, + "verdict_recommendation": { + "type": "string", + "enum": ["credible", "likely", "inconclusive", "not_substantiated", "benign"], + }, + "verdict_confidence": {"type": "integer", "minimum": 0, "maximum": 100}, + "verdict_rationale": {"type": "string", "maxLength": 8000}, + "verdict_evidence_refs": { + "type": "array", + "maxItems": 50, + "items": {"type": "string", "maxLength": 200}, + }, }, } diff --git a/secopsai/research_pipeline.py b/secopsai/research_pipeline.py index 05ada0a..8d0b1b0 100644 --- a/secopsai/research_pipeline.py +++ b/secopsai/research_pipeline.py @@ -4,6 +4,7 @@ import hashlib import json +import os import secrets from contextlib import closing from typing import Any, Dict, Iterable, Optional @@ -13,7 +14,11 @@ from secopsai.research_analysis import compare_intakes from secopsai.research_cases import add_evidence, get_case from secopsai.research_intake import attach_intake_result, collect_package_intake -from secopsai.research_workflow import build_evidence_matrix, publication_safety_check +from secopsai.research_workflow import ( + build_evidence_matrix, + publication_safety_check, + record_verdict, +) SCHEMA_VERSION = "secopsai.research.investigation-pipeline.v1" @@ -685,6 +690,12 @@ def reconcile_intelligence_job(job: Dict[str, Any], *, db_path: Optional[str] = {"pipeline_id": pipeline_id, "pending_review_items": summary["pending_review_items"]}, ) connection.commit() + if str(os.environ.get("SECOPSAI_RESEARCH_AUTONOMY_MODE") or "").strip().lower() == "agent_review": + return agent_complete_pipeline( + pipeline_id, + actor="secopsai-agent-autonomy", + db_path=db_path, + ) return get_pipeline(pipeline_id, db_path=db_path) @@ -795,6 +806,146 @@ def auto_review_pipeline( return get_pipeline(pipeline_id, db_path=db_path) +def agent_complete_pipeline( + pipeline_id: str, + *, + actor: str = "secopsai-agent-autonomy", + db_path: Optional[str] = None, +) -> Dict[str, Any]: + """Complete safe review work and record a bounded, evidence-linked agent verdict.""" + pipeline = get_pipeline(pipeline_id, db_path=db_path) + if pipeline["status"] == "awaiting_review": + pipeline = auto_review_pipeline(pipeline_id, actor=actor, db_path=db_path) + elif pipeline["status"] != "succeeded": + raise ValueError("agent completion requires a pipeline awaiting review or already succeeded") + + case = get_case(pipeline["case_id"], db_path=db_path) + actor_id = f"{actor}:{pipeline_id}"[:160] + existing = next( + (item for item in case.get("verdicts") or [] if str(item.get("actor") or "") == actor_id), + None, + ) + verdict_recorded = bool(existing) + verdict = str(existing.get("verdict") or "") if existing else "" + confidence = int(existing.get("confidence") or 0) if existing else 0 + + if not existing: + analyze = next( + (step for step in pipeline["steps"] if step["step_key"] == "analyze_research_case"), + {}, + ) + envelope = analyze.get("result") or {} + data = envelope.get("data") if isinstance(envelope.get("data"), dict) else envelope + requested = str(data.get("verdict_recommendation") or "inconclusive").strip().lower() + if requested not in {"credible", "likely", "inconclusive", "not_substantiated", "benign"}: + requested = "inconclusive" + try: + confidence = max(0, min(int(data.get("verdict_confidence") or 0), 100)) + except (TypeError, ValueError): + confidence = 0 + rationale = _clean(data.get("verdict_rationale"), 8000) + contradictions = data.get("contradictions") if isinstance(data.get("contradictions"), list) else [] + unsupported = data.get("unsupported_claims") if isinstance(data.get("unsupported_claims"), list) else [] + decision_context = " ".join( + _clean(value, 2000) + for value in ( + rationale, + data.get("summary"), + data.get("risk_assessment"), + *(data.get("confirmed_facts") if isinstance(data.get("confirmed_facts"), list) else []), + *(data.get("limitations") if isinstance(data.get("limitations"), list) else []), + ) + ).lower() + local_absence_only = any( + marker in decision_context + for marker in ( + "not installed locally", + "not referenced locally", + "not present locally", + "absent from the local", + "no local dependency", + "no matching dependency", + "not found in the local repository", + ) + ) + active_evidence = [ + item + for item in case.get("evidence") or [] + if str(item.get("status") or "active") == "active" + and item.get("evidence_id") + and str((item.get("metadata") or {}).get("pipeline_id") or "") == pipeline_id + ] + evidence_ids = [str(item["evidence_id"]) for item in active_evidence[:50]] + strong_proof = any( + str(item.get("evidence_type") or "") == "sandbox_analysis" + or bool((item.get("metadata") or {}).get("advisory_backed")) + for item in active_evidence + ) + + verdict = requested + guardrail_notes: list[str] = [] + if not evidence_ids or confidence < 60: + verdict = "inconclusive" + guardrail_notes.append("insufficient linked evidence or confidence") + if requested == "credible" and not strong_proof: + verdict = "likely" if evidence_ids and confidence >= 70 else "inconclusive" + guardrail_notes.append("credible requires advisory-backed or sandbox evidence") + if contradictions and verdict == "credible": + verdict = "likely" + guardrail_notes.append("unresolved contradictions prevent a credible verdict") + if requested in {"benign", "not_substantiated"} and local_absence_only: + verdict = "inconclusive" + guardrail_notes.append("local absence cannot establish package benignness") + if requested in {"benign", "not_substantiated"} and not strong_proof: + verdict = "inconclusive" + guardrail_notes.append("an exonerating verdict requires advisory-backed or sandbox evidence") + if unsupported and verdict == "credible": + verdict = "likely" + guardrail_notes.append("unsupported claims prevent a credible verdict") + + if evidence_ids: + rationale_text = rationale or "Agent completed bounded review of normalized research evidence." + if guardrail_notes: + rationale_text += " Guardrails: " + "; ".join(guardrail_notes) + "." + record_verdict( + pipeline["case_id"], + verdict=verdict, + confidence=confidence, + rationale=f"[pipeline {pipeline_id}] {rationale_text}", + evidence_ids=evidence_ids, + actor=actor_id, + db_path=db_path, + ) + verdict_recorded = True + + safety = publication_safety_check(pipeline["case_id"], actor=actor_id, db_path=db_path) + current = get_pipeline(pipeline_id, db_path=db_path) + _set_pipeline( + pipeline_id, + "succeeded", + current_step="agent_review_complete", + summary={ + **current.get("summary", {}), + "autonomy_mode": "agent_review", + "human_review_required": False, + "verdict_recorded": verdict_recorded, + "agent_verdict": verdict or None, + "agent_verdict_confidence": confidence, + "publication_preflight": safety["status"], + "disclosure_sent": False, + "sandbox_submitted": False, + "published": False, + "human_gates_remaining": [ + "external sandbox submission", + "external disclosure delivery", + "final publication approval", + ], + }, + db_path=db_path, + ) + return get_pipeline(pipeline_id, db_path=db_path) + + def review_pipeline_item( pipeline_id: str, item_id: str, diff --git a/tests/test_intelligence.py b/tests/test_intelligence.py index bcecfb5..1382d7d 100644 --- a/tests/test_intelligence.py +++ b/tests/test_intelligence.py @@ -217,6 +217,7 @@ def runner(command): home=tmp_path, platform_name="darwin", runner=runner, + autonomy_mode="agent_review", ) plist_path = Path(result["path"]) with plist_path.open("rb") as handle: @@ -225,10 +226,23 @@ def runner(command): assert payload["Label"] == "ai.secopsai.codex-bridge" assert "SECOPSAI_CORE_READ_TOKEN" not in encoded assert "OPENAI_API_KEY" not in encoded + assert payload["EnvironmentVariables"]["SECOPSAI_RESEARCH_AUTONOMY_MODE"] == "agent_review" + assert result["autonomy_mode"] == "agent_review" assert result["credentials_persisted"] is False assert any(command[1] == "bootstrap" for command in calls) +def test_bridge_service_rejects_unknown_autonomy_mode(tmp_path: Path): + with pytest.raises(ValueError, match="supervised or agent_review"): + install_service( + db_path=str(tmp_path / "core.db"), + start=False, + home=tmp_path, + platform_name="darwin", + autonomy_mode="publish_everything", + ) + + def test_bridge_lists_opencodex_models(monkeypatch): monkeypatch.setattr("secopsai.codex_bridge.shutil.which", lambda value: f"/usr/local/bin/{value}") monkeypatch.setenv("SECOPSAI_BRIDGE_LIVE_MODELS", "true") diff --git a/tests/test_research_pipeline.py b/tests/test_research_pipeline.py index 25b338e..fb1612b 100644 --- a/tests/test_research_pipeline.py +++ b/tests/test_research_pipeline.py @@ -10,6 +10,7 @@ from secopsai.research_cases import add_evidence, add_subject, create_case, get_case from secopsai.research_intake import SafeFetcher from secopsai.research_pipeline import ( + agent_complete_pipeline, get_pipeline, pipeline_intelligence_context, resume_investigation_pipeline, @@ -96,6 +97,10 @@ def _bridge_result(action: str) -> dict: "evidence": ["The package was collected and hash identified without execution."], "recommended_actions": ["Verify publisher ownership."], "limitations": ["No runtime behavior was observed."], + "verdict_recommendation": "inconclusive", + "verdict_confidence": 50, + "verdict_rationale": "Evidence remains incomplete.", + "verdict_evidence_refs": [], } if action == "analyze_research_case": data.update( @@ -105,6 +110,10 @@ def _bridge_result(action: str) -> dict: "unsupported_claims": ["Victim impact has not been established."], "contradictions": [], "missing_evidence": ["Publisher confirmation is missing."], + "verdict_recommendation": "likely", + "verdict_confidence": 78, + "verdict_rationale": "Static package and comparison evidence supports a likely verdict while runtime behavior remains unobserved.", + "verdict_evidence_refs": ["collect_subject", "compare_packages"], } ) elif action == "generate_analyst_brief": @@ -233,6 +242,84 @@ def test_auto_review_pipeline(tmp_path, monkeypatch): assert completed["review_summary"]["rejected"] == 0 +def test_agent_complete_records_bounded_verdict_without_external_actions(tmp_path, monkeypatch): + db = str(tmp_path / "research.db") + monkeypatch.setenv("SECOPSAI_RESEARCH_QUARANTINE", str(tmp_path / "quarantine")) + case = _case(db) + pipeline = start_investigation_pipeline( + case["case_id"], + reference_ecosystem="npm", + reference_package="legitimate-pkg", + reference_version="1.0.0", + db_path=db, + fetcher=_fetcher(), + ) + _complete_bridge_queue(db) + + completed = agent_complete_pipeline(pipeline["pipeline_id"], db_path=db) + stored = get_case(case["case_id"], db_path=db) + + assert completed["status"] == "succeeded" + assert completed["summary"]["autonomy_mode"] == "agent_review" + assert completed["summary"]["verdict_recorded"] is True + assert completed["summary"]["agent_verdict"] == "likely" + assert completed["summary"]["disclosure_sent"] is False + assert completed["summary"]["sandbox_submitted"] is False + assert completed["summary"]["published"] is False + assert stored["verdicts"][0]["verdict"] == "likely" + assert stored["verdicts"][0]["actor"].startswith("secopsai-agent-autonomy:") + assert stored["verdicts"][0]["evidence_ids"] + accepted_ids = { + item["evidence_id"] + for item in stored["evidence"] + if (item.get("metadata") or {}).get("pipeline_id") == pipeline["pipeline_id"] + } + assert set(stored["verdicts"][0]["evidence_ids"]) <= accepted_ids + + +def test_agent_completion_cannot_treat_local_absence_as_benign(tmp_path, monkeypatch): + db = str(tmp_path / "research.db") + monkeypatch.setenv("SECOPSAI_RESEARCH_QUARANTINE", str(tmp_path / "quarantine")) + case = _case(db) + pipeline = start_investigation_pipeline(case["case_id"], db_path=db, fetcher=_fetcher()) + + while True: + job = claim_next_job(provider="test-codex", worker_id="test-worker", db_path=db) + if job is None: + break + result = _bridge_result(job["action"]) + if job["action"] == "analyze_research_case": + result["data"].update( + { + "verdict_recommendation": "benign", + "verdict_confidence": 99, + "verdict_rationale": "The package is not found in the local repository and no matching dependency exists.", + } + ) + complete_job(job["job_id"], result=result, actor="test-worker", db_path=db) + + completed = agent_complete_pipeline(pipeline["pipeline_id"], db_path=db) + stored = get_case(case["case_id"], db_path=db) + assert completed["summary"]["agent_verdict"] == "inconclusive" + assert stored["verdicts"][0]["verdict"] == "inconclusive" + assert "local absence cannot establish package benignness" in stored["verdicts"][0]["rationale"] + + +def test_agent_review_mode_completes_pipeline_when_last_job_finishes(tmp_path, monkeypatch): + db = str(tmp_path / "research.db") + monkeypatch.setenv("SECOPSAI_RESEARCH_QUARANTINE", str(tmp_path / "quarantine")) + monkeypatch.setenv("SECOPSAI_RESEARCH_AUTONOMY_MODE", "agent_review") + case = _case(db) + pipeline = start_investigation_pipeline(case["case_id"], db_path=db, fetcher=_fetcher()) + _complete_bridge_queue(db) + + completed = get_pipeline(pipeline["pipeline_id"], db_path=db) + assert completed["status"] == "succeeded" + assert completed["current_step"] == "agent_review_complete" + assert completed["summary"]["agent_verdict"] == "likely" + assert completed["review_summary"]["pending"] == 0 + + def test_pipeline_does_not_guess_a_legitimate_reference(tmp_path, monkeypatch): db = str(tmp_path / "research.db") monkeypatch.setenv("SECOPSAI_RESEARCH_QUARANTINE", str(tmp_path / "quarantine")) @@ -303,6 +390,34 @@ def fake_start(case_id: str, **kwargs): assert called["actor"] == "cli-test" +def test_pipeline_cli_dispatches_agent_completion(monkeypatch, capsys): + called = {} + + def fake_complete(pipeline_id: str, **kwargs): + called.update({"pipeline_id": pipeline_id, **kwargs}) + return {"pipeline_id": pipeline_id, "status": "succeeded"} + + monkeypatch.setattr(cli_module, "agent_complete_pipeline", fake_complete) + status = cli_module.main( + [ + "--json", + "research", + "pipeline", + "agent-complete", + "RPL-AAAAAAAAAAAAAAAA", + "--actor", + "agent-test", + "--db-path", + "/tmp/research-agent-test.db", + ] + ) + output = json.loads(capsys.readouterr().out) + assert status == 0 + assert output["status"] == "succeeded" + assert called["pipeline_id"] == "RPL-AAAAAAAAAAAAAAAA" + assert called["actor"] == "agent-test" + + def test_bridge_request_contains_only_normalized_pipeline_context(tmp_path, monkeypatch): db = str(tmp_path / "research.db") monkeypatch.setenv("SECOPSAI_RESEARCH_QUARANTINE", str(tmp_path / "quarantine"))