diff --git a/.coverage b/.coverage deleted file mode 100644 index 287232d1..00000000 Binary files a/.coverage and /dev/null differ diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..deeed802 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,24 @@ +.git +.github +.agents +.ruff_cache +.pytest_cache +pytest-cache-files-* +__pycache__ +*.pyc +*.pyo +*.egg-info +.coverage +coverage.json +coverage.xml +htmlcov +dist +build +node_modules +frontend/node_modules +frontend/.next +agentwatch-landing/node_modules +agentwatch-landing/.next +.env +.env.* +*.log diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed38dd08..129267eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,7 +177,9 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - name: Install pip-audit - run: pip install pip-audit + run: | + pip install -e ".[crypto]" + pip install pip-audit - name: Audit dependencies - run: pip-audit --desc --requirement <(pip-compile pyproject.toml --quiet 2>/dev/null || echo "") + run: pip-audit --desc continue-on-error: true diff --git a/.gitignore b/.gitignore index 6118d28f..d364158f 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ build/ coverage.json coverage.xml htmlcov/ +site/ # Node node_modules/ diff --git a/Dockerfile.frontend b/Dockerfile.frontend deleted file mode 100644 index f554d197..00000000 --- a/Dockerfile.frontend +++ /dev/null @@ -1,19 +0,0 @@ -FROM node:20-alpine AS builder -WORKDIR /app -COPY package.json package-lock.json ./ -RUN npm ci -COPY . . -ENV NEXT_TELEMETRY_DISABLED=1 -RUN npm run build - -FROM node:20-alpine AS runner -WORKDIR /app -ENV NODE_ENV=production -ENV NEXT_TELEMETRY_DISABLED=1 -RUN addgroup --system --gid 1001 nodejs && adduser --system --uid 1001 nextjs -COPY --from=builder /app/public ./public -COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ -COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static -USER nextjs -EXPOSE 3000 -CMD ["node", "server.js"] diff --git a/README.md b/README.md index 1db66423..9a7ff101 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ [![Tests](https://img.shields.io/badge/tests-205_passing-brightgreen?style=flat-square)](https://github.com/sreerevanth/AgentWatch) [![Coverage](https://img.shields.io/codecov/c/github/sreerevanth/AgentWatch?style=flat-square)](https://codecov.io/gh/sreerevanth/AgentWatch) [![License](https://img.shields.io/badge/license-Apache_2.0-blue?style=flat-square)](LICENSE) -[![Python](https://img.shields.io/badge/python-3.10+-blue?style=flat-square&logo=python)](https://python.org) +[![Python](https://img.shields.io/badge/python-3.12+-blue?style=flat-square&logo=python)](https://python.org) [![Discord](https://img.shields.io/badge/Discord-Join_Community-5865F2?style=flat-square&logo=discord)](https://discord.gg/n2RzUmZ4) [![Stars](https://img.shields.io/github/stars/sreerevanth/AgentWatch?style=flat-square&color=gold)](https://github.com/sreerevanth/AgentWatch/stargazers) [![Forks](https://img.shields.io/github/forks/sreerevanth/AgentWatch?style=flat-square&color=orange)](https://github.com/sreerevanth/AgentWatch/network) diff --git a/VISION.md b/VISION.md new file mode 100644 index 00000000..68c023da --- /dev/null +++ b/VISION.md @@ -0,0 +1,404 @@ +# What is AgentWatch? + +AgentWatch is the control plane for safe, reliable AI agent execution. + +It began as an AI agent observability project, but the repository now shows a broader identity: a hybrid SDK, runtime guardrail, telemetry system, governance layer, and developer platform for agents that use tools, memory, multiple models, multiple frameworks, and long-running workflows. + +The strongest product evidence is: + +- `agentwatch.watch()` instruments existing agents and framework objects without requiring users to rewrite their agent code. +- The core event schema normalizes agent lifecycle, tool calls, memory, safety, confidence, rollback, and multi-agent events. +- The `SafetyEngine` evaluates tool calls before execution, attaches reasoning confidence, estimates blast radius, enforces policies, and can block or require approval. +- The FastAPI server exposes sessions, events, replay, rollback, safety policy, safety checks, dashboard summaries, compliance reports, entitlement usage, tenants, metrics, and WebSocket streaming. +- The CLI exposes watch, replay, session inspection, export, scoring, rollback, pruning, safety checks, policy inspection, cost reports, evaluation, server controls, comparison, red-team checks, upgrade/licensing, MCP, and diagnostics. +- The dashboard presents live events, sessions, safety blocks, confidence trends, replay, rollback, policy editing surfaces, memory, multi-agent, security, benchmark, compliance, cost, sandbox, and safety lab views. +- The repository contains importable modules for observability, reasoning audit, safety, memory, multi-agent orchestration, cost governance, compliance, security, rollback, replay, protocol, plugins, cloud tenancy, and deployment. + +AgentWatch is therefore not only an observability layer. It is an execution governance platform for autonomous and semi-autonomous AI systems. + +# Why AgentWatch Exists + +AI agents fail differently from normal software. + +Traditional applications usually fail through errors, crashes, bad latency, or bad status codes. Agents can fail while looking successful: they can hallucinate tool arguments, drift away from the original goal, call the wrong system, leak sensitive data, loop until budget is exhausted, create inconsistent multi-agent state, or perform irreversible actions before anyone notices. + +Existing observability tools are strongest after execution. AgentWatch is designed around the earlier moment: before an action reaches the outside world. + +AgentWatch exists to make agent behavior inspectable, governable, reversible where possible, and preventable where necessary. + +# The Problem + +The single problem AgentWatch solves: + +> Teams cannot safely operate AI agents in production unless they can observe, evaluate, constrain, and intervene in agent actions before those actions cause damage. + +Who experiences this: + +- AI application developers adding tool use to LLM workflows. +- Platform teams standardizing agent infrastructure across frameworks. +- Security teams worried about command execution, exfiltration, prompt injection, and unsafe tools. +- Compliance and governance teams that need audit trails, retention controls, redaction, and explainable decision records. +- Engineering managers responsible for reliability, cost, and incident response. +- Operators supervising long-running or multi-agent workflows. + +Why existing tools fail: + +- Post-hoc tracing explains what happened after the action has already run. +- Generic APM does not model reasoning steps, tool intent, goal drift, hallucinated arguments, or agent memory. +- Framework-specific monitoring fragments across LangChain, CrewAI, AutoGen, LangGraph, Claude Code, OpenAI Agents, AutoGPT, Smolagents, OpenClaw, and custom agents. +- Simple guardrails usually sit at prompt or response boundaries, not around every tool action. +- Cost dashboards rarely understand agent loops, task complexity, model failover, or per-step attribution. +- Compliance systems usually require humans to reconstruct agent decisions from scattered logs. + +What makes AgentWatch different: + +- It treats each agent action as a governed event, not just a log line. +- It combines telemetry, safety, reasoning audit, policy, cost, memory, replay, and governance around one normalized event model. +- It provides both developer-local workflows and platform/API/dashboard workflows. +- It has an explicit pre-execution enforcement path, including block, approval, and rollback-oriented workflows. + +# Our Mission + +Make AI agents safe enough to operate in real production environments by giving teams a unified control plane for observing, auditing, constraining, replaying, and governing agent execution. + +# Our Vision + +AgentWatch should become the default safety and reliability layer between AI agents and the systems they affect. + +In five years, a production agent stack should not connect directly to tools, APIs, databases, file systems, or other agents without a control plane that can answer: + +- What is the agent trying to do? +- Why is it trying to do it? +- Is the action safe? +- Is the reasoning trustworthy enough? +- What is the blast radius? +- Does policy allow it? +- Should a human approve it? +- Can the action be replayed or rolled back? +- What did this cost? +- What evidence is available for audit and compliance? + +AgentWatch should be that control plane. + +# Core Product Pillars + +Every future feature should strengthen at least one of these pillars. + +## 1. Pre-Execution Safety + +AgentWatch must prevent high-risk agent actions before they execute. This includes destructive command detection, risky database operations, remote code execution patterns, exfiltration attempts, prompt injection, sandbox simulation, blast-radius estimation, human approval gates, and policy enforcement. + +## 2. Unified Agent Telemetry + +AgentWatch must capture agent behavior as normalized, typed events across frameworks. Sessions, traces, spans, tool calls, reasoning artifacts, memory events, cost metadata, confidence scores, and task graph events should share one coherent model. + +## 3. Reasoning Trust + +AgentWatch must evaluate whether an agent's reasoning and intended action remain coherent, grounded, goal-aligned, and resistant to drift or hallucination. The independent auditor, confidence scoring, semantic drift, hallucination checks, fingerprinting, dual evaluation, and benchmark harness all belong here. + +## 4. Operational Control + +AgentWatch must help operators intervene in running or completed sessions. Replay, counterfactual simulation, rollback, checkpoints, alerts, live dashboards, CLI inspection, session sharing, and server controls belong here. + +## 5. Multi-Agent and Memory Governance + +AgentWatch must model state that crosses a single call: causal memory, temporal decay, memory health, identity, natural-language memory query, inter-agent DAGs, crew context, propagation, deadlock, consensus, spawning, and attribution. + +## 6. Cost and Model Governance + +AgentWatch must make agent spend predictable and controllable. Token budgets, per-session tracking, cost reporting, model comparison, degradation routing, complexity-based routing, anomaly detection, ROI, and semantic caching belong here. + +## 7. Compliance and Trust Evidence + +AgentWatch must produce evidence for regulated and enterprise adoption. Audit logs, RBAC, tenant isolation, redaction, GDPR, HIPAA, EU AI Act, ISO 42001, data residency, security reports, entitlement checks, and signed or immutable records belong here. + +## 8. Open Integration Surface + +AgentWatch must be easy to adopt and extend. Framework adapters, REST APIs, WebSockets, CLI, MCP server, Open Reasoning Trace schema, badge checks, plugins, deployment templates, and OpenTelemetry export belong here. + +# Product Identity + +AgentWatch is a hybrid product. + +It is a library because the package exposes `watch()`, adapters, event schemas, safety engines, memory engines, reasoning modules, and utility APIs directly to Python users. + +It is an SDK because developers integrate it into LangChain, CrewAI, AutoGPT, Claude Code, LangGraph, AutoGen, Smolagents, OpenAI Agents, OpenClaw, or custom agents. + +It is a runtime guardrail because it intercepts tool-like calls, evaluates safety before execution, and can raise `AgentWatchBlockedError`. + +It is a control plane because the server, dashboard, CLI, WebSocket stream, tenants, policies, compliance reports, and operational commands create a central operating surface for many agent sessions. + +It is an observability layer because it captures traces, spans, sessions, events, dashboards, metrics, OTEL-compatible telemetry, replay, and live streams. + +It is a security and governance layer because it implements risk scoring, policy enforcement, redaction, OWASP checks, sandboxing, audit logs, RBAC, entitlement controls, and compliance reports. + +It is not yet a fully integrated platform in every area. Several advanced modules are implemented and tested as Python components but are not consistently exposed through mounted API routes or complete dashboard workflows. + +The objective identity is: + +> AgentWatch is a hybrid AI agent execution control plane: part SDK, part runtime guardrail, part observability platform, part governance and security layer. + +# System Architecture + +The system is organized around a normalized event pipeline. + +## Instrumentation Layer + +The `watch()` API and adapters instrument framework-specific agents and translate actions into `AgentEvent` records. Evidence includes adapters for Claude Code, LangChain, CrewAI, AutoGPT, OpenAI Agents, OpenClaw, LangGraph, AutoGen, and Smolagents, plus the generic wrapper. + +## Event Model and Bus + +The core schema defines event types for lifecycle, planning, tool execution, memory, safety, multi-agent messages, rollback, confidence, and custom events. The event bus routes these events to collectors, live streams, forwarders, and other subscribers. + +## Runtime Enforcement + +The safety engine evaluates tool calls with pattern-based risk scoring, intent normalizers, reasoning audit, blast-radius estimation, policy DSL evaluation, static policy fallback, and optional approval callbacks. The sync path blocks approval-required actions fail-safe. + +## Persistence and API + +The backend uses FastAPI, SQLAlchemy async, PostgreSQL, Redis, and Celery. The API stores sessions/events, streams WebSockets, exposes Prometheus metrics, supports pruning, handles rollback, and reports governance/compliance outputs. + +## Operator Surfaces + +The CLI and dashboard provide developer and operator workflows: watch, replay, list, export, score, safety check, rollback, cost report, eval, red-team, server status, live event feed, session detail, replay studio, safety lab, sandbox, and domain pages. + +## Advanced Capability Modules + +Separate packages cover reasoning, memory, orchestration, cost, governance, security, platform, protocol, plugins, replay, rollback, monitoring, telemetry, tracing, validation, and infrastructure routing. + +## Deployment + +The repo includes Docker Compose, Dockerfiles, Render/Railway/Fly configuration, a Helm chart, Grafana dashboards, CI workflows, load tests, and benchmark scripts. + +# Guiding Principles + +## Product Principles + +- Prevent harm before explaining it. +- Treat actions, not responses, as the primary unit of risk. +- Make every agent action attributable to a session, actor, policy, cost, and confidence state. +- Work across frameworks instead of forcing one orchestration stack. +- Keep local developer adoption simple while supporting enterprise controls. +- Prefer one coherent control plane over scattered point tools. + +## Design Philosophy + +- Operator views must answer what happened, what is happening, why it matters, and what can be done next. +- Safety UI should expose decisions, evidence, matched policies, and blast-radius reasoning. +- Advanced capability pages must not exist as empty product promises; if a page ships, the backend route and data contract should ship with it. +- The dashboard should be an operational cockpit, not a marketing site. + +## Engineering Philosophy + +- The normalized event schema is the foundation; new features should enrich or consume it, not bypass it. +- Guardrails should fail closed when security or approval state is uncertain. +- Runtime instrumentation should avoid crashing host agents except when intentionally blocking a dangerous action. +- Modules should become end-to-end workflows before new domains are added. +- Tests should validate behavior across the full chain: adapter to event, safety decision, persistence, API, dashboard/CLI. +- Security-sensitive shell execution must stay routed through the central CLI utility. + +## User Philosophy + +- Developers should add AgentWatch without rewriting their agent. +- Operators should understand and act without reading raw logs. +- Security and compliance teams should get evidence, not anecdotes. +- Teams should be able to self-host and keep sensitive traces inside their environment. + +# Scope + +## What Belongs Inside AgentWatch + +- Agent instrumentation and framework adapters. +- Normalized event schemas and trace collection. +- Pre-execution risk scoring, blocking, approval, and policy enforcement. +- Reasoning confidence, hallucination, drift, quality, benchmark, and trust evaluation. +- Replay, counterfactual simulation, checkpointing, rollback, and incident review. +- Causal memory and memory governance when tied to agent reliability, auditability, or safety. +- Multi-agent tracing, propagation, consensus, deadlock, task graph, and attribution when tied to execution reliability. +- Cost tracking, budget enforcement, model routing, semantic caching, anomaly detection, and ROI reporting. +- Compliance, audit, redaction, RBAC, tenancy, data residency, entitlement, and security reporting. +- Dashboard, CLI, REST API, WebSocket, MCP, plugin, OTEL, Grafana, and deployment integrations. +- Benchmarks and tests that measure safety, reasoning, overhead, and resilience. + +## What Should Never Belong Inside AgentWatch + +- A general-purpose agent framework that competes with LangChain, CrewAI, AutoGen, or LangGraph. +- A general chat application or end-user assistant unrelated to agent execution control. +- A generic APM replacement for non-agent services. +- A standalone vector database, model provider, or LLM gateway unless directly required for safety, routing, or cost governance. +- Prompt libraries or business workflow templates that are not tied to governance, evaluation, or observability. +- Marketing-only dashboard pages without backend routes and working data contracts. +- Compliance claims that cannot be generated from real telemetry, configuration, and audit evidence. +- Autonomous remediation that mutates production systems without policy, auditability, approval controls, and rollback planning. + +# Non Goals + +- AgentWatch should not become the agent orchestration engine. +- AgentWatch should not hide unsafe behavior behind prettier observability. +- AgentWatch should not require one vendor's model, cloud, or framework. +- AgentWatch should not collect private chain-of-thought unless explicitly provided and permitted by the host system. +- AgentWatch should not claim production readiness for modules that are test-only, simulated, or not wired to an operational surface. +- AgentWatch should not prioritize breadth of feature count over end-to-end reliability. + +# Repository Assessment + +## Feature Inventory + +| Capability | Repository Evidence | Purpose | Beneficiary | Layer | Completeness | Overlap / Belongs | +|---|---|---|---|---|---|---| +| Universal `watch()` SDK | `agentwatch/core/watcher.py`, root exports | Attach observability and safety to existing agents | Developers | SDK/runtime | Strong for generic and several framework paths | Belongs; core adoption surface | +| Framework adapters | `agentwatch/adapters/*` | Normalize framework-specific events | Developers/platform teams | SDK/integration | Broad coverage; depth varies by framework | Belongs; should be tested against real framework versions | +| Event schema | `agentwatch/core/schema.py` | Common representation for sessions/actions/safety/memory/cost | All users | Core | Strong | Belongs; should remain canonical | +| Event bus | `agentwatch/core/event_bus.py` | Route events to collectors/streams/forwarders | Platform/operators | Core/runtime | Strong | Belongs | +| HTTP forwarding | `agentwatch/core/http_forwarder.py` | Send local events to API | Developers/platform | Integration | Present | Belongs | +| Trace collection/spans | `agentwatch/tracing`, `agentwatch/telemetry` | Capture spans, trajectories, sampling, OTEL export, live streams | Operators/SRE | Observability | Implemented modules; API integration partial | Belongs | +| API server | `agentwatch/api/server.py` | Central REST/WebSocket/metrics surface | Dashboard, CLI, integrations | Platform | Strong for sessions/safety/replay/tenant basics | Belongs; route modularization needed | +| Dashboard | `frontend/pages/*` | Operator UI for sessions, live feed, safety, replay, domain pages | Operators | Product UI | Core dashboard works; many domain pages hit missing routes | Belongs; incomplete chains need priority | +| CLI | `agentwatch/cli/main.py` | Local developer/operator control | Developers/operators | DX/control | Broad and useful | Belongs; should keep secure command utility rule | +| Safety engine | `agentwatch/core/safety.py`, risk/injection/blast policy modules | Block/approve risky actions before execution | Security/operators | Runtime enforcement | Strong core | Belongs; central pillar | +| Policy DSL | `agentwatch/core/policy_dsl.py`, `policy_loader.py`, CLI/page surfaces | Human-readable runtime policy | Security/platform | Governance | Implemented; frontend endpoints drift from backend | Belongs; needs single API contract | +| Human approval | Safety policy and approval callback | Pause risky actions for review | Operators/security | Runtime control | CLI callback exists; full workflow partial | Belongs | +| Sandbox/safety lab | `agentwatch/security/sandbox.py`, frontend sandbox/safety-lab | Simulate command risk and threat path | Developers/security | Security/DX | Safety lab wired to `/safety/check`; sandbox page calls missing `/security/sandbox/simulate` | Belongs; consolidate | +| OWASP checks | `agentwatch/security/owasp.py`, tests, security page | Detect agentic security vectors | Security | Security/compliance | Module/test present; page calls missing route | Belongs; route needed | +| Red-team harness | `agentwatch/security/redteam.py`, payload corpus, CLI, Celery task | Test guardrail resilience without executing attacks | Security | Security/eval | Implemented harness | Belongs | +| Redaction | `agentwatch/security/redaction.py`, GDPR/HIPAA patterns | Scrub PII/PHI from telemetry | Compliance/security | Governance | Implemented, opt-in | Belongs | +| Reasoning auditor | `agentwatch/reasoning/auditor.py` and related modules | Score trust, hallucination, drift, quality, calibration | Developers/operators | Reasoning trust | Broad module coverage; benchmark semantics depend on judge | Belongs | +| Confidence scoring | `agentwatch/scoring/confidence.py`, API confidence endpoint | Summarize session trust/anomalies | Operators | Observability/reasoning | Present | Belongs | +| Benchmarks/eval | `benchmarks`, `agentwatch/eval`, CLI eval | Test reasoning and safety behavior | Developers/platform | Evaluation | Harness present; saved mock result should not be overclaimed | Belongs | +| Replay | `agentwatch/replay`, API, dashboard | Time-travel session review | Operators | Operational control | Strong | Belongs | +| Counterfactual simulation | `agentwatch/replay/counterfactual.py`, API simulate | Ask what changes if a step differs | Developers/operators | Operational control | Implemented with user-supplied step behavior; not full autonomous replay | Belongs | +| Rollback | `agentwatch/rollback`, CLI/API/session UI | Restore filesystem/git checkpoints | Operators | Operational control | Present; high-risk area | Belongs with strict safety/audit | +| Memory engine | `agentwatch/memory/*` | Store/query/govern causal and temporal memory | Developers/operators | Memory governance | Implemented modules; dashboard routes missing | Belongs if tied to reliability/governance | +| Multi-agent orchestration analysis | `agentwatch/orchestration/*` | Model DAGs, deadlocks, trust, propagation, consensus | Multi-agent builders | Multi-agent governance | Implemented modules; dashboard routes missing | Belongs | +| Cost intelligence | `agentwatch/cost/*`, CLI cost | Track budgets, compare models, route, predict, cache, detect anomalies | Engineering/finance | Cost governance | Modules and CLI present; dashboard route missing | Belongs | +| Compliance/governance | `agentwatch/governance/*`, API reports | Produce audit/compliance evidence | Compliance/security | Governance | Modules plus some mounted reports | Belongs; claims must be evidence-backed | +| RBAC/auth/tenant | `agentwatch/api/auth.py`, `tenant_auth.py`, models | Auth, permissions, tenant isolation | Enterprise/platform | Platform/security | Present; dashboard auth forwarding mismatch | Belongs | +| Entitlements/licensing | `security/license.py`, `checkout.py`, entitlement API, CLI upgrade | Premium feature gating and abuse detection | Maintainers/cloud users | Commercial platform | Present | Belongs if product includes cloud/commercial path | +| Plugins | `agentwatch/plugins/*`, manifest schema | Extend safely with permissions | Ecosystem | Platform | Registry/sandbox present | Belongs but should remain constrained | +| Protocol/MCP/badge | `agentwatch/protocol/*`, CLI MCP | Open trace schema and interoperability | Ecosystem/frameworks | Protocol | Present | Belongs; strategic differentiator | +| Monitoring/metrics | `agentwatch/monitoring`, `/metrics`, Grafana JSON | Operate AgentWatch itself | SRE/platform | Ops | Present | Belongs | +| Deployment | Docker, Helm, Render, Railway, Fly | Self-host and deploy | Platform/devops | Infra | Present but inconsistent completeness | Belongs; harden | +| Landing site | `agentwatch-landing` | Public positioning, docs/blog/contributors | Community/users | Marketing/docs | Separate app; contains unverified claims | Belongs as website, not source of product truth | + +## Architecture Drift and Technical Debt + +- Package/runtime versioning is now aligned at `0.2.0`, but generated examples, Helm metadata, docs, and landing-page release claims still need a single release authority. +- Python support is now documented as 3.12+ in package and primary setup docs; secondary docs and examples should continue to be checked during release work. +- `MASTERLIST_STATUS.md` explicitly says some new modules do not have wired API routes and frontend pages render empty states when endpoints are missing. +- Frontend hooks call routes not mounted in `agentwatch/api/server.py`, including `/memory/graph`, `/memory/query`, `/orchestration/dag`, `/orchestration/deadlock`, `/security/owasp`, `/security/sandbox/simulate`, `/cost/summary`, `/governance/compliance/status`, `/reasoning/benchmark/latest`, `/reasoning/benchmark/run`, `/policies/current`, and `/policies/preview`. +- Backend exposes `/api/v1/safety/policy`, while frontend policy hooks use `/policies/current`. +- Backend `/api/v1/safety/blocked` returns an object containing `blocked_events`, while the dashboard hook types it as an array. +- Backend `/api/v1/sessions` returns `{ sessions, total }`, while the dashboard hook types it as `AgentSession[]`. +- Backend `/api/v1/sessions/{id}/checkpoints` response shape should be verified against the frontend's direct array expectation. +- Frontend API proxy forwards `Authorization` but not `X-Api-Key`, while protected backend endpoints rely on `X-Api-Key`. +- Docker Compose sets `AGENTWATCH_ENV=production` for the API but does not set `AGENTWATCH_API_KEY`, which conflicts with the backend fail-closed production model. +- Helm chart covers the API deployment/service but not a complete platform release including frontend, PostgreSQL templates, Redis templates, workers, ingress, secrets, and auth values. +- Benchmark README warns the default heuristic auditor is not semantic; saved results use a mock judge. Performance and accuracy claims should distinguish harness capability from real auditor accuracy. +- Landing blog/source claims include broad market and performance assertions that are not clearly tied to repository evidence. +- Stale setup references were corrected in the main developer setup and CLI getting-started docs; remaining docs should be periodically checked against package metadata. +- The API server is large and centralizes many concerns in one module, increasing coupling across auth, storage, telemetry, safety, rollback, governance, tenants, metrics, and WebSockets. +- There are duplicate or overlapping concepts: governance RBAC vs API auth, memory decay vs temporal decay, telemetry collector vs tracing collector, sandbox page vs safety lab, policy DSL routes vs safety policy routes, platform tenant store vs API tenant store. +- `__main__.py` at the repository root is empty. +- The landing app has its own Next.js 16 rules and separate dependency set; it must remain isolated from the dashboard. + +# Current Strengths + +- Clear differentiated core: pre-execution safety plus reasoning-aware agent telemetry. +- Strong normalized event model that can support observability, governance, memory, cost, and multi-agent use cases. +- Broad framework adapter surface and a one-line adoption story. +- Useful developer CLI with real operational workflows. +- FastAPI backend already exposes the core session, event, replay, rollback, safety, dashboard, compliance, tenant, metrics, and WebSocket surfaces. +- Safety engine includes practical bypass-resistant command intent normalizers. +- Replay and rollback give AgentWatch an operational control story beyond passive logging. +- Rich module inventory creates a credible roadmap for a wider control plane. +- Self-hosting and deployment assets exist across Docker, Render, Railway, Fly, Helm, Grafana, and CI. +- Test suite is broad and includes security, API, CLI, safety, reasoning, memory, cost, governance, protocol, and adapter coverage. + +# Current Weaknesses + +- Product scope has expanded faster than end-to-end integration. +- Several dashboard pages are ahead of mounted backend routes. +- Versioning and public claims are inconsistent. +- Some advanced features are implemented as isolated modules rather than coherent API/dashboard workflows. +- The API server has become a large coupling point. +- Production auth defaults and compose configuration conflict. +- Compliance and benchmark claims need stricter evidence discipline. +- Documentation quality is uneven and includes stale setup instructions. +- The product has both open-source/self-hosted and commercial/entitlement directions, but the boundary is not yet clearly documented. + +# Future Direction + +## Immediate Priorities + +1. Reconcile product identity and versioning across `pyproject.toml`, package `__version__`, README, Helm, docs, and landing site. +2. Fix the dashboard/API contract mismatches for sessions, blocked events, policies, memory, multi-agent, security, cost, compliance, and benchmark surfaces. +3. Decide which advanced pages should ship now, which should be hidden, and which need backend routes first. +4. Fix production auth/deployment defaults, including Docker Compose `AGENTWATCH_API_KEY` handling and dashboard proxy key forwarding. +5. Split or modularize the API server by capability while preserving the public API. +6. Add end-to-end tests for the most important workflows: watch to event to safety to persistence to dashboard/CLI. +7. Mark benchmark modes clearly: heuristic, mock judge, external LLM judge, and production calibration. +8. Update stale docs and remove conflicting installation/version claims. + +## Medium-Term Priorities + +1. Turn the strongest module families into complete product workflows: + - Safety and policy center. + - Replay and rollback studio. + - Reasoning trust dashboard. + - Memory governance dashboard. + - Multi-agent causal DAG. + - Cost governance and routing. + - Compliance evidence center. +2. Define stable API contracts for each product pillar. +3. Harden tenant isolation, RBAC, API key propagation, audit logging, and data residency. +4. Expand real framework integration tests against supported versions. +5. Publish the Open Reasoning Trace schema as a formal spec with validation, examples, and compatibility tests. +6. Make plugin permissions and sandbox enforcement production-grade before inviting third-party plugins. +7. Build a measured overhead benchmark and publish latency/cost/accuracy results with reproducible methodology. + +## Long-Term Vision + +Over the next 3-5 years, AgentWatch should become: + +- The standard control plane for tool-using AI agents. +- The open trace schema for agent reasoning, tool execution, safety decisions, and governance evidence. +- The self-hosted enterprise platform for agent observability, safety, compliance, and cost governance. +- The interoperability layer across agent frameworks. +- The operational evidence system for regulated agent deployments. + +## Future Ecosystem Opportunities + +- AgentWatch-compatible badge for frameworks and tools. +- Open Reasoning Trace adoption across agent frameworks. +- Managed AgentWatch Cloud for teams that do not want to operate Postgres/Redis/API/dashboard. +- Policy packs for industries such as finance, healthcare, legal, and infrastructure. +- Plugin marketplace for auditor models, security rules, exporters, compliance templates, and custom dashboards. +- Anonymous failure benchmark, opt-in only, built from scrubbed and governed traces. +- MCP-based agent self-inspection tools. + +# Success Metrics + +AgentWatch should measure success by operational outcomes, not feature count. + +- Percentage of dangerous actions blocked before execution. +- False positive and false negative rates for safety decisions. +- Median and p99 overhead added by instrumentation and safety checks. +- Time from integration to first visible session. +- Percentage of supported dashboard pages backed by real API routes and tests. +- Number of frameworks with verified adapter tests. +- Mean time to diagnose a failed agent session using replay. +- Cost reduction from budget enforcement, model routing, and caching. +- Compliance evidence completeness for audit requests. +- Tenant isolation and auth test coverage. +- Production deployments with configured API keys, retention, redaction, and alerting. +- User retention for CLI, dashboard, and API workflows. + +# Final Statement + +AgentWatch should not be guided by the old assumption that it is merely AI agent observability. + +The repository shows a more ambitious and more valuable product: an AI agent execution control plane that observes, audits, constrains, replays, governs, and helps reverse agent behavior across frameworks. + +The path forward is not to add more disconnected features. The path forward is to make the existing pillars coherent, end-to-end, evidence-backed, and hard to misuse. + +AgentWatch wins if teams trust it enough to put it between their agents and production. diff --git a/__main__.py b/__main__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/agentwatch/cli/demo.py b/agentwatch/cli/demo.py index 776bee80..485f21b7 100644 --- a/agentwatch/cli/demo.py +++ b/agentwatch/cli/demo.py @@ -569,7 +569,7 @@ async def run_demo(): print( bold(""" +--------------------------------------------------------------+ -| AgentWatch - Demo Suite v0.1.0 | +| AgentWatch - Demo Suite v0.2.0 | | Reliability, Safety & Observability Layer for AI Agents | +--------------------------------------------------------------+ """) diff --git a/benchmarks/generate_cases.py b/benchmarks/generate_cases.py index 34a3a068..808bec10 100644 --- a/benchmarks/generate_cases.py +++ b/benchmarks/generate_cases.py @@ -1,5 +1,6 @@ import json import uuid +from pathlib import Path from typing import Any from agentwatch.core.schema import AgentEvent, EventType, ExecutionStatus, ToolCallData @@ -22,8 +23,6 @@ def create_event( return event.model_dump_for_storage() -from pathlib import Path - OUTPUT_PATH = Path(__file__).parent / "test_cases.json" @@ -87,7 +86,7 @@ def generate_cases(): } ) - with open(OUTPUT_PATH, "w") as f: + with open(OUTPUT_PATH, "w", encoding="utf-8") as f: json.dump(cases, f, indent=2) diff --git a/benchmarks/run_eval.py b/benchmarks/run_eval.py index 29292ec3..a8ac05ac 100644 --- a/benchmarks/run_eval.py +++ b/benchmarks/run_eval.py @@ -1,8 +1,10 @@ import asyncio import json +import math import os import sys import time +from pathlib import Path from statistics import mean, median from rich.console import Console @@ -12,10 +14,6 @@ from agentwatch.reasoning.auditor import ReasoningAuditor -import math -from pathlib import Path - - def safe_div(n, d): return n / d if d else 0.0 @@ -49,7 +47,7 @@ async def run_evaluation(): results_dir = base_dir / "results" output_path = results_dir / "eval_latest.json" - with open(input_path) as f: + with open(input_path, encoding="utf-8") as f: cases = json.load(f) if use_mock_judge: @@ -123,7 +121,7 @@ async def run_evaluation(): "confusion_matrix": {"tp": tp, "fp": fp, "tn": tn, "fn": fn}, } - with open(output_path, "w") as f: + with open(output_path, "w", encoding="utf-8") as f: json.dump(results, f, indent=2) console.print(f"[green]Results saved to {output_path}[/green]") diff --git a/docker-compose.yml b/docker-compose.yml index 04caf985..5746b82e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,7 +70,7 @@ services: frontend: build: context: ./frontend - dockerfile: ../Dockerfile.frontend + dockerfile: Dockerfile environment: AGENTWATCH_API_URL: http://api:8000 NEXT_PUBLIC_API_URL: /api/v1 diff --git a/docs/architecture.md b/docs/architecture.md index 434193df..c168ce77 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -61,7 +61,7 @@ graph TD --- ## Detailed Guides -- [Detailed Architecture Internals](file:///d:/AgentWatch-Ed/AgentWatch/docs/architecture_detailed.md) -- [Developer Setup Manual](file:///d:/AgentWatch-Ed/AgentWatch/docs/developer_setup.md) -- [Custom Adapters Tutorial](file:///d:/AgentWatch-Ed/AgentWatch/docs/custom_adapters_tutorial.md) -- [Extended Getting Started Guide](file:///d:/AgentWatch-Ed/AgentWatch/docs/getting_started_extended.md) +- [Detailed Architecture Internals](architecture_detailed.md) +- [Developer Setup Manual](developer_setup.md) +- [Custom Adapters Tutorial](custom_adapters_tutorial.md) +- [Extended Getting Started Guide](getting_started_extended.md) diff --git a/docs/architecture_detailed.md b/docs/architecture_detailed.md index c0a61b07..45e5bfd7 100644 --- a/docs/architecture_detailed.md +++ b/docs/architecture_detailed.md @@ -1,4 +1,4 @@ -# Detailed Architecture Internals (ELUSoC_2026) +# Detailed Architecture Internals ## Event Bus Architecture The event bus distributes asynchronous task tracing payloads to multiple targets. It runs on a thread-safe registry with reader-writer locks. diff --git a/docs/cli/getting-started.md b/docs/cli/getting-started.md index cb604b3a..a3106d5f 100644 --- a/docs/cli/getting-started.md +++ b/docs/cli/getting-started.md @@ -2,10 +2,10 @@ ## Install -AgentWatch requires Python 3.10+. +AgentWatch requires Python 3.12 or newer. ```bash -pip install agentwatch +pip install agentwatch-ai ``` To work from a checkout of the repository: diff --git a/docs/custom_adapters_tutorial.md b/docs/custom_adapters_tutorial.md index cb23b9b2..071370a8 100644 --- a/docs/custom_adapters_tutorial.md +++ b/docs/custom_adapters_tutorial.md @@ -1,4 +1,4 @@ -# Custom Adapters Tutorial (ELUSoC_2026) +# Custom Adapters Tutorial ## Implementation Steps To build a custom wrapper for framework targets: diff --git a/docs/developer_setup.md b/docs/developer_setup.md index 90b63d10..af3d87db 100644 --- a/docs/developer_setup.md +++ b/docs/developer_setup.md @@ -1,4 +1,4 @@ -# Developer Setup Guide (ELUSoC_2026) +# Developer Setup Guide ## Environment Requirements Prepare a Python environment running version 3.12 or newer: @@ -9,12 +9,12 @@ python --version ## Installation 1. Clone the repository fork: ```bash - git clone https://github.com/DebasmitaBose0/AgentWatch-Ed.git + git clone https://github.com/sreerevanth/AgentWatch.git ``` 2. Navigate and install dependencies: ```bash cd AgentWatch - pip install -e .[dev] + pip install -e ".[dev]" ``` 3. Run verification tests: ```bash diff --git a/docs/getting_started_extended.md b/docs/getting_started_extended.md index a62dc796..4b58a961 100644 --- a/docs/getting_started_extended.md +++ b/docs/getting_started_extended.md @@ -1,4 +1,4 @@ -# Extended Getting Started Guide (ELUSoC_2026) +# Extended Getting Started Guide ## Setup Configuration Declare parameters in a local `.env` file: diff --git a/docs/issue-498-resolved.md b/docs/issue-498-resolved.md deleted file mode 100644 index d17180c5..00000000 --- a/docs/issue-498-resolved.md +++ /dev/null @@ -1,7 +0,0 @@ -# Issue #498 Resolution Note - -Issue [#498](https://github.com/sreerevanth/AgentWatch/issues/498) requested collapsing `TemporalDecayManager` into `ForgettingEngine`. - -This work was completed and merged via [PR #533](https://github.com/sreerevanth/AgentWatch/pull/533) which closed issue [#476](https://github.com/sreerevanth/AgentWatch/issues/476) — the original task. The merge made `TemporalDecayManager` a redundant abstraction, fully resolving the intent of #498. - -Closes #498 \ No newline at end of file diff --git "a/how HEAD\357\200\272__main__.py" "b/how HEAD\357\200\272__main__.py" new file mode 100644 index 00000000..0e2d66a4 --- /dev/null +++ "b/how HEAD\357\200\272__main__.py" @@ -0,0 +1,45 @@ +warning: in the working copy of '.github/workflows/ci.yml', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of '.gitignore', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'README.md', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'benchmarks/generate_cases.py', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'benchmarks/run_eval.py', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'docker-compose.yml', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'docs/architecture.md', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'docs/architecture_detailed.md', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'docs/cli/getting-started.md', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'docs/custom_adapters_tutorial.md', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'docs/developer_setup.md', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'docs/getting_started_extended.md', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'mkdocs.yml', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'pyproject.toml', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'tests/conftest.py', LF will be replaced by CRLF the next time Git touches it + .coverage | Bin 69632 -> 0 bytes + .github/workflows/ci.yml | 6 +++-- + .gitignore | 1 + + Dockerfile.frontend | 19 -------------- + README.md | 2 +- + __main__.py | 0 + agentwatch/cli/demo.py | 2 +- + benchmarks/generate_cases.py | 5 ++-- + benchmarks/run_eval.py | 10 +++----- + docker-compose.yml | 2 +- + docs/architecture.md | 8 +++--- + docs/architecture_detailed.md | 2 +- + docs/cli/getting-started.md | 4 +-- + docs/custom_adapters_tutorial.md | 2 +- + docs/developer_setup.md | 6 ++--- + docs/getting_started_extended.md | 2 +- + docs/issue-498-resolved.md | 7 ----- + mkdocs.yml | 5 ++++ + pyproject.toml | 4 +-- + scripts/owasp_test_harness.py | 8 ++---- + tests/conftest.py | 7 ----- + tests/test_cli_share.py | 4 +-- + tests/test_db_failure.py | 1 - + tests/test_platform.py | 4 +-- + tests/test_router.py | 1 + + tests/test_semantic_caching.py | 54 +++++++++++++++++++++++---------------- + tests/test_server.py | 5 ++-- + tests/test_sync_safety.py | 13 +++++----- + tests/test_telemetry.py | 4 +++ + 29 files changed, 84 insertions(+), 104 deletions(-) diff --git a/issues.md b/issues.md new file mode 100644 index 00000000..ba1b7057 --- /dev/null +++ b/issues.md @@ -0,0 +1,337 @@ +# AgentWatch Engineering Audit + +Audit date: 2026-07-12 + +Scope reviewed: Python package, frontend dashboard, landing page, documentation, Docker assets, GitHub Actions, tests, packaging metadata, and generated/local artifacts present in the repository workspace. + +## Architecture + +### Issue 1: Three independent applications share one repository root + +- Severity: Medium +- Location: `agentwatch/`, `frontend/`, `agentwatch-landing/` +- Description: The repository contains a Python package, a Next.js 14 dashboard, and a Next.js 16 landing page with independent dependency graphs and runtime conventions. +- Why it matters: Contributors can accidentally mix dependencies, Dockerfiles, or components across apps, especially because both frontend apps use similar framework names but incompatible major versions. +- Recommended fix: Keep project-local config and Docker assets inside each app, document boundaries in onboarding docs, and avoid shared frontend source unless a deliberate package boundary is introduced. + +### Issue 2: Public package exports depend on wildcard re-exports + +- Severity: Low +- Location: `agentwatch/**/__init__.py` +- Description: Package entrypoints intentionally use wildcard re-exports. +- Why it matters: Wildcard exports make public API ownership harder to audit and can obscure unused imports. +- Recommended fix: Keep current behavior for compatibility, but define explicit `__all__` lists over time for stable public modules. + +## Technical Debt + +### Issue 3: Large CLI module has too many responsibilities + +- Severity: Medium +- Location: `agentwatch/cli/main.py` +- Description: The CLI contains command routing, rendering, API calls, upgrade/checkout logic, policy display, server startup, export, compare, and live watch behavior in a single large module. +- Why it matters: Large command modules are hard to test in isolation and make small CLI changes risky. +- Recommended fix: Split by command family behind the existing Typer command surface without changing CLI behavior. + +### Issue 4: Optional telemetry behavior was not reflected consistently in tests + +- Severity: Medium +- Location: `tests/test_telemetry.py` +- Description: Several reasoning trace tests assumed OpenTelemetry was available even when the package correctly degrades without the optional OTel dependency. +- Why it matters: Environments without optional telemetry dependencies fail tests even though runtime behavior is intentionally optional. +- Recommended fix: Guard OTel-specific tests with the same `_OTEL_AVAILABLE` skip condition used elsewhere. Fixed in this pass. + +## Duplicate Code + +### Issue 5: Duplicate dashboard Dockerfiles + +- Severity: Medium +- Location: `Dockerfile.frontend`, `frontend/Dockerfile`, `docker-compose.yml` +- Description: Two Dockerfiles built the same dashboard app, but CI used `frontend/Dockerfile` while Compose used the root `Dockerfile.frontend`. +- Why it matters: Duplicate build definitions drift and cause environment-specific production bugs. +- Recommended fix: Standardize on the app-local `frontend/Dockerfile` and update Compose to use it. Fixed in this pass; root duplicate removed. + +## Dead Code + +### Issue 6: One-off issue resolution note in docs + +- Severity: Low +- Location: `docs/issue-498-resolved.md` +- Description: The file documented a closed issue/PR relationship rather than durable product or developer guidance. +- Why it matters: Stale project-management notes clutter published documentation and confuse contributors. +- Recommended fix: Keep issue history in GitHub and remove the note from docs. Fixed in this pass. + +## Unused Files + +### Issue 7: Generated Python bytecode and tool caches were present in the workspace + +- Severity: Low +- Location: `agentwatch/**/__pycache__/`, `tests/__pycache__/`, `.ruff_cache/`, `pytest-cache-files-*` +- Description: Generated caches were present under the repository tree, and several were permission-restricted. +- Why it matters: Generated artifacts slow audits, pollute searches, and can break traversal on Windows. +- Recommended fix: Keep these ignored and remove them after validation runs. Cleaned during this pass; validation tools recreated some caches locally. + +### Issue 8: Coverage artifact deletion was already pending + +- Severity: Low +- Location: `.coverage` +- Description: `.coverage` was already deleted in the working tree before this cleanup started. +- Why it matters: Coverage outputs should not be source-controlled. +- Recommended fix: Keep coverage artifacts ignored and do not commit regenerated coverage files. + +### Issue 9: Root `__main__.py` deletion was already pending + +- Severity: Low +- Location: `__main__.py` +- Description: A root-level `__main__.py` was already deleted before this cleanup started. +- Why it matters: A root entrypoint outside the installed package is ambiguous when `agentwatch/__main__.py` exists. +- Recommended fix: Keep the package entrypoint and avoid root-level executable shims unless they are documented and tested. + +## Legacy Code + +### Issue 10: Historical labels remained in documentation headings + +- Severity: Low +- Location: `docs/architecture_detailed.md`, `docs/custom_adapters_tutorial.md`, `docs/developer_setup.md`, `docs/getting_started_extended.md` +- Description: Several docs included stale event/project suffixes in headings. +- Why it matters: Historical labels make current docs look forked or stale. +- Recommended fix: Use neutral durable document titles. Fixed before/within this cleanup set. + +## Naming Problems + +### Issue 11: Mixed getting-started filename styles + +- Severity: Low +- Location: `docs/getting-started.md`, `docs/getting_started_extended.md`, `docs/cli/getting-started.md` +- Description: Documentation filenames mix kebab-case and snake_case. +- Why it matters: Inconsistent naming makes docs harder to discover and link. +- Recommended fix: Prefer kebab-case for docs pages in a future compatibility-preserving docs rename pass with redirects or link updates. + +## Folder Structure + +### Issue 12: Build output was not ignored + +- Severity: Medium +- Location: `site/`, `.gitignore` +- Description: Running `mkdocs build` generates a root `site/` directory that was not ignored. +- Why it matters: Generated docs output can be accidentally committed. +- Recommended fix: Add `site/` to `.gitignore`. Fixed in this pass. + +## API Consistency + +### Issue 13: API response conventions are not centrally documented + +- Severity: Low +- Location: `agentwatch/api/server.py`, `docs/API-reference.md` +- Description: API routes exist across safety, sessions, policies, telemetry, compliance, and security, but response envelope conventions are not documented in one place. +- Why it matters: Contributors may add inconsistent response shapes. +- Recommended fix: Add a contributor-facing API style section documenting existing conventions before accepting new API work. + +## Documentation + +### Issue 14: Durable docs existed outside MkDocs navigation + +- Severity: Medium +- Location: `mkdocs.yml`, `docs/API-reference.md`, `docs/architecture_detailed.md`, `docs/custom_adapters_tutorial.md`, `docs/developer_setup.md`, `docs/getting_started_extended.md` +- Description: MkDocs built successfully but reported several durable docs pages that were not included in nav. +- Why it matters: Unlisted docs are effectively hidden from published documentation. +- Recommended fix: Add durable pages to MkDocs nav and remove stale notes. Fixed in this pass. + +### Issue 15: Documentation had stale absolute local links + +- Severity: Medium +- Location: `docs/architecture.md` +- Description: Architecture docs previously referenced local `file:///` paths from another machine. +- Why it matters: Absolute local links break for every contributor and in published docs. +- Recommended fix: Use relative links. Fixed before/within this cleanup set. + +### Issue 16: README and package metadata version references must stay synchronized + +- Severity: Low +- Location: `README.md`, `pyproject.toml`, `agentwatch/cli/demo.py` +- Description: Version and Python support references were being updated across multiple files. +- Why it matters: Drift between package metadata, badges, and CLI output undermines release trust. +- Recommended fix: Keep version bumps atomic and consider a small release checklist that lists all version-bearing files. + +## Testing + +### Issue 17: Full test suite is sensitive to local Windows temp permissions + +- Severity: Medium +- Location: `tests/`, local temp path `C:\Users\sreer_mg\AppData\Local\Temp\pytest-of-sreer_mg` +- Description: Many tests using pytest temp fixtures failed locally because pytest could not access its temp root. +- Why it matters: Contributors on Windows can see unrelated fixture setup errors even when code is correct. +- Recommended fix: Document setting a writable temp directory for Windows test runs or configure CI/dev scripts to use a repo-local temp root. + +### Issue 18: Tests ran under Python 3.14 despite project targeting Python 3.12 + +- Severity: Medium +- Location: Local validation environment, `pyproject.toml` +- Description: Local pytest used Python 3.14.2 while the project requires and CI targets Python 3.12. +- Why it matters: Python 3.14 can surface unrelated deprecations or dependency compatibility behavior not representative of CI. +- Recommended fix: Use a Python 3.12 virtual environment for official validation and document this explicitly in developer setup. + +### Issue 19: Sentence-transformers mock is critical test infrastructure + +- Severity: High +- Location: `tests/conftest.py` +- Description: Tests rely on a mock to avoid model downloads and hangs. +- Why it matters: Removing or weakening the mock makes test runs slow and network-dependent. +- Recommended fix: Keep the mock in `conftest.py` and add comments explaining why it exists. Existing behavior preserved. + +## CI/CD + +### Issue 20: CI security audit command used a missing tool and shell-specific process substitution + +- Severity: High +- Location: `.github/workflows/ci.yml` +- Description: The security job attempted `pip-audit --requirement <(pip-compile pyproject.toml ...)` without installing `pip-compile`. +- Why it matters: The job can silently audit an empty requirement stream or behave differently by shell. +- Recommended fix: Install the package and run `pip-audit --desc` directly. Fixed in this pass. + +### Issue 21: PR workflow intentionally does not hard-fail lint + +- Severity: Low +- Location: `.github/workflows/test-on-pr.yml` +- Description: Ruff runs with `continue-on-error: true`; pytest is the hard gate. +- Why it matters: This is documented, but contributors may assume lint failures block merges. +- Recommended fix: Keep behavior if intentional; otherwise make lint a required check after the backlog is clean. + +## Security + +### Issue 22: Security baseline is large and should be periodically reviewed + +- Severity: Medium +- Location: `bandit-baseline.json`, `.github/workflows/bandit.yml` +- Description: Bandit runs with a committed baseline. +- Why it matters: Baselines can hide real regressions if not reviewed and refreshed deliberately. +- Recommended fix: Add a scheduled baseline review process and require justification for new baseline entries. + +### Issue 23: Docker API image runs as root + +- Severity: Medium +- Location: `Dockerfile.api` +- Description: The API image uses `python:3.12-slim` without creating a non-root runtime user. +- Why it matters: Running as root increases container escape blast radius. +- Recommended fix: Add a non-root user and adjust ownership in a focused Docker hardening pass. + +## Packaging + +### Issue 24: `typer[all]` extra is obsolete with current Typer + +- Severity: Medium +- Location: `pyproject.toml` +- Description: Installing the project reported that Typer no longer provides the `all` extra. +- Why it matters: Obsolete extras create install warnings and reduce trust in packaging metadata. +- Recommended fix: Replace `typer[all]` with explicit dependencies actually needed by the CLI, such as `typer`, `rich`, and `shellingham` if required. + +### Issue 25: Docs install caused local dependency resolver conflict + +- Severity: Medium +- Location: `pyproject.toml`, local Python environment +- Description: Installing `.[docs]` pulled `python-dotenv==1.0.1` through `litellm`, conflicting with another installed package requiring `python-dotenv>=1.1.0`. +- Why it matters: Broad unpinned dependencies can make editable installs unstable in shared environments. +- Recommended fix: Validate package extras in isolated Python 3.12 environments and consider dependency upper/lower-bound review. + +## Performance + +### Issue 26: Repository traversal was slowed by generated dependency/build artifacts + +- Severity: Low +- Location: `node_modules/`, `.next/`, `.ruff_cache/`, `site/`, `pytest-cache-files-*` +- Description: Generated artifacts significantly increase scan volume when present. +- Why it matters: Large generated trees slow audits, grep, and contributor tooling. +- Recommended fix: Keep generated outputs ignored and avoid running audits without excluding dependency/build directories. + +## Dependencies + +### Issue 27: Dashboard dependencies include deprecated or vulnerable packages + +- Severity: High +- Location: `frontend/package.json`, `frontend/package-lock.json` +- Description: `npm ci` warnings reported deprecated packages and a security warning for `next@14.2.30`. +- Why it matters: Frontend dependency drift can introduce security exposure. +- Recommended fix: Run a dedicated dashboard dependency update pass, staying within Next.js 14 unless deliberately upgrading the app. + +### Issue 28: Landing dependency installation was not validated + +- Severity: Medium +- Location: `agentwatch-landing/package.json`, `agentwatch-landing/package-lock.json` +- Description: Landing `npm ci` was not completed because dependency installation approval was interrupted and the sandbox cache lacked tarballs. +- Why it matters: Next.js 16 app validity cannot be confirmed without lockfile install and build. +- Recommended fix: Re-run `npm ci && npm run build` in `agentwatch-landing/` with network access in a clean workspace. + +## Build System + +### Issue 29: API Docker context lacked root `.dockerignore` + +- Severity: Medium +- Location: `.dockerignore`, `Dockerfile.api` +- Description: API Docker builds copied the whole repository context without a root Docker ignore file. +- Why it matters: Build contexts can include caches, node modules, docs output, coverage, and local env files. +- Recommended fix: Add a root `.dockerignore`. Fixed in this pass. + +## Type Safety + +### Issue 30: Frontend type-check could not be validated locally + +- Severity: Medium +- Location: `frontend/` +- Description: `npm run type-check` failed because `tsc` was unavailable after `npm ci` could not complete. +- Why it matters: Dashboard type safety remains unverified in this cleanup run. +- Recommended fix: Complete `npm ci` from `frontend/package-lock.json`, then run `npm run type-check`. + +## Logging + +### Issue 31: CLI and demo use direct terminal printing extensively + +- Severity: Low +- Location: `agentwatch/cli/main.py`, `agentwatch/cli/demo.py`, `agentwatch/core/safety.py` +- Description: CLI paths intentionally use Rich/print output while library modules use logging. +- Why it matters: The boundary is acceptable, but direct prints in non-CLI library paths can complicate automation. +- Recommended fix: Keep interactive CLI output in CLI modules; evaluate library print paths for conversion to callbacks or logger-based output. + +## Error Handling + +### Issue 32: Local tool cache permission errors affect validation repeatability + +- Severity: Low +- Location: `.ruff_cache/`, `.pytest_cache/`, `pytest-cache-files-*` +- Description: Ruff and pytest emitted permission warnings when writing caches. +- Why it matters: Cache failures create noisy validation logs and can leave undeletable generated folders. +- Recommended fix: Run validation with `ruff --no-cache` in constrained workspaces and configure pytest cache/temp paths to writable locations. + +## Config + +### Issue 33: Environment variable names differ between Docker and frontend config + +- Severity: Low +- Location: `docker-compose.yml`, `frontend/pages/api/v1/[...path].ts`, `frontend/next.config.js` +- Description: Compose sets both `AGENTWATCH_API_URL` and `NEXT_PUBLIC_API_URL`; frontend runtime proxy behavior depends on server-side env. +- Why it matters: Contributors may set the wrong variable when debugging frontend API proxying. +- Recommended fix: Document frontend runtime and build-time environment variables in one dashboard setup section. + +## Repository Hygiene + +### Issue 34: `VISION.md` is untracked + +- Severity: Medium +- Location: `VISION.md` +- Description: `VISION.md` exists in the workspace but is not tracked. +- Why it matters: Product vision docs should be versioned if they are authoritative; otherwise they should not live in the repo root. +- Recommended fix: Decide whether to commit it as canonical vision documentation or remove it from the workspace. + +### Issue 35: Working tree was dirty before cleanup + +- Severity: Medium +- Location: Multiple files +- Description: Several docs, benchmarks, tests, version metadata, `.coverage`, and root `__main__.py` were already modified/deleted before this pass. +- Why it matters: Broad cleanup work is harder to review when mixed with pre-existing changes. +- Recommended fix: Split commits by intent: pre-existing version/doc updates, cleanup changes, test hygiene, and generated artifact removal. + +### Issue 36: Line-ending churn warnings appear on Windows + +- Severity: Low +- Location: Multiple text files +- Description: Git warned that LF will be replaced by CRLF the next time Git touches several files. +- Why it matters: Uncontrolled line-ending churn creates noisy diffs. +- Recommended fix: Add or verify `.gitattributes` with normalized text handling. diff --git a/mkdocs.yml b/mkdocs.yml index d64649e7..1ec64dcd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -30,8 +30,13 @@ nav: - API Reference: cli/api.md - Guides: - Getting Started (project): getting-started.md + - Extended Getting Started: getting_started_extended.md + - Developer Setup: developer_setup.md - Architecture: architecture.md + - Detailed Architecture: architecture_detailed.md + - API Reference: API-reference.md - Troubleshooting: TROUBLESHOOTING.md - Adapters: + - Custom Adapters: custom_adapters_tutorial.md - Claude Code: adapters/claude-code.md - LangChain: adapters/langchain.md diff --git a/pyproject.toml b/pyproject.toml index 4abb4a8e..80993f66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agentwatch-ai" -version = "0.1.0" +version = "0.2.0" description = "The Reliability, Safety, and Observability Layer for AI Agents" readme = "README.md" license = { text = "Apache-2.0" } @@ -130,4 +130,4 @@ ignore = [ [tool.mypy] python_version = "3.12" strict = false -ignore_missing_imports = true \ No newline at end of file +ignore_missing_imports = true diff --git a/scripts/owasp_test_harness.py b/scripts/owasp_test_harness.py index 0daa051d..22805f8e 100644 --- a/scripts/owasp_test_harness.py +++ b/scripts/owasp_test_harness.py @@ -265,12 +265,8 @@ def main() -> int: parser = argparse.ArgumentParser( description="OWASP Agentic Top 10 automated test harness for AgentWatch" ) - parser.add_argument( - "--json", action="store_true", help="Output JSON report instead of text" - ) - parser.add_argument( - "--fail-fast", action="store_true", help="Stop on first critical finding" - ) + parser.add_argument("--json", action="store_true", help="Output JSON report instead of text") + parser.add_argument("--fail-fast", action="store_true", help="Stop on first critical finding") args = parser.parse_args() result = run_harness(fail_fast=args.fail_fast) diff --git a/tests/conftest.py b/tests/conftest.py index aa95196b..7ba066dd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,6 @@ import datetime -# Mock EmbeddingProvider._load to prevent downloading SentenceTransformer weights in tests import agentwatch.memory.engine - -# Mock sentence-transformers to bypass heavy model loading and downloads in tests import agentwatch.scoring.drift if not hasattr(datetime, "UTC"): @@ -11,10 +8,6 @@ agentwatch.scoring.drift._st_model = agentwatch.scoring.drift._ST_UNAVAILABLE -# Mock EmbeddingProvider._load to prevent downloading SentenceTransformer weights in tests -import agentwatch.memory.engine # noqa: E402 - - async def mock_load(self): self._disabled = True diff --git a/tests/test_cli_share.py b/tests/test_cli_share.py index d04a53c0..b284fa45 100644 --- a/tests/test_cli_share.py +++ b/tests/test_cli_share.py @@ -80,9 +80,7 @@ def test_share_dry_run_does_not_upload(trace_file, mock_post): def test_share_custom_url(trace_file, mock_post): mock_instance, _ = mock_post - result = runner.invoke( - app, ["share", str(trace_file), "--share-url", "https://my.host/"] - ) + result = runner.invoke(app, ["share", str(trace_file), "--share-url", "https://my.host/"]) assert result.exit_code == 0 assert mock_instance.post.call_args[0][0] == "https://my.host/api/v1/share" diff --git a/tests/test_db_failure.py b/tests/test_db_failure.py index 9a61c35a..7d81c812 100644 --- a/tests/test_db_failure.py +++ b/tests/test_db_failure.py @@ -24,7 +24,6 @@ def test_health_check_reports_db_status(client): assert response.status_code == expected_http - def test_system_status_endpoint(client): # Protect with API key if needed, but in local dev it's usually None response = client.get("/api/v1/system/status") diff --git a/tests/test_platform.py b/tests/test_platform.py index 6412f8cd..62eab95f 100644 --- a/tests/test_platform.py +++ b/tests/test_platform.py @@ -85,9 +85,7 @@ def test_sanitize_trace_strips_secret_keys_at_any_depth(): def test_sanitize_trace_masks_free_text_pii(): payload = { - "events": [ - {"tool_call": {"raw_command": "mail bob@acme.com about SSN 123-45-6789"}} - ] + "events": [{"tool_call": {"raw_command": "mail bob@acme.com about SSN 123-45-6789"}}] } cmd = sanitize_trace(payload)["events"][0]["tool_call"]["raw_command"] assert "bob@acme.com" not in cmd diff --git a/tests/test_router.py b/tests/test_router.py index e5932a69..f19ab40a 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -58,6 +58,7 @@ def test_circuit_breaker_transitions(mock_sleep, mock_config): mock_completion.side_effect = Exception("API error") import contextlib + for _ in range(5): with contextlib.suppress(Exception): router.completion([{"role": "user", "content": "hi"}]) diff --git a/tests/test_semantic_caching.py b/tests/test_semantic_caching.py index c488a5d3..cf0f84a4 100644 --- a/tests/test_semantic_caching.py +++ b/tests/test_semantic_caching.py @@ -11,18 +11,19 @@ async def test_semantic_cache_exact_match(monkeypatch): async def mock_embed(self, texts): return [[1.0, 0.0] for _ in texts] - + monkeypatch.setattr(EmbeddingProvider, "embed", mock_embed) - + cache = SemanticCache() prompt = "What is the capital of France?" response = "Paris" - + await cache.set(prompt, response) - + hit = await cache.get(prompt) assert hit == response + @pytest.mark.asyncio async def test_semantic_cache_fuzzy_match(monkeypatch): async def mock_embed(self, texts): @@ -35,15 +36,17 @@ async def mock_embed(self, texts): else: res.append([0.0, 1.0]) return res + monkeypatch.setattr(EmbeddingProvider, "embed", mock_embed) cache = SemanticCache(similarity_threshold=0.90) - + await cache.set("write a python script", "print('hello')") hit = await cache.get("write a python code") - + assert hit == "print('hello')" + @pytest.mark.asyncio async def test_semantic_cache_miss(monkeypatch): async def mock_embed(self, texts): @@ -54,74 +57,81 @@ async def mock_embed(self, texts): else: res.append([0.0, 1.0]) return res + monkeypatch.setattr(EmbeddingProvider, "embed", mock_embed) - + cache = SemanticCache(similarity_threshold=0.90) await cache.set("hello", "hi") hit = await cache.get("goodbye") - + assert hit is None + @pytest.mark.asyncio async def test_semantic_cache_ttl(monkeypatch): async def mock_embed(self, texts): return [[1.0, 0.0] for _ in texts] - + monkeypatch.setattr(EmbeddingProvider, "embed", mock_embed) - + cache = SemanticCache(ttl_days=1) await cache.set("test", "test response") - + # Force expiration by rewriting created_at cache._cache[0].created_at = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=2) - + hit = await cache.get("test") assert hit is None + @pytest.mark.asyncio async def test_adapter_base_hooks(monkeypatch): async def mock_embed(self, texts): return [[1.0, 0.0] for _ in texts] + monkeypatch.setattr(EmbeddingProvider, "embed", mock_embed) - + cache = SemanticCache() set_semantic_cache(cache) - + # Store response await after_provider_call("What is the largest ocean?", "Pacific Ocean") - + # Check cache hit hit = await before_provider_call("What is the largest ocean?") assert hit == "Pacific Ocean" - + # Clean up set_semantic_cache(None) + @pytest.mark.asyncio async def test_cache_hit_skips_provider_call(monkeypatch): """ Verify that a cached response is returned without incurring downstream LLM API latency or cost. """ + async def mock_embed(self, texts): return [[1.0, 0.0] for _ in texts] + monkeypatch.setattr(EmbeddingProvider, "embed", mock_embed) - + cache = SemanticCache() set_semantic_cache(cache) - + # Pre-warm cache await after_provider_call("What is the largest ocean?", "Pacific Ocean") - + provider_called = False - + # Simulate an adapter flow hit = await before_provider_call("What is the largest ocean?") if not hit: provider_called = True # Real provider call would go here - + assert hit == "Pacific Ocean" assert provider_called is False - + # Clean up set_semantic_cache(None) diff --git a/tests/test_server.py b/tests/test_server.py index 7a35d1da..e21398d7 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -64,7 +64,8 @@ def test_health_healthy_db_and_redis(client, mock_db_session, mock_redis_healthy def test_health_db_unavailable(client, mock_redis_healthy): with patch( - "agentwatch.api.server._db_session_factory", side_effect=Exception("Database is Unavailable") + "agentwatch.api.server._db_session_factory", + side_effect=Exception("Database is Unavailable"), ): response = client.get("/health") @@ -84,7 +85,7 @@ def test_health_redis_unavailable(client, mock_db_session): mock_client = AsyncMock() mock_client.ping = AsyncMock(side_effect=Exception("Redis is Unavailable")) mock_client.aclose = AsyncMock() - + mock_redis.return_value = mock_client response = client.get("/health") diff --git a/tests/test_sync_safety.py b/tests/test_sync_safety.py index 7c8e98e5..c7e53a52 100644 --- a/tests/test_sync_safety.py +++ b/tests/test_sync_safety.py @@ -68,9 +68,11 @@ def test_sync_vs_async_parity(): assert (len(reasons_sync) > 0) == blocked_sync, ( "Metadata discrepancy: blocked is True but no reasons provided" ) + + def test_tool_parameter_scope_constraints(): """ - Test feature [ELUSOC] from Issue #364: + Test feature [ELUSOC] from Issue #364: Ensures policy DSL blocks tool calls if input arguments violate threshold parameters. """ # 1. Define a YAML rule that restricts high-value transfers @@ -87,24 +89,23 @@ def test_tool_parameter_scope_constraints(): allowed_tool_call = ToolCallData( tool_name="request_funds", raw_command="", - arguments={"amount": 250} # Natively parsed by policy_dsl via args.amount + arguments={"amount": 250}, # Natively parsed by policy_dsl via args.amount ) blocked_allow, reasons_allow = safety_engine.check_tool_call_sync(allowed_tool_call) - + assert blocked_allow is False assert len(reasons_allow) == 0 # 3. Test outside safe parameters (amount = 600, blocked) blocked_tool_call = ToolCallData( - tool_name="request_funds", - raw_command="", - arguments={"amount": 600} + tool_name="request_funds", raw_command="", arguments={"amount": 600} ) blocked_deny, reasons_deny = safety_engine.check_tool_call_sync(blocked_tool_call) assert blocked_deny is True assert "rule_matched:high_value_funds_restriction" in reasons_deny + if __name__ == "__main__": test_sync_vs_async_parity() test_tool_parameter_scope_constraints() diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 8e3e4700..28e4cbce 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -119,6 +119,7 @@ def test_export_gives_up_after_max_retries(): assert exporter.export.call_count == 3 +@pytest.mark.skipif(not _OTEL_AVAILABLE, reason="requires opentelemetry-sdk") def test_export_reasoning_trace_success(): provider = TelemetryProvider() provider.initialize() @@ -160,6 +161,7 @@ def test_export_reasoning_trace_success(): mock_span.set_attribute.assert_any_call("custom", "val") +@pytest.mark.skipif(not _OTEL_AVAILABLE, reason="requires opentelemetry-sdk") def test_export_reasoning_trace_hierarchy(): provider = TelemetryProvider() provider.initialize() @@ -190,6 +192,7 @@ def test_export_reasoning_trace_hierarchy(): # we just need to ensure the call count is 2 and no exceptions occurred +@pytest.mark.skipif(not _OTEL_AVAILABLE, reason="requires opentelemetry-sdk") def test_export_reasoning_trace_malformed_ids(): provider = TelemetryProvider() provider.initialize() @@ -214,6 +217,7 @@ def test_export_reasoning_trace_malformed_ids(): mock_span.set_attribute.assert_any_call("agentwatch.span_id", "also-not-a-uuid") +@pytest.mark.skipif(not _OTEL_AVAILABLE, reason="requires opentelemetry-sdk") def test_export_reasoning_trace_missing_end_time(): provider = TelemetryProvider() provider.initialize()