diff --git a/.gitignore b/.gitignore index 70cf721..ff83fbb 100644 --- a/.gitignore +++ b/.gitignore @@ -227,6 +227,7 @@ Release_Guide.md # Task tracking (local-only) /05_TASKS.md +DECISIONS.md # Local documentation artifacts openai-docs.md @@ -234,3 +235,8 @@ tiktoken-docs.md # Local data directory /data/ + +# Local config files (contains local settings, use env vars for secrets) +config.yaml +config.groq.yaml +config.*.yaml diff --git a/QUICKSTART.md b/QUICKSTART.md new file mode 100644 index 0000000..c15c151 --- /dev/null +++ b/QUICKSTART.md @@ -0,0 +1,112 @@ +# QUICKSTART.md — Coding Agent Quick Reference + +> Get productive in <60 seconds. + +--- + +## Identity + +| Field | Value | +|-------|-------| +| Repo | `openagent-eval` | +| Package | `openagent_eval` | +| CLI | `oaeval` | + +--- + +## First Commands + +```bash +uv sync --group dev # Install deps +pytest # Run tests +ruff check . # Lint +``` + +--- + +## How to Run + +```bash +oaeval run config.yaml # Run evaluation +oaeval audit --corpus ./docs # Audit corpus +oaeval diagnose --report r.json # Blame attribution +oaeval synth --corpus ./docs # Generate test data +oaeval test config.yaml -t faithfulness:gte:0.8 # CI/CD +``` + +--- + +## Where to Put Code + +| What | Where | +|------|-------| +| New metric | `metrics/{category}/new_metric.py` | +| New LLM | `providers/llm/new_provider.py` | +| New retriever | `providers/retrievers/new_retriever.py` | +| New command | `cli/commands/new_command.py` | + +--- + +## Interfaces + +```python +# Metric +from openagent_eval.metrics.base import BaseMetric, MetricResult +class MyMetric(BaseMetric): + name = "my_metric" + def evaluate(self, **kwargs) -> MetricResult: + return MetricResult(score=0.95, reason="Because...", metadata={}) + +# LLM Provider +from openagent_eval.providers.base.llm import LLMProvider +class MyLLM(LLMProvider): + async def generate(self, prompt: str, **kwargs) -> str: + return "response" + async def get_token_count(self, text: str) -> int: + return len(text.split()) + +# Retriever +from openagent_eval.providers.base.retriever import Retriever +from openagent_eval.providers.models import Document +class MyRetriever(Retriever): + async def retrieve(self, query: str, k: int = 5) -> list[Document]: + return [Document(content="...", metadata={}, score=0.9)] +``` + +--- + +## Testing & Git + +```bash +pytest # All tests +pytest tests/unit/test_metrics/ # Module +pytest -k "test_faithfulness" # One test +``` + +- **Never** develop on `main` +- Branch: `feature/{desc}`, `fix/{desc}`, `docs/{desc}` +- Push → PR → user reviews + +--- + +## Context Files + +| File | Purpose | +|------|---------| +| `PROJECT.md` | What we're building | +| `ARCHITECTURE.md` | How it's built | +| `AGENT.md` | Coding rules | +| `DECISIONS.md` | Why we chose this | +| `CONTEXT.md` | Current state | +| `INSTRUCTIONS.md` | Writing rules | + +--- + +## Rules + +1. Never develop on `main` +2. Never raise generic `Exception` +3. Never put business logic in CLI +4. Always use typed exceptions +5. Always implement ABCs +6. Always create PR for review diff --git a/config.groq.yaml b/config.groq.yaml deleted file mode 100644 index 626a96e..0000000 --- a/config.groq.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# OpenAgent Eval Configuration - Groq + Mock Retriever -# Uses Groq's Llama 3.3 70B for generation, mock retriever for offline testing - -dataset: - path: data/sample_questions.json - -llm: - provider: groq - model: llama-3.3-70b-versatile - temperature: 0.0 - -retriever: - provider: mock - -metrics: - retrieval: - - context_precision - - context_recall - - mrr - generation: - - faithfulness - - answer_relevancy - - similarity - performance: - - latency - cost: - - token_count - -report: - output: terminal - output_dir: ./reports - -verbose: true -parallel: false -max_workers: 1 diff --git a/config.yaml b/config.yaml deleted file mode 100644 index bd9fcd5..0000000 --- a/config.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# OpenAgent Eval Configuration -# See documentation for options: https://github.com/OpenAgentHQ/openagent-eval - -dataset: - path: data/sample_questions.json - format: json - # limit: 100 - -llm: - provider: openai - model: gpt-4o-mini - temperature: 0.0 - -retrieval: - # provider: chromadb - # collection_name: my_collection - -evaluation: - metrics: - - answer_relevancy - - faithfulness - - context_precision - - context_recall - -report: - output: terminal - output_dir: ./reports