diff --git a/.gitignore b/.gitignore index fa29d07..30a6d42 100644 --- a/.gitignore +++ b/.gitignore @@ -221,3 +221,6 @@ __marimo__/ .opencode/ .agents/ .tmp/ + +# Release guide (local-only) +Release_Guide.md diff --git a/docs/sdk/quickstart.md b/docs/sdk/quickstart.md new file mode 100644 index 0000000..fd4e069 --- /dev/null +++ b/docs/sdk/quickstart.md @@ -0,0 +1,67 @@ +# Quickstart + +!!! note "Prerequisites" + - Python 3.11+ + - Install the package: `pip install openagent-eval` + - A compatible LLM provider (e.g., OpenAI) and any required credentials are available as environment variables. + +The following example shows the smallest possible workflow: create a configuration, run the evaluation engine on a tiny in‑memory dataset, and print the report summary. + +```python +"""Minimal OpenAgent Eval example. +Runs an async engine with a single question and prints the summary. +""" + +import asyncio +from pathlib import Path + +from openagent_eval.config.models import Config, LLMConfig, RetrieverConfig, MetricsConfig, DatasetConfig +from openagent_eval.core.engine import Engine + +# 1️⃣ Build a configuration +config = Config( + dataset=DatasetConfig(path="data/sample.json", limit=1), + llm=LLMConfig(provider="openai", model="gpt-4o-mini", temperature=0.0), + retriever=RetrieverConfig(provider="mock"), # simple mock retriever + metrics=MetricsConfig( + retrieval=[], + generation=["faithfulness"], + performance=[], + cost=[], + ), +) + +# 2️⃣ Create the engine +engine = Engine(config) + +# 3️⃣ Define a tiny dataset inline (overrides the path above) +# The engine expects a list of dicts matching `DatasetItemModel`. +dataset = [ + { + "question": "What is the capital of France?", + "ground_truth": "Paris", + "metadata": {}, + } +] + +# 4️⃣ Run the evaluation (async) +async def main(): + report = await engine.run(dataset) + # The report contains a handy `summary` dict – print the most useful bits. + print("✅ Evaluation complete") + print("Total items:", report.summary["total_items"]) + print("Successful:", report.summary["successful_evaluations"]) + print("Metrics summary:", report.summary.get("metrics_summary", {})) + # Optionally persist the report using ReportManager (see next docs page). + +asyncio.run(main()) +``` + +**What you just saw** +- `Config` ties together dataset, LLM, retriever, and metrics. +- `Engine` orchestrates provider construction, pipeline execution, and report generation. +- The example uses the built‑in `mock` retriever so no external service is needed. + +Next, dive deeper into the mental model that powers this SDK. + +[Next: Concepts →](concepts.md) diff --git a/openagent_eval/__init__.py b/openagent_eval/__init__.py index cffca59..906abb7 100644 --- a/openagent_eval/__init__.py +++ b/openagent_eval/__init__.py @@ -1,4 +1,4 @@ """OpenAgent Eval - Open-source CLI framework for evaluating RAG systems and AI Agents.""" __version__ = "0.4.5" -__author__ = "OpenAgent Eval Contributors" +__author__ = "OpenAgentHQ" diff --git a/pyproject.toml b/pyproject.toml index d17e8f1..480108e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,10 @@ readme = "README.md" license = {text = "Apache-2.0"} requires-python = ">=3.11" authors = [ - {name = "OpenAgent Eval Contributors"} + {name = "Himanshu Kumar"} +] +Maintainers = [ + {name = "OpenAgentHQ Team", email = "openagenthq@gmail.com"} ] keywords = ["rag", "evaluation", "ai", "llm", "metrics", "cli", "nli", "corpus-health", "diagnosis", "synthetic-data"] classifiers = [