diff --git a/docs/sdk/quickstart.md b/docs/sdk/quickstart.md deleted file mode 100644 index fd4e069..0000000 --- a/docs/sdk/quickstart.md +++ /dev/null @@ -1,67 +0,0 @@ -# Quickstart - -!!! note "Prerequisites" - - Python 3.11+ - - Install the package: `pip install openagent-eval` - - A compatible LLM provider (e.g., OpenAI) and any required credentials are available as environment variables. - -The following example shows the smallest possible workflow: create a configuration, run the evaluation engine on a tiny in‑memory dataset, and print the report summary. - -```python -"""Minimal OpenAgent Eval example. -Runs an async engine with a single question and prints the summary. -""" - -import asyncio -from pathlib import Path - -from openagent_eval.config.models import Config, LLMConfig, RetrieverConfig, MetricsConfig, DatasetConfig -from openagent_eval.core.engine import Engine - -# 1️⃣ Build a configuration -config = Config( - dataset=DatasetConfig(path="data/sample.json", limit=1), - llm=LLMConfig(provider="openai", model="gpt-4o-mini", temperature=0.0), - retriever=RetrieverConfig(provider="mock"), # simple mock retriever - metrics=MetricsConfig( - retrieval=[], - generation=["faithfulness"], - performance=[], - cost=[], - ), -) - -# 2️⃣ Create the engine -engine = Engine(config) - -# 3️⃣ Define a tiny dataset inline (overrides the path above) -# The engine expects a list of dicts matching `DatasetItemModel`. -dataset = [ - { - "question": "What is the capital of France?", - "ground_truth": "Paris", - "metadata": {}, - } -] - -# 4️⃣ Run the evaluation (async) -async def main(): - report = await engine.run(dataset) - # The report contains a handy `summary` dict – print the most useful bits. - print("✅ Evaluation complete") - print("Total items:", report.summary["total_items"]) - print("Successful:", report.summary["successful_evaluations"]) - print("Metrics summary:", report.summary.get("metrics_summary", {})) - # Optionally persist the report using ReportManager (see next docs page). - -asyncio.run(main()) -``` - -**What you just saw** -- `Config` ties together dataset, LLM, retriever, and metrics. -- `Engine` orchestrates provider construction, pipeline execution, and report generation. -- The example uses the built‑in `mock` retriever so no external service is needed. - -Next, dive deeper into the mental model that powers this SDK. - -[Next: Concepts →](concepts.md) diff --git a/openagent_eval/__init__.py b/openagent_eval/__init__.py index 906abb7..c2aba67 100644 --- a/openagent_eval/__init__.py +++ b/openagent_eval/__init__.py @@ -1,4 +1,4 @@ """OpenAgent Eval - Open-source CLI framework for evaluating RAG systems and AI Agents.""" -__version__ = "0.4.5" +__version__ = "0.4.6" __author__ = "OpenAgentHQ" diff --git a/pyproject.toml b/pyproject.toml index 480108e..df022b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openagent-eval" -version = "0.4.5" +version = "0.4.6" description = "Open-source CLI framework for evaluating RAG systems and AI Agents" readme = "README.md" license = {text = "Apache-2.0"}