Add RAGAS DeepEval and Python OTLP hooks

README.md

@@ -15,12 +15,12 @@ content by default.
 
 - OpenTelemetry GenAI spans: `gen_ai.chat`, `gen_ai.embeddings`, `gen_ai.execute_tool`, `gen_ai.agent`, and `gen_ai.workflow`
 - Provider adapters for OpenAI chat, OpenAI-compatible APIs, Anthropic, Cohere, AWS Bedrock, Google Vertex, and Ollama
-- Promptfoo result conversion with run, case, dataset, score, assertion, provenance, and evidence metadata
+- Framework adapters for Promptfoo, RAGAS, and DeepEval with run, case, dataset, score, provenance, and evidence metadata
 - RAG scoring for context precision, recall, faithfulness, MRR, NDCG, citation coverage, top-k relevance, and context-token use
 - Privacy controls for opt-in content capture, redaction-to-string, redaction-to-fingerprint, truncation flags, and event caps
 - A versioned `eval2otel.v1` contract backed by conformance fixtures
 - Operational telemetry about Eval2Otel itself: conversion count, warnings, dropped events, redactions, truncations, and duration
-- A Python contract scaffold for teams that want the same Eval2Otel payload shape outside TypeScript
+- A Python SDK preview with optional OTLP spans, content events, PII redaction, and provider instrumentation hooks
 
 ## Install
 
@@ -164,12 +164,16 @@ Supported adapter modes:
 Every adapter result includes structured warnings and raw payload evidence
 hashes, so conversion failures can be reported without dumping raw payloads.
 
-## Promptfoo Adapter
+## Framework Adapters
 
 Promptfoo results can be converted directly into Eval2Otel results:
 
 ```ts
-import { convertPromptfooToEvalResults } from 'eval2otel';
+import {
+  convertDeepEvalToEvalResults,
+  convertPromptfooToEvalResults,
+  convertRagasToEvalResults,
+} from 'eval2otel';
 
 const { evalResults, warnings } = convertPromptfooToEvalResults(promptfooJson, {
   runId: 'promptfoo-nightly',
@@ -191,6 +195,31 @@ The adapter preserves Promptfoo success, score, assertion counts, failed
 assertion warnings, metric names, run identity, case identity, and payload
 hashes.
 
+RAGAS and DeepEval exports use the same conversion shape:
+
+```ts
+const ragas = convertRagasToEvalResults(ragasJson, {
+  runId: 'ragas-nightly',
+  datasetId: 'rag-evals',
+  defaultModel: 'gpt-4o-mini',
+});
+
+const deepeval = convertDeepEvalToEvalResults(deepevalJson, {
+  runId: 'deepeval-nightly',
+  includeExplanations: true,
+  defaultModel: 'gpt-4o-mini',
+});
+
+for (const result of [...ragas.evalResults, ...deepeval.evalResults]) {
+  eval2otel.processEvaluation(result);
+}
+```
+
+RAGAS rows populate RAG metrics such as context precision, context recall,
+answer relevance, and faithfulness. DeepEval rows preserve metric scores,
+failed-metric warnings, expected-output fingerprints, and retrieval context as
+RAG chunk evidence.
+
 ## RAG Telemetry
 
 RAG payloads can include retrieval inputs, chunk metadata, explicit eval scores,
@@ -338,23 +367,25 @@ Each provider-native line should look like:
 {"startTime":1725170000000,"endTime":1725170001200,"request":{"model":"gpt-4o-mini","messages":[{"role":"user","content":"hi"}]},"response":{"id":"chatcmpl-1","object":"chat.completion","model":"gpt-4o-mini","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}}
 ```
 
-## Python Scaffold
+## Python SDK Preview
 
-The TypeScript package is the production emitter today. The Python directory
-contains a small contract-first scaffold with the same `eval2otel.v1`
-provenance/evidence/report vocabulary:
+The Python package mirrors the `eval2otel.v1` provenance, evidence, and
+conversion-report vocabulary. It can run contract-only with no OpenTelemetry
+dependency, or emit real spans when the optional OTel extras are installed:
 
 ```bash
+pip install -e "python[otel]"
 PYTHONPATH=python python3 -m unittest discover -s python/tests
 ```
 
 ```python
-from eval2otel import Eval2Otel
+from eval2otel import instrument_all
 
-client = Eval2Otel(service_name="pytest", semconv_version="1.37.0")
+client = instrument_all()
 report = client.process_evaluation({
     "id": "py-case-1",
     "model": "gpt-4o-mini",
+    "system": "openai",
     "operation": "chat",
     "request": {"model": "gpt-4o-mini"},
     "response": {},
@@ -364,6 +395,18 @@ report = client.process_evaluation({
 assert report.contract_version == "eval2otel.v1"
 ```
 
+`instrument_all()` honors the common OTLP environment variables plus:
+
+- `EVAL2OTEL_SERVICE_NAME`
+- `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`
+- `EVAL2OTEL_SAMPLE_RATE`
+- `EVAL2OTEL_REDACT_PII`
+- `EVAL2OTEL_PROVIDERS`
+
+Provider hooks are optional. If provider packages and compatible
+OpenLLMetry/OpenInference instrumentors are installed, Eval2Otel invokes them;
+otherwise it returns structured handles explaining what was available.
+
 See [python/README.md](./python/README.md).
 
 ## Configuration

docs/contract/eval2otel-v1.md

@@ -43,6 +43,7 @@ attributes intentionally aligned with:
   exists yet;
 - Eval2Otel-owned `evalops.*` contract, privacy, provenance, and evidence
   attributes;
+- framework-specific `eval.*` diagnostic attributes;
 - provider-prefixed diagnostic attributes such as `openai.*`,
   `anthropic.*`, and `google.vertex.*`.
 
@@ -131,8 +132,15 @@ Provider-native adapters should return a `ProviderConversionResult` with:
 - structured warnings;
 - evidence containing at least `rawPayloadSha256`.
 
-Promptfoo conversion additionally emits `eval.promptfoo.*` attributes for pass
-state, score, assertion counts, failed assertion counts, and metric names.
+Framework adapters additionally emit namespaced `eval.*` attributes:
+
+- Promptfoo: `eval.promptfoo.*` for pass state, score, assertion counts, failed
+  assertion counts, and metric names.
+- RAGAS: `eval.ragas.*` for source metric values, metric names, and reference
+  fingerprints. Shared RAG quality fields are also copied into
+  `EvalResult.rag.metrics`.
+- DeepEval: `eval.deepeval.*` for pass state, failed metric counts, normalized
+  metric scores, metric names, and expected-output fingerprints.
 
 ## Operational Telemetry
 

python/README.md

@@ -1,23 +1,34 @@
-# eval2otel Python Scaffold
+# eval2otel Python SDK Preview
 
-This directory is a contract-first Python SDK scaffold. It mirrors the
-`eval2otel.v1` provenance, evidence, and conversion-report vocabulary used by
-the TypeScript package without adding a Python OpenTelemetry dependency yet.
+The Python package mirrors the TypeScript `eval2otel.v1` contract and can also
+emit OpenTelemetry spans when the optional OTel extras are installed. Without
+those extras, it still validates Eval2Otel payloads and returns conversion
+reports.
+
+```bash
+pip install -e ".[otel]"
+```
 
 ```python
-from eval2otel import Eval2Otel
+from eval2otel import instrument_all
 
-client = Eval2Otel(service_name="eval-runner", semconv_version="1.37.0")
+client = instrument_all()
 report = client.process_evaluation({
     "id": "case-1",
     "timestamp": 1700000000000,
     "model": "gpt-4o-mini",
     "system": "openai",
     "operation": "chat",
     "request": {"model": "gpt-4o-mini"},
-    "response": {},
-    "usage": {},
+    "response": {"model": "gpt-4o-mini"},
+    "usage": {"inputTokens": 12, "outputTokens": 8},
     "performance": {"duration": 0.25},
+    "conversation": {
+        "messages": [
+            {"role": "user", "content": "What shipped?"},
+            {"role": "assistant", "content": "Eval2Otel Python OTLP hooks shipped."}
+        ]
+    },
     "provenance": {
         "sourceFramework": "deepeval",
         "runId": "nightly",
@@ -26,7 +37,50 @@ report = client.process_evaluation({
 })
 
 assert report.contract_version == "eval2otel.v1"
+client.shutdown()
 ```
 
-The intended next step is a real OTLP/OpenTelemetry emitter behind this same
-API, plus framework adapters for Python-native eval runners.
+## Environment
+
+`instrument_all()` reads:
+
+- `OTEL_SERVICE_NAME` or `EVAL2OTEL_SERVICE_NAME`
+- `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` or `OTEL_EXPORTER_OTLP_ENDPOINT`
+- `OTEL_EXPORTER_OTLP_PROTOCOL`
+- `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`
+- `EVAL2OTEL_SAMPLE_RATE`
+- `EVAL2OTEL_REDACT_PII`
+- `EVAL2OTEL_PROVIDERS`
+
+Content capture is off by default. When
+`OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true`, message content is
+emitted as span events and sampled by `EVAL2OTEL_SAMPLE_RATE`. When
+`EVAL2OTEL_REDACT_PII=true`, the built-in redactor masks common emails,
+bearer tokens, secret assignments, and long number sequences before content is
+emitted.
+
+## Provider Hooks
+
+`instrument_all()` returns `client.instrumentation_handles` when provider
+patching is enabled. Each handle reports whether the provider package was
+available, whether a compatible instrumentor was invoked, and the reason when it
+could not be instrumented.
+
+Supported provider names:
+
+- `openai`
+- `anthropic`
+- `google-generativeai`
+- `bedrock`
+- `cohere`
+- `huggingface`
+
+Set `EVAL2OTEL_PROVIDERS=openai,anthropic` to limit discovery.
+
+## Development
+
+From the repository root:
+
+```bash
+PYTHONPATH=python python3 -m unittest discover -s python/tests
+```

python/eval2otel/__init__.py

@@ -9,8 +9,20 @@
     Eval2OtelProvenance,
     EvalResult,
     build_eval2otel_attributes,
+    build_span_attributes,
     sha256_payload,
 )
+from .instrumentations import (
+    ProviderInstrumentationHandle,
+    instrument_all_providers,
+    instrument_anthropic,
+    instrument_bedrock,
+    instrument_cohere,
+    instrument_google_generativeai,
+    instrument_huggingface,
+    instrument_openai,
+)
+from .privacy import redact_pii
 
 __all__ = [
     "EVAL2OTEL_CONTRACT_VERSION",
@@ -22,6 +34,16 @@
     "Eval2OtelProvenance",
     "EvalResult",
     "build_eval2otel_attributes",
+    "build_span_attributes",
     "instrument_all",
+    "instrument_all_providers",
+    "instrument_anthropic",
+    "instrument_bedrock",
+    "instrument_cohere",
+    "instrument_google_generativeai",
+    "instrument_huggingface",
+    "instrument_openai",
+    "ProviderInstrumentationHandle",
+    "redact_pii",
     "sha256_payload",
 ]

python/eval2otel/auto.py

@@ -1,22 +1,68 @@
 from __future__ import annotations
 
+import os
+from typing import Callable, Iterable
+
 from .contract import Eval2Otel
+from .instrumentations import instrument_all_providers
+from .privacy import redact_pii
 
 
 def instrument_all(
-    service_name: str,
+    service_name: str | None = None,
     service_version: str | None = None,
     semconv_version: str = "unspecified",
+    capture_content: bool | None = None,
+    sample_content_rate: float | None = None,
+    endpoint: str | None = None,
+    exporter_protocol: str | None = None,
+    patch_providers: bool = True,
+    providers: Iterable[str] | None = None,
+    redact: Callable[[str], str | None] | None = None,
 ) -> Eval2Otel:
-    """Create the contract-first client.
-
-    The scaffold deliberately has no OpenTelemetry dependency yet. The API gives
-    Python adopters a stable construction point while the emitter grows behind
-    the same contract.
-    """
+    """Create an Eval2Otel client from explicit args and OTEL/EVAL2OTEL env."""
 
-    return Eval2Otel(
-        service_name=service_name,
+    selected_providers = tuple(providers) if providers is not None else _env_list("EVAL2OTEL_PROVIDERS")
+    client = Eval2Otel(
+        service_name=service_name or os.getenv("OTEL_SERVICE_NAME") or os.getenv("EVAL2OTEL_SERVICE_NAME") or "eval2otel-python",
         service_version=service_version,
         semconv_version=semconv_version,
+        capture_content=_env_bool("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", default=False)
+        if capture_content is None else capture_content,
+        sample_content_rate=_clamp_sample_rate(_env_float("EVAL2OTEL_SAMPLE_RATE", default=1.0))
+        if sample_content_rate is None else sample_content_rate,
+        endpoint=endpoint or os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") or os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"),
+        exporter_protocol=exporter_protocol or os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL"),
+        redact=redact if redact is not None else redact_pii if _env_bool("EVAL2OTEL_REDACT_PII", default=False) else None,
     )
+    if patch_providers:
+        setattr(client, "instrumentation_handles", instrument_all_providers(selected_providers or None))
+    return client
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    return value.lower() in {"1", "true", "yes", "on"}
+
+
+def _env_float(name: str, default: float) -> float:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    try:
+        return float(value)
+    except ValueError:
+        return default
+
+
+def _env_list(name: str) -> tuple[str, ...]:
+    value = os.getenv(name)
+    if not value:
+        return ()
+    return tuple(part.strip() for part in value.split(",") if part.strip())
+
+
+def _clamp_sample_rate(value: float) -> float:
+    return min(1.0, max(0.0, value))