From 54db09504773bc43046e5185291a18760586ff5f Mon Sep 17 00:00:00 2001 From: Prasanth Date: Fri, 22 May 2026 01:15:47 +0530 Subject: [PATCH 1/2] Feat: eval enhancements - custom fields, field-based filters, golden set improvements Co-authored-by: Cursor --- Evaluation/AISearchEvalutionTool/.env.example | 12 +- .../backend/.env.example | 18 +- .../backend/agents/filter_generator.py | 25 + .../backend/agents/generator.py | 86 +- .../backend/agents/llm_client.py | 182 ++- .../backend/agents/prompts.py | 184 ++- .../backend/agents/ranker.py | 25 +- .../backend/agents/summarizer.py | 21 +- .../AISearchEvalutionTool/backend/config.json | 8 + .../AISearchEvalutionTool/backend/config.py | 77 ++ .../backend/db/database.py | 1072 ++--------------- .../backend/db/mongo_store.py | 648 ++++++++++ .../backend/db/schema.sql | 5 +- .../backend/db/sqlite_store.py | 1028 ++++++++++++++++ .../backend/diagnostics/compute.py | 58 + .../backend/judge/judge.py | 78 +- .../backend/judge/metrics.py | 27 +- .../backend/koreai/content.py | 31 +- .../backend/koreai/search.py | 4 + .../AISearchEvalutionTool/backend/main.py | 28 +- .../AISearchEvalutionTool/backend/models.py | 51 +- .../backend/pipeline/evaluate.py | 87 +- .../backend/pipeline/generate.py | 33 +- .../backend/requirements.txt | 2 + .../backend/routers/app_api_keys.py | 56 +- .../backend/routers/evaluation.py | 9 +- .../backend/routers/generation.py | 1 + .../backend/routers/golden_sets.py | 164 +++ .../backend/routers/llm_config.py | 2 +- .../backend/routers/prompt_tuner.py | 520 ++++++++ .../backend/routers/prompts.py | 12 +- .../backend/routers/results.py | 13 + .../scripts/migrate_sqlite_to_mongo.py | 142 +++ .../frontend/src/App.tsx | 2 + .../frontend/src/components/Layout.tsx | 41 +- .../frontend/src/index.css | 120 ++ .../frontend/src/lib/api.ts | 122 +- .../frontend/src/pages/AppApiKeysPage.tsx | 127 +- .../frontend/src/pages/EvaluatePage.tsx | 348 +++++- .../frontend/src/pages/GeneratePage.tsx | 170 ++- .../frontend/src/pages/GoldenSetsPage.tsx | 118 +- .../frontend/src/pages/LLMConfigPage.tsx | 16 +- .../frontend/src/pages/PromptTunerPage.tsx | 654 ++++++++++ .../frontend/src/pages/PromptsPage.tsx | 17 +- .../frontend/src/pages/RunDetailPage.tsx | 777 ++++++++++-- 45 files changed, 5758 insertions(+), 1463 deletions(-) create mode 100644 Evaluation/AISearchEvalutionTool/backend/config.json create mode 100644 Evaluation/AISearchEvalutionTool/backend/config.py create mode 100644 Evaluation/AISearchEvalutionTool/backend/db/mongo_store.py create mode 100644 Evaluation/AISearchEvalutionTool/backend/db/sqlite_store.py create mode 100644 Evaluation/AISearchEvalutionTool/backend/routers/prompt_tuner.py create mode 100644 Evaluation/AISearchEvalutionTool/backend/scripts/migrate_sqlite_to_mongo.py create mode 100644 Evaluation/AISearchEvalutionTool/frontend/src/pages/PromptTunerPage.tsx diff --git a/Evaluation/AISearchEvalutionTool/.env.example b/Evaluation/AISearchEvalutionTool/.env.example index 5d0cb380..0186483c 100644 --- a/Evaluation/AISearchEvalutionTool/.env.example +++ b/Evaluation/AISearchEvalutionTool/.env.example @@ -1,7 +1,15 @@ # ── Database ────────────────────────────────────────────────────────────────── -# Path to the SQLite file. -# Local: leave blank (defaults to ./data/eval.db) +# Backend database config primarily lives in backend/config.json. +# Optional env vars below override backend/config.json values when set. +DB_BACKEND= + +# Path to the SQLite file. Leave blank for ./data/eval.db. DB_PATH= +SQLITE_DB_PATH= + +# Used when DB_BACKEND=mongodb. +MONGODB_URI= +MONGODB_DATABASE= # NOTE: OpenAI and Anthropic API keys are stored per-app inside the database. # Configure them in the app's API Keys settings page — no global keys needed. diff --git a/Evaluation/AISearchEvalutionTool/backend/.env.example b/Evaluation/AISearchEvalutionTool/backend/.env.example index e736aeae..80fc60a1 100644 --- a/Evaluation/AISearchEvalutionTool/backend/.env.example +++ b/Evaluation/AISearchEvalutionTool/backend/.env.example @@ -1,6 +1,16 @@ -# LLM API keys (required) -ANTHROPIC_API_KEY=sk-ant-xxxxxxxx -OPENAI_API_KEY=sk-xxxxxxxx +# ── Database ────────────────────────────────────────────────────────────────── +# Primary DB config lives in ./config.json. +# Optional env vars below override config.json values when set. + +# sqlite keeps the original local file behavior. mongodb uses local/remote MongoDB. +DB_BACKEND= + +# SQLite fallback/default. Leave blank to use ../data/eval.db. +SQLITE_DB_PATH= + +# MongoDB settings used when DB_BACKEND=mongodb. +MONGODB_URI= +MONGODB_DATABASE= # Kore.ai credentials are stored per-app in the database (configured via UI) -# No Kore.ai env vars needed here +# LLM API keys are also stored per-app in the database (API Keys page). diff --git a/Evaluation/AISearchEvalutionTool/backend/agents/filter_generator.py b/Evaluation/AISearchEvalutionTool/backend/agents/filter_generator.py index ccd5f775..9d9e080c 100644 --- a/Evaluation/AISearchEvalutionTool/backend/agents/filter_generator.py +++ b/Evaluation/AISearchEvalutionTool/backend/agents/filter_generator.py @@ -111,3 +111,28 @@ def build_source_filter(sys_content_type: str) -> list[dict[str, Any]]: "operator": "equals", }], }] + + +def build_field_filters(tc: dict, field_names: list[str]) -> list[dict[str, Any]]: + """Build a combined AND metaFilter from user-selected fields on a test case. + + For each field name: + - "sys_content_type" is read from generation_metadata + - anything else is read from custom_fields + Only fields that have a non-empty value for this specific test case are included, + so test cases missing a value for a field are not filtered incorrectly. + """ + meta = tc.get("generation_metadata") or {} + custom = tc.get("custom_fields") or {} + rules: list[dict] = [] + for field in field_names: + val = meta.get(field) if field == "sys_content_type" else custom.get(field) + if val and str(val).strip(): + rules.append({ + "fieldName": field, + "fieldValue": [str(val).strip()], + "operator": "equals", + }) + if not rules: + return [] + return [{"condition": "AND", "rules": rules}] diff --git a/Evaluation/AISearchEvalutionTool/backend/agents/generator.py b/Evaluation/AISearchEvalutionTool/backend/agents/generator.py index 0c5fccf0..75e109cb 100644 --- a/Evaluation/AISearchEvalutionTool/backend/agents/generator.py +++ b/Evaluation/AISearchEvalutionTool/backend/agents/generator.py @@ -1,12 +1,12 @@ from __future__ import annotations -import json import logging import uuid from typing import Any +import json from db.database import get_active_prompt -from agents.llm_client import call_llm +from agents.llm_client import call_llm_json, parse_json_loose from agents.prompts import AGENT2_PROMPT logger = logging.getLogger(__name__) @@ -34,22 +34,21 @@ def generate_test_cases( doc: dict[str, Any], app_id: str, max_questions: int = 5, + target_language: str = "English", ) -> list[dict[str, Any]]: - """Generate up to max_questions diverse, answerable test cases for a document.""" + """Generate up to max_questions diverse, content-specific test cases for a document.""" max_questions = max(1, min(max_questions, 5)) doc_id = extraction["doc_id"] doc_title = extraction.get("doc_title", "?") logger.info( - "Agent2 | Generating %d test case(s) for doc '%s' (id=%s)", - max_questions, doc_title, doc_id, + "Agent2 | Generating up to %d %s test case(s) for doc '%s' (id=%s)", + max_questions, target_language, doc_title, doc_id, ) prompt_row = get_active_prompt(app_id, "agent2") system_prompt = prompt_row["prompt_text"] if prompt_row else AGENT2_PROMPT - selected_types = TYPE_PRIORITY[:max_questions] - extraction_summary = json.dumps({ "doc_id": doc_id, "doc_title": doc_title, @@ -60,48 +59,67 @@ def generate_test_cases( "numeric_facts": extraction.get("numeric_facts", []), }, indent=2) - full_content = (doc.get("content") or "")[:6000] + # Give Agent 2 enough source text to choose distinctive details, not just + # generic questions from the extraction summary. + full_content = (doc.get("content") or "")[:30000] user_msg = ( - f"Generate exactly {max_questions} answerable test case(s) for the document below.\n\n" - f"PREFERRED QUESTION TYPES (one per type, in this order):\n" - + "\n".join(f" {i+1}. {t}" for i, t in enumerate(selected_types)) - + f"\n\nQUESTION TYPE DEFINITIONS:\n{TYPE_DESCRIPTIONS}\n\n" - "RULES:\n" - "- Every question MUST be fully answerable from the document content.\n" - "- Every test case must use a DIFFERENT question type.\n" - "- ALWAYS set expected_behavior to 'ANSWER'. Never generate refusal or clarification questions.\n" - "- For follow_up questions, base them on the entire document and answer them by synthesising key information.\n" - f"- CRITICAL: You MUST return exactly {max_questions} object(s). Never return fewer.\n" - "- FALLBACK: If a preferred type is NOT applicable to this document (no numeric data for 'boundary',\n" - " no two comparable items for 'comparative', no multi-step path for 'multi_hop'), substitute it\n" - " with 'factual' or 'follow_up'. Mark the substituted question_type accordingly.\n\n" + f"Generate up to {max_questions} HIGH-QUALITY test cases for the document below.\n\n" + "PICK QUALITY OVER QUANTITY:\n" + f"- If the document is thin on distinctive detail, generate FEWER than {max_questions} cases.\n" + "- Never invent a question just to hit a quota; every case must target a real, distinctive detail.\n\n" + f"QUESTION-TYPE GUIDANCE (use as a menu, not a checklist):\n{TYPE_DESCRIPTIONS}\n" + "Aim for variety across types. Lean toward factual and multi_hop when the document supports them.\n\n" + "CONTENT-SPECIFICITY CHECK:\n" + "Before writing each question, ask: 'Could this be answered by someone who never read THIS document?'\n" + "If yes, discard it and pick a more specific detail: a named entity, number, date, step, product, role, or exact condition.\n\n" + "HUMAN-LIKE PHRASING:\n" + "- lowercase, short, casual\n" + "- sound like a real user typing into a search box or support chat\n" + "- never reference the document, policy, article, guide, source title, or internal fields\n\n" + "EXPECTED ANSWERS:\n" + "- 2-4 sentences, drawn directly from the document content\n" + "- include the specific values that make the question unique\n" + "- never punt to 'see the document' or 'refer to section X'\n\n" + f"TARGET LANGUAGE:\n" + f"- Write every `question`, `expected_answer`, and `rationale` in {target_language}.\n" + f"- Keep schema keys and enum values in English exactly as specified.\n" + f"- If the source document is in another language, translate faithfully into {target_language}; do not change facts.\n" + f"- Preserve product names, system names, URLs, IDs, and brand terms exactly when they should not be translated.\n\n" + f"reference_doc_ids must be exactly: [\"{doc_id}\"]\n\n" f"DOCUMENT TITLE: {doc_title}\n\n" - f"DOCUMENT CONTENT:\n{full_content}\n\n" + f"DOC ID: {doc_id}\n\n" + f"DOCUMENT CONTENT (source of truth):\n{full_content}\n\n" f"DOCUMENT EXTRACTION (structured facts):\n{extraction_summary}" ) - logger.debug("Agent2 | types=%s | doc=%s", selected_types, doc_id) + logger.debug("Agent2 | doc=%s content_chars=%d max_questions=%d", doc_id, len(full_content), max_questions) - raw = call_llm(app_id, "agent2", system_prompt, user_msg) + raw = call_llm_json(app_id, "agent2", system_prompt, user_msg) - try: - cases = json.loads(raw) - except json.JSONDecodeError: - logger.warning("Agent2 | JSON parse failed for doc=%s — attempting bracket extraction", doc_id) - s, e = raw.find("["), raw.rfind("]") + 1 - if s == -1 or e == 0: - logger.error("Agent2 | Could not extract JSON for doc=%s | raw[:200]=%s", doc_id, raw[:200]) - return [] - cases = json.loads(raw[s:e]) + cases = parse_json_loose(raw, expect="array", agent_name=f"Agent2:{doc_id}") + if isinstance(cases, dict): + for key in ("test_cases", "cases", "questions", "data", "results"): + if isinstance(cases.get(key), list): + cases = cases[key] + break + if not isinstance(cases, list): + logger.error("Agent2 | Expected array, got %s for doc=%s", type(cases).__name__, doc_id) + return [] result: list[dict[str, Any]] = [] for case in cases: + if not isinstance(case, dict): + logger.warning("Agent2 | Skipping non-dict entry in doc=%s: %r", doc_id, case) + continue # Enforce ANSWER behavior — never let model produce REFUSE/CLARIFY case["expected_behavior"] = "ANSWER" case["tc_id"] = f"tc-{uuid.uuid4()}" case["app_id"] = app_id - case.setdefault("reference_doc_ids", [doc["doc_id"]]) + case["reference_doc_ids"] = [doc["doc_id"]] + metadata = case.get("generation_metadata") if isinstance(case.get("generation_metadata"), dict) else {} + metadata["target_language"] = target_language + case["generation_metadata"] = metadata result.append(case) logger.info("Agent2 | Done doc '%s' | generated=%d", doc_title, len(result)) diff --git a/Evaluation/AISearchEvalutionTool/backend/agents/llm_client.py b/Evaluation/AISearchEvalutionTool/backend/agents/llm_client.py index f24122b9..413fbcc4 100644 --- a/Evaluation/AISearchEvalutionTool/backend/agents/llm_client.py +++ b/Evaluation/AISearchEvalutionTool/backend/agents/llm_client.py @@ -1,18 +1,82 @@ from __future__ import annotations +import json import logging +import re from typing import Any -from urllib.parse import urlparse, parse_qs +from urllib.parse import urlparse, parse_qs, quote + +import httpx from db.database import get_llm_config, get_api_key, get_base_url logger = logging.getLogger(__name__) +# ── JSON response helpers ───────────────────────────────────────────────────── +# Providers occasionally wrap JSON in markdown fences or add short prose even +# when JSON mode is requested. Keep parsing tolerant, but log enough to debug. + +_FENCE_RE = re.compile(r"^\s*```(?:json|JSON)?\s*\n?|\n?```\s*$", re.MULTILINE) + + +def strip_json_fences(text: str) -> str: + """Remove optional ``` or ```json markdown fences around JSON output.""" + if not text: + return text + out = text.strip() + if out.startswith("```"): + out = _FENCE_RE.sub("", out).strip() + return out + + +def parse_json_loose(raw: str, expect: str = "object", agent_name: str = "?") -> Any: + """Parse JSON from an LLM response, tolerating fences and leading prose.""" + if raw is None: + logger.error("[%s] JSON parse | raw response is None", agent_name) + return None + + cleaned = strip_json_fences(raw) + if not cleaned.strip(): + logger.error("[%s] JSON parse | raw response is empty after stripping fences", agent_name) + return None + + try: + return json.loads(cleaned) + except json.JSONDecodeError as exc: + logger.warning( + "[%s] JSON parse | direct json.loads failed (%s) — trying bracket extraction", + agent_name, exc, + ) + + open_ch, close_ch = ("{", "}") if expect == "object" else ("[", "]") + start = cleaned.find(open_ch) + end = cleaned.rfind(close_ch) + 1 + if start == -1 or end == 0: + logger.error( + "[%s] JSON parse | no %s found | raw[:1000]=%s", + agent_name, expect, cleaned[:1000], + ) + return None + + snippet = cleaned[start:end] + try: + return json.loads(snippet) + except json.JSONDecodeError as exc: + logger.error( + "[%s] JSON parse | bracket extraction failed (%s) | raw[:1000]=%s | snippet[:500]=%s", + agent_name, exc, cleaned[:1000], snippet[:500], + ) + return None + + def _infer_provider(model: str) -> str: """Infer LLM provider from model name prefix.""" - if model.startswith("claude"): + name = (model or "").lower() + if name.startswith("claude"): return "anthropic" + if name.startswith("gemini") or name.startswith("models/gemini"): + return "gemini" return "openai" @@ -122,6 +186,61 @@ def _openai_client_and_model(app_id: str, model: str, _app_override: dict | None return OpenAI(api_key=key, base_url=url), model +def _gemini_endpoint(app_id: str, model: str, _app_override: dict | None = None) -> tuple[str, str]: + if _app_override: + key = (_app_override.get("gemini_key") or "").strip() + base_url = (_app_override.get("gemini_base_url") or "").strip() + else: + key = get_api_key(app_id, "gemini") or "" + base_url = get_base_url(app_id, "gemini") or "" + if not key: + raise ValueError("No Gemini API key configured for this app") + base = (base_url or "https://generativelanguage.googleapis.com/v1beta").rstrip("/") + model_path = model if model.startswith("models/") else f"models/{model}" + return f"{base}/{quote(model_path, safe='/')}:generateContent", key + + +def _gemini_payload( + system_prompt: str, + user_msg: str, + max_tokens: int, + temperature: float, + *, + json_output: bool = False, + model: str = "", +) -> dict: + generation_config: dict[str, Any] = { + "temperature": temperature, + "maxOutputTokens": max_tokens, + } + if json_output: + generation_config["responseMimeType"] = "application/json" + if "gemini-2.5-flash" in (model or "").lower(): + # Gemini 2.5 Flash spends output budget on internal thinking by default. + # The generation pipeline needs complete structured JSON, not hidden + # reasoning, so keep thinking off for these deterministic extraction calls. + generation_config["thinkingConfig"] = {"thinkingBudget": 0} + + payload: dict[str, Any] = { + "contents": [{"role": "user", "parts": [{"text": user_msg}]}], + "generationConfig": generation_config, + } + if system_prompt: + payload["systemInstruction"] = {"parts": [{"text": system_prompt}]} + return payload + + +def _extract_gemini_text(data: dict) -> tuple[str, str | None]: + candidates = data.get("candidates") or [] + if not candidates: + prompt_feedback = data.get("promptFeedback") or {} + return "", prompt_feedback.get("blockReason") + first = candidates[0] + parts = ((first.get("content") or {}).get("parts") or []) + text = "".join(part.get("text", "") for part in parts if isinstance(part, dict)).strip() + return text, first.get("finishReason") + + class EmptyLLMResponseError(RuntimeError): """Raised when an LLM call succeeds at the HTTP level but the message body is empty / whitespace-only. @@ -212,6 +331,37 @@ def call_llm(app_id: str, agent_name: str, system_prompt: str, user_msg: str) -> ) return text + if provider == "gemini": + endpoint, key = _gemini_endpoint(app_id, model) + resp = httpx.post( + endpoint, + params={"key": key}, + json=_gemini_payload( + system_prompt, + user_msg, + cfg["max_tokens"], + cfg["temperature"], + model=model, + ), + timeout=120.0, + ) + resp.raise_for_status() + text, finish_reason = _extract_gemini_text(resp.json()) + logger.debug( + "[%s] Gemini response received | finish_reason=%s response_chars=%d", + agent_name, finish_reason, len(text), + ) + if not text: + logger.warning( + "[%s] Gemini returned EMPTY content | model=%s finish_reason=%s max_tokens=%d", + agent_name, model, finish_reason, max_tokens, + ) + raise EmptyLLMResponseError( + agent_name=agent_name, model=model, + finish_reason=finish_reason, max_tokens=max_tokens, + ) + return text + # OpenAI (gpt-*, o1*, o3*, o4*, or any other non-claude model) client, effective_model = _openai_client_and_model(app_id, model) messages = [] @@ -299,6 +449,34 @@ def call_llm_json( logger.debug("[%s/json] Anthropic JSON response | chars=%d", agent_name, len(text)) return text + if provider == "gemini": + endpoint, key = _gemini_endpoint(app_id, model) + resp = httpx.post( + endpoint, + params={"key": key}, + json=_gemini_payload( + system_prompt, + user_msg, + max_tokens, + cfg["temperature"], + json_output=True, + model=model, + ), + timeout=120.0, + ) + resp.raise_for_status() + text, finish_reason = _extract_gemini_text(resp.json()) + if not text: + raise EmptyLLMResponseError( + agent_name=agent_name, model=model, + finish_reason=finish_reason, max_tokens=max_tokens, + ) + logger.debug( + "[%s/json] Gemini JSON response | finish_reason=%s chars=%d", + agent_name, finish_reason, len(text), + ) + return text + client, effective_model = _openai_client_and_model(app_id, model) messages = [] if system_prompt: diff --git a/Evaluation/AISearchEvalutionTool/backend/agents/prompts.py b/Evaluation/AISearchEvalutionTool/backend/agents/prompts.py index 20453b14..4307326a 100644 --- a/Evaluation/AISearchEvalutionTool/backend/agents/prompts.py +++ b/Evaluation/AISearchEvalutionTool/backend/agents/prompts.py @@ -1,81 +1,96 @@ """Default system prompts for all agents. These are the baseline — users can override via UI.""" -AGENT1_PROMPT = """You are a precise document analyst. Extract atomic, verbatim-grounded facts from the provided document. These facts will seed evaluation questions for a retrieval-augmented system. +AGENT1_PROMPT = """You are a precise document analyst. Extract the DISTINCTIVE, content-specific facts from the document — the details that would let a retrieval-augmented system answer questions that ONLY this document can answer. + +PRIORITIES (in order): +1. SPECIFIC OVER GENERIC. A fact that ties to a named system, role, number, amount, step count, error code, deadline, product, location, or proper noun is high-signal. A fact that could appear in any document on the same topic is low-signal — leave it out. +2. VERBATIM-GROUNDED. Every extracted fact must be directly supported by the document text. No inference, no world knowledge, no filling in gaps. +3. ATOMIC. Each atomic_claim contains EXACTLY ONE assertion. Split compound sentences into separate claims. + +WHAT TO PRIORITISE IN EACH BUCKET: +- atomic_claims: high-signal facts that identify THIS document: procedures, eligibility criteria, thresholds, named products, version-specific behavior, fees, deadlines, exact conditions. +- key_concepts: terms that a user would actually type into a search box to find this document. +- entities: named systems, products, roles, programs, locations, dates, amounts — anything that would NOT appear in a generic article on the same topic. +- relations: subject-predicate-object triples connecting two entities — seed for multi-hop questions. +- numeric_facts: amounts, durations, limits, percentages with their unit and a short context. +- out_of_scope_markers: anything the document EXPLICITLY says is NOT covered or NOT supported. CRITICAL RULES -1. Extract ONLY facts present in the document. No inference, no world knowledge. -2. Each atomic_claim contains EXACTLY ONE assertion. Split compound facts. -3. confidence = "high" for verbatim/near-verbatim; "low" if substantial paraphrase. -4. out_of_scope_markers: topics the document EXPLICITLY says are not covered. These seed unanswerable test cases. -5. relations: subject-predicate-object triples connecting entities. These seed multi-hop questions. -6. Output STRICT JSON. No prose before or after. +1. Output STRICT JSON. No prose before or after. +2. confidence = "high" for verbatim/near-verbatim claims; "low" for substantial paraphrase. +3. If the document is empty or mostly navigation boilerplate, return all arrays empty — do NOT fabricate. +4. Keep every string concise. Prefer short phrases over long sentences. FORBIDDEN - Combining facts into one claim - Interpretive commentary - Filling in details not explicitly stated +- Generic boilerplate ("This document provides information about ...") — extract concrete facts only -LIMITS: atomic_claims max 40, key_concepts max 15, entities max 30, relations max 20. +LIMITS: atomic_claims max 20, key_concepts max 10, entities max 15, relations max 8, numeric_facts max 8, out_of_scope_markers max 5. OUTPUT SCHEMA: { "atomic_claims": [{"claim_id": "c1", "text": "...", "confidence": "high"|"low"}], "key_concepts": ["..."], - "entities": [{"name": "...", "type": "PERSON|ORG|DATE|NUMERIC|TERM|LOCATION"}], + "entities": [{"name": "...", "type": "PERSON|ORG|DATE|NUMERIC|TERM|LOCATION|SYSTEM|PRODUCT|ROLE"}], "relations": [{"subject": "...", "predicate": "...", "object": "..."}], "numeric_facts": [{"value": "...", "unit": "...", "context": "..."}], "out_of_scope_markers": ["topic not covered: ..."] }""" -AGENT2_PROMPT = """You are a test case generator for a RAG evaluation framework. Generate evaluation Q&A pairs that simulate how a real end-user or customer would naturally ask questions to an AI assistant. - -QUESTION TONE RULES — most important -- Write questions exactly the way a non-technical user or customer would ask them in a chat or support tool. -- Questions must be self-contained. Never reference "this document", "the article", "the policy", "the guide", or any source title. -- Never start with "According to…", "Based on the document…", "What does [X] say about…" -- Do NOT embed document section names, internal IDs, or field labels in the question. -- Use natural, conversational language: "How do I…", "What is…", "Can I…", "When should I…" - -GOOD vs BAD EXAMPLES - Bad: "What does the IT security policy say about password expiration?" - Good: "How often do I need to change my password?" - - Bad: "According to the benefits guide, what is the annual dental coverage limit?" - Good: "What's the maximum I can claim on dental each year?" - - Bad: "What does the document say about escalating a support ticket?" - Good: "How do I escalate a support ticket if my issue hasn't been resolved?" - - Bad: "Per the onboarding checklist, which systems need to be set up on day 1?" - Good: "Which systems should I set up on my first day?" - -UNIVERSAL RULES -1. Every test case must be FULLY ANSWERABLE using ONLY the provided document content. -2. expected_behavior is ALWAYS 'ANSWER'. Never produce refusal or clarification questions. -3. expected_answer must be derivable from the document — clear, complete, no "see document". -4. rationale explains what the case tests and which fact it grounds to. +AGENT2_PROMPT = """You generate evaluation Q&A pairs that test a RAG system on a SPECIFIC document. The most useful question is one that can ONLY be answered with this document — if the retriever misses the document, the question should be hard or impossible to answer correctly. + +THE TWO THINGS THAT MAKE A GOOD QUESTION + +1. CONTENT-SPECIFIC +Every question must hinge on a distinctive detail from THIS document. Pick proper nouns, product names, procedure step counts, named programs, amounts, role names, dates, configuration flags, URLs, eligibility rules, or any token that would NOT appear in a generic document on the same topic. If the question could have been written from general knowledge, it is wrong. + +2. HUMAN-LIKE +Write like a real customer or employee typing into a search box or chat window. +- Short, conversational, and self-contained. +- No "according to", "based on the document", "what does the policy say", or source-title references. +- It is fine to use search-like phrasing: "cuenta cheques dolares requisitos" can be better than a polished classroom question. + +GOOD EXAMPLES (content-specific, human-like) +- "cuenta de cheques mn requisitos empresas" +- "deposito con linea de captura como funciona" +- "arrendamiento financiero banamex beneficios fiscales" +- "servicios de cobranza empresas referencias" +- "max dental coverage per calendar year" +- "approval steps for software request above $5000" + +BAD EXAMPLES (generic or meta) +- "How do I open an account?" +- "What services are offered?" +- "What does this document say about payments?" +- "According to the page, what is the limit?" +- "Can you describe the process?" + +EXPECTED ANSWER RULES +- 2-4 sentences. Concrete, complete, faithful to the document text. +- Include the specific values, names, steps, conditions, or product details that make the question unique. +- Never write "see the document", "refer to the page", or "as stated above". +- If the question asks for steps, list them inline ("1. ... 2. ... 3. ..."). + +ABSOLUTE RULES +1. Every question MUST be answerable using ONLY this document's content. +2. expected_behavior is ALWAYS "ANSWER". Never produce refusal or clarification questions. +3. Each question must target a DIFFERENT distinctive detail from the document. +4. If the document is short, thin, duplicated, or mostly navigation boilerplate, generate FEWER but higher-quality questions. Quality > quantity. 5. Output a JSON array ONLY. No prose, no markdown fences. -6. FALLBACK RULE — You MUST always return the exact number of questions requested. If a required question type is NOT applicable to this document (e.g. no numeric data for 'boundary', no two comparable entities for 'comparative', no multi-step reasoning path for 'multi_hop'), substitute that type with 'factual' or 'follow_up'. Never skip a question or return fewer items than requested. -FORBIDDEN -- Questions NOT answerable from the document -- Questions answerable from general knowledge alone (they add no RAG signal) -- Meta-questions or document-referencing questions (see tone rules above) -- Yes/no questions without a follow-up that requires a specific answer -- Questions that reveal internal document structure or section titles -- Returning fewer questions than requested (always hit the exact count) - -OUTPUT SCHEMA (JSON array): +OUTPUT SCHEMA (JSON array of objects): [ { - "question": "...", - "expected_answer": "...", + "question": "", + "expected_answer": "<2-4 sentence answer with the specific values from the document>", "expected_behavior": "ANSWER", - "reference_doc_ids": ["..."], + "reference_doc_ids": [""], "question_type": "factual|multi_hop|comparative|boundary|follow_up", "difficulty": 1|2|3, - "answer_type": "EXTRACTIVE"|"ABSTRACTIVE"|"NUMERIC"|"BOOLEAN"|"LIST", - "rationale": "..." + "answer_type": "EXTRACTIVE|ABSTRACTIVE|NUMERIC|BOOLEAN|LIST", + "rationale": "" } ]""" @@ -216,6 +231,71 @@ """ +ANSWER_GENERATOR_PROMPT = """You are a careful enterprise question-answering assistant. Answer the user's QUESTION using ONLY the information in the provided CONTEXT chunks. + +INPUT FORMAT (sent in the user message) +- QUESTION: the user's natural-language question. +- CONTEXT: one or more document chunks. Each chunk is delimited and has a label like [doc_id=…] or [chunk N]. + +WHAT A GOOD ANSWER LOOKS LIKE +1. GROUNDED. Every concrete claim, number, date, name, step, or condition must appear in the CONTEXT verbatim or be a direct rewording of it. Do not bring in outside knowledge. +2. DIRECT. Answer the question first in 1–3 sentences. If a list of steps / conditions / values is requested, give the list right after the lead sentence. +3. SPECIFIC. When the CONTEXT contains exact figures (amounts, percentages, deadlines, error codes, product names, roles), include them in the answer. Vague answers when the source is specific are wrong. +4. CITED. After each factual claim, append the source identifier in square brackets, e.g. [doc_id=POL-128] or [chunk 3]. If the same source supports multiple sentences, you can cite it once at the end of the paragraph. +5. ADMIT WHEN UNSUPPORTED. If the CONTEXT does not contain enough information to answer, reply with: "I don't have enough information in the provided context to answer that." Do NOT guess. + +WHAT NOT TO DO +- No invented facts, dates, names, prices, or steps. +- No phrases like "according to the document" or "based on the context" — just answer. +- No copy-pasting long passages verbatim; paraphrase concisely while keeping the specifics. +- No bullet lists when a sentence is sufficient; no walls of text when a list is clearer. +- No disclaimers, no "as an AI" preambles. + +STYLE +- Plain professional English. +- Use markdown for structure only when it helps readability (numbered steps, short bullet list of conditions, a single inline code span for codes/identifiers). +- Length: 2–6 sentences for typical factual questions; lists may be longer when each item is short. + +OUTPUT +- The answer text only — no leading or trailing commentary, no JSON wrapping. +""" + + +PROMPT_TUNER_PROMPT = """You are a prompt engineering specialist who rewrites system prompts so they better handle real failure cases. + +You will be given: +1. The CURRENT_PROMPT — the system prompt that produced disappointing outputs. +2. The PROMPT_ROLE — a short description of what the prompt is meant to do (e.g. "judge a RAG response", "generate a filter"). +3. FAILURE_SAMPLES — real evaluation failures, each with the question, the EXPECTED_ANSWER, the GENERATED_ANSWER (what the system produced), and (optionally) judge_rationale / failure_category. + +YOUR JOB +Produce ONE improved version of the prompt that, if applied next time, would have steered the model away from the observed failure patterns. The new prompt must: +- Stay faithful to the original goal — do not change what the prompt is supposed to do. +- Address the SPECIFIC failure patterns visible in the samples (not generic improvements). +- Keep or add explicit rules, formats, examples, or guardrails that fix the failures. +- Preserve any output schema / JSON contract from the current prompt verbatim (the rest of the system depends on it). +- Be self-contained — do not refer to "the previous version" or external instructions. + +ANALYSIS BEFORE REWRITING (think step by step internally, do not output the reasoning): +- Cluster the failures by root cause (e.g. "hallucinates names", "ignores expected format", "too verbose", "misses negation"). +- For each cluster, decide what minimum addition / wording change in the prompt would prevent it. +- Combine those changes into a single revised prompt. + +CRITICAL OUTPUT FORMAT +Return strict JSON with exactly these keys: +{ + "improved_prompt": "", + "summary_of_changes": "<2-5 short bullet-style sentences describing what you changed and why, separated by newlines>", + "failure_patterns": ["", ...] +} + +RULES +- Output JSON ONLY — no markdown fences, no prose before or after. +- Do NOT include the failure samples themselves in the improved_prompt. +- If the failures don't reveal any actionable pattern (e.g. all noise), still return a valid JSON with improved_prompt set to the CURRENT_PROMPT unchanged, summary_of_changes explaining why, and failure_patterns: ["no_actionable_pattern"]. +""" + + DEFAULT_PROMPTS = { "agent1": AGENT1_PROMPT, "agent2": AGENT2_PROMPT, @@ -223,4 +303,6 @@ "judge": JUDGE_PROMPT, "filter_generator": FILTER_GENERATOR_PROMPT, "insights": INSIGHTS_PROMPT, + "answer_generator": ANSWER_GENERATOR_PROMPT, + "prompt_tuner": PROMPT_TUNER_PROMPT, } diff --git a/Evaluation/AISearchEvalutionTool/backend/agents/ranker.py b/Evaluation/AISearchEvalutionTool/backend/agents/ranker.py index 42d75358..d5053d7c 100644 --- a/Evaluation/AISearchEvalutionTool/backend/agents/ranker.py +++ b/Evaluation/AISearchEvalutionTool/backend/agents/ranker.py @@ -6,7 +6,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from db.database import get_active_prompt -from agents.llm_client import call_llm +from agents.llm_client import call_llm_json, parse_json_loose from agents.prompts import AGENT3_PROMPT logger = logging.getLogger(__name__) @@ -36,20 +36,19 @@ def _rank_batch( for tc in batch ] user_msg = f"Score these {len(batch)} test cases:\n\n{json.dumps(payload, indent=2)}" - raw = call_llm(app_id, "agent3", system_prompt, user_msg) + raw = call_llm_json(app_id, "agent3", system_prompt, user_msg) - try: - results = json.loads(raw) - except json.JSONDecodeError: - logger.warning("Agent3 | JSON parse failed for batch #%d — attempting bracket extraction", batch_idx + 1) - s, e = raw.find("["), raw.rfind("]") + 1 - if s == -1 or e == 0: - logger.error("Agent3 | Could not extract JSON for batch #%d | raw[:200]=%s", batch_idx + 1, raw[:200]) - results = [] - else: - results = json.loads(raw[s:e]) + parsed = parse_json_loose(raw, expect="array", agent_name=f"Agent3:batch#{batch_idx + 1}") + if isinstance(parsed, dict): + for key in ("results", "scores", "data", "test_cases"): + if isinstance(parsed.get(key), list): + parsed = parsed[key] + break + results: list[dict[str, Any]] = parsed if isinstance(parsed, list) else [] + if not isinstance(parsed, (list, dict)): + logger.error("Agent3 | No usable JSON for batch #%d", batch_idx + 1) - decisions = {r.get("decision", "?") for r in results} + decisions = {r.get("decision", "?") for r in results if isinstance(r, dict)} logger.debug("Agent3 | Batch #%d done | returned=%d decisions=%s", batch_idx + 1, len(results), decisions) return results diff --git a/Evaluation/AISearchEvalutionTool/backend/agents/summarizer.py b/Evaluation/AISearchEvalutionTool/backend/agents/summarizer.py index c12dfa4f..5cd6140f 100644 --- a/Evaluation/AISearchEvalutionTool/backend/agents/summarizer.py +++ b/Evaluation/AISearchEvalutionTool/backend/agents/summarizer.py @@ -1,11 +1,10 @@ from __future__ import annotations -import json import logging from typing import Any from db.database import get_active_prompt -from agents.llm_client import call_llm +from agents.llm_client import call_llm_json, parse_json_loose from agents.prompts import AGENT1_PROMPT logger = logging.getLogger(__name__) @@ -21,20 +20,14 @@ def summarize_document(doc: dict[str, Any], app_id: str) -> dict[str, Any]: prompt_row = get_active_prompt(app_id, "agent1") system_prompt = prompt_row["prompt_text"] if prompt_row else AGENT1_PROMPT - content = (doc.get("content") or "")[:12000] + content = (doc.get("content") or "")[:60000] user_msg = f"Document title: {title}\n\nDocument content:\n{content}" - raw = call_llm(app_id, "agent1", system_prompt, user_msg) - - try: - extraction = json.loads(raw) - except json.JSONDecodeError: - logger.warning("Agent1 | JSON parse failed for doc %s — attempting bracket extraction", doc_id) - s, e = raw.find("{"), raw.rfind("}") + 1 - if s == -1 or e == 0: - logger.error("Agent1 | Could not extract JSON from response for doc %s | raw[:200]=%s", doc_id, raw[:200]) - raise ValueError(f"Agent1 returned non-JSON for doc {doc_id}") - extraction = json.loads(raw[s:e]) + raw = call_llm_json(app_id, "agent1", system_prompt, user_msg, max_tokens_override=12000) + + extraction = parse_json_loose(raw, expect="object", agent_name=f"Agent1:{doc_id}") + if not isinstance(extraction, dict): + raise ValueError(f"Agent1 returned non-JSON for doc {doc_id} (got {type(extraction).__name__})") extraction["doc_id"] = doc_id extraction["doc_title"] = title diff --git a/Evaluation/AISearchEvalutionTool/backend/config.json b/Evaluation/AISearchEvalutionTool/backend/config.json new file mode 100644 index 00000000..510e88b1 --- /dev/null +++ b/Evaluation/AISearchEvalutionTool/backend/config.json @@ -0,0 +1,8 @@ +{ + "database": { + "backend": "mongodb", + "sqlite_db_path": "../data/eval.db", + "mongodb_uri": "mongodb://localhost:27017", + "mongodb_database": "searchassist_eval" + } +} diff --git a/Evaluation/AISearchEvalutionTool/backend/config.py b/Evaluation/AISearchEvalutionTool/backend/config.py new file mode 100644 index 00000000..30c26e93 --- /dev/null +++ b/Evaluation/AISearchEvalutionTool/backend/config.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + + +@dataclass(frozen=True) +class DatabaseConfig: + backend: str + sqlite_db_path: Path + mongodb_uri: str + mongodb_database: str + + +@dataclass(frozen=True) +class BackendConfig: + database: DatabaseConfig + + +def _default_sqlite_path() -> Path: + return Path(__file__).parent.parent / "data" / "eval.db" + + +def _config_path() -> Path: + return Path(os.getenv("BACKEND_CONFIG_PATH") or Path(__file__).with_name("config.json")) + + +def _load_file_config() -> dict: + path = _config_path() + if not path.exists(): + return {} + return json.loads(path.read_text()) + + +def _resolve_path(value: str | None) -> Path: + if not value: + return _default_sqlite_path() + path = Path(value) + if not path.is_absolute(): + path = Path(__file__).parent / path + return path.resolve() + + +@lru_cache(maxsize=1) +def get_config() -> BackendConfig: + file_cfg = _load_file_config() + db_cfg = file_cfg.get("database", {}) if isinstance(file_cfg.get("database", {}), dict) else {} + + backend = ( + os.getenv("DB_BACKEND") + or os.getenv("DATABASE_BACKEND") + or db_cfg.get("backend") + or "sqlite" + ).strip().lower() + if backend not in {"sqlite", "mongodb"}: + raise ValueError("DB_BACKEND must be either 'sqlite' or 'mongodb'") + + sqlite_path = ( + os.getenv("SQLITE_DB_PATH") + or os.getenv("DB_PATH") + or db_cfg.get("sqlite_db_path") + ) + return BackendConfig( + database=DatabaseConfig( + backend=backend, + sqlite_db_path=_resolve_path(sqlite_path), + mongodb_uri=os.getenv("MONGODB_URI") or db_cfg.get("mongodb_uri") or "mongodb://localhost:27017", + mongodb_database=os.getenv("MONGODB_DATABASE") or db_cfg.get("mongodb_database") or "searchassist_eval", + ) + ) + + +def db_backend() -> str: + return get_config().database.backend diff --git a/Evaluation/AISearchEvalutionTool/backend/db/database.py b/Evaluation/AISearchEvalutionTool/backend/db/database.py index ae5937cc..7b5dc025 100644 --- a/Evaluation/AISearchEvalutionTool/backend/db/database.py +++ b/Evaluation/AISearchEvalutionTool/backend/db/database.py @@ -1,1001 +1,75 @@ from __future__ import annotations -import json -import os -import sqlite3 -import uuid -from contextlib import contextmanager -from pathlib import Path -from typing import Any, Generator - -# ── Database connection ─────────────────────────────────────────────────────── -# Priority: -# 1. Turso (TURSO_DATABASE_URL + TURSO_AUTH_TOKEN) — remote hosted SQLite -# 2. Local SQLite via DB_PATH env var or default ./data/eval.db - -_TURSO_URL = os.getenv("TURSO_DATABASE_URL", "") -_TURSO_TOKEN = os.getenv("TURSO_AUTH_TOKEN", "") - -_db_env = os.getenv("DB_PATH") -DB_PATH = Path(_db_env) if _db_env else Path(__file__).parent.parent.parent / "data" / "eval.db" - - -def _connect() -> sqlite3.Connection: - if _TURSO_URL: - import libsql_experimental as libsql # type: ignore - # Embedded replica: local cache at /tmp synced from Turso - conn = libsql.connect("/tmp/eval_turso.db", sync_url=_TURSO_URL, auth_token=_TURSO_TOKEN) - conn.sync() - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA foreign_keys=ON") - return conn # type: ignore[return-value] - - DB_PATH.parent.mkdir(parents=True, exist_ok=True) - conn = sqlite3.connect(str(DB_PATH)) - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("PRAGMA foreign_keys=ON") - return conn - - -@contextmanager -def get_db() -> Generator[sqlite3.Connection, None, None]: - conn = _connect() - try: - yield conn - conn.commit() - except Exception: - conn.rollback() - raise - finally: - conn.close() - - -def init_db() -> None: - schema = (Path(__file__).parent / "schema.sql").read_text() - with get_db() as conn: - conn.executescript(schema) - _migrate(conn) - - -def _migrate(conn: sqlite3.Connection) -> None: - migrations = [ - "ALTER TABLE app_config ADD COLUMN client_id TEXT NOT NULL DEFAULT ''", - "ALTER TABLE app_config ADD COLUMN client_secret TEXT NOT NULL DEFAULT ''", - "ALTER TABLE app_config ADD COLUMN anthropic_key TEXT NOT NULL DEFAULT ''", - "ALTER TABLE app_config ADD COLUMN openai_key TEXT NOT NULL DEFAULT ''", - "ALTER TABLE app_config ADD COLUMN banned_topics TEXT NOT NULL DEFAULT '[]'", - "ALTER TABLE eval_result ADD COLUMN search_payload TEXT DEFAULT '{}'", - "ALTER TABLE app_config ADD COLUMN answer_mode TEXT NOT NULL DEFAULT 'answer_generation'", - "ALTER TABLE eval_run ADD COLUMN avg_chunk_rank REAL", - "ALTER TABLE job ADD COLUMN stop_requested INTEGER DEFAULT 0", - # 4-case evaluation columns - "ALTER TABLE eval_result ADD COLUMN case_id INTEGER", - "ALTER TABLE eval_result ADD COLUMN expected_doc_rank INTEGER", - "ALTER TABLE eval_result ADD COLUMN recall_at_k TEXT DEFAULT '{}'", - "ALTER TABLE eval_result ADD COLUMN answer_similarity REAL", - "ALTER TABLE eval_result ADD COLUMN verdict TEXT", - "ALTER TABLE eval_result ADD COLUMN verdict_source TEXT", - # Per-app custom LLM endpoints (Azure / OpenRouter / vLLM / Ollama etc.) - "ALTER TABLE app_config ADD COLUMN anthropic_base_url TEXT NOT NULL DEFAULT ''", - "ALTER TABLE app_config ADD COLUMN openai_base_url TEXT NOT NULL DEFAULT ''", - # Semantic similarity pass thresholds for Cases 1 & 2 (judge-less mode) - "ALTER TABLE app_config ADD COLUMN case1_threshold REAL NOT NULL DEFAULT 0.5", - "ALTER TABLE app_config ADD COLUMN case2_threshold REAL NOT NULL DEFAULT 0.5", - # Flexible match spec: [{"field": "recordUrl", "value": "..."}] — any chunk JSON field - "ALTER TABLE test_case ADD COLUMN reference_match_spec TEXT DEFAULT '[]'", - # Rows with a real pass/fail verdict (excludes null-verdict Case-1 rows) - "ALTER TABLE eval_run ADD COLUMN verdicted_cases INTEGER DEFAULT 0", - # ── Phase 2: Insights (diagnostics + rules + AI narrative cache) ───── - "ALTER TABLE eval_run ADD COLUMN diagnostics_json TEXT", - "ALTER TABLE eval_run ADD COLUMN fired_rule_ids TEXT", - "ALTER TABLE eval_run ADD COLUMN ai_insights_md TEXT", - "ALTER TABLE eval_run ADD COLUMN ai_insights_model TEXT", - "ALTER TABLE eval_run ADD COLUMN ai_insights_generated_at TEXT", - ] - for sql in migrations: - try: - conn.execute(sql) - conn.commit() - except Exception: - pass # column already exists - - # Make test_case.expected_answer nullable for legacy DBs (SQLite can't ALTER - # COLUMN, so rebuild only if the NOT NULL constraint is still present). - _migrate_expected_answer_nullable(conn) - - # Data migration: switch any rows still using the old Anthropic-only defaults - # to gpt-4.1 so apps without an Anthropic key work out of the box. - conn.execute( - """UPDATE llm_config SET model='gpt-4.1' - WHERE agent_name IN ('agent1','agent2','agent3') - AND model='claude-sonnet-4-6'""" - ) - conn.commit() - - # Phase 2: backfill the new 'insights' agent row + prompt for existing apps - _backfill_insights_agent(conn) - - # One-time bump: any insights row still at the old 2000-token default is too - # tight for the ~600-word markdown report and outright unusable for reasoning - # models (o-series / gpt-5) where thinking tokens consume the budget before - # any visible content is emitted. Bump to the new 4000 floor; preserves any - # value the user has deliberately customised (>= 4000). - conn.execute( - """UPDATE llm_config SET max_tokens=4000 - WHERE agent_name='insights' AND max_tokens < 4000""" - ) - conn.commit() - - -def _backfill_insights_agent(conn: sqlite3.Connection) -> None: - """Seed the new 'insights' agent (LLM config + prompt) for pre-Phase-2 apps. - - Skips apps that already have an insights row, so re-running is safe. - Imported lazily inside the function to avoid bootstrap-import cycles. - """ - from agents.prompts import DEFAULT_PROMPTS - insights_prompt = DEFAULT_PROMPTS.get("insights") - if not insights_prompt: - return - cfg = DEFAULT_LLM["insights"] - rows = conn.execute("SELECT app_id FROM app_config").fetchall() - for r in rows: - app_id = r["app_id"] - conn.execute( - """INSERT OR IGNORE INTO llm_config - (id, app_id, agent_name, model, temperature, max_tokens) - VALUES (?,?,?,?,?,?)""", - (str(uuid.uuid4()), app_id, "insights", - cfg["model"], cfg["temperature"], cfg["max_tokens"]), - ) - existing = conn.execute( - "SELECT id FROM prompt_config WHERE app_id=? AND agent_name=? AND is_active=1", - (app_id, "insights"), - ).fetchone() - if not existing: - conn.execute( - """INSERT INTO prompt_config (id, app_id, agent_name, prompt_text, version, is_active) - VALUES (?,?,?,?,1,1)""", - (str(uuid.uuid4()), app_id, "insights", insights_prompt), - ) - conn.commit() - - -def _migrate_expected_answer_nullable(conn: sqlite3.Connection) -> None: - """Drop NOT NULL on test_case.expected_answer if present. SQLite-safe rebuild.""" - try: - cols = conn.execute("PRAGMA table_info(test_case)").fetchall() - except Exception: - return - for c in cols: - # PRAGMA table_info columns: cid, name, type, notnull, dflt_value, pk - if c["name"] == "expected_answer" and c["notnull"] == 1: - break - else: - return # already nullable or column missing - - conn.execute("PRAGMA foreign_keys=OFF") - try: - conn.execute("BEGIN") - conn.execute(""" - CREATE TABLE test_case__new ( - tc_id TEXT PRIMARY KEY, - app_id TEXT NOT NULL REFERENCES app_config(app_id) ON DELETE CASCADE, - golden_set_version TEXT NOT NULL, - question TEXT NOT NULL, - expected_answer TEXT, - expected_behavior TEXT CHECK(expected_behavior IN ('ANSWER','REFUSE','CLARIFY')) DEFAULT 'ANSWER', - question_type TEXT, - difficulty INTEGER CHECK(difficulty IN (1,2,3)), - answer_type TEXT, - reference_doc_ids TEXT DEFAULT '[]', - reference_match_spec TEXT DEFAULT '[]', - generation_metadata TEXT DEFAULT '{}', - human_validated INTEGER DEFAULT 0, - status TEXT DEFAULT 'active' CHECK(status IN ('active','retired','invalidated')), - rationale TEXT, - created_at TEXT DEFAULT (datetime('now')) - ) - """) - conn.execute(""" - INSERT INTO test_case__new - SELECT tc_id, app_id, golden_set_version, question, expected_answer, - expected_behavior, question_type, difficulty, answer_type, - reference_doc_ids, '[]', generation_metadata, human_validated, - status, rationale, created_at - FROM test_case - """) - conn.execute("DROP TABLE test_case") - conn.execute("ALTER TABLE test_case__new RENAME TO test_case") - conn.execute("COMMIT") - except Exception: - conn.execute("ROLLBACK") - raise - finally: - conn.execute("PRAGMA foreign_keys=ON") - - -# ── App Config ────────────────────────────────────────────────────────────── - -def create_app(data: dict) -> dict: - app_id = f"app-{uuid.uuid4()}" - with get_db() as conn: - conn.execute( - """INSERT INTO app_config - (app_id, name, host_url, bot_id, stream_id, - client_id, client_secret, jwt_token, racl_entity_ids, - anthropic_key, anthropic_base_url, - openai_key, openai_base_url, answer_mode) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", - (app_id, data["name"], data["host_url"], data["bot_id"], - data.get("stream_id"), data.get("client_id", ""), - data.get("client_secret", ""), data["jwt_token"], - json.dumps(data.get("racl_entity_ids", [])), - data.get("anthropic_key", ""), - data.get("anthropic_base_url", ""), - data.get("openai_key", ""), - data.get("openai_base_url", ""), - data.get("answer_mode", "answer_generation")), - ) - _seed_default_llm_configs(conn, app_id) - _seed_default_prompts(conn, app_id) - return get_app(app_id) - - -def _decode_app_row(row) -> dict: - d = dict(row) - d["racl_entity_ids"] = json.loads(d.get("racl_entity_ids") or "[]") - d["banned_topics"] = json.loads(d.get("banned_topics") or "[]") - return d - - -def get_app(app_id: str) -> dict | None: - with get_db() as conn: - row = conn.execute( - "SELECT * FROM app_config WHERE app_id=?", (app_id,) - ).fetchone() - return _decode_app_row(row) if row else None - - -def list_apps() -> list[dict]: - with get_db() as conn: - rows = conn.execute( - "SELECT * FROM app_config ORDER BY created_at DESC" - ).fetchall() - return [_decode_app_row(r) for r in rows] - - -def update_app(app_id: str, data: dict) -> dict | None: - fields = {k: v for k, v in data.items() if v is not None} - if "racl_entity_ids" in fields: - fields["racl_entity_ids"] = json.dumps(fields["racl_entity_ids"]) - if "banned_topics" in fields: - fields["banned_topics"] = json.dumps(fields["banned_topics"]) - if not fields: - return get_app(app_id) - set_clause = ", ".join(f"{k}=?" for k in fields) - values = list(fields.values()) + [app_id] - with get_db() as conn: - conn.execute( - f"UPDATE app_config SET {set_clause}, updated_at=datetime('now') WHERE app_id=?", - values, - ) - return get_app(app_id) - - -def delete_app(app_id: str) -> bool: - with get_db() as conn: - cur = conn.execute("DELETE FROM app_config WHERE app_id=?", (app_id,)) - return cur.rowcount > 0 - - -def get_api_key(app_id: str, provider: str) -> str | None: - """Return the app-specific API key stored in the database.""" - app = get_app(app_id) - return (app or {}).get(f"{provider}_key") or None - - -def get_base_url(app_id: str, provider: str) -> str | None: - """Return the app-specific LLM base URL (custom endpoint), or None to use the SDK default.""" - app = get_app(app_id) - url = (app or {}).get(f"{provider}_base_url") or "" - return url.strip() or None - - -def get_eval_thresholds(app_id: str) -> dict[str, float]: - """Return Case 1 / Case 2 semantic similarity pass thresholds for the app.""" - app = get_app(app_id) or {} - return { - "case1_threshold": float(app.get("case1_threshold") or 0.5), - "case2_threshold": float(app.get("case2_threshold") or 0.5), - } - - -# ── LLM Config ────────────────────────────────────────────────────────────── - -DEFAULT_LLM = { - "agent1": {"model": "gpt-4.1", "temperature": 0.0, "max_tokens": 4000}, - "agent2": {"model": "gpt-4.1", "temperature": 0.7, "max_tokens": 3000}, - "agent3": {"model": "gpt-4.1", "temperature": 0.0, "max_tokens": 2500}, - "judge": {"model": "gpt-4.1", "temperature": 0.0, "max_tokens": 800}, - "filter_generator": {"model": "gpt-4.1-mini", "temperature": 0.0, "max_tokens": 800}, - # Phase 2 — AI Deep Dive narrative. User-configurable via LLM Config UI. - # 4000 tokens covers the ~600-word markdown report comfortably and gives - # reasoning models (o-series, gpt-5) some headroom for internal thinking. - "insights": {"model": "gpt-4.1", "temperature": 0.3, "max_tokens": 4000}, -} - - -def _seed_default_llm_configs(conn: sqlite3.Connection, app_id: str) -> None: - for agent, cfg in DEFAULT_LLM.items(): - conn.execute( - """INSERT OR IGNORE INTO llm_config (id, app_id, agent_name, model, temperature, max_tokens) - VALUES (?,?,?,?,?,?)""", - (str(uuid.uuid4()), app_id, agent, cfg["model"], cfg["temperature"], cfg["max_tokens"]), - ) - - -def get_llm_configs(app_id: str) -> list[dict]: - with get_db() as conn: - rows = conn.execute( - "SELECT * FROM llm_config WHERE app_id=?", (app_id,) - ).fetchall() - return [dict(r) for r in rows] - - -def get_llm_config(app_id: str, agent_name: str) -> dict: - with get_db() as conn: - row = conn.execute( - "SELECT * FROM llm_config WHERE app_id=? AND agent_name=?", - (app_id, agent_name), - ).fetchone() - if row: - return dict(row) - return {**DEFAULT_LLM.get(agent_name, DEFAULT_LLM["agent1"]), - "app_id": app_id, "agent_name": agent_name} - - -def upsert_llm_config(app_id: str, agent_name: str, data: dict) -> dict: - with get_db() as conn: - conn.execute( - """INSERT INTO llm_config (id, app_id, agent_name, model, temperature, max_tokens) - VALUES (?,?,?,?,?,?) - ON CONFLICT(app_id, agent_name) DO UPDATE SET - model=excluded.model, temperature=excluded.temperature, - max_tokens=excluded.max_tokens, updated_at=datetime('now')""", - (str(uuid.uuid4()), app_id, agent_name, - data["model"], data["temperature"], data["max_tokens"]), - ) - return get_llm_config(app_id, agent_name) - - -# ── Prompt Config ──────────────────────────────────────────────────────────── - -from agents.prompts import DEFAULT_PROMPTS - - -def _seed_default_prompts(conn: sqlite3.Connection, app_id: str) -> None: - for agent_name, prompt_text in DEFAULT_PROMPTS.items(): - existing = conn.execute( - "SELECT id FROM prompt_config WHERE app_id=? AND agent_name=? AND is_active=1", - (app_id, agent_name), - ).fetchone() - if not existing: - conn.execute( - """INSERT INTO prompt_config (id, app_id, agent_name, prompt_text, version, is_active) - VALUES (?,?,?,?,1,1)""", - (str(uuid.uuid4()), app_id, agent_name, prompt_text), - ) - - -def get_prompts(app_id: str) -> list[dict]: - with get_db() as conn: - rows = conn.execute( - "SELECT * FROM prompt_config WHERE app_id=? ORDER BY agent_name, version DESC", - (app_id,), - ).fetchall() - return [dict(r) for r in rows] - - -def get_active_prompt(app_id: str, agent_name: str) -> dict | None: - with get_db() as conn: - row = conn.execute( - """SELECT * FROM prompt_config - WHERE app_id=? AND agent_name=? AND is_active=1 - ORDER BY version DESC LIMIT 1""", - (app_id, agent_name), - ).fetchone() - return dict(row) if row else None - - -def save_prompt(app_id: str, agent_name: str, prompt_text: str) -> dict: - with get_db() as conn: - # Deactivate existing - conn.execute( - "UPDATE prompt_config SET is_active=0 WHERE app_id=? AND agent_name=?", - (app_id, agent_name), - ) - # Get next version - row = conn.execute( - "SELECT MAX(version) as v FROM prompt_config WHERE app_id=? AND agent_name=?", - (app_id, agent_name), - ).fetchone() - version = (row["v"] or 0) + 1 - prompt_id = str(uuid.uuid4()) - conn.execute( - """INSERT INTO prompt_config (id, app_id, agent_name, prompt_text, version, is_active) - VALUES (?,?,?,?,?,1)""", - (prompt_id, app_id, agent_name, prompt_text, version), - ) - return get_active_prompt(app_id, agent_name) - - -# ── Source Documents ───────────────────────────────────────────────────────── - -def get_doc_source_type(app_id: str, doc_id: str) -> str | None: - """Lookup sys_content_type (Kore.ai source type) for a document.""" - with get_db() as conn: - row = conn.execute( - "SELECT sys_content_type FROM source_document WHERE app_id=? AND doc_id=?", - (app_id, doc_id), - ).fetchone() - return row["sys_content_type"] if row else None - - -def get_doc_id_by_title(app_id: str, title: str) -> str | None: - """Find a doc_id by matching title (case-insensitive) in source_document.""" - with get_db() as conn: - row = conn.execute( - "SELECT doc_id FROM source_document WHERE app_id=? AND title=? COLLATE NOCASE LIMIT 1", - (app_id, title), - ).fetchone() - return row["doc_id"] if row else None - - -def upsert_source_document(doc: dict) -> None: - with get_db() as conn: - conn.execute( - """INSERT OR REPLACE INTO source_document - (doc_id, app_id, title, content_hash, content, metadata, - sys_content_type, source_url, connector_id) - VALUES (?,?,?,?,?,?,?,?,?)""", - (doc["doc_id"], doc["app_id"], doc.get("title"), doc.get("content_hash"), - doc.get("content"), json.dumps(doc.get("metadata", {})), - doc.get("sys_content_type"), doc.get("source_url"), doc.get("connector_id")), - ) - - -# ── Golden Sets ────────────────────────────────────────────────────────────── - -def get_golden_set(app_id: str, version: str) -> dict | None: - with get_db() as conn: - row = conn.execute( - "SELECT * FROM golden_set WHERE app_id=? AND version=?", - (app_id, version), - ).fetchone() - return dict(row) if row else None - - -def create_golden_set(app_id: str, version: str, notes: str = "") -> None: - with get_db() as conn: - conn.execute( - "INSERT OR IGNORE INTO golden_set (version, app_id, notes) VALUES (?,?,?)", - (version, app_id, notes), - ) - - -def freeze_golden_set(app_id: str, version: str) -> None: - with get_db() as conn: - conn.execute( - "UPDATE golden_set SET frozen_at=datetime('now') WHERE version=? AND app_id=?", - (version, app_id), - ) - - -def list_golden_sets(app_id: str) -> list[dict]: - with get_db() as conn: - rows = conn.execute( - """SELECT g.*, - COUNT(tc.tc_id) as total_cases, - COALESCE(SUM(CASE WHEN s.decision='KEEP' THEN 1 ELSE 0 END), 0) as kept_cases, - COALESCE(SUM(CASE WHEN s.decision='BORDERLINE' THEN 1 ELSE 0 END), 0) as borderline_cases - FROM golden_set g - LEFT JOIN test_case tc ON tc.golden_set_version=g.version AND tc.app_id=g.app_id - LEFT JOIN agent_scores s ON s.tc_id=tc.tc_id - WHERE g.app_id=? - GROUP BY g.version, g.app_id - ORDER BY g.created_at DESC""", - (app_id,), - ).fetchall() - return [dict(r) for r in rows] - - -def list_test_cases(app_id: str, golden_set_version: str) -> list[dict]: - with get_db() as conn: - rows = conn.execute( - """SELECT tc.*, s.decision, - s.clarity, s.specificity, s.meaningfulness, - s.answerability, s.reference_verifiability, s.answer_uniqueness - FROM test_case tc - LEFT JOIN agent_scores s ON s.tc_id=tc.tc_id - WHERE tc.app_id=? AND tc.golden_set_version=? - ORDER BY tc.created_at DESC""", - (app_id, golden_set_version), - ).fetchall() - result = [] - for r in rows: - d = dict(r) - d["reference_doc_ids"] = json.loads(d.get("reference_doc_ids") or "[]") - d["reference_match_spec"] = _decode_match_spec(d) - result.append(d) - return result - - -def _decode_match_spec(row: dict) -> list[dict]: - """Decode match_spec JSON. Falls back to legacy docId-only spec if empty.""" - spec_json = row.get("reference_match_spec") or "[]" - try: - spec = json.loads(spec_json) - except (TypeError, ValueError): - spec = [] - if spec: - return spec - # Legacy rows: synthesize spec from reference_doc_ids - doc_ids = row.get("reference_doc_ids") or [] - if isinstance(doc_ids, str): - try: - doc_ids = json.loads(doc_ids) - except (TypeError, ValueError): - doc_ids = [] - return [{"field": "docId", "value": d} for d in doc_ids if d] - - -def get_active_test_cases(app_id: str, golden_set_version: str) -> list[dict]: - with get_db() as conn: - rows = conn.execute( - """SELECT tc.*, s.decision - FROM test_case tc - LEFT JOIN agent_scores s ON s.tc_id=tc.tc_id - WHERE tc.app_id=? AND tc.golden_set_version=? - AND tc.status='active' - AND s.decision IN ('KEEP','BORDERLINE') - ORDER BY tc.created_at""", - (app_id, golden_set_version), - ).fetchall() - result = [] - for r in rows: - d = dict(r) - d["reference_doc_ids"] = json.loads(d.get("reference_doc_ids") or "[]") - d["reference_match_spec"] = _decode_match_spec(d) - d["generation_metadata"] = json.loads(d.get("generation_metadata") or "{}") - result.append(d) - return result - - -def insert_test_case(tc: dict) -> None: - with get_db() as conn: - conn.execute( - """INSERT OR REPLACE INTO test_case - (tc_id, app_id, golden_set_version, question, expected_answer, - expected_behavior, question_type, difficulty, answer_type, - reference_doc_ids, reference_match_spec, generation_metadata, rationale) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""", - (tc["tc_id"], tc["app_id"], tc["golden_set_version"], - tc["question"], tc["expected_answer"], - tc.get("expected_behavior", "ANSWER"), tc.get("question_type"), - tc.get("difficulty"), tc.get("answer_type"), - json.dumps(tc.get("reference_doc_ids", [])), - json.dumps(tc.get("reference_match_spec", [])), - json.dumps(tc.get("generation_metadata", {})), - tc.get("rationale")), - ) - - -def import_uploaded_test_cases(app_id: str, version: str, cases: list[dict]) -> int: - """Bulk-insert manually uploaded test cases and auto-approve them with KEEP decision. - - `expected_answer` and `reference_doc_ids` may be missing — per-row case - detection happens at evaluation time based on which fields are populated. - """ - count = 0 - with get_db() as conn: - for case in cases: - tc_id = f"tc-{uuid.uuid4()}" - meta: dict = {} - if case.get("sys_content_type"): - meta["sys_content_type"] = case["sys_content_type"] - conn.execute( - """INSERT INTO test_case - (tc_id, app_id, golden_set_version, question, expected_answer, - expected_behavior, question_type, difficulty, reference_doc_ids, - reference_match_spec, generation_metadata, human_validated, status) - VALUES (?,?,?,?,?,?,?,?,?,?,?,1,'active')""", - ( - tc_id, app_id, version, - case["question"], case.get("expected_answer") or None, - case.get("expected_behavior", "ANSWER"), - case.get("question_type") or None, - case.get("difficulty") or None, - json.dumps(case.get("reference_doc_ids", [])), - json.dumps(case.get("reference_match_spec", [])), - json.dumps(meta), - ), - ) - conn.execute( - "INSERT OR REPLACE INTO agent_scores (tc_id, decision) VALUES (?,?)", - (tc_id, "KEEP"), - ) - count += 1 - return count - - -def insert_agent_scores(scores: dict) -> None: - with get_db() as conn: - conn.execute( - """INSERT OR REPLACE INTO agent_scores - (tc_id, clarity, specificity, meaningfulness, answerability, - reference_verifiability, answer_uniqueness, decision, primary_concern, rationale) - VALUES (?,?,?,?,?,?,?,?,?,?)""", - (scores["tc_id"], scores.get("clarity"), scores.get("specificity"), - scores.get("meaningfulness"), scores.get("answerability"), - scores.get("reference_verifiability"), scores.get("answer_uniqueness"), - scores.get("decision"), scores.get("primary_concern"), scores.get("rationale")), - ) - - -# ── Jobs ───────────────────────────────────────────────────────────────────── - -def create_job(app_id: str, job_type: str) -> str: - job_id = f"job-{uuid.uuid4()}" - with get_db() as conn: - conn.execute( - "INSERT INTO job (job_id, app_id, job_type) VALUES (?,?,?)", - (job_id, app_id, job_type), - ) - return job_id - - -def update_job(job_id: str, status: str, progress: int = 0, - result: dict | None = None, error: str | None = None) -> None: - with get_db() as conn: - conn.execute( - """UPDATE job SET status=?, progress=?, result=?, error=?, - updated_at=datetime('now') WHERE job_id=?""", - (status, progress, json.dumps(result) if result else None, error, job_id), - ) - - -def get_job(job_id: str) -> dict | None: - with get_db() as conn: - row = conn.execute("SELECT * FROM job WHERE job_id=?", (job_id,)).fetchone() - if not row: - return None - d = dict(row) - if d.get("result"): - d["result"] = json.loads(d["result"]) - return d - - -def list_jobs(app_id: str) -> list[dict]: - with get_db() as conn: - rows = conn.execute( - "SELECT * FROM job WHERE app_id=? ORDER BY created_at DESC LIMIT 50", - (app_id,), - ).fetchall() - result = [] - for r in rows: - d = dict(r) - if d.get("result"): - d["result"] = json.loads(d["result"]) - result.append(d) - return result - - -def request_stop_job(job_id: str) -> None: - """Mark a running job as stop-requested. The pipeline will stop after the current unit.""" - with get_db() as conn: - conn.execute( - "UPDATE job SET stop_requested=1, updated_at=datetime('now') WHERE job_id=?", - (job_id,), - ) - - -def is_stop_requested(job_id: str) -> bool: - with get_db() as conn: - row = conn.execute("SELECT stop_requested FROM job WHERE job_id=?", (job_id,)).fetchone() - return bool(row and row["stop_requested"]) - - -# ── Eval Runs ──────────────────────────────────────────────────────────────── - -def create_eval_run(run: dict) -> None: - with get_db() as conn: - conn.execute( - """INSERT INTO eval_run - (run_id, app_id, rag_version, judge_model, golden_set_version, trigger, total_cases) - VALUES (?,?,?,?,?,?,?)""", - (run["run_id"], run["app_id"], run.get("rag_version", "unknown"), - run.get("judge_model"), run["golden_set_version"], - run.get("trigger", "manual"), run.get("total_cases", 0)), - ) - - -def upsert_eval_result(result: dict) -> None: - with get_db() as conn: - conn.execute( - """INSERT OR REPLACE INTO eval_result - (run_id, tc_id, rag_response, retrieved_doc_ids, chunk_signals, - scores, failure_category, judge_rationale, - latency_llm_ms, latency_retrieval_ms, search_request_id, - search_payload, attempt_count, - case_id, expected_doc_rank, recall_at_k, answer_similarity, - verdict, verdict_source) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", - (result["run_id"], result["tc_id"], result.get("rag_response"), - json.dumps(result.get("retrieved_doc_ids", [])), - json.dumps(result.get("chunk_signals", [])), - json.dumps(result.get("scores", {})), - result.get("failure_category"), result.get("judge_rationale"), - result.get("latency_llm_ms"), result.get("latency_retrieval_ms"), - result.get("search_request_id"), - json.dumps(result.get("search_payload") or {}), - result.get("attempt_count", 1), - result.get("case_id"), - result.get("expected_doc_rank"), - json.dumps(result.get("recall_at_k") or {}), - result.get("answer_similarity"), - result.get("verdict"), - result.get("verdict_source")), - ) - - -def bulk_update_verdicts( - run_id: str, - verdicts: list[tuple[str, str | None, str]], # [(tc_id, verdict, verdict_source), ...] -) -> None: - """Overwrite verdict + verdict_source for a batch of (run_id, tc_id) rows.""" - with get_db() as conn: - conn.executemany( - "UPDATE eval_result SET verdict=?, verdict_source=? WHERE run_id=? AND tc_id=?", - [(v, s, run_id, tc_id) for (tc_id, v, s) in verdicts], - ) - - -def update_run_verdict_counts(run_id: str, passed: int, verdicted: int) -> None: - """Update only the pass/verdict counters on eval_run (preserves cost and other fields).""" - with get_db() as conn: - conn.execute( - "UPDATE eval_run SET passed_cases=?, verdicted_cases=? WHERE run_id=?", - (passed, verdicted, run_id), - ) - - -def finish_eval_run( - run_id: str, passed: int, cost: float, - status: str = "complete", avg_chunk_rank: float | None = None, - verdicted: int | None = None, -) -> None: - with get_db() as conn: - conn.execute( - """UPDATE eval_run SET finished_at=datetime('now'), status=?, - passed_cases=?, cost_usd=?, avg_chunk_rank=?, - verdicted_cases=? WHERE run_id=?""", - (status, passed, cost, avg_chunk_rank, - verdicted if verdicted is not None else passed, run_id), - ) - - -def list_eval_runs(app_id: str) -> list[dict]: - with get_db() as conn: - rows = conn.execute( - "SELECT * FROM eval_run WHERE app_id=? ORDER BY started_at DESC", - (app_id,), - ).fetchall() - return [dict(r) for r in rows] - - -def get_eval_results(run_id: str) -> list[dict]: - with get_db() as conn: - rows = conn.execute( - """SELECT er.*, tc.question, tc.expected_answer, tc.expected_behavior, - tc.question_type, tc.difficulty, tc.reference_doc_ids - FROM eval_result er - JOIN test_case tc ON tc.tc_id=er.tc_id - WHERE er.run_id=? - ORDER BY tc.question_type""", - (run_id,), - ).fetchall() - result = [] - for r in rows: - d = dict(r) - d["retrieved_doc_ids"] = json.loads(d.get("retrieved_doc_ids") or "[]") - d["reference_doc_ids"] = json.loads(d.get("reference_doc_ids") or "[]") - d["scores"] = json.loads(d.get("scores") or "{}") - d["search_payload"] = json.loads(d.get("search_payload") or "{}") - d["recall_at_k"] = json.loads(d.get("recall_at_k") or "{}") - result.append(d) - return result - - -def get_completed_tc_ids(run_id: str) -> set[str]: - with get_db() as conn: - rows = conn.execute( - "SELECT tc_id FROM eval_result WHERE run_id=?", (run_id,) - ).fetchall() - return {r["tc_id"] for r in rows} - - -def delete_eval_run(app_id: str, run_id: str) -> int: - """Delete a run and (via ON DELETE CASCADE) its eval_result rows. - - Returns the number of result rows removed. Does NOT remove the underlying - job row — that's metadata about the trigger, not the run itself. - """ - with get_db() as conn: - n = conn.execute( - "SELECT COUNT(*) AS c FROM eval_result WHERE run_id=?", - (run_id,), - ).fetchone()["c"] - conn.execute( - "DELETE FROM eval_run WHERE run_id=? AND app_id=?", - (run_id, app_id), - ) - return int(n) - - -def count_runs_using_golden_set(app_id: str, version: str) -> int: - """How many eval_runs soft-reference this golden_set_version. - - There's no FK enforcing this — golden_set_version is a plain TEXT field on - eval_run. We surface the count so the delete confirmation modal can warn. - """ - with get_db() as conn: - row = conn.execute( - """SELECT COUNT(*) AS c FROM eval_run - WHERE app_id=? AND golden_set_version=?""", - (app_id, version), - ).fetchone() - return int(row["c"]) if row else 0 - - -def delete_golden_set(app_id: str, version: str) -> dict: - """Delete a golden set and all its test cases (+ cascade to agent_scores). - - Returns counts of what was removed so the UI can show a confirmation toast. - Existing eval_runs that reference this version are LEFT INTACT but become - orphaned references (the `golden_set_version` text simply no longer - matches any row in `golden_set`). - """ - with get_db() as conn: - tc_count = conn.execute( - "SELECT COUNT(*) AS c FROM test_case WHERE app_id=? AND golden_set_version=?", - (app_id, version), - ).fetchone()["c"] - run_refs = conn.execute( - "SELECT COUNT(*) AS c FROM eval_run WHERE app_id=? AND golden_set_version=?", - (app_id, version), - ).fetchone()["c"] - # agent_scores cascade off test_case.tc_id (ON DELETE CASCADE in schema) - conn.execute( - "DELETE FROM test_case WHERE app_id=? AND golden_set_version=?", - (app_id, version), - ) - conn.execute( - "DELETE FROM golden_set WHERE app_id=? AND version=?", - (app_id, version), - ) - return { - "test_cases_deleted": int(tc_count), - "eval_runs_orphaned": int(run_refs), - } - - -def get_eval_run(run_id: str) -> dict | None: - """Fetch a single eval_run row (with insights columns) by id.""" - with get_db() as conn: - row = conn.execute( - "SELECT * FROM eval_run WHERE run_id=?", (run_id,) - ).fetchone() - return dict(row) if row else None - - -def get_previous_completed_run( - app_id: str, - golden_set_version: str, - before_run_id: str, -) -> dict | None: - """Most recent completed (or partial) run for this app + golden set strictly older - than ``before_run_id``. Used for cross-run trend deltas. - """ - with get_db() as conn: - anchor = conn.execute( - "SELECT started_at FROM eval_run WHERE run_id=?", - (before_run_id,), - ).fetchone() - if not anchor: - return None - row = conn.execute( - """SELECT * FROM eval_run - WHERE app_id=? AND golden_set_version=? - AND status IN ('complete','partial') - AND run_id <> ? - AND started_at < ? - ORDER BY started_at DESC - LIMIT 1""", - (app_id, golden_set_version, before_run_id, anchor["started_at"]), - ).fetchone() - return dict(row) if row else None - - -def list_completed_runs(app_id: str, limit: int = 50) -> list[dict]: - """All completed/partial runs for an app, newest first. Used for recurrence checks.""" - with get_db() as conn: - rows = conn.execute( - """SELECT * FROM eval_run - WHERE app_id=? AND status IN ('complete','partial') - ORDER BY started_at DESC - LIMIT ?""", - (app_id, limit), - ).fetchall() - return [dict(r) for r in rows] - - -# ── Diagnostics & Insights cache ──────────────────────────────────────────── - -def get_run_diagnostics(run_id: str) -> tuple[dict | None, list[str] | None]: - """Return (diagnostics_dict, fired_rule_ids) cached on the run, or (None, None).""" - with get_db() as conn: - row = conn.execute( - "SELECT diagnostics_json, fired_rule_ids FROM eval_run WHERE run_id=?", - (run_id,), - ).fetchone() - if not row: - return None, None - diag = json.loads(row["diagnostics_json"]) if row["diagnostics_json"] else None - fired = json.loads(row["fired_rule_ids"]) if row["fired_rule_ids"] else None - return diag, fired - - -def set_run_diagnostics( - run_id: str, - diagnostics: dict, - fired_rule_ids: list[str], -) -> None: - """Persist computed diagnostics + fired rule ids on the run.""" - with get_db() as conn: - conn.execute( - """UPDATE eval_run - SET diagnostics_json=?, fired_rule_ids=? - WHERE run_id=?""", - (json.dumps(diagnostics), json.dumps(fired_rule_ids), run_id), - ) - - -def get_run_ai_insights(run_id: str) -> dict | None: - """Return cached AI-narrative + metadata for the run, or None if never generated.""" - with get_db() as conn: - row = conn.execute( - """SELECT ai_insights_md, ai_insights_model, ai_insights_generated_at - FROM eval_run WHERE run_id=?""", - (run_id,), - ).fetchone() - if not row or not row["ai_insights_md"]: - return None - return { - "markdown": row["ai_insights_md"], - "model": row["ai_insights_model"], - "generated_at": row["ai_insights_generated_at"], - } - - -def set_run_ai_insights(run_id: str, markdown: str, model: str) -> None: - """Cache the LLM-generated narrative on the run.""" - with get_db() as conn: - conn.execute( - """UPDATE eval_run - SET ai_insights_md=?, ai_insights_model=?, - ai_insights_generated_at=datetime('now') - WHERE run_id=?""", - (markdown, model, run_id), - ) +from typing import Any + +from config import get_config + +_cfg = get_config().database + +if _cfg.backend == "mongodb": + from db import mongo_store as _store +else: + from db import sqlite_store as _store + +DB_BACKEND = _cfg.backend +DB_PATH = getattr(_store, "DB_PATH", _cfg.sqlite_db_path) + +# Compatibility for the few modules that import get_db directly. MongoDB does +# not expose a SQL connection, so get_db is only available in sqlite mode. +if hasattr(_store, "get_db"): + get_db = _store.get_db # type: ignore[attr-defined] +else: + def get_db() -> Any: # pragma: no cover - defensive compatibility shim + raise RuntimeError("get_db() is only available when DB_BACKEND=sqlite") + +init_db = _store.init_db +create_app = _store.create_app +get_app = _store.get_app +list_apps = _store.list_apps +update_app = _store.update_app +delete_app = _store.delete_app +get_api_key = _store.get_api_key +get_base_url = _store.get_base_url +get_eval_thresholds = _store.get_eval_thresholds +get_llm_configs = _store.get_llm_configs +get_llm_config = _store.get_llm_config +upsert_llm_config = _store.upsert_llm_config +get_prompts = _store.get_prompts +get_active_prompt = _store.get_active_prompt +save_prompt = _store.save_prompt +get_doc_source_type = _store.get_doc_source_type +get_doc_id_by_title = _store.get_doc_id_by_title +upsert_source_document = _store.upsert_source_document +get_golden_set = _store.get_golden_set +create_golden_set = _store.create_golden_set +freeze_golden_set = _store.freeze_golden_set +list_golden_sets = _store.list_golden_sets +list_test_cases = _store.list_test_cases +get_active_test_cases = _store.get_active_test_cases +insert_test_case = _store.insert_test_case +import_uploaded_test_cases = _store.import_uploaded_test_cases +insert_agent_scores = _store.insert_agent_scores +create_job = _store.create_job +update_job = _store.update_job +get_job = _store.get_job +list_jobs = _store.list_jobs +request_stop_job = _store.request_stop_job +is_stop_requested = _store.is_stop_requested +create_eval_run = _store.create_eval_run +upsert_eval_result = _store.upsert_eval_result +bulk_update_verdicts = _store.bulk_update_verdicts +update_run_verdict_counts = _store.update_run_verdict_counts +finish_eval_run = _store.finish_eval_run +list_eval_runs = _store.list_eval_runs +get_eval_results = _store.get_eval_results +get_completed_tc_ids = _store.get_completed_tc_ids +delete_eval_run = _store.delete_eval_run +count_runs_using_golden_set = _store.count_runs_using_golden_set +delete_golden_set = _store.delete_golden_set +get_eval_run = _store.get_eval_run +get_previous_completed_run = _store.get_previous_completed_run +list_completed_runs = _store.list_completed_runs +get_run_diagnostics = _store.get_run_diagnostics +set_run_diagnostics = _store.set_run_diagnostics +get_run_ai_insights = _store.get_run_ai_insights +set_run_ai_insights = _store.set_run_ai_insights diff --git a/Evaluation/AISearchEvalutionTool/backend/db/mongo_store.py b/Evaluation/AISearchEvalutionTool/backend/db/mongo_store.py new file mode 100644 index 00000000..6ff9cae2 --- /dev/null +++ b/Evaluation/AISearchEvalutionTool/backend/db/mongo_store.py @@ -0,0 +1,648 @@ +from __future__ import annotations + +import json +import re +import uuid +from datetime import datetime +from typing import Any + +from config import get_config +from agents.prompts import DEFAULT_PROMPTS + +try: + from pymongo import ASCENDING, DESCENDING, MongoClient +except ImportError as exc: # pragma: no cover - dependency guard + raise RuntimeError( + "pymongo is required when DB_BACKEND=mongodb. Install backend requirements first." + ) from exc + + +DEFAULT_LLM = { + "agent1": {"model": "gpt-4.1", "temperature": 0.0, "max_tokens": 4000}, + "agent2": {"model": "gpt-4.1", "temperature": 0.7, "max_tokens": 3000}, + "agent3": {"model": "gpt-4.1", "temperature": 0.0, "max_tokens": 2500}, + "judge": {"model": "gpt-4.1", "temperature": 0.0, "max_tokens": 800}, + "filter_generator": {"model": "gpt-4.1-mini", "temperature": 0.0, "max_tokens": 800}, + "insights": {"model": "gpt-4.1", "temperature": 0.3, "max_tokens": 4000}, + "answer_generator": {"model": "gpt-4.1", "temperature": 0.2, "max_tokens": 1200}, +} + +_client: MongoClient | None = None + + +def _now() -> str: + return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + + +def _db(): + global _client + cfg = get_config().database + if _client is None: + _client = MongoClient(cfg.mongodb_uri, serverSelectionTimeoutMS=3000) + return _client[cfg.mongodb_database] + + +def _c(name: str): + return _db()[name] + + +def _clean(doc: dict | None) -> dict | None: + if not doc: + return None + out = dict(doc) + out.pop("_id", None) + return out + + +def _ensure_list(value: Any) -> list: + if isinstance(value, list): + return value + if isinstance(value, str): + try: + parsed = json.loads(value or "[]") + return parsed if isinstance(parsed, list) else [] + except (TypeError, ValueError): + return [] + return [] + + +def _ensure_dict(value: Any) -> dict: + if isinstance(value, dict): + return value + if isinstance(value, str): + try: + parsed = json.loads(value or "{}") + return parsed if isinstance(parsed, dict) else {} + except (TypeError, ValueError): + return {} + return {} + + +def _decode_app(doc: dict | None) -> dict | None: + app = _clean(doc) + if not app: + return None + app["racl_entity_ids"] = _ensure_list(app.get("racl_entity_ids")) + app["banned_topics"] = _ensure_list(app.get("banned_topics")) + app.setdefault("is_active", True) + return app + + +def init_db() -> None: + db = _db() + db.app_config.create_index([("app_id", ASCENDING)], unique=True) + db.llm_config.create_index([("app_id", ASCENDING), ("agent_name", ASCENDING)], unique=True) + db.prompt_config.create_index([("app_id", ASCENDING), ("agent_name", ASCENDING), ("version", DESCENDING)]) + db.source_document.create_index([("doc_id", ASCENDING), ("app_id", ASCENDING)], unique=True) + db.golden_set.create_index([("version", ASCENDING), ("app_id", ASCENDING)], unique=True) + db.test_case.create_index([("tc_id", ASCENDING)], unique=True) + db.test_case.create_index([("app_id", ASCENDING), ("golden_set_version", ASCENDING), ("status", ASCENDING)]) + db.agent_scores.create_index([("tc_id", ASCENDING)], unique=True) + db.job.create_index([("job_id", ASCENDING)], unique=True) + db.job.create_index([("app_id", ASCENDING), ("created_at", DESCENDING)]) + db.eval_run.create_index([("run_id", ASCENDING)], unique=True) + db.eval_run.create_index([("app_id", ASCENDING), ("started_at", DESCENDING)]) + db.eval_result.create_index([("run_id", ASCENDING), ("tc_id", ASCENDING)], unique=True) + + for app in db.app_config.find({}, {"app_id": 1}): + _seed_default_llm_configs(app["app_id"]) + _seed_default_prompts(app["app_id"]) + + +def _seed_default_llm_configs(app_id: str) -> None: + now = _now() + for agent, cfg in DEFAULT_LLM.items(): + _c("llm_config").update_one( + {"app_id": app_id, "agent_name": agent}, + { + "$setOnInsert": { + "id": str(uuid.uuid4()), + "app_id": app_id, + "agent_name": agent, + "model": cfg["model"], + "temperature": cfg["temperature"], + "max_tokens": cfg["max_tokens"], + "updated_at": now, + } + }, + upsert=True, + ) + + +def _seed_default_prompts(app_id: str) -> None: + now = _now() + for agent_name, prompt_text in DEFAULT_PROMPTS.items(): + existing = _c("prompt_config").find_one( + {"app_id": app_id, "agent_name": agent_name, "is_active": True} + ) + if not existing: + _c("prompt_config").insert_one({ + "id": str(uuid.uuid4()), + "app_id": app_id, + "agent_name": agent_name, + "prompt_text": prompt_text, + "version": 1, + "is_active": True, + "created_at": now, + }) + + +def create_app(data: dict) -> dict: + app_id = f"app-{uuid.uuid4()}" + now = _now() + _c("app_config").insert_one({ + "app_id": app_id, + "name": data["name"], + "host_url": data["host_url"], + "bot_id": data["bot_id"], + "stream_id": data.get("stream_id"), + "client_id": data.get("client_id", ""), + "client_secret": data.get("client_secret", ""), + "jwt_token": data.get("jwt_token", ""), + "racl_entity_ids": data.get("racl_entity_ids", []), + "anthropic_key": data.get("anthropic_key", ""), + "anthropic_base_url": data.get("anthropic_base_url", ""), + "openai_key": data.get("openai_key", ""), + "openai_base_url": data.get("openai_base_url", ""), + "gemini_key": data.get("gemini_key", ""), + "gemini_base_url": data.get("gemini_base_url", ""), + "banned_topics": data.get("banned_topics", []), + "answer_mode": data.get("answer_mode", "answer_generation"), + "case1_threshold": float(data.get("case1_threshold") or 0.5), + "case2_threshold": float(data.get("case2_threshold") or 0.5), + "is_active": True, + "created_at": now, + "updated_at": now, + }) + _seed_default_llm_configs(app_id) + _seed_default_prompts(app_id) + return get_app(app_id) + + +def get_app(app_id: str) -> dict | None: + return _decode_app(_c("app_config").find_one({"app_id": app_id})) + + +def list_apps() -> list[dict]: + return [ + _decode_app(row) + for row in _c("app_config").find({}).sort("created_at", DESCENDING) + ] + + +def update_app(app_id: str, data: dict) -> dict | None: + fields = {k: v for k, v in data.items() if v is not None} + if not fields: + return get_app(app_id) + fields["updated_at"] = _now() + result = _c("app_config").update_one({"app_id": app_id}, {"$set": fields}) + return get_app(app_id) if result.matched_count else None + + +def delete_app(app_id: str) -> bool: + result = _c("app_config").delete_one({"app_id": app_id}) + if not result.deleted_count: + return False + tc_ids = [d["tc_id"] for d in _c("test_case").find({"app_id": app_id}, {"tc_id": 1})] + run_ids = [d["run_id"] for d in _c("eval_run").find({"app_id": app_id}, {"run_id": 1})] + for name in ( + "llm_config", "prompt_config", "source_document", "golden_set", + "test_case", "job", "eval_run", + ): + _c(name).delete_many({"app_id": app_id}) + if tc_ids: + _c("agent_scores").delete_many({"tc_id": {"$in": tc_ids}}) + if run_ids: + _c("eval_result").delete_many({"run_id": {"$in": run_ids}}) + return True + + +def get_api_key(app_id: str, provider: str) -> str | None: + app = get_app(app_id) + return (app or {}).get(f"{provider}_key") or None + + +def get_base_url(app_id: str, provider: str) -> str | None: + app = get_app(app_id) + url = (app or {}).get(f"{provider}_base_url") or "" + return url.strip() or None + + +def get_eval_thresholds(app_id: str) -> dict[str, float]: + app = get_app(app_id) or {} + return { + "case1_threshold": float(app.get("case1_threshold") or 0.5), + "case2_threshold": float(app.get("case2_threshold") or 0.5), + } + + +def get_llm_configs(app_id: str) -> list[dict]: + return [_clean(r) for r in _c("llm_config").find({"app_id": app_id})] + + +def get_llm_config(app_id: str, agent_name: str) -> dict: + row = _c("llm_config").find_one({"app_id": app_id, "agent_name": agent_name}) + if row: + return _clean(row) + return {**DEFAULT_LLM.get(agent_name, DEFAULT_LLM["agent1"]), "app_id": app_id, "agent_name": agent_name} + + +def upsert_llm_config(app_id: str, agent_name: str, data: dict) -> dict: + _c("llm_config").update_one( + {"app_id": app_id, "agent_name": agent_name}, + { + "$set": { + "model": data["model"], + "temperature": data["temperature"], + "max_tokens": data["max_tokens"], + "updated_at": _now(), + }, + "$setOnInsert": {"id": str(uuid.uuid4()), "app_id": app_id, "agent_name": agent_name}, + }, + upsert=True, + ) + return get_llm_config(app_id, agent_name) + + +def get_prompts(app_id: str) -> list[dict]: + return [_clean(r) for r in _c("prompt_config").find({"app_id": app_id}).sort([("agent_name", ASCENDING), ("version", DESCENDING)])] + + +def get_active_prompt(app_id: str, agent_name: str) -> dict | None: + return _clean(_c("prompt_config").find_one( + {"app_id": app_id, "agent_name": agent_name, "is_active": True}, + sort=[("version", DESCENDING)], + )) + + +def save_prompt(app_id: str, agent_name: str, prompt_text: str) -> dict: + coll = _c("prompt_config") + coll.update_many({"app_id": app_id, "agent_name": agent_name}, {"$set": {"is_active": False}}) + latest = coll.find_one({"app_id": app_id, "agent_name": agent_name}, sort=[("version", DESCENDING)]) + version = int((latest or {}).get("version") or 0) + 1 + coll.insert_one({ + "id": str(uuid.uuid4()), + "app_id": app_id, + "agent_name": agent_name, + "prompt_text": prompt_text, + "version": version, + "is_active": True, + "created_at": _now(), + }) + return get_active_prompt(app_id, agent_name) + + +def get_doc_source_type(app_id: str, doc_id: str) -> str | None: + row = _c("source_document").find_one({"app_id": app_id, "doc_id": doc_id}, {"sys_content_type": 1}) + return row.get("sys_content_type") if row else None + + +def get_doc_id_by_title(app_id: str, title: str) -> str | None: + row = _c("source_document").find_one( + {"app_id": app_id, "title": {"$regex": f"^{re.escape(title)}$", "$options": "i"}}, + {"doc_id": 1}, + ) + return row.get("doc_id") if row else None + + +def upsert_source_document(doc: dict) -> None: + payload = { + "doc_id": doc["doc_id"], + "app_id": doc["app_id"], + "title": doc.get("title"), + "content_hash": doc.get("content_hash"), + "content": doc.get("content"), + "metadata": _ensure_dict(doc.get("metadata")), + "sys_content_type": doc.get("sys_content_type"), + "source_url": doc.get("source_url"), + "connector_id": doc.get("connector_id"), + "ingested_at": _now(), + } + _c("source_document").update_one( + {"doc_id": doc["doc_id"], "app_id": doc["app_id"]}, + {"$set": payload}, + upsert=True, + ) + + +def get_golden_set(app_id: str, version: str) -> dict | None: + return _clean(_c("golden_set").find_one({"app_id": app_id, "version": version})) + + +def create_golden_set(app_id: str, version: str, notes: str = "") -> None: + _c("golden_set").update_one( + {"app_id": app_id, "version": version}, + {"$setOnInsert": {"version": version, "app_id": app_id, "created_at": _now(), "frozen_at": None, "parent_version": None, "notes": notes}}, + upsert=True, + ) + + +def freeze_golden_set(app_id: str, version: str) -> None: + _c("golden_set").update_one({"app_id": app_id, "version": version}, {"$set": {"frozen_at": _now()}}) + + +def list_golden_sets(app_id: str) -> list[dict]: + out = [] + for gs in _c("golden_set").find({"app_id": app_id}).sort("created_at", DESCENDING): + d = _clean(gs) + cases = list(_c("test_case").find({"app_id": app_id, "golden_set_version": d["version"]}, {"tc_id": 1})) + tc_ids = [c["tc_id"] for c in cases] + scores = list(_c("agent_scores").find({"tc_id": {"$in": tc_ids}}, {"decision": 1})) if tc_ids else [] + d["total_cases"] = len(cases) + d["kept_cases"] = sum(1 for s in scores if s.get("decision") == "KEEP") + d["borderline_cases"] = sum(1 for s in scores if s.get("decision") == "BORDERLINE") + out.append(d) + return out + + +def _decode_match_spec(row: dict) -> list[dict]: + spec = _ensure_list(row.get("reference_match_spec")) + if spec: + return spec + return [{"field": "docId", "value": d} for d in _ensure_list(row.get("reference_doc_ids")) if d] + + +def _with_score(tc: dict) -> dict: + d = _clean(tc) + score = _clean(_c("agent_scores").find_one({"tc_id": d["tc_id"]})) or {} + d.update({k: v for k, v in score.items() if k not in {"tc_id"}}) + d["reference_doc_ids"] = _ensure_list(d.get("reference_doc_ids")) + d["reference_match_spec"] = _decode_match_spec(d) + return d + + +def list_test_cases(app_id: str, golden_set_version: str) -> list[dict]: + rows = _c("test_case").find({"app_id": app_id, "golden_set_version": golden_set_version}).sort("created_at", DESCENDING) + return [_with_score(r) for r in rows] + + +def get_active_test_cases(app_id: str, golden_set_version: str) -> list[dict]: + rows = _c("test_case").find({"app_id": app_id, "golden_set_version": golden_set_version, "status": "active"}).sort("created_at", ASCENDING) + out = [] + for row in rows: + d = _with_score(row) + if d.get("decision") in ("KEEP", "BORDERLINE"): + d["generation_metadata"] = _ensure_dict(d.get("generation_metadata")) + out.append(d) + return out + + +def insert_test_case(tc: dict) -> None: + payload = { + **tc, + "reference_doc_ids": _ensure_list(tc.get("reference_doc_ids")), + "reference_match_spec": _ensure_list(tc.get("reference_match_spec")), + "generation_metadata": _ensure_dict(tc.get("generation_metadata")), + "human_validated": bool(tc.get("human_validated", False)), + "status": tc.get("status", "active"), + "created_at": tc.get("created_at") or _now(), + } + _c("test_case").update_one({"tc_id": tc["tc_id"]}, {"$set": payload}, upsert=True) + + +def import_uploaded_test_cases(app_id: str, version: str, cases: list[dict]) -> int: + count = 0 + for case in cases: + tc_id = f"tc-{uuid.uuid4()}" + meta = {} + if case.get("sys_content_type"): + meta["sys_content_type"] = case["sys_content_type"] + insert_test_case({ + "tc_id": tc_id, + "app_id": app_id, + "golden_set_version": version, + "question": case["question"], + "expected_answer": case.get("expected_answer") or None, + "expected_behavior": case.get("expected_behavior", "ANSWER"), + "question_type": case.get("question_type") or None, + "difficulty": case.get("difficulty") or None, + "reference_doc_ids": case.get("reference_doc_ids", []), + "reference_match_spec": case.get("reference_match_spec", []), + "generation_metadata": meta, + "custom_fields": case.get("custom_fields") or {}, + "human_validated": True, + "status": "active", + }) + insert_agent_scores({"tc_id": tc_id, "decision": "KEEP"}) + count += 1 + return count + + +def insert_agent_scores(scores: dict) -> None: + payload = { + "tc_id": scores["tc_id"], + "clarity": scores.get("clarity"), + "specificity": scores.get("specificity"), + "meaningfulness": scores.get("meaningfulness"), + "answerability": scores.get("answerability"), + "reference_verifiability": scores.get("reference_verifiability"), + "answer_uniqueness": scores.get("answer_uniqueness"), + "decision": scores.get("decision"), + "primary_concern": scores.get("primary_concern"), + "rationale": scores.get("rationale"), + } + if isinstance(scores.get("scores"), dict): + payload.update(scores["scores"]) + _c("agent_scores").update_one({"tc_id": scores["tc_id"]}, {"$set": payload}, upsert=True) + + +def create_job(app_id: str, job_type: str) -> str: + job_id = f"job-{uuid.uuid4()}" + now = _now() + _c("job").insert_one({ + "job_id": job_id, "app_id": app_id, "job_type": job_type, + "status": "running", "progress": 0, "result": None, "error": None, + "stop_requested": False, "created_at": now, "updated_at": now, + }) + return job_id + + +def update_job(job_id: str, status: str, progress: int = 0, result: dict | None = None, error: str | None = None) -> None: + _c("job").update_one( + {"job_id": job_id}, + {"$set": {"status": status, "progress": progress, "result": result, "error": error, "updated_at": _now()}}, + ) + + +def get_job(job_id: str) -> dict | None: + return _clean(_c("job").find_one({"job_id": job_id})) + + +def list_jobs(app_id: str) -> list[dict]: + return [_clean(r) for r in _c("job").find({"app_id": app_id}).sort("created_at", DESCENDING).limit(50)] + + +def request_stop_job(job_id: str) -> None: + _c("job").update_one({"job_id": job_id}, {"$set": {"stop_requested": True, "updated_at": _now()}}) + + +def is_stop_requested(job_id: str) -> bool: + row = _c("job").find_one({"job_id": job_id}, {"stop_requested": 1}) + return bool(row and row.get("stop_requested")) + + +def create_eval_run(run: dict) -> None: + _c("eval_run").insert_one({ + "run_id": run["run_id"], + "app_id": run["app_id"], + "started_at": run.get("started_at") or _now(), + "finished_at": None, + "rag_version": run.get("rag_version", "unknown"), + "judge_model": run.get("judge_model"), + "golden_set_version": run["golden_set_version"], + "trigger": run.get("trigger", "manual"), + "status": "running", + "cost_usd": 0, + "total_cases": run.get("total_cases", 0), + "passed_cases": 0, + "verdicted_cases": 0, + }) + + +def upsert_eval_result(result: dict) -> None: + payload = { + **result, + "retrieved_doc_ids": _ensure_list(result.get("retrieved_doc_ids")), + "chunk_signals": _ensure_list(result.get("chunk_signals")), + "scores": _ensure_dict(result.get("scores")), + "search_payload": _ensure_dict(result.get("search_payload")), + "search_response": _ensure_dict(result.get("search_response")), + "recall_at_k": _ensure_dict(result.get("recall_at_k")), + "attempt_count": result.get("attempt_count", 1), + } + _c("eval_result").update_one( + {"run_id": result["run_id"], "tc_id": result["tc_id"]}, + {"$set": payload}, + upsert=True, + ) + + +def bulk_update_verdicts(run_id: str, verdicts: list[tuple[str, str | None, str]]) -> None: + coll = _c("eval_result") + for tc_id, verdict, source in verdicts: + coll.update_one({"run_id": run_id, "tc_id": tc_id}, {"$set": {"verdict": verdict, "verdict_source": source}}) + + +def update_run_verdict_counts(run_id: str, passed: int, verdicted: int) -> None: + _c("eval_run").update_one({"run_id": run_id}, {"$set": {"passed_cases": passed, "verdicted_cases": verdicted}}) + + +def finish_eval_run(run_id: str, passed: int, cost: float, status: str = "complete", avg_chunk_rank: float | None = None, verdicted: int | None = None) -> None: + _c("eval_run").update_one( + {"run_id": run_id}, + {"$set": { + "finished_at": _now(), "status": status, "passed_cases": passed, + "cost_usd": cost, "avg_chunk_rank": avg_chunk_rank, + "verdicted_cases": verdicted if verdicted is not None else passed, + }}, + ) + + +def list_eval_runs(app_id: str) -> list[dict]: + return [_clean(r) for r in _c("eval_run").find({"app_id": app_id}).sort("started_at", DESCENDING)] + + +def get_eval_results(run_id: str) -> list[dict]: + out = [] + for er in _c("eval_result").find({"run_id": run_id}): + d = _clean(er) + tc = _clean(_c("test_case").find_one({"tc_id": d["tc_id"]})) or {} + d.update({ + "question": tc.get("question"), + "expected_answer": tc.get("expected_answer"), + "expected_behavior": tc.get("expected_behavior", "ANSWER"), + "question_type": tc.get("question_type"), + "difficulty": tc.get("difficulty"), + "reference_doc_ids": _ensure_list(tc.get("reference_doc_ids")), + }) + d["retrieved_doc_ids"] = _ensure_list(d.get("retrieved_doc_ids")) + d["scores"] = _ensure_dict(d.get("scores")) + d["search_payload"] = _ensure_dict(d.get("search_payload")) + d["search_response"] = _ensure_dict(d.get("search_response")) + d["recall_at_k"] = _ensure_dict(d.get("recall_at_k")) + out.append(d) + return sorted(out, key=lambda r: str(r.get("question_type") or "")) + + +def get_completed_tc_ids(run_id: str) -> set[str]: + return {r["tc_id"] for r in _c("eval_result").find({"run_id": run_id}, {"tc_id": 1})} + + +def delete_eval_run(app_id: str, run_id: str) -> int: + n = _c("eval_result").count_documents({"run_id": run_id}) + result = _c("eval_run").delete_one({"run_id": run_id, "app_id": app_id}) + if result.deleted_count: + _c("eval_result").delete_many({"run_id": run_id}) + return int(n) + + +def count_runs_using_golden_set(app_id: str, version: str) -> int: + return int(_c("eval_run").count_documents({"app_id": app_id, "golden_set_version": version})) + + +def delete_golden_set(app_id: str, version: str) -> dict: + tc_ids = [r["tc_id"] for r in _c("test_case").find({"app_id": app_id, "golden_set_version": version}, {"tc_id": 1})] + run_refs = count_runs_using_golden_set(app_id, version) + deleted = _c("test_case").delete_many({"app_id": app_id, "golden_set_version": version}).deleted_count + if tc_ids: + _c("agent_scores").delete_many({"tc_id": {"$in": tc_ids}}) + _c("golden_set").delete_one({"app_id": app_id, "version": version}) + return {"test_cases_deleted": int(deleted), "eval_runs_orphaned": int(run_refs)} + + +def get_eval_run(run_id: str) -> dict | None: + return _clean(_c("eval_run").find_one({"run_id": run_id})) + + +def get_previous_completed_run(app_id: str, golden_set_version: str, before_run_id: str) -> dict | None: + anchor = get_eval_run(before_run_id) + if not anchor: + return None + return _clean(_c("eval_run").find_one( + { + "app_id": app_id, + "golden_set_version": golden_set_version, + "status": {"$in": ["complete", "partial"]}, + "run_id": {"$ne": before_run_id}, + "started_at": {"$lt": anchor.get("started_at", "")}, + }, + sort=[("started_at", DESCENDING)], + )) + + +def list_completed_runs(app_id: str, limit: int = 50) -> list[dict]: + return [_clean(r) for r in _c("eval_run").find( + {"app_id": app_id, "status": {"$in": ["complete", "partial"]}} + ).sort("started_at", DESCENDING).limit(limit)] + + +def get_run_diagnostics(run_id: str) -> tuple[dict | None, list[str] | None]: + row = _c("eval_run").find_one({"run_id": run_id}, {"diagnostics_json": 1, "fired_rule_ids": 1}) + if not row: + return None, None + return _ensure_dict(row.get("diagnostics_json")) or None, _ensure_list(row.get("fired_rule_ids")) or None + + +def set_run_diagnostics(run_id: str, diagnostics: dict, fired_rule_ids: list[str]) -> None: + _c("eval_run").update_one( + {"run_id": run_id}, + {"$set": {"diagnostics_json": diagnostics, "fired_rule_ids": fired_rule_ids}}, + ) + + +def get_run_ai_insights(run_id: str) -> dict | None: + row = _c("eval_run").find_one({"run_id": run_id}, {"ai_insights_md": 1, "ai_insights_model": 1, "ai_insights_generated_at": 1}) + if not row or not row.get("ai_insights_md"): + return None + return { + "markdown": row.get("ai_insights_md"), + "model": row.get("ai_insights_model"), + "generated_at": row.get("ai_insights_generated_at"), + } + + +def set_run_ai_insights(run_id: str, markdown: str, model: str) -> None: + _c("eval_run").update_one( + {"run_id": run_id}, + {"$set": {"ai_insights_md": markdown, "ai_insights_model": model, "ai_insights_generated_at": _now()}}, + ) diff --git a/Evaluation/AISearchEvalutionTool/backend/db/schema.sql b/Evaluation/AISearchEvalutionTool/backend/db/schema.sql index 86d170f9..ebe900e1 100644 --- a/Evaluation/AISearchEvalutionTool/backend/db/schema.sql +++ b/Evaluation/AISearchEvalutionTool/backend/db/schema.sql @@ -15,6 +15,8 @@ CREATE TABLE IF NOT EXISTS app_config ( anthropic_base_url TEXT NOT NULL DEFAULT '', openai_key TEXT NOT NULL DEFAULT '', openai_base_url TEXT NOT NULL DEFAULT '', + gemini_key TEXT NOT NULL DEFAULT '', + gemini_base_url TEXT NOT NULL DEFAULT '', banned_topics TEXT NOT NULL DEFAULT '[]', -- JSON array of strings answer_mode TEXT NOT NULL DEFAULT 'answer_generation', -- 'answer_generation' | 'extract_only' case1_threshold REAL NOT NULL DEFAULT 0.5, -- semantic Q↔Answer similarity pass threshold @@ -158,7 +160,8 @@ CREATE TABLE IF NOT EXISTS eval_result ( latency_llm_ms INTEGER, latency_retrieval_ms INTEGER, search_request_id TEXT, - search_payload TEXT DEFAULT '{}', + search_payload TEXT DEFAULT '{}', -- request body sent to Kore.ai (JSON) + search_response TEXT DEFAULT '{}', -- full raw Kore.ai response (JSON) attempt_count INTEGER DEFAULT 1, case_id INTEGER, -- 1..4 derived from test_case columns present expected_doc_rank INTEGER, -- 1-based rank of expected doc in retrieved (null if none/not found) diff --git a/Evaluation/AISearchEvalutionTool/backend/db/sqlite_store.py b/Evaluation/AISearchEvalutionTool/backend/db/sqlite_store.py new file mode 100644 index 00000000..57bb4b09 --- /dev/null +++ b/Evaluation/AISearchEvalutionTool/backend/db/sqlite_store.py @@ -0,0 +1,1028 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +import uuid +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Generator + +from config import get_config + +# ── Database connection ─────────────────────────────────────────────────────── +# Priority: +# 1. Turso (TURSO_DATABASE_URL + TURSO_AUTH_TOKEN) — remote hosted SQLite +# 2. Local SQLite via DB_PATH env var or default ./data/eval.db + +_TURSO_URL = os.getenv("TURSO_DATABASE_URL", "") +_TURSO_TOKEN = os.getenv("TURSO_AUTH_TOKEN", "") + +DB_PATH = get_config().database.sqlite_db_path + + +def _connect() -> sqlite3.Connection: + if _TURSO_URL: + import libsql_experimental as libsql # type: ignore + # Embedded replica: local cache at /tmp synced from Turso + conn = libsql.connect("/tmp/eval_turso.db", sync_url=_TURSO_URL, auth_token=_TURSO_TOKEN) + conn.sync() + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys=ON") + return conn # type: ignore[return-value] + + DB_PATH.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(str(DB_PATH)) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA foreign_keys=ON") + return conn + + +@contextmanager +def get_db() -> Generator[sqlite3.Connection, None, None]: + conn = _connect() + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + +def init_db() -> None: + schema = (Path(__file__).parent / "schema.sql").read_text() + with get_db() as conn: + conn.executescript(schema) + _migrate(conn) + + +def _migrate(conn: sqlite3.Connection) -> None: + migrations = [ + "ALTER TABLE app_config ADD COLUMN client_id TEXT NOT NULL DEFAULT ''", + "ALTER TABLE app_config ADD COLUMN client_secret TEXT NOT NULL DEFAULT ''", + "ALTER TABLE app_config ADD COLUMN anthropic_key TEXT NOT NULL DEFAULT ''", + "ALTER TABLE app_config ADD COLUMN openai_key TEXT NOT NULL DEFAULT ''", + "ALTER TABLE app_config ADD COLUMN banned_topics TEXT NOT NULL DEFAULT '[]'", + "ALTER TABLE eval_result ADD COLUMN search_payload TEXT DEFAULT '{}'", + "ALTER TABLE app_config ADD COLUMN answer_mode TEXT NOT NULL DEFAULT 'answer_generation'", + "ALTER TABLE eval_run ADD COLUMN avg_chunk_rank REAL", + "ALTER TABLE job ADD COLUMN stop_requested INTEGER DEFAULT 0", + # 4-case evaluation columns + "ALTER TABLE eval_result ADD COLUMN case_id INTEGER", + "ALTER TABLE eval_result ADD COLUMN expected_doc_rank INTEGER", + "ALTER TABLE eval_result ADD COLUMN recall_at_k TEXT DEFAULT '{}'", + "ALTER TABLE eval_result ADD COLUMN answer_similarity REAL", + "ALTER TABLE eval_result ADD COLUMN verdict TEXT", + "ALTER TABLE eval_result ADD COLUMN verdict_source TEXT", + # Per-app custom LLM endpoints (Azure / OpenRouter / vLLM / Ollama etc.) + "ALTER TABLE app_config ADD COLUMN anthropic_base_url TEXT NOT NULL DEFAULT ''", + "ALTER TABLE app_config ADD COLUMN openai_base_url TEXT NOT NULL DEFAULT ''", + "ALTER TABLE app_config ADD COLUMN gemini_key TEXT NOT NULL DEFAULT ''", + "ALTER TABLE app_config ADD COLUMN gemini_base_url TEXT NOT NULL DEFAULT ''", + # Semantic similarity pass thresholds for Cases 1 & 2 (judge-less mode) + "ALTER TABLE app_config ADD COLUMN case1_threshold REAL NOT NULL DEFAULT 0.5", + "ALTER TABLE app_config ADD COLUMN case2_threshold REAL NOT NULL DEFAULT 0.5", + # Flexible match spec: [{"field": "recordUrl", "value": "..."}] — any chunk JSON field + "ALTER TABLE test_case ADD COLUMN reference_match_spec TEXT DEFAULT '[]'", + # Rows with a real pass/fail verdict (excludes null-verdict Case-1 rows) + "ALTER TABLE eval_run ADD COLUMN verdicted_cases INTEGER DEFAULT 0", + # ── Phase 2: Insights (diagnostics + rules + AI narrative cache) ───── + "ALTER TABLE eval_run ADD COLUMN diagnostics_json TEXT", + "ALTER TABLE eval_run ADD COLUMN fired_rule_ids TEXT", + "ALTER TABLE eval_run ADD COLUMN ai_insights_md TEXT", + "ALTER TABLE eval_run ADD COLUMN ai_insights_model TEXT", + "ALTER TABLE eval_run ADD COLUMN ai_insights_generated_at TEXT", + # Full raw Kore.ai response per test case (for debugging / UI display) + "ALTER TABLE eval_result ADD COLUMN search_response TEXT DEFAULT '{}'", + ] + for sql in migrations: + try: + conn.execute(sql) + conn.commit() + except Exception: + pass # column already exists + + # Make test_case.expected_answer nullable for legacy DBs (SQLite can't ALTER + # COLUMN, so rebuild only if the NOT NULL constraint is still present). + _migrate_expected_answer_nullable(conn) + + # Data migration: switch any rows still using the old Anthropic-only defaults + # to gpt-4.1 so apps without an Anthropic key work out of the box. + conn.execute( + """UPDATE llm_config SET model='gpt-4.1' + WHERE agent_name IN ('agent1','agent2','agent3') + AND model='claude-sonnet-4-6'""" + ) + conn.commit() + + # Phase 2: backfill the new 'insights' agent row + prompt for existing apps + _backfill_insights_agent(conn) + # Backfill the new 'answer_generator' agent row + prompt for existing apps + _backfill_agent(conn, "answer_generator") + + # One-time bump: any insights row still at the old 2000-token default is too + # tight for the ~600-word markdown report and outright unusable for reasoning + # models (o-series / gpt-5) where thinking tokens consume the budget before + # any visible content is emitted. Bump to the new 4000 floor; preserves any + # value the user has deliberately customised (>= 4000). + conn.execute( + """UPDATE llm_config SET max_tokens=4000 + WHERE agent_name='insights' AND max_tokens < 4000""" + ) + conn.commit() + + +def _backfill_insights_agent(conn: sqlite3.Connection) -> None: + """Seed the new 'insights' agent (LLM config + prompt) for pre-Phase-2 apps. + + Skips apps that already have an insights row, so re-running is safe. + """ + _backfill_agent(conn, "insights") + + +def _backfill_agent(conn: sqlite3.Connection, agent_name: str) -> None: + """Seed an agent's default LLM config + active prompt for every existing app. + + Idempotent: relies on ``INSERT OR IGNORE`` for the llm_config row and on an + existence check for the prompt row, so re-running is safe. + Imported lazily inside the function to avoid bootstrap-import cycles. + """ + from agents.prompts import DEFAULT_PROMPTS + prompt_text = DEFAULT_PROMPTS.get(agent_name) + cfg = DEFAULT_LLM.get(agent_name) + if not prompt_text or not cfg: + return + rows = conn.execute("SELECT app_id FROM app_config").fetchall() + for r in rows: + app_id = r["app_id"] + conn.execute( + """INSERT OR IGNORE INTO llm_config + (id, app_id, agent_name, model, temperature, max_tokens) + VALUES (?,?,?,?,?,?)""", + (str(uuid.uuid4()), app_id, agent_name, + cfg["model"], cfg["temperature"], cfg["max_tokens"]), + ) + existing = conn.execute( + "SELECT id FROM prompt_config WHERE app_id=? AND agent_name=? AND is_active=1", + (app_id, agent_name), + ).fetchone() + if not existing: + conn.execute( + """INSERT INTO prompt_config (id, app_id, agent_name, prompt_text, version, is_active) + VALUES (?,?,?,?,1,1)""", + (str(uuid.uuid4()), app_id, agent_name, prompt_text), + ) + conn.commit() + + +def _migrate_expected_answer_nullable(conn: sqlite3.Connection) -> None: + """Drop NOT NULL on test_case.expected_answer if present. SQLite-safe rebuild.""" + try: + cols = conn.execute("PRAGMA table_info(test_case)").fetchall() + except Exception: + return + for c in cols: + # PRAGMA table_info columns: cid, name, type, notnull, dflt_value, pk + if c["name"] == "expected_answer" and c["notnull"] == 1: + break + else: + return # already nullable or column missing + + conn.execute("PRAGMA foreign_keys=OFF") + try: + conn.execute("BEGIN") + conn.execute(""" + CREATE TABLE test_case__new ( + tc_id TEXT PRIMARY KEY, + app_id TEXT NOT NULL REFERENCES app_config(app_id) ON DELETE CASCADE, + golden_set_version TEXT NOT NULL, + question TEXT NOT NULL, + expected_answer TEXT, + expected_behavior TEXT CHECK(expected_behavior IN ('ANSWER','REFUSE','CLARIFY')) DEFAULT 'ANSWER', + question_type TEXT, + difficulty INTEGER CHECK(difficulty IN (1,2,3)), + answer_type TEXT, + reference_doc_ids TEXT DEFAULT '[]', + reference_match_spec TEXT DEFAULT '[]', + generation_metadata TEXT DEFAULT '{}', + human_validated INTEGER DEFAULT 0, + status TEXT DEFAULT 'active' CHECK(status IN ('active','retired','invalidated')), + rationale TEXT, + created_at TEXT DEFAULT (datetime('now')) + ) + """) + conn.execute(""" + INSERT INTO test_case__new + SELECT tc_id, app_id, golden_set_version, question, expected_answer, + expected_behavior, question_type, difficulty, answer_type, + reference_doc_ids, '[]', generation_metadata, human_validated, + status, rationale, created_at + FROM test_case + """) + conn.execute("DROP TABLE test_case") + conn.execute("ALTER TABLE test_case__new RENAME TO test_case") + conn.execute("COMMIT") + except Exception: + conn.execute("ROLLBACK") + raise + finally: + conn.execute("PRAGMA foreign_keys=ON") + + +# ── App Config ────────────────────────────────────────────────────────────── + +def create_app(data: dict) -> dict: + app_id = f"app-{uuid.uuid4()}" + with get_db() as conn: + conn.execute( + """INSERT INTO app_config + (app_id, name, host_url, bot_id, stream_id, + client_id, client_secret, jwt_token, racl_entity_ids, + anthropic_key, anthropic_base_url, + openai_key, openai_base_url, + gemini_key, gemini_base_url, answer_mode) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", + (app_id, data["name"], data["host_url"], data["bot_id"], + data.get("stream_id"), data.get("client_id", ""), + data.get("client_secret", ""), data["jwt_token"], + json.dumps(data.get("racl_entity_ids", [])), + data.get("anthropic_key", ""), + data.get("anthropic_base_url", ""), + data.get("openai_key", ""), + data.get("openai_base_url", ""), + data.get("gemini_key", ""), + data.get("gemini_base_url", ""), + data.get("answer_mode", "answer_generation")), + ) + _seed_default_llm_configs(conn, app_id) + _seed_default_prompts(conn, app_id) + return get_app(app_id) + + +def _decode_app_row(row) -> dict: + d = dict(row) + d["racl_entity_ids"] = json.loads(d.get("racl_entity_ids") or "[]") + d["banned_topics"] = json.loads(d.get("banned_topics") or "[]") + return d + + +def get_app(app_id: str) -> dict | None: + with get_db() as conn: + row = conn.execute( + "SELECT * FROM app_config WHERE app_id=?", (app_id,) + ).fetchone() + return _decode_app_row(row) if row else None + + +def list_apps() -> list[dict]: + with get_db() as conn: + rows = conn.execute( + "SELECT * FROM app_config ORDER BY created_at DESC" + ).fetchall() + return [_decode_app_row(r) for r in rows] + + +def update_app(app_id: str, data: dict) -> dict | None: + fields = {k: v for k, v in data.items() if v is not None} + if "racl_entity_ids" in fields: + fields["racl_entity_ids"] = json.dumps(fields["racl_entity_ids"]) + if "banned_topics" in fields: + fields["banned_topics"] = json.dumps(fields["banned_topics"]) + if not fields: + return get_app(app_id) + set_clause = ", ".join(f"{k}=?" for k in fields) + values = list(fields.values()) + [app_id] + with get_db() as conn: + conn.execute( + f"UPDATE app_config SET {set_clause}, updated_at=datetime('now') WHERE app_id=?", + values, + ) + return get_app(app_id) + + +def delete_app(app_id: str) -> bool: + with get_db() as conn: + cur = conn.execute("DELETE FROM app_config WHERE app_id=?", (app_id,)) + return cur.rowcount > 0 + + +def get_api_key(app_id: str, provider: str) -> str | None: + """Return the app-specific API key stored in the database.""" + app = get_app(app_id) + return (app or {}).get(f"{provider}_key") or None + + +def get_base_url(app_id: str, provider: str) -> str | None: + """Return the app-specific LLM base URL (custom endpoint), or None to use the SDK default.""" + app = get_app(app_id) + url = (app or {}).get(f"{provider}_base_url") or "" + return url.strip() or None + + +def get_eval_thresholds(app_id: str) -> dict[str, float]: + """Return Case 1 / Case 2 semantic similarity pass thresholds for the app.""" + app = get_app(app_id) or {} + return { + "case1_threshold": float(app.get("case1_threshold") or 0.5), + "case2_threshold": float(app.get("case2_threshold") or 0.5), + } + + +# ── LLM Config ────────────────────────────────────────────────────────────── + +DEFAULT_LLM = { + "agent1": {"model": "gpt-4.1", "temperature": 0.0, "max_tokens": 4000}, + "agent2": {"model": "gpt-4.1", "temperature": 0.7, "max_tokens": 3000}, + "agent3": {"model": "gpt-4.1", "temperature": 0.0, "max_tokens": 2500}, + "judge": {"model": "gpt-4.1", "temperature": 0.0, "max_tokens": 800}, + "filter_generator": {"model": "gpt-4.1-mini", "temperature": 0.0, "max_tokens": 800}, + # Phase 2 — AI Deep Dive narrative. User-configurable via LLM Config UI. + # 4000 tokens covers the ~600-word markdown report comfortably and gives + # reasoning models (o-series, gpt-5) some headroom for internal thinking. + "insights": {"model": "gpt-4.1", "temperature": 0.3, "max_tokens": 4000}, + # Answer generator — RAG-style "given context + question produce an answer" + # prompt. Editable + tunable; the actual answer rendering still goes through + # Kore.ai today, this row exists so users can manage / fine-tune the prompt. + "answer_generator": {"model": "gpt-4.1", "temperature": 0.2, "max_tokens": 1200}, +} + + +def _seed_default_llm_configs(conn: sqlite3.Connection, app_id: str) -> None: + for agent, cfg in DEFAULT_LLM.items(): + conn.execute( + """INSERT OR IGNORE INTO llm_config (id, app_id, agent_name, model, temperature, max_tokens) + VALUES (?,?,?,?,?,?)""", + (str(uuid.uuid4()), app_id, agent, cfg["model"], cfg["temperature"], cfg["max_tokens"]), + ) + + +def get_llm_configs(app_id: str) -> list[dict]: + with get_db() as conn: + rows = conn.execute( + "SELECT * FROM llm_config WHERE app_id=?", (app_id,) + ).fetchall() + return [dict(r) for r in rows] + + +def get_llm_config(app_id: str, agent_name: str) -> dict: + with get_db() as conn: + row = conn.execute( + "SELECT * FROM llm_config WHERE app_id=? AND agent_name=?", + (app_id, agent_name), + ).fetchone() + if row: + return dict(row) + return {**DEFAULT_LLM.get(agent_name, DEFAULT_LLM["agent1"]), + "app_id": app_id, "agent_name": agent_name} + + +def upsert_llm_config(app_id: str, agent_name: str, data: dict) -> dict: + with get_db() as conn: + conn.execute( + """INSERT INTO llm_config (id, app_id, agent_name, model, temperature, max_tokens) + VALUES (?,?,?,?,?,?) + ON CONFLICT(app_id, agent_name) DO UPDATE SET + model=excluded.model, temperature=excluded.temperature, + max_tokens=excluded.max_tokens, updated_at=datetime('now')""", + (str(uuid.uuid4()), app_id, agent_name, + data["model"], data["temperature"], data["max_tokens"]), + ) + return get_llm_config(app_id, agent_name) + + +# ── Prompt Config ──────────────────────────────────────────────────────────── + +from agents.prompts import DEFAULT_PROMPTS + + +def _seed_default_prompts(conn: sqlite3.Connection, app_id: str) -> None: + for agent_name, prompt_text in DEFAULT_PROMPTS.items(): + existing = conn.execute( + "SELECT id FROM prompt_config WHERE app_id=? AND agent_name=? AND is_active=1", + (app_id, agent_name), + ).fetchone() + if not existing: + conn.execute( + """INSERT INTO prompt_config (id, app_id, agent_name, prompt_text, version, is_active) + VALUES (?,?,?,?,1,1)""", + (str(uuid.uuid4()), app_id, agent_name, prompt_text), + ) + + +def get_prompts(app_id: str) -> list[dict]: + with get_db() as conn: + rows = conn.execute( + "SELECT * FROM prompt_config WHERE app_id=? ORDER BY agent_name, version DESC", + (app_id,), + ).fetchall() + return [dict(r) for r in rows] + + +def get_active_prompt(app_id: str, agent_name: str) -> dict | None: + with get_db() as conn: + row = conn.execute( + """SELECT * FROM prompt_config + WHERE app_id=? AND agent_name=? AND is_active=1 + ORDER BY version DESC LIMIT 1""", + (app_id, agent_name), + ).fetchone() + return dict(row) if row else None + + +def save_prompt(app_id: str, agent_name: str, prompt_text: str) -> dict: + with get_db() as conn: + # Deactivate existing + conn.execute( + "UPDATE prompt_config SET is_active=0 WHERE app_id=? AND agent_name=?", + (app_id, agent_name), + ) + # Get next version + row = conn.execute( + "SELECT MAX(version) as v FROM prompt_config WHERE app_id=? AND agent_name=?", + (app_id, agent_name), + ).fetchone() + version = (row["v"] or 0) + 1 + prompt_id = str(uuid.uuid4()) + conn.execute( + """INSERT INTO prompt_config (id, app_id, agent_name, prompt_text, version, is_active) + VALUES (?,?,?,?,?,1)""", + (prompt_id, app_id, agent_name, prompt_text, version), + ) + return get_active_prompt(app_id, agent_name) + + +# ── Source Documents ───────────────────────────────────────────────────────── + +def get_doc_source_type(app_id: str, doc_id: str) -> str | None: + """Lookup sys_content_type (Kore.ai source type) for a document.""" + with get_db() as conn: + row = conn.execute( + "SELECT sys_content_type FROM source_document WHERE app_id=? AND doc_id=?", + (app_id, doc_id), + ).fetchone() + return row["sys_content_type"] if row else None + + +def get_doc_id_by_title(app_id: str, title: str) -> str | None: + """Find a doc_id by matching title (case-insensitive) in source_document.""" + with get_db() as conn: + row = conn.execute( + "SELECT doc_id FROM source_document WHERE app_id=? AND title=? COLLATE NOCASE LIMIT 1", + (app_id, title), + ).fetchone() + return row["doc_id"] if row else None + + +def upsert_source_document(doc: dict) -> None: + with get_db() as conn: + conn.execute( + """INSERT OR REPLACE INTO source_document + (doc_id, app_id, title, content_hash, content, metadata, + sys_content_type, source_url, connector_id) + VALUES (?,?,?,?,?,?,?,?,?)""", + (doc["doc_id"], doc["app_id"], doc.get("title"), doc.get("content_hash"), + doc.get("content"), json.dumps(doc.get("metadata", {})), + doc.get("sys_content_type"), doc.get("source_url"), doc.get("connector_id")), + ) + + +# ── Golden Sets ────────────────────────────────────────────────────────────── + +def get_golden_set(app_id: str, version: str) -> dict | None: + with get_db() as conn: + row = conn.execute( + "SELECT * FROM golden_set WHERE app_id=? AND version=?", + (app_id, version), + ).fetchone() + return dict(row) if row else None + + +def create_golden_set(app_id: str, version: str, notes: str = "") -> None: + with get_db() as conn: + conn.execute( + "INSERT OR IGNORE INTO golden_set (version, app_id, notes) VALUES (?,?,?)", + (version, app_id, notes), + ) + + +def freeze_golden_set(app_id: str, version: str) -> None: + with get_db() as conn: + conn.execute( + "UPDATE golden_set SET frozen_at=datetime('now') WHERE version=? AND app_id=?", + (version, app_id), + ) + + +def list_golden_sets(app_id: str) -> list[dict]: + with get_db() as conn: + rows = conn.execute( + """SELECT g.*, + COUNT(tc.tc_id) as total_cases, + COALESCE(SUM(CASE WHEN s.decision='KEEP' THEN 1 ELSE 0 END), 0) as kept_cases, + COALESCE(SUM(CASE WHEN s.decision='BORDERLINE' THEN 1 ELSE 0 END), 0) as borderline_cases + FROM golden_set g + LEFT JOIN test_case tc ON tc.golden_set_version=g.version AND tc.app_id=g.app_id + LEFT JOIN agent_scores s ON s.tc_id=tc.tc_id + WHERE g.app_id=? + GROUP BY g.version, g.app_id + ORDER BY g.created_at DESC""", + (app_id,), + ).fetchall() + return [dict(r) for r in rows] + + +def list_test_cases(app_id: str, golden_set_version: str) -> list[dict]: + with get_db() as conn: + rows = conn.execute( + """SELECT tc.*, s.decision, + s.clarity, s.specificity, s.meaningfulness, + s.answerability, s.reference_verifiability, s.answer_uniqueness + FROM test_case tc + LEFT JOIN agent_scores s ON s.tc_id=tc.tc_id + WHERE tc.app_id=? AND tc.golden_set_version=? + ORDER BY tc.created_at DESC""", + (app_id, golden_set_version), + ).fetchall() + result = [] + for r in rows: + d = dict(r) + d["reference_doc_ids"] = json.loads(d.get("reference_doc_ids") or "[]") + d["reference_match_spec"] = _decode_match_spec(d) + result.append(d) + return result + + +def _decode_match_spec(row: dict) -> list[dict]: + """Decode match_spec JSON. Falls back to legacy docId-only spec if empty.""" + spec_json = row.get("reference_match_spec") or "[]" + try: + spec = json.loads(spec_json) + except (TypeError, ValueError): + spec = [] + if spec: + return spec + # Legacy rows: synthesize spec from reference_doc_ids + doc_ids = row.get("reference_doc_ids") or [] + if isinstance(doc_ids, str): + try: + doc_ids = json.loads(doc_ids) + except (TypeError, ValueError): + doc_ids = [] + return [{"field": "docId", "value": d} for d in doc_ids if d] + + +def get_active_test_cases(app_id: str, golden_set_version: str) -> list[dict]: + with get_db() as conn: + rows = conn.execute( + """SELECT tc.*, s.decision + FROM test_case tc + LEFT JOIN agent_scores s ON s.tc_id=tc.tc_id + WHERE tc.app_id=? AND tc.golden_set_version=? + AND tc.status='active' + AND s.decision IN ('KEEP','BORDERLINE') + ORDER BY tc.created_at""", + (app_id, golden_set_version), + ).fetchall() + result = [] + for r in rows: + d = dict(r) + d["reference_doc_ids"] = json.loads(d.get("reference_doc_ids") or "[]") + d["reference_match_spec"] = _decode_match_spec(d) + d["generation_metadata"] = json.loads(d.get("generation_metadata") or "{}") + result.append(d) + return result + + +def insert_test_case(tc: dict) -> None: + with get_db() as conn: + conn.execute( + """INSERT OR REPLACE INTO test_case + (tc_id, app_id, golden_set_version, question, expected_answer, + expected_behavior, question_type, difficulty, answer_type, + reference_doc_ids, reference_match_spec, generation_metadata, rationale) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""", + (tc["tc_id"], tc["app_id"], tc["golden_set_version"], + tc["question"], tc["expected_answer"], + tc.get("expected_behavior", "ANSWER"), tc.get("question_type"), + tc.get("difficulty"), tc.get("answer_type"), + json.dumps(tc.get("reference_doc_ids", [])), + json.dumps(tc.get("reference_match_spec", [])), + json.dumps(tc.get("generation_metadata", {})), + tc.get("rationale")), + ) + + +def import_uploaded_test_cases(app_id: str, version: str, cases: list[dict]) -> int: + """Bulk-insert manually uploaded test cases and auto-approve them with KEEP decision. + + `expected_answer` and `reference_doc_ids` may be missing — per-row case + detection happens at evaluation time based on which fields are populated. + """ + count = 0 + with get_db() as conn: + for case in cases: + tc_id = f"tc-{uuid.uuid4()}" + meta: dict = {} + if case.get("sys_content_type"): + meta["sys_content_type"] = case["sys_content_type"] + if case.get("custom_fields"): + meta["custom_fields"] = case["custom_fields"] + conn.execute( + """INSERT INTO test_case + (tc_id, app_id, golden_set_version, question, expected_answer, + expected_behavior, question_type, difficulty, reference_doc_ids, + reference_match_spec, generation_metadata, human_validated, status) + VALUES (?,?,?,?,?,?,?,?,?,?,?,1,'active')""", + ( + tc_id, app_id, version, + case["question"], case.get("expected_answer") or None, + case.get("expected_behavior", "ANSWER"), + case.get("question_type") or None, + case.get("difficulty") or None, + json.dumps(case.get("reference_doc_ids", [])), + json.dumps(case.get("reference_match_spec", [])), + json.dumps(meta), + ), + ) + conn.execute( + "INSERT OR REPLACE INTO agent_scores (tc_id, decision) VALUES (?,?)", + (tc_id, "KEEP"), + ) + count += 1 + return count + + +def insert_agent_scores(scores: dict) -> None: + with get_db() as conn: + conn.execute( + """INSERT OR REPLACE INTO agent_scores + (tc_id, clarity, specificity, meaningfulness, answerability, + reference_verifiability, answer_uniqueness, decision, primary_concern, rationale) + VALUES (?,?,?,?,?,?,?,?,?,?)""", + (scores["tc_id"], scores.get("clarity"), scores.get("specificity"), + scores.get("meaningfulness"), scores.get("answerability"), + scores.get("reference_verifiability"), scores.get("answer_uniqueness"), + scores.get("decision"), scores.get("primary_concern"), scores.get("rationale")), + ) + + +# ── Jobs ───────────────────────────────────────────────────────────────────── + +def create_job(app_id: str, job_type: str) -> str: + job_id = f"job-{uuid.uuid4()}" + with get_db() as conn: + conn.execute( + "INSERT INTO job (job_id, app_id, job_type) VALUES (?,?,?)", + (job_id, app_id, job_type), + ) + return job_id + + +def update_job(job_id: str, status: str, progress: int = 0, + result: dict | None = None, error: str | None = None) -> None: + with get_db() as conn: + conn.execute( + """UPDATE job SET status=?, progress=?, result=?, error=?, + updated_at=datetime('now') WHERE job_id=?""", + (status, progress, json.dumps(result) if result else None, error, job_id), + ) + + +def get_job(job_id: str) -> dict | None: + with get_db() as conn: + row = conn.execute("SELECT * FROM job WHERE job_id=?", (job_id,)).fetchone() + if not row: + return None + d = dict(row) + if d.get("result"): + d["result"] = json.loads(d["result"]) + return d + + +def list_jobs(app_id: str) -> list[dict]: + with get_db() as conn: + rows = conn.execute( + "SELECT * FROM job WHERE app_id=? ORDER BY created_at DESC LIMIT 50", + (app_id,), + ).fetchall() + result = [] + for r in rows: + d = dict(r) + if d.get("result"): + d["result"] = json.loads(d["result"]) + result.append(d) + return result + + +def request_stop_job(job_id: str) -> None: + """Mark a running job as stop-requested. The pipeline will stop after the current unit.""" + with get_db() as conn: + conn.execute( + "UPDATE job SET stop_requested=1, updated_at=datetime('now') WHERE job_id=?", + (job_id,), + ) + + +def is_stop_requested(job_id: str) -> bool: + with get_db() as conn: + row = conn.execute("SELECT stop_requested FROM job WHERE job_id=?", (job_id,)).fetchone() + return bool(row and row["stop_requested"]) + + +# ── Eval Runs ──────────────────────────────────────────────────────────────── + +def create_eval_run(run: dict) -> None: + with get_db() as conn: + conn.execute( + """INSERT INTO eval_run + (run_id, app_id, rag_version, judge_model, golden_set_version, trigger, total_cases) + VALUES (?,?,?,?,?,?,?)""", + (run["run_id"], run["app_id"], run.get("rag_version", "unknown"), + run.get("judge_model"), run["golden_set_version"], + run.get("trigger", "manual"), run.get("total_cases", 0)), + ) + + +def upsert_eval_result(result: dict) -> None: + with get_db() as conn: + conn.execute( + """INSERT OR REPLACE INTO eval_result + (run_id, tc_id, rag_response, retrieved_doc_ids, chunk_signals, + scores, failure_category, judge_rationale, + latency_llm_ms, latency_retrieval_ms, search_request_id, + search_payload, search_response, attempt_count, + case_id, expected_doc_rank, recall_at_k, answer_similarity, + verdict, verdict_source) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", + (result["run_id"], result["tc_id"], result.get("rag_response"), + json.dumps(result.get("retrieved_doc_ids", [])), + json.dumps(result.get("chunk_signals", [])), + json.dumps(result.get("scores", {})), + result.get("failure_category"), result.get("judge_rationale"), + result.get("latency_llm_ms"), result.get("latency_retrieval_ms"), + result.get("search_request_id"), + json.dumps(result.get("search_payload") or {}), + json.dumps(result.get("search_response") or {}), + result.get("attempt_count", 1), + result.get("case_id"), + result.get("expected_doc_rank"), + json.dumps(result.get("recall_at_k") or {}), + result.get("answer_similarity"), + result.get("verdict"), + result.get("verdict_source")), + ) + + +def bulk_update_verdicts( + run_id: str, + verdicts: list[tuple[str, str | None, str]], # [(tc_id, verdict, verdict_source), ...] +) -> None: + """Overwrite verdict + verdict_source for a batch of (run_id, tc_id) rows.""" + with get_db() as conn: + conn.executemany( + "UPDATE eval_result SET verdict=?, verdict_source=? WHERE run_id=? AND tc_id=?", + [(v, s, run_id, tc_id) for (tc_id, v, s) in verdicts], + ) + + +def update_run_verdict_counts(run_id: str, passed: int, verdicted: int) -> None: + """Update only the pass/verdict counters on eval_run (preserves cost and other fields).""" + with get_db() as conn: + conn.execute( + "UPDATE eval_run SET passed_cases=?, verdicted_cases=? WHERE run_id=?", + (passed, verdicted, run_id), + ) + + +def finish_eval_run( + run_id: str, passed: int, cost: float, + status: str = "complete", avg_chunk_rank: float | None = None, + verdicted: int | None = None, +) -> None: + with get_db() as conn: + conn.execute( + """UPDATE eval_run SET finished_at=datetime('now'), status=?, + passed_cases=?, cost_usd=?, avg_chunk_rank=?, + verdicted_cases=? WHERE run_id=?""", + (status, passed, cost, avg_chunk_rank, + verdicted if verdicted is not None else passed, run_id), + ) + + +def list_eval_runs(app_id: str) -> list[dict]: + with get_db() as conn: + rows = conn.execute( + "SELECT * FROM eval_run WHERE app_id=? ORDER BY started_at DESC", + (app_id,), + ).fetchall() + return [dict(r) for r in rows] + + +def get_eval_results(run_id: str) -> list[dict]: + with get_db() as conn: + rows = conn.execute( + """SELECT er.*, tc.question, tc.expected_answer, tc.expected_behavior, + tc.question_type, tc.difficulty, tc.reference_doc_ids + FROM eval_result er + JOIN test_case tc ON tc.tc_id=er.tc_id + WHERE er.run_id=? + ORDER BY tc.question_type""", + (run_id,), + ).fetchall() + result = [] + for r in rows: + d = dict(r) + d["retrieved_doc_ids"] = json.loads(d.get("retrieved_doc_ids") or "[]") + d["reference_doc_ids"] = json.loads(d.get("reference_doc_ids") or "[]") + d["scores"] = json.loads(d.get("scores") or "{}") + d["search_payload"] = json.loads(d.get("search_payload") or "{}") + d["search_response"] = json.loads(d.get("search_response") or "{}") + d["recall_at_k"] = json.loads(d.get("recall_at_k") or "{}") + result.append(d) + return result + + +def get_completed_tc_ids(run_id: str) -> set[str]: + with get_db() as conn: + rows = conn.execute( + "SELECT tc_id FROM eval_result WHERE run_id=?", (run_id,) + ).fetchall() + return {r["tc_id"] for r in rows} + + +def delete_eval_run(app_id: str, run_id: str) -> int: + """Delete a run and (via ON DELETE CASCADE) its eval_result rows. + + Returns the number of result rows removed. Does NOT remove the underlying + job row — that's metadata about the trigger, not the run itself. + """ + with get_db() as conn: + n = conn.execute( + "SELECT COUNT(*) AS c FROM eval_result WHERE run_id=?", + (run_id,), + ).fetchone()["c"] + conn.execute( + "DELETE FROM eval_run WHERE run_id=? AND app_id=?", + (run_id, app_id), + ) + return int(n) + + +def count_runs_using_golden_set(app_id: str, version: str) -> int: + """How many eval_runs soft-reference this golden_set_version. + + There's no FK enforcing this — golden_set_version is a plain TEXT field on + eval_run. We surface the count so the delete confirmation modal can warn. + """ + with get_db() as conn: + row = conn.execute( + """SELECT COUNT(*) AS c FROM eval_run + WHERE app_id=? AND golden_set_version=?""", + (app_id, version), + ).fetchone() + return int(row["c"]) if row else 0 + + +def delete_golden_set(app_id: str, version: str) -> dict: + """Delete a golden set and all its test cases (+ cascade to agent_scores). + + Returns counts of what was removed so the UI can show a confirmation toast. + Existing eval_runs that reference this version are LEFT INTACT but become + orphaned references (the `golden_set_version` text simply no longer + matches any row in `golden_set`). + """ + with get_db() as conn: + tc_count = conn.execute( + "SELECT COUNT(*) AS c FROM test_case WHERE app_id=? AND golden_set_version=?", + (app_id, version), + ).fetchone()["c"] + run_refs = conn.execute( + "SELECT COUNT(*) AS c FROM eval_run WHERE app_id=? AND golden_set_version=?", + (app_id, version), + ).fetchone()["c"] + # agent_scores cascade off test_case.tc_id (ON DELETE CASCADE in schema) + conn.execute( + "DELETE FROM test_case WHERE app_id=? AND golden_set_version=?", + (app_id, version), + ) + conn.execute( + "DELETE FROM golden_set WHERE app_id=? AND version=?", + (app_id, version), + ) + return { + "test_cases_deleted": int(tc_count), + "eval_runs_orphaned": int(run_refs), + } + + +def get_eval_run(run_id: str) -> dict | None: + """Fetch a single eval_run row (with insights columns) by id.""" + with get_db() as conn: + row = conn.execute( + "SELECT * FROM eval_run WHERE run_id=?", (run_id,) + ).fetchone() + return dict(row) if row else None + + +def get_previous_completed_run( + app_id: str, + golden_set_version: str, + before_run_id: str, +) -> dict | None: + """Most recent completed (or partial) run for this app + golden set strictly older + than ``before_run_id``. Used for cross-run trend deltas. + """ + with get_db() as conn: + anchor = conn.execute( + "SELECT started_at FROM eval_run WHERE run_id=?", + (before_run_id,), + ).fetchone() + if not anchor: + return None + row = conn.execute( + """SELECT * FROM eval_run + WHERE app_id=? AND golden_set_version=? + AND status IN ('complete','partial') + AND run_id <> ? + AND started_at < ? + ORDER BY started_at DESC + LIMIT 1""", + (app_id, golden_set_version, before_run_id, anchor["started_at"]), + ).fetchone() + return dict(row) if row else None + + +def list_completed_runs(app_id: str, limit: int = 50) -> list[dict]: + """All completed/partial runs for an app, newest first. Used for recurrence checks.""" + with get_db() as conn: + rows = conn.execute( + """SELECT * FROM eval_run + WHERE app_id=? AND status IN ('complete','partial') + ORDER BY started_at DESC + LIMIT ?""", + (app_id, limit), + ).fetchall() + return [dict(r) for r in rows] + + +# ── Diagnostics & Insights cache ──────────────────────────────────────────── + +def get_run_diagnostics(run_id: str) -> tuple[dict | None, list[str] | None]: + """Return (diagnostics_dict, fired_rule_ids) cached on the run, or (None, None).""" + with get_db() as conn: + row = conn.execute( + "SELECT diagnostics_json, fired_rule_ids FROM eval_run WHERE run_id=?", + (run_id,), + ).fetchone() + if not row: + return None, None + diag = json.loads(row["diagnostics_json"]) if row["diagnostics_json"] else None + fired = json.loads(row["fired_rule_ids"]) if row["fired_rule_ids"] else None + return diag, fired + + +def set_run_diagnostics( + run_id: str, + diagnostics: dict, + fired_rule_ids: list[str], +) -> None: + """Persist computed diagnostics + fired rule ids on the run.""" + with get_db() as conn: + conn.execute( + """UPDATE eval_run + SET diagnostics_json=?, fired_rule_ids=? + WHERE run_id=?""", + (json.dumps(diagnostics), json.dumps(fired_rule_ids), run_id), + ) + + +def get_run_ai_insights(run_id: str) -> dict | None: + """Return cached AI-narrative + metadata for the run, or None if never generated.""" + with get_db() as conn: + row = conn.execute( + """SELECT ai_insights_md, ai_insights_model, ai_insights_generated_at + FROM eval_run WHERE run_id=?""", + (run_id,), + ).fetchone() + if not row or not row["ai_insights_md"]: + return None + return { + "markdown": row["ai_insights_md"], + "model": row["ai_insights_model"], + "generated_at": row["ai_insights_generated_at"], + } + + +def set_run_ai_insights(run_id: str, markdown: str, model: str) -> None: + """Cache the LLM-generated narrative on the run.""" + with get_db() as conn: + conn.execute( + """UPDATE eval_run + SET ai_insights_md=?, ai_insights_model=?, + ai_insights_generated_at=datetime('now') + WHERE run_id=?""", + (markdown, model, run_id), + ) diff --git a/Evaluation/AISearchEvalutionTool/backend/diagnostics/compute.py b/Evaluation/AISearchEvalutionTool/backend/diagnostics/compute.py index 0efb34b2..89ec412d 100644 --- a/Evaluation/AISearchEvalutionTool/backend/diagnostics/compute.py +++ b/Evaluation/AISearchEvalutionTool/backend/diagnostics/compute.py @@ -93,6 +93,7 @@ def compute_run_diagnostics(run_id: str) -> dict[str, Any]: "by_question_type": _compute_by_qtype(results), "by_failure_category": _bucket_count(results, lambda r: r.get("failure_category") or "none"), "retrieval": _compute_retrieval(results), + "chunk_lifecycle": _compute_chunk_lifecycle(results), "judge_metric_avgs": _compute_judge_avgs(results), "weakest_judge_metric": None, # filled in below "latency_ms": _compute_latency(results), @@ -254,6 +255,63 @@ def _compute_retrieval(results: list[dict]) -> dict[str, Any]: } +def _compute_chunk_lifecycle(results: list[dict]) -> dict[str, Any]: + """Pipeline accuracy across the Kore.ai chunk lifecycle. + + Each test case has, at most, ONE "matched chunk" — the first chunk whose + fields satisfy the test case's reference_match_spec. Three boolean flags + from Kore.ai tell us how far that chunk made it down the pipeline: + + chunkQualified → got past Kore.ai's internal scoring threshold + (a.k.a. the "shortlist" / "retrieval" stage). + sentToLLM → was included in the LLM context for answer generation. + usedInAnswer → the LLM actually cited / used it in its answer. + + Accuracy for each stage = (# cases where that flag is True) / + (# cases that had a matched chunk to begin with). Cases without a + reference doc (case 1 / 2) are excluded from the denominator — there is + no expected chunk to track. We also report a top-line ``has_matched_chunk`` + count so callers can see the funnel depth. + """ + cases_with_ref = [ + r for r in results + if (r.get("reference_doc_ids") or []) + ] + matched_rows = [ + r for r in cases_with_ref + if (r.get("scores") or {}).get("matched_chunk_qualified") is not None + or (r.get("scores") or {}).get("matched_chunk_sent_to_llm") is not None + or (r.get("scores") or {}).get("matched_chunk_used_in_answer") is not None + ] + + def _count_true(rows: list[dict], key: str) -> int: + return sum( + 1 for r in rows + if (r.get("scores") or {}).get(key) is True + ) + + qualified = _count_true(matched_rows, "matched_chunk_qualified") + sent_llm = _count_true(matched_rows, "matched_chunk_sent_to_llm") + used_answer = _count_true(matched_rows, "matched_chunk_used_in_answer") + + denom = len(matched_rows) + def _rate(numer: int) -> float | None: + if denom == 0: + return None + return round(numer / denom, 4) + + return { + "cases_with_reference": len(cases_with_ref), + "cases_with_matched_chunk": denom, + "retrieval_qualified": qualified, + "sent_to_llm": sent_llm, + "used_in_answer": used_answer, + "retrieval_accuracy": _rate(qualified), + "sent_to_llm_accuracy": _rate(sent_llm), + "answer_gen_accuracy": _rate(used_answer), + } + + def _compute_judge_avgs(results: list[dict]) -> dict[str, float | None]: """Run-wide average of each judge rubric metric (1-5 scale).""" out: dict[str, float | None] = {} diff --git a/Evaluation/AISearchEvalutionTool/backend/judge/judge.py b/Evaluation/AISearchEvalutionTool/backend/judge/judge.py index 053dbed0..94681976 100644 --- a/Evaluation/AISearchEvalutionTool/backend/judge/judge.py +++ b/Evaluation/AISearchEvalutionTool/backend/judge/judge.py @@ -1,12 +1,11 @@ from __future__ import annotations -import json import logging import time from typing import Any -from db.database import get_llm_config, get_active_prompt, get_api_key, get_base_url -from agents.llm_client import _infer_provider, _openai_client_and_model, _openai_call_kwargs, _is_o_series_model +from db.database import get_llm_config, get_active_prompt +from agents.llm_client import _infer_provider, call_llm_json, parse_json_loose from agents.prompts import JUDGE_PROMPT # Backoff delays (seconds) between judge retries on rate-limit errors. @@ -114,6 +113,13 @@ def _call_judge( user: str, max_tokens_override: int | None = None, ) -> dict: + """Call the judge LLM and parse its JSON output. + + Delegates to :func:`agents.llm_client.call_llm_json`, which supports + Anthropic, OpenAI and Gemini uniformly — historically this helper only + handled Anthropic and OpenAI which silently broke Gemini-based judges. + Rate-limit / 429 retries are kept here on top of the underlying call. + """ model: str = cfg["model"] max_tokens = max_tokens_override if max_tokens_override is not None else cfg["max_tokens"] provider = _infer_provider(model) @@ -128,55 +134,17 @@ def _call_judge( ) time.sleep(sleep_secs) - logger.debug("Judge | Calling %s via %s | max_tokens=%d attempt=%d", model, provider, max_tokens, attempt) - raw = "" + logger.debug( + "Judge | Calling %s via %s | max_tokens=%d attempt=%d", + model, provider, max_tokens, attempt, + ) try: - if provider == "anthropic": - import anthropic - client = anthropic.Anthropic( - api_key=get_api_key(app_id, "anthropic"), - base_url=get_base_url(app_id, "anthropic"), - ) - message = client.messages.create( - model=model, - max_tokens=max_tokens, - temperature=0, - system=system, - messages=[{"role": "user", "content": user}], - ) - raw = message.content[0].text.strip() - else: - client, effective_model = _openai_client_and_model(app_id, model) - messages = [] - if system: - messages.append({"role": "system", "content": system}) - messages.append({"role": "user", "content": user}) - extra: dict = {} - if not _is_o_series_model(effective_model): - extra["response_format"] = {"type": "json_object"} - resp = client.chat.completions.create( - model=effective_model, - messages=messages, - **extra, - **_openai_call_kwargs(effective_model, max_tokens, 0.0), - ) - raw = resp.choices[0].message.content - - return json.loads(raw) - - except json.JSONDecodeError as exc: - # Try bracket extraction as fallback - s, e = raw.find("{"), raw.rfind("}") + 1 - if s != -1 and e > 0: - try: - return json.loads(raw[s:e]) - except Exception: - pass - logger.error("Judge | JSON parse failed | model=%s | raw[:200]=%s | error: %s", model, raw[:200], exc) - return {} - + raw = call_llm_json( + app_id, "judge", system, user, + max_tokens_override=max_tokens, + ) except Exception as exc: - # Detect rate-limit errors from both OpenAI and Anthropic + # Detect rate-limit errors from any provider so we can sleep+retry. exc_str = str(exc).lower() is_rate_limit = ( "rate limit" in exc_str @@ -191,11 +159,19 @@ def _call_judge( attempt, len(delays), model, exc, ) continue # sleep handled at top of next iteration - logger.error( "Judge | API call FAILED | model=%s provider=%s attempt=%d | error: %s", model, provider, attempt, exc, exc_info=True, ) return {} + parsed = parse_json_loose(raw, expect="object", agent_name="judge") + if isinstance(parsed, dict): + return parsed + logger.error( + "Judge | JSON parse failed | model=%s | raw[:200]=%s", + model, (raw or "")[:200], + ) + return {} + return {} diff --git a/Evaluation/AISearchEvalutionTool/backend/judge/metrics.py b/Evaluation/AISearchEvalutionTool/backend/judge/metrics.py index 6a744a44..1a857d67 100644 --- a/Evaluation/AISearchEvalutionTool/backend/judge/metrics.py +++ b/Evaluation/AISearchEvalutionTool/backend/judge/metrics.py @@ -10,6 +10,7 @@ from typing import Any +from agents.llm_client import _infer_provider from db.database import get_api_key, get_llm_config from judge.embeddings import semantic_similarity @@ -118,10 +119,14 @@ def qualified_chunks_count(chunk_signals: list[dict]) -> int: def judge_configured(app: dict) -> bool: - """A judge is 'configured' when the API key for its provider is present.""" + """A judge is 'configured' when the API key for its provider is present. + + Provider is inferred from the judge agent's model name (anthropic / openai / + gemini), matching the same logic the LLM client itself uses — that way a + Gemini-based judge is correctly detected, not silently rejected. + """ cfg = get_llm_config(app["app_id"], "judge") - model = (cfg.get("model") or "").lower() - provider = "anthropic" if "claude" in model else "openai" + provider = _infer_provider(cfg.get("model") or "") key = get_api_key(app["app_id"], provider) or "" return bool(key.strip()) @@ -184,6 +189,7 @@ def derive_verdict( case2_threshold: float = DEFAULT_CASE2_THRESHOLD, chunk_rank: int | None = None, answer_mode: str = "answer_generation", + top_k_pass: int = TOP_K_PASS, ) -> tuple[str | None, str]: """Return (verdict, verdict_source). @@ -192,7 +198,7 @@ def derive_verdict( answer_mode='extract_only': Pass/fail is purely retrieval-based for all cases that have a reference doc - (cases 3 & 4): expected doc chunk must be in top TOP_K_PASS chunks. + (cases 3 & 4): expected doc chunk must be in top ``top_k_pass`` chunks. Cases 1 & 2 without a reference doc fall back to semantic similarity. answer_mode='answer_generation': @@ -203,7 +209,7 @@ def derive_verdict( """ if answer_mode == "extract_only": if case_id in (3, 4): - return _verdict_retrieval(chunk_rank, judge_scores or {}) + return _verdict_retrieval(chunk_rank, judge_scores or {}, top_k=top_k_pass) # Cases 1 & 2 in extract mode: no generated answer, use semantic if available return _verdict_no_judge( case_id, expected_doc_rank_val, similarity, @@ -222,16 +228,17 @@ def derive_verdict( def _verdict_retrieval( chunk_rank: int | None, scores: dict, + top_k: int = TOP_K_PASS, ) -> tuple[str | None, str]: - """Pass if expected document chunk is in top TOP_K_PASS (extract_only mode).""" + """Pass if expected document chunk is in top ``top_k`` (extract_only mode).""" if scores.get("toxicity_detected") or scores.get("bias_detected") or scores.get("banned_topic_violation"): - return "fail", f"Safety violation (top-{TOP_K_PASS} chunk rule)" + return "fail", f"Safety violation (top-{top_k} chunk rule)" if chunk_rank is None: - return "fail", f"Expected doc not found in top {TOP_K_PASS} chunks" - ok = chunk_rank <= TOP_K_PASS + return "fail", f"Expected doc not found in top {top_k} chunks" + ok = chunk_rank <= top_k return ( ("pass" if ok else "fail"), - f"Chunk rank {chunk_rank} {'≤' if ok else '>'} top-{TOP_K_PASS} (extract_only)", + f"Chunk rank {chunk_rank} {'≤' if ok else '>'} top-{top_k} (extract_only)", ) diff --git a/Evaluation/AISearchEvalutionTool/backend/koreai/content.py b/Evaluation/AISearchEvalutionTool/backend/koreai/content.py index a525e44e..254f5861 100644 --- a/Evaluation/AISearchEvalutionTool/backend/koreai/content.py +++ b/Evaluation/AISearchEvalutionTool/backend/koreai/content.py @@ -13,6 +13,7 @@ def fetch_documents( app: dict, filters: dict[str, Any] | None = None, extraction_type: str | None = None, + source_id: str | None = None, max_docs: int = 10, ) -> Generator[dict[str, Any], None, None]: """Yield documents from Content by Condition API with cursor pagination.""" @@ -22,8 +23,8 @@ def fetch_documents( unlimited = max_docs == 0 logger.info( - "Kore.ai | Fetching documents | app=%s extraction_type=%s max_docs=%s filters=%s", - app_id, extraction_type, "unlimited" if unlimited else max_docs, filters, + "Kore.ai | Fetching documents | app=%s extraction_type=%s source_id=%s max_docs=%s filters=%s", + app_id, extraction_type, source_id, "unlimited" if unlimited else max_docs, filters, ) query: dict[str, Any] = dict(filters or {}) @@ -63,18 +64,29 @@ def fetch_documents( if not unlimited and fetched >= max_docs: logger.debug("Kore.ai | Reached max_docs=%d — stopping", max_docs) return + item_source_id = item.get("connectorId") or item.get("extractionSourceId") + if source_id and item_source_id != source_id: + continue source = item.get("_source", {}) - content = source.get("content", "") or "" + content = ( + source.get("content") + or source.get("page_body") + or source.get("text") + or source.get("page_preview") + or source.get("page_html") + or "" + ) title = ( source.get("title") or source.get("file_title") + or source.get("page_title") + or source.get("recordTitle") or item.get("_id", "Untitled") ) - connector_id = item.get("connectorId") or item.get("extractionSourceId") logger.debug( "Kore.ai | Doc #%d: id=%s title='%s' connector=%s content_chars=%d", - fetched + 1, item["_id"], title, connector_id, len(content), + fetched + 1, item["_id"], title, item_source_id, len(content), ) yield { @@ -85,8 +97,13 @@ def fetch_documents( "content": content, "metadata": source, "sys_content_type": source.get("sys_content_type"), - "source_url": source.get("url") or source.get("base_url"), - "connector_id": connector_id, + "source_url": ( + source.get("url") + or source.get("page_url") + or source.get("base_url") + or source.get("sys_source_url") + ), + "connector_id": item_source_id, } fetched += 1 diff --git a/Evaluation/AISearchEvalutionTool/backend/koreai/search.py b/Evaluation/AISearchEvalutionTool/backend/koreai/search.py index eef25788..bc44e397 100644 --- a/Evaluation/AISearchEvalutionTool/backend/koreai/search.py +++ b/Evaluation/AISearchEvalutionTool/backend/koreai/search.py @@ -78,6 +78,10 @@ def query_rag( result = _parse_response(raw, answer_mode=answer_mode) # Expose the exact payload that was sent so callers can store/show it for debugging result["search_payload"] = payload + # Expose the full raw response so the UI / persistence layer can show it + # exactly as Kore.ai returned it. Useful for debugging chunk scoring, + # answer payload structure, and parameters the parser ignored. + result["search_response"] = raw logger.info( "Kore.ai | RAG response | app=%s valid=%s cited_docs=%d result_docs=%d " diff --git a/Evaluation/AISearchEvalutionTool/backend/main.py b/Evaluation/AISearchEvalutionTool/backend/main.py index 8baa0780..2b2bb6fc 100644 --- a/Evaluation/AISearchEvalutionTool/backend/main.py +++ b/Evaluation/AISearchEvalutionTool/backend/main.py @@ -19,11 +19,12 @@ logging.getLogger("anthropic").setLevel(logging.WARNING) logging.getLogger("openai").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("pymongo").setLevel(logging.WARNING) logger = logging.getLogger(__name__) from db.database import init_db -from routers import apps, sources, llm_config, prompts, generation, evaluation, golden_sets, results, app_api_keys, query +from routers import apps, sources, llm_config, prompts, generation, evaluation, golden_sets, results, app_api_keys, query, prompt_tuner app = FastAPI(title="RAG Evaluator API", version="1.0.1") @@ -63,6 +64,7 @@ def startup(): app.include_router(results.router, prefix="/api") app.include_router(app_api_keys.router, prefix="/api") app.include_router(query.router, prefix="/api") +app.include_router(prompt_tuner.router, prefix="/api") @app.get("/api/health") @@ -72,10 +74,29 @@ def health(): @app.get("/api/debug/db") def debug_db(): - """Shows database location, size, and table row counts.""" + """Shows configured database backend, location, and row/collection counts.""" + from config import get_config + cfg = get_config().database + if cfg.backend == "mongodb": + from pymongo import MongoClient + client = MongoClient(cfg.mongodb_uri, serverSelectionTimeoutMS=2000) + db = client[cfg.mongodb_database] + # Force connection errors to surface in this debug endpoint. + client.admin.command("ping") + collections = {} + for name in db.list_collection_names(): + collections[name] = db[name].count_documents({}) + client.close() + return { + "backend": "mongodb", + "mongodb_uri": cfg.mongodb_uri, + "database": cfg.mongodb_database, + "collections": collections, + } + + import os import sqlite3 as _sq from db.database import DB_PATH - import os path = str(DB_PATH) exists = os.path.exists(path) size_kb = round(os.path.getsize(path) / 1024, 2) if exists else 0 @@ -88,6 +109,7 @@ def debug_db(): tables[t] = count conn.close() return { + "backend": "sqlite", "db_path": path, "exists": exists, "size_kb": size_kb, diff --git a/Evaluation/AISearchEvalutionTool/backend/models.py b/Evaluation/AISearchEvalutionTool/backend/models.py index 5dfdbce7..55163145 100644 --- a/Evaluation/AISearchEvalutionTool/backend/models.py +++ b/Evaluation/AISearchEvalutionTool/backend/models.py @@ -55,7 +55,7 @@ class AppConfigResponse(BaseModel): class LLMConfigUpdate(BaseModel): model: str temperature: float = Field(ge=0.0, le=2.0) - max_tokens: int = Field(gt=0, le=16000) + max_tokens: int = Field(gt=0, le=32000) class LLMConfigResponse(BaseModel): @@ -101,7 +101,7 @@ class ContentSourceResponse(BaseModel): Used for Websites (sys_content_type=web) and Documents (sys_content_type=file), which Kore.ai's public API does not expose via a dedicated listing endpoint. """ - source_id: str # extractionSourceId — the parent container + source_id: str # extractionSourceId — top-level source metadata name: str # sys_source_name (falls back to URL or source_id) sys_content_type: str # echoed for the UI badge ("web" | "file") records_count: int # docs observed in this source @@ -120,6 +120,7 @@ class GenerationRequest(BaseModel): file_source_ids: list[str] = [] # extractionSourceId for sys_content_type=file uploads max_docs_per_source: int = Field(default=10, ge=0, description="Max docs per source. 0 = all documents (no cap).") max_questions_per_doc: int = Field(default=5, ge=1, le=5) + target_language: str = Field(default="English", min_length=1, max_length=80) filters: dict[str, Any] = {} @@ -134,10 +135,14 @@ class EvaluationRequest(BaseModel): # ── Meta-filter / RACL controls ────────────────────────────────────── filter_mode: str = Field( default="none", - pattern="^(none|auto_source|custom_prompt)$", - description="'none' = no filters, 'auto_source' = derive sys_content_type from each TC's reference doc, 'custom_prompt' = LLM generates filters per question (mapper script loaded from DB if configured)", + pattern="^(none|field_filters|custom_prompt)$", + description="'none' = no filters, 'field_filters' = use test-case column values as metaFilters, 'custom_prompt' = LLM generates filters per question", ) filter_prompt: str | None = Field(default=None, description="Unused — prompt is read from DB (Prompts & Models page)") + filter_fields: list[str] | None = Field( + default=None, + description="Field names to use when filter_mode='field_filters'. Values are read from each test case's generation_metadata / custom_fields.", + ) enable_racl: bool = Field(default=False, description="If true, send user_email as customData.userContext.userId") user_email: str | None = Field(default=None, description="RACL user identifier (required when enable_racl=true)") @@ -154,6 +159,30 @@ class EvaluationRequest(BaseModel): description="Override the app's answer_mode for this run. None = use app default.", ) + # ── Per-run verdict configuration overrides ─────────────────────────── + judge_mode: str = Field( + default="auto", + pattern="^(auto|force_on|force_off)$", + description=( + "'auto' = use the LLM judge if its API key is configured, else " + "fall back to semantic similarity. 'force_on' = require the judge " + "(fail if not configured). 'force_off' = skip the judge, use " + "semantic similarity only." + ), + ) + case1_threshold: float | None = Field( + default=None, ge=0.0, le=1.0, + description="Override Q↔Answer relevance pass threshold for Cases 1 & 3 (no-judge fallback). None = use app setting.", + ) + case2_threshold: float | None = Field( + default=None, ge=0.0, le=1.0, + description="Override Answer↔Expected similarity pass threshold for Cases 2 & 4 (no-judge fallback). None = use app setting.", + ) + top_k_pass: int | None = Field( + default=None, ge=1, le=100, + description="Override the top-K chunk pass threshold used in extract_only retrieval verdicts. None = default (5).", + ) + class MapperTestRequest(BaseModel): script: str @@ -204,6 +233,10 @@ class TestCaseResponse(BaseModel): human_validated: bool status: str decision: str | None + primary_concern: str | None = None + rationale: str | None = None + case_id: int | None = None + custom_fields: dict[str, Any] = {} scores: dict[str, Any] | None @@ -225,6 +258,12 @@ class EvalRunResponse(BaseModel): avg_chunk_rank: float | None = None +class DocLabel(BaseModel): + """Compact per-document label info derived from chunk_signals.""" + title: str | None = None + url: str | None = None + + class EvalResultResponse(BaseModel): tc_id: str question: str @@ -242,6 +281,10 @@ class EvalResultResponse(BaseModel): latency_retrieval_ms: float | None doc_retrieved: bool search_payload: dict[str, Any] | None = None + search_response: dict[str, Any] | None = None + # docId → {title, url} — built from stored chunk_signals so the UI can show + # record titles instead of raw doc IDs without sending full chunk payloads + doc_label_map: dict[str, DocLabel] = {} # 4-case evaluation fields case_id: int | None = None expected_doc_rank: int | None = None diff --git a/Evaluation/AISearchEvalutionTool/backend/pipeline/evaluate.py b/Evaluation/AISearchEvalutionTool/backend/pipeline/evaluate.py index d3545a8c..d965544f 100644 --- a/Evaluation/AISearchEvalutionTool/backend/pipeline/evaluate.py +++ b/Evaluation/AISearchEvalutionTool/backend/pipeline/evaluate.py @@ -11,7 +11,7 @@ from typing import Any from concurrent.futures import ThreadPoolExecutor, as_completed -from agents.filter_generator import build_source_filter, generate_meta_filters +from agents.filter_generator import build_field_filters, build_source_filter, generate_meta_filters from db.database import ( create_eval_run, finish_eval_run, get_active_test_cases, get_completed_tc_ids, get_doc_source_type, get_eval_thresholds, @@ -148,10 +148,15 @@ def run_evaluation( sample_mode: str = "first", filter_mode: str = "none", filter_prompt: str | None = None, + filter_fields: list[str] | None = None, enable_racl: bool = False, user_email: str | None = None, answer_mode_override: str | None = None, question_types: list[str] | None = None, + judge_mode: str = "auto", + case1_threshold: float | None = None, + case2_threshold: float | None = None, + top_k_pass: int | None = None, ) -> dict[str, Any]: # Apply per-run answer mode override if provided if answer_mode_override: @@ -162,11 +167,25 @@ def run_evaluation( logger.info( "Eval | Starting evaluation | app=%s version=%s rag_version=%s run_id=%s " - "max_cases=%s sample_mode=%s filter_mode=%s racl=%s answer_mode=%s", + "max_cases=%s sample_mode=%s filter_mode=%s racl=%s answer_mode=%s " + "judge_mode=%s c1_threshold=%s c2_threshold=%s top_k_pass=%s", app_id, golden_set_version, rag_version, run_id, max_cases, sample_mode, filter_mode, enable_racl, app.get("answer_mode"), + judge_mode, case1_threshold, case2_threshold, top_k_pass, ) + # ── Validate judge_mode against actual configuration ──────────────── + if judge_mode == "force_on" and not judge_configured(app): + msg = ( + "Verdict configuration requires the LLM judge, but no API key is " + "configured for the judge agent's provider. Either set a key on " + "the API Keys page, switch the judge agent's model on Prompts & " + "Models, or change Judge mode to 'auto' / 'off'." + ) + logger.warning("Eval | %s | run_id=%s", msg, run_id) + update_job(job_id, "failed", error=msg) + return {} + try: test_cases = get_active_test_cases(app_id, golden_set_version) if not test_cases: @@ -226,11 +245,16 @@ def _run_one(tc: dict) -> tuple[dict | None, dict]: app_id=app_id, filter_mode=filter_mode, filter_prompt=filter_prompt, + filter_fields=filter_fields, ) return _evaluate_one( run_id, tc, app, meta_filters=meta_filters, user_email=racl_user, + judge_mode=judge_mode, + case1_threshold_override=case1_threshold, + case2_threshold_override=case2_threshold, + top_k_pass_override=top_k_pass, ), tc with ThreadPoolExecutor(max_workers=MAX_EVAL_WORKERS) as executor: @@ -345,15 +369,26 @@ def _resolve_meta_filters( app_id: str, filter_mode: str, filter_prompt: str | None, + filter_fields: list[str] | None = None, ) -> list[dict]: """Resolve metaFilters for a single test case based on the configured mode. - Priority: - 1. auto_source — look up sys_content_type from source_document table - 2. custom_prompt — LLM-generated filters from question - 3. per-row — sys_content_type stored in the test case's generation_metadata - (set when the Excel sheet has a sys_content_type column) + Modes: + field_filters — use values from the test case's own fields (generation_metadata / + custom_fields) for the field names selected by the user. + Only applies filters for fields that have a non-empty value on + this specific test case. + custom_prompt — LLM-generated filters per question. + none — no filters. """ + if filter_mode == "field_filters": + fields = filter_fields or [] + if not fields: + return [] + result = build_field_filters(tc, fields) + logger.debug("Eval | tc=%s field_filters=%s → %d rules", tc["tc_id"], fields, len(result)) + return result + if filter_mode == "auto_source": ref_docs = tc.get("reference_doc_ids") or [] if not ref_docs: @@ -385,6 +420,10 @@ def _evaluate_one( app: dict, meta_filters: list[dict] | None = None, user_email: str | None = None, + judge_mode: str = "auto", + case1_threshold_override: float | None = None, + case2_threshold_override: float | None = None, + top_k_pass_override: int | None = None, ) -> dict | None: expected_behavior = tc.get("expected_behavior", "ANSWER") banned_topics = app.get("banned_topics") or [] @@ -397,8 +436,28 @@ def _evaluate_one( match_spec = [{"field": "docId", "value": d} for d in legacy_ref_ids] case_id = detect_case(tc) - has_judge = judge_configured(app) - thresholds = get_eval_thresholds(app["app_id"]) + + # Resolve the effective verdict knobs for this run. judge_mode='auto' falls + # back to actual configuration; 'force_off' skips the judge even when keys + # are present; 'force_on' is validated upstream and treated as 'auto' here. + if judge_mode == "force_off": + has_judge = False + else: + has_judge = judge_configured(app) + + app_thresholds = get_eval_thresholds(app["app_id"]) + case1_threshold = ( + case1_threshold_override + if case1_threshold_override is not None + else app_thresholds["case1_threshold"] + ) + case2_threshold = ( + case2_threshold_override + if case2_threshold_override is not None + else app_thresholds["case2_threshold"] + ) + from judge.metrics import TOP_K_PASS as _DEFAULT_TOP_K + top_k_pass = top_k_pass_override if top_k_pass_override is not None else _DEFAULT_TOP_K for attempt, delay in enumerate(RETRY_DELAYS, 1): try: @@ -470,10 +529,11 @@ def _evaluate_one( judge_scores=judge_scores, expected_doc_rank_val=rank, similarity=similarity, qa_relevance=qa_relevance, - case1_threshold=thresholds["case1_threshold"], - case2_threshold=thresholds["case2_threshold"], + case1_threshold=case1_threshold, + case2_threshold=case2_threshold, chunk_rank=chunk_rank, answer_mode=_answer_mode, + top_k_pass=top_k_pass, ) # Compose stored "scores" payload — includes legacy fields the UI uses @@ -497,13 +557,13 @@ def _evaluate_one( failure_category = "retrieval_miss" if not rank else "none" elif case_id in (1, 3): # Q↔Answer relevance proxy - if qa_relevance is not None and qa_relevance < thresholds["case1_threshold"]: + if qa_relevance is not None and qa_relevance < case1_threshold: failure_category = "off_topic" else: failure_category = "none" elif case_id in (2, 4): # Answer↔Expected similarity - if similarity is not None and similarity < thresholds["case2_threshold"]: + if similarity is not None and similarity < case2_threshold: failure_category = "low_similarity" else: failure_category = "none" @@ -526,6 +586,7 @@ def _evaluate_one( "latency_retrieval_ms": rag.get("latency_retrieval_ms"), "search_request_id": rag.get("search_request_id"), "search_payload": rag.get("search_payload"), + "search_response": rag.get("search_response"), "attempt_count": attempt, # 4-case fields "case_id": case_id, diff --git a/Evaluation/AISearchEvalutionTool/backend/pipeline/generate.py b/Evaluation/AISearchEvalutionTool/backend/pipeline/generate.py index 4ad5b5b6..d29fa764 100644 --- a/Evaluation/AISearchEvalutionTool/backend/pipeline/generate.py +++ b/Evaluation/AISearchEvalutionTool/backend/pipeline/generate.py @@ -30,19 +30,32 @@ def _process_doc( total_docs: int, lock: threading.Lock, max_questions_per_doc: int = 5, + target_language: str = "English", ) -> list[dict[str, Any]]: """Run Agent 1 + Agent 2 for a single document (runs in a thread).""" doc_id = doc.get("doc_id", "?") title = doc.get("title", "?") - logger.info("Pipeline | Processing doc '%s' (id=%s) | max_questions=%d", title, doc_id, max_questions_per_doc) + logger.info( + "Pipeline | Processing doc '%s' (id=%s) | max_questions=%d language=%s", + title, doc_id, max_questions_per_doc, target_language, + ) upsert_source_document(doc) extraction = summarize_document(doc, app_id) - raw_cases = generate_test_cases(extraction, doc, app_id, max_questions=max_questions_per_doc) + raw_cases = generate_test_cases( + extraction, + doc, + app_id, + max_questions=max_questions_per_doc, + target_language=target_language, + ) for tc in raw_cases: tc["golden_set_version"] = golden_set_version tc["app_id"] = app_id + metadata = tc.get("generation_metadata") if isinstance(tc.get("generation_metadata"), dict) else {} + metadata["target_language"] = target_language + tc["generation_metadata"] = metadata with lock: completed_counter[0] += 1 @@ -66,15 +79,16 @@ def run_generation( job_id: str, web_source_ids: list[str] | None = None, file_source_ids: list[str] | None = None, + target_language: str = "English", ) -> dict[str, Any]: app_id = app["app_id"] web_source_ids = web_source_ids or [] file_source_ids = file_source_ids or [] logger.info( "Pipeline | Generation started | app=%s version=%s max_docs=%d max_q_per_doc=%d " - "connectors=%s web_sources=%s file_sources=%s", + "language=%s connectors=%s web_sources=%s file_sources=%s", app_id, golden_set_version, max_docs_per_source, max_questions_per_doc, - connector_ids, web_source_ids, file_source_ids, + target_language, connector_ids, web_source_ids, file_source_ids, ) try: @@ -121,10 +135,14 @@ def run_generation( # ── Web crawls (sys_content_type='web', scoped by extractionSourceId) ── for src_id in web_source_ids: - web_filters = {**filters, "sys_content_type": "web", "extractionSourceId": src_id} + # Kore.ai's content condition API filters only trained index fields. + # extractionSourceId is returned as top-level metadata, so fetch web + # records by sys_content_type and apply the source-id match client-side. + web_filters = {**filters, "sys_content_type": "web"} web_docs = list(fetch_documents( app, filters=web_filters, extraction_type=None, + source_id=src_id, max_docs=max_docs_per_source, )) logger.info("Pipeline | Web source %s → %d pages", src_id, len(web_docs)) @@ -132,10 +150,11 @@ def run_generation( # ── Uploaded files (sys_content_type='file', scoped by extractionSourceId) ── for src_id in file_source_ids: - file_filters = {**filters, "sys_content_type": "file", "extractionSourceId": src_id} + file_filters = {**filters, "sys_content_type": "file"} file_docs = list(fetch_documents( app, filters=file_filters, extraction_type=None, + source_id=src_id, max_docs=max_docs_per_source, )) logger.info("Pipeline | File source %s → %d documents", src_id, len(file_docs)) @@ -204,7 +223,7 @@ def run_generation( executor.submit( _process_doc, doc, app_id, golden_set_version, job_id, completed_counter, len(valid_docs), lock, - max_questions_per_doc, + max_questions_per_doc, target_language, ): doc for doc in valid_docs } diff --git a/Evaluation/AISearchEvalutionTool/backend/requirements.txt b/Evaluation/AISearchEvalutionTool/backend/requirements.txt index 79657b14..93afec6e 100644 --- a/Evaluation/AISearchEvalutionTool/backend/requirements.txt +++ b/Evaluation/AISearchEvalutionTool/backend/requirements.txt @@ -4,6 +4,8 @@ anthropic>=0.40.0 openai>=1.50.0 httpx>=0.27.0 pydantic>=2.7.0 +eval-type-backport +pymongo>=4.0.0 python-dotenv>=1.0.0 python-multipart>=0.0.9 openpyxl>=3.1.0 diff --git a/Evaluation/AISearchEvalutionTool/backend/routers/app_api_keys.py b/Evaluation/AISearchEvalutionTool/backend/routers/app_api_keys.py index af586e66..a615c230 100644 --- a/Evaluation/AISearchEvalutionTool/backend/routers/app_api_keys.py +++ b/Evaluation/AISearchEvalutionTool/backend/routers/app_api_keys.py @@ -1,11 +1,19 @@ from __future__ import annotations import anthropic +import httpx import openai from fastapi import APIRouter, HTTPException from pydantic import BaseModel from db.database import get_app, update_app -from agents.llm_client import _openai_client_and_model, _openai_call_kwargs, _parse_azure_endpoint +from agents.llm_client import ( + _extract_gemini_text, + _gemini_endpoint, + _gemini_payload, + _openai_client_and_model, + _openai_call_kwargs, + _parse_azure_endpoint, +) router = APIRouter(prefix="/apps/{app_id}/api-keys", tags=["app-api-keys"]) @@ -15,6 +23,8 @@ class AppApiKeyUpdate(BaseModel): anthropic_base_url: str | None = None openai_key: str | None = None openai_base_url: str | None = None + gemini_key: str | None = None + gemini_base_url: str | None = None case1_threshold: float | None = None case2_threshold: float | None = None @@ -32,6 +42,7 @@ def get_app_api_keys(app_id: str): raise HTTPException(404, "App not found") ant_key = app.get("anthropic_key", "") oai_key = app.get("openai_key", "") + gemini_key = app.get("gemini_key", "") return { "anthropic_key_set": bool(ant_key), "anthropic_key_preview": _mask(ant_key), @@ -39,6 +50,9 @@ def get_app_api_keys(app_id: str): "openai_key_set": bool(oai_key), "openai_key_preview": _mask(oai_key), "openai_base_url": app.get("openai_base_url", "") or "", + "gemini_key_set": bool(gemini_key), + "gemini_key_preview": _mask(gemini_key), + "gemini_base_url": app.get("gemini_base_url", "") or "", "case1_threshold": float(app.get("case1_threshold") or 0.5), "case2_threshold": float(app.get("case2_threshold") or 0.5), } @@ -57,6 +71,10 @@ def set_app_api_keys(app_id: str, body: AppApiKeyUpdate): data["openai_key"] = body.openai_key.strip() if body.openai_base_url is not None: data["openai_base_url"] = body.openai_base_url.strip() + if body.gemini_key is not None: + data["gemini_key"] = body.gemini_key.strip() + if body.gemini_base_url is not None: + data["gemini_base_url"] = body.gemini_base_url.strip() if body.case1_threshold is not None: data["case1_threshold"] = max(0.0, min(1.0, float(body.case1_threshold))) if body.case2_threshold is not None: @@ -95,6 +113,42 @@ def test_app_anthropic(app_id: str, body: TestKeyRequest = TestKeyRequest()): raise HTTPException(502, str(exc)) +@router.post("/test-gemini") +def test_app_gemini(app_id: str, body: TestKeyRequest = TestKeyRequest()): + app = get_app(app_id) + if not app: + raise HTTPException(404, "App not found") + key = (body.key or "").strip() or app.get("gemini_key", "") + if not key: + raise HTTPException(400, "No Gemini API key provided or saved") + + raw_url = (body.base_url if body.base_url is not None else app.get("gemini_base_url", "")) or "" + base_url = raw_url.strip() + if base_url and not base_url.startswith("http"): + raise HTTPException(400, "Invalid base URL — must start with http(s)://") + + test_app = {**app, "gemini_key": key, "gemini_base_url": base_url} + try: + endpoint, api_key = _gemini_endpoint(test_app["app_id"], "gemini-2.5-flash", _app_override=test_app) + resp = httpx.post( + endpoint, + params={"key": api_key}, + json=_gemini_payload("", "Say hello in exactly 3 words.", 60, 0.0), + timeout=60.0, + ) + resp.raise_for_status() + text, _finish_reason = _extract_gemini_text(resp.json()) + return {"ok": True, "response": text.strip()} + except httpx.HTTPStatusError as exc: + status = exc.response.status_code + detail = exc.response.text + if status in (401, 403): + raise HTTPException(status, "Invalid Gemini API key or insufficient permissions") + raise HTTPException(502, detail) + except Exception as exc: + raise HTTPException(502, str(exc)) + + @router.post("/test-openai") def test_app_openai(app_id: str, body: TestKeyRequest = TestKeyRequest()): app = get_app(app_id) diff --git a/Evaluation/AISearchEvalutionTool/backend/routers/evaluation.py b/Evaluation/AISearchEvalutionTool/backend/routers/evaluation.py index 6ba9546a..8299156e 100644 --- a/Evaluation/AISearchEvalutionTool/backend/routers/evaluation.py +++ b/Evaluation/AISearchEvalutionTool/backend/routers/evaluation.py @@ -21,10 +21,15 @@ def start_evaluation(app_id: str, body: EvaluationRequest, bg: BackgroundTasks): sample_mode=body.sample_mode, filter_mode=body.filter_mode, filter_prompt=body.filter_prompt, + filter_fields=body.filter_fields, enable_racl=body.enable_racl, user_email=body.user_email, answer_mode_override=body.answer_mode_override, question_types=body.question_types, + judge_mode=body.judge_mode, + case1_threshold=body.case1_threshold, + case2_threshold=body.case2_threshold, + top_k_pass=body.top_k_pass, job_id=job_id, ) return get_job(job_id) @@ -50,7 +55,7 @@ def test_filter_prompt(app_id: str, body: FilterPromptTestRequest): if not app: raise HTTPException(404, "App not found") try: - from agents.llm_client import call_llm + from agents.llm_client import call_llm_json from agents.prompts import FILTER_GENERATOR_PROMPT if body.prompt_text is not None: @@ -59,7 +64,7 @@ def test_filter_prompt(app_id: str, body: FilterPromptTestRequest): row = get_active_prompt(app_id, "filter_generator") system_prompt = row["prompt_text"] if row else FILTER_GENERATOR_PROMPT - raw = call_llm(app_id, "filter_generator", system_prompt, f"Question: {body.question}") + raw = call_llm_json(app_id, "filter_generator", system_prompt, f"Question: {body.question}") return {"raw_response": raw, "error": None} except Exception as exc: return {"raw_response": None, "error": str(exc)} diff --git a/Evaluation/AISearchEvalutionTool/backend/routers/generation.py b/Evaluation/AISearchEvalutionTool/backend/routers/generation.py index 1d7ed88c..1db31428 100644 --- a/Evaluation/AISearchEvalutionTool/backend/routers/generation.py +++ b/Evaluation/AISearchEvalutionTool/backend/routers/generation.py @@ -21,6 +21,7 @@ def start_generation(app_id: str, body: GenerationRequest, bg: BackgroundTasks): file_source_ids=body.file_source_ids, max_docs_per_source=body.max_docs_per_source, max_questions_per_doc=body.max_questions_per_doc, + target_language=body.target_language, filters=body.filters, job_id=job_id, ) diff --git a/Evaluation/AISearchEvalutionTool/backend/routers/golden_sets.py b/Evaluation/AISearchEvalutionTool/backend/routers/golden_sets.py index e187e568..d129ff10 100644 --- a/Evaluation/AISearchEvalutionTool/backend/routers/golden_sets.py +++ b/Evaluation/AISearchEvalutionTool/backend/routers/golden_sets.py @@ -169,6 +169,20 @@ def _resolve_match_spec(app_id: str, row_norm: dict[str, str]) -> tuple[list[dic return [], [] +_ALL_KNOWN_COLS = ( + set(_DOC_ID_COLS) | set(_URL_COLS) | set(_TITLE_COLS) | _STANDARD_COLS + | {"sys_content_type", "docid", "recordurl", "recordtitle"} +) + + +def _extract_custom_fields(norm: dict[str, str]) -> dict[str, str]: + """Return any columns that are not part of the standard schema.""" + return { + k: v for k, v in norm.items() + if k not in _ALL_KNOWN_COLS and v + } + + def _parse_csv(content: bytes, app_id: str) -> list[dict]: text = content.decode("utf-8-sig") reader = csv.DictReader(io.StringIO(text)) @@ -188,6 +202,7 @@ def _parse_csv(content: bytes, app_id: str) -> list[dict]: "reference_doc_ids": ref_ids, "reference_match_spec": match_spec, "sys_content_type": norm.get("sys_content_type") or None, + "custom_fields": _extract_custom_fields(norm), }) return rows @@ -228,6 +243,7 @@ def _cell(row_vals: tuple, col: str) -> str: "reference_doc_ids": ref_ids, "reference_match_spec": match_spec, "sys_content_type": norm.get("sys_content_type") or None, + "custom_fields": _extract_custom_fields(norm), }) wb.close() return cases @@ -387,11 +403,159 @@ def get_test_cases(app_id: str, version: str): "human_validated": bool(r.get("human_validated")), "status": r.get("status", "active"), "decision": r.get("decision"), + "primary_concern": r.get("primary_concern"), + "rationale": r.get("rationale"), + "case_id": r.get("case_id"), + "custom_fields": r.get("custom_fields") or {}, "scores": scores, }) return result +# ── Filter options ─────────────────────────────────────────────────────────── + +_FILTER_FIELD_LABELS: dict[str, str] = { + "sys_content_type": "Source Type (sys_content_type)", +} + + +@router.get("/{version}/filter-options") +def get_filter_options(app_id: str, version: str): + """Return which meta-filter fields are populated across all test cases in this golden set. + + Used by the Evaluate page to show only fields that actually have data. + Response: {"fields": [{"name": str, "label": str, "count": int}]} + """ + if not get_app(app_id): + raise HTTPException(404, "App not found") + cases = list_test_cases(app_id, version) + counts: dict[str, int] = {} + for tc in cases: + meta = tc.get("generation_metadata") or {} + if str(meta.get("sys_content_type") or "").strip(): + counts["sys_content_type"] = counts.get("sys_content_type", 0) + 1 + for k, v in (tc.get("custom_fields") or {}).items(): + if k and str(v or "").strip(): + counts[k] = counts.get(k, 0) + 1 + fields = [ + { + "name": name, + "label": _FILTER_FIELD_LABELS.get(name, name.replace("_", " ").title()), + "count": count, + } + for name, count in sorted(counts.items(), key=lambda x: -x[1]) + ] + return {"fields": fields} + + +# ── Export ────────────────────────────────────────────────────────────────── + +@router.get("/{version}/export") +def export_golden_set(app_id: str, version: str): + """Stream all test cases for a golden set as a .xlsx file.""" + if not get_app(app_id): + raise HTTPException(404, "App not found") + cases = list_test_cases(app_id, version) + if not cases: + raise HTTPException(404, "No test cases found for this golden set") + + try: + import openpyxl + from openpyxl.styles import Font, PatternFill, Alignment + from openpyxl.utils import get_column_letter + except ImportError: + raise HTTPException(500, "openpyxl is required for export") + + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "Test Cases" + + HDR_FILL = PatternFill(start_color="1F2937", end_color="1F2937", fill_type="solid") + HDR_FONT = Font(color="FFFFFF", bold=True) + KEEP_FILL = PatternFill(start_color="D1FAE5", end_color="D1FAE5", fill_type="solid") + DROP_FILL = PatternFill(start_color="FEE2E2", end_color="FEE2E2", fill_type="solid") + BORDER_FILL = PatternFill(start_color="FEF3C7", end_color="FEF3C7", fill_type="solid") + + # Collect all custom field keys across all test cases (preserving first-seen order) + all_custom_keys: list[str] = [] + seen_custom_keys: set[str] = set() + for tc in cases: + for k in (tc.get("custom_fields") or {}).keys(): + if k not in seen_custom_keys: + all_custom_keys.append(k) + seen_custom_keys.add(k) + + base_headers = [ + "Question", + "Expected Answer", + "Expected Behavior", + "Case ID", + "Question Type", + "Difficulty", + "Reference Doc IDs", + "Status", + "Decision", + "Primary Concern", + "Rationale", + ] + headers = base_headers + [k.replace("_", " ").title() for k in all_custom_keys] + base_widths = [60, 50, 18, 8, 18, 10, 50, 12, 12, 25, 70] + all_widths = base_widths + [25] * len(all_custom_keys) + + ws.append(headers) + for i, cell in enumerate(ws[1], start=1): + cell.fill = HDR_FILL + cell.font = HDR_FONT + cell.alignment = Alignment(horizontal="left", wrap_text=True) + ws.column_dimensions[get_column_letter(i)].width = all_widths[i - 1] + ws.row_dimensions[1].height = 30 + + for tc in cases: + decision = (tc.get("decision") or "").upper() + ref_ids = ", ".join(tc.get("reference_doc_ids") or []) + custom = tc.get("custom_fields") or {} + row = [ + tc.get("question"), + tc.get("expected_answer"), + tc.get("expected_behavior"), + tc.get("case_id"), + tc.get("question_type"), + tc.get("difficulty"), + ref_ids, + tc.get("status"), + tc.get("decision"), + tc.get("primary_concern"), + tc.get("rationale"), + ] + [custom.get(k, "") for k in all_custom_keys] + ws.append(row) + row_idx = ws.max_row + if decision == "KEEP": + fill = KEEP_FILL + elif decision == "DROP": + fill = DROP_FILL + elif decision == "BORDERLINE": + fill = BORDER_FILL + else: + fill = None + if fill: + for col_idx in range(1, len(headers) + 1): + ws.cell(row=row_idx, column=col_idx).fill = fill + ws.row_dimensions[row_idx].height = 15 + + ws.freeze_panes = "A2" + + buf = io.BytesIO() + wb.save(buf) + buf.seek(0) + safe_version = version.replace("/", "-") + filename = f"golden-set-{safe_version}.xlsx" + return StreamingResponse( + buf, + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + # ── Delete ────────────────────────────────────────────────────────────────── @router.get("/{version}/delete-impact") diff --git a/Evaluation/AISearchEvalutionTool/backend/routers/llm_config.py b/Evaluation/AISearchEvalutionTool/backend/routers/llm_config.py index 3f8552c9..41be61d8 100644 --- a/Evaluation/AISearchEvalutionTool/backend/routers/llm_config.py +++ b/Evaluation/AISearchEvalutionTool/backend/routers/llm_config.py @@ -4,7 +4,7 @@ router = APIRouter(prefix="/apps/{app_id}/llm-config", tags=["llm-config"]) -VALID_AGENTS = {"agent1", "agent2", "agent3", "judge"} +VALID_AGENTS = {"agent1", "agent2", "agent3", "judge", "filter_generator", "insights", "answer_generator"} @router.get("", response_model=list[LLMConfigResponse]) diff --git a/Evaluation/AISearchEvalutionTool/backend/routers/prompt_tuner.py b/Evaluation/AISearchEvalutionTool/backend/routers/prompt_tuner.py new file mode 100644 index 00000000..33d48fef --- /dev/null +++ b/Evaluation/AISearchEvalutionTool/backend/routers/prompt_tuner.py @@ -0,0 +1,520 @@ +"""Prompt Fine-Tuner — produces a revised system prompt from real failure samples. + +Workflow (frontend): + 1. Pick an agent (which prompt to improve). + 2. Pick an evaluation run that has failed cases. + 3. The UI shows the active prompt + a sample of failed (question / expected / generated) trios. + 4. User clicks "Generate improved prompt" → POST /finetune. + 5. Backend asks the configured LLM to rewrite the prompt around those failures. + 6. UI shows the diff; user can save the result as a new prompt version + (via the existing PUT /apps/{app_id}/prompts/{agent_name} endpoint). + +The router intentionally does NOT persist the new prompt — the user must +explicitly confirm and save through the existing prompts endpoint. This keeps +the regular prompt-edit history and version semantics intact. +""" +from __future__ import annotations + +import json +import logging +from typing import Any + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +from agents.llm_client import call_llm_json, parse_json_loose +from agents.prompts import DEFAULT_PROMPTS, PROMPT_TUNER_PROMPT +from db.database import ( + get_active_prompt, get_api_key, get_app, get_eval_results, get_eval_run, + get_llm_config, list_eval_runs, +) + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/apps/{app_id}/prompt-tuner", tags=["prompt-tuner"]) + +TUNABLE_AGENTS = { + "agent1", "agent2", "agent3", "judge", + "filter_generator", "insights", "answer_generator", +} + +# Friendly descriptions surfaced to the LLM so it knows what the prompt is for. +AGENT_ROLES = { + "agent1": "Document analyst — extracts distinctive, content-specific facts from a single document so a retriever can later match questions back to it.", + "agent2": "Test-case generator — writes search-engine-style Q&A pairs that can ONLY be answered using the given document.", + "agent3": "Test-case quality auditor — scores generated Q&A pairs on a 6-dimensional rubric and decides KEEP / BORDERLINE / DROP.", + "judge": "RAG response judge — scores a RAG system's answer vs. the expected answer across groundedness, relevance, completeness, safety, etc.", + "filter_generator": "Search filter generator — converts a user question into Kore.ai Advance Search metaFilter groups.", + "insights": "Run-level analyst — writes a markdown report explaining what's going right / wrong in one evaluation run.", + "answer_generator": "Answer generator — given a question and retrieved CONTEXT chunks, produces the final grounded answer the RAG system serves to the user. This is the prompt whose failures show up as 'generated_answer' in the evaluation results.", +} + +MAX_FAILURE_SAMPLES = 20 +TRUNC_QUESTION = 240 +TRUNC_EXPECTED = 400 +TRUNC_GENERATED = 500 +TRUNC_RATIONALE = 300 + + +# ── Schemas ───────────────────────────────────────────────────────────────── + + +class FailureSample(BaseModel): + tc_id: str + question: str + expected_answer: str | None = None + generated_answer: str | None = None + failure_category: str | None = None + judge_rationale: str | None = None + expected_doc_rank: int | None = None + question_type: str | None = None + case_id: int | None = None + + +class FailuresResponse(BaseModel): + run_id: str + agent_name: str + total_failures: int + returned: int + failures: list[FailureSample] + + +class FineTuneRequest(BaseModel): + agent_name: str = Field(..., description="Which agent's prompt to fine-tune") + current_prompt: str | None = Field( + default=None, + description="Existing prompt to improve. If omitted, the app's active prompt for this agent is used.", + ) + run_id: str | None = Field( + default=None, + description="Evaluation run to pull failures from. If omitted, supply failures explicitly.", + ) + failures: list[FailureSample] | None = Field( + default=None, + description="Optional explicit list of failure samples (overrides run-derived sampling).", + ) + max_samples: int = Field( + default=10, ge=1, le=MAX_FAILURE_SAMPLES, + description="Cap on how many failures to send to the LLM.", + ) + user_notes: str | None = Field( + default=None, + description="Optional free-form guidance (e.g. 'be stricter on hallucination', 'shorter answers').", + ) + + +class FineTuneResponse(BaseModel): + agent_name: str + model: str + llm_slot: str | None = None + current_prompt: str + improved_prompt: str + summary_of_changes: str + failure_patterns: list[str] + samples_used: int + raw_response: str + + +# ── Helpers ───────────────────────────────────────────────────────────────── + + +def _truncate(text: str | None, limit: int) -> str: + if not text: + return "" + text = str(text).strip() + return text if len(text) <= limit else text[: limit - 1] + "…" + + +def _resolve_current_prompt(app_id: str, agent_name: str, override: str | None) -> str: + """Pick the prompt to improve. Prefer the explicit override, then the active row, then the default.""" + if override is not None and override.strip(): + return override + row = get_active_prompt(app_id, agent_name) + if row and row.get("prompt_text"): + return row["prompt_text"] + default = DEFAULT_PROMPTS.get(agent_name) + if not default: + raise HTTPException( + 404, + f"No active or default prompt found for agent '{agent_name}'. " + "Set a prompt first on the Prompts & Models page.", + ) + return default + + +def _failed_rows_from_run(run_id: str) -> list[dict]: + """Return rows from the run that look like failures (verdict=fail OR passed=False).""" + rows = get_eval_results(run_id) + fails: list[dict] = [] + for r in rows: + verdict = r.get("verdict") + if verdict == "fail": + fails.append(r) + elif verdict not in ("pass", "fail"): + # No verdict — only treat as a failure if the judge clearly disliked it. + scores = r.get("scores") or {} + judge_red_flags = ( + scores.get("toxicity_detected") or scores.get("bias_detected") + or scores.get("banned_topic_violation") + ) + if judge_red_flags: + fails.append(r) + return fails + + +def _row_to_sample(r: dict) -> FailureSample: + scores = r.get("scores") or {} + return FailureSample( + tc_id=str(r.get("tc_id") or ""), + question=_truncate(r.get("question"), TRUNC_QUESTION), + expected_answer=_truncate(r.get("expected_answer"), TRUNC_EXPECTED) or None, + generated_answer=_truncate(r.get("rag_response"), TRUNC_GENERATED) or None, + failure_category=r.get("failure_category"), + judge_rationale=_truncate(r.get("judge_rationale"), TRUNC_RATIONALE) or None, + expected_doc_rank=r.get("expected_doc_rank"), + question_type=r.get("question_type"), + case_id=r.get("case_id"), + ) + + +def _infer_provider(model: str) -> str: + """Same logic the LLM client uses — picks the right key by model prefix.""" + name = (model or "").lower() + if name.startswith("claude"): + return "anthropic" + if name.startswith("gemini") or name.startswith("models/gemini"): + return "gemini" + return "openai" + + +def _available_providers(app_id: str) -> set[str]: + """Return the set of provider keys that are currently stored for this app.""" + out: set[str] = set() + for provider in ("openai", "anthropic", "gemini"): + if (get_api_key(app_id, provider) or "").strip(): + out.add(provider) + return out + + +def _pick_llm_slot( + app_id: str, target_agent: str, providers: set[str], +) -> tuple[str | None, dict]: + """Choose which agent's LLM config to drive the tuner with. + + Preference order: + 1. The target agent's own slot (most predictable for the user). + 2. The 'insights' slot — heavy-reasoning workhorse. + 3. Any other tunable agent whose model maps to a configured provider. + Returns ``(slot_name, cfg)`` or ``(None, {})`` when nothing fits. + """ + preference: list[str] = [target_agent] + if "insights" not in preference: + preference.append("insights") + for name in sorted(TUNABLE_AGENTS): + if name not in preference: + preference.append(name) + + for slot in preference: + try: + cfg = get_llm_config(app_id, slot) + except Exception: + continue + model = (cfg or {}).get("model") or "" + if not model: + continue + if _infer_provider(model) in providers: + return slot, cfg + return None, {} + + +def _humanise_llm_error(exc: Exception, *, model: str, llm_slot: str) -> tuple[str, int]: + """Turn raw provider errors into a short, actionable message + HTTP status.""" + try: + import openai + if isinstance(exc, openai.AuthenticationError): + return ( + f"The LLM provider rejected the API key for model '{model}' " + f"(agent slot '{llm_slot}'). Open the API Keys page, save a " + f"valid key for that provider, then retry.", + 401, + ) + if isinstance(exc, openai.PermissionDeniedError): + return ( + f"The API key for model '{model}' does not have access to that " + f"model. Check the key's permissions or switch to a model the " + f"key can use on the Prompts & Models page.", + 403, + ) + if isinstance(exc, openai.RateLimitError): + return ( + f"Rate-limited by the LLM provider while tuning with '{model}'. " + f"Wait a moment and try again, or pick a different model.", + 429, + ) + except ImportError: + pass + try: + import anthropic + if isinstance(exc, anthropic.AuthenticationError): + return ( + f"Invalid Anthropic API key for model '{model}'. Set a working " + f"key on the API Keys page and retry.", + 401, + ) + except ImportError: + pass + + # Default: keep the message short so the frontend can render it cleanly. + detail = str(exc) + if len(detail) > 400: + detail = detail[:400] + "…" + return (f"LLM call failed ({model}): {detail}", 502) + + +def _rank_failures(rows: list[dict]) -> list[dict]: + """Order failures so the LLM sees the most informative ones first. + + Priority: + 1. Rows where both expected_answer and rag_response are non-empty + (these give the clearest signal of where the prompt diverges). + 2. Rows with a judge_rationale (explains *why* it failed). + 3. Rows with a specific failure_category that isn't 'none'. + 4. Everything else. + """ + def key(r: dict) -> tuple: + has_pair = bool((r.get("expected_answer") or "").strip()) and bool((r.get("rag_response") or "").strip()) + has_rationale = bool((r.get("judge_rationale") or "").strip()) + has_category = bool((r.get("failure_category") or "").strip()) and r.get("failure_category") != "none" + # Lower tuple sorts first — invert booleans + return (not has_pair, not has_rationale, not has_category) + + return sorted(rows, key=key) + + +# ── Endpoints ─────────────────────────────────────────────────────────────── + + +@router.get("/agents") +def list_tunable_agents(app_id: str) -> dict[str, Any]: + """Return the agents that have tunable prompts, with their role descriptions.""" + if not get_app(app_id): + raise HTTPException(404, "App not found") + return { + "agents": [ + {"agent_name": name, "description": AGENT_ROLES.get(name, "")} + for name in sorted(TUNABLE_AGENTS) + ] + } + + +@router.get("/runs") +def list_runs_with_failures(app_id: str) -> dict[str, Any]: + """List completed runs along with a failure_count, sorted by most recent first.""" + if not get_app(app_id): + raise HTTPException(404, "App not found") + runs = list_eval_runs(app_id) + out = [] + for r in runs: + total = r.get("total_cases") or 0 + passed = r.get("passed_cases") or 0 + verdicted = r.get("verdicted_cases") or 0 + # Best-effort failure count: prefer (verdicted - passed), fall back to (total - passed). + if verdicted: + failed = max(verdicted - passed, 0) + else: + failed = max(total - passed, 0) + out.append({ + "run_id": r.get("run_id"), + "started_at": r.get("started_at"), + "finished_at": r.get("finished_at"), + "status": r.get("status"), + "golden_set_version": r.get("golden_set_version"), + "rag_version": r.get("rag_version"), + "total_cases": total, + "failure_count": failed, + }) + return {"runs": out} + + +@router.get("/runs/{run_id}/failures", response_model=FailuresResponse) +def get_run_failures( + app_id: str, + run_id: str, + agent_name: str, + limit: int = 20, +) -> FailuresResponse: + """Return a ranked list of failure samples (Q + expected + generated) for the run. + + The agent_name is accepted for parity with the fine-tune endpoint but + currently doesn't filter the rows — every agent reuses the same RAG + failure trace. Future versions could attach per-agent metadata. + """ + if agent_name not in TUNABLE_AGENTS: + raise HTTPException(400, f"agent_name must be one of {sorted(TUNABLE_AGENTS)}") + if not get_app(app_id): + raise HTTPException(404, "App not found") + run = get_eval_run(run_id) + if not run or run.get("app_id") != app_id: + raise HTTPException(404, "Run not found") + + fails = _failed_rows_from_run(run_id) + ranked = _rank_failures(fails) + limit = max(1, min(limit, MAX_FAILURE_SAMPLES)) + samples = [_row_to_sample(r) for r in ranked[:limit]] + return FailuresResponse( + run_id=run_id, + agent_name=agent_name, + total_failures=len(fails), + returned=len(samples), + failures=samples, + ) + + +@router.post("/finetune", response_model=FineTuneResponse) +def finetune_prompt(app_id: str, body: FineTuneRequest) -> FineTuneResponse: + """Generate an improved prompt by feeding the current prompt + real failures to an LLM.""" + if body.agent_name not in TUNABLE_AGENTS: + raise HTTPException(400, f"agent_name must be one of {sorted(TUNABLE_AGENTS)}") + if not get_app(app_id): + raise HTTPException(404, "App not found") + + current_prompt = _resolve_current_prompt(app_id, body.agent_name, body.current_prompt) + + # Gather failure samples — either from a run or from the explicit list. + samples: list[FailureSample] + if body.failures: + samples = body.failures[: body.max_samples] + elif body.run_id: + run = get_eval_run(body.run_id) + if not run or run.get("app_id") != app_id: + raise HTTPException(404, "Run not found") + fails = _failed_rows_from_run(body.run_id) + ranked = _rank_failures(fails) + samples = [_row_to_sample(r) for r in ranked[: body.max_samples]] + else: + raise HTTPException( + 400, + "Provide either run_id or an explicit failures list to fine-tune against.", + ) + + if not samples: + raise HTTPException( + 422, + "No failure samples available — nothing to learn from. Pick a run with failures " + "or supply failures explicitly.", + ) + + # Pick the LLM slot to drive the tuner. Try, in order: + # 1. The target agent's own LLM config (whatever model the user already + # trusts for this prompt). Usually the safest choice. + # 2. The 'insights' slot (heavy-reasoning workhorse). + # 3. Any other tunable agent whose model maps to a provider with a key. + # If nothing has a configured key, fail early with an actionable 400. + available_providers = _available_providers(app_id) + if not available_providers: + raise HTTPException( + 400, + "No LLM API key configured for this app. Open the API Keys page and " + "set at least one provider key (OpenAI, Anthropic or Gemini) before " + "running the prompt tuner.", + ) + + llm_slot, cfg = _pick_llm_slot(app_id, body.agent_name, available_providers) + if not llm_slot: + raise HTTPException( + 400, + "No agent has an LLM model configured for a provider whose API key " + "is set. Either add an API key for the model you want to use " + "(API Keys page) or change an agent's model on the Prompts & Models " + "page to a provider whose key is set: " + f"{', '.join(sorted(available_providers))}.", + ) + model = cfg.get("model", "") + + user_msg = _build_user_message( + agent_name=body.agent_name, + current_prompt=current_prompt, + samples=samples, + user_notes=body.user_notes, + ) + + logger.info( + "Prompt tuner | starting | app=%s target=%s llm_slot=%s model=%s samples=%d", + app_id, body.agent_name, llm_slot, model, len(samples), + ) + + # Route the call through the picked agent slot so call_llm_json reads its + # model / temp / max_tokens. The system_prompt stays the meta-prompt — the + # LLM is rewriting another prompt, not playing whatever role the slot is for. + try: + raw = call_llm_json(app_id, llm_slot, PROMPT_TUNER_PROMPT, user_msg) + except Exception as exc: + msg, status = _humanise_llm_error(exc, model=model, llm_slot=llm_slot) + logger.error( + "Prompt tuner | LLM call failed | model=%s slot=%s | %s", + model, llm_slot, exc, exc_info=True, + ) + raise HTTPException(status, msg) + + parsed = parse_json_loose(raw, expect="object", agent_name="prompt_tuner") + if not isinstance(parsed, dict): + logger.error( + "Prompt tuner | could not parse JSON | raw[:500]=%s", + (raw or "")[:500], + ) + raise HTTPException( + 502, + "The LLM returned a response we couldn't parse as JSON. Try again, " + "or simplify the failures list.", + ) + + improved = (parsed.get("improved_prompt") or "").strip() + summary = (parsed.get("summary_of_changes") or "").strip() + patterns = parsed.get("failure_patterns") or [] + if not isinstance(patterns, list): + patterns = [str(patterns)] + patterns = [str(p) for p in patterns] + + if not improved: + improved = current_prompt + summary = summary or "The model did not propose a change." + + return FineTuneResponse( + agent_name=body.agent_name, + model=model, + llm_slot=llm_slot, + current_prompt=current_prompt, + improved_prompt=improved, + summary_of_changes=summary, + failure_patterns=patterns, + samples_used=len(samples), + raw_response=raw or "", + ) + + +def _build_user_message( + *, + agent_name: str, + current_prompt: str, + samples: list[FailureSample], + user_notes: str | None, +) -> str: + """Compose the structured payload the meta-LLM sees.""" + role = AGENT_ROLES.get(agent_name, "") + serialised_samples = [s.model_dump(exclude_none=True) for s in samples] + parts = [ + f"# PROMPT_ROLE\n{role or agent_name}\n", + f"# CURRENT_PROMPT (agent_name = {agent_name})\n```\n{current_prompt}\n```\n", + ] + if user_notes and user_notes.strip(): + parts.append(f"# USER_NOTES\n{user_notes.strip()}\n") + parts.append( + f"# FAILURE_SAMPLES ({len(samples)} cases — JSON array, each item has the " + f"question, expected_answer, generated_answer, and any judge metadata)\n" + f"```json\n{json.dumps(serialised_samples, indent=2, ensure_ascii=False)}\n```\n" + ) + parts.append( + "Now produce the JSON object specified in the system prompt. " + "Respond with JSON ONLY, no markdown fences, no extra prose." + ) + return "\n".join(parts) diff --git a/Evaluation/AISearchEvalutionTool/backend/routers/prompts.py b/Evaluation/AISearchEvalutionTool/backend/routers/prompts.py index a02bc385..31a9ef61 100644 --- a/Evaluation/AISearchEvalutionTool/backend/routers/prompts.py +++ b/Evaluation/AISearchEvalutionTool/backend/routers/prompts.py @@ -5,7 +5,17 @@ router = APIRouter(prefix="/apps/{app_id}/prompts", tags=["prompts"]) -VALID_AGENTS = {"agent1", "agent2", "agent3", "judge"} +VALID_AGENTS = { + "agent1", "agent2", "agent3", "judge", + "filter_generator", "filter_mapper", "insights", + "answer_generator", +} + +TUNABLE_AGENTS = { + "agent1", "agent2", "agent3", "judge", + "filter_generator", "insights", + "answer_generator", +} @router.get("", response_model=list[PromptConfigResponse]) diff --git a/Evaluation/AISearchEvalutionTool/backend/routers/results.py b/Evaluation/AISearchEvalutionTool/backend/routers/results.py index 4b001371..8af85e16 100644 --- a/Evaluation/AISearchEvalutionTool/backend/routers/results.py +++ b/Evaluation/AISearchEvalutionTool/backend/routers/results.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import io import json import logging @@ -66,6 +68,15 @@ def get_run_results(app_id: str, run_id: str): passed = verdict == "pass" else: passed = _is_pass(scores, expected_behavior) + # Build docId → {title, url} from stored chunk_signals (deduped by docId) + doc_label_map: dict[str, dict] = {} + for chunk in r.get("chunk_signals") or []: + doc_id = chunk.get("docId") + if doc_id and doc_id not in doc_label_map: + doc_label_map[doc_id] = { + "title": chunk.get("recordTitle"), + "url": chunk.get("recordUrl"), + } result.append({ "tc_id": r["tc_id"], "question": r["question"], @@ -83,6 +94,8 @@ def get_run_results(app_id: str, run_id: str): "latency_retrieval_ms": r.get("latency_retrieval_ms"), "doc_retrieved": scores.get("doc_retrieved", False), "search_payload": r.get("search_payload") or {}, + "search_response": r.get("search_response") or {}, + "doc_label_map": doc_label_map, # 4-case fields "case_id": r.get("case_id"), "expected_doc_rank": r.get("expected_doc_rank"), diff --git a/Evaluation/AISearchEvalutionTool/backend/scripts/migrate_sqlite_to_mongo.py b/Evaluation/AISearchEvalutionTool/backend/scripts/migrate_sqlite_to_mongo.py new file mode 100644 index 00000000..94fade68 --- /dev/null +++ b/Evaluation/AISearchEvalutionTool/backend/scripts/migrate_sqlite_to_mongo.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path +from typing import Any + +from pymongo import ASCENDING, MongoClient + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from config import get_config # noqa: E402 + + +COLLECTIONS = [ + "app_config", "llm_config", "prompt_config", "source_document", + "golden_set", "test_case", "agent_scores", "job", "eval_run", "eval_result", +] + +JSON_FIELDS = { + "app_config": {"racl_entity_ids": [], "banned_topics": []}, + "source_document": {"metadata": {}}, + "test_case": {"reference_doc_ids": [], "reference_match_spec": [], "generation_metadata": {}}, + "job": {"result": None}, + "eval_run": {"diagnostics_json": None, "fired_rule_ids": None}, + "eval_result": { + "retrieved_doc_ids": [], + "chunk_signals": [], + "scores": {}, + "search_payload": {}, + "search_response": {}, + "recall_at_k": {}, + }, +} + +KEYS = { + "app_config": ("app_id",), + "llm_config": ("app_id", "agent_name"), + "prompt_config": ("id",), + "source_document": ("doc_id", "app_id"), + "golden_set": ("version", "app_id"), + "test_case": ("tc_id",), + "agent_scores": ("tc_id",), + "job": ("job_id",), + "eval_run": ("run_id",), + "eval_result": ("run_id", "tc_id"), +} + + +def _decode(value: Any, default: Any) -> Any: + if value in (None, ""): + return default + if isinstance(value, (list, dict)): + return value + try: + return json.loads(value) + except (TypeError, ValueError): + return default + + +def _read_table(conn: sqlite3.Connection, table: str) -> list[dict]: + try: + rows = conn.execute(f"SELECT * FROM {table}").fetchall() + except sqlite3.OperationalError: + return [] + result = [] + for row in rows: + doc = dict(row) + for field, default in JSON_FIELDS.get(table, {}).items(): + if field in doc: + doc[field] = _decode(doc[field], default) + if "is_active" in doc: + doc["is_active"] = bool(doc["is_active"]) + if "human_validated" in doc: + doc["human_validated"] = bool(doc["human_validated"]) + if "stop_requested" in doc: + doc["stop_requested"] = bool(doc["stop_requested"]) + result.append(doc) + return result + + +def _ensure_indexes(db) -> None: + db.app_config.create_index([("app_id", ASCENDING)], unique=True) + db.llm_config.create_index([("app_id", ASCENDING), ("agent_name", ASCENDING)], unique=True) + db.prompt_config.create_index([("id", ASCENDING)], unique=True) + db.source_document.create_index([("doc_id", ASCENDING), ("app_id", ASCENDING)], unique=True) + db.golden_set.create_index([("version", ASCENDING), ("app_id", ASCENDING)], unique=True) + db.test_case.create_index([("tc_id", ASCENDING)], unique=True) + db.agent_scores.create_index([("tc_id", ASCENDING)], unique=True) + db.job.create_index([("job_id", ASCENDING)], unique=True) + db.eval_run.create_index([("run_id", ASCENDING)], unique=True) + db.eval_result.create_index([("run_id", ASCENDING), ("tc_id", ASCENDING)], unique=True) + + +def migrate(sqlite_path: Path, mongo_uri: str, mongo_db: str, dry_run: bool) -> None: + if not sqlite_path.exists(): + raise FileNotFoundError(f"SQLite database not found: {sqlite_path}") + + conn = sqlite3.connect(str(sqlite_path)) + conn.row_factory = sqlite3.Row + client = MongoClient(mongo_uri, serverSelectionTimeoutMS=3000) + db = client[mongo_db] + + if not dry_run: + _ensure_indexes(db) + + print(f"SQLite: {sqlite_path}") + print(f"MongoDB: {mongo_uri}/{mongo_db}") + print(f"Mode: {'dry-run' if dry_run else 'write'}") + + for table in COLLECTIONS: + docs = _read_table(conn, table) + print(f"{table}: {len(docs)} row(s)") + if dry_run: + continue + key_fields = KEYS[table] + coll = db[table] + for doc in docs: + query = {field: doc[field] for field in key_fields} + coll.update_one(query, {"$set": doc}, upsert=True) + + conn.close() + client.close() + + +def main() -> None: + cfg = get_config().database + parser = argparse.ArgumentParser(description="Migrate AISearch evaluator SQLite data into MongoDB.") + parser.add_argument("--sqlite-path", default=str(cfg.sqlite_db_path)) + parser.add_argument("--mongo-uri", default=cfg.mongodb_uri) + parser.add_argument("--mongo-db", default=cfg.mongodb_database) + parser.add_argument("--dry-run", action="store_true", help="Print counts without writing to MongoDB.") + args = parser.parse_args() + migrate(Path(args.sqlite_path), args.mongo_uri, args.mongo_db, args.dry_run) + + +if __name__ == "__main__": + main() diff --git a/Evaluation/AISearchEvalutionTool/frontend/src/App.tsx b/Evaluation/AISearchEvalutionTool/frontend/src/App.tsx index f18aea8b..a41c45f2 100644 --- a/Evaluation/AISearchEvalutionTool/frontend/src/App.tsx +++ b/Evaluation/AISearchEvalutionTool/frontend/src/App.tsx @@ -6,6 +6,7 @@ import GeneratePage from "@/pages/GeneratePage"; import EvaluatePage from "@/pages/EvaluatePage"; import GoldenSetsPage from "@/pages/GoldenSetsPage"; import PromptsPage from "@/pages/PromptsPage"; +import PromptTunerPage from "@/pages/PromptTunerPage"; import LLMConfigPage from "@/pages/LLMConfigPage"; import ResultsPage from "@/pages/ResultsPage"; import RunDetailPage from "@/pages/RunDetailPage"; @@ -22,6 +23,7 @@ export default function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/Evaluation/AISearchEvalutionTool/frontend/src/components/Layout.tsx b/Evaluation/AISearchEvalutionTool/frontend/src/components/Layout.tsx index 19e61502..df5cbe6f 100644 --- a/Evaluation/AISearchEvalutionTool/frontend/src/components/Layout.tsx +++ b/Evaluation/AISearchEvalutionTool/frontend/src/components/Layout.tsx @@ -1,4 +1,4 @@ -import { useState } from "react"; +import { useEffect, useState } from "react"; import { Outlet, NavLink, useParams, useNavigate } from "react-router-dom"; import { useQuery } from "@tanstack/react-query"; import { appsApi } from "@/lib/api"; @@ -6,17 +6,18 @@ import type { AppConfig } from "@/lib/api"; import { cn } from "@/lib/utils"; import { LayoutDashboard, Database, Zap, FlaskConical, BookOpen, - Settings2, BarChart3, ChevronDown, Plus, Bot, KeySquare + Settings2, BarChart3, ChevronDown, Plus, Bot, KeySquare, Moon, Sun, Wand2 } from "lucide-react"; const appNavItems = (appId: string) => [ - { to: `/apps/${appId}/sources`, label: "Sources", icon: Database }, - { to: `/apps/${appId}/generate`, label: "Generate", icon: Zap }, - { to: `/apps/${appId}/golden-sets`, label: "Golden Sets", icon: BookOpen }, - { to: `/apps/${appId}/evaluate`, label: "Evaluate", icon: FlaskConical }, - { to: `/apps/${appId}/results`, label: "Results", icon: BarChart3 }, - { to: `/apps/${appId}/prompts`, label: "Prompts & Models", icon: Settings2 }, - { to: `/apps/${appId}/api-keys`, label: "API Keys", icon: KeySquare }, + { to: `/apps/${appId}/sources`, label: "Sources", icon: Database }, + { to: `/apps/${appId}/generate`, label: "Generate", icon: Zap }, + { to: `/apps/${appId}/golden-sets`, label: "Golden Sets", icon: BookOpen }, + { to: `/apps/${appId}/evaluate`, label: "Evaluate", icon: FlaskConical }, + { to: `/apps/${appId}/results`, label: "Results", icon: BarChart3 }, + { to: `/apps/${appId}/prompts`, label: "Prompts & Models", icon: Settings2 }, + { to: `/apps/${appId}/prompt-tuner`, label: "Prompt Tuner", icon: Wand2 }, + { to: `/apps/${appId}/api-keys`, label: "API Keys", icon: KeySquare }, // TODO: missing QueryPage — add { to: `/apps/${appId}/query`, label: "Query", icon: Search } once QueryPage.tsx is created ]; @@ -24,6 +25,16 @@ export default function Layout() { const { appId } = useParams<{ appId?: string }>(); const navigate = useNavigate(); const [sidebarOpen, setSidebarOpen] = useState(true); + const [darkMode, setDarkMode] = useState(() => { + const saved = localStorage.getItem("theme"); + if (saved) return saved === "dark"; + return window.matchMedia?.("(prefers-color-scheme: dark)").matches ?? false; + }); + + useEffect(() => { + document.documentElement.classList.toggle("dark", darkMode); + localStorage.setItem("theme", darkMode ? "dark" : "light"); + }, [darkMode]); const { data: apps = [] } = useQuery({ queryKey: ["apps"], @@ -92,6 +103,18 @@ export default function Layout() { {/* Main content */}
+
+ +
diff --git a/Evaluation/AISearchEvalutionTool/frontend/src/index.css b/Evaluation/AISearchEvalutionTool/frontend/src/index.css index b177726a..221eef13 100644 --- a/Evaluation/AISearchEvalutionTool/frontend/src/index.css +++ b/Evaluation/AISearchEvalutionTool/frontend/src/index.css @@ -11,7 +11,127 @@ body { margin: 0; } +html.dark { + color-scheme: dark; +} + +html.dark body { + background: #000000; +} + #root { height: 100vh; width: 100%; } + +/* Dark theme compatibility layer for the existing Tailwind utility-heavy UI. */ +html.dark .bg-white, +html.dark .bg-gray-50 { + background-color: #050505 !important; +} + +html.dark .bg-gray-100 { + background-color: #111111 !important; +} + +html.dark .bg-gray-200 { + background-color: #1f1f1f !important; +} + +/* Soften every neutral border so cards / panels separate from the page through + surface contrast rather than visible 1px lines. The values below are tuned + so a card edge is just barely perceptible against the #050505 surface — no + "deep" outlines. */ +html.dark .border-gray-100 { + border-color: rgb(255 255 255 / 0.04) !important; +} +html.dark .border-gray-200, +html.dark .border-gray-300 { + border-color: rgb(255 255 255 / 0.06) !important; +} + +/* Match the same softness for divide-y / divide-x stripes between rows. */ +html.dark .divide-gray-50 > :not([hidden]) ~ :not([hidden]), +html.dark .divide-gray-100 > :not([hidden]) ~ :not([hidden]) { + border-color: rgb(255 255 255 / 0.05) !important; +} +html.dark .divide-gray-200 > :not([hidden]) ~ :not([hidden]) { + border-color: rgb(255 255 255 / 0.07) !important; +} + +html.dark .text-gray-900, +html.dark .text-gray-800, +html.dark .text-gray-700 { + color: #e2e8f0 !important; +} + +html.dark .text-gray-600, +html.dark .text-gray-500 { + color: #cbd5e1 !important; +} + +html.dark .text-gray-400, +html.dark .text-gray-300 { + color: #94a3b8 !important; +} + +html.dark input, +html.dark textarea, +html.dark select { + background-color: #050505 !important; + border-color: #262626 !important; + color: #e2e8f0 !important; +} + +html.dark input::placeholder, +html.dark textarea::placeholder { + color: #64748b !important; +} + +html.dark .hover\:bg-gray-50:hover, +html.dark .hover\:bg-gray-100:hover { + background-color: #111111 !important; +} + +html.dark .bg-violet-50, +html.dark .bg-violet-100 { + background-color: rgb(91 33 182 / 0.24) !important; +} + +html.dark .text-violet-700, +html.dark .text-violet-600, +html.dark .text-violet-500 { + color: #c4b5fd !important; +} + +/* Accent (violet) borders also stay subtle in dark mode — they were drawing + too much attention as bright purple outlines. */ +html.dark .border-violet-100, +html.dark .border-violet-200, +html.dark .border-violet-300 { + border-color: rgb(167 139 250 / 0.18) !important; +} + +html.dark .bg-green-50, +html.dark .bg-emerald-50 { + background-color: rgb(20 83 45 / 0.24) !important; +} + +html.dark .bg-red-50 { + background-color: rgb(127 29 29 / 0.24) !important; +} + +html.dark .bg-amber-50 { + background-color: rgb(120 53 15 / 0.24) !important; +} + +/* Drop heavy drop-shadows in dark mode — they read as halos on a black page. + Keep just enough to lift floating elements (popovers, modals via .shadow-lg) + above the surface. */ +html.dark .shadow, +html.dark .shadow-sm { + box-shadow: none !important; +} +html.dark .shadow-lg { + box-shadow: 0 8px 24px rgb(0 0 0 / 0.45) !important; +} diff --git a/Evaluation/AISearchEvalutionTool/frontend/src/lib/api.ts b/Evaluation/AISearchEvalutionTool/frontend/src/lib/api.ts index 4e5d9184..96008602 100644 --- a/Evaluation/AISearchEvalutionTool/frontend/src/lib/api.ts +++ b/Evaluation/AISearchEvalutionTool/frontend/src/lib/api.ts @@ -106,6 +106,10 @@ export interface TestCase { human_validated: boolean; status: string; decision: string | null; + primary_concern: string | null; + rationale: string | null; + case_id: number | null; + custom_fields: Record; scores: Record | null; } @@ -125,6 +129,11 @@ export interface EvalRun { avg_chunk_rank: number | null; } +export interface DocLabel { + title: string | null; + url: string | null; +} + export interface EvalResult { tc_id: string; question: string; @@ -142,6 +151,9 @@ export interface EvalResult { latency_retrieval_ms: number | null; doc_retrieved: boolean; search_payload?: Record | null; + search_response?: Record | null; + /** docId → {title, url} built from stored chunk_signals */ + doc_label_map?: Record; // 4-case evaluation fields case_id: number | null; expected_doc_rank: number | null; @@ -202,6 +214,7 @@ export const generationApi = { file_source_ids?: string[]; max_docs_per_source: number; max_questions_per_doc: number; + target_language: string; filters: Record; }) => api.post(`/apps/${appId}/generation/start`, data).then((r) => r.data), getJob: (appId: string, jobId: string) => @@ -214,9 +227,11 @@ export const generationApi = { api.post(`/apps/${appId}/generation/jobs/${jobId}/stop`).then((r) => r.data), }; -export type FilterMode = "none" | "auto_source" | "custom_prompt"; +export type FilterMode = "none" | "field_filters" | "custom_prompt"; export type ScriptLang = "python" | "js"; +export type JudgeMode = "auto" | "force_on" | "force_off"; + export const evaluationApi = { start: (appId: string, data: { golden_set_version: string; @@ -225,10 +240,15 @@ export const evaluationApi = { sample_mode: "first" | "random"; filter_mode: FilterMode; filter_prompt: string | null; + filter_fields: string[] | null; enable_racl: boolean; user_email: string | null; answer_mode_override: AnswerMode | null; question_types: string[] | null; + judge_mode?: JudgeMode; + case1_threshold?: number | null; + case2_threshold?: number | null; + top_k_pass?: number | null; }) => api.post(`/apps/${appId}/evaluation/start`, data).then((r) => r.data), getJob: (appId: string, jobId: string) => api.get(`/apps/${appId}/evaluation/jobs/${jobId}`).then((r) => r.data), @@ -286,6 +306,12 @@ export const goldenSetsApi = { ).then((r) => r.data); }, templateUrl: (appId: string) => `/api/apps/${appId}/golden-sets/template`, + exportUrl: (appId: string, version: string) => + `/api/apps/${appId}/golden-sets/${encodeURIComponent(version)}/export`, + filterOptions: (appId: string, version: string) => + api.get<{ fields: { name: string; label: string; count: number }[] }>( + `/apps/${appId}/golden-sets/${encodeURIComponent(version)}/filter-options`, + ).then((r) => r.data), getDeleteImpact: (appId: string, version: string) => api.get( `/apps/${appId}/golden-sets/${encodeURIComponent(version)}/delete-impact`, @@ -360,6 +386,16 @@ export interface RunDiagnostics { recall_at_5: number | null; recall_at_10: number | null; }; + chunk_lifecycle?: { + cases_with_reference: number; + cases_with_matched_chunk: number; + retrieval_qualified: number; + sent_to_llm: number; + used_in_answer: number; + retrieval_accuracy: number | null; + sent_to_llm_accuracy: number | null; + answer_gen_accuracy: number | null; + }; judge_metric_avgs: Record; weakest_judge_metric: string | null; latency_ms: { @@ -453,6 +489,9 @@ export interface ApiKeyStatus { openai_key_set: boolean; openai_key_preview: string; openai_base_url: string; + gemini_key_set: boolean; + gemini_key_preview: string; + gemini_base_url: string; case1_threshold: number; case2_threshold: number; } @@ -462,6 +501,8 @@ export interface AppApiKeysUpdate { anthropic_base_url?: string; openai_key?: string; openai_base_url?: string; + gemini_key?: string; + gemini_base_url?: string; case1_threshold?: number; case2_threshold?: number; } @@ -474,4 +515,83 @@ export const appApiKeysApi = { api.post<{ ok: boolean; response: string }>(`/apps/${appId}/api-keys/test-anthropic`, { key: key || null, base_url: baseUrl || null }).then((r) => r.data), testOpenAI: (appId: string, key?: string, baseUrl?: string) => api.post<{ ok: boolean; response: string }>(`/apps/${appId}/api-keys/test-openai`, { key: key || null, base_url: baseUrl || null }).then((r) => r.data), + testGemini: (appId: string, key?: string, baseUrl?: string) => + api.post<{ ok: boolean; response: string }>(`/apps/${appId}/api-keys/test-gemini`, { key: key || null, base_url: baseUrl || null }).then((r) => r.data), +}; + +// ── Prompt Tuner ───────────────────────────────────────────────────────────── + +export interface TunableAgent { + agent_name: string; + description: string; +} + +export interface PromptTunerRunSummary { + run_id: string; + started_at: string; + finished_at: string | null; + status: string; + golden_set_version: string; + rag_version: string; + total_cases: number; + failure_count: number; +} + +export interface FailureSample { + tc_id: string; + question: string; + expected_answer?: string | null; + generated_answer?: string | null; + failure_category?: string | null; + judge_rationale?: string | null; + expected_doc_rank?: number | null; + question_type?: string | null; + case_id?: number | null; +} + +export interface PromptTunerFailuresResponse { + run_id: string; + agent_name: string; + total_failures: number; + returned: number; + failures: FailureSample[]; +} + +export interface PromptTunerFineTuneRequest { + agent_name: string; + current_prompt?: string | null; + run_id?: string | null; + failures?: FailureSample[] | null; + max_samples?: number; + user_notes?: string | null; +} + +export interface PromptTunerFineTuneResponse { + agent_name: string; + model: string; + llm_slot?: string | null; + current_prompt: string; + improved_prompt: string; + summary_of_changes: string; + failure_patterns: string[]; + samples_used: number; + raw_response: string; +} + +export const promptTunerApi = { + listAgents: (appId: string) => + api.get<{ agents: TunableAgent[] }>(`/apps/${appId}/prompt-tuner/agents`).then((r) => r.data.agents), + listRuns: (appId: string) => + api.get<{ runs: PromptTunerRunSummary[] }>(`/apps/${appId}/prompt-tuner/runs`).then((r) => r.data.runs), + getFailures: (appId: string, runId: string, agentName: string, limit = 20) => + api.get( + `/apps/${appId}/prompt-tuner/runs/${runId}/failures`, + { params: { agent_name: agentName, limit } }, + ).then((r) => r.data), + fineTune: (appId: string, body: PromptTunerFineTuneRequest) => + api.post( + `/apps/${appId}/prompt-tuner/finetune`, + body, + { timeout: 180_000 }, + ).then((r) => r.data), }; diff --git a/Evaluation/AISearchEvalutionTool/frontend/src/pages/AppApiKeysPage.tsx b/Evaluation/AISearchEvalutionTool/frontend/src/pages/AppApiKeysPage.tsx index 5c27e135..68466b83 100644 --- a/Evaluation/AISearchEvalutionTool/frontend/src/pages/AppApiKeysPage.tsx +++ b/Evaluation/AISearchEvalutionTool/frontend/src/pages/AppApiKeysPage.tsx @@ -5,14 +5,15 @@ import { appApiKeysApi } from "@/lib/api"; import type { AppApiKeysUpdate } from "@/lib/api"; import { CheckCircle, XCircle, Eye, EyeOff, Loader2, Send, - Link as LinkIcon, Gauge, Terminal, ChevronDown, ChevronUp, + Link as LinkIcon, Terminal, ChevronDown, ChevronUp, AlertCircle, ArrowRight, Check, Save, } from "lucide-react"; import { cn } from "@/lib/utils"; // ── cURL parser ────────────────────────────────────────────────────────────── -type CurlResult = { provider: "anthropic" | "openai" | null; apiKey: string; baseUrl: string; error: string | null }; +type Provider = "anthropic" | "openai" | "gemini"; +type CurlResult = { provider: Provider | null; apiKey: string; baseUrl: string; error: string | null }; function parseCurl(raw: string): CurlResult { const s = raw.replace(/\\\s*\n/g, " ").replace(/\r?\n/g, " "); @@ -29,8 +30,9 @@ function parseCurl(raw: string): CurlResult { if (headers["authorization"]?.toLowerCase().startsWith("bearer ")) apiKey = headers["authorization"].slice(7).trim(); else if (headers["api-key"]) apiKey = headers["api-key"].trim(); else if (headers["x-api-key"]) apiKey = headers["x-api-key"].trim(); - if (!apiKey) return { provider: null, apiKey: "", baseUrl: "", error: "No API key found (Authorization: Bearer, api-key, or x-api-key)" }; - let provider: "anthropic" | "openai" | null = null; + else if (headers["x-goog-api-key"]) apiKey = headers["x-goog-api-key"].trim(); + if (!apiKey) return { provider: null, apiKey: "", baseUrl: "", error: "No API key found (Authorization: Bearer, api-key, x-api-key, or x-goog-api-key)" }; + let provider: Provider | null = null; let baseUrl = ""; try { const p = new URL(rawUrl); @@ -38,6 +40,11 @@ function parseCurl(raw: string): CurlResult { if (host.includes("anthropic.com") || apiKey.startsWith("sk-ant-") || "anthropic-version" in headers) { provider = "anthropic"; if (!host.includes("api.anthropic.com")) baseUrl = `${p.protocol}//${p.host}`; + } else if (host.includes("generativelanguage.googleapis.com")) { + provider = "gemini"; + const modelsIndex = p.pathname.indexOf("/models/"); + baseUrl = modelsIndex > -1 ? `${p.protocol}//${p.host}${p.pathname.slice(0, modelsIndex)}` : `${p.protocol}//${p.host}`; + if (baseUrl === "https://generativelanguage.googleapis.com/v1beta") baseUrl = ""; } else { provider = "openai"; const isAzure = host.endsWith(".openai.azure.com") || host.endsWith(".cognitiveservices.azure.com") || p.pathname.includes("/openai/deployments/"); @@ -67,26 +74,26 @@ export default function AppApiKeysPage() { const [anthropicUrl, setAnthropicUrl] = useState(""); const [openaiKey, setOpenaiKey] = useState(""); const [openaiUrl, setOpenaiUrl] = useState(""); - const [case1Threshold, setCase1Threshold] = useState(0.5); - const [case2Threshold, setCase2Threshold] = useState(0.5); - + const [geminiKey, setGeminiKey] = useState(""); + const [geminiUrl, setGeminiUrl] = useState(""); const [showAnthropic, setShowAnthropic] = useState(false); const [showOpenai, setShowOpenai] = useState(false); + const [showGemini, setShowGemini] = useState(false); const [anthropicTest, setAnthropicTest] = useState<{ ok: boolean; response: string } | null>(null); const [openaiTest, setOpenaiTest] = useState<{ ok: boolean; response: string } | null>(null); + const [geminiTest, setGeminiTest] = useState<{ ok: boolean; response: string } | null>(null); const [curlOpen, setCurlOpen] = useState(false); const [curlRaw, setCurlRaw] = useState(""); const [curlResult, setCurlResult] = useState(null); - const [thresholdSaved, setThresholdSaved] = useState(false); const [anthropicSaved, setAnthropicSaved] = useState(false); const [openaiSaved, setOpenaiSaved] = useState(false); + const [geminiSaved, setGeminiSaved] = useState(false); useEffect(() => { if (status) { setAnthropicUrl(status.anthropic_base_url || ""); setOpenaiUrl(status.openai_base_url || ""); - setCase1Threshold(status.case1_threshold ?? 0.5); - setCase2Threshold(status.case2_threshold ?? 0.5); + setGeminiUrl(status.gemini_base_url || ""); } }, [status]); @@ -110,9 +117,14 @@ export default function AppApiKeysPage() { onSuccess: () => { qc.invalidateQueries({ queryKey: ["app-api-keys", appId] }); setOpenaiKey(""); setOpenaiSaved(true); setTimeout(() => setOpenaiSaved(false), 2000); }, }); - const saveThresholds = useMutation({ - mutationFn: () => appApiKeysApi.set(appId!, { case1_threshold: case1Threshold, case2_threshold: case2Threshold }), - onSuccess: () => { qc.invalidateQueries({ queryKey: ["app-api-keys", appId] }); setThresholdSaved(true); setTimeout(() => setThresholdSaved(false), 2000); }, + const saveGemini = useMutation({ + mutationFn: () => { + const body: AppApiKeysUpdate = {}; + if (geminiKey.trim()) body.gemini_key = geminiKey.trim(); + if (geminiUrl !== (status?.gemini_base_url ?? "")) body.gemini_base_url = geminiUrl.trim(); + return appApiKeysApi.set(appId!, body); + }, + onSuccess: () => { qc.invalidateQueries({ queryKey: ["app-api-keys", appId] }); setGeminiKey(""); setGeminiSaved(true); setTimeout(() => setGeminiSaved(false), 2000); }, }); const testAnthropicMutation = useMutation({ @@ -127,18 +139,26 @@ export default function AppApiKeysPage() { onError: (e: { response?: { data?: { detail?: string } } }) => setOpenaiTest({ ok: false, response: e.response?.data?.detail ?? "Test failed" }), }); + const testGeminiMutation = useMutation({ + mutationFn: () => appApiKeysApi.testGemini(appId!, geminiKey.trim() || undefined, geminiUrl.trim() || undefined), + onSuccess: (d) => setGeminiTest(d), + onError: (e: { response?: { data?: { detail?: string } } }) => setGeminiTest({ ok: false, response: e.response?.data?.detail ?? "Test failed" }), + }); + function applyCurl(r: CurlResult) { if (r.error) return; if (r.provider === "anthropic") { setAnthropicKey(r.apiKey); if (r.baseUrl) setAnthropicUrl(r.baseUrl); setAnthropicTest(null); } + else if (r.provider === "gemini") { setGeminiKey(r.apiKey); if (r.baseUrl) setGeminiUrl(r.baseUrl); setGeminiTest(null); } else { setOpenaiKey(r.apiKey); if (r.baseUrl) setOpenaiUrl(r.baseUrl); setOpenaiTest(null); } setCurlOpen(false); setCurlRaw(""); setCurlResult(null); } const anthropicDirty = anthropicKey.trim() || anthropicUrl !== (status?.anthropic_base_url ?? ""); const openaiDirty = openaiKey.trim() || openaiUrl !== (status?.openai_base_url ?? ""); - const thresholdsDirty = case1Threshold !== (status?.case1_threshold ?? 0.5) || case2Threshold !== (status?.case2_threshold ?? 0.5); + const geminiDirty = geminiKey.trim() || geminiUrl !== (status?.gemini_base_url ?? ""); const canTestAnthropic = !!(anthropicKey.trim() || status?.anthropic_key_set); const canTestOpenai = !!(openaiKey.trim() || status?.openai_key_set); + const canTestGemini = !!(geminiKey.trim() || status?.gemini_key_set); if (isLoading) return
Loading...
; @@ -146,7 +166,7 @@ export default function AppApiKeysPage() {

API Keys

-

Set keys for Anthropic and OpenAI. You can test before saving.

+

Set keys for Anthropic, OpenAI, and Gemini. You can test before saving.

{/* cURL import */} @@ -179,7 +199,7 @@ export default function AppApiKeysPage() { ) : ( <>
- {curlResult.provider === "anthropic" ? "Anthropic" : "OpenAI / compatible"} + {curlResult.provider === "anthropic" ? "Anthropic" : curlResult.provider === "gemini" ? "Gemini" : "OpenAI / compatible"}

Key{curlResult.apiKey.slice(0, 10)}{"•".repeat(8)}

@@ -188,7 +208,7 @@ export default function AppApiKeysPage() { )} @@ -244,33 +264,30 @@ export default function AppApiKeysPage() { onTest={() => { setOpenaiTest(null); testOpenaiMutation.mutate(); }} testResult={openaiTest} /> + setShowGemini((v) => !v)} + onKeyChange={(v) => { setGeminiKey(v); setGeminiTest(null); }} + onUrlChange={(v) => { setGeminiUrl(v); setGeminiTest(null); }} + isDirty={!!geminiDirty} + isSaving={saveGemini.isPending} + saved={geminiSaved} + onSave={() => saveGemini.mutate()} + canTest={canTestGemini} + isTesting={testGeminiMutation.isPending} + onTest={() => { setGeminiTest(null); testGeminiMutation.mutate(); }} + testResult={geminiTest} + />
- {/* Evaluation thresholds */} -
-
-
- - Evaluation Thresholds -
- -
-

Semantic similarity pass cutoff when no judge LLM is configured. Range 0–1, default 0.50.

-
- - -
-
); } @@ -364,29 +381,3 @@ function ProviderRow({ ); } -// ── Threshold input ─────────────────────────────────────────────────────────── - -function ThresholdInput({ label, value, onChange }: { label: string; value: number; onChange: (v: number) => void }) { - const clamp = (v: number) => Math.max(0, Math.min(1, v)); - return ( -
-
- - onChange(clamp(parseFloat(e.target.value) || 0))} - className="w-16 text-right font-mono text-xs px-2 py-1 border border-gray-200 rounded focus:outline-none focus:ring-2 focus:ring-violet-500" - /> -
- onChange(clamp(parseFloat(e.target.value)))} className="w-full accent-violet-600" /> -
- {[0.4, 0.5, 0.6, 0.7, 0.8].map((p) => ( - - ))} -
-
- ); -} diff --git a/Evaluation/AISearchEvalutionTool/frontend/src/pages/EvaluatePage.tsx b/Evaluation/AISearchEvalutionTool/frontend/src/pages/EvaluatePage.tsx index c80a42fc..dd746f78 100644 --- a/Evaluation/AISearchEvalutionTool/frontend/src/pages/EvaluatePage.tsx +++ b/Evaluation/AISearchEvalutionTool/frontend/src/pages/EvaluatePage.tsx @@ -1,9 +1,10 @@ import { useState } from "react"; import { useParams } from "react-router-dom"; import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query"; -import { goldenSetsApi, evaluationApi, appsApi } from "@/lib/api"; -import type { Job, AnswerMode, FilterMode } from "@/lib/api"; -import { FlaskConical, Loader2, CheckCircle, XCircle, ChevronRight, Filter, UserCircle, Zap, FileText, Square, Info, ChevronDown, ChevronUp } from "lucide-react"; +import { goldenSetsApi, evaluationApi, appsApi, appApiKeysApi } from "@/lib/api"; +import type { Job, AnswerMode, FilterMode, JudgeMode } from "@/lib/api"; + +import { FlaskConical, Loader2, CheckCircle, XCircle, ChevronRight, Filter, UserCircle, Zap, FileText, Square, Info, ChevronDown, ChevronUp, Scale, Sparkles, SlidersHorizontal } from "lucide-react"; import { cn } from "@/lib/utils"; export default function EvaluatePage() { @@ -15,6 +16,7 @@ export default function EvaluatePage() { const [sampleMode, setSampleMode] = useState<"first" | "random">("first"); // Advanced filters const [filterMode, setFilterMode] = useState("none"); + const [filterFields, setFilterFields] = useState([]); const [enableRacl, setEnableRacl] = useState(false); const [userEmail, setUserEmail] = useState(""); // Answer mode: null = inherit from app config, otherwise an explicit override for this run @@ -22,6 +24,12 @@ export default function EvaluatePage() { // Question type filter: empty = all types const [selectedQTypes, setSelectedQTypes] = useState([]); const [showCasesPanel, setShowCasesPanel] = useState(false); + // Verdict configuration overrides (per-run) + const [showVerdictPanel, setShowVerdictPanel] = useState(false); + const [judgeMode, setJudgeMode] = useState("auto"); + const [case1Threshold, setCase1Threshold] = useState(null); + const [case2Threshold, setCase2Threshold] = useState(null); + const [topKPass, setTopKPass] = useState(null); const [activeJobId, setActiveJobId] = useState(null); const queryClient = useQueryClient(); @@ -48,6 +56,34 @@ export default function EvaluatePage() { enabled: !!appId, }); + // Available filter fields for the selected golden set (only fetched when needed) + const { data: filterOptions } = useQuery({ + queryKey: ["filter-options", appId, selectedVersion], + queryFn: () => goldenSetsApi.filterOptions(appId!, selectedVersion), + enabled: !!appId && !!selectedVersion && filterMode === "field_filters", + staleTime: 30_000, + }); + const availableFilterFields = filterOptions?.fields ?? []; + + // App-level thresholds (used as the placeholder/default for overrides) + const { data: apiKeyStatus } = useQuery({ + queryKey: ["app-api-keys", appId], + queryFn: () => appApiKeysApi.get(appId!), + enabled: !!appId, + }); + const appCase1 = apiKeyStatus?.case1_threshold ?? 0.5; + const appCase2 = apiKeyStatus?.case2_threshold ?? 0.5; + const judgeKeySet = + apiKeyStatus?.anthropic_key_set || apiKeyStatus?.openai_key_set || apiKeyStatus?.gemini_key_set; + const effectiveCase1 = case1Threshold ?? appCase1; + const effectiveCase2 = case2Threshold ?? appCase2; + const effectiveTopK = topKPass ?? 5; + const verdictOverrideCount = + (judgeMode !== "auto" ? 1 : 0) + + (case1Threshold !== null ? 1 : 0) + + (case2Threshold !== null ? 1 : 0) + + (topKPass !== null ? 1 : 0); + const { data: jobs = [] } = useQuery({ queryKey: ["eval-jobs", appId], queryFn: () => evaluationApi.listJobs(appId!), @@ -85,10 +121,15 @@ export default function EvaluatePage() { sample_mode: sampleMode, filter_mode: filterMode, filter_prompt: null, + filter_fields: filterMode === "field_filters" && filterFields.length > 0 ? filterFields : null, enable_racl: enableRacl, user_email: enableRacl && userEmail.trim() ? userEmail.trim() : null, answer_mode_override: answerModeOverride, question_types: selectedQTypes.length > 0 ? selectedQTypes : null, + judge_mode: judgeMode, + case1_threshold: case1Threshold, + case2_threshold: case2Threshold, + top_k_pass: topKPass, }), onSuccess: (job: Job) => setActiveJobId(job.job_id), }); @@ -499,6 +540,168 @@ export default function EvaluatePage() { + {/* ── Verdict Configuration ───────────────────────── */} +
+ + + {showVerdictPanel && ( +
+ {/* Judge mode */} +
+
+
+ +

Judge mode

+
+ {judgeMode !== "auto" && ( + + )} +
+

+ Controls whether the LLM judge scores each answer. + {!judgeKeySet && ( + + No LLM API key found for any provider — judge can't run, falls back to semantic similarity. + + )} +

+
+ {([ + { id: "auto", label: "Auto", desc: "Judge if API key is configured" }, + { id: "force_on", label: "Always on", desc: "Require LLM judge — fails if no key" }, + { id: "force_off", label: "Never", desc: "Skip judge, semantic only" }, + ] as const).map((m) => ( + + ))} +
+
+ + {/* Thresholds — only meaningful when judge isn't running */} +
+
+ +

Semantic-similarity thresholds

+ + (used when judge is off or unavailable) + +
+ + setCase1Threshold(v)} + onReset={() => setCase1Threshold(null)} + /> + setCase2Threshold(v)} + onReset={() => setCase2Threshold(null)} + /> +
+ + {/* Top-K — retrieval verdict threshold (extract_only mode) */} +
+
+
+ +

Retrieval top-K pass threshold

+
+ {topKPass !== null && ( + + )} +
+

+ For extract_only answer mode (Cases 3 & 4): pass if the expected document's chunk is in the top {effectiveTopK} retrieved chunks. +

+
+ setTopKPass(Math.max(1, Math.min(50, Number(e.target.value))))} + className="w-20 px-3 py-1.5 text-sm border border-gray-200 rounded-lg focus:outline-none focus:ring-2 focus:ring-violet-500" + /> + setTopKPass(Number(e.target.value))} + className="flex-1 accent-violet-600" + /> +
+
+ + {verdictOverrideCount > 0 && ( + + )} +
+ )} +
+ {/* ── Advanced filters / RACL ──────────────────────── */}
@@ -510,9 +713,21 @@ export default function EvaluatePage() {

Meta-filter strategy

{([ - { id: "none" as FilterMode, label: "No filters", desc: "Send queries to RAG without any metaFilter" }, - { id: "auto_source" as FilterMode, label: "Auto from source", desc: "Auto-attach sys_content_type filter based on each test case's source connector" }, - { id: "custom_prompt" as FilterMode, label: "Custom prompt", desc: "Use the Filter Generator LLM to produce metaFilters per question (configure prompt & mapper in Prompts & Models)" }, + { + id: "none" as FilterMode, + label: "No filters", + desc: "Only the question is sent — no metaFilter is attached to the RAG query", + }, + { + id: "field_filters" as FilterMode, + label: "Filters", + desc: "Apply filters from the golden set columns — choose which fields to include below", + }, + { + id: "custom_prompt" as FilterMode, + label: "Custom prompt", + desc: "Use the Filter Generator LLM to produce metaFilters per question (configure prompt & mapper in Prompts & Models)", + }, ]).map((m) => (
+ onChange(Number(e.target.value))} + className="w-full accent-violet-600" + /> +

{helpText}

+
+ ); +} + function StatusDot({ status }: { status: string }) { return (
a.localeCompare(b)); + export default function GeneratePage() { const { appId } = useParams<{ appId: string }>(); + const qc = useQueryClient(); const [selectedConnectorIds, setSelectedConnectorIds] = useState([]); const [selectedWebIds, setSelectedWebIds] = useState([]); @@ -18,8 +38,13 @@ export default function GeneratePage() { const [maxDocs, setMaxDocs] = useState(10); const [allDocs, setAllDocs] = useState(false); const [maxQuestionsPerDoc, setMaxQuestionsPerDoc] = useState(5); + const [targetLanguage, setTargetLanguage] = useState("English"); + const [languageQuery, setLanguageQuery] = useState(""); + const [languageOpen, setLanguageOpen] = useState(false); const [goldenSetVersion, setGoldenSetVersion] = useState("v1.0.0"); const [activeJobId, setActiveJobId] = useState(null); + const [freezeSavedVersion, setFreezeSavedVersion] = useState(null); + const [freezeError, setFreezeError] = useState(null); const { data: sources = [] } = useQuery({ queryKey: ["sources", appId], @@ -44,6 +69,13 @@ export default function GeneratePage() { const totalSelected = selectedConnectorIds.length + selectedWebIds.length + selectedFileIds.length; const totalAvailable = sources.length + webCrawls.length + uploadedDocs.length; + const normalizedLanguageQuery = languageQuery.trim(); + const filteredLanguages = LANGUAGE_OPTIONS.filter((lang) => + lang.toLowerCase().includes(normalizedLanguageQuery.toLowerCase()) + ); + const canUseCustomLanguage = + normalizedLanguageQuery.length > 0 && + !LANGUAGE_OPTIONS.some((lang) => lang.toLowerCase() === normalizedLanguageQuery.toLowerCase()); const { data: llmConfigs = [] } = useQuery({ queryKey: ["llm-configs", appId], @@ -63,6 +95,12 @@ export default function GeneratePage() { enabled: !!appId, }); + const { data: goldenSets = [] } = useQuery({ + queryKey: ["golden-sets", appId], + queryFn: () => goldenSetsApi.list(appId!), + enabled: !!appId, + }); + const { data: activeJob } = useQuery({ queryKey: ["gen-job", appId, activeJobId], queryFn: () => generationApi.getJob(appId!, activeJobId!), @@ -82,6 +120,7 @@ export default function GeneratePage() { file_source_ids: selectedFileIds, max_docs_per_source: allDocs ? 0 : maxDocs, max_questions_per_doc: maxQuestionsPerDoc, + target_language: targetLanguage, filters: {}, }), onSuccess: (job: Job) => setActiveJobId(job.job_id), @@ -89,6 +128,17 @@ export default function GeneratePage() { const freezeMutation = useMutation({ mutationFn: (version: string) => generationApi.freeze(appId!, version), + onMutate: () => { + setFreezeError(null); + setFreezeSavedVersion(null); + }, + onSuccess: (_data, version) => { + setFreezeSavedVersion(version); + qc.invalidateQueries({ queryKey: ["golden-sets", appId] }); + }, + onError: (err: { response?: { data?: { detail?: string } }; message?: string }) => { + setFreezeError(err.response?.data?.detail ?? err.message ?? "Failed to save golden set."); + }, }); const stopMutation = useMutation({ @@ -101,6 +151,8 @@ export default function GeneratePage() { const isRunning = activeJob?.status === "running"; const isDone = activeJob?.status === "complete"; + const currentGoldenSet = goldenSets.find((g) => g.version === goldenSetVersion); + const isFrozen = Boolean(currentGoldenSet?.frozen_at || freezeSavedVersion === goldenSetVersion); return (
@@ -287,6 +339,80 @@ export default function GeneratePage() { />

Semver string for this golden set batch

+ +
+ +
+ + {languageOpen && ( +
+
+ + setLanguageQuery(e.target.value)} + placeholder="Search languages..." + className="w-full pl-8 pr-3 py-1.5 text-sm border border-gray-200 rounded-md focus:outline-none focus:ring-2 focus:ring-violet-500" + /> +
+
+ {canUseCustomLanguage && ( + + )} + {filteredLanguages.length === 0 && !canUseCustomLanguage ? ( +
No languages found
+ ) : ( + filteredLanguages.map((lang) => ( + + )) + )} +
+
+ )} +
+

+ Questions, expected answers, and rationales will be generated in this language. +

+
@@ -314,6 +440,7 @@ export default function GeneratePage() { + @@ -356,18 +483,39 @@ export default function GeneratePage() { {/* Save option (previously "Freeze") */} {isDone && activeJob && !(activeJob.result?.stopped_early as boolean) && ( -
-

Ready to save for evaluation?

-

- Saving locks this golden set so it can be used for evaluation runs. You can still view it in the Draft state on the Golden Sets page. +

+

+ {isFrozen ? "Saved for evaluation" : "Ready to save for evaluation?"} +

+

+ {isFrozen + ? `${goldenSetVersion} is locked and can now be used for evaluation runs.` + : "Saving locks this golden set so it can be used for evaluation runs. You can still view it in the Draft state on the Golden Sets page."}

+ {freezeError && ( +

{freezeError}

+ )}
)} diff --git a/Evaluation/AISearchEvalutionTool/frontend/src/pages/GoldenSetsPage.tsx b/Evaluation/AISearchEvalutionTool/frontend/src/pages/GoldenSetsPage.tsx index 50ac1ebd..84ee4492 100644 --- a/Evaluation/AISearchEvalutionTool/frontend/src/pages/GoldenSetsPage.tsx +++ b/Evaluation/AISearchEvalutionTool/frontend/src/pages/GoldenSetsPage.tsx @@ -5,7 +5,7 @@ import { goldenSetsApi, generationApi } from "@/lib/api"; import type { GoldenSet, TestCase } from "@/lib/api"; import { BookOpen, Lock, ChevronDown, ChevronRight, CheckCircle, Clock, - Upload, X, FileText, Download, Loader2, AlertCircle, Trash2, + Upload, X, FileText, Download, Loader2, AlertCircle, Trash2, ChevronUp, } from "lucide-react"; import { cn } from "@/lib/utils"; import ConfirmDialog from "@/components/ConfirmDialog"; @@ -177,6 +177,14 @@ export default function GoldenSetsPage() { Save )} + + +