From bef60ddebef94638c08290cb86498d5bf7dab64c Mon Sep 17 00:00:00 2001 From: niranjank-Kore Date: Mon, 11 Aug 2025 18:58:15 +0530 Subject: [PATCH 1/2] feat: Add RAG evaluation improvements and optimizations - Add async API calls for better performance - Add context relevancy evaluation functionality - Add RAG system analysis capabilities - Add LLM evaluator for comprehensive assessment - Update prompts for better evaluation accuracy - Add timing and performance monitoring - Improve error handling and logging - Add batch processing utilities --- Evaluation/RAG_Evaluator/src/LLMEvaluator.py | 1389 +++++++++++++++++ Evaluation/RAG_Evaluator/src/api/XOSearch.py | 2 +- .../RAG_Evaluator/src/async_api_calls.py | 323 ++++ .../RAG_Evaluator/src/check_relevance.py | 314 ++++ .../RAG_Evaluator/src/config/config.json | 5 +- Evaluation/RAG_Evaluator/src/main.py | 703 ++++++++- .../RAG_Evaluator/src/prompts/prompts.json | 7 +- Evaluation/RAG_Evaluator/src/rag_analysis.py | 436 ++++++ .../RAG_Evaluator/src/utils/__init__.py | 34 + 9 files changed, 3185 insertions(+), 28 deletions(-) create mode 100644 Evaluation/RAG_Evaluator/src/LLMEvaluator.py create mode 100644 Evaluation/RAG_Evaluator/src/async_api_calls.py create mode 100644 Evaluation/RAG_Evaluator/src/check_relevance.py create mode 100644 Evaluation/RAG_Evaluator/src/rag_analysis.py diff --git a/Evaluation/RAG_Evaluator/src/LLMEvaluator.py b/Evaluation/RAG_Evaluator/src/LLMEvaluator.py new file mode 100644 index 00000000..f2a8d33b --- /dev/null +++ b/Evaluation/RAG_Evaluator/src/LLMEvaluator.py @@ -0,0 +1,1389 @@ +#!/usr/bin/env python3 +""" +Comprehensive RAG Evaluation Script + +This script evaluates RAG systems across three key dimensions: +1. Answer Correctness - How accurate the generated answer is compared to ground truth +2. Answer Relevance - How relevant the answer is to the user query +3. Context Relevance - How relevant the retrieved context is to the user query + +Usage: + python comprehensive_evaluator.py --input data.xlsx --sheet Sheet1 + python comprehensive_evaluator.py --input data.xlsx --sheet Sheet1 --model azure + python comprehensive_evaluator.py --input data.xlsx --sheet Sheet1 --output results.xlsx +""" + +import os +import sys +import argparse +import pandas as pd +import asyncio +import time +from datetime import datetime +from openai import OpenAI +from tqdm import tqdm +import json +from typing import List, Dict, Tuple, Optional +from openai import AzureOpenAI + +# Add the current directory to the path so we can import our modules +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +from config.configManager import ConfigManager +from evaluators.baseEvaluator import BaseEvaluator +from async_api_calls import call_search_api_async, AsyncAnswerProcessor + + +class ComprehensiveEvaluator(BaseEvaluator): + """ + Comprehensive evaluator for RAG systems that evaluates: + - Answer Correctness + - Answer Relevance + - Context Relevance + """ + + def __init__(self, model_name: str, openai_client, model_type: str = "openai"): + self.model_name = model_name + self.openai_client = openai_client + self.model_type = model_type + self.config = ConfigManager().get_config() + # Create semaphore for concurrency control (max 5 concurrent requests) + self.semaphore = asyncio.Semaphore(5) + + def evaluate_answer_correctness(self, query: str, answer: str, ground_truth: str) -> Dict: + """ + Evaluate how correct the generated answer is compared to ground truth using RAGAS framework. + + Args: + query: User query + answer: Generated answer + ground_truth: Expected correct answer + + Returns: + Dictionary with correctness score and explanation + """ + prompt = f"""You are an evaluator assessing the quality of a generated answer from a language model using the RAGAS framework. You are provided with the following: + +Query: {query} + +Ground Truth: {ground_truth} + +Generated Answer: {answer} + +You need to evaluate two metrics: + +Answer Correctness: How factually and semantically correct is the Generated Answer, based on both the Query and the Ground Truth? + +Focus on factual alignment and whether the core meaning matches the Ground Truth. + +Score on a scale from 0 (completely incorrect) to 1 (fully correct). + +Answer Relevancy: How relevant is the Generated Answer to the Query? + +Consider how well the answer addresses the user's intent and question. + +Score on a scale from 0 (completely irrelevant) to 1 (fully relevant). + +Provide your evaluation in this exact format: +Correctness Score: [0-1] +Relevance Score: [0-1] +Explanation: [2-3 sentences explaining both scores]""" + + messages = [ + {"role": "system", "content": "You are an expert evaluator for question-answering systems using the RAGAS framework."}, + {"role": "user", "content": prompt} + ] + + response = self.attempt_api_call(messages) + if response: + correctness_score, answer_relevance_score, explanation = self._parse_ragas_response(response) + # Convert 0-1 scale to 0-5 scale for consistency + correctness_0_5 = int(correctness_score * 5) if correctness_score >= 0 else -1 + relevance_0_5 = int(answer_relevance_score * 5) if answer_relevance_score >= 0 else -1 + + return { + "correctness_score": correctness_0_5, + "answer_relevance_score": relevance_0_5, + "correctness_explanation": explanation, + "answer_relevance_explanation": explanation + } + else: + return { + "correctness_score": -1, + "answer_relevance_score": -1, + "correctness_explanation": "Failed to get evaluation response", + "answer_relevance_explanation": "Failed to get evaluation response" + } + + def evaluate_answer_relevance(self, query: str, answer: str) -> Dict: + """ + Evaluate how relevant the generated answer is to the user query using RAGAS framework. + This method now uses the same unified prompt as evaluate_answer_correctness. + + Args: + query: User query + answer: Generated answer + + Returns: + Dictionary with relevance score and explanation + """ + # Use the same unified evaluation as evaluate_answer_correctness + # but with empty ground truth for relevance-only evaluation + return self.evaluate_answer_correctness(query, answer, "") + + def evaluate_context_relevance(self, query: str, context: str) -> Dict: + """ + Evaluate how relevant the retrieved context is to the user query. + + Args: + query: User query + context: Retrieved context + + Returns: + Dictionary with context relevance score and explanation + """ + prompt = f"""You are an evaluator assessing the quality of retrieved context for a question-answering system using the RAGAS framework. You are provided with the following: + +Query: {query} + +Retrieved Context: {context} + +You need to evaluate: + +Context Relevance: How relevant is the Retrieved Context to the Query? + +Consider how well the context would help answer the user's question and whether it contains useful information. + +Score on a scale from 0 (completely irrelevant) to 1 (fully relevant). + +Provide your evaluation in this exact format: +Context Relevance Score: [0-1] +Explanation: [2-3 sentences explaining your score]""" + + messages = [ + {"role": "system", "content": "You are an expert evaluator for information retrieval systems using the RAGAS framework."}, + {"role": "user", "content": prompt} + ] + + response = self.attempt_api_call(messages) + if response: + score, explanation = self._parse_score_response_ragas(response, "Context Relevance Score") + # Convert 0-1 scale to 0-5 scale for consistency + score_0_5 = int(score * 5) if score >= 0 else -1 + return { + "context_relevance_score": score_0_5, + "context_relevance_explanation": explanation + } + else: + return { + "context_relevance_score": -1, + "context_relevance_explanation": "Failed to get evaluation response" + } + + async def evaluate_all_metrics(self, query: str, answer: str, ground_truth: str, context: str) -> Dict: + """ + Evaluate all three metrics (correctness, relevance, context relevance) in a single OpenAI call. + + Args: + query: User query + answer: Generated answer + ground_truth: Expected correct answer + context: Retrieved context + + Returns: + Dictionary with all three scores and explanations + """ + prompt = f"""You are an expert evaluator of question-answering systems using the RAGAS framework. Be strict in your evaluation ā€” only give 100% if the answer is fully complete, factually accurate, and the context is completely supportive of the answer. Minor omissions or partial correctness should reduce the score. + +You are provided with the following: + +Query: {query} + +Ground Truth: {ground_truth} + +Generated Answer: {answer} + +Retrieved Context: {context} + +--- + +Evaluate the following three metrics, each scored between 0% and 100%, based on the definitions below: + +šŸ“Œ Metric Definitions: + +1. **Answer Relevancy** +Measures how well the Generated Answer addresses the user's original query, regardless of factual accuracy. +- Check if the answer directly relates to the question. +- Focus on topical and intent alignment. +**Key Question:** Is this answer relevant to what the user asked? +Score Range: +0% = Completely irrelevant +100% = Fully aligned and directly relevant to the query + +2. **Answer Correctness** +Measures whether the Generated Answer is factually correct and semantically aligned with the Ground Truth. +- Check for factual accuracy. +- Ensure the meaning is preserved. +- Penalize if information is missing, wrong, or hallucinated. +**Key Question:** Is this answer factually and semantically correct compared to the ground truth? +Score Range: +0% = Entirely incorrect or misleading +100% = Fully accurate and matches the ground truth + +3. **Context Relevancy** +Measures how relevant and useful the Retrieved Context is for answering the Query, regardless of how the model used it. +- Check if the context contains enough information to answer the question. +**Key Question:** Would this context help someone answer the question correctly? +**Important Note:** Context Relevancy should be evaluated using only the Query and the Retrieved Context. Do not consider the Ground Truth or Generated Answer in this score. +Score Range: +0% = Totally unrelated or unhelpful +100% = Highly relevant to the query + +--- + +šŸ“ Output Format: +Return your evaluation in this exact format: + +Correctness Score: [0ā€“100]% +Correctness Explanation: [Explain why you gave that score.] + +Answer Relevance Score: [0ā€“100]% +Answer Relevance Explanation: [Explain why you gave that score.] + +Context Relevance Score: [0ā€“100]% +Context Relevance Explanation: [Explain why you gave that score.] + +Each explanation must be specific to the metric. Do not repeat the same explanation for multiple metrics.""" + + messages = [ + {"role": "system", "content": "You are an expert evaluator for question-answering systems using the RAGAS framework. Be strict in your evaluation ā€” only give 100% if the answer is fully complete, factually accurate, and the context is completely supportive of the answer."}, + {"role": "user", "content": prompt} + ] + response = await self.attempt_api_call_async(messages) + if response: + correctness_score, answer_relevance_score, context_score, explanations = self._parse_all_metrics_response(response) + + return { + "correctness_score": correctness_score, + "answer_relevance_score": answer_relevance_score, + "context_relevance_score": context_score, + "correctness_explanation": explanations['correctness'], + "answer_relevance_explanation": explanations['relevance'], + "context_relevance_explanation": explanations['context'] + } + else: + return { + "correctness_score": -1, + "answer_relevance_score": -1, + "context_relevance_score": -1, + "correctness_explanation": "Failed to get evaluation response", + "answer_relevance_explanation": "Failed to get evaluation response", + "context_relevance_explanation": "Failed to get evaluation response" + } + + def _parse_ragas_response(self, response: str) -> Tuple[float, float, str]: + """ + Parse RAGAS evaluation response to extract correctness score, relevance score, and explanation. + + Args: + response: Raw response from evaluation API + + Returns: + Tuple of (correctness_score, answer_relevance_score, explanation) + """ + try: + import re + + # Extract correctness score + correctness_pattern = r'Correctness Score:\s*([0-9]*\.?[0-9]+)' + correctness_match = re.search(correctness_pattern, response, re.IGNORECASE) + correctness_score = float(correctness_match.group(1)) if correctness_match else -1 + + # Extract relevance score + answer_relevance_pattern = r'Answer Relevance Score:\s*([0-9]*\.?[0-9]+)' + answer_relevance_match = re.search(answer_relevance_pattern, response, re.IGNORECASE) + answer_relevance_score = float(answer_relevance_match.group(1)) if answer_relevance_match else -1 + + # Extract explanation + explanation_pattern = r'Explanation:\s*(.+?)(?:\n|$)' + explanation_match = re.search(explanation_pattern, response, re.IGNORECASE | re.DOTALL) + explanation = explanation_match.group(1).strip() if explanation_match else "No explanation provided" + + # Validate score ranges + if correctness_score < 0 or correctness_score > 1: + correctness_score = -1 + if answer_relevance_score < 0 or answer_relevance_score > 1: + answer_relevance_score = -1 + + return correctness_score, answer_relevance_score, explanation + + except Exception as e: + print(f"Error parsing RAGAS response: {str(e)}") + return -1, -1, f"Error parsing response: {str(e)}" + + def _parse_score_response_ragas(self, response: str, score_type: str) -> Tuple[float, str]: + """ + Parse evaluation response to extract score and explanation for RAGAS framework (0-1 scale). + + Args: + response: Raw response from evaluation API + score_type: Type of score being parsed + + Returns: + Tuple of (score, explanation) + """ + try: + import re + + # Extract score using regex for 0-1 scale + score_pattern = f"{score_type}:\\s*([0-9]*\\.?[0-9]+)" + score_match = re.search(score_pattern, response, re.IGNORECASE) + + if score_match: + score = float(score_match.group(1)) + else: + # Try to find any number between 0-1 in the response + score_match = re.search(r'\b([0-9]*\.?[0-9]+)\b', response) + score = float(score_match.group(1)) if score_match else -1 + + # Extract explanation + explanation_pattern = r'Explanation:\s*(.+?)(?:\n|$)' + explanation_match = re.search(explanation_pattern, response, re.IGNORECASE | re.DOTALL) + explanation = explanation_match.group(1).strip() if explanation_match else "No explanation provided" + + # Validate score range (0-1 for RAGAS) + if score < 0 or score > 1: + score = -1 + explanation = "Invalid score extracted from response" + + return score, explanation + + except Exception as e: + print(f"Error parsing {score_type} response: {str(e)}") + return -1, f"Error parsing response: {str(e)}" + + def _parse_all_metrics_response(self, response: str) -> Tuple[float, float, float, Dict[str, str]]: + """ + Parse evaluation response to extract all three scores and their individual explanations. + + Args: + response: Raw response from evaluation API + + Returns: + Tuple of (correctness_score, answer_relevance_score, context_score, explanations_dict) + """ + try: + import re + + # Patterns to extract scores - match the exact format from the prompt + correctness_score = float(re.search(r'Correctness Score:\s*([0-9]+)%', response, re.IGNORECASE).group(1)) + answer_relevance_score = float(re.search(r'Answer Relevance Score:\s*([0-9]+)%', response, re.IGNORECASE).group(1)) + context_score = float(re.search(r'Context Relevance Score:\s*([0-9]+)%', response, re.IGNORECASE).group(1)) + + # More precise extraction of explanations using lookahead and lookbehind + # Find the start and end of each explanation section + correctness_pattern = r'Correctness Explanation:\s*(.*?)(?=\n\s*Answer Relevance Score:)' + relevance_pattern = r'Answer Relevance Explanation:\s*(.*?)(?=\n\s*Context Relevance Score:)' + context_pattern = r'Context Relevance Explanation:\s*(.*?)(?=\n\s*$|\Z)' + + # Extract explanations with fallback patterns + correctness_exp = re.search(correctness_pattern, response, re.DOTALL | re.IGNORECASE) + relevance_exp = re.search(relevance_pattern, response, re.DOTALL | re.IGNORECASE) + context_exp = re.search(context_pattern, response, re.DOTALL | re.IGNORECASE) + + # Fallback patterns if the above don't work + if not correctness_exp: + correctness_pattern_fallback = r'Correctness Explanation:\s*(.*?)(?=\n\s*[A-Z][a-z]+)' + correctness_exp = re.search(correctness_pattern_fallback, response, re.DOTALL | re.IGNORECASE) + + if not relevance_exp: + relevance_pattern_fallback = r'Answer Relevance Explanation:\s*(.*?)(?=\n\s*[A-Z][a-z]+)' + relevance_exp = re.search(relevance_pattern_fallback, response, re.DOTALL | re.IGNORECASE) + + if not context_exp: + context_pattern_fallback = r'Context Relevance Explanation:\s*(.*?)(?=\n\s*$|\Z)' + context_exp = re.search(context_pattern_fallback, response, re.DOTALL | re.IGNORECASE) + + explanations = { + 'correctness': correctness_exp.group(1).strip() if correctness_exp else "No correctness explanation provided", + 'relevance': relevance_exp.group(1).strip() if relevance_exp else "No relevance explanation provided", + 'context': context_exp.group(1).strip() if context_exp else "No context explanation provided" + } + + return correctness_score, answer_relevance_score, context_score, explanations + + except Exception as e: + print(f"āŒ Error parsing all metrics response: {str(e)}") + print(f"šŸ” Raw response for debugging:") + print(response) + return -1, -1, -1, { + 'correctness': 'Error parsing response - check format', + 'relevance': 'Error parsing response - check format', + 'context': 'Error parsing response - check format' + } + + async def evaluate_comprehensive(self, queries: List[str], answers: List[str], + ground_truths: List[str], contexts: List[str]) -> pd.DataFrame: + """ + Perform comprehensive evaluation across all three dimensions using a single OpenAI call per sample. + + Args: + queries: List of user queries + answers: List of generated answers + ground_truths: List of ground truth answers + contexts: List of retrieved contexts + + Returns: + DataFrame with comprehensive evaluation results + """ + results = [] + + print(f"šŸ”„ Starting comprehensive evaluation for {len(queries)} samples...") + print(f"šŸ“Š Using {self.model_name} OpenAI call per sample for all three metrics") + + # Create tasks for concurrent evaluation + tasks = [] + for i, (query, answer, ground_truth, context) in enumerate(zip(queries, answers, ground_truths, contexts)): + task = self.evaluate_single_sample(i, query, answer, ground_truth, context) + tasks.append(task) + + # Execute all tasks concurrently with progress bar + print(f"šŸš€ Running {len(tasks)} evaluations concurrently...") + + # Create progress bar + progress_bar = tqdm(total=len(tasks), desc="Evaluating", unit="sample") + + # Track completed tasks + completed_results = [] + completed_count = 0 + + # Process tasks as they complete + for coro in asyncio.as_completed(tasks): + try: + result = await coro + completed_results.append(result) + completed_count += 1 + progress_bar.update(1) + progress_bar.set_postfix({ + 'Completed': f"{completed_count}/{len(tasks)}", + 'Success': f"{len([r for r in completed_results if not isinstance(r, Exception)])}" + }) + except Exception as e: + print(f"āŒ Error in evaluation: {str(e)}") + completed_count += 1 + progress_bar.update(1) + + progress_bar.close() + + # Sort results by sample_id to maintain order + completed_results.sort(key=lambda x: x.get('sample_id', 0) if not isinstance(x, Exception) else 0) + + # Process results and handle any exceptions + processed_results = [] + for i, result in enumerate(completed_results): + if isinstance(result, Exception): + print(f"āŒ Error evaluating sample {i+1}: {str(result)}") + # Create error result + error_result = { + "query": queries[i] if i < len(queries) else "Error", + "answer": answers[i] if i < len(answers) else "Error", + "ground_truth": ground_truths[i] if i < len(ground_truths) else "Error", + "context": contexts[i] if i < len(contexts) else "Error", + "sample_id": i + 1, + "correctness_score": -1, + "answer_relevance_score": -1, + "context_relevance_score": -1, + "correctness_explanation": f"Evaluation failed: {str(result)}", + "answer_relevance_explanation": f"Evaluation failed: {str(result)}", + "context_relevance_explanation": f"Evaluation failed: {str(result)}" + } + processed_results.append(error_result) + else: + processed_results.append(result) + + successful_count = len([r for r in processed_results if r.get('correctness_score', -1) >= 0]) + print(f"āœ… Completed {successful_count}/{len(tasks)} evaluations successfully") + + df = pd.DataFrame(processed_results) + df = self._format_dataframe_columns(df) + return df + + async def evaluate_single_sample(self, index: int, query: str, answer: str, ground_truth: str, context: str) -> Dict: + """ + Evaluate a single sample asynchronously. + + Args: + index: Sample index + query: User query + answer: Generated answer + ground_truth: Expected correct answer + context: Retrieved context + + Returns: + Dictionary with evaluation results + """ + result = { + "query": query, + "answer": answer, + "ground_truth": ground_truth, + "context": context, + } + + # Evaluate all three metrics in a single OpenAI call + all_metrics_result = await self.evaluate_all_metrics(query, answer, ground_truth, context) + result.update(all_metrics_result) + + return result + + def _format_dataframe_columns(self, df: pd.DataFrame) -> pd.DataFrame: + """ + Ensure DataFrame columns are properly formatted and not truncated. + + Args: + df: Input DataFrame + + Returns: + Formatted DataFrame + """ + # Ensure all explanation columns have proper content + explanation_columns = ['correctness_explanation', 'answer_relevance_explanation', 'context_relevance_explanation'] + + for col in explanation_columns: + if col in df.columns: + # Fill empty or very short explanations + df[col] = df[col].fillna("No explanation provided") + df[col] = df[col].apply(lambda x: "No explanation provided" if len(str(x)) < 5 else x) + + + return df + + def evaluate(self, queries: List[str], answers: List[str], + ground_truths: List[str], contexts: List[str]) -> pd.DataFrame: + """ + Abstract method implementation - alias for evaluate_comprehensive. + + Args: + queries: List of user queries + answers: List of generated answers + ground_truths: List of ground truth answers + contexts: List of retrieved contexts + + Returns: + DataFrame with evaluation results + """ + return self.evaluate_comprehensive(queries, answers, ground_truths, contexts) + + def process_results(self, results: pd.DataFrame) -> Dict: + """ + Abstract method implementation - alias for generate_summary_report. + + Args: + results: DataFrame with evaluation results + + Returns: + Dictionary with summary statistics + """ + return self.generate_summary_report(results) + + def attempt_api_call(self, messages: List[Dict]) -> Optional[str]: + """ + Attempt to make an API call to the OpenAI model. + + Args: + messages: List of message dictionaries for the API call + + Returns: + Response string if successful, None if failed + """ + try: + response = self.openai_client.chat.completions.create( + model=self.model_name, + messages=messages, + temperature=0.0, + max_tokens=500 + ) + return response.choices[0].message.content.strip() + except Exception as e: + error_msg = str(e) + print(f"āŒ API call failed: {error_msg}") + + # Provide specific guidance for common Azure errors + if "404" in error_msg and "Resource not found" in error_msg: + print("šŸ” Azure 404 Error - Possible causes:") + print(" ā€¢ Model deployment name is incorrect") + print(" ā€¢ Model deployment doesn't exist in your Azure resource") + print(" ā€¢ API version is not supported") + print(f" ā€¢ Current model: {self.model_name}") + print(f" ā€¢ Current model type: {self.model_type}") + elif "401" in error_msg: + print("šŸ” Azure 401 Error - Authentication failed:") + print(" ā€¢ Check if AZURE_OPENAI_API_KEY is set correctly") + print(" ā€¢ Verify the API key has access to the Azure resource") + elif "403" in error_msg: + print("šŸ” Azure 403 Error - Access denied:") + print(" ā€¢ Check if your API key has permission to use this model") + print(" ā€¢ Verify the model deployment is active") + + return None + + async def attempt_api_call_async(self, messages: List[Dict]) -> Optional[str]: + """ + Attempt to make an async API call to the OpenAI model. + + Args: + messages: List of message dictionaries for the API call + + Returns: + Response string if successful, None if failed + """ + async with self.semaphore: # Limit concurrent requests + try: + response = await asyncio.get_event_loop().run_in_executor( + None, + lambda: self.openai_client.chat.completions.create( + model=self.model_name, + messages=messages, + temperature=0.0, + max_tokens=500 + ) + ) + return response.choices[0].message.content.strip() + except Exception as e: + error_msg = str(e) + print(f"āŒ Async API call failed: {error_msg}") + + # Provide specific guidance for common Azure errors + if "404" in error_msg and "Resource not found" in error_msg: + print("šŸ” Azure 404 Error - Possible causes:") + print(" ā€¢ Model deployment name is incorrect") + print(" ā€¢ Model deployment doesn't exist in your Azure resource") + print(" ā€¢ API version is not supported") + print(f" ā€¢ Current model: {self.model_name}") + print(f" ā€¢ Current model type: {self.model_type}") + elif "401" in error_msg: + print("šŸ” Azure 401 Error - Authentication failed:") + print(" ā€¢ Check if AZURE_OPENAI_API_KEY is set correctly") + print(" ā€¢ Verify the API key has access to the Azure resource") + elif "403" in error_msg: + print("šŸ” Azure 403 Error - Access denied:") + print(" ā€¢ Check if your API key has permission to use this model") + print(" ā€¢ Verify the model deployment is active") + elif "rate limit" in error_msg.lower(): + print("šŸ” Rate limit error - Consider reducing concurrency") + + return None + + def generate_summary_report(self, results_df: pd.DataFrame) -> Dict: + """ + Generate a comprehensive summary report of evaluation results. + + Args: + results_df: DataFrame with evaluation results + + Returns: + Dictionary with summary statistics + """ + if results_df.empty: + return {"error": "No results to analyze"} + + # Filter valid results (scores >= 0) + valid_correctness = results_df[results_df['correctness_score'] >= 0] + valid_relevance = results_df[results_df['answer_relevance_score'] >= 0] + valid_context = results_df[results_df['context_relevance_score'] >= 0] + + summary = { + "total_samples": len(results_df), + "evaluation_timestamp": datetime.now().isoformat(), + "model_used": self.model_name, + "model_type": self.model_type, + + # Answer Correctness Statistics (0-100% scale) + "answer_correctness": { + "total_evaluations": len(results_df), + "valid_evaluations": len(valid_correctness), + "average_score": valid_correctness['correctness_score'].mean() if not valid_correctness.empty else 0, + "median_score": valid_correctness['correctness_score'].median() if not valid_correctness.empty else 0, + "min_score": valid_correctness['correctness_score'].min() if not valid_correctness.empty else 0, + "max_score": valid_correctness['correctness_score'].max() if not valid_correctness.empty else 0, + "std_score": valid_correctness['correctness_score'].std() if not valid_correctness.empty else 0, + "scale": "0-100%" + }, + + # Answer Relevance Statistics (0-100% scale) + "answer_relevance": { + "total_evaluations": len(results_df), + "valid_evaluations": len(valid_relevance), + "average_score": valid_relevance['answer_relevance_score'].mean() if not valid_relevance.empty else 0, + "median_score": valid_relevance['answer_relevance_score'].median() if not valid_relevance.empty else 0, + "min_score": valid_relevance['answer_relevance_score'].min() if not valid_relevance.empty else 0, + "max_score": valid_relevance['answer_relevance_score'].max() if not valid_relevance.empty else 0, + "std_score": valid_relevance['answer_relevance_score'].std() if not valid_relevance.empty else 0, + "scale": "0-100%" + }, + + # Context Relevance Statistics (0-100% scale) + "context_relevance": { + "total_evaluations": len(results_df), + "valid_evaluations": len(valid_context), + "average_score": valid_context['context_relevance_score'].mean() if not valid_context.empty else 0, + "median_score": valid_context['context_relevance_score'].median() if not valid_context.empty else 0, + "min_score": valid_context['context_relevance_score'].min() if not valid_context.empty else 0, + "max_score": valid_context['context_relevance_score'].max() if not valid_context.empty else 0, + "std_score": valid_context['context_relevance_score'].std() if not valid_context.empty else 0, + "scale": "0-100%" + } + } + + return summary + + +def setup_openai_client(model_type: str = "openai"): + """ + Setup OpenAI client based on model type. + + Args: + model_type: "openai" or "azure" + + Returns: + OpenAI client instance + """ + config_manager = ConfigManager() + config = config_manager.get_config() + + if model_type == "azure": + azure_conf = config["azure"] + return AzureOpenAI( + api_key=os.getenv("AZURE_OPENAI_API_KEY"), + api_version=azure_conf["openai_api_version"], + azure_endpoint=azure_conf["base_url"] + ) + else: + return OpenAI(api_key=os.getenv("OPENAI_API_KEY")) + + +def get_model_name(model_type: str = "openai"): + """ + Get model name from config. + + Args: + model_type: "openai" or "azure" + + Returns: + Model name string + """ + config_manager = ConfigManager() + config = config_manager.get_config() + + if model_type == "azure": + # For Azure, this should be the deployment name from Azure Portal + return config["azure"]["model_deployment"] + else: + return config["openai"]["model_name"] + + +async def fetch_missing_data_with_search_api(queries: List[str], answers: List[str], + ground_truths: List[str], contexts: List[str]) -> Tuple[List[str], List[str], List[str], List[str]]: + """ + Fetch missing answers or contexts using Search API. + + Args: + queries: List of user queries + answers: List of generated answers + ground_truths: List of ground truth answers + contexts: List of retrieved contexts + + Returns: + Updated lists with missing data filled + """ + config_manager = ConfigManager() + config = config_manager.get_config() + max_concurrent_requests = config.get("max_concurrent_requests_for_search_api", 2) + # Determine API type based on config + api_type = 'UXO' # default + if config.get('SA'): + api_type = 'SA' + elif config.get('UXO'): + api_type = 'UXO' + + print(f"šŸ” Checking for missing data and fetching from {api_type} Search API...") + + # Find indices where we need to fetch data + missing_indices = [] + for i, (query, answer, context) in enumerate(zip(queries, answers, contexts)): + # Check if answer is missing or empty + if not answer or answer.strip() == "" or answer == "Failed to get response": + missing_indices.append(i) + # Check if context is missing or empty + elif not context or (isinstance(context, list) and len(context) == 0) or context == "": + missing_indices.append(i) + + if not missing_indices: + print("āœ… No missing data found, proceeding with existing data") + return queries, answers, ground_truths, contexts + + print(f"šŸ“Š Found {len(missing_indices)} samples with missing data, fetching from Search API...") + + # Prepare queries for missing data + missing_queries = [queries[i] for i in missing_indices] + missing_ground_truths = [ground_truths[i] for i in missing_indices] + + # Generate persistent filename for this operation + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + persistent_filename = f"outputs/search_api_results_{api_type}_{timestamp}.xlsx" + + try: + # Call Search API for missing data with persistent file saving + api_results = await call_search_api_async(missing_queries, missing_ground_truths, api_type, + save_batches=True, persistent_filename=persistent_filename) + + # Update the original lists with fetched data + for i, api_result in zip(missing_indices, api_results): + if api_result and 'answer' in api_result and 'context' in api_result: + # Update answer + if not answers[i] or answers[i].strip() == "" or answers[i] == "Failed to get response": + answers[i] = api_result['answer'] + + # Update context + if not contexts[i] or (isinstance(contexts[i], list) and len(contexts[i]) == 0) or contexts[i] == "": + contexts[i] = api_result['context'] + + print(f"āœ… Fetched data for query {i+1}: {queries[i][:50]}...") + else: + print(f"āš ļø Failed to fetch data for query {i+1}") + + print(f"āœ… Successfully fetched data for {len([r for r in api_results if r])} samples") + print(f"šŸ’¾ All batch data saved to persistent file: {persistent_filename}") + + except Exception as e: + print(f"āŒ Error fetching data from Search API: {str(e)}") + print("āš ļø Proceeding with existing data") + + return queries, answers, ground_truths, contexts + + +async def fetch_all_data_from_search_api(queries: List[str], ground_truths: List[str]) -> Tuple[List[str], List[str], List[str], List[str]]: + """ + Fetch all answers and contexts from Search API. + + Args: + queries: List of user queries + ground_truths: List of ground truth answers + + Returns: + Lists of queries, answers, ground_truths, and contexts + """ + config_manager = ConfigManager() + config = config_manager.get_config() + max_concurrent_requests = config.get("max_concurrent_requests_for_search_api", 2) + # Determine API type based on config + api_type = 'UXO' # default + if config.get('SA'): + api_type = 'SA' + elif config.get('UXO'): + api_type = 'UXO' + + print(f"šŸ” Fetching all answers and contexts from {api_type} Search API...") + print(f"šŸ“Š Fetching data for {len(queries)} queries...") + + # Generate persistent filename for this operation + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + persistent_filename = f"outputs/sa_api_outputs/search_api_results_{api_type}_{timestamp}.xlsx" + + try: + # Call Search API for all data with persistent file saving + api_results = await call_search_api_async(queries, ground_truths, api_type, + save_batches=True, persistent_filename=persistent_filename, max_concurrent=max_concurrent_requests) + + # Extract answers and contexts from API results + answers = [] + contexts = [] + + print(f"šŸ“Š Processing {len(api_results)} API results...") + api_progress = tqdm(api_results, desc="Processing API results", unit="result") + + for i, api_result in enumerate(api_progress): + if api_result and 'answer' in api_result and 'context' in api_result: + answers.append(api_result['answer']) + contexts.append(api_result['context']) + api_progress.set_postfix({'Success': f"{len(answers)}/{i+1}"}) + else: + # Fallback if API call failed + answers.append("Failed to get response") + contexts.append([]) + api_progress.set_postfix({'Failed': f"{i+1-len(answers)}/{i+1}"}) + + api_progress.close() + + print(f"āœ… Successfully fetched data for {len([r for r in api_results if r])} samples") + print(f"šŸ’¾ All batch data saved to persistent file: {persistent_filename}") + + return queries, answers, ground_truths, contexts + + except Exception as e: + print(f"āŒ Error fetching data from Search API: {str(e)}") + print("āš ļø Proceeding with empty data") + # Return empty data if API fails + answers = ["Failed to get response"] * len(queries) + contexts = [[]] * len(queries) + return queries, answers, ground_truths, contexts + + +def load_data_from_excel(file_path: str, sheet_name: str = None, use_search_api: bool = False) -> Tuple[List[str], List[str], List[str], List[str]]: + """ + Load data from Excel file. + + Args: + file_path: Path to Excel file + sheet_name: Sheet name to load (optional) + use_search_api: Whether to fetch answers/contexts from Search API + + Returns: + Tuple of (queries, answers, ground_truths, contexts) + """ + try: + if sheet_name: + df = pd.read_excel(file_path, sheet_name=sheet_name, engine='openpyxl') + else: + df = pd.read_excel(file_path, engine='openpyxl') + + # Handle different column name variations + query_col = None + for col in ['query', 'user_input', 'question']: + if col in df.columns: + query_col = col + break + + answer_col = None + for col in ['answer', 'response', 'prediction']: + if col in df.columns: + answer_col = col + break + + ground_truth_col = None + for col in ['ground_truth', 'reference', 'expected_answer']: + if col in df.columns: + ground_truth_col = col + break + + context_col = None + for col in ['context', 'contexts', 'retrieved_contexts']: + if col in df.columns: + context_col = col + break + + # Check required columns based on use_search_api flag + missing_cols = [] + if not query_col: + missing_cols.append("query/user_input/question") + if not ground_truth_col: + missing_cols.append("ground_truth/reference/expected_answer") + + # Only require answer/context columns if not using Search API + if not use_search_api: + if not answer_col: + missing_cols.append("answer/response/prediction") + if not context_col: + missing_cols.append("context/contexts/retrieved_contexts") + + if missing_cols: + raise ValueError(f"Missing required columns: {', '.join(missing_cols)}") + + queries = df[query_col].fillna('').tolist() + ground_truths = df[ground_truth_col].fillna('').tolist() + + # Handle answers and contexts based on use_search_api flag + if use_search_api: + # When using Search API, we don't need existing answers/contexts + # They will be fetched from the API + answers = [""] * len(queries) # Empty answers, will be filled by API + contexts = [[]] * len(queries) # Empty contexts, will be filled by API + else: + # Use existing answers and contexts from the file + answers = df[answer_col].fillna('').tolist() + contexts = df[context_col].fillna('').tolist() + + # Convert contexts to string if they're lists + processed_contexts = [] + for context in contexts: + if isinstance(context, list): + processed_contexts.append('\n'.join(context)) + else: + processed_contexts.append(str(context)) + + return queries, answers, ground_truths, processed_contexts + + except Exception as e: + print(f"āŒ Error loading data from Excel file: {str(e)}") + raise + + +def print_summary_report(summary: Dict): + """ + Print a formatted summary report. + + Args: + summary: Summary dictionary from evaluation + """ + print("\n" + "="*80) + print("šŸ“Š COMPREHENSIVE EVALUATION SUMMARY") + print("="*80) + + print(f"šŸ“‹ Total Samples: {summary['total_samples']}") + print(f"šŸ¤– Model Used: {summary['model_used']} ({summary['model_type'].upper()})") + print(f"ā° Evaluation Time: {summary['evaluation_timestamp']}") + + print("\n" + "-"*80) + print("āœ… ANSWER CORRECTNESS (0-100% Scale)") + print("-"*80) + correctness = summary['answer_correctness'] + print(f" Valid Evaluations: {correctness['valid_evaluations']}/{correctness['total_evaluations']}") + print(f" Average Score: {correctness['average_score']:.1f}%") + print(f" Median Score: {correctness['median_score']:.1f}%") + print(f" Score Range: {correctness['min_score']:.1f}% - {correctness['max_score']:.1f}%") + print(f" Standard Deviation: {correctness['std_score']:.1f}%") + + print("\n" + "-"*80) + print("šŸ” ANSWER RELEVANCE (0-100% Scale)") + print("-"*80) + relevance = summary['answer_relevance'] + print(f" Valid Evaluations: {relevance['valid_evaluations']}/{relevance['total_evaluations']}") + print(f" Average Score: {relevance['average_score']:.1f}%") + print(f" Median Score: {relevance['median_score']:.1f}%") + print(f" Score Range: {relevance['min_score']:.1f}% - {relevance['max_score']:.1f}%") + print(f" Standard Deviation: {relevance['std_score']:.1f}%") + + print("\n" + "-"*80) + print("šŸ“š CONTEXT RELEVANCE (0-100% Scale)") + print("-"*80) + context = summary['context_relevance'] + print(f" Valid Evaluations: {context['valid_evaluations']}/{context['total_evaluations']}") + print(f" Average Score: {context['average_score']:.1f}%") + print(f" Median Score: {context['median_score']:.1f}%") + print(f" Score Range: {context['min_score']:.1f}% - {context['max_score']:.1f}%") + print(f" Standard Deviation: {context['std_score']:.1f}%") + + print("\n" + "="*80) + + +def ensure_outputs_directory(): + """Ensure the outputs directory exists.""" + if not os.path.exists("outputs"): + os.makedirs("outputs", exist_ok=True) + print("šŸ“ Created outputs directory") + + +async def main(): + """Main function to handle command line arguments and run evaluation.""" + parser = argparse.ArgumentParser( + description='Comprehensive RAG Evaluation Script - Multi-Sheet Support with Search API Integration', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Evaluate all sheets using data from input file (with missing data auto-fetch) + python comprehensive_evaluator.py --input data.xlsx + + # Evaluate all sheets using Search API for all answers and contexts + python comprehensive_evaluator.py --input data.xlsx --use_search_api + + # Evaluate specific sheet using Search API with high concurrency + python comprehensive_evaluator.py --input data.xlsx --sheet Sheet1 --use_search_api --concurrency 10 + + # Evaluate first 5 samples from all sheets using input file data + python comprehensive_evaluator.py --input data.xlsx --sample 5 + + # Evaluate specific sheet with custom output using Search API + python comprehensive_evaluator.py --input data.xlsx --sheet Sheet1 --output results.xlsx --use_search_api + """ + ) + + parser.add_argument('--input', type=str, required=True, + help='Path to input Excel file') + parser.add_argument('--sheet', type=str, default=None, + help='Sheet name to evaluate (defaults to all sheets)') + parser.add_argument('--output', type=str, default=None, + help='Output file path (default: comprehensive_evaluation_.xlsx)') + parser.add_argument('--sample', type=int, default=None, + help='Number of samples to evaluate (default: all)') + parser.add_argument('--use_search_api', action='store_true', + help='Fetch answers and contexts from Search API instead of using input file data') + parser.add_argument('--concurrency', type=int, default=5, + help='Number of concurrent API calls (default: 5, overrides config setting)') + + args = parser.parse_args() + + try: + # Ensure outputs directory exists + ensure_outputs_directory() + + # Auto-detect model type from config + config_manager = ConfigManager() + config = config_manager.get_config() + model_type = config["model_type"] + # Check which model configuration is available + if not model_type: + if "azure" in config and config["azure"].get("model_name") and config["azure"]["model_name"] != "": + model_type = "azure" + print("šŸ¤– Using Azure OpenAI model from config") + else: + model_type = "openai" + print("šŸ¤– Using OpenAI model from config") + + print("šŸš€ Starting Comprehensive RAG Evaluation") + print("="*60) + print(f"šŸ“ Input File: {args.input}") + print(f"šŸ¤– Model Type: {model_type.upper()}") + if args.sample: + print(f"šŸ“Š Sample Size: {args.sample}") + if args.use_search_api: + print(f"šŸ” Using Search API to fetch answers and contexts") + else: + print(f"šŸ“„ Using answers and contexts from input file") + print() + + # Setup evaluator + print("šŸ”„ Setting up evaluator...") + openai_client = setup_openai_client(model_type) + model_name = get_model_name(model_type) + evaluator = ComprehensiveEvaluator(model_name, openai_client, model_type) + + # Set concurrency level from config or command line + config_concurrency = config.get("max_concurrent_requests_for_evaluation", 5) + concurrency_level = args.concurrency if args.concurrency != 5 else config_concurrency + evaluator.semaphore = asyncio.Semaphore(concurrency_level) + print(f"āš” Concurrency level set to {concurrency_level} concurrent API calls (from config: {config_concurrency})") + + # Show search API concurrency setting + search_api_concurrency = config.get("max_concurrent_requests_for_search_api", 2) + print(f"šŸ” Search API concurrency: {search_api_concurrency} concurrent requests") + + if args.concurrency != 5: + print(f"šŸ”§ Command line concurrency ({args.concurrency}) overrides config setting ({config_concurrency})") + else: + print(f"šŸ”§ Using concurrency setting from config file: {config_concurrency}") + + # Get all sheet names from the Excel file + try: + excel_file = pd.ExcelFile(args.input, engine='openpyxl') + sheet_names = excel_file.sheet_names + print(f"šŸ“‹ Found {len(sheet_names)} sheets: {', '.join(sheet_names)}") + except Exception as e: + print(f"āŒ Error reading Excel file: {str(e)}") + return + + # Filter sheets if specific sheet is requested + if args.sheet: + if args.sheet in sheet_names: + sheet_names = [args.sheet] + print(f"šŸ“‹ Processing specific sheet: {args.sheet}") + else: + print(f"āŒ Sheet '{args.sheet}' not found. Available sheets: {', '.join(sheet_names)}") + return + else: + print(f"šŸ“‹ Processing all {len(sheet_names)} sheets") + + # Initialize results storage + all_results = [] + all_summaries = [] + + # Generate output filename + if not args.output: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + args.output = f"outputs/comprehensive_evaluation_{timestamp}.xlsx" + + print(f"šŸ’¾ Results will be saved to: {args.output}") + + # Create output directory if it doesn't exist + output_dir = os.path.dirname(args.output) + if output_dir and not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + + # Process each sheet + print(f"šŸ“‹ Processing {len(sheet_names)} sheets...") + sheet_progress = tqdm(sheet_names, desc="Processing sheets", unit="sheet") + + for i, current_sheet_name in enumerate(sheet_progress): + sheet_progress.set_description(f"Processing {current_sheet_name}") + print(f"\nšŸ“‹ Processing sheet: {current_sheet_name}") + print("-" * 60) + + try: + # Load data from current sheet + print(f"šŸ”„ Loading data from sheet '{current_sheet_name}'...") + queries, answers, ground_truths, contexts = load_data_from_excel(args.input, current_sheet_name,args.use_search_api) + + if args.sample: + queries = queries[:args.sample] + ground_truths = ground_truths[:args.sample] + + # Only slice answers and contexts if not using Search API + if not args.use_search_api: + answers = answers[:args.sample] + contexts = contexts[:args.sample] + print(f"āœ… Loaded {len(queries)} samples from sheet '{current_sheet_name}'") + + if len(queries) == 0: + print(f"āš ļø No data found in sheet '{current_sheet_name}', skipping...") + continue + + # Handle data fetching based on use_search_api flag + if args.use_search_api: + print(f"šŸ” Fetching data from Search API for {len(queries)} queries...") + queries, answers, ground_truths, contexts = await fetch_all_data_from_search_api(queries, ground_truths) + print(f"āœ… Data fetching completed") + else: + # Check for missing data and fetch from Search API if needed + print(f"šŸ” Checking for missing answers or contexts...") + queries, answers, ground_truths, contexts = await fetch_missing_data_with_search_api( + queries, answers, ground_truths, contexts + ) + + # Run evaluation + print(f"šŸ”„ Running comprehensive evaluation for sheet '{current_sheet_name}'...") + start_time = time.time() + + results_df = await evaluator.evaluate_comprehensive(queries, answers, ground_truths, contexts) + + evaluation_time = time.time() - start_time + + # Generate summary + print(f"šŸ”„ Generating summary report for sheet '{current_sheet_name}'...") + summary = evaluator.generate_summary_report(results_df) + + # Add sheet information to summary + summary['sheet_name'] = current_sheet_name + summary['evaluation_time'] = evaluation_time + + # Print summary for this sheet + print(f"\nšŸ“Š RESULTS FOR SHEET: {current_sheet_name}") + print("=" * 60) + print_summary_report(summary) + + # Store results + all_results.append((current_sheet_name, results_df)) + all_summaries.append(summary) + + # Save this sheet's results immediately + print(f"šŸ’¾ Saving results for sheet '{current_sheet_name}'...") + try: + # Ensure the output directory exists + output_dir = os.path.dirname(args.output) + if output_dir and not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + + with pd.ExcelWriter(args.output, engine='openpyxl', mode='a' if i > 0 else 'w') as writer: + # Save detailed results for this sheet + results_df.to_excel(writer, sheet_name=f'{current_sheet_name}', index=False) + + # Save summary for this sheet + + + print(f"āœ… Sheet '{current_sheet_name}' results saved successfully") + except Exception as save_error: + print(f"āš ļø Warning: Could not save sheet '{current_sheet_name}' results: {str(save_error)}") + # Try to save to a different file as fallback + try: + fallback_file = args.output.replace('.xlsx', f'_{current_sheet_name}_only.xlsx') + results_df.to_excel(fallback_file, sheet_name=current_sheet_name, index=False) + print(f"āœ… Sheet saved to fallback file: {fallback_file}") + except Exception as fallback_error: + print(f"āŒ Could not save to fallback file either: {str(fallback_error)}") + + print(f"āœ… Sheet '{current_sheet_name}' completed in {evaluation_time:.2f} seconds") + + except Exception as e: + print(f"āŒ Error processing sheet '{current_sheet_name}': {str(e)}") + import traceback + traceback.print_exc() + continue + + # Generate overall summary + if all_summaries: + print(f"\nšŸŽ‰ OVERALL EVALUATION SUMMARY") + print("=" * 80) + print(f"šŸ“‹ Total Sheets Processed: {len(all_summaries)}") + print(f"šŸ¤– Model Used: {model_name} ({model_type.upper()})") + + # Calculate overall statistics + total_samples = sum(s['total_samples'] for s in all_summaries) + total_evaluation_time = sum(s['evaluation_time'] for s in all_summaries) + successful_samples = sum(s['answer_correctness']['valid_evaluations'] for s in all_summaries) + + print(f"šŸ“Š Total Samples Evaluated: {total_samples}") + print(f"āœ… Successful Evaluations: {successful_samples}") + print(f"ā° Total Evaluation Time: {total_evaluation_time:.2f} seconds") + print(f"šŸš€ Average Time per Sample: {total_evaluation_time/total_samples:.2f} seconds") + + # Print per-sheet summary + print(f"\nšŸ“‹ PER-SHEET SUMMARY:") + print("-" * 80) + for summary in all_summaries: + sheet_name = summary['sheet_name'] + correctness_avg = summary['answer_correctness']['average_score'] + relevance_avg = summary['answer_relevance']['average_score'] + context_avg = summary['context_relevance']['average_score'] + samples = summary['total_samples'] + eval_time = summary['evaluation_time'] + + print(f"šŸ“„ {sheet_name}:") + print(f" Samples: {samples}") + print(f" Answer Correctness: {correctness_avg:.1f}%") + print(f" Answer Relevance: {relevance_avg:.1f}%") + print(f" Context Relevance: {context_avg:.1f}%") + print(f" Time: {eval_time:.2f}s") + print() + + # Save overall summary to the same file + print(f"šŸ’¾ Saving overall summary...") + try: + # Check if the file exists, if not create it in write mode + file_exists = os.path.exists(args.output) + mode = 'a' if file_exists else 'w' + + with pd.ExcelWriter(args.output, engine='openpyxl', mode=mode) as writer: + # Save overall summary + summary_df = pd.DataFrame(all_summaries) + summary_df.to_excel(writer, sheet_name='Overall_Summary', index=False) + + # Auto-adjust column widths for overall summary + worksheet = writer.sheets['Overall_Summary'] + for column in worksheet.columns: + max_length = 0 + column_letter = column[0].column_letter + for cell in column: + try: + if len(str(cell.value)) > max_length: + max_length = len(str(cell.value)) + except: + pass + adjusted_width = min(max_length + 2, 50) # Cap at 50 characters + worksheet.column_dimensions[column_letter].width = adjusted_width + + print(f"āœ… Overall summary saved successfully") + print(f"šŸŽ‰ All results saved to: {args.output}") + print(f"šŸŽ‰ Evaluation completed successfully!") + + except Exception as save_error: + print(f"āš ļø Warning: Could not save overall summary: {str(save_error)}") + # Try to save to a different file as fallback + try: + fallback_file = args.output.replace('.xlsx', '_summary_only.xlsx') + summary_df = pd.DataFrame(all_summaries) + summary_df.to_excel(fallback_file, sheet_name='Overall_Summary', index=False) + print(f"āœ… Overall summary saved to fallback file: {fallback_file}") + except Exception as fallback_error: + print(f"āŒ Could not save to fallback file either: {str(fallback_error)}") + print(f"āœ… Individual sheet results were saved successfully") + + else: + print("āŒ No sheets were processed successfully") + print(f"šŸ“ No files were saved") + + except Exception as e: + print(f"āŒ Error during evaluation: {str(e)}") + import traceback + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/Evaluation/RAG_Evaluator/src/api/XOSearch.py b/Evaluation/RAG_Evaluator/src/api/XOSearch.py index c6d9f643..6b3c8667 100644 --- a/Evaluation/RAG_Evaluator/src/api/XOSearch.py +++ b/Evaluation/RAG_Evaluator/src/api/XOSearch.py @@ -77,7 +77,7 @@ def get_bot_response(api: XOSearchAPI, query: str, truth: str) -> Optional[Dict] 'query': query, 'ground_truth': truth, 'context': context_data, - 'context_url': context_url, + 'context_url': "", 'answer': bot_answer } diff --git a/Evaluation/RAG_Evaluator/src/async_api_calls.py b/Evaluation/RAG_Evaluator/src/async_api_calls.py new file mode 100644 index 00000000..8194bca1 --- /dev/null +++ b/Evaluation/RAG_Evaluator/src/async_api_calls.py @@ -0,0 +1,323 @@ +import aiohttp +import asyncio +import pandas as pd +import os +from datetime import datetime +from typing import Dict, List, Tuple, Optional +from config.configManager import ConfigManager +from utils.jti import JTI + +def generate_JWT_token(client_id, client_secret): + jwt_token = JTI.get_hs_key(client_id, client_secret, "JWT", "HS256") + return jwt_token + +def save_batch_to_persistent_file(batch_results: List[Dict], batch_number: int, api_type: str, + output_dir: str = "outputs/sa_api_outputs", filename: str = None): + """ + Save batch results to a single persistent file that accumulates all batches. + + Args: + batch_results: List of results from the current batch + batch_number: Current batch number + api_type: Type of API used (SA or UXO) + output_dir: Directory to save the persistent file (default: outputs) + filename: Optional custom filename (if None, auto-generates) + """ + try: + # Create output directory if it doesn't exist + if not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + + # Generate filename if not provided + if filename is None: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"{output_dir}/all_batches_{api_type}_{timestamp}.xlsx" + + # Convert batch results to DataFrame format + df_data = [] + for result in batch_results: + # Convert context list to string if it's a list + context_str = result.get('context', []) + if isinstance(context_str, list): + context_str = '\n'.join(context_str) + + df_data.append({ + 'query': result.get('query', ''), + 'ground_truth': result.get('ground_truth', ''), + 'answer': result.get('answer', ''), + 'context': context_str, + 'context_url': result.get('context_url', ''), + 'batch_number': batch_number, + 'api_type': api_type, + 'timestamp': datetime.now().strftime("%Y-%m-%d %H:%M:%S") + }) + + # Create DataFrame for current batch + current_batch_df = pd.DataFrame(df_data) + + # Check if file already exists + if os.path.exists(filename): + try: + # Load existing data + existing_df = pd.read_excel(filename, engine='openpyxl') + + # Append new batch data + combined_df = pd.concat([existing_df, current_batch_df], ignore_index=True) + + # Save combined data + combined_df.to_excel(filename, index=False) + + print(f"āœ… Batch {batch_number} appended to existing file: {filename}") + print(f" šŸ“Š Total samples in file: {len(combined_df)}") + print(f" šŸ“Š New samples added: {len(current_batch_df)}") + + except Exception as e: + print(f"āš ļø Error reading existing file, creating new file: {str(e)}") + current_batch_df.to_excel(filename, index=False) + print(f"āœ… Batch {batch_number} saved to new file: {filename}") + print(f" šŸ“Š Samples in file: {len(current_batch_df)}") + else: + # Create new file with first batch + current_batch_df.to_excel(filename, index=False) + print(f"āœ… Batch {batch_number} saved to new file: {filename}") + print(f" šŸ“Š Samples in file: {len(current_batch_df)}") + + return filename + + except Exception as e: + print(f"āŒ Error saving batch {batch_number} to persistent file: {str(e)}") + return None + +class AsyncXOSearchAPI: + def __init__(self): + config = ConfigManager().get_config() + self.client_id = config.get('UXO').get('client_id') + self.client_secret = config.get('UXO').get('client_secret') + self.auth_token = generate_JWT_token(self.client_id, self.client_secret) + self.app_id = config.get('UXO').get('app_id') + self.domain = config.get('UXO').get('domain') + self.base_url = f'https://{self.domain}/api/public/bot/{self.app_id}' + + async def _make_request(self, session: aiohttp.ClientSession, endpoint: str, data: Dict) -> Optional[Dict]: + headers = { + 'auth': self.auth_token, + 'Content-Type': 'application/json' + } + try: + print("url is ", f"{self.base_url}/{endpoint}") + async with session.post(f"{self.base_url}/{endpoint}", json=data, headers=headers) as response: + response.raise_for_status() + return await response.json() + except aiohttp.ClientError as e: + print(f"Request failed: {e}") + return None + + async def advanced_search(self, session: aiohttp.ClientSession, query: str) -> Optional[Dict]: + data = { + "query": query, + "includeChunksInResponse": True + } + print("Making async SA search call for query:", query) + return await self._make_request(session, 'advancedSearch', data) + + +class AsyncSearchAssistAPI: + def __init__(self): + config = ConfigManager().get_config() + self.client_id = config.get('SA').get('client_id') + self.client_secret = config.get('SA').get('client_secret') + self.auth_token = generate_JWT_token(self.client_id, self.client_secret) + self.app_id = config.get('SA').get('app_id') + self.domain = config.get('SA').get('domain') + self.base_url = f'https://{self.domain}/searchassistapi/external/stream/{self.app_id}' + + async def _make_request(self, session: aiohttp.ClientSession, endpoint: str, data: Dict) -> Optional[Dict]: + headers = { + 'auth': self.auth_token, + 'Content-Type': 'application/json' + } + try: + async with session.post(f"{self.base_url}/{endpoint}", json=data, headers=headers) as response: + response.raise_for_status() + return await response.json() + except aiohttp.ClientError as e: + print(f"Request failed: {e}") + return None + + async def advanced_search(self, session: aiohttp.ClientSession, query: str) -> Optional[Dict]: + data = { + "query": query, + "includeChunksInResponse": True + } + print("Making async SA search call for query:", query) + return await self._make_request(session, 'advancedSearch', data) + + +class AsyncAnswerProcessor: + @staticmethod + def get_context(answer: Dict) -> Tuple[List[str], str]: + contexts = [] + context_urls = set() + + # Handle XO Search format + if 'chunk_result' in answer: + for chunk in answer.get('chunk_result', {}).get('generative', []): + source = chunk.get('_source', {}) + if source.get('sentToLLM'): + contexts.append(source.get('chunkText', '')) + context_urls.add(source.get('recordUrl', '')) + + # Handle SearchAssist format + elif 'template' in answer: + for chunk in answer.get('template', {}).get('chunk_result', {}).get('generative', []): + source = chunk.get('_source', {}) + if source.get('sentToLLM'): + contexts.append(source.get('chunkText', '')) + context_urls.add(source.get('recordUrl', '')) + + return contexts, ",".join(context_urls) + + @staticmethod + def extract_answer(answer: Dict) -> str: + # Handle XO Search format + if 'response' in answer: + center_panel = (answer.get('response', {}) + .get('answer_payload', {}) + .get('center_panel', {})) + # Handle SearchAssist format + elif 'template' in answer: + center_panel = (answer.get('template', {}) + .get('graph_answer', {}) + .get('payload', {}) + .get('center_panel', {})) + else: + return "No Answer Found" + + if not center_panel: + return "No Answer Found" + snippet_content = center_panel.get('data', [{}])[0].get('snippet_content', [{}]) + answer_string = " ".join(content.get('answer_fragment', "No Answer Found") for content in snippet_content) if snippet_content else "No Answer Found" + return answer_string + + +async def get_async_bot_response(api_type: str, session: aiohttp.ClientSession, query: str, truth: str) -> Optional[Dict]: + """ + Get bot response asynchronously + + Args: + api_type: 'SA' for SearchAssist or 'UXO' for XO Search + session: aiohttp session + query: search query + truth: ground truth + """ + config_manager = ConfigManager() + config = config_manager.get_config() + + if api_type == 'SA' and config.get('SA'): + api = AsyncSearchAssistAPI() + elif api_type == 'UXO' and config.get('UXO'): + api = AsyncXOSearchAPI() + else: + print(f"API type {api_type} not configured") + return None + + answer = await api.advanced_search(session, query) + if not answer: + return None + + context_data, context_url = AsyncAnswerProcessor.get_context(answer) + bot_answer = AsyncAnswerProcessor.extract_answer(answer) + + return { + 'query': query, + 'ground_truth': truth, + 'context': context_data, + 'context_url': context_url, + 'answer': bot_answer + } + + +async def call_search_api_async(queries: List[str], ground_truths: List[str], api_type: str = 'UXO', + max_concurrent: int = 3, save_batches: bool = False, + persistent_filename: str = None) -> List[Dict]: + """ + Call search API asynchronously for multiple queries with configurable concurrency limit + + Args: + queries: List of search queries + ground_truths: List of ground truths + api_type: 'SA' for SearchAssist or 'UXO' for XO Search + max_concurrent: Maximum number of concurrent API calls (default: 3) + save_batches: Whether to save data after each batch (default: True) + persistent_filename: Optional custom filename for persistent saving + """ + results = [] + batch_number = 1 + + async with aiohttp.ClientSession() as session: + # Process queries in batches to limit concurrency + for i in range(0, len(queries), max_concurrent): + batch_queries = queries[i:i + max_concurrent] + batch_ground_truths = ground_truths[i:i + max_concurrent] + + print(f"šŸ”„ Processing batch {batch_number}: {len(batch_queries)} queries (max {max_concurrent} concurrent)") + + # Create tasks for current batch + tasks = [] + for query, truth in zip(batch_queries, batch_ground_truths): + task = get_async_bot_response(api_type, session, query, truth) + tasks.append(task) + + # Execute current batch concurrently + batch_responses = await asyncio.gather(*tasks, return_exceptions=True) + + # Process batch results + batch_results = [] + for j, response in enumerate(batch_responses): + query_index = i + j + if isinstance(response, Exception): + print(f"āŒ Error processing query {query_index}: {response}") + batch_result = { + 'query': queries[query_index], + 'ground_truth': ground_truths[query_index], + 'context': [], + 'context_url': '', + 'answer': "Failed to get response" + } + elif response: + batch_result = response + print(f"āœ… Successfully processed query {query_index}: {queries[query_index][:50]}...") + else: + batch_result = { + 'query': queries[query_index], + 'ground_truth': ground_truths[query_index], + 'context': [], + 'context_url': '', + 'answer': "Failed to get response" + } + + batch_results.append(batch_result) + results.append(batch_result) + + # Save batch data to persistent file after processing + if save_batches: + save_batch_to_persistent_file(batch_results, batch_number, api_type, + filename=persistent_filename) + + print(f"āœ… Batch {batch_number} completed: {len(batch_results)} queries processed") + batch_number += 1 + + return results + + +# Example usage +if __name__ == "__main__": + async def main(): + queries = ["what is eva?", "how does it work?"] + ground_truths = ["Example ground truth 1", "Example ground truth 2"] + + results = await call_search_api_async(queries, ground_truths, 'UXO') + for result in results: + print(result) + + asyncio.run(main()) \ No newline at end of file diff --git a/Evaluation/RAG_Evaluator/src/check_relevance.py b/Evaluation/RAG_Evaluator/src/check_relevance.py new file mode 100644 index 00000000..48816d67 --- /dev/null +++ b/Evaluation/RAG_Evaluator/src/check_relevance.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +""" +Context Relevance Checker + +This script reads query and retrieved_contexts from Ragas output files +and evaluates the relevance of each context for its corresponding query. + +Usage: + python check_relevance.py + python check_relevance.py --input "outputs/NVR_Evaluation_evaluation_output_*.xlsx" + python check_relevance.py --sample 5 # Check only first 5 samples + python check_relevance.py --output results.csv # Save results to CSV +""" + +import os +import sys +import argparse +import pandas as pd +import glob +from openai import OpenAI +from loguru import logger +from tqdm import tqdm +from datetime import datetime + +# Add the current directory to the path so we can import our modules +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +from evaluators.relevanceEvaluator import RelevanceEvaluator +from config.configManager import ConfigManager + + +def find_latest_ragas_output(): + """ + Find the latest Ragas output file in the outputs directory. + + Returns: + Path to the latest output file + """ + outputs_dir = "outputs" + pattern = os.path.join(outputs_dir, "*evaluation_output*.xlsx") + + files = glob.glob(pattern) + if not files: + print(f"āŒ No Ragas output files found in {outputs_dir}") + return None + + # Sort by modification time to get the latest + latest_file = max(files, key=os.path.getmtime) + print(f"šŸ“ Found latest Ragas output: {latest_file}") + return latest_file + + +def extract_data_from_ragas_output(file_path): + """ + Extract user_input and retrieved_contexts from the Ragas output file. + + Args: + file_path: Path to the Ragas output Excel file + + Returns: + DataFrame with user_input and retrieved_contexts columns + """ + try: + print(f"Reading Ragas output file: {file_path}") + df = pd.read_excel(file_path) + + print(f"File shape: {df.shape}") + print(f"Available columns: {list(df.columns)}") + + # Check if required columns exist + required_columns = ['user_input', 'retrieved_contexts'] + missing_columns = [col for col in required_columns if col not in df.columns] + + if missing_columns: + print(f"āŒ Missing columns: {missing_columns}") + print(f"Available columns: {list(df.columns)}") + return None + + # Extract the required columns + extracted_data = df[['user_input', 'retrieved_contexts']].copy() + + # Remove any rows with missing data + extracted_data = extracted_data.dropna() + + print(f"āœ… Successfully extracted {len(extracted_data)} rows") + print(f"Columns: {list(extracted_data.columns)}") + + return extracted_data + + except FileNotFoundError: + print(f"āŒ File not found: {file_path}") + return None + except Exception as e: + print(f"āŒ Error reading file: {str(e)}") + return None + + +def setup_openai_client(model_type="openai"): + """Setup OpenAI client with API key from environment variable.""" + # Try to get API key from environment variable first + api_key = os.getenv('OPENAI_API_KEY') + + if not api_key: + print("āŒ OpenAI API key not found in environment variable OPENAI_API_KEY") + print("Please set your API key using: export OPENAI_API_KEY='your_api_key'") + return None + + return OpenAI(api_key=api_key) + + +def get_model_from_config(model_type="openai"): + """Get the model name from config.json.""" + try: + config = ConfigManager().get_config() + + if model_type == "azure": + model_name = config.get('azure', {}).get('model_name', 'gpt-4') + print(f"šŸ¤– Using Azure model from config: {model_name}") + else: + model_name = config.get('openai', {}).get('model_name', 'gpt-3.5-turbo') + print(f"šŸ¤– Using OpenAI model from config: {model_name}") + + return model_name + except Exception as e: + print(f"āš ļø Error reading config, using default model: {str(e)}") + return 'gpt-3.5-turbo' if model_type != "azure" else 'gpt-4' + + +def evaluate_relevance_batch(data_df, sample_size=None, output_file='relevance_results.csv', model_type="openai"): + """ + Evaluate relevance for all query-context pairs in the dataset. + + Args: + data_df: DataFrame with user_input and retrieved_contexts columns + sample_size: Number of samples to evaluate (None for all) + output_file: File to save results (optional) + model_type: "openai" or "azure" to specify which model to use + + Returns: + DataFrame with evaluation results + """ + # Setup OpenAI client + openai_client = setup_openai_client(model_type) + if not openai_client: + return None + + # Get model from config + model_name = get_model_from_config(model_type) + evaluator = RelevanceEvaluator(model_name, openai_client, model_type) + + # Limit to sample size if specified + if sample_size: + data_df = data_df.head(sample_size) + print(f"Evaluating first {sample_size} samples...") + + results = [] + + print(f"Evaluating relevance for {len(data_df)} query-context pairs using {model_type.upper()}...") + + for idx, (_, row) in enumerate(tqdm(data_df.iterrows(), total=len(data_df), desc="Evaluating")): + query = row['user_input'] + context = row['retrieved_contexts'] + + try: + # Evaluate relevance + result = evaluator.evaluate_single_pair(query, context) + + # Add row index for reference + result['row_index'] = idx + 1 + + results.append(result) + + except Exception as e: + print(f"Error evaluating row {idx + 1}: {str(e)}") + results.append({ + 'row_index': idx + 1, + 'query': query, + 'context_chunk': context, + 'relevance_score': -1, + 'explanation': f"Error: {str(e)}" + }) + + # Convert to DataFrame + results_df = pd.DataFrame(results) + + # Add summary statistics + valid_results = results_df[results_df['relevance_score'] >= 0] + if not valid_results.empty: + summary = { + 'total_evaluations': len(results_df), + 'valid_evaluations': len(valid_results), + 'average_relevance_score': valid_results['relevance_score'].mean(), + 'median_relevance_score': valid_results['relevance_score'].median(), + 'min_relevance_score': valid_results['relevance_score'].min(), + 'max_relevance_score': valid_results['relevance_score'].max(), + 'std_relevance_score': valid_results['relevance_score'].std(), + 'score_distribution': valid_results['relevance_score'].value_counts().sort_index().to_dict() + } + + print(f"\nšŸ“Š EVALUATION SUMMARY") + print("=" * 50) + for key, value in summary.items(): + if key == 'score_distribution': + print(f"{key}:") + for score, count in value.items(): + print(f" Score {score}: {count} evaluations") + else: + print(f"{key}: {value}") + + # Save results if output file specified + if not output_file: + output_file = f'relevance_results_from_ragas_{datetime.now().strftime("%Y%m%d_%H%M%S")}.csv' + results_df.to_csv(output_file, index=False) + print(f"\nšŸ’¾ Results saved to: {output_file}") + + return results_df + + +def display_sample_results(results_df, num_samples=3): + """Display sample results.""" + print(f"\nšŸ“‹ SAMPLE RESULTS") + print("=" * 50) + + for i, (_, row) in enumerate(results_df.head(num_samples).iterrows()): + print(f"\n--- Sample {i+1} ---") + print(f"Row: {row['row_index']}") + print(f"Query: {row['query']}") + print(f"Context: {row['context_chunk'][:150]}...") + print(f"Relevance Score: {row['relevance_score']}/5") + print(f"Explanation: {row['explanation']}") + + # Provide interpretation + score = row['relevance_score'] + if score == 5: + print("āœ… Highly relevant") + elif score == 4: + print("āœ… Mostly relevant") + elif score == 3: + print("āš ļø Related but limited") + elif score == 2: + print("āš ļø Moderately related") + elif score == 1: + print("āŒ Slightly related") + elif score == 0: + print("āŒ Completely irrelevant") + else: + print("ā“ Unable to evaluate") + + +def main(): + """Main function to handle command line arguments and run evaluation.""" + parser = argparse.ArgumentParser( + description="Check relevance of retrieved contexts from Ragas output files", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python check_relevance.py + python check_relevance.py --input "outputs/NVR_Evaluation_evaluation_output_*.xlsx" + python check_relevance.py --sample 10 + python check_relevance.py --output relevance_results.csv + """ + ) + + parser.add_argument("--input", "-i", help="Path to Ragas output file (default: latest)") + parser.add_argument("--sample", "-s", type=int, help="Number of samples to evaluate (default: all)") + parser.add_argument("--output", "-o", help="Output CSV file to save results") + parser.add_argument("--verbose", "-v", action="store_true", help="Show detailed output") + + args = parser.parse_args() + + print("Context Relevance Checker (Ragas Output)") + print("=" * 60) + print("Reading from Ragas output file and evaluating relevance...") + print() + + try: + # Determine input file + if args.input: + input_file = args.input + else: + input_file = find_latest_ragas_output() + if not input_file: + return + + # Extract data from Ragas output file + data_df = extract_data_from_ragas_output(input_file) + + if data_df is None: + print("āŒ Failed to extract data from Ragas output file") + return + + # Evaluate relevance + results_df = evaluate_relevance_batch( + data_df, + sample_size=args.sample, + output_file=args.output + ) + + if results_df is None: + print("āŒ Failed to evaluate relevance") + return + + # Display sample results + display_sample_results(results_df, num_samples=3) + + print(f"\nāœ… Evaluation completed successfully!") + + except Exception as e: + logger.error(f"Error during evaluation: {str(e)}") + print(f"āŒ Error: {str(e)}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/Evaluation/RAG_Evaluator/src/config/config.json b/Evaluation/RAG_Evaluator/src/config/config.json index 0a59bc6e..d363e208 100644 --- a/Evaluation/RAG_Evaluator/src/config/config.json +++ b/Evaluation/RAG_Evaluator/src/config/config.json @@ -25,5 +25,8 @@ "url": "", "dbName": "", "collectionName": "" - } + }, + "model_type": "openai", + "max_concurrent_requests_for_search_api": 2, + "max_concurrent_requests_for_evaluation": 2 } \ No newline at end of file diff --git a/Evaluation/RAG_Evaluator/src/main.py b/Evaluation/RAG_Evaluator/src/main.py index 2bfe2d01..f4da5d5b 100644 --- a/Evaluation/RAG_Evaluator/src/main.py +++ b/Evaluation/RAG_Evaluator/src/main.py @@ -2,6 +2,8 @@ import os import argparse import traceback +import asyncio +import time from datetime import datetime from openai import OpenAI from config.configManager import ConfigManager @@ -10,6 +12,13 @@ from utils.evaluationResult import ResultsConverter from utils.dbservice import dbService +# Import optimized write functions +from utils.optimized_writer import unified_api_write, conditional_write_results, update_results_in_memory +from utils.batch_processor import process_sheets_with_batch_processor + +# Import async API functions +from async_api_calls import call_search_api_async + def call_search_api(queries, ground_truths): config_manager = ConfigManager() config = config_manager.get_config() @@ -36,6 +45,27 @@ def call_search_api(queries, ground_truths): return results +async def call_search_api_async_wrapper(queries, ground_truths, max_concurrent=3): + """ + Async wrapper for search API calls with configurable concurrency limit + + Args: + queries: List of search queries + ground_truths: List of ground truths + max_concurrent: Maximum number of concurrent API calls (default: 10) + """ + config_manager = ConfigManager() + config = config_manager.get_config() + # Determine API type based on config + api_type = 'UXO' # default + if config.get('SA'): + api_type = 'SA' + elif config.get('UXO'): + api_type = 'UXO' + + return await call_search_api_async(queries, ground_truths, api_type, max_concurrent) + + def load_data_and_call_api(excel_file, sheet_name, config): df = pd.read_excel(excel_file, sheet_name=sheet_name, engine='openpyxl') queries = df['query'].fillna('').tolist() @@ -66,6 +96,51 @@ def load_data_and_call_api(excel_file, sheet_name, config): ) +async def load_data_and_call_api_async(excel_file, sheet_name, config): + """ + Async version of load_data_and_call_api with timing + """ + start_time = time.time() + + df = pd.read_excel(excel_file, sheet_name=sheet_name, engine='openpyxl') + queries = df['query'].fillna('').tolist() + ground_truths = df['ground_truth'].fillna('').tolist() + + print(f"šŸ”„ Starting Search API calls for {len(queries)} queries...") + api_start_time = time.time() + + api_results = await call_search_api_async_wrapper(queries, ground_truths) + api_end_time = time.time() + api_duration = api_end_time - api_start_time + + + + print(f"āœ… Search API calls completed in {api_duration:.2f} seconds") + print(f"šŸ“Š Average time per query: {api_duration/len(queries):.2f} seconds") + + # Create a new DataFrame with API results + results_df = pd.DataFrame(api_results) + current_file_dir = os.path.dirname(os.path.abspath(__file__)) + relative_output_dir = os.path.join(current_file_dir, "outputs", "sa_api_outputs") + os.makedirs(relative_output_dir, exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") + base_filename = os.path.splitext(os.path.basename(excel_file))[0] + output_filename = f"{base_filename}_async_sa_api_results_{timestamp}.xlsx" + output_file_path = os.path.join(relative_output_dir, output_filename) + results_df.to_excel(output_file_path, index=False) + + print(f"Async API results saved to {output_file_path}") + + # Return the data in the format expected by the evaluators + return ( + results_df['query'].tolist(), + results_df['answer'].tolist(), + results_df['ground_truth'].tolist(), + results_df['context'].tolist() + ), api_duration + + def load_data(excel_file, sheet_name): if sheet_name: df = pd.read_excel(excel_file, sheet_name=sheet_name, engine='openpyxl') @@ -80,10 +155,24 @@ def load_data(excel_file, sheet_name): return queries, answers, ground_truths, contexts -def evaluate_with_ragas_and_crag(excel_file, sheet_name, config, run_ragas=True, run_crag=True, use_search_api= False, llm_model=""): +async def evaluate_with_ragas_and_crag_async(excel_file, sheet_name, config, run_ragas=True, run_crag=True, use_search_api=False, llm_model="", use_async_ragas=False): + """ + Async version of evaluate_with_ragas_and_crag with optional async Ragas and timing + """ + pipeline_start_time = time.time() + search_api_time = 0 + ragas_time = 0 + crag_time = 0 + try: if use_search_api: - queries, answers, ground_truths, contexts = load_data_and_call_api(excel_file, sheet_name, config) + print(f"šŸ”„ Loading data and calling Search API...") + data_result = await load_data_and_call_api_async(excel_file, sheet_name, config) + queries, answers, ground_truths, contexts = data_result[0] + print('quiries: ', queries) + print('answers: ', answers) + print('ground_truths: ', ground_truths) + search_api_time = data_result[1] else: queries, answers, ground_truths, contexts = load_data(excel_file, sheet_name) @@ -92,16 +181,40 @@ def evaluate_with_ragas_and_crag(excel_file, sheet_name, config, run_ragas=True, total_set_result = {} # Initialize as empty dict instead of None if run_ragas: - ragas_evaluator = RagasEvaluator() - ragas_eval_result = ragas_evaluator.evaluate(queries, answers, ground_truths, contexts, model=llm_model) + print("šŸ”„ Running Ragas evaluation...") + ragas_start_time = time.time() + + if use_async_ragas: + print("šŸ”„ Running Ragas evaluation asynchronously...") + from evaluators.asyncRagasEvaluator import AsyncRagasEvaluator + ragas_evaluator = AsyncRagasEvaluator() + ragas_eval_result = await ragas_evaluator.evaluate_async(queries, answers, ground_truths, contexts, model=llm_model) + else: + print("šŸ”„ Running Ragas evaluation synchronously...") + from evaluators.ragasEvaluator import RagasEvaluator + ragas_evaluator = RagasEvaluator() + ragas_eval_result = ragas_evaluator.evaluate(queries, answers, ground_truths, contexts, model=llm_model) + + ragas_end_time = time.time() + print('ragas_eval_result: ', ragas_eval_result) + ragas_time = ragas_end_time - ragas_start_time + print(f"āœ… Ragas evaluation completed in {ragas_time:.2f} seconds") + ragas_results = ragas_eval_result[0] # DataFrame total_set_result = ragas_eval_result[1].__dict__ if len(ragas_eval_result) > 1 else {} # Convert result object to dict if run_crag: + print("šŸ”„ Running Crag evaluation...") + crag_start_time = time.time() + openai_client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) crag_evaluator = CragEvaluator(config['openai']['model_name'], openai_client) crag_results = crag_evaluator.evaluate(queries, answers, ground_truths, contexts) + crag_end_time = time.time() + crag_time = crag_end_time - crag_start_time + print(f"āœ… Crag evaluation completed in {crag_time:.2f} seconds") + result_converter = ResultsConverter(ragas_results, crag_results) if run_ragas: @@ -119,15 +232,313 @@ def evaluate_with_ragas_and_crag(excel_file, sheet_name, config, run_ragas=True, elif len(crag_results.index) > 0: final_results = result_converter.get_crag_results() + # Calculate total pipeline time + pipeline_end_time = time.time() + total_pipeline_time = pipeline_end_time - pipeline_start_time + + # Print timing summary + print("\nā±ļø TIMING SUMMARY") + print("=" * 50) + if use_search_api: + print(f"šŸ” Search API Calls: {search_api_time:.2f}s") + if run_ragas: + print(f"šŸ“Š Ragas Evaluation: {ragas_time:.2f}s") + if run_crag: + print(f"šŸŽÆ Crag Evaluation: {crag_time:.2f}s") + print(f"šŸš€ Total Pipeline Time: {total_pipeline_time:.2f}s") + + # Calculate percentages + if total_pipeline_time > 0: + if use_search_api: + search_percent = (search_api_time / total_pipeline_time) * 100 + print(f" šŸ“ˆ Search API: {search_percent:.1f}% of total time") + if run_ragas: + ragas_percent = (ragas_time / total_pipeline_time) * 100 + print(f" šŸ“ˆ Ragas: {ragas_percent:.1f}% of total time") + if run_crag: + crag_percent = (crag_time / total_pipeline_time) * 100 + print(f" šŸ“ˆ Crag: {crag_percent:.1f}% of total time") + return final_results, total_set_result except Exception as e: print("Encountered error while running evaluation: ", traceback.format_exc()) return pd.DataFrame([]), {} + +def evaluate_with_ragas_and_crag(excel_file, sheet_name, config, run_ragas=True, run_crag=True, use_search_api=False, llm_model=""): + """ + Synchronous version with timing + """ + pipeline_start_time = time.time() + search_api_time = 0 + ragas_time = 0 + crag_time = 0 + + try: + if use_search_api: + print(f"šŸ”„ Loading data and calling Search API...") + search_start_time = time.time() + queries, answers, ground_truths, contexts = load_data_and_call_api(excel_file, sheet_name, config) + search_end_time = time.time() + search_api_time = search_end_time - search_start_time + print(f"āœ… Search API calls completed in {search_api_time:.2f} seconds") + else: + queries, answers, ground_truths, contexts = load_data(excel_file, sheet_name) + + ragas_results = pd.DataFrame([]) + crag_results = pd.DataFrame([]) + total_set_result = {} # Initialize as empty dict instead of None + + if run_ragas: + print("šŸ”„ Running Ragas evaluation...") + ragas_start_time = time.time() + ragas_evaluator = RagasEvaluator() + ragas_eval_result = ragas_evaluator.evaluate(queries, answers, ground_truths, contexts, model=llm_model) + ragas_end_time = time.time() + ragas_time = ragas_end_time - ragas_start_time + print(f"āœ… Ragas evaluation completed in {ragas_time:.2f} seconds") + + ragas_results = ragas_eval_result[0] # DataFrame + total_set_result = ragas_eval_result[1].__dict__ if len(ragas_eval_result) > 1 else {} # Convert result object to dict + + if run_crag: + print("šŸ”„ Running Crag evaluation...") + crag_start_time = time.time() + openai_client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) + crag_evaluator = CragEvaluator(config['openai']['model_name'], openai_client) + crag_results = crag_evaluator.evaluate(queries, answers, ground_truths, contexts) + crag_end_time = time.time() + crag_time = crag_end_time - crag_start_time + print(f"āœ… Crag evaluation completed in {crag_time:.2f} seconds") + + result_converter = ResultsConverter(ragas_results, crag_results) + + if run_ragas: + result_converter.convert_ragas_results() + + if run_crag: + result_converter.convert_crag_results() + + # Single return point based on review feedback + final_results = pd.DataFrame([]) + if len(ragas_results.index) > 0 and len(crag_results.index) > 0: + combined_results = result_converter.get_combined_results() + return combined_results, total_set_result if 'total_set_result' in locals() else {} + elif len(ragas_results.index) > 0: + return result_converter.get_ragas_results(), total_set_result if 'total_set_result' in locals() else {} + elif len(crag_results.index) > 0: + final_results = result_converter.get_crag_results() + return final_results, {} + + # Calculate total pipeline time + pipeline_end_time = time.time() + total_pipeline_time = pipeline_end_time - pipeline_start_time + + # Print timing summary + print("\nā±ļø TIMING SUMMARY") + print("=" * 50) + if use_search_api: + print(f"šŸ” Search API Calls: {search_api_time:.2f}s") + if run_ragas: + print(f"šŸ“Š Ragas Evaluation: {ragas_time:.2f}s") + if run_crag: + print(f"šŸŽÆ Crag Evaluation: {crag_time:.2f}s") + print(f"šŸš€ Total Pipeline Time: {total_pipeline_time:.2f}s") + + # Calculate percentages + if total_pipeline_time > 0: + if use_search_api: + search_percent = (search_api_time / total_pipeline_time) * 100 + print(f" šŸ“ˆ Search API: {search_percent:.1f}% of total time") + if run_ragas: + ragas_percent = (ragas_time / total_pipeline_time) * 100 + print(f" šŸ“ˆ Ragas: {ragas_percent:.1f}% of total time") + if run_crag: + crag_percent = (crag_time / total_pipeline_time) * 100 + print(f" šŸ“ˆ Crag: {crag_percent:.1f}% of total time") + + return pd.DataFrame([]), {} + + except Exception as e: + print("Encountered error while running evaluation: ", traceback.format_exc()) + return pd.DataFrame([]), {} + + +def add_context_relevancy_to_output(output_file_path, model_type="openai", sheet_name=None): + """ + Add context relevancy scores to the existing Ragas output file. + + Args: + output_file_path: Path to the Ragas output Excel file + model_type: "openai" or "azure" to specify which model to use + sheet_name: Optional sheet name to filter results for context relevancy. + """ + try: + if sheet_name: + print(f"šŸ”„ Adding Context Relevancy to sheet '{sheet_name}' in output file: {output_file_path}") + else: + print(f"šŸ”„ Adding Context Relevancy to output file: {output_file_path}") + + # Check if output file exists + if not os.path.exists(output_file_path): + print(f"āŒ Output file not found: {output_file_path}") + return False + + # Check file size with retry logic + max_retries = 3 + for attempt in range(max_retries): + file_size = os.path.getsize(output_file_path) + print(f"šŸ“Š File size (attempt {attempt + 1}): {file_size} bytes") + + if file_size > 0: + break + elif attempt < max_retries - 1: + print(f"āš ļø File is empty, waiting 2 seconds before retry...") + import time as time_module + time_module.sleep(2) + else: + print(f"āŒ File is still empty after {max_retries} attempts") + return False + + # Import the check_relevance functionality + from check_relevance import extract_data_from_ragas_output, evaluate_relevance_batch, setup_openai_client, get_model_from_config + + # Extract data from the Ragas output file for specific sheet + data_df = extract_data_from_ragas_output(output_file_path, sheet_name) + + if data_df is None: + if sheet_name: + print(f"āŒ Failed to extract data from Ragas output file for sheet '{sheet_name}'") + else: + print("āŒ Failed to extract data from Ragas output file") + return False + + # Evaluate context relevancy for this sheet + if sheet_name: + print(f"šŸ”„ Evaluating context relevancy for sheet '{sheet_name}' using {model_type.upper()}...") + else: + print(f"šŸ”„ Evaluating context relevancy for existing data using {model_type.upper()}...") + results_df = evaluate_relevance_batch(data_df, output_file=None, model_type=model_type) # Don't save to new file + + if results_df is None: + if sheet_name: + print(f"āŒ Failed to evaluate context relevancy for sheet '{sheet_name}'") + else: + print("āŒ Failed to evaluate context relevancy") + return False + + # Read the original output file with error handling + try: + original_df = pd.read_excel(output_file_path, sheet_name=sheet_name if sheet_name else 0, engine='openpyxl') + except Exception as read_error: + print(f"āŒ Error reading original file: {str(read_error)}") + print(f"Trying alternative reading method...") + try: + original_df = pd.read_excel(output_file_path, sheet_name=sheet_name if sheet_name else 0) + except Exception as alt_read_error: + print(f"āŒ Alternative reading method also failed: {str(alt_read_error)}") + return False + + # Add context relevancy scores to the original DataFrame + if 'relevance_score' in results_df.columns: + # Match rows by query to add context relevancy scores + relevancy_scores = [] + for _, original_row in original_df.iterrows(): + query = original_row.get('user_input', '') + # Find matching row in results + matching_row = results_df[results_df['query'] == query] + if not matching_row.empty: + relevancy_scores.append(matching_row.iloc[0]['relevance_score']) + else: + relevancy_scores.append(-1) # No match found + + original_df['context_relevancy'] = relevancy_scores + + # Save back to the same file with sheet name + try: + with pd.ExcelWriter(output_file_path, engine='openpyxl', mode='a', if_sheet_exists='replace') as writer: + original_df.to_excel(writer, sheet_name=sheet_name if sheet_name else 'Sheet1', index=False) + except Exception as write_error: + print(f"āŒ Error writing to file: {str(write_error)}") + return False + + if sheet_name: + print(f"āœ… Context Relevancy scores added to sheet '{sheet_name}'") + else: + print("āœ… Context Relevancy scores added to output file") + + # Show summary + valid_scores = original_df['context_relevancy'][original_df['context_relevancy'] >= 0] + if not valid_scores.empty: + print(f"šŸ“Š Context Relevancy Summary for {sheet_name if sheet_name else 'main sheet'}:") + print(f" Total evaluations: {len(original_df)}") + print(f" Valid scores: {len(valid_scores)}") + print(f" Average score: {valid_scores.mean():.2f}/5") + print(f" Min score: {valid_scores.min():.2f}/5") + print(f" Max score: {valid_scores.max():.2f}/5") + + return True + else: + if sheet_name: + print(f"āŒ Context relevancy column not found in results for sheet '{sheet_name}'") + else: + print("āŒ Context relevancy column not found in results") + return False + + except Exception as e: + if sheet_name: + print(f"āŒ Error adding context relevancy for sheet '{sheet_name}': {str(e)}") + else: + print(f"āŒ Error adding context relevancy: {str(e)}") + import traceback + traceback.print_exc() + return False + +def add_rag_analysis_to_output(output_file_path, model_type="openai"): + """ + Add RAG system analysis to the evaluation output. + + Args: + output_file_path: Path to the evaluation output Excel file + model_type: "openai" or "azure" to specify which model to use for analysis + """ + try: + print(f"šŸ” Adding RAG System Analysis to output file: {output_file_path}") + + # Import the RAG analysis functionality + from rag_analysis import RAGAnalyzer + + # Create analyzer + analyzer = RAGAnalyzer(model_type=model_type) + + # Run analysis + analysis_result = analyzer.analyze_rag_system(output_file_path) + + if analysis_result: + print("āœ… RAG System Analysis completed successfully!") + + # Save analysis to a separate file + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + analysis_file = f"rag_analysis_report_{timestamp}.txt" + analyzer.save_analysis_results(analysis_result, analysis_file) + + return True + else: + print("āŒ Failed to generate RAG analysis") + return False + + except Exception as e: + print(f"āŒ Error adding RAG analysis: {str(e)}") + import traceback + traceback.print_exc() + return False + # for running from api -def run(input_file, sheet_name="", evaluate_ragas=False, evaluate_crag=False, use_search_api=False, llm_model=None, save_db=False): +def run(input_file, sheet_name="", evaluate_ragas=False, evaluate_crag=False, use_search_api=False, llm_model=None, save_db=False, use_async_api=False, use_async_ragas=False ,run_analysis=False): try: + overall_start_time = time.time() + config_manager = ConfigManager() config = config_manager.get_config() @@ -158,31 +569,153 @@ def run(input_file, sheet_name="", evaluate_ragas=False, evaluate_crag=False, us run_crag = True run_ragas = True + print("šŸš€ Starting RAG Evaluation Pipeline") + print("=" * 60) + if run_ragas: + print("āœ… Ragas evaluation started") + if use_async_ragas: + print("āš” Using async Ragas evaluation") + if run_crag: + print("āœ… Crag evaluation started") + if use_async_api: + print("āš” Using async API calls for better performance") + print(f"šŸ“ Output will be saved to: {output_filename}") + print() + + pipeline_start_time = time.time() with pd.ExcelWriter(output_file_path, engine='openpyxl') as writer: - for sheet_name in sheet_names: - print(f"Processing sheet: {sheet_name}") - results = evaluate_with_ragas_and_crag(input_file, sheet_name, config, - run_crag=run_crag, - run_ragas=run_ragas, - use_search_api=use_search_api, - llm_model=llm_model) + sheets_processed = False + for i, current_sheet_name in enumerate(sheet_names): + print(f"šŸ“‹ Processing sheet {i+1}/{len(sheet_names)}: {current_sheet_name}") + + if use_async_api or use_async_ragas: + # Use async evaluation + results = asyncio.run(evaluate_with_ragas_and_crag_async(input_file, current_sheet_name, config, + run_crag=run_crag, + run_ragas=run_ragas, + use_search_api=use_search_api, + llm_model=llm_model, + use_async_ragas=use_async_ragas)) + else: + # Use sync evaluation + results = evaluate_with_ragas_and_crag(input_file, current_sheet_name, config, + run_crag=run_crag, + run_ragas=run_ragas, + use_search_api=use_search_api, + llm_model=llm_model) # Handle the case where results might be None or empty if results and len(results) >= 1 and not results[0].empty: - results[0].to_excel(writer, sheet_name=sheet_name, index=False) + # Add context relevancy scores if Ragas was run + if run_ragas: + print(f"šŸ”„ Adding Context Relevancy for sheet '{current_sheet_name}'...") + model_type = "azure" if llm_model == "azure" else "openai" + + # Import the check_relevance functionality + from check_relevance import extract_data_from_ragas_output, evaluate_relevance_batch, setup_openai_client, get_model_from_config + + # Extract data from the results DataFrame + data_df = pd.DataFrame({ + 'user_input': results[0]['user_input'].tolist(), + 'retrieved_contexts': results[0]['retrieved_contexts'].tolist() + }) + + # Evaluate context relevancy + print(f"šŸ”„ Evaluating context relevancy for sheet '{current_sheet_name}' using {model_type.upper()}...") + relevancy_results_df = evaluate_relevance_batch(data_df, output_file=None, model_type=model_type) + + if relevancy_results_df is not None and 'relevance_score' in relevancy_results_df.columns: + # Add context relevancy scores to the results DataFrame + relevancy_scores = [] + for _, original_row in results[0].iterrows(): + query = original_row.get('user_input', '') + # Find matching row in relevancy results + matching_row = relevancy_results_df[relevancy_results_df['query'] == query] + if not matching_row.empty: + relevancy_scores.append(matching_row.iloc[0]['relevance_score']) + else: + relevancy_scores.append(-1) # No match found + + results[0]['context_relevancy'] = relevancy_scores + print(f"āœ… Context Relevancy scores added to sheet '{current_sheet_name}'") + + # Show summary + valid_scores = [score for score in relevancy_scores if score >= 0] + if valid_scores: + print(f"šŸ“Š Context Relevancy Summary for {current_sheet_name}:") + print(f" Total evaluations: {len(results[0])}") + print(f" Valid scores: {len(valid_scores)}") + print(f" Average score: {sum(valid_scores)/len(valid_scores):.2f}/5") + print(f" Min score: {min(valid_scores):.2f}/5") + print(f" Max score: {max(valid_scores):.2f}/5") + else: + print(f"āš ļø Failed to calculate context relevancy for sheet '{current_sheet_name}'") + + # Write results to Excel with context relevancy included + results[0].to_excel(writer, sheet_name=current_sheet_name, index=False) if(save_db): dbService(results[0], results[1], timestamp) - print(f"Results for sheet '{sheet_name}' saved to '{output_filename}'.") + print(f"āœ… Results for sheet '{current_sheet_name}' saved to '{output_filename}'.") + sheets_processed = True else: - print(f"No results to save for sheet '{sheet_name}'. Skipping.") + print(f"āš ļø No results to save for sheet '{current_sheet_name}'. Skipping.") + + # If no sheets were processed, create a dummy sheet to avoid Excel error + if not sheets_processed: + print("āš ļø No results generated for any sheet. Creating empty result file.") + empty_df = pd.DataFrame({'message': ['No evaluation results generated']}) + empty_df.to_excel(writer, sheet_name='No_Results', index=False) + + pipeline_end_time = time.time() + pipeline_time = pipeline_end_time - pipeline_start_time + + # Add RAG analysis if requested (only once for all sheets) + run_analysis = False # Default to False, can be set based on args if needed + if run_analysis: + print("\nšŸ” Running RAG System Analysis...") + analysis_start_time = time.time() + model_type = "azure" if llm_model == "azure" else "openai" + analysis_success = add_rag_analysis_to_output(output_file_path, model_type) + analysis_end_time = time.time() + analysis_time = analysis_end_time - analysis_start_time + + if analysis_success: + print(f"āœ… RAG System Analysis completed successfully ({analysis_time:.2f}s)") + else: + print("āš ļø Failed to complete RAG System Analysis") + + # Calculate overall execution time + overall_end_time = time.time() + total_execution_time = overall_end_time - overall_start_time - print(f"All results have been saved to '{output_filename}'.") - return f"All results have been saved to '{output_filename}'." + print(f"\nšŸŽ‰ All results have been saved to '{output_filename}'.") + print("šŸ“Š The output file includes Context Relevancy scores and RAG analysis report.") + + # Print comprehensive timing summary + print("\n" + "="*60) + print("šŸ“Š COMPREHENSIVE TIMING SUMMARY") + print("="*60) + print(f"šŸš€ Total Execution Time: {total_execution_time:.2f}s") + print(f"šŸ“‹ Pipeline Processing: {pipeline_time:.2f}s") + if run_analysis: + print(f"šŸ” RAG Analysis: {analysis_time:.2f}s") + + # Calculate time breakdown percentages + if total_execution_time > 0: + pipeline_percent = (pipeline_time / total_execution_time) * 100 + print(f" šŸ“ˆ Pipeline: {pipeline_percent:.1f}% of total time") + if run_analysis: + analysis_percent = (analysis_time / total_execution_time) * 100 + print(f" šŸ“ˆ RAG Analysis: {analysis_percent:.1f}% of total time") + + return f"All results have been saved to '{output_filename}'. Total execution time: {total_execution_time:.2f}s" except Exception as e: raise Exception(f"RAG Evaluation has been failed with an error: {e}") def main(): try: + overall_start_time = time.time() + # Setup command-line argument parsing parser = argparse.ArgumentParser(description='Evaluate Ragas and Crag based on Excel input.') parser.add_argument('--input_file', type=str, required=True, help='Path to the input Excel file.') @@ -192,6 +725,8 @@ def main(): parser.add_argument('--use_search_api', action='store_true', help='Use SearchAssist API to fetch responses.') parser.add_argument('--llm_model', type=str, help="Use Azure OpenAI to evaluate the responses.") parser.add_argument('--save_db', action='store_true', help='Save the results to MongoDB.') + parser.add_argument('--use_async_api', action='store_true', help='Use async API calls for better performance.') + parser.add_argument('--use_async_ragas', action='store_true', help='Use async Ragas evaluation for better performance.') args = parser.parse_args() config_manager = ConfigManager() @@ -222,10 +757,36 @@ def main(): llm_model = args.llm_model + print("šŸš€ Starting RAG Evaluation Pipeline") + print("=" * 60) + if run_ragas: + print("āœ… Ragas evaluation ") + if args.use_async_ragas: + print("āš” Using async Ragas evaluation") + if run_crag: + print("āœ… Crag evaluation") + if args.use_async_api: + print("āš” Using async API calls for better performance") + print(f"šŸ“ Output will be saved to: {output_filename}") + print() + + pipeline_start_time = time.time() with pd.ExcelWriter(output_file_path, engine='openpyxl') as writer: - for sheet_name in sheet_names: - print(f"Processing sheet: {sheet_name}") - results = evaluate_with_ragas_and_crag(args.input_file, sheet_name, config, + sheets_processed = False + for i, current_sheet_name in enumerate(sheet_names): + print(f"šŸ“‹ Processing sheet {i+1}/{len(sheet_names)}: {current_sheet_name}") + + if args.use_async_api or args.use_async_ragas: + # Use async evaluation + results = asyncio.run(evaluate_with_ragas_and_crag_async(args.input_file, current_sheet_name, config, + run_crag=run_crag, + run_ragas=run_ragas, + use_search_api=args.use_search_api, + llm_model=llm_model, + use_async_ragas=args.use_async_ragas)) + else: + # Use sync evaluation + results = evaluate_with_ragas_and_crag(args.input_file, current_sheet_name, config, run_crag=run_crag, run_ragas=run_ragas, use_search_api=args.use_search_api, @@ -233,14 +794,108 @@ def main(): # Handle the case where results might be None or empty if results and len(results) >= 1 and not results[0].empty: - results[0].to_excel(writer, sheet_name=sheet_name, index=False) + # Add context relevancy scores if Ragas was run + if run_ragas: + print(f"šŸ”„ Adding Context Relevancy for sheet '{current_sheet_name}'...") + model_type = "azure" if llm_model == "azure" else "openai" + + # Import the check_relevance functionality + from check_relevance import extract_data_from_ragas_output, evaluate_relevance_batch, setup_openai_client, get_model_from_config + + # Extract data from the results DataFrame + data_df = pd.DataFrame({ + 'user_input': results[0]['user_input'].tolist(), + 'retrieved_contexts': results[0]['retrieved_contexts'].tolist() + }) + + # Evaluate context relevancy + print(f"šŸ”„ Evaluating context relevancy for sheet '{current_sheet_name}' using {model_type.upper()}...") + relevancy_results_df = evaluate_relevance_batch(data_df, output_file=None, model_type=model_type) + + if relevancy_results_df is not None and 'relevance_score' in relevancy_results_df.columns: + # Add context relevancy scores to the results DataFrame + relevancy_scores = [] + for _, original_row in results[0].iterrows(): + query = original_row.get('user_input', '') + # Find matching row in relevancy results + matching_row = relevancy_results_df[relevancy_results_df['query'] == query] + if not matching_row.empty: + relevancy_scores.append(matching_row.iloc[0]['relevance_score']) + else: + relevancy_scores.append(-1) # No match found + + results[0]['context_relevancy'] = relevancy_scores + print(f"āœ… Context Relevancy scores added to sheet '{current_sheet_name}'") + + # Show summary + valid_scores = [score for score in relevancy_scores if score >= 0] + if valid_scores: + print(f"šŸ“Š Context Relevancy Summary for {current_sheet_name}:") + print(f" Total evaluations: {len(results[0])}") + print(f" Valid scores: {len(valid_scores)}") + print(f" Average score: {sum(valid_scores)/len(valid_scores):.2f}/5") + print(f" Min score: {min(valid_scores):.2f}/5") + print(f" Max score: {max(valid_scores):.2f}/5") + else: + print(f"āš ļø Failed to calculate context relevancy for sheet '{current_sheet_name}'") + + # Write results to Excel with context relevancy included + results[0].to_excel(writer, sheet_name=current_sheet_name, index=False) if(args.save_db): dbService(results[0], results[1], timestamp) - print(f"Results for sheet '{sheet_name}' saved to '{output_filename}'.") + print(f"āœ… Results for sheet '{current_sheet_name}' saved to '{output_filename}'.") + sheets_processed = True else: - print(f"No results to save for sheet '{sheet_name}'. Skipping.") + print(f"āš ļø No results to save for sheet '{current_sheet_name}'. Skipping.") + + # If no sheets were processed, create a dummy sheet to avoid Excel error + if not sheets_processed: + print("āš ļø No results generated for any sheet. Creating empty result file.") + empty_df = pd.DataFrame({'message': ['No evaluation results generated']}) + empty_df.to_excel(writer, sheet_name='No_Results', index=False) + + pipeline_end_time = time.time() + pipeline_time = pipeline_end_time - pipeline_start_time + + # Add RAG analysis if requested (only once for all sheets) + run_analysis = False # Default to False, can be set based on args if needed + if run_analysis: + print("\nšŸ” Running RAG System Analysis...") + analysis_start_time = time.time() + model_type = "azure" if llm_model == "azure" else "openai" + analysis_success = add_rag_analysis_to_output(output_file_path, model_type) + analysis_end_time = time.time() + analysis_time = analysis_end_time - analysis_start_time + + if analysis_success: + print(f"āœ… RAG System Analysis completed successfully ({analysis_time:.2f}s)") + else: + print("āš ļø Failed to complete RAG System Analysis") + + # Calculate overall execution time + overall_end_time = time.time() + total_execution_time = overall_end_time - overall_start_time - print(f"All results have been saved to '{output_filename}'.") + print(f"\nšŸŽ‰ All results have been saved to '{output_filename}'.") + print("šŸ“Š The output file includes Context Relevancy scores and RAG analysis report.") + + # Print comprehensive timing summary + print("\n" + "="*60) + print("šŸ“Š COMPREHENSIVE TIMING SUMMARY") + print("="*60) + print(f"šŸš€ Total Execution Time: {total_execution_time:.2f}s") + print(f"šŸ“‹ Pipeline Processing: {pipeline_time:.2f}s") + if run_analysis: + print(f"šŸ” RAG Analysis: {analysis_time:.2f}s") + + # Calculate time breakdown percentages + if total_execution_time > 0: + pipeline_percent = (pipeline_time / total_execution_time) * 100 + print(f" šŸ“ˆ Pipeline: {pipeline_percent:.1f}% of total time") + if run_analysis: + analysis_percent = (analysis_time / total_execution_time) * 100 + print(f" šŸ“ˆ RAG Analysis: {analysis_percent:.1f}% of total time") + except Exception as e: raise Exception("RAG Evaluation has been failed with an error!!!") diff --git a/Evaluation/RAG_Evaluator/src/prompts/prompts.json b/Evaluation/RAG_Evaluator/src/prompts/prompts.json index 7cda9325..bce9eca9 100644 --- a/Evaluation/RAG_Evaluator/src/prompts/prompts.json +++ b/Evaluation/RAG_Evaluator/src/prompts/prompts.json @@ -1,3 +1,6 @@ { - "cragEvaluationPrompt": "# Task: \\r\\nYou are given a Question, a model Prediction, and a list of Ground Truth answers, judge whether the model Prediction matches any answer from the list of Ground Truth answers. Follow the instructions step by step to make a judgement. \\r\\n1. If the model prediction matches any provided answers from the Ground Truth Answer list, \\\"Accuracy\\\" should be \\\"True\\\"; otherwise, \\\"Accuracy\\\" should be \\\"False\\\".\\r\\n2. If the model prediction says that it couldn't answer the question or it doesn't have enough information, \\\"Accuracy\\\" should always be \\\"False\\\".\\r\\n3. If the Ground Truth is \\\"invalid question\\\", \\\"Accuracy\\\" is \\\"True\\\" only if the model prediction is exactly \\\"invalid question\\\".\\r\\n# Output: \\r\\nRespond with only a single JSON string with an \\\"Accuracy\\\" field which is \\\"True\\\" or \\\"False\\\".\\r\\n# Examples:\\r\\nQuestion: how many seconds is 3 minutes 15 seconds?\\r\\nGround truth: [\\\"195 seconds\\\"]\\r\\nPrediction: 3 minutes 15 seconds is 195 seconds.\\r\\nAccuracy: True\\r\\n\\r\\nQuestion: Who authored The Taming of the Shrew (published in 2002)?\\r\\nGround truth: [\\\"William Shakespeare\\\", \\\"Roma Gill\\\"]\\r\\nPrediction: The author to The Taming of the Shrew is Roma Shakespeare.\\r\\nAccuracy: False\\r\\n\\r\\nQuestion: Who played Sheldon in Big Bang Theory?\\r\\nGround truth: [\\\"Jim Parsons\\\", \\\"Iain Armitage\\\"]\\r\\nPrediction: I am sorry I don't know.\\r\\nAccuracy: False" -} \ No newline at end of file + "cragEvaluationPrompt": "# Task: \\r\\nYou are given a Question, a model Prediction, and a list of Ground Truth answers, judge whether the model Prediction matches any answer from the list of Ground Truth answers. Follow the instructions step by step to make a judgement. \\r\\n1. If the model prediction matches any provided answers from the Ground Truth Answer list, \\\"Accuracy\\\" should be \\\"True\\\"; otherwise, \\\"Accuracy\\\" should be \\\"False\\\".\\r\\n2. If the model prediction says that it couldn't answer the question or it doesn't have enough information, \\\"Accuracy\\\" should always be \\\"False\\\".\\r\\n3. If the Ground Truth is \\\"invalid question\\\", \\\"Accuracy\\\" is \\\"True\\\" only if the model prediction is exactly \\\"invalid question\\\".\\r\\n# Output: \\r\\nRespond with only a single JSON string with an \\\"Accuracy\\\" field which is \\\"True\\\" or \\\"False\\\".\\r\\n# Examples:\\r\\nQuestion: how many seconds is 3 minutes 15 seconds?\\r\\nGround truth: [\\\"195 seconds\\\"]\\r\\nPrediction: 3 minutes 15 seconds is 195 seconds.\\r\\nAccuracy: True\\r\\n\\r\\nQuestion: Who authored The Taming of the Shrew (published in 2002)?\\r\\nGround truth: [\\\"William Shakespeare\\\", \\\"Roma Gill\\\"]\\r\\nPrediction: The author to The Taming of the Shrew is Roma Shakespeare.\\r\\nAccuracy: False\\r\\n\\r\\nQuestion: Who played Sheldon in Big Bang Theory?\\r\\nGround truth: [\\\"Jim Parsons\\\", \\\"Iain Armitage\\\"]\\r\\nPrediction: I am sorry I don't know.\\r\\nAccuracy: False", + "relevanceEvaluationPrompt": "You are an expert evaluator helping assess the quality of information retrieval for question answering systems.\\r\\n\\r\\nGiven a user query and a retrieved context passage, your task is to evaluate:\\r\\n\\r\\n1. **Relevance**: How directly the context relates to the query.\\r\\n2. **Usefulness**: How helpful this context would be in answering the query.\\r\\n\\r\\nPlease return:\\r\\n- A **Relevance score** from 0 to 5, where:\\r\\n - 0 = Completely irrelevant\\r\\n - 1 = Slightly related\\r\\n - 2 = Moderately related\\r\\n - 3 = Related but lacking detail\\r\\n - 4 = Mostly relevant and partially helpful\\r\\n - 5 = Highly relevant and directly helpful for answering the query\\r\\n\\r\\n- A **short explanation** justifying your score.\\r\\n\\r\\n---\\r\\n\\r\\n**Query:**\\r\\n{query}\\r\\n\\r\\n**Retrieved Context:**\\r\\n{context_chunk}\\r\\n\\r\\n---\\r\\n\\r\\nYour response should be in this exact format:\\r\\n\\r\\nRelevance Score: [0ā€“5]\\r\\nExplanation: [1ā€“3 sentences explaining your score]" +} + + \ No newline at end of file diff --git a/Evaluation/RAG_Evaluator/src/rag_analysis.py b/Evaluation/RAG_Evaluator/src/rag_analysis.py new file mode 100644 index 00000000..29441864 --- /dev/null +++ b/Evaluation/RAG_Evaluator/src/rag_analysis.py @@ -0,0 +1,436 @@ +#!/usr/bin/env python3 +""" +RAG System Analysis Tool + +This script analyzes the complete evaluation output and generates detailed insights +about RAG system performance, including recommendations and improvements. + +Usage: + python rag_analysis.py --input "outputs/NVR_Evaluation_evaluation_output_*.xlsx" + python rag_analysis.py --input "outputs/NVR_Evaluation_evaluation_output_*.xlsx" --model azure +""" + +import os +import sys +import argparse +import pandas as pd +import json +from datetime import datetime +from openai import OpenAI +from loguru import logger +from tqdm import tqdm +import numpy as np + +# Add the current directory to the path so we can import our modules +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +from config.configManager import ConfigManager + + +class RAGAnalyzer: + def __init__(self, model_type="openai"): + self.model_type = model_type + self.config = ConfigManager().get_config() + self.openai_client = self._setup_client() + self.model_name = self._get_model_name() + + def _setup_client(self): + """Setup OpenAI client with API key from environment variable.""" + api_key = os.getenv('OPENAI_API_KEY') + + if not api_key: + print("āŒ OpenAI API key not found in environment variable OPENAI_API_KEY") + print("Please set your API key using: export OPENAI_API_KEY='your_api_key'") + return None + + return OpenAI(api_key=api_key) + + def _get_model_name(self): + """Get the model name from config.json.""" + try: + if self.model_type == "azure": + model_name = self.config.get('azure', {}).get('model_name', 'gpt-4') + print(f"šŸ¤– Using Azure model for analysis: {model_name}") + else: + model_name = self.config.get('openai', {}).get('model_name', 'gpt-4o-mini') + print(f"šŸ¤– Using OpenAI model for analysis: {model_name}") + + return model_name + except Exception as e: + print(f"āš ļø Error reading config, using default model: {str(e)}") + return 'gpt-4o-mini' if self.model_type != "azure" else 'gpt-4' + + def load_evaluation_data(self, file_path): + """ + Load evaluation data from Excel file, excluding retrieved_contexts. + + Args: + file_path: Path to the evaluation output Excel file + + Returns: + DataFrame with evaluation data (excluding retrieved_contexts) + """ + try: + print(f"šŸ“Š Loading evaluation data from: {file_path}") + df = pd.read_excel(file_path) + + # Remove retrieved_contexts column if it exists + if 'retrieved_contexts' in df.columns: + df = df.drop(columns=['retrieved_contexts']) + print("āœ… Removed retrieved_contexts column") + + print(f"šŸ“ˆ Data shape: {df.shape}") + print(f"šŸ“‹ Available columns: {list(df.columns)}") + + return df + + except Exception as e: + print(f"āŒ Error loading data: {str(e)}") + return None + + def _convert_to_json_serializable(self, obj): + """ + Convert pandas/numpy objects to JSON serializable types. + + Args: + obj: Object to convert + + Returns: + JSON serializable object + """ + if pd.isna(obj): + return None + elif isinstance(obj, (np.integer, np.int64, np.int32)): + return int(obj) + elif isinstance(obj, (np.floating, np.float64, np.float32)): + return float(obj) + elif isinstance(obj, (np.bool_)): + return bool(obj) + elif isinstance(obj, (list, tuple)): + return [self._convert_to_json_serializable(item) for item in obj] + elif isinstance(obj, dict): + return {key: self._convert_to_json_serializable(value) for key, value in obj.items()} + else: + return obj + + def prepare_analysis_data(self, df): + """ + Prepare data for analysis by creating summary statistics. + + Args: + df: DataFrame with evaluation results + + Returns: + Dictionary with analysis data + """ + analysis_data = { + "total_samples": len(df), + "columns": list(df.columns), + "summary_stats": {} + } + + # Calculate summary statistics for numeric columns + numeric_columns = df.select_dtypes(include=['number']).columns + for col in numeric_columns: + if col in df.columns: + # Convert pandas numeric types to native Python types for JSON serialization + analysis_data["summary_stats"][col] = { + "mean": self._convert_to_json_serializable(df[col].mean()), + "median": self._convert_to_json_serializable(df[col].median()), + "min": self._convert_to_json_serializable(df[col].min()), + "max": self._convert_to_json_serializable(df[col].max()), + "std": self._convert_to_json_serializable(df[col].std()), + "count": self._convert_to_json_serializable(df[col].count()) + } + + # Sample data for detailed analysis + sample_size = min(10, len(df)) + # Convert DataFrame to records and handle numeric types + sample_records = df.head(sample_size).to_dict('records') + + # Convert all values to JSON serializable types + converted_records = [] + for record in sample_records: + converted_record = {} + for key, value in record.items(): + converted_record[key] = self._convert_to_json_serializable(value) + converted_records.append(converted_record) + + analysis_data["sample_data"] = converted_records + + return analysis_data + + def generate_analysis_prompt(self, analysis_data): + """ + Generate the analysis prompt for the LLM. + + Args: + analysis_data: Dictionary with analysis data + + Returns: + Formatted prompt string + """ + prompt = f""" +You are an expert in evaluating Retrieval-Augmented Generation (RAG) pipelines. + +Below is the output of a RAGAS evaluation on a dataset, including the following metrics: + +- Answer Correctness +- Faithfulness +- Context Relevancy + +The RAG system works by retrieving **relevant chunks (context)** for a given **user query**, and then passing that context to an LLM to **generate the answer**. + +EVALUATION DATA SUMMARY: +- Total samples: {analysis_data['total_samples']} +- Available metrics: {', '.join(analysis_data['columns'])} + +SUMMARY STATISTICS: +{json.dumps(analysis_data['summary_stats'], indent=2)} + +SAMPLE DATA (first 10 entries): +{json.dumps(analysis_data['sample_data'], indent=2)} + +--- + +Please analyze the RAGAS evaluation and generate a **comprehensive and human-understandable performance report**. The report should include the following: + +--- + +1. **EXECUTIVE SUMMARY** (2ā€“3 sentences) + - Overall performance of the RAG system. + - Key strengths and weaknesses. + +2. **DETAILED METRIC ANALYSIS** + - Breakdown and interpretation of each metric. + - Highlight areas of strong and weak performance. + - Describe the impact of these scores on the user experience. + +3. **PROBLEMS IDENTIFIED** + - Specific issues in the RAG pipeline such as: + - Context chunks not relevant to the query + - Important context missing (low recall) + - LLM generating incorrect or hallucinated answers + - Answers not directly answering the user's query + - Low-quality or unclear answers + +4. **RECOMMENDATIONS (Understandable to Non-Engineers)** + - **Context Retrieval**: Are the right chunks being retrieved? Should embedding model, chunk size, or retrieval filtering be improved? + - **Prompt Quality**: Is the prompt to the LLM clear and effective? Should it be improved to better guide the LLM? + - **Answer Generation**: How can answer relevance, clarity, and correctness be improved? + +5. **IMPROVEMENT STRATEGIES** + - Concrete, actionable improvements across: + - Chunking strategy + - Retrieval model or ranking technique + - Prompt engineering + - LLM usage or fine-tuning + - Post-processing or QA validation steps + +6. **OVERALL ASSESSMENT** + - Summary of system quality. + - Confidence level in recommendations. + - Priority list for fixes or next steps. + +Be concise, specific, and use plain language so that both engineers and product stakeholders can easily understand the insights and recommendations. + +""" + return prompt + + def call_llm_for_analysis(self, prompt): + """ + Call LLM for analysis with retry logic. + + Args: + prompt: Analysis prompt + + Returns: + LLM response or None if failed + """ + if not self.openai_client: + return None + + for attempt in range(3): + try: + # Prepare API call parameters + api_params = { + "model": self.model_name, + "messages": [ + {"role": "system", "content": "You are an expert RAG system analyst. Provide detailed, actionable insights."}, + {"role": "user", "content": prompt} + ], + "temperature": 0.3, + "max_tokens": 2000 + } + + # Add Azure-specific parameters if using Azure + if self.model_type == "azure": + azure_config = self.config.get('azure', {}) + api_params.update({ + "api_version": azure_config.get('openai_api_version', '2024-02-15-preview'), + "azure_endpoint": azure_config.get('base_url'), + "azure_deployment": azure_config.get('model_deployment', self.model_name) + }) + + response = self.openai_client.chat.completions.create(**api_params) + return response.choices[0].message.content.strip() + + except Exception as e: + logger.warning(f"Analysis API call attempt {attempt + 1} failed: {str(e)}") + if attempt == 2: + logger.error(f"All analysis API call attempts failed") + return None + continue + + return None + + def save_analysis_results(self, analysis_text, output_file=None): + """ + Save analysis results to file. + + Args: + analysis_text: Analysis text from LLM + output_file: Output file path (optional) + """ + if not output_file: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + output_file = f"rag_analysis_report_{timestamp}.txt" + + try: + with open(output_file, 'w', encoding='utf-8') as f: + f.write("RAG SYSTEM ANALYSIS REPORT\n") + f.write("=" * 50 + "\n") + f.write(f"Generated on: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") + f.write(f"Model used: {self.model_name} ({self.model_type.upper()})\n") + f.write("=" * 50 + "\n\n") + f.write(analysis_text) + + print(f"šŸ’¾ Analysis report saved to: {output_file}") + + except Exception as e: + print(f"āŒ Error saving analysis report: {str(e)}") + + def analyze_rag_system(self, file_path, output_file=None): + """ + Complete RAG system analysis pipeline. + + Args: + file_path: Path to evaluation output file + output_file: Output file for analysis report (optional) + """ + print("šŸ” Starting RAG System Analysis") + print("=" * 50) + + # Load data + df = self.load_evaluation_data(file_path) + if df is None: + print("āŒ Failed to load evaluation data") + return None + + # Prepare analysis data + print("šŸ“Š Preparing analysis data...") + analysis_data = self.prepare_analysis_data(df) + + # Generate analysis prompt + print("šŸ¤– Generating analysis prompt...") + prompt = self.generate_analysis_prompt(analysis_data) + + # Call LLM for analysis + print("šŸ§  Calling LLM for comprehensive analysis...") + analysis_result = self.call_llm_for_analysis(prompt) + + if analysis_result: + print("āœ… Analysis completed successfully!") + + # Display analysis + print("\n" + "=" * 60) + print("šŸ“‹ RAG SYSTEM ANALYSIS REPORT") + print("=" * 60) + print(analysis_result) + print("=" * 60) + + # Save results + self.save_analysis_results(analysis_result, output_file) + + return analysis_result + else: + print("āŒ Failed to generate analysis") + return None + + +def find_latest_evaluation_output(): + """ + Find the latest evaluation output file in the outputs directory. + + Returns: + Path to the latest output file + """ + outputs_dir = "outputs" + pattern = os.path.join(outputs_dir, "*evaluation_output*.xlsx") + + import glob + files = glob.glob(pattern) + if not files: + print(f"āŒ No evaluation output files found in {outputs_dir}") + return None + + # Sort by modification time to get the latest + latest_file = max(files, key=os.path.getmtime) + print(f"šŸ“ Found latest evaluation output: {latest_file}") + return latest_file + + +def main(): + """Main function to handle command line arguments and run analysis.""" + parser = argparse.ArgumentParser( + description="Analyze RAG system performance from evaluation output", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python rag_analysis.py + python rag_analysis.py --input "outputs/NVR_Evaluation_evaluation_output_*.xlsx" + python rag_analysis.py --input "outputs/NVR_Evaluation_evaluation_output_*.xlsx" --model azure + python rag_analysis.py --output analysis_report.txt + """ + ) + + parser.add_argument("--input", "-i", help="Path to evaluation output file (default: latest)") + parser.add_argument("--model", "-m", choices=["openai", "azure"], default="openai", + help="Model type to use for analysis (default: openai)") + parser.add_argument("--output", "-o", help="Output file for analysis report") + + args = parser.parse_args() + + print("RAG System Analysis Tool") + print("=" * 60) + print("Analyzing RAG system performance and generating recommendations...") + print() + + try: + # Determine input file + if args.input: + input_file = args.input + else: + input_file = find_latest_evaluation_output() + if not input_file: + return + + # Create analyzer + analyzer = RAGAnalyzer(model_type=args.model) + + # Run analysis + result = analyzer.analyze_rag_system(input_file, args.output) + + if result: + print(f"\nāœ… RAG system analysis completed successfully!") + print("šŸ“Š Check the analysis report for detailed insights and recommendations.") + else: + print(f"\nāŒ RAG system analysis failed!") + + except Exception as e: + # logger.error(f"Error during analysis: {str(e)}") + print(f"āŒ Error: {str(e)}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/Evaluation/RAG_Evaluator/src/utils/__init__.py b/Evaluation/RAG_Evaluator/src/utils/__init__.py index e69de29b..ff678caf 100644 --- a/Evaluation/RAG_Evaluator/src/utils/__init__.py +++ b/Evaluation/RAG_Evaluator/src/utils/__init__.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +""" +Utils package for RAG evaluation pipeline optimizations +""" + +from .optimized_writer import ( + optimized_write_results, + batch_write_results, + unified_api_write, + update_results_in_memory, + conditional_write_results, + consolidate_sheet_results +) + +from .batch_processor import ( + BatchProcessor, + create_batch_processor, + process_sheets_with_batch_processor +) + +__all__ = [ + 'optimized_write_results', + 'batch_write_results', + 'unified_api_write', + 'update_results_in_memory', + 'conditional_write_results', + 'consolidate_sheet_results', + 'BatchProcessor', + 'create_batch_processor', + 'process_sheets_with_batch_processor', + 'ConsolidatedWriter', + 'create_consolidated_writer', + 'write_all_results_consolidated' +] From c20b77b5cca9749b152b087dbf9dead52734f00d Mon Sep 17 00:00:00 2001 From: niranjank-Kore Date: Mon, 11 Aug 2025 19:04:34 +0530 Subject: [PATCH 2/2] added context_url --- Evaluation/RAG_Evaluator/src/api/XOSearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Evaluation/RAG_Evaluator/src/api/XOSearch.py b/Evaluation/RAG_Evaluator/src/api/XOSearch.py index 6b3c8667..c6d9f643 100644 --- a/Evaluation/RAG_Evaluator/src/api/XOSearch.py +++ b/Evaluation/RAG_Evaluator/src/api/XOSearch.py @@ -77,7 +77,7 @@ def get_bot_response(api: XOSearchAPI, query: str, truth: str) -> Optional[Dict] 'query': query, 'ground_truth': truth, 'context': context_data, - 'context_url': "", + 'context_url': context_url, 'answer': bot_answer }