} Session ID or null if failed
+ */
+ async createUserSession() {
+ try {
+ console.log('๐ Creating user session...');
+
+ const response = await fetch(API_ENDPOINTS.CREATE_SESSION, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ client_info: {
+ timestamp: new Date().toISOString(),
+ timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+ language: navigator.language
+ }
+ })
+ });
+
+ if (response.ok) {
+ const data = await response.json();
+ this.sessionId = data.session_id;
+ console.log('โ
User session created:', this.sessionId);
+
+ // Store session in local storage for page refresh persistence
+ localStorage.setItem('ragEvaluatorSessionId', this.sessionId);
+
+ // Update UI to show session info
+ this.updateSessionInfo();
+
+ return this.sessionId;
+ } else {
+ await ErrorHandler.handleNetworkError(response, 'Session creation');
+ return null;
+ }
+ } catch (error) {
+ ErrorHandler.handleError(error, 'Session creation');
+ return null;
+ }
+ }
+
+ async validateExistingSession() {
+ // Validate an existing session from localStorage
+ const storedSessionId = localStorage.getItem('ragEvaluatorSessionId');
+ if (!storedSessionId) {
+ return false;
+ }
+
+ try {
+ const response = await fetch(`${API_ENDPOINTS.SESSION_STATUS}/${storedSessionId}`);
+ if (response.ok) {
+ const data = await response.json();
+ if (data.is_valid) {
+ this.sessionId = storedSessionId;
+ console.log('โ
Restored existing session:', this.sessionId);
+ this.updateSessionInfo();
+ return true;
+ }
+ }
+ } catch (error) {
+ console.warn('โ ๏ธ Error validating existing session:', error);
+ // Clear invalid session
+ localStorage.removeItem('ragEvaluatorSessionId');
+ }
+
+ return false;
+ }
+
+ updateSessionInfo() {
+ // Update UI with session information
+ if (this.sessionId) {
+ // Add session indicator to header
+ const header = document.querySelector('.header-content');
+ if (header && !document.getElementById('session-indicator')) {
+ const sessionIndicator = document.createElement('div');
+ sessionIndicator.id = 'session-indicator';
+ sessionIndicator.className = 'session-indicator';
+ sessionIndicator.innerHTML = `
+
+
+
Session: ${this.sessionId.substring(0, 8)}...
+
+
+
+
+
+
+
+ Batch Size
+ ${presetData.batch}
+
+
+ Concurrent
+ ${presetData.concurrent}
+
+
+
+
+
Benefits & Features
+ ${presetData.benefits.map(benefit => `
+
+
+ ${benefit.emoji}
+
+
${benefit.text}
+
+ `).join('')}
+
+
+
+
Performance Characteristics
+
+
+
+
+
${presetData.stats.speed}%
+
+
+
+
+
๐พ
+
Resource Usage
+
+
+
+
${presetData.stats.resource}%
+
+
+
+
+
+
+
${presetData.stats.stability}%
+
+
+
+
+
+ Best for: ${presetData.recommendation}
+
+
+
All sheets ';
+
+ this.validateForm();
+ this.estimateProcessingTime(); // Reset estimates
+ }
+
+ async loadSheetNames(file) {
+ console.log('๐ Loading sheet names for file:', file.name);
+ try {
+ const formData = new FormData();
+ formData.append('file', file);
+
+ const response = await fetch('/api/get-sheet-names', {
+ method: 'POST',
+ body: formData
+ });
+
+ console.log('๐ก Sheet names API response status:', response.status);
+
+ if (response.ok) {
+ const data = await response.json();
+ console.log('๐ Sheet names data received:', data);
+
+ if (data.sheet_names && data.sheet_names.length > 0) {
+ console.log('โ
Found sheets:', data.sheet_names);
+ this.populateSheetDropdown(data.sheet_names);
+ this.showToast(`Found ${data.sheet_names.length} sheet(s) in the Excel file`, 'info');
+ } else {
+ console.warn('โ ๏ธ No sheet names found in response');
+ this.populateSheetDropdown([]);
+ }
+
+ // Get row count for better estimation
+ if (data.row_counts) {
+ this.fileRowCounts = data.row_counts;
+ console.log('๐ File row counts:', this.fileRowCounts);
+ this.estimateProcessingTime(); // Re-estimate with actual data
+ }
+ } else {
+ const errorText = await response.text();
+ console.error('โ Failed to load sheet names:', response.status, errorText);
+ this.showToast('Failed to analyze Excel file sheets', 'error');
+ }
+ } catch (error) {
+ console.error('โ Error loading sheet names:', error);
+ this.showToast('Error analyzing Excel file: ' + error.message, 'error');
+ }
+ }
+
+ populateSheetDropdown(sheetNames) {
+ console.log('๐ Populating sheet dropdown with:', sheetNames);
+ const sheetSelect = document.getElementById('sheet-name');
+ if (sheetSelect) {
+ // Always start with "All sheets" option which is selected by default
+ sheetSelect.innerHTML = 'All sheets ';
+
+ if (sheetNames && sheetNames.length > 0) {
+ sheetNames.forEach((sheetName, index) => {
+ const option = document.createElement('option');
+ option.value = sheetName;
+ option.textContent = sheetName;
+ sheetSelect.appendChild(option);
+ console.log(`๐ Added sheet option ${index + 1}: "${sheetName}"`);
+ });
+
+ console.log(`โ
Successfully populated dropdown with ${sheetNames.length} sheet(s) + "All sheets" option`);
+ } else {
+ console.log('โน๏ธ No individual sheets to add, only "All sheets" option available');
+ }
+
+ // Add event listener for sheet selection changes to update estimates
+ sheetSelect.removeEventListener('change', this.handleSheetChange); // Remove existing
+ this.handleSheetChange = () => {
+ console.log('๐ Sheet selection changed, updating estimates...');
+ this.estimateProcessingTime();
+ };
+ sheetSelect.addEventListener('change', this.handleSheetChange);
+ } else {
+ console.error('โ Could not find sheet-name dropdown element');
+ }
+ }
+
+ estimateProcessingTime() {
+ const estimatedTimeElement = document.getElementById('estimated-time');
+ const totalQueriesElement = document.getElementById('total-queries');
+
+ if (!this.uploadedFile) {
+ // No file uploaded - show placeholders
+ if (estimatedTimeElement) {
+ estimatedTimeElement.textContent = '--';
+ estimatedTimeElement.title = 'Upload a file to see estimation';
+ }
+ if (totalQueriesElement) {
+ totalQueriesElement.textContent = '--';
+ totalQueriesElement.title = 'Upload a file to see query count';
+ }
+
+ // Clear estimation details
+ const detailsContainer = document.getElementById('estimation-details');
+ if (detailsContainer) {
+ detailsContainer.style.display = 'none';
+ }
+ return;
+ }
+
+ // Show loading while calculating
+ if (estimatedTimeElement) estimatedTimeElement.textContent = 'Calculating...';
+ if (totalQueriesElement) totalQueriesElement.textContent = 'Analyzing...';
+
+ // Get current settings
+ const batchSize = parseInt(document.getElementById(ELEMENTS.BATCH_SIZE)?.value || CONFIG.DEFAULT_BATCH_SIZE);
+ const maxConcurrent = parseInt(document.getElementById(ELEMENTS.MAX_CONCURRENT)?.value || CONFIG.DEFAULT_MAX_CONCURRENT);
+ const useSearchApi = document.getElementById(ELEMENTS.USE_SEARCH_API)?.checked || false;
+ const evaluateRagas = document.getElementById(ELEMENTS.EVALUATE_RAGAS)?.checked || false;
+ const evaluateCrag = document.getElementById(ELEMENTS.EVALUATE_CRAG)?.checked || false;
+ const evaluateLlm = document.getElementById(ELEMENTS.EVALUATE_LLM)?.checked || false;
+ const selectedSheet = document.getElementById(ELEMENTS.SHEET_SELECT)?.value || '';
+
+ // Calculate actual query count
+ let totalQueries = 0;
+
+ if (this.fileRowCounts) {
+ // Use actual row counts from Excel file
+ if (selectedSheet && this.fileRowCounts[selectedSheet]) {
+ // Specific sheet selected
+ totalQueries = this.fileRowCounts[selectedSheet];
+ console.log(`๐ Using row count for sheet '${selectedSheet}': ${totalQueries}`);
+ } else {
+ // All sheets or no specific sheet - sum all rows
+ totalQueries = Object.values(this.fileRowCounts).reduce((sum, count) => sum + count, 0);
+ console.log(`๐ Using total row count across all sheets: ${totalQueries}`);
+ }
+ } else {
+ // Fallback to file size estimation (less accurate)
+ totalQueries = Math.max(10, Math.floor(this.uploadedFile.size / (2 * 1024))); // Assume ~2KB per row
+ console.log(`๐ Fallback estimation based on file size: ${totalQueries} queries`);
+ }
+
+ // Improved time estimation based on real-world benchmarks
+ let timePerQuery = this.calculateTimePerQuery(useSearchApi, evaluateRagas, evaluateCrag, evaluateLlm);
+
+ // Account for batch processing and concurrency
+ const effectiveBatchSize = Math.min(batchSize, totalQueries);
+ const parallelBatches = Math.min(maxConcurrent, Math.ceil(totalQueries / effectiveBatchSize));
+ const totalBatches = Math.ceil(totalQueries / effectiveBatchSize);
+ const batchesPerRound = parallelBatches;
+ const totalRounds = Math.ceil(totalBatches / batchesPerRound);
+
+ // Calculate total time considering parallel processing
+ const totalTime = totalRounds * timePerQuery * effectiveBatchSize / parallelBatches;
+
+ // Add overhead for initialization, file I/O, etc.
+ const overheadTime = Math.max(CONFIG.BASE_OVERHEAD, totalQueries * CONFIG.OVERHEAD_PER_QUERY);
+ const finalTime = totalTime + overheadTime;
+
+ console.log(`๐ Estimation details:`, {
+ totalQueries,
+ timePerQuery: `${timePerQuery}s`,
+ effectiveBatchSize,
+ parallelBatches,
+ totalBatches,
+ totalRounds,
+ processingTime: `${totalTime}s`,
+ overheadTime: `${overheadTime}s`,
+ finalTime: `${finalTime}s`
+ });
+
+ // Update UI with calculated values
+ if (estimatedTimeElement) {
+ estimatedTimeElement.textContent = UIUtils.formatTime(finalTime);
+ estimatedTimeElement.title = `Based on ${totalQueries} queries with current settings`;
+ }
+
+ if (totalQueriesElement) {
+ totalQueriesElement.textContent = totalQueries.toLocaleString();
+ totalQueriesElement.title = selectedSheet ?
+ `From sheet: ${selectedSheet}` :
+ 'Across all sheets';
+ }
+
+ // Update estimates info
+ this.updateEstimationDetails(totalQueries, finalTime, useSearchApi, evaluateRagas, evaluateCrag, evaluateLlm);
+
+ // Show estimation details
+ const detailsContainer = document.getElementById('estimation-details');
+ if (detailsContainer) {
+ detailsContainer.style.display = 'block';
+ }
+
+ // No longer needed - preset details are shown in modal
+ }
+
+ /**
+ * Calculate estimated processing time per query based on enabled features
+ * @param {boolean} useSearchApi - Whether search API is enabled
+ * @param {boolean} evaluateRagas - Whether RAGAS evaluation is enabled
+ * @param {boolean} evaluateCrag - Whether CRAG evaluation is enabled
+ * @param {boolean} evaluateLlm - Whether LLM evaluation is enabled
+ * @returns {number} Estimated time per query in seconds
+ */
+ calculateTimePerQuery(useSearchApi, evaluateRagas, evaluateCrag, evaluateLlm) {
+ // Base time for basic processing
+ let timePerQuery = CONFIG.BASE_PROCESSING_TIME;
+
+ // Search API time (if enabled)
+ if (useSearchApi) {
+ timePerQuery += CONFIG.SEARCH_API_TIME;
+ }
+
+ // Calculate parallel evaluation time - evaluators run concurrently
+ const evaluationTimes = [];
+
+ // RAGAS evaluation time (most time-consuming)
+ if (evaluateRagas) {
+ evaluationTimes.push(CONFIG.RAGAS_EVAL_TIME);
+ }
+
+ // CRAG evaluation time
+ if (evaluateCrag) {
+ evaluationTimes.push(CONFIG.CRAG_EVAL_TIME);
+ }
+
+ // LLM evaluation time (3 API calls per query, but concurrent)
+ if (evaluateLlm) {
+ evaluationTimes.push(CONFIG.LLM_EVAL_TIME);
+ }
+
+ // When running in parallel, take the maximum time (longest running evaluator)
+ // rather than sum of all times
+ if (evaluationTimes.length > 0) {
+ timePerQuery += Math.max(...evaluationTimes);
+ }
+
+ // Minimum time
+ return Math.max(CONFIG.MIN_PROCESSING_TIME, timePerQuery);
+ }
+
+ updateEstimationDetails(queries, time, useSearchApi, evaluateRagas, evaluateCrag, evaluateLlm) {
+ // Create or update estimation breakdown
+ let detailsContainer = document.getElementById('estimation-details');
+ if (!detailsContainer) {
+ detailsContainer = document.createElement('div');
+ detailsContainer.id = 'estimation-details';
+ detailsContainer.className = 'estimation-details';
+
+ const parentContainer = document.querySelector('.run-info');
+ if (parentContainer) {
+ parentContainer.appendChild(detailsContainer);
+ }
+ }
+
+ const components = [];
+ if (useSearchApi) components.push('Search API');
+ if (evaluateRagas) components.push('RAGAS Evaluation');
+ if (evaluateCrag) components.push('CRAG Evaluation');
+ if (evaluateLlm) components.push('LLM Evaluation');
+
+ detailsContainer.innerHTML = `
+
+
+ Processing:
+ ${components.join(' + ') || 'Basic processing'}
+
+
+ Queries:
+ ${queries.toLocaleString()} rows detected
+
+
+ Est. Time:
+ ${this.formatTime(time)}
+
+
+
${name}
+
${count} questions affected
+
+
+
No specific recommendations at this time. Your system appears to be performing well.
+
${recommendation}
+ `;
+
+ recommendationsList.appendChild(recommendationItem);
+ });
+ }
+
+ /**
+ * Display detailed analysis table in the UI
+ * @param {Array} analysisList - List of analysis items
+ */
+ displayDetailedAnalysisTable(analysisList) {
+ const tableBody = document.getElementById('analysis-table-body');
+ if (!tableBody) return;
+
+ tableBody.innerHTML = '';
+
+ if (analysisList.length === 0) {
+ tableBody.innerHTML = `
+
+
+
+
+ `;
+ return;
+ }
+
+ analysisList.forEach(analysis => {
+ const row = document.createElement('tr');
+
+ // Truncate query if too long
+ const query = analysis.query && analysis.query.length > 50
+ ? analysis.query.substring(0, 50) + '...'
+ : analysis.query || 'N/A';
+
+ row.innerHTML = `
+ ${query}
+
+
+ ${this.formatCategoryName(analysis.category)}
+
+
+ ${analysis.unused_count || 0}
+ ${this.formatScore(analysis.context_relevance_score)}
+ ${this.formatScore(analysis.ground_truth_validity_score)}
+ ${this.formatScore(analysis.context_overload_score)}
+
+ ${this.truncateText(analysis.reasoning || 'N/A', 60)}
+
+ `;
+
+ tableBody.appendChild(row);
+ });
+ }
+
+ /**
+ * Format category name for display
+ * @param {string} category - Category string
+ * @returns {string} Formatted category name
+ */
+ formatCategoryName(category) {
+ const categoryNames = {
+ 'context_irrelevant': 'Context Irrelevant',
+ 'ground_truth_invalid': 'Ground Truth Invalid',
+ 'context_overload': 'Context Overload',
+ 'answer_generation_failure': 'Answer Generation Failure',
+ 'mixed_issues': 'Mixed Issues'
+ };
+
+ return categoryNames[category] || category;
+ }
+
+ /**
+ * Format score for display
+ * @param {number|string} score - Score value
+ * @returns {string} Formatted score
+ */
+ formatScore(score) {
+ if (score === null || score === undefined || score === 'N/A') {
+ return 'N/A';
+ }
+
+ if (typeof score === 'number') {
+ return score.toFixed(3);
+ }
+
+ return score;
+ }
+
+ /**
+ * Truncate text for display
+ * @param {string} text - Text to truncate
+ * @param {number} maxLength - Maximum length
+ * @returns {string} Truncated text
+ */
+ truncateText(text, maxLength) {
+ if (!text || text.length <= maxLength) {
+ return text;
+ }
+
+ return text.substring(0, maxLength) + '...';
+ }
+
+ /**
+ * Toggle quality analysis section visibility
+ */
+ toggleQualityAnalysis() {
+ const qualityContainer = document.getElementById('quality-analysis-container');
+ const toggleButton = document.getElementById('toggle-quality-analysis');
+
+ if (!qualityContainer || !toggleButton) {
+ console.error('โ Quality analysis elements not found');
+ return;
+ }
+
+ const isVisible = qualityContainer.style.display !== 'none';
+
+ if (isVisible) {
+ qualityContainer.style.display = 'none';
+ toggleButton.innerHTML = '
+
+
No detailed results available. This may happen if:
+
+ The evaluation is still processing
+ No evaluation methods were performed (RAGAS, LLM, or CRAG)
+ The results file could not be read
+ The evaluation encountered errors during processing
+
+
+
+
+
+ ${sheetNames.map((sheetName, index) => `
+
+ (${sheetGroups[sheetName].length})
+
+ `).join('')}
+
+
+
+
+ ${sheetNames.map((sheetName, index) => `
+
+ ${this.createSingleSheetTable(sheetGroups[sheetName], sheetName)}
+
+ `).join('')}
+
+
+ ${sheetName ? `
+
+
+ ` : ''}
+
+ ${hasEvaluationMetrics ? `
+
+
+
+
+
+
+ ๐ Overview
+ ๐ Correlations
+ ๐ฏ Performance
+ ๐ Retrieval Quality
+ ๐ก Insights
+
+
+
+
+
+
+
+
Overall Performance Summary
+
+
+
+
+
+
+
+
Per-Metric Score Range Distribution
+
+
+
+
+
+
+
+
+
Metric Correlation Matrix
+
+
+
Metric Relationships
+
+
+
Correlation Insights
+
+
+
+
+
+
+
+
+
+
+
๐ Retrieval Quality Overview
+
+
+
+
๐ Chunk Utilization Distribution
+
+
+
๐ Retrieval Quality Insights
+
+
+
+
โก System Efficiency Analysis
+
+
+
+
๐ฏ Chunk Utilization Analysis
+
+
+
+
+
+
+
+
+
๐ฏ Model Performance Analysis
+
+
+
+
๐ Statistical Summary
+
+
+
+
๐ก Recommendations
+
+
+
+
+
+ ` : ''}
+
+
+
+
+
+
+
+
+ #
+ ${orderedColumns.map(col => `
+
+ ${this.formatColumnHeader(col)}
+
+ `).join('')}
+
+
+
+ ${detailedResults.map((row, index) => `
+
+ ${index + 1}
+ ${orderedColumns.map(col => `
+
+ ${this.formatCellValue(col, row[col])}
+
+ `).join('')}
+
+ `).join('')}
+
+
+
+
+ ${detailedResults.length >= 50 ? `
+
+ Showing first 50 results per sheet. Download the full results file for complete data.
+
+ ` : ''}
+
N/A ';
+ }
+
+ const col = columnName.toLowerCase();
+
+ // Format evaluation metrics (RAGAS, LLM, CRAG)
+ if (col.includes('relevancy') || col.includes('faithfulness') ||
+ col.includes('recall') || col.includes('precision') ||
+ col.includes('correctness') || col.includes('similarity') ||
+ col.includes('accuracy') || col.includes('llm') || col.includes('crag')) {
+
+ const numValue = parseFloat(value);
+ if (!isNaN(numValue)) {
+ // Smart formatting: detect if value is already a percentage (>1) or decimal (0-1)
+ let displayValue, normalizedValue;
+
+ if (numValue <= 1) {
+ // Value is likely a decimal (0-1), convert to percentage for display
+ displayValue = (numValue * 100).toFixed(1) + '%';
+ normalizedValue = numValue;
+ } else {
+ // Value is likely already a percentage, display as-is
+ displayValue = numValue.toFixed(1) + '%';
+ normalizedValue = numValue / 100;
+ }
+
+ const scoreClass = normalizedValue >= 0.8 ? 'score-good' :
+ normalizedValue >= 0.6 ? 'score-medium' : 'score-low';
+
+ return `${displayValue} `;
+ }
+ }
+
+ // Format long text fields
+ if (col === 'query' || col === 'answer' || col === 'ground_truth' || col.includes('context')) {
+ const strValue = String(value);
+ if (strValue.length > 100) {
+ const truncated = strValue.substring(0, 100) + '...';
+ return `${this.escapeHtml(truncated)} `;
+ }
+ return `${this.escapeHtml(strValue)} `;
+ }
+
+ // Format numbers
+ if (!isNaN(value) && value !== '') {
+ const numValue = parseFloat(value);
+ if (Number.isInteger(numValue)) {
+ return `${numValue} `;
+ } else {
+ // Show more precision for non-metric numbers, less aggressive rounding
+ const decimalPlaces = numValue < 1 ? 6 : 4;
+ return `${numValue.toFixed(decimalPlaces)} `;
+ }
+ }
+
+ // Default formatting
+ const strValue = String(value);
+ if (strValue.length > 50) {
+ const truncated = strValue.substring(0, 50) + '...';
+ return `${this.escapeHtml(truncated)} `;
+ }
+
+ return this.escapeHtml(strValue);
+ }
+
+ getCellTitle(value) {
+ if (value === null || value === undefined || value === 'N/A') {
+ return 'No data available';
+ }
+ return String(value).length > 50 ? String(value) : '';
+ }
+
+ escapeHtml(text) {
+ const div = document.createElement('div');
+ div.textContent = text;
+ return div.innerHTML;
+ }
+
+ formatTokenCount(tokens) {
+ if (tokens === null || tokens === undefined || tokens === 0) {
+ return 'N/A ';
+ }
+ return Number(tokens).toLocaleString();
+ }
+
+ formatCurrency(amount) {
+ if (amount === null || amount === undefined || amount === 0) {
+ return '$0.00 ';
+ }
+ return `$${Number(amount).toFixed(4)}`;
+ }
+
+ formatCostPerQuery(totalCost, totalQueries) {
+ if (!totalCost || !totalQueries || totalCost === 0 || totalQueries === 0) {
+ return 'N/A ';
+ }
+ const costPerQuery = totalCost / totalQueries;
+ return `$${costPerQuery.toFixed(6)}`;
+ }
+
+ createTokenUsageSection(stats) {
+ // Check if we have real token data
+ const hasRealTokenData = this.hasRealTokenUsageData(stats);
+
+ if (!hasRealTokenData) {
+ console.log('โ ๏ธ No valid token usage data available, hiding detailed section');
+ return '';
+ }
+
+ console.log('โ
Token usage data detected, showing detailed breakdown section');
+
+ // Build the section with available data
+ let tokenCards = '';
+
+ // Always show total tokens if available
+ if (stats.total_tokens && stats.total_tokens > 0) {
+ tokenCards += `
+
+ ${this.formatTokenCount(stats.total_tokens)}
+ Total Tokens
+
`;
+ }
+
+ // Show breakdown if available
+ if (stats.prompt_tokens && stats.prompt_tokens > 0) {
+ tokenCards += `
+
+ ${this.formatTokenCount(stats.prompt_tokens)}
+ Prompt Tokens
+
`;
+ }
+
+ if (stats.completion_tokens && stats.completion_tokens > 0) {
+ tokenCards += `
+
+ ${this.formatTokenCount(stats.completion_tokens)}
+ Completion Tokens
+
`;
+ }
+
+ // Show cost per query if both cost and query count are available
+ if (stats.estimated_cost_usd && stats.total_processed &&
+ stats.estimated_cost_usd > 0 && stats.total_processed > 0) {
+ tokenCards += `
+
+ ${this.formatCostPerQuery(stats.estimated_cost_usd, stats.total_processed)}
+ Cost per Query
+
`;
+ }
+
+ return `
+
+
+
+ ${tokenCards}
+
+
+
+
+
+
+
+
+
+ ${stats.total_processed || 0}
+ Total Processed
+
+
+ ${stats.successful_queries || 0}
+ Successful
+
+
+ ${stats.failed_queries || 0}
+ Failed
+
+ ${this.hasRealTokenUsageData(stats) ? `
+
+ ${this.formatTokenCount(stats.total_tokens)}
+ Total Tokens
+
+
+ ${this.formatCurrency(stats.estimated_cost_usd)}
+ Estimated Cost
+
` : ''}
+
+
+
+ ${this.createTokenUsageSection(stats)}
+
+
+
+
+
+
+
+
RAGAS: ${config.evaluate_ragas ? 'โ
Enabled' : 'โ Disabled'}
+
CRAG: ${config.evaluate_crag ? 'โ
Enabled' : 'โ Disabled'}
+
LLM Evaluation: ${config.evaluate_llm ? 'โ
Enabled' : 'โ Disabled'}
+
Search API: ${config.use_search_api ? 'โ
Enabled' : 'โ Disabled'}
+
LLM Model: ${config.llm_model || 'Default'}
+
Batch Size: ${config.batch_size || 10}
+
+
+
+
+
+
${message}
+ `;
+
+ document.body.appendChild(toast);
+
+ setTimeout(() => {
+ if (toast.parentNode) {
+ toast.style.animation = 'slideInRight 0.3s ease reverse';
+ setTimeout(() => {
+ if (toast.parentNode) toast.parentNode.removeChild(toast);
+ }, 300);
+ }
+ }, 4000);
+ }
+
+ getToastIcon(type) {
+ const icons = {
+ 'success': 'check-circle',
+ 'warning': 'exclamation-triangle',
+ 'error': 'times-circle',
+ 'info': 'info-circle'
+ };
+ return icons[type] || 'info-circle';
+ }
+
+ getEvaluationMethodsText() {
+ const config = this.resultData?.config_used || {};
+ const enabledMethods = [];
+ if (config.evaluate_ragas) enabledMethods.push('RAGAS');
+ if (config.evaluate_llm) enabledMethods.push('LLM');
+ if (config.evaluate_crag) enabledMethods.push('CRAG');
+
+ return enabledMethods.length > 0 ? enabledMethods.join(' + ') : 'Unknown Methods';
+ }
+
+ createOverviewCharts(sheetId, analysisData) {
+ // 1. Summary Radar Chart
+ this.createSummaryRadarChart(`summary-radar-${sheetId}`, analysisData);
+
+ // 2. Per-Metric Score Range Histograms
+ this.createMetricRangeHistograms(`metric-histograms-${sheetId}`, analysisData);
+
+ // 3. Key Statistics
+ this.populateKeyStatistics(`key-stats-${sheetId}`, analysisData);
+ }
+
+
+
+ createCorrelationCharts(sheetId, analysisData) {
+ // 1. Correlation heatmap
+ this.createCorrelationHeatmap(`correlation-matrix-${sheetId}`, analysisData);
+
+ // 2. Scatter plot matrix
+ this.createScatterMatrix(`scatter-matrix-${sheetId}`, analysisData);
+
+ // 3. Correlation insights
+ this.populateCorrelationInsights(`correlation-insights-${sheetId}`, analysisData);
+ }
+
+ createPerformanceCharts(sheetId, analysisData) {
+ // 1. Query performance heatmap
+ this.createQueryHeatmap(`performance-heatmap-${sheetId}`, analysisData);
+
+ // 2. Top/bottom performers
+ this.createTopBottomChart(`top-bottom-queries-${sheetId}`, analysisData);
+
+ // 3. Outliers analysis
+ this.populateOutliersAnalysis(`outliers-analysis-${sheetId}`, analysisData);
+ }
+
+ generateInsights(sheetId, analysisData) {
+ console.log('๐ฏ Generating insights for sheet:', sheetId);
+ console.log('๐ Analysis data insights:', analysisData.insights);
+
+ // Populate all insight sections
+ this.populateModelInsights(`model-insights-${sheetId}`, analysisData);
+ this.populateStatisticalInsights(`statistical-insights-${sheetId}`, analysisData);
+ this.populateRecommendations(`recommendations-${sheetId}`, analysisData);
+
+ // Populate retrieval quality tab
+ this.populateRetrievalQualityTab(sheetId, analysisData);
+
+ console.log('โ
Insights generation completed for sheet:', sheetId);
+ }
+
+ createSummaryRadarChart(canvasId, analysisData) {
+ const canvas = document.getElementById(canvasId);
+ if (!canvas) return;
+
+ const ctx = canvas.getContext('2d');
+ const metrics = Object.keys(analysisData.statistics);
+ const avgScores = metrics.map(metric => analysisData.statistics[metric].mean);
+
+ new Chart(ctx, {
+ type: 'radar',
+ data: {
+ labels: metrics.map(m => this.formatMetricName(m)),
+ datasets: [{
+ label: 'Average Scores',
+ data: avgScores,
+ backgroundColor: 'rgba(59, 130, 246, 0.1)',
+ borderColor: 'rgba(59, 130, 246, 0.8)',
+ borderWidth: 2,
+ pointBackgroundColor: 'rgba(59, 130, 246, 1)',
+ pointBorderColor: '#ffffff',
+ pointBorderWidth: 2,
+ pointRadius: 5
+ }]
+ },
+ options: {
+ responsive: true,
+ maintainAspectRatio: false,
+ plugins: {
+ legend: { display: false },
+ tooltip: {
+ callbacks: {
+ label: (context) => `${context.label}: ${(context.raw * 100).toFixed(1)}%`
+ }
+ }
+ },
+ scales: {
+ r: {
+ beginAtZero: true,
+ max: 1,
+ ticks: {
+ stepSize: 0.2,
+ callback: (value) => `${(value * 100).toFixed(0)}%`
+ }
+ }
+ }
+ }
+ });
+ }
+
+ createMetricRangeHistograms(containerId, analysisData) {
+ console.log('๐ Creating per-metric range histograms for container:', containerId);
+
+ const container = document.getElementById(containerId);
+ if (!container) {
+ console.error('โ Container element not found:', containerId);
+ return;
+ }
+
+ // Define score ranges (same as before)
+ const ranges = [
+ { label: 'Excellent', shortLabel: 'Exc', min: 0.9, max: 1.0, color: '#10b981' },
+ { label: 'Good', shortLabel: 'Good', min: 0.75, max: 0.9, color: '#059669' },
+ { label: 'Fair', shortLabel: 'Fair', min: 0.6, max: 0.75, color: '#f59e0b' },
+ { label: 'Poor', shortLabel: 'Poor', min: 0.4, max: 0.6, color: '#ef4444' },
+ { label: 'Very Poor', shortLabel: 'V.Poor', min: 0.0, max: 0.4, color: '#dc2626' }
+ ];
+
+ // Get metrics that have actual score data (not justification columns)
+ const metricsWithScores = Object.entries(analysisData.scores).filter(([metric, scores]) =>
+ !metric.includes('Justification') && !metric.includes('Test') && scores && scores.length > 0
+ );
+
+ console.log('๐ Metrics with scores:', metricsWithScores.map(([metric]) => metric));
+
+ if (metricsWithScores.length === 0) {
+ container.innerHTML = 'No metric data available
';
+ return;
+ }
+
+ // Create histogram container HTML with enhanced styles for full-width layout
+ let histogramHTML = `
+
+
+ `;
+
+ metricsWithScores.forEach(([metric, scores], metricIndex) => {
+ const cleanMetricName = metric.replace('LLM ', '').replace(' Relevancy', ' Rel.').replace(' Correctness', ' Corr.').replace(' Ground Truth Validity', ' GT Val.').replace(' Answer Completeness', ' Comp.').replace(' Justification', ' Just.');
+ const canvasId = `histogram-${containerId}-${metricIndex}`;
+
+ histogramHTML += `
+
+ ${cleanMetricName}
+
+ `;
+ });
+
+ histogramHTML += '
+ Overall Performance
+ ${(overallMean * 100).toFixed(1)}%
+
+
+ Best Metric
+ ${this.formatMetricName(bestMetric.metric)} (${(bestMetric.score * 100).toFixed(1)}%)
+
+
+ Needs Improvement
+ ${this.formatMetricName(worstMetric.metric)} (${(worstMetric.score * 100).toFixed(1)}%)
+
+
+ Score Consistency
+ ฯ = ${overallStdDev.toFixed(3)}
+
+
+ Total Evaluations
+ ${analysisData.rawData.length}
+
+
+ Metrics Analyzed
+ ${analysisData.metrics.length}
+
+ `;
+ }
+
+
+
+
+
+ formatMetricName(metric) {
+ return metric
+ .replace(/_/g, ' ')
+ .replace(/([A-Z])/g, ' $1')
+ .replace(/\b\w/g, l => l.toUpperCase())
+ .trim();
+ }
+
+
+
+ createHistogram(canvasId, scores, metricName) {
+ const canvas = document.getElementById(canvasId);
+ if (!canvas) return;
+
+ const ctx = canvas.getContext('2d');
+
+ // Create bins for histogram
+ const bins = 10;
+ const binSize = 1 / bins;
+ const binCounts = new Array(bins).fill(0);
+ const binLabels = [];
+
+ for (let i = 0; i < bins; i++) {
+ const start = i * binSize;
+ const end = (i + 1) * binSize;
+ binLabels.push(`${(start * 100).toFixed(0)}-${(end * 100).toFixed(0)}%`);
+ }
+
+ scores.forEach(score => {
+ const binIndex = Math.min(Math.floor(score / binSize), bins - 1);
+ binCounts[binIndex]++;
+ });
+
+ new Chart(ctx, {
+ type: 'bar',
+ data: {
+ labels: binLabels,
+ datasets: [{
+ data: binCounts,
+ backgroundColor: 'rgba(59, 130, 246, 0.6)',
+ borderColor: 'rgba(59, 130, 246, 0.8)',
+ borderWidth: 1,
+ borderRadius: 4
+ }]
+ },
+ options: {
+ responsive: true,
+ maintainAspectRatio: false,
+ plugins: {
+ legend: { display: false },
+ tooltip: {
+ callbacks: {
+ title: () => metricName,
+ label: (context) => `${context.raw} queries in ${context.label} range`
+ }
+ }
+ },
+ scales: {
+ y: {
+ beginAtZero: true,
+ ticks: { stepSize: 1 }
+ },
+ x: {
+ ticks: {
+ maxRotation: 45,
+ font: { size: 8 }
+ }
+ }
+ }
+ }
+ });
+ }
+
+ createCorrelationHeatmap(canvasId, analysisData) {
+ const canvas = document.getElementById(canvasId);
+ if (!canvas) return;
+
+ const ctx = canvas.getContext('2d');
+ const metrics = Object.keys(analysisData.statistics);
+
+ // Create correlation matrix data
+ const correlationMatrix = [];
+ const labels = metrics.map(m => this.formatMetricName(m));
+
+ for (let i = 0; i < metrics.length; i++) {
+ const row = [];
+ for (let j = 0; j < metrics.length; j++) {
+ if (i === j) {
+ row.push(1); // Perfect correlation with self
+ } else {
+ const key1 = `${metrics[i]}_${metrics[j]}`;
+ const key2 = `${metrics[j]}_${metrics[i]}`;
+ const correlation = analysisData.correlations[key1] || analysisData.correlations[key2] || 0;
+ row.push(correlation);
+ }
+ }
+ correlationMatrix.push(row);
+ }
+
+ // Create heatmap using scatter plot
+ const scatterData = [];
+ correlationMatrix.forEach((row, i) => {
+ row.forEach((correlation, j) => {
+ scatterData.push({
+ x: j,
+ y: metrics.length - 1 - i, // Flip Y axis
+ v: correlation
+ });
+ });
+ });
+
+ new Chart(ctx, {
+ type: 'scatter',
+ data: {
+ datasets: [{
+ label: 'Correlation',
+ data: scatterData,
+ backgroundColor: (context) => {
+ const correlation = context.raw.v;
+ const intensity = Math.abs(correlation);
+ const hue = correlation >= 0 ? 120 : 0; // Green for positive, red for negative
+ return `hsla(${hue}, 70%, 50%, ${intensity})`;
+ },
+ pointRadius: 15,
+ pointHoverRadius: 18
+ }]
+ },
+ options: {
+ responsive: true,
+ maintainAspectRatio: false,
+ plugins: {
+ legend: { display: false },
+ tooltip: {
+ callbacks: {
+ title: () => 'Metric Correlation',
+ label: (context) => {
+ const i = metrics.length - 1 - context.raw.y;
+ const j = context.raw.x;
+ return `${labels[i]} โ ${labels[j]}: ${context.raw.v.toFixed(3)}`;
+ }
+ }
+ }
+ },
+ scales: {
+ x: {
+ type: 'linear',
+ position: 'bottom',
+ min: -0.5,
+ max: metrics.length - 0.5,
+ ticks: {
+ stepSize: 1,
+ callback: (value) => labels[Math.round(value)] || ''
+ }
+ },
+ y: {
+ type: 'linear',
+ min: -0.5,
+ max: metrics.length - 0.5,
+ ticks: {
+ stepSize: 1,
+ callback: (value) => labels[metrics.length - 1 - Math.round(value)] || ''
+ }
+ }
+ }
+ }
+ });
+ }
+
+ createScatterMatrix(canvasId, analysisData) {
+ const canvas = document.getElementById(canvasId);
+ if (!canvas) return;
+
+ const ctx = canvas.getContext('2d');
+ const metrics = Object.keys(analysisData.scores);
+
+ if (metrics.length < 2) {
+ canvas.parentElement.innerHTML = 'Need at least 2 metrics for scatter analysis
';
+ return;
+ }
+
+ // Create scatter plot for the two most correlated metrics
+ const correlations = Object.entries(analysisData.correlations);
+ const strongestCorrelation = correlations.reduce((strongest, [pair, correlation]) =>
+ Math.abs(correlation) > Math.abs(strongest.correlation) ? { pair, correlation } : strongest,
+ { pair: '', correlation: 0 }
+ );
+
+ if (!strongestCorrelation.pair) return;
+
+ const [metric1, metric2] = strongestCorrelation.pair.split('_');
+ const scores1 = analysisData.scores[metric1];
+ const scores2 = analysisData.scores[metric2];
+
+ if (!scores1 || !scores2) return;
+
+ const scatterData = scores1.map((score1, index) => ({
+ x: score1,
+ y: scores2[index] || 0
+ })).filter(point => point.y !== 0);
+
+ new Chart(ctx, {
+ type: 'scatter',
+ data: {
+ datasets: [{
+ label: `${this.formatMetricName(metric1)} vs ${this.formatMetricName(metric2)}`,
+ data: scatterData,
+ backgroundColor: 'rgba(59, 130, 246, 0.6)',
+ borderColor: 'rgba(59, 130, 246, 0.8)',
+ borderWidth: 1,
+ pointRadius: 4,
+ pointHoverRadius: 6
+ }]
+ },
+ options: {
+ responsive: true,
+ maintainAspectRatio: false,
+ plugins: {
+ legend: { display: false },
+ tooltip: {
+ callbacks: {
+ label: (context) => {
+ return `${this.formatMetricName(metric1)}: ${(context.raw.x * 100).toFixed(1)}%, ${this.formatMetricName(metric2)}: ${(context.raw.y * 100).toFixed(1)}%`;
+ }
+ }
+ }
+ },
+ scales: {
+ x: {
+ title: {
+ display: true,
+ text: this.formatMetricName(metric1)
+ },
+ min: 0,
+ max: 1,
+ ticks: {
+ callback: (value) => `${(value * 100).toFixed(0)}%`
+ }
+ },
+ y: {
+ title: {
+ display: true,
+ text: this.formatMetricName(metric2)
+ },
+ min: 0,
+ max: 1,
+ ticks: {
+ callback: (value) => `${(value * 100).toFixed(0)}%`
+ }
+ }
+ }
+ }
+ });
+ }
+
+ populateCorrelationInsights(elementId, analysisData) {
+ const element = document.getElementById(elementId);
+ if (!element) return;
+
+ const correlations = Object.entries(analysisData.correlations);
+ const strongCorrelations = correlations.filter(([_, corr]) => Math.abs(corr) > 0.5);
+
+ let content = '';
+
+ if (strongCorrelations.length === 0) {
+ content = `
+
+ ๐ Weak Correlations: No strong correlations detected between metrics.
+ This suggests each metric captures unique aspects of performance.
+
+ `;
+ } else {
+ content = ``;
+ strongCorrelations.forEach(([pair, correlation]) => {
+ const [metric1, metric2] = pair.split('_');
+ const strength = Math.abs(correlation) > 0.8 ? 'very strong' : 'strong';
+ const direction = correlation > 0 ? 'positive' : 'negative';
+
+ content += `
+
+ ๐ ${strength.charAt(0).toUpperCase() + strength.slice(1)} ${direction} correlation:
+ ${this.formatMetricName(metric1)} and ${this.formatMetricName(metric2)}
+ (r = ${correlation.toFixed(3)})
+
+ `;
+ });
+ content += `
+ ${totalQueries} queries totalPage 1 of ${totalPages}
+
+
+
+ โ Previous
+
+
+ Next โ
+
+
+ `;
+
+ // Add CSS for navigation buttons
+ const style = document.createElement('style');
+ style.textContent = `
+ .btn-heatmap {
+ background: #3b82f6;
+ color: white;
+ border: none;
+ padding: 6px 12px;
+ border-radius: 4px;
+ cursor: pointer;
+ margin: 0 2px;
+ font-size: 12px;
+ }
+ .btn-heatmap:hover:not(:disabled) {
+ background: #2563eb;
+ }
+ .btn-heatmap:disabled {
+ background: #9ca3af;
+ cursor: not-allowed;
+ }
+ `;
+ document.head.appendChild(style);
+
+ container.insertBefore(controlsDiv, container.firstChild);
+
+ // Render initial page
+ const canvas = document.getElementById(canvasId);
+ this.renderHeatmapPage(canvas, analysisData, currentPage, queriesPerPage);
+
+ // Add navigation event listeners
+ document.getElementById(`prev-${canvasId}`).addEventListener('click', () => {
+ if (currentPage > 0) {
+ currentPage--;
+ this.renderHeatmapPage(canvas, analysisData, currentPage, queriesPerPage);
+ this.updateNavigationState(canvasId, currentPage, totalPages);
+ }
+ });
+
+ document.getElementById(`next-${canvasId}`).addEventListener('click', () => {
+ if (currentPage < totalPages - 1) {
+ currentPage++;
+ this.renderHeatmapPage(canvas, analysisData, currentPage, queriesPerPage);
+ this.updateNavigationState(canvasId, currentPage, totalPages);
+ }
+ });
+ }
+
+ updateNavigationState(canvasId, currentPage, totalPages) {
+ document.getElementById(`current-page-${canvasId}`).textContent = currentPage + 1;
+ document.getElementById(`prev-${canvasId}`).disabled = currentPage === 0;
+ document.getElementById(`next-${canvasId}`).disabled = currentPage === totalPages - 1;
+ }
+
+ renderHeatmapPage(canvas, analysisData, currentPage, queriesPerPage) {
+ const startIdx = currentPage * queriesPerPage;
+ const endIdx = Math.min(startIdx + queriesPerPage, analysisData.rawData.length);
+ const pageQueries = endIdx - startIdx;
+
+ // Create subset of data for current page
+ const pageData = analysisData.rawData.slice(startIdx, endIdx);
+ const pageAnalysisData = { ...analysisData, rawData: pageData };
+
+ this.renderFullHeatmap(canvas, pageAnalysisData, pageQueries, startIdx);
+ }
+
+ renderFullHeatmap(canvas, analysisData, maxQueries, startOffset = 0) {
+ const ctx = canvas.getContext('2d');
+ const metrics = analysisData.metrics;
+
+ // Clear any existing chart
+ if (canvas.chart) {
+ canvas.chart.destroy();
+ }
+
+ // Adaptive point sizing based on data density
+ const pointRadius = this.calculatePointRadius(maxQueries, metrics.length);
+
+ // Prepare data for heatmap
+ const heatmapData = [];
+ for (let queryIdx = 0; queryIdx < maxQueries; queryIdx++) {
+ metrics.forEach((metric, metricIdx) => {
+ const score = parseFloat(analysisData.rawData[queryIdx][metric]);
+ if (!isNaN(score)) {
+ heatmapData.push({
+ x: metricIdx,
+ y: queryIdx,
+ v: score,
+ actualQueryIdx: startOffset + queryIdx // Track actual query index
+ });
+ }
+ });
+ }
+
+ const chart = new Chart(ctx, {
+ type: 'scatter',
+ data: {
+ datasets: [{
+ label: 'Query Performance',
+ data: heatmapData,
+ backgroundColor: (context) => {
+ const score = context.raw.v;
+ if (score >= 0.8) return 'rgba(16, 185, 129, 0.8)'; // Green
+ if (score >= 0.6) return 'rgba(245, 158, 11, 0.8)'; // Yellow
+ return 'rgba(239, 68, 68, 0.8)'; // Red
+ },
+ pointRadius: pointRadius,
+ pointHoverRadius: Math.min(pointRadius + 2, 12)
+ }]
+ },
+ options: {
+ responsive: true,
+ maintainAspectRatio: false,
+ plugins: {
+ legend: { display: false },
+ tooltip: {
+ callbacks: {
+ title: () => 'Query Performance',
+ label: (context) => {
+ const actualQueryIdx = context.raw.actualQueryIdx || context.raw.y;
+ const metricIdx = context.raw.x;
+ const metric = metrics[metricIdx];
+ return `Query ${actualQueryIdx + 1} - ${this.formatMetricName(metric)}: ${(context.raw.v * 100).toFixed(1)}%`;
+ }
+ }
+ }
+ },
+ scales: {
+ x: {
+ type: 'linear',
+ min: -0.5,
+ max: metrics.length - 0.5,
+ ticks: {
+ stepSize: 1,
+ callback: (value) => {
+ const metric = metrics[Math.round(value)];
+ return metric ? this.formatMetricName(metric).substring(0, 8) + '...' : '';
+ },
+ maxRotation: 45,
+ font: { size: Math.max(8, 12 - Math.floor(metrics.length / 5)) }
+ },
+ title: {
+ display: true,
+ text: 'Metrics'
+ }
+ },
+ y: {
+ type: 'linear',
+ min: -0.5,
+ max: maxQueries - 0.5,
+ ticks: {
+ stepSize: Math.max(1, Math.floor(maxQueries / 10)),
+ callback: (value) => {
+ const actualQueryIdx = startOffset + Math.round(value);
+ return `Q${actualQueryIdx + 1}`;
+ },
+ font: { size: Math.max(8, 12 - Math.floor(maxQueries / 20)) }
+ },
+ title: {
+ display: true,
+ text: 'Queries'
+ }
+ }
+ }
+ }
+ });
+
+ // Store chart reference for cleanup
+ canvas.chart = chart;
+ }
+
+ calculatePointRadius(queryCount, metricCount) {
+ // Adaptive point sizing based on density
+ const density = queryCount * metricCount;
+ if (density > 1000) return 3; // Very dense
+ if (density > 500) return 4; // Dense
+ if (density > 250) return 6; // Medium
+ return 8; // Sparse
+ }
+
+ createTopBottomChart(canvasId, analysisData) {
+ const canvas = document.getElementById(canvasId);
+ if (!canvas) return;
+
+ const ctx = canvas.getContext('2d');
+ const topPerformers = analysisData.performance.topPerformers.slice(0, 5);
+ const bottomPerformers = analysisData.performance.bottomPerformers.slice(0, 5);
+
+ const allQueries = [...topPerformers, ...bottomPerformers];
+ const labels = allQueries.map((query, idx) =>
+ idx < 5 ? `Top ${idx + 1}` : `Bottom ${idx - 4}`
+ );
+ const scores = allQueries.map(query => query.averageScore);
+ const colors = allQueries.map((_, idx) =>
+ idx < 5 ? '#10b981' : '#ef4444'
+ );
+
+ new Chart(ctx, {
+ type: 'bar',
+ data: {
+ labels: labels,
+ datasets: [{
+ data: scores,
+ backgroundColor: colors,
+ borderRadius: 4,
+ borderSkipped: false
+ }]
+ },
+ options: {
+ responsive: true,
+ maintainAspectRatio: false,
+ plugins: {
+ legend: { display: false },
+ tooltip: {
+ callbacks: {
+ title: (context) => {
+ const query = allQueries[context[0].dataIndex];
+ return query.query.substring(0, 50) + '...';
+ },
+ label: (context) => `Average Score: ${(context.raw * 100).toFixed(1)}%`
+ }
+ }
+ },
+ scales: {
+ y: {
+ beginAtZero: true,
+ max: 1,
+ ticks: {
+ callback: (value) => `${(value * 100).toFixed(0)}%`
+ }
+ }
+ }
+ }
+ });
+ }
+
+ populateOutliersAnalysis(elementId, analysisData) {
+ const element = document.getElementById(elementId);
+ if (!element) return;
+
+ const outliers = analysisData.performance.outliers;
+
+ if (outliers.length === 0) {
+ element.innerHTML = `
+
+ โ
No outliers detected: All queries show consistent performance patterns.
+
+ `;
+ return;
+ }
+
+ let content = `
+
+ ๐ฏ ${outliers.length} outlier queries detected:
+
+ `;
+
+ outliers.slice(0, 5).forEach((outlier, idx) => {
+ const performance = outlier.averageScore >= 0.8 ? 'exceptionally high' : 'unusually low';
+ const scoreColor = outlier.averageScore >= 0.8 ? 'excellent' : 'poor';
+
+ content += `
+
+ `;
+ });
+
+ if (outliers.length > 5) {
+ content += `... and ${outliers.length - 5} more outliers
`;
+ }
+
+ element.innerHTML = content;
+ }
+
+ populateModelInsights(elementId, analysisData) {
+ const element = document.getElementById(elementId);
+ if (!element) {
+ console.error('โ Element not found:', elementId);
+ return;
+ }
+
+ const insights = analysisData.insights;
+ console.log('๐ฏ Populating model insights for element:', elementId);
+ console.log('๐ Available insights:', insights);
+ let content = '';
+
+ if (insights.strengths.length > 0) {
+ content += `๐ช Strengths:
`;
+ insights.strengths.forEach(strength => {
+ content += `${strength}
`;
+ });
+ }
+
+ if (insights.weaknesses.length > 0) {
+ content += `โ ๏ธ Areas for Improvement:
`;
+ insights.weaknesses.forEach(weakness => {
+ content += `${weakness}
`;
+ });
+ }
+
+ // Add new insight categories
+ console.log('๐ Checking retrieval insights:', insights.retrieval);
+ if (insights.retrieval && insights.retrieval.length > 0) {
+ content += `๐ Retrieval Quality:
`;
+ insights.retrieval.forEach(retrieval => {
+ content += `${retrieval}
`;
+ });
+ }
+
+ console.log('โก Checking efficiency insights:', insights.efficiency);
+ if (insights.efficiency && insights.efficiency.length > 0) {
+ content += `โก System Efficiency:
`;
+ insights.efficiency.forEach(efficiency => {
+ content += `${efficiency}
`;
+ });
+ }
+
+ console.log('๐ฏ Checking utilization insights:', insights.utilization);
+ if (insights.utilization && insights.utilization.length > 0) {
+ content += `๐ฏ Chunk Utilization:
`;
+ insights.utilization.forEach(utilization => {
+ content += `${utilization}
`;
+ });
+ }
+
+ element.innerHTML = content;
+
+ // Log summary of what was generated
+ const insightCounts = {
+ strengths: insights.strengths ? insights.strengths.length : 0,
+ weaknesses: insights.weaknesses ? insights.weaknesses.length : 0,
+ retrieval: insights.retrieval ? insights.retrieval.length : 0,
+ efficiency: insights.efficiency ? insights.efficiency.length : 0,
+ utilization: insights.utilization ? insights.utilization.length : 0
+ };
+ console.log('๐ Insight counts generated for', elementId, ':', insightCounts);
+ }
+
+ populateRetrievalQualityTab(sheetId, analysisData) {
+ console.log('๐ Populating retrieval quality tab for sheet:', sheetId);
+
+ // Populate retrieval overview
+ this.populateRetrievalOverview(`retrieval-overview-${sheetId}`, analysisData);
+
+ // Charts removed - Ground Truth Validity and Answer Completeness pie charts
+
+ // Populate insight sections
+ this.populateRetrievalInsights(`retrieval-insights-${sheetId}`, analysisData);
+ this.populateEfficiencyInsights(`efficiency-insights-${sheetId}`, analysisData);
+ this.populateUtilizationInsights(`utilization-insights-${sheetId}`, analysisData);
+
+ // Create chunk utilization charts
+ this.createChunkUtilizationChart(`chunk-utilization-chart-${sheetId}`, analysisData);
+
+ console.log('โ
Retrieval quality tab populated for sheet:', sheetId);
+ }
+
+ populateRetrievalOverview(elementId, analysisData) {
+ const element = document.getElementById(elementId);
+ if (!element) {
+ console.error('โ Retrieval overview element not found:', elementId);
+ return;
+ }
+
+ const extendedMetrics = this.extractExtendedMetrics(analysisData);
+ const hasChunkData = this.checkForChunkData(analysisData);
+
+ let content = `
+
+
+ `;
+
+ // Add available metrics
+ if (extendedMetrics.ground_truth_validity !== undefined) {
+ const gtValidity = extendedMetrics.ground_truth_validity;
+ const gtClass = gtValidity >= 0.9 ? 'excellent' : gtValidity >= 0.8 ? 'good' : gtValidity >= 0.7 ? 'fair' : 'poor';
+ content += `
+
+
โ
+
Ground Truth Validity
+
${(gtValidity * 100).toFixed(1)}%
+
${gtValidity >= 0.9 ? 'Excellent' : gtValidity >= 0.8 ? 'Good' : gtValidity >= 0.7 ? 'Fair' : 'Poor'}
+
+ `;
+ }
+
+ if (extendedMetrics.answer_completeness !== undefined) {
+ const completeness = extendedMetrics.answer_completeness;
+ const compClass = completeness >= 0.9 ? 'excellent' : completeness >= 0.8 ? 'good' : completeness >= 0.7 ? 'fair' : 'poor';
+ content += `
+
+
๐ฏ
+
Answer Completeness
+
${(completeness * 100).toFixed(1)}%
+
${completeness >= 0.9 ? 'Excellent' : completeness >= 0.8 ? 'Good' : completeness >= 0.7 ? 'Fair' : 'Poor'}
+
+ `;
+ }
+
+ // Add chunk metrics if available
+ if (hasChunkData) {
+ if (extendedMetrics.retrieved_chunk_count !== undefined) {
+ content += `
+
+
๐
+
Retrieved Chunks
+
${extendedMetrics.retrieved_chunk_count.toFixed(1)}
+
Chunks
+
+ `;
+ }
+
+ if (extendedMetrics.sent_to_llm_chunk_count !== undefined) {
+ content += `
+
+
๐ค
+
Sent to LLM
+
${extendedMetrics.sent_to_llm_chunk_count.toFixed(1)}
+
Chunks
+
+ `;
+ }
+
+ if (extendedMetrics.used_in_answer_chunk_count !== undefined) {
+ content += `
+
+
๐ฏ
+
Used in Answer
+
${extendedMetrics.used_in_answer_chunk_count.toFixed(1)}
+
Chunks
+
+ `;
+ }
+
+ if (extendedMetrics.total_chunks_used !== undefined) {
+ content += `
+
+
๐
+
Total Used
+
${extendedMetrics.total_chunks_used.toFixed(1)}
+
Chunks
+
+ `;
+ }
+
+ if (extendedMetrics.best_support_rank !== undefined) {
+ const rank = extendedMetrics.best_support_rank;
+ const rankClass = rank <= 3 ? 'excellent' : rank <= 10 ? 'good' : 'fair';
+ content += `
+
+
๐
+
Best Support Rank
+
${rank.toFixed(1)}
+
${rank <= 3 ? 'Excellent' : rank <= 10 ? 'Good' : 'Fair'}
+
+ `;
+ }
+ }
+
+ content += `
+
+
+
+ ${hasChunkData ? 'โ
' : 'โ ๏ธ'}
+ ${hasChunkData ? 'Comprehensive chunk analysis available' : 'Basic retrieval analysis - enable chunk tracking for detailed insights'}
+
+ ${!hasChunkData ? `
+
+
+ How to enable chunk analysis
+
+
To get detailed chunk analysis:
+
+ Check "Use Search API" in the configuration
+ Ensure your RAG system returns chunk metadata
+ Look for columns like "Retrieved Chunk Count", "Total Chunks Used" in results
+ Chunk analysis provides insights into retrieval efficiency and quality
+
+
Available columns in your data:
+
${analysisData.rawData && analysisData.rawData.length > 0 ? Object.keys(analysisData.rawData[0]).join(', ') : 'No data available'}
+
+
+ ` : ''}
+
+
';
+
+ if (insights.retrieval && insights.retrieval.length > 0) {
+ insights.retrieval.forEach(insight => {
+ content += `
${insight}
`;
+ });
+ } else {
+ content += '
No retrieval quality insights available
';
+ }
+
+ // Add detailed metric analysis
+ if (extendedMetrics.ground_truth_validity !== undefined) {
+ const gtValidity = extendedMetrics.ground_truth_validity;
+ content += `
+
+
Ground Truth Validity Analysis
+
+
+ Score:
+ ${(gtValidity * 100).toFixed(1)}%
+
+
+ Performance Level:
+ ${gtValidity >= 0.9 ? 'Excellent' : gtValidity >= 0.8 ? 'Good' : gtValidity >= 0.7 ? 'Fair' : 'Poor'}
+
+
+ Interpretation:
+ ${this.getGTValidityInterpretation(gtValidity)}
+
+
+
+ `;
+ }
+
+ content += '
';
+
+ if (insights.efficiency && insights.efficiency.length > 0) {
+ insights.efficiency.forEach(insight => {
+ content += `
${insight}
`;
+ });
+ } else {
+ content += '
No efficiency insights available
';
+ }
+
+ // Add efficiency score calculation
+ if (extendedMetrics.ground_truth_validity !== undefined && extendedMetrics.answer_completeness !== undefined) {
+ const gtValidity = extendedMetrics.ground_truth_validity;
+ const completeness = extendedMetrics.answer_completeness;
+ const efficiencyScore = (gtValidity + completeness) / 2;
+ const gap = Math.abs(gtValidity - completeness);
+
+ content += `
+
+
System Efficiency Analysis
+
+
+ Overall Efficiency Score:
+ ${(efficiencyScore * 100).toFixed(1)}%
+
+
+ Performance Gap:
+ ${(gap * 100).toFixed(1)}%
+
+
+ Balance Assessment:
+ ${this.getEfficiencyBalanceAssessment(gtValidity, completeness)}
+
+
+
+ `;
+ }
+
+ content += '
';
+
+ if (insights.utilization && insights.utilization.length > 0) {
+ insights.utilization.forEach(insight => {
+ content += `
${insight}
`;
+ });
+ } else {
+ content += '
No utilization insights available
';
+ }
+
+ // Add detailed chunk utilization analysis
+ if (extendedMetrics.chunks_used_top5 !== undefined ||
+ extendedMetrics.chunks_used_top10 !== undefined ||
+ extendedMetrics.chunks_used_top20 !== undefined) {
+
+ content += `
+
+
Chunk Utilization Distribution Analysis
+
+ `;
+
+ if (extendedMetrics.chunks_used_top5 !== undefined) {
+ const top5 = extendedMetrics.chunks_used_top5;
+ content += `
+
+ Top 5 Chunks Used:
+ ${top5} chunks
+ ${this.getChunkUtilizationAssessment(top5, 'top5')}
+
+ `;
+ }
+
+ if (extendedMetrics.chunks_used_top10 !== undefined) {
+ const top10 = extendedMetrics.chunks_used_top10;
+ content += `
+
+ Chunks 5-10 Used:
+ ${top10} chunks
+ ${this.getChunkUtilizationAssessment(top10, 'top10')}
+
+ `;
+ }
+
+ if (extendedMetrics.chunks_used_top20 !== undefined) {
+ const top20 = extendedMetrics.chunks_used_top20;
+ content += `
+
+ Chunks 10-20 Used:
+ ${top20} chunks
+ ${this.getChunkUtilizationAssessment(top20, 'top20')}
+
+ `;
+ }
+
+ // Add utilization efficiency analysis
+ if (extendedMetrics.retrieved_chunk_count !== undefined &&
+ extendedMetrics.used_in_answer_chunk_count !== undefined) {
+ const retrieved = extendedMetrics.retrieved_chunk_count;
+ const used = extendedMetrics.used_in_answer_chunk_count;
+ const efficiency = retrieved > 0 ? (used / retrieved) * 100 : 0;
+
+ content += `
+
+
+
+
+ Retrieved:
+ ${retrieved.toFixed(1)} chunks
+
+
+ Used:
+ ${used.toFixed(1)} chunks
+
+
+ Efficiency:
+ ${efficiency.toFixed(1)}%
+
+
+
+ `;
+ }
+
+ content += `
+
+
+ `;
+ }
+
+ content += '
+
๐
+
Chunk utilization data not available
+
Enable chunk tracking for detailed utilization analysis
+
${stat}
`;
+ });
+ } else {
+ content = `๐ Standard statistical patterns detected across all metrics.
`;
+ }
+
+ element.innerHTML = content;
+ }
+
+ populateRecommendations(elementId, analysisData) {
+ const element = document.getElementById(elementId);
+ if (!element) return;
+
+ const insights = analysisData.insights;
+ let content = '';
+
+ if (insights.recommendations && insights.recommendations.length > 0) {
+ // Add a header explaining the evaluation-based recommendations
+ content += `
+
+ `;
+
+ insights.recommendations.forEach((recommendation, index) => {
+ // Extract emoji and text for better formatting
+ const emojiMatch = recommendation.match(/^([\u{1F300}-\u{1F9FF}][\u{FE00}-\u{FE0F}]?|[\u{2600}-\u{27BF}])/u);
+ const emoji = emojiMatch ? emojiMatch[0] : '๐ก';
+ const text = recommendation.replace(/^[\u{1F300}-\u{1F9FF}][\u{FE00}-\u{FE0F}]?[\u{2600}-\u{27BF}]?\s*/u, '');
+
+ // Determine priority class based on emoji
+ let priorityClass = 'recommendation-normal';
+ if (emoji === 'โ
') priorityClass = 'recommendation-success';
+ else if (emoji === 'โ' || emoji === '๐') priorityClass = 'recommendation-critical';
+ else if (emoji === '๐ ๏ธ' || emoji === '๐งช') priorityClass = 'recommendation-action';
+
+ content += `
+
+
+ ${emoji}
+ ${text}
+
+
+ ๐ Based on: `;
+
+ const metricDisplay = [];
+ if (metricValues.context_relevancy !== undefined) {
+ metricDisplay.push(`Context Relevancy: ${(metricValues.context_relevancy * 100).toFixed(1)}%`);
+ }
+ if (metricValues.answer_correctness !== undefined) {
+ metricDisplay.push(`Answer Correctness: ${(metricValues.answer_correctness * 100).toFixed(1)}%`);
+ }
+ if (metricValues.ground_truth_validity !== undefined) {
+ metricDisplay.push(`GT Validity: ${(metricValues.ground_truth_validity * 100).toFixed(1)}%`);
+ }
+ if (metricValues.answer_completeness !== undefined) {
+ metricDisplay.push(`Completeness: ${(metricValues.answer_completeness * 100).toFixed(1)}%`);
+ }
+ if (metricValues.retrieved_chunk_count !== undefined) {
+ metricDisplay.push(`Retrieved: ${metricValues.retrieved_chunk_count.toFixed(1)}`);
+ }
+ if (metricValues.sent_to_llm_chunk_count !== undefined) {
+ metricDisplay.push(`Sent to LLM: ${metricValues.sent_to_llm_chunk_count.toFixed(1)}`);
+ }
+ if (metricValues.used_in_answer_chunk_count !== undefined) {
+ metricDisplay.push(`Used: ${metricValues.used_in_answer_chunk_count.toFixed(1)}`);
+ }
+ if (metricValues.total_chunks_used !== undefined) {
+ metricDisplay.push(`Total Used: ${metricValues.total_chunks_used.toFixed(1)}`);
+ }
+ if (metricValues.best_support_rank !== undefined) {
+ metricDisplay.push(`Support Rank: ${metricValues.best_support_rank.toFixed(1)}`);
+ }
+
+ content += metricDisplay.join(', ');
+ content += `
+
+ `;
+ }
+ } else {
+ content = `
+
+
+
+ ๐
+ Performance appears balanced across metrics.
+
+
+
+ ๐
+ Consider running evaluation with larger dataset for clearer insights.
+
+
+
+ ๐
+ Monitor performance trends over time to identify patterns.
+
+
"),
+ "dbName": os.getenv("DB_NAME", ""),
+ "collectionName": os.getenv("COLLECTION_NAME", "")
+ }
+}
diff --git a/Evaluation/RAG_Evaluator/src/utils/evaluationResult.py b/Evaluation/RAG_Evaluator/src/utils/evaluationResult.py
index 62bd16d1..05b1de72 100644
--- a/Evaluation/RAG_Evaluator/src/utils/evaluationResult.py
+++ b/Evaluation/RAG_Evaluator/src/utils/evaluationResult.py
@@ -2,25 +2,40 @@
class ResultsConverter:
- def __init__(self, ragas_results: pd.DataFrame, crag_results: pd.DataFrame):
+ def __init__(self, ragas_results: pd.DataFrame, crag_results: pd.DataFrame, llm_results: pd.DataFrame = None):
self.ragas_results = ragas_results
self.crag_results = crag_results
+ self.llm_results = llm_results if llm_results is not None else pd.DataFrame()
def convert_ragas_results(self):
- """Converts column name 'question' to 'query' in the Ragas Results DataFrame."""
- if 'question' in self.ragas_results.columns:
- self.ragas_results.rename(columns={'question': 'query'}, inplace=True)
- print("Converted 'question' to 'query' in Ragas results.")
- else:
- print("'question' column not found in Ragas results.")
+ """Converts column names in the Ragas Results DataFrame for consistency."""
+ # Handle common RAGAS column name mappings
+ column_mappings = {
+ 'question': 'query',
+ 'user_input': 'query',
+ 'response': 'answer',
+ 'retrieved_contexts': 'context',
+ 'reference': 'ground_truth'
+ }
+
+ # Apply mappings if columns exist
+ columns_to_rename = {}
+ for old_name, new_name in column_mappings.items():
+ if old_name in self.ragas_results.columns:
+ columns_to_rename[old_name] = new_name
+
+ if columns_to_rename:
+ self.ragas_results.rename(columns=columns_to_rename, inplace=True)
def convert_crag_results(self):
"""Converts column name 'prediction' to 'answer' in the CRAG Results DataFrame."""
if 'prediction' in self.crag_results.columns:
self.crag_results.rename(columns={'prediction': 'answer'}, inplace=True)
- print("Converted 'prediction' to 'answer' in CRAG results.")
- else:
- print("'prediction' column not found in CRAG results.")
+
+ def convert_llm_results(self):
+ """Converts column names in the LLM Results DataFrame for consistency."""
+ # LLM results should already have standardized column names
+ pass
def get_crag_results(self):
return self.crag_results
@@ -28,14 +43,28 @@ def get_crag_results(self):
def get_ragas_results(self):
return self.ragas_results
+ def get_llm_results(self):
+ return self.llm_results
+
def get_combined_results(self):
- """Combines the converted Ragas results and CRAG results DataFrames."""
- # Assuming both DataFrames have the same number of rows
- if len(self.ragas_results) != len(self.crag_results):
- print("Warning: Ragas and CRAG results have different row counts. Combining may lead to misalignment.")
+ """Combines the converted Ragas, CRAG, and LLM results DataFrames."""
+ dfs_to_combine = []
+
+ # Add non-empty DataFrames to combination list
+ if not self.ragas_results.empty:
+ dfs_to_combine.append(self.ragas_results.reset_index(drop=True))
+
+ if not self.crag_results.empty:
+ dfs_to_combine.append(self.crag_results.reset_index(drop=True))
+
+ if not self.llm_results.empty:
+ dfs_to_combine.append(self.llm_results.reset_index(drop=True))
+
+ if not dfs_to_combine:
+ return pd.DataFrame()
# Combine the DataFrames
- combined_results = pd.concat([self.ragas_results.reset_index(drop=True), self.crag_results.reset_index(drop=True)], axis=1)
+ combined_results = pd.concat(dfs_to_combine, axis=1)
# Remove duplicate columns while keeping the first occurrence
combined_results = combined_results.loc[:, ~combined_results.columns.duplicated()]
@@ -43,20 +72,3 @@ def get_combined_results(self):
return combined_results
-# Example usage:
-if __name__ == "__main__":
- # Sample DataFrames (replace these with actual DataFrames in practice)
- ragas_results = pd.DataFrame({'question': ['What is AI?', 'Define ML.'], 'other_col': [1, 2]})
- crag_results = pd.DataFrame({'prediction': ['AI is a field.', 'ML is a subset of AI.'], 'another_col': [3, 4]})
-
- converter = ResultsConverter(ragas_results, crag_results)
-
- # Convert columns
- converter.convert_ragas_results()
- converter.convert_crag_results()
-
- # Get combined results
- combined_results = converter.get_combined_results()
-
- print("\nCombined Results:")
- print(combined_results)
\ No newline at end of file
diff --git a/Evaluation/RAG_Evaluator/src/utils/sessionManager.py b/Evaluation/RAG_Evaluator/src/utils/sessionManager.py
new file mode 100644
index 00000000..2588431a
--- /dev/null
+++ b/Evaluation/RAG_Evaluator/src/utils/sessionManager.py
@@ -0,0 +1,244 @@
+# Session Manager for User-level File Separation
+# This module handles unique session IDs and file mapping for multi-user support
+
+import uuid
+import time
+import json
+import os
+import threading
+from datetime import datetime, timedelta
+from typing import Dict, Optional, List
+import logging
+
+logger = logging.getLogger(__name__)
+
+class SessionManager:
+ def __init__(self, base_output_dir: str = "outputs"):
+ self.base_output_dir = base_output_dir
+ self.sessions_file = os.path.join(base_output_dir, "sessions.json")
+ self.sessions: Dict[str, dict] = {}
+ self.lock = threading.RLock()
+ self.load_sessions()
+
+ # Create base output directory if it doesn't exist
+ os.makedirs(base_output_dir, exist_ok=True)
+
+ def create_session(self) -> str:
+ """Create a new user session and return session ID"""
+ with self.lock:
+ session_id = str(uuid.uuid4())
+ timestamp = datetime.now().isoformat()
+
+ session_data = {
+ "session_id": session_id,
+ "created_at": timestamp,
+ "last_accessed": timestamp,
+ "output_files": [],
+ "status": "active",
+ "user_ip": None, # Can be set later
+ "user_agent": None # Can be set later
+ }
+
+ self.sessions[session_id] = session_data
+
+ # Create user-specific directory
+ session_dir = self.get_session_directory(session_id)
+ os.makedirs(session_dir, exist_ok=True)
+
+ self.save_sessions()
+ logger.info(f"Created new session: {session_id}")
+ return session_id
+
+ def get_session_directory(self, session_id: str) -> str:
+ """Get the directory path for a specific session"""
+ return os.path.join(self.base_output_dir, f"session_{session_id}")
+
+ def add_output_file(self, session_id: str, file_path: str) -> bool:
+ """Add an output file to a session"""
+ with self.lock:
+ if session_id not in self.sessions:
+ logger.error(f"Session not found: {session_id}")
+ return False
+
+ # Update last accessed time
+ self.sessions[session_id]["last_accessed"] = datetime.now().isoformat()
+
+ # Add file to session
+ if file_path not in self.sessions[session_id]["output_files"]:
+ self.sessions[session_id]["output_files"].append({
+ "file_path": file_path,
+ "created_at": datetime.now().isoformat(),
+ "file_size": os.path.getsize(file_path) if os.path.exists(file_path) else 0
+ })
+
+ self.save_sessions()
+ logger.info(f"Added output file to session {session_id}: {file_path}")
+ return True
+
+ return False
+
+ def get_session_files(self, session_id: str) -> List[dict]:
+ """Get all output files for a session"""
+ with self.lock:
+ if session_id not in self.sessions:
+ return []
+
+ # Update last accessed time
+ self.sessions[session_id]["last_accessed"] = datetime.now().isoformat()
+ self.save_sessions()
+
+ return self.sessions[session_id]["output_files"]
+
+ def get_latest_file(self, session_id: str) -> Optional[str]:
+ """Get the most recent output file for a session"""
+ files = self.get_session_files(session_id)
+ if not files:
+ return None
+
+ # Sort by creation time and return the latest
+ latest_file = max(files, key=lambda f: f["created_at"])
+ file_path = latest_file["file_path"]
+
+ # Verify file still exists
+ if os.path.exists(file_path):
+ return file_path
+ else:
+ logger.warning(f"File not found: {file_path}")
+ return None
+
+ def is_valid_session(self, session_id: str) -> bool:
+ """Check if a session ID is valid and active"""
+ with self.lock:
+ return session_id in self.sessions and self.sessions[session_id]["status"] == "active"
+
+ def update_session_metadata(self, session_id: str, user_ip: str = None, user_agent: str = None):
+ """Update session metadata with user information"""
+ with self.lock:
+ if session_id in self.sessions:
+ if user_ip:
+ self.sessions[session_id]["user_ip"] = user_ip
+ if user_agent:
+ self.sessions[session_id]["user_agent"] = user_agent
+
+ self.sessions[session_id]["last_accessed"] = datetime.now().isoformat()
+ self.save_sessions()
+
+ def cleanup_old_sessions(self, max_age_hours: int = 24):
+ """Clean up old sessions and their files"""
+ with self.lock:
+ current_time = datetime.now()
+ sessions_to_remove = []
+
+ for session_id, session_data in self.sessions.items():
+ last_accessed = datetime.fromisoformat(session_data["last_accessed"])
+ age_hours = (current_time - last_accessed).total_seconds() / 3600
+
+ if age_hours > max_age_hours:
+ sessions_to_remove.append(session_id)
+
+ for session_id in sessions_to_remove:
+ self.remove_session(session_id)
+
+ logger.info(f"Cleaned up {len(sessions_to_remove)} old sessions")
+ return len(sessions_to_remove)
+
+ def remove_session(self, session_id: str):
+ """Remove a session and clean up its files"""
+ with self.lock:
+ if session_id not in self.sessions:
+ return
+
+ session_data = self.sessions[session_id]
+
+ # Remove output files
+ for file_info in session_data["output_files"]:
+ file_path = file_info["file_path"]
+ try:
+ if os.path.exists(file_path):
+ os.remove(file_path)
+ logger.info(f"Removed file: {file_path}")
+ except Exception as e:
+ logger.error(f"Error removing file {file_path}: {e}")
+
+ # Remove session directory
+ session_dir = self.get_session_directory(session_id)
+ try:
+ if os.path.exists(session_dir):
+ import shutil
+ shutil.rmtree(session_dir)
+ logger.info(f"Removed session directory: {session_dir}")
+ except Exception as e:
+ logger.error(f"Error removing session directory {session_dir}: {e}")
+
+ # Remove from sessions
+ del self.sessions[session_id]
+ self.save_sessions()
+ logger.info(f"Removed session: {session_id}")
+
+ def get_session_stats(self) -> dict:
+ """Get statistics about all sessions"""
+ with self.lock:
+ total_sessions = len(self.sessions)
+ active_sessions = sum(1 for s in self.sessions.values() if s["status"] == "active")
+ total_files = sum(len(s["output_files"]) for s in self.sessions.values())
+
+ return {
+ "total_sessions": total_sessions,
+ "active_sessions": active_sessions,
+ "total_files": total_files,
+ "base_output_dir": self.base_output_dir
+ }
+
+ def save_sessions(self):
+ """Save sessions to disk"""
+ try:
+ with open(self.sessions_file, 'w') as f:
+ json.dump(self.sessions, f, indent=2)
+ except Exception as e:
+ logger.error(f"Error saving sessions: {e}")
+
+ def load_sessions(self):
+ """Load sessions from disk"""
+ try:
+ if os.path.exists(self.sessions_file):
+ with open(self.sessions_file, 'r') as f:
+ self.sessions = json.load(f)
+ logger.info(f"Loaded {len(self.sessions)} sessions from disk")
+ else:
+ self.sessions = {}
+ logger.info("No existing sessions file found, starting fresh")
+ except Exception as e:
+ logger.error(f"Error loading sessions: {e}")
+ self.sessions = {}
+
+# Global session manager instance
+_session_manager = None
+
+def get_session_manager() -> SessionManager:
+ """Get the global session manager instance"""
+ global _session_manager
+ if _session_manager is None:
+ # Use the outputs directory from the app structure
+ output_dir = os.path.join(os.path.dirname(__file__), "../outputs")
+ _session_manager = SessionManager(output_dir)
+ return _session_manager
+
+def create_user_session() -> str:
+ """Convenience function to create a new user session"""
+ return get_session_manager().create_session()
+
+def get_user_session_directory(session_id: str) -> str:
+ """Convenience function to get session directory"""
+ return get_session_manager().get_session_directory(session_id)
+
+def add_session_file(session_id: str, file_path: str) -> bool:
+ """Convenience function to add file to session"""
+ return get_session_manager().add_output_file(session_id, file_path)
+
+def get_session_latest_file(session_id: str) -> Optional[str]:
+ """Convenience function to get latest file for session"""
+ return get_session_manager().get_latest_file(session_id)
+
+def validate_session(session_id: str) -> bool:
+ """Convenience function to validate session"""
+ return get_session_manager().is_valid_session(session_id)
\ No newline at end of file
diff --git a/Evaluation/RAG_Evaluator/src/utils/unusedChunkAnalyzer.py b/Evaluation/RAG_Evaluator/src/utils/unusedChunkAnalyzer.py
new file mode 100644
index 00000000..7bb9248c
--- /dev/null
+++ b/Evaluation/RAG_Evaluator/src/utils/unusedChunkAnalyzer.py
@@ -0,0 +1,663 @@
+"""
+Unused Chunk Analysis Module
+============================
+Advanced analysis of questions where chunks were sent to LLM but not used in answers.
+
+This module provides comprehensive categorization and analysis of unused chunks to identify:
+1. Context relevance issues from qualified chunks
+2. Ground truth validity problems
+3. Context overload causing answer generation failures
+
+Author: RAG Evaluator Team
+Version: 1.0.0
+"""
+
+import json
+import logging
+from typing import Dict, List, Optional, Tuple, Any
+from dataclasses import dataclass, field
+from enum import Enum
+
+# Try to import pandas, but handle gracefully if not available
+try:
+ import pandas as pd
+ PANDAS_AVAILABLE = True
+except ImportError:
+ PANDAS_AVAILABLE = False
+ pd = None
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+class UnusedChunkCategory(Enum):
+ """Categories for questions with unused chunks sent to LLM."""
+ CONTEXT_IRRELEVANT = "context_irrelevant"
+ GROUND_TRUTH_INVALID = "ground_truth_invalid"
+ CONTEXT_OVERLOAD = "context_overload"
+ ANSWER_GENERATION_FAILURE = "answer_generation_failure"
+ MIXED_ISSUES = "mixed_issues"
+
+@dataclass
+class UnusedChunkAnalysis:
+ """Data structure for unused chunk analysis."""
+ query: str
+ answer: str
+ ground_truth: str
+ sent_to_llm_chunks: List[str]
+ used_chunks: List[str]
+ unused_chunks: List[str]
+ chunk_qualification_stats: Dict[str, int]
+ total_sent_to_llm: int
+ total_used: int
+ unused_count: int
+ category: UnusedChunkCategory
+ reasoning: str
+ context_relevance_score: Optional[float] = None
+ ground_truth_validity_score: Optional[float] = None
+ context_overload_score: Optional[float] = None
+
+@dataclass
+class UnusedChunkSummary:
+ """Summary statistics for unused chunk analysis."""
+ total_questions_analyzed: int
+ questions_with_unused_chunks: int
+ category_distribution: Dict[str, int]
+ avg_unused_chunks_per_question: float
+ avg_context_relevance_score: float
+ avg_ground_truth_validity_score: float
+ avg_context_overload_score: float
+ recommendations: List[str]
+
+class UnusedChunkAnalyzer:
+ """
+ Analyzes questions where chunks were sent to LLM but not used in answers.
+
+ This class provides comprehensive categorization and analysis to identify
+ the root causes of chunk underutilization in RAG systems.
+ """
+
+ # Thresholds for categorization
+ CONTEXT_OVERLOAD_THRESHOLD = 15 # More than 15 chunks sent to LLM
+ LOW_UTILIZATION_THRESHOLD = 0.3 # Less than 30% of sent chunks used
+ HIGH_UNUSED_THRESHOLD = 0.7 # More than 70% of sent chunks unused
+
+ @staticmethod
+ def analyze_unused_chunks(
+ queries: List[str],
+ answers: List[str],
+ ground_truths: List[str],
+ chunk_statistics_list: List[Dict],
+ llm_evaluation_results: Optional[List[Dict]] = None
+ ) -> Tuple[List[UnusedChunkAnalysis], UnusedChunkSummary]:
+ """
+ Analyze questions with unused chunks sent to LLM.
+
+ Args:
+ queries: List of user queries
+ answers: List of generated answers
+ ground_truths: List of ground truth answers
+ chunk_statistics_list: List of chunk statistics dictionaries
+ llm_evaluation_results: Optional LLM evaluation results for additional context
+
+ Returns:
+ Tuple of (analysis_list, summary_statistics)
+ """
+ try:
+ logger.info(f"๐ Starting unused chunk analysis for {len(queries)} queries")
+
+ analysis_list = []
+ questions_with_unused_chunks = 0
+
+ for i, (query, answer, ground_truth, chunk_stats) in enumerate(
+ zip(queries, answers, ground_truths, chunk_statistics_list)
+ ):
+ # Check if this question has unused chunks
+ sent_to_llm_count = chunk_stats.get('sent_to_llm_chunk_count', 0)
+ used_count = chunk_stats.get('used_in_answer_chunk_count', 0)
+
+ if sent_to_llm_count > 0 and sent_to_llm_count > used_count:
+ questions_with_unused_chunks += 1
+
+ # Analyze this question
+ analysis = UnusedChunkAnalyzer._analyze_single_question(
+ query, answer, ground_truth, chunk_stats,
+ llm_evaluation_results[i] if llm_evaluation_results else None
+ )
+ analysis_list.append(analysis)
+
+ # Generate summary statistics
+ summary = UnusedChunkAnalyzer._generate_summary(
+ analysis_list, len(queries), questions_with_unused_chunks
+ )
+
+ logger.info(f"โ
Unused chunk analysis completed: {questions_with_unused_chunks} questions analyzed")
+ return analysis_list, summary
+
+ except Exception as e:
+ logger.error(f"โ Error in unused chunk analysis: {e}")
+ return [], UnusedChunkAnalyzer._create_empty_summary()
+
+ @staticmethod
+ def _analyze_single_question(
+ query: str,
+ answer: str,
+ ground_truth: str,
+ chunk_stats: Dict,
+ llm_eval: Optional[Dict] = None
+ ) -> UnusedChunkAnalysis:
+ """
+ Analyze a single question for unused chunk issues.
+
+ Args:
+ query: User query
+ answer: Generated answer
+ ground_truth: Expected answer
+ chunk_stats: Chunk statistics for this question
+ llm_eval: Optional LLM evaluation results
+
+ Returns:
+ UnusedChunkAnalysis object
+ """
+ sent_to_llm_chunks = chunk_stats.get('sent_to_llm_chunk_ids', [])
+ used_chunks = chunk_stats.get('used_in_answer_chunk_ids', [])
+ unused_chunks = [chunk for chunk in sent_to_llm_chunks if chunk not in used_chunks]
+
+ sent_count = len(sent_to_llm_chunks)
+ used_count = len(used_chunks)
+ unused_count = len(unused_chunks)
+
+ # Determine category and reasoning
+ category, reasoning = UnusedChunkAnalyzer._categorize_question(
+ query, answer, ground_truth, chunk_stats, llm_eval
+ )
+
+ # Calculate scores
+ context_relevance_score = UnusedChunkAnalyzer._calculate_context_relevance_score(
+ chunk_stats, llm_eval
+ )
+ ground_truth_validity_score = UnusedChunkAnalyzer._calculate_ground_truth_validity_score(
+ query, ground_truth, llm_eval
+ )
+ context_overload_score = UnusedChunkAnalyzer._calculate_context_overload_score(
+ sent_count, used_count, unused_count
+ )
+
+ return UnusedChunkAnalysis(
+ query=query,
+ answer=answer,
+ ground_truth=ground_truth,
+ sent_to_llm_chunks=sent_to_llm_chunks,
+ used_chunks=used_chunks,
+ unused_chunks=unused_chunks,
+ chunk_qualification_stats=chunk_stats.get('chunk_qualification_stats', {}),
+ total_sent_to_llm=sent_count,
+ total_used=used_count,
+ unused_count=unused_count,
+ category=category,
+ reasoning=reasoning,
+ context_relevance_score=context_relevance_score,
+ ground_truth_validity_score=ground_truth_validity_score,
+ context_overload_score=context_overload_score
+ )
+
+ @staticmethod
+ def _categorize_question(
+ query: str,
+ answer: str,
+ ground_truth: str,
+ chunk_stats: Dict,
+ llm_eval: Optional[Dict] = None
+ ) -> Tuple[UnusedChunkCategory, str]:
+ """
+ Categorize the question based on unused chunk analysis.
+
+ Args:
+ query: User query
+ answer: Generated answer
+ ground_truth: Expected answer
+ chunk_stats: Chunk statistics
+ llm_eval: Optional LLM evaluation results
+
+ Returns:
+ Tuple of (category, reasoning)
+ """
+ sent_count = chunk_stats.get('sent_to_llm_chunk_count', 0)
+ used_count = chunk_stats.get('used_in_answer_chunk_count', 0)
+ unused_count = sent_count - used_count
+
+ # Check for context overload
+ if sent_count > UnusedChunkAnalyzer.CONTEXT_OVERLOAD_THRESHOLD:
+ return UnusedChunkCategory.CONTEXT_OVERLOAD, f"Too many chunks ({sent_count}) sent to LLM, causing information overload"
+
+ # Check utilization ratio
+ utilization_ratio = used_count / sent_count if sent_count > 0 else 0
+ if utilization_ratio < UnusedChunkAnalyzer.LOW_UTILIZATION_THRESHOLD:
+ return UnusedChunkCategory.CONTEXT_IRRELEVANT, f"Low chunk utilization ({utilization_ratio:.2%}), suggesting irrelevant context"
+
+ # Check for answer generation issues
+ if answer.strip().lower() in ['', 'no answer', 'i cannot answer', 'insufficient information']:
+ return UnusedChunkCategory.ANSWER_GENERATION_FAILURE, "LLM failed to generate meaningful answer despite having context"
+
+ # Check ground truth validity using LLM evaluation if available
+ if llm_eval and 'ground_truth_validity_score' in llm_eval:
+ validity_score = llm_eval['ground_truth_validity_score']
+ if validity_score < 0.6: # Low validity threshold
+ return UnusedChunkCategory.GROUND_TRUTH_INVALID, f"Ground truth validity score ({validity_score:.2f}) suggests invalid reference answer"
+
+ # Default to mixed issues if no clear category
+ return UnusedChunkCategory.MIXED_ISSUES, "Multiple factors contributing to chunk underutilization"
+
+ @staticmethod
+ def _calculate_context_relevance_score(chunk_stats: Dict, llm_eval: Optional[Dict] = None) -> Optional[float]:
+ """Calculate context relevance score."""
+ if llm_eval and 'context_relevancy_score' in llm_eval:
+ return llm_eval['context_relevancy_score']
+
+ # Fallback: use chunk qualification stats
+ qualification_stats = chunk_stats.get('chunk_qualification_stats', {})
+ qualified_count = qualification_stats.get('qualified', 0)
+ total_sent = chunk_stats.get('sent_to_llm_chunk_count', 0)
+
+ if total_sent > 0:
+ return qualified_count / total_sent
+ return None
+
+ @staticmethod
+ def _calculate_ground_truth_validity_score(
+ query: str,
+ ground_truth: str,
+ llm_eval: Optional[Dict] = None
+ ) -> Optional[float]:
+ """Calculate ground truth validity score."""
+ if llm_eval and 'ground_truth_validity_score' in llm_eval:
+ return llm_eval['ground_truth_validity_score']
+
+ # Fallback: basic heuristics
+ if not ground_truth or ground_truth.strip() == '':
+ return 0.0
+ if ground_truth.lower() in ['invalid question', 'cannot answer', 'no answer']:
+ return 0.3
+ return 0.7 # Default moderate validity
+
+ @staticmethod
+ def _calculate_context_overload_score(sent_count: int, used_count: int, unused_count: int) -> float:
+ """Calculate context overload score (0-1, higher = more overload)."""
+ if sent_count == 0:
+ return 0.0
+
+ # Factors: high chunk count, low utilization, high unused ratio
+ chunk_count_factor = min(sent_count / 20.0, 1.0) # Normalize to 0-1
+ utilization_factor = 1.0 - (used_count / sent_count)
+ unused_ratio_factor = unused_count / sent_count
+
+ # Weighted combination
+ overload_score = (chunk_count_factor * 0.4 +
+ utilization_factor * 0.3 +
+ unused_ratio_factor * 0.3)
+
+ return min(overload_score, 1.0)
+
+ @staticmethod
+ def _generate_summary(
+ analysis_list: List[UnusedChunkAnalysis],
+ total_questions: int,
+ questions_with_unused_chunks: int
+ ) -> UnusedChunkSummary:
+ """Generate summary statistics for unused chunk analysis."""
+ if not analysis_list:
+ return UnusedChunkAnalyzer._create_empty_summary()
+
+ # Category distribution
+ category_distribution = {}
+ for analysis in analysis_list:
+ category = analysis.category.value
+ category_distribution[category] = category_distribution.get(category, 0) + 1
+
+ # Calculate averages
+ total_unused = sum(analysis.unused_count for analysis in analysis_list)
+ avg_unused_chunks = total_unused / len(analysis_list) if analysis_list else 0
+
+ # Calculate average scores
+ context_scores = [a.context_relevance_score for a in analysis_list if a.context_relevance_score is not None]
+ validity_scores = [a.ground_truth_validity_score for a in analysis_list if a.ground_truth_validity_score is not None]
+ overload_scores = [a.context_overload_score for a in analysis_list if a.context_overload_score is not None]
+
+ avg_context_relevance = sum(context_scores) / len(context_scores) if context_scores else 0.0
+ avg_ground_truth_validity = sum(validity_scores) / len(validity_scores) if validity_scores else 0.0
+ avg_context_overload = sum(overload_scores) / len(overload_scores) if overload_scores else 0.0
+
+ # Generate recommendations
+ recommendations = UnusedChunkAnalyzer._generate_recommendations(
+ analysis_list, category_distribution
+ )
+
+ return UnusedChunkSummary(
+ total_questions_analyzed=total_questions,
+ questions_with_unused_chunks=questions_with_unused_chunks,
+ category_distribution=category_distribution,
+ avg_unused_chunks_per_question=round(avg_unused_chunks, 2),
+ avg_context_relevance_score=round(avg_context_relevance, 3),
+ avg_ground_truth_validity_score=round(avg_ground_truth_validity, 3),
+ avg_context_overload_score=round(avg_context_overload, 3),
+ recommendations=recommendations
+ )
+
+ @staticmethod
+ def _generate_recommendations(
+ analysis_list: List[UnusedChunkAnalysis],
+ category_distribution: Dict[str, int]
+ ) -> List[str]:
+ """Generate actionable recommendations based on analysis."""
+ recommendations = []
+
+ # Context overload recommendations
+ if category_distribution.get('context_overload', 0) > 0:
+ recommendations.append("๐ Reduce chunks sent to LLM to prevent information overload")
+ recommendations.append("โก Implement dynamic chunk selection based on query complexity")
+
+ # Context irrelevance recommendations
+ if category_distribution.get('context_irrelevant', 0) > 0:
+ recommendations.append("๐ฏ Improve retrieval relevance by fine-tuning embedding models")
+ recommendations.append("๐ Implement chunk pre-filtering based on semantic similarity")
+
+ # Ground truth validity recommendations
+ if category_distribution.get('ground_truth_invalid', 0) > 0:
+ recommendations.append("โ
Review and validate ground truth answers for accuracy")
+ recommendations.append("๐ Implement ground truth quality assessment pipeline")
+
+ # Answer generation failure recommendations
+ if category_distribution.get('answer_generation_failure', 0) > 0:
+ recommendations.append("๐ค Optimize LLM prompts for better context utilization")
+ recommendations.append("๐ Implement answer generation quality checks")
+
+ # General recommendations
+ if len(analysis_list) > 0:
+ avg_unused = sum(a.unused_count for a in analysis_list) / len(analysis_list)
+ if avg_unused > 5:
+ recommendations.append("โ๏ธ Balance chunk quantity vs. quality for optimal performance")
+
+ recommendations.append("๐ Monitor chunk utilization patterns for continuous improvement")
+
+ return recommendations
+
+ @staticmethod
+ def _create_empty_summary() -> UnusedChunkSummary:
+ """Create empty summary structure."""
+ return UnusedChunkSummary(
+ total_questions_analyzed=0,
+ questions_with_unused_chunks=0,
+ category_distribution={},
+ avg_unused_chunks_per_question=0.0,
+ avg_context_relevance_score=0.0,
+ avg_ground_truth_validity_score=0.0,
+ avg_context_overload_score=0.0,
+ recommendations=[]
+ )
+
+ @staticmethod
+ def format_analysis_for_excel(analysis_list: List[UnusedChunkAnalysis]) -> List[Dict]:
+ """
+ Format unused chunk analysis for Excel export.
+
+ Args:
+ analysis_list: List of UnusedChunkAnalysis objects
+
+ Returns:
+ List of dictionaries formatted for Excel
+ """
+ formatted_data = []
+
+ for analysis in analysis_list:
+ formatted_row = {
+ 'Query': analysis.query,
+ 'Generated Answer': analysis.answer,
+ 'Ground Truth': analysis.ground_truth,
+ 'Sent to LLM Chunk Count': analysis.total_sent_to_llm,
+ 'Used Chunk Count': analysis.total_used,
+ 'Unused Chunk Count': analysis.unused_count,
+ 'Unused Chunk IDs': json.dumps(analysis.unused_chunks, separators=(',', ':')),
+ 'Category': analysis.category.value,
+ 'Reasoning': analysis.reasoning,
+ 'Context Relevance Score': analysis.context_relevance_score or 'N/A',
+ 'Ground Truth Validity Score': analysis.ground_truth_validity_score or 'N/A',
+ 'Context Overload Score': analysis.context_overload_score or 'N/A',
+ 'Chunk Qualification Stats': json.dumps(analysis.chunk_qualification_stats, separators=(',', ':'))
+ }
+ formatted_data.append(formatted_row)
+
+ return formatted_data
+
+ @staticmethod
+ def format_summary_for_excel(summary: UnusedChunkSummary) -> Dict:
+ """
+ Format summary statistics for Excel export.
+
+ Args:
+ summary: UnusedChunkSummary object
+
+ Returns:
+ Dictionary formatted for Excel
+ """
+ return {
+ 'Total Questions Analyzed': summary.total_questions_analyzed,
+ 'Questions with Unused Chunks': summary.questions_with_unused_chunks,
+ 'Percentage with Unused Chunks': f"{(summary.questions_with_unused_chunks / summary.total_questions_analyzed * 100):.1f}%" if summary.total_questions_analyzed > 0 else "0%",
+ 'Average Unused Chunks per Question': summary.avg_unused_chunks_per_question,
+ 'Average Context Relevance Score': summary.avg_context_relevance_score,
+ 'Average Ground Truth Validity Score': summary.avg_ground_truth_validity_score,
+ 'Average Context Overload Score': summary.avg_context_overload_score,
+ 'Category Distribution': json.dumps(summary.category_distribution, separators=(',', ':')),
+ 'Recommendations': ' | '.join(summary.recommendations)
+ }
+
+ @staticmethod
+ def _create_quality_analysis_tab(summary_list: List[Dict]):
+ """
+ Create a comprehensive Quality Analysis tab for Excel export.
+
+ Args:
+ summary_list: List of unused chunk summary dictionaries
+
+ Returns:
+ DataFrame formatted for Quality Analysis tab
+ """
+ try:
+ if not PANDAS_AVAILABLE:
+ logger.error("Pandas is not available. Cannot create quality analysis tab.")
+ return None
+
+ if not summary_list:
+ return pd.DataFrame({
+ 'Metric': ['No Data Available'],
+ 'Value': ['No unused chunk analysis performed'],
+ 'Description': ['Please ensure chunk statistics are available']
+ })
+
+ # Aggregate data across all sheets
+ total_questions = sum(s.get('total_questions_analyzed', 0) for s in summary_list)
+ total_with_unused = sum(s.get('questions_with_unused_chunks', 0) for s in summary_list)
+
+ # Calculate weighted averages
+ total_weight = sum(s.get('total_questions_analyzed', 0) for s in summary_list)
+ weighted_avg_unused = 0
+ weighted_avg_context_relevance = 0
+ weighted_avg_ground_truth_validity = 0
+ weighted_avg_context_overload = 0
+
+ if total_weight > 0:
+ for summary in summary_list:
+ weight = summary.get('total_questions_analyzed', 0) / total_weight
+ weighted_avg_unused += summary.get('avg_unused_chunks_per_question', 0) * weight
+ weighted_avg_context_relevance += summary.get('avg_context_relevance_score', 0) * weight
+ weighted_avg_ground_truth_validity += summary.get('avg_ground_truth_validity_score', 0) * weight
+ weighted_avg_context_overload += summary.get('avg_context_overload_score', 0) * weight
+
+ # Aggregate category distribution
+ combined_category_dist = {}
+ for summary in summary_list:
+ category_dist = summary.get('category_distribution', {})
+ for category, count in category_dist.items():
+ combined_category_dist[category] = combined_category_dist.get(category, 0) + count
+
+ # Collect all recommendations
+ all_recommendations = []
+ for summary in summary_list:
+ recommendations = summary.get('recommendations', [])
+ all_recommendations.extend(recommendations)
+
+ # Remove duplicates while preserving order
+ unique_recommendations = []
+ for rec in all_recommendations:
+ if rec not in unique_recommendations:
+ unique_recommendations.append(rec)
+
+ # Create comprehensive quality analysis DataFrame
+ quality_data = [
+ # Overall Statistics
+ {
+ 'Metric': 'Total Questions Analyzed',
+ 'Value': str(total_questions),
+ 'Description': 'Total number of questions across all sheets'
+ },
+ {
+ 'Metric': 'Questions with Unused Chunks',
+ 'Value': str(total_with_unused),
+ 'Description': 'Questions where chunks were sent to LLM but not used'
+ },
+ {
+ 'Metric': 'Percentage with Unused Chunks',
+ 'Value': f"{(total_with_unused / total_questions * 100):.1f}%" if total_questions > 0 else "0%",
+ 'Description': 'Percentage of questions experiencing chunk underutilization'
+ },
+
+ # Average Metrics
+ {
+ 'Metric': 'Average Unused Chunks per Question',
+ 'Value': f"{weighted_avg_unused:.2f}",
+ 'Description': 'Weighted average of unused chunks across all sheets'
+ },
+ {
+ 'Metric': 'Average Context Relevance Score',
+ 'Value': f"{weighted_avg_context_relevance:.3f}",
+ 'Description': 'Weighted average context relevance (0-1, higher is better)'
+ },
+ {
+ 'Metric': 'Average Ground Truth Validity Score',
+ 'Value': f"{weighted_avg_ground_truth_validity:.3f}",
+ 'Description': 'Weighted average ground truth validity (0-1, higher is better)'
+ },
+ {
+ 'Metric': 'Average Context Overload Score',
+ 'Value': f"{weighted_avg_context_overload:.3f}",
+ 'Description': 'Weighted average context overload (0-1, lower is better)'
+ },
+
+ # Category Distribution
+ {
+ 'Metric': 'Context Irrelevant Issues',
+ 'Value': str(combined_category_dist.get('context_irrelevant', 0)),
+ 'Description': 'Questions with low chunk utilization due to irrelevant context'
+ },
+ {
+ 'Metric': 'Ground Truth Invalid Issues',
+ 'Value': str(combined_category_dist.get('ground_truth_invalid', 0)),
+ 'Description': 'Questions with invalid or incorrect ground truth answers'
+ },
+ {
+ 'Metric': 'Context Overload Issues',
+ 'Value': str(combined_category_dist.get('context_overload', 0)),
+ 'Description': 'Questions with too many chunks causing information overload'
+ },
+ {
+ 'Metric': 'Answer Generation Failures',
+ 'Value': str(combined_category_dist.get('answer_generation_failure', 0)),
+ 'Description': 'Questions where LLM failed to generate meaningful answers'
+ },
+ {
+ 'Metric': 'Mixed Issues',
+ 'Value': str(combined_category_dist.get('mixed_issues', 0)),
+ 'Description': 'Questions with multiple contributing factors'
+ },
+
+ # Recommendations
+ {
+ 'Metric': 'Total Recommendations',
+ 'Value': str(len(unique_recommendations)),
+ 'Description': 'Number of actionable recommendations generated'
+ }
+ ]
+
+ # Add individual recommendations
+ for i, recommendation in enumerate(unique_recommendations, 1):
+ quality_data.append({
+ 'Metric': f'Recommendation {i}',
+ 'Value': recommendation,
+ 'Description': 'Actionable improvement suggestion'
+ })
+
+ return pd.DataFrame(quality_data)
+
+ except Exception as e:
+ logger.error(f"Error creating quality analysis tab: {e}")
+ return pd.DataFrame({
+ 'Metric': ['Error'],
+ 'Value': [f'Failed to create quality analysis: {str(e)}'],
+ 'Description': ['Please check logs for details']
+ })
+
+ @staticmethod
+ def _create_detailed_analysis_tab(analysis_list: List[UnusedChunkAnalysis]):
+ """
+ Create a detailed analysis tab showing individual questions with unused chunks.
+
+ Args:
+ analysis_list: List of UnusedChunkAnalysis objects
+
+ Returns:
+ DataFrame formatted for detailed analysis tab
+ """
+ try:
+ if not PANDAS_AVAILABLE:
+ logger.error("Pandas is not available. Cannot create detailed analysis tab.")
+ return None
+
+ if not analysis_list:
+ return pd.DataFrame({
+ 'Query': ['No Data Available'],
+ 'Generated Answer': ['No unused chunk analysis performed'],
+ 'Ground Truth': ['Please ensure chunk statistics are available'],
+ 'Category': ['N/A'],
+ 'Reasoning': ['N/A']
+ })
+
+ # Convert analysis objects to DataFrame format
+ detailed_data = []
+ for analysis in analysis_list:
+ row_data = {
+ 'Query': analysis.query,
+ 'Generated Answer': analysis.answer,
+ 'Ground Truth': analysis.ground_truth,
+ 'Sent to LLM Chunk Count': analysis.total_sent_to_llm,
+ 'Used Chunk Count': analysis.total_used,
+ 'Unused Chunk Count': analysis.unused_count,
+ 'Unused Chunk IDs': json.dumps(analysis.unused_chunks, separators=(',', ':')),
+ 'Category': analysis.category.value,
+ 'Reasoning': analysis.reasoning,
+ 'Context Relevance Score': analysis.context_relevance_score or 'N/A',
+ 'Ground Truth Validity Score': analysis.ground_truth_validity_score or 'N/A',
+ 'Context Overload Score': analysis.context_overload_score or 'N/A',
+ 'Chunk Qualification Stats': json.dumps(analysis.chunk_qualification_stats, separators=(',', ':'))
+ }
+ detailed_data.append(row_data)
+
+ return pd.DataFrame(detailed_data)
+
+ except Exception as e:
+ logger.error(f"Error creating detailed analysis tab: {e}")
+ return pd.DataFrame({
+ 'Query': ['Error'],
+ 'Generated Answer': [f'Failed to create detailed analysis: {str(e)}'],
+ 'Ground Truth': ['Please check logs for details'],
+ 'Category': ['Error'],
+ 'Reasoning': ['Error']
+ })
diff --git a/Evaluation/RAG_Evaluator/start_ui.py b/Evaluation/RAG_Evaluator/start_ui.py
new file mode 100644
index 00000000..2fe39329
--- /dev/null
+++ b/Evaluation/RAG_Evaluator/start_ui.py
@@ -0,0 +1,257 @@
+#!/usr/bin/env python3
+# Prevent Python from creating .pyc files
+import sys
+sys.dont_write_bytecode = True
+
+"""
+RAG Evaluator UI Startup Script
+===============================
+
+This script starts the RAG Evaluator web interface with the enhanced UI.
+
+Usage:
+ python start_ui.py
+
+The UI will be available at:
+ - Main Interface: http://localhost:8001
+ - API Documentation: http://localhost:8001/api/docs
+ - Interactive API: http://localhost:8001/api/redoc
+
+Features:
+ - Modern drag & drop file upload
+ - Real-time progress tracking
+ - Performance optimization settings
+ - Beautiful result visualization
+ - Downloadable reports
+"""
+
+import os
+import sys
+import webbrowser
+import time
+from pathlib import Path
+
+def check_dependencies():
+ """Check if required dependencies are installed"""
+ required_packages = [
+ 'fastapi',
+ 'uvicorn',
+ 'pandas',
+ 'openpyxl',
+ 'aiohttp',
+ 'ragas',
+ 'openai',
+ 'langchain_openai',
+ 'datasets',
+ 'transformers'
+ ]
+
+ missing_packages = []
+
+ for package in required_packages:
+ try:
+ __import__(package)
+ except ImportError:
+ missing_packages.append(package)
+
+ if missing_packages:
+ print("โ Missing required packages:")
+ for package in missing_packages:
+ print(f" โข {package}")
+
+ print("\n๐ก Installation options:")
+ print(" 1. Install all requirements:")
+ print(" pip install -r src/requirements.txt")
+ print("\n 2. Install missing packages individually:")
+ print(f" pip install {' '.join(missing_packages)}")
+
+ # Provide specific guidance for problematic packages
+ if 'ragas' in missing_packages:
+ print("\n ๐ก For RAGAS: pip install ragas")
+ if 'transformers' in missing_packages:
+ print(" ๐ก For Transformers: pip install transformers torch")
+ if any(pkg in missing_packages for pkg in ['openai', 'langchain_openai']):
+ print(" ๐ก For OpenAI: pip install openai langchain-openai")
+
+ print("\nโ ๏ธ Note: Some packages require additional system dependencies")
+ print(" See README.md for detailed installation instructions")
+ return False
+
+ return True
+
+def start_server():
+ """Start the FastAPI server"""
+ try:
+ # Add src directory to Python path
+ src_path = Path(__file__).parent / "src"
+ if str(src_path) not in sys.path:
+ sys.path.insert(0, str(src_path))
+
+ # Change to src directory
+ os.chdir(src_path)
+
+ # Import and start the app
+ from routes.app import app
+ import uvicorn
+
+ print("๐ Starting RAG Evaluator UI Server...")
+ print("=" * 60)
+ print("๐ Main Interface: http://localhost:8001")
+ print("๐ API Documentation: http://localhost:8001/api/docs")
+ print("๐ Interactive API: http://localhost:8001/api/redoc")
+ print("๐ก Health Check: http://localhost:8001/api/health")
+ print("=" * 60)
+ print("๐ฏ Features:")
+ print(" โข Drag & drop file upload")
+ print(" โข Real-time progress tracking")
+ print(" โข Performance optimization presets")
+ print(" โข Beautiful result visualization")
+ print(" โข Async batch processing (3-5x faster)")
+ print("=" * 60)
+ print("โน๏ธ Press Ctrl+C to stop the server")
+ print("")
+
+ # Open browser after a short delay
+ def open_browser():
+ time.sleep(2)
+ try:
+ webbrowser.open("http://localhost:8001")
+ print("๐ Opened browser automatically")
+ except Exception as e:
+ print(f"๐ป Could not open browser automatically: {e}")
+ print("๐ป Please open http://localhost:8001 in your browser manually")
+
+ import threading
+ browser_thread = threading.Thread(target=open_browser)
+ browser_thread.daemon = True
+ browser_thread.start()
+
+ # Start the server
+ uvicorn.run(
+ app,
+ host="0.0.0.0",
+ port=8001,
+ reload=False, # Disable reload in production
+ access_log=True
+ )
+
+ except KeyboardInterrupt:
+ print("\n")
+ print("๐ Server stopped by user")
+ print("๐ Thank you for using RAG Evaluator!")
+ except ImportError as e:
+ print(f"โ Import error: {e}")
+ print("๐ก Make sure you're in the RAG_Evaluator directory")
+ print("๐ก Try: cd RAG_Evaluator && python start_ui.py")
+ print("๐ก Install missing dependencies: pip install -r src/requirements.txt")
+ except PermissionError as e:
+ print(f"โ Permission error: {e}")
+ print("๐ก Try running with appropriate permissions")
+ print("๐ก Check if port 8001 is available or in use by another application")
+ except OSError as e:
+ if "Address already in use" in str(e):
+ print("โ Port 8001 is already in use")
+ print("๐ก Stop any other instances of the application")
+ print("๐ก Or wait a few moments and try again")
+ else:
+ print(f"โ Network error: {e}")
+ print("๐ก Check your network configuration")
+ except Exception as e:
+ print(f"โ Unexpected error starting server: {e}")
+ print("๐ก Check the error message above and try again")
+ print("๐ก For support, see README.md or create an issue on GitHub")
+
+def show_help():
+ """Show help information"""
+ print(__doc__)
+ print("\n๐ง Troubleshooting:")
+ print(" โข Make sure you're in the RAG_Evaluator directory")
+ print(" โข Install dependencies: pip install -r src/requirements.txt")
+ print(" โข Check Python version: Python 3.8+ required")
+ print(" โข Ensure you have sufficient disk space (>1GB recommended)")
+ print(" โข Check internet connectivity for API calls")
+ print(" โข For issues, check the console output for error details")
+ print("\n๐ Command Line Options:")
+ print(" โข python start_ui.py - Start the server")
+ print(" โข python start_ui.py --help - Show this help")
+ print(" โข python start_ui.py --version - Show version information")
+ print("\n๐ Accessing the Application:")
+ print(" โข Web Interface: http://localhost:8001")
+ print(" โข API Docs: http://localhost:8001/api/docs")
+ print(" โข Health Check: http://localhost:8001/api/health")
+ print("\n๐ For detailed documentation, see README.md and UI_README.md")
+
+def check_python_version():
+ """Check if Python version is compatible"""
+ if sys.version_info < (3, 8):
+ print("โ Python 3.8 or higher is required")
+ print(f"๐ก Current Python version: {sys.version}")
+ print("๐ก Please upgrade Python and try again")
+ return False
+ return True
+
+def main():
+ """Main entry point"""
+
+ # Check command line arguments
+ if len(sys.argv) > 1:
+ if sys.argv[1] in ['-h', '--help', 'help']:
+ show_help()
+ return
+ elif sys.argv[1] in ['-v', '--version', 'version']:
+ print("๐ง RAG Evaluator UI v2.0.0 - Async Batch Processing Edition")
+ print("๐
Built: 2024")
+ print(f"๐ Python: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro} (3.8+ required)")
+ print("๐ FastAPI + Modern Web UI")
+ print("๐ RAGAS + CRAG + LLM Evaluation")
+ print("โก 3-5x faster async processing")
+ return
+
+ print("๐ง RAG Evaluator - Advanced Performance Testing Suite")
+ print("=" * 60)
+
+ # Check Python version
+ if not check_python_version():
+ return
+
+ # Check if we're in the right directory
+ if not Path("src/routes/app.py").exists():
+ print("โ Error: Not in RAG_Evaluator directory")
+ print("๐ก Please run this script from the RAG_Evaluator root directory")
+ print("๐ก Example: cd RAG_Evaluator && python start_ui.py")
+ return
+
+ # Check dependencies
+ print("๐ Checking dependencies...")
+ if not check_dependencies():
+ return
+
+ print("โ
All dependencies found")
+
+ # Check for optional dependencies
+ optional_packages = ['pymongo']
+ missing_optional = []
+
+ for package in optional_packages:
+ try:
+ __import__(package)
+ except ImportError:
+ missing_optional.append(package)
+
+ if missing_optional:
+ print("โน๏ธ Optional features available with additional packages:")
+ if 'pymongo' in missing_optional:
+ print(" โข MongoDB result storage: pip install pymongo")
+
+ # System info
+ print(f"๐ Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}")
+ print(f"๐ Working directory: {os.getcwd()}")
+ print(f"๐ง Platform: {sys.platform}")
+
+ print("")
+
+ # Start the server
+ start_server()
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/Evaluation/RAG_Evaluator/test_ui_integration.html b/Evaluation/RAG_Evaluator/test_ui_integration.html
new file mode 100644
index 00000000..e24ca03b
--- /dev/null
+++ b/Evaluation/RAG_Evaluator/test_ui_integration.html
@@ -0,0 +1,379 @@
+
+
+
+ Test Quality Analysis UI Integration
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
100
+
Total Processed
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
100
+
Total Questions
+
+
+
+
25
+
With Unused Chunks
+
+
+
+
25%
+
Unused Chunks %
+
+
+
+
3.2
+
Avg Unused/Question
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Query
+ Category
+ Unused Chunks
+ Context Relevance
+ Ground Truth Validity
+ Context Overload
+ Reasoning
+
+
+
+
+
+
+
+
+
+
+
+