diff --git a/app/components/Metrics/Dashboard.tsx b/app/components/Metrics/Dashboard.tsx index 13c92fa5..2c986f17 100644 --- a/app/components/Metrics/Dashboard.tsx +++ b/app/components/Metrics/Dashboard.tsx @@ -50,6 +50,24 @@ interface MetricsDashboardProps { agents: { usage: Array<{ agent_name: string; usage_count: number }>; }; + failures?: { + stats: { + totalFailures: number; + failuresByType: Array<{ type: string; count: number }>; + failuresByTheme: Array<{ theme: string; count: number }>; + topFailureReasons: Array<{ reason: string; count: number }>; + }; + recentFailures: Array<{ + id: number; + user_prompt: string; + failure_type?: string; + failure_summary?: string; + request_theme?: string; + created_at: Date; + }>; + requestThemes: Array<{ theme: string; count: number }>; + tagDistribution: Array<{ tag: string; count: number }>; + }; vercel?: { pageViews?: { total: number; @@ -75,7 +93,7 @@ interface MetricsDashboardProps { } export const MetricsDashboard: React.FC = ({ data }) => { - const { jobs, users, mcp, agents } = data; + const { jobs, users, mcp, agents, failures } = data; // Prepare data for tables const topMCPTools = (mcp.toolUsage || []).slice(0, 10); @@ -331,6 +349,230 @@ export const MetricsDashboard: React.FC = ({ data }) => { + {/* Failure Metrics */} + {failures && ( + <> + + + + Failure Analysis + + + + Total Failures + + {failures.stats.totalFailures.toLocaleString()} + + + + + + {/* Failures by Type */} + + + Failures by Type + + + + + + + + + + + {failures.stats.failuresByType.length > 0 ? ( + failures.stats.failuresByType.map((item) => ( + + + + + )) + ) : ( + + + + )} + +
Failure Type + Count +
{item.type} + {item.count.toLocaleString()} +
+ No failure data available +
+ + + + {/* Request Themes */} + + + Request Themes (What People Are Asking) + + + + + + + + + + + {failures.requestThemes.length > 0 ? ( + failures.requestThemes.slice(0, 10).map((item) => ( + + + + + )) + ) : ( + + + + )} + +
Theme + Count +
{item.theme} + {item.count.toLocaleString()} +
+ No theme data available +
+ + + + + {/* What Platform Can't Do */} + + + What Platform Cannot Accomplish + + + + + + + + + + + + + {failures.recentFailures.length > 0 ? ( + failures.recentFailures.slice(0, 10).map((failure) => ( + + + + + + + )) + ) : ( + + + + )} + +
User RequestFailure TypeSummaryDate
+ + {failure.user_prompt.substring(0, 100)} + {failure.user_prompt.length > 100 ? '...' : ''} + + + {failure.failure_type || 'Unknown'} + + + {failure.failure_summary + ? failure.failure_summary.substring(0, 150) + : 'N/A'} + {failure.failure_summary && + failure.failure_summary.length > 150 + ? '...' + : ''} + + + {new Date( + failure.created_at + ).toLocaleDateString()} +
+ No failure data available +
+ + + + {/* Tag Distribution */} + {failures.tagDistribution.length > 0 && ( + + + Request Tags Distribution + + + + + + + + + + + {failures.tagDistribution.slice(0, 15).map((item) => ( + + + + + ))} + +
Tag + Count +
{item.tag} + {item.count.toLocaleString()} +
+ + + )} + + + )} + {/* Vercel Metrics */} = {}; + allMetrics.forEach((metric) => { + if (metric.request_theme) { + themeCounts[metric.request_theme] = + (themeCounts[metric.request_theme] || 0) + 1; + } + }); + + const requestThemes = Object.entries(themeCounts) + .map(([theme, count]) => ({ theme, count })) + .sort((a, b) => b.count - a.count) + .slice(0, 20); + + // Get tag distribution + const tagCounts: Record = {}; + allMetrics.forEach((metric) => { + if (metric.detected_tags && Array.isArray(metric.detected_tags)) { + metric.detected_tags.forEach((tag) => { + tagCounts[tag] = (tagCounts[tag] || 0) + 1; + }); + } + }); + + const tagDistribution = Object.entries(tagCounts) + .map(([tag, count]) => ({ tag, count })) + .sort((a, b) => b.count - a.count); + + return res.status(200).json({ + stats, + recentFailures: recentFailures.slice(0, 20), + requestThemes, + tagDistribution, + timestamp: new Date().toISOString(), + }); + } catch (error) { + console.error('[Metrics Failures] Error:', error); + return res.status(500).json({ + error: + error instanceof Error ? error.message : 'Internal server error', + }); + } +} + diff --git a/app/pages/api/metrics/index.ts b/app/pages/api/metrics/index.ts index add9dc90..532705fe 100644 --- a/app/pages/api/metrics/index.ts +++ b/app/pages/api/metrics/index.ts @@ -1,6 +1,6 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { isUserWhitelisted } from '@/services/metrics/whitelist'; -import { JobDB, MessageDB, UserAvailableToolDB, UserMCPServerDB } from '@/services/database/db'; +import { JobDB, MessageDB, UserAvailableToolDB, UserMCPServerDB, FailureMetricsDB } from '@/services/database/db'; export default async function handler( req: NextApiRequest, @@ -61,6 +61,54 @@ export default async function handler( const thirtyDaysAgo = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000); const mcpUsageLast30Days = await MessageDB.getMCPUsageByDateRange(thirtyDaysAgo, now); + // Fetch failure metrics (handle case where table doesn't exist yet) + let failureData = null; + try { + const failureStats = await FailureMetricsDB.getFailureStats(); + const recentFailures = await FailureMetricsDB.getFailureMetrics({ + isFailure: true, + limit: 50, + }); + + // Get request themes and tags + const allMetrics = await FailureMetricsDB.getFailureMetrics({ + limit: 1000, + }); + + const themeCounts: Record = {}; + const tagCounts: Record = {}; + allMetrics.forEach((metric) => { + if (metric.request_theme) { + themeCounts[metric.request_theme] = (themeCounts[metric.request_theme] || 0) + 1; + } + if (metric.detected_tags && Array.isArray(metric.detected_tags)) { + metric.detected_tags.forEach((tag) => { + tagCounts[tag] = (tagCounts[tag] || 0) + 1; + }); + } + }); + + const requestThemes = Object.entries(themeCounts) + .map(([theme, count]) => ({ theme, count })) + .sort((a, b) => b.count - a.count) + .slice(0, 20); + + const tagDistribution = Object.entries(tagCounts) + .map(([tag, count]) => ({ tag, count })) + .sort((a, b) => b.count - a.count); + + failureData = { + stats: failureStats, + recentFailures: recentFailures.slice(0, 20), + requestThemes, + tagDistribution, + }; + } catch (error) { + // Table might not exist yet - log but don't fail the whole request + console.warn('[Metrics] Failure metrics table may not exist yet:', error instanceof Error ? error.message : String(error)); + failureData = null; + } + // Fetch Vercel metrics if available let vercelMetrics = null; try { @@ -102,6 +150,7 @@ export default async function handler( agents: { usage: agentUsage, }, + failures: failureData, vercel: vercelMetrics, timestamp: new Date().toISOString(), }); diff --git a/app/pages/api/v1/chat/index.ts b/app/pages/api/v1/chat/index.ts index de9b7aad..b3802d9f 100644 --- a/app/pages/api/v1/chat/index.ts +++ b/app/pages/api/v1/chat/index.ts @@ -121,6 +121,24 @@ export default async function handler( enhancedChatRequest.similarPrompts || [] ); + // Track failure metrics asynchronously (fire and forget) + const userPrompt = message; + const assistantResponse = enhancedResponse.content || ''; + const jobId = enhancedChatRequest.conversationId || ''; + if (userPrompt && assistantResponse) { + import('@/services/metrics/failure-tracking').then(({ failureTrackingService }) => { + failureTrackingService.trackResponse({ + jobId, + walletAddress, + agentName: currentAgent, + userPrompt, + assistantResponse, + }).catch((err) => { + console.error('[CHAT] Error tracking failure metrics:', err); + }); + }); + } + return res.status(200).json({ response: enhancedResponse, current_agent: currentAgent, diff --git a/app/pages/api/v1/chat/orchestrate.ts b/app/pages/api/v1/chat/orchestrate.ts index 1ffab255..08ea5c77 100644 --- a/app/pages/api/v1/chat/orchestrate.ts +++ b/app/pages/api/v1/chat/orchestrate.ts @@ -1,3 +1,7 @@ +import { + rateLimitErrorResponse, + withRateLimit, +} from '@/middleware/rate-limiting'; import { initializeAgents } from '@/services/agents/initialize'; import { createOrchestrator } from '@/services/agents/orchestrator'; import { ChatRequest } from '@/services/agents/types'; @@ -8,7 +12,6 @@ import { } from '@/services/utils/errors'; import { NextApiRequest, NextApiResponse } from 'next'; import { v4 as uuidv4 } from 'uuid'; -import { withRateLimit, rateLimitErrorResponse } from '@/middleware/rate-limiting'; // Timeout configuration const ORCHESTRATION_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes @@ -16,14 +19,18 @@ const AGENT_SELECTION_TIMEOUT_MS = 30 * 1000; // 30 seconds const AGENT_EXECUTION_TIMEOUT_MS = 4 * 60 * 1000; // 4 minutes // Timeout utility function -function withTimeout(promise: Promise, timeoutMs: number, operation: string): Promise { +function withTimeout( + promise: Promise, + timeoutMs: number, + operation: string +): Promise { return Promise.race([ promise, new Promise((_, reject) => { setTimeout(() => { reject(new Error(`${operation} timed out after ${timeoutMs}ms`)); }, timeoutMs); - }) + }), ]); } @@ -44,7 +51,7 @@ export default async function handler( const startTime = Date.now(); let chatRequest: ChatRequest; let walletAddress: string | undefined; - + try { // Wrap entire orchestration in timeout const result = await withTimeout( @@ -52,29 +59,34 @@ export default async function handler( ORCHESTRATION_TIMEOUT_MS, 'Orchestration' ); - + const processingTime = Date.now() - startTime; - console.log(`[ORCHESTRATION] Completed successfully in ${processingTime}ms`); - + console.log( + `[ORCHESTRATION] Completed successfully in ${processingTime}ms` + ); + return res.status(200).json({ ...result, _meta: { processingTime, - timestamp: new Date().toISOString() - } + timestamp: new Date().toISOString(), + }, }); } catch (error) { const processingTime = Date.now() - startTime; - console.error(`[ORCHESTRATION ERROR] Failed after ${processingTime}ms:`, error); - + console.error( + `[ORCHESTRATION ERROR] Failed after ${processingTime}ms:`, + error + ); + const { error: errorMessage, statusCode } = createSafeErrorResponse(error); - return res.status(statusCode).json({ + return res.status(statusCode).json({ error: errorMessage, _meta: { processingTime, timestamp: new Date().toISOString(), - failed: true - } + failed: true, + }, }); } } @@ -83,6 +95,12 @@ async function executeOrchestration(req: NextApiRequest) { const chatRequest: ChatRequest = req.body; const walletAddress = req.body.walletAddress; + // Track failure metrics asynchronously (don't await to avoid blocking) + let userPrompt = ''; + let assistantResponse = ''; + let jobId = ''; + let agentName = ''; + // Generate request ID if not provided if (!chatRequest.requestId) { chatRequest.requestId = uuidv4(); @@ -97,8 +115,7 @@ async function executeOrchestration(req: NextApiRequest) { throw new ValidationError('Missing prompt content'); } } catch (error) { - const { error: errorMessage, statusCode } = - createSafeErrorResponse(error); + const { error: errorMessage, statusCode } = createSafeErrorResponse(error); throw new Error(errorMessage); } @@ -129,12 +146,43 @@ async function executeOrchestration(req: NextApiRequest) { 'Agent execution' ); + // Store for failure tracking + userPrompt = chatRequest.prompt?.content || ''; + assistantResponse = agentResponse.content || ''; + jobId = chatRequest.conversationId || ''; + agentName = currentAgent; + + // Track failure metrics asynchronously (fire and forget) + if (userPrompt && assistantResponse) { + import('@/services/metrics/failure-tracking').then( + ({ failureTrackingService }) => { + failureTrackingService + .trackResponse({ + jobId, + walletAddress, + agentName, + userPrompt, + assistantResponse, + }) + .catch((err) => { + console.error( + '[ORCHESTRATION] Error tracking failure metrics:', + err + ); + }); + } + ); + } + return { response: agentResponse, current_agent: currentAgent, }; } catch (error) { - console.error(`[ORCHESTRATION] Execution failed for request ${chatRequest.requestId}:`, error); + console.error( + `[ORCHESTRATION] Execution failed for request ${chatRequest.requestId}:`, + error + ); throw error; } -} \ No newline at end of file +} diff --git a/app/scripts/run-failure-metrics-migration.js b/app/scripts/run-failure-metrics-migration.js new file mode 100644 index 00000000..026b20d6 --- /dev/null +++ b/app/scripts/run-failure-metrics-migration.js @@ -0,0 +1,39 @@ +const { Pool } = require('pg'); +const fs = require('fs'); +const path = require('path'); + +// Load environment variables +require('dotenv').config({ path: path.join(__dirname, '../.env.local') }); + +async function runMigration() { + const pool = new Pool({ + connectionString: process.env.DATABASE_URL, + max: 20, + idleTimeoutMillis: 30000, + connectionTimeoutMillis: 2000, + }); + + try { + console.log('Running migration: 017-create-failure-metrics-table.sql'); + + // Read the migration file + const migrationPath = path.join( + __dirname, + '../migrations/017-create-failure-metrics-table.sql' + ); + const migrationSQL = fs.readFileSync(migrationPath, 'utf8'); + + // Execute the migration + await pool.query(migrationSQL); + + console.log('Migration completed successfully!'); + } catch (error) { + console.error('Migration failed:', error); + process.exit(1); + } finally { + await pool.end(); + } +} + +runMigration(); + diff --git a/app/services/database/db.ts b/app/services/database/db.ts index 5d2f52e4..b2c46df8 100644 --- a/app/services/database/db.ts +++ b/app/services/database/db.ts @@ -2551,6 +2551,171 @@ export class ReferralDB { } } +export interface FailureMetric { + id: number; + job_id: string; + message_id?: string; + user_id?: string; + wallet_address?: string; + agent_name?: string; + user_prompt: string; + assistant_response: string; + is_failure: boolean; + failure_type?: string; + failure_reason?: string; + failure_summary?: string; + detected_tags?: string[]; + request_theme?: string; + created_at: Date; + updated_at: Date; +} + +export class FailureMetricsDB { + static async createFailureMetric( + metric: Omit + ): Promise { + const query = ` + INSERT INTO failure_metrics ( + job_id, message_id, user_id, wallet_address, agent_name, + user_prompt, assistant_response, is_failure, failure_type, + failure_reason, failure_summary, detected_tags, request_theme + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + RETURNING *; + `; + + const result = await pool.query(query, [ + metric.job_id, + metric.message_id ?? null, + metric.user_id ?? null, + metric.wallet_address ?? null, + metric.agent_name ?? null, + metric.user_prompt, + metric.assistant_response, + metric.is_failure, + metric.failure_type ?? null, + metric.failure_reason ?? null, + metric.failure_summary ?? null, + metric.detected_tags ?? null, + metric.request_theme ?? null, + ]); + + return result.rows[0]; + } + + static async getFailureMetrics(filters?: { + walletAddress?: string; + isFailure?: boolean; + failureType?: string; + startDate?: Date; + endDate?: Date; + limit?: number; + }): Promise { + let query = 'SELECT * FROM failure_metrics WHERE 1=1'; + const values: any[] = []; + let paramCount = 0; + + if (filters?.walletAddress) { + paramCount++; + query += ` AND wallet_address = $${paramCount}`; + values.push(filters.walletAddress); + } + + if (filters?.isFailure !== undefined) { + paramCount++; + query += ` AND is_failure = $${paramCount}`; + values.push(filters.isFailure); + } + + if (filters?.failureType) { + paramCount++; + query += ` AND failure_type = $${paramCount}`; + values.push(filters.failureType); + } + + if (filters?.startDate) { + paramCount++; + query += ` AND created_at >= $${paramCount}`; + values.push(filters.startDate); + } + + if (filters?.endDate) { + paramCount++; + query += ` AND created_at <= $${paramCount}`; + values.push(filters.endDate); + } + + query += ' ORDER BY created_at DESC'; + + if (filters?.limit) { + paramCount++; + query += ` LIMIT $${paramCount}`; + values.push(filters.limit); + } + + const result = await pool.query(query, values); + return result.rows; + } + + static async getFailureStats(): Promise<{ + totalFailures: number; + failuresByType: Array<{ type: string; count: number }>; + failuresByTheme: Array<{ theme: string; count: number }>; + topFailureReasons: Array<{ reason: string; count: number }>; + }> { + const totalQuery = ` + SELECT COUNT(*) as count + FROM failure_metrics + WHERE is_failure = true; + `; + const totalResult = await pool.query(totalQuery); + + const typeQuery = ` + SELECT failure_type as type, COUNT(*) as count + FROM failure_metrics + WHERE is_failure = true AND failure_type IS NOT NULL + GROUP BY failure_type + ORDER BY count DESC; + `; + const typeResult = await pool.query(typeQuery); + + const themeQuery = ` + SELECT request_theme as theme, COUNT(*) as count + FROM failure_metrics + WHERE is_failure = true AND request_theme IS NOT NULL + GROUP BY request_theme + ORDER BY count DESC + LIMIT 20; + `; + const themeResult = await pool.query(themeQuery); + + const reasonQuery = ` + SELECT failure_reason as reason, COUNT(*) as count + FROM failure_metrics + WHERE is_failure = true AND failure_reason IS NOT NULL + GROUP BY failure_reason + ORDER BY count DESC + LIMIT 10; + `; + const reasonResult = await pool.query(reasonQuery); + + return { + totalFailures: parseInt(totalResult.rows[0]?.count || '0'), + failuresByType: typeResult.rows.map((r) => ({ + type: r.type, + count: parseInt(r.count), + })), + failuresByTheme: themeResult.rows.map((r) => ({ + theme: r.theme, + count: parseInt(r.count), + })), + topFailureReasons: reasonResult.rows.map((r) => ({ + reason: r.reason, + count: parseInt(r.count), + })), + }; + } +} + // Export pool for direct database access export { pool }; @@ -2562,6 +2727,7 @@ export class Database { static UserDB = UserDB; static SharedJobDB = SharedJobDB; static ReferralDB = ReferralDB; + static FailureMetricsDB = FailureMetricsDB; } export default { @@ -2579,4 +2745,5 @@ export default { UserMemoriesDB, SharedJobDB, ReferralDB, + FailureMetricsDB, }; diff --git a/app/services/metrics/failure-analysis.ts b/app/services/metrics/failure-analysis.ts new file mode 100644 index 00000000..e5625958 --- /dev/null +++ b/app/services/metrics/failure-analysis.ts @@ -0,0 +1,284 @@ +/** + * Failure Analysis Service + * + * Analyzes assistant responses to detect failures and extract information about + * what the platform couldn't accomplish. + */ + +export interface FailureAnalysis { + isFailure: boolean; + failureType?: string; + failureReason?: string; + failureSummary?: string; + detectedTags?: string[]; + requestTheme?: string; +} + +export class FailureAnalysisService { + // Keywords that indicate a failure or inability to complete a task + private failureIndicators = [ + 'cannot', + "can't", + 'unable to', + 'not able to', + 'failed', + 'error', + 'does not exist', + 'not found', + 'not available', + 'not supported', + 'not implemented', + 'not possible', + 'sorry', + 'apologize', + 'unfortunately', + "i don't", + "i can't", + "i'm unable", + 'i cannot', + 'limitation', + 'restriction', + 'not capable', + 'not designed', + 'not configured', + 'missing', + 'broken', + 'not working', + 'not functioning', + ]; + + // Failure type patterns + private failureTypePatterns: Array<{ + type: string; + patterns: RegExp[]; + }> = [ + { + type: 'agent_not_found', + patterns: [ + /agent.*not.*found/i, + /agent.*does.*not.*exist/i, + /no.*agent.*available/i, + /agent.*not.*available/i, + ], + }, + { + type: 'capability_limitation', + patterns: [ + /cannot.*perform/i, + /not.*capable/i, + /not.*designed.*to/i, + /limitation/i, + /not.*supported/i, + /not.*implemented/i, + ], + }, + { + type: 'could_not_answer', + patterns: [ + /cannot.*answer/i, + /unable.*to.*answer/i, + /don.*t.*know/i, + /not.*sure/i, + /don.*t.*have.*information/i, + /lack.*information/i, + ], + }, + { + type: 'error', + patterns: [ + /error.*occurred/i, + /something.*went.*wrong/i, + /failed.*to/i, + /encountered.*error/i, + /an.*error/i, + ], + }, + ]; + + /** + * Analyze a response to detect if it indicates a failure + */ + analyzeResponse( + userPrompt: string, + assistantResponse: string + ): FailureAnalysis { + const responseLower = assistantResponse.toLowerCase(); + const promptLower = userPrompt.toLowerCase(); + + // Check for failure indicators + const hasFailureIndicator = this.failureIndicators.some((indicator) => + responseLower.includes(indicator) + ); + + if (!hasFailureIndicator) { + return { + isFailure: false, + detectedTags: this.extractTags(userPrompt), + requestTheme: this.extractTheme(userPrompt), + }; + } + + // Determine failure type + let failureType = 'unknown'; + for (const { type, patterns } of this.failureTypePatterns) { + if (patterns.some((pattern) => pattern.test(assistantResponse))) { + failureType = type; + break; + } + } + + // Extract failure reason + const failureReason = this.extractFailureReason(assistantResponse); + + // Generate summary + const failureSummary = this.generateFailureSummary( + userPrompt, + assistantResponse, + failureType, + failureReason + ); + + return { + isFailure: true, + failureType, + failureReason, + failureSummary, + detectedTags: this.extractTags(userPrompt), + requestTheme: this.extractTheme(userPrompt), + }; + } + + /** + * Extract failure reason from response + */ + private extractFailureReason(response: string): string { + // Look for sentences that contain failure indicators + const sentences = response.split(/[.!?]+/); + const failureSentences = sentences.filter((sentence) => { + const lower = sentence.toLowerCase(); + return this.failureIndicators.some((indicator) => + lower.includes(indicator) + ); + }); + + if (failureSentences.length > 0) { + // Return the first failure sentence, cleaned up + return failureSentences[0].trim(); + } + + // Fallback: return first 200 chars of response + return response.substring(0, 200).trim(); + } + + /** + * Generate a summary of what couldn't be done + */ + private generateFailureSummary( + userPrompt: string, + assistantResponse: string, + failureType: string, + failureReason: string + ): string { + const theme = this.extractTheme(userPrompt); + const tags = this.extractTags(userPrompt); + + let summary = `User requested: ${theme || 'a task'}`; + if (tags.length > 0) { + summary += ` (tags: ${tags.join(', ')})`; + } + + summary += `. Platform was unable to: `; + + switch (failureType) { + case 'agent_not_found': + summary += `find or access the required agent/service`; + break; + case 'capability_limitation': + summary += `perform this action due to platform limitations`; + break; + case 'could_not_answer': + summary += `provide the requested information`; + break; + case 'error': + summary += `complete the request due to an error`; + break; + default: + summary += `complete the request`; + } + + if (failureReason) { + summary += `. Reason: ${failureReason.substring(0, 150)}`; + } + + return summary; + } + + /** + * Extract tags/themes from user prompt using simple keyword matching + */ + private extractTags(prompt: string): string[] { + const tags: string[] = []; + const lower = prompt.toLowerCase(); + + // Common request categories + const tagPatterns: Array<{ tag: string; patterns: RegExp[] }> = [ + { + tag: 'crypto', + patterns: [/crypto|bitcoin|ethereum|blockchain|token|coin|nft|defi/i], + }, + { + tag: 'trading', + patterns: [/trade|buy|sell|order|price|market|exchange/i], + }, + { + tag: 'research', + patterns: [/research|analyze|investigate|study|find.*information/i], + }, + { + tag: 'code', + patterns: [/code|program|script|function|api|develop|build/i], + }, + { + tag: 'data', + patterns: [/data|database|query|fetch|retrieve|get.*data/i], + }, + { + tag: 'web', + patterns: [/website|web|url|link|page|browse|scrape/i], + }, + { + tag: 'agent', + patterns: [/agent|assistant|bot|automation/i], + }, + { + tag: 'mcp', + patterns: [/mcp|server|tool|integration/i], + }, + ]; + + for (const { tag, patterns } of tagPatterns) { + if (patterns.some((pattern) => pattern.test(prompt))) { + tags.push(tag); + } + } + + return tags; + } + + /** + * Extract main theme/category from user prompt + */ + private extractTheme(prompt: string): string { + const tags = this.extractTags(prompt); + if (tags.length > 0) { + return tags[0]; // Return first tag as main theme + } + + // Fallback: extract first few words as theme + const words = prompt.split(/\s+/).slice(0, 3); + return words.join(' ').toLowerCase(); + } +} + +export const failureAnalysisService = new FailureAnalysisService(); + diff --git a/app/services/metrics/failure-tracking.ts b/app/services/metrics/failure-tracking.ts new file mode 100644 index 00000000..dea2d398 --- /dev/null +++ b/app/services/metrics/failure-tracking.ts @@ -0,0 +1,97 @@ +/** + * Failure Tracking Service + * + * Tracks and stores failure metrics when responses are generated. + * This should be called after each assistant response is created. + */ + +import { FailureMetricsDB } from '../database/db'; +import { failureAnalysisService } from './failure-analysis'; + +export interface TrackFailureParams { + jobId: string; + messageId?: string; + userId?: string; + walletAddress?: string; + agentName?: string; + userPrompt: string; + assistantResponse: string; +} + +export class FailureTrackingService { + /** + * Track a response and analyze it for failures + * This should be called asynchronously (fire and forget) to not block the response + */ + async trackResponse(params: TrackFailureParams): Promise { + try { + // Extract text content from response (handles both string and object formats) + const responseText = this.extractTextContent(params.assistantResponse); + const promptText = this.extractTextContent(params.userPrompt); + + if (!responseText || !promptText) { + return; // Skip if we can't extract text + } + + // Analyze the response for failures + const analysis = failureAnalysisService.analyzeResponse( + promptText, + responseText + ); + + // Store the metric (always store, even if not a failure, for analytics) + await FailureMetricsDB.createFailureMetric({ + job_id: params.jobId, + message_id: params.messageId, + user_id: params.userId, + wallet_address: params.walletAddress, + agent_name: params.agentName, + user_prompt: promptText, + assistant_response: responseText, + is_failure: analysis.isFailure, + failure_type: analysis.failureType || undefined, + failure_reason: analysis.failureReason || undefined, + failure_summary: analysis.failureSummary || undefined, + detected_tags: analysis.detectedTags || undefined, + request_theme: analysis.requestTheme || undefined, + }); + + if (analysis.isFailure) { + console.log( + `[FailureTracking] Detected failure: ${analysis.failureType} - ${analysis.failureSummary}` + ); + } + } catch (error) { + // Don't throw - we don't want to break the response flow + console.error('[FailureTracking] Error tracking response:', error); + } + } + + /** + * Extract text content from message content (handles both string and object formats) + */ + private extractTextContent(content: any): string { + if (typeof content === 'string') { + return content; + } + + if (typeof content === 'object' && content !== null) { + // Try common fields + if (content.text) return String(content.text); + if (content.content) return String(content.content); + if (content.message) return String(content.message); + + // If it's an array, try to extract text from elements + if (Array.isArray(content)) { + return content + .map((item) => this.extractTextContent(item)) + .join(' '); + } + } + + return String(content || ''); + } +} + +export const failureTrackingService = new FailureTrackingService(); +