Skip to content

feat: migration to add failure and theming statistics #210

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
daleyu merged 2 commits into from
Nov 17, 2025
Merged

feat: migration to add failure and theming statistics #210

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
25e560c
feat: migration to add failure and theming statistics
daleyu Nov 17, 2025
7e11ad3
fix: typescript errors
daleyu Nov 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
244 changes: 243 additions & 1 deletion app/components/Metrics/Dashboard.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,24 @@ interface MetricsDashboardProps {
agents: {
usage: Array<{ agent_name: string; usage_count: number }>;
};
failures?: {
stats: {
totalFailures: number;
failuresByType: Array<{ type: string; count: number }>;
failuresByTheme: Array<{ theme: string; count: number }>;
topFailureReasons: Array<{ reason: string; count: number }>;
};
recentFailures: Array<{
id: number;
user_prompt: string;
failure_type?: string;
failure_summary?: string;
request_theme?: string;
created_at: Date;
}>;
requestThemes: Array<{ theme: string; count: number }>;
tagDistribution: Array<{ tag: string; count: number }>;
};
vercel?: {
pageViews?: {
total: number;
Expand All @@ -75,7 +93,7 @@ interface MetricsDashboardProps {
}

export const MetricsDashboard: React.FC<MetricsDashboardProps> = ({ data }) => {
const { jobs, users, mcp, agents } = data;
const { jobs, users, mcp, agents, failures } = data;

// Prepare data for tables
const topMCPTools = (mcp.toolUsage || []).slice(0, 10);
Expand Down Expand Up @@ -331,6 +349,230 @@ export const MetricsDashboard: React.FC<MetricsDashboardProps> = ({ data }) => {
</TableContainer>
</Box>

{/* Failure Metrics */}
{failures && (
<>
<Divider borderColor="rgba(255, 255, 255, 0.1)" />
<Box>
<Heading size="lg" color="white" mb={4}>
Failure Analysis
</Heading>
<SimpleGrid
columns={{ base: 1, md: 2, lg: 4 }}
spacing={4}
mb={6}
>
<Stat
bg="rgba(255, 255, 255, 0.05)"
p={4}
borderRadius="lg"
border="1px solid rgba(255, 255, 255, 0.1)"
>
<StatLabel color="gray.400">Total Failures</StatLabel>
<StatNumber color="red.400">
{failures.stats.totalFailures.toLocaleString()}
</StatNumber>
</Stat>
</SimpleGrid>

<SimpleGrid columns={{ base: 1, lg: 2 }} spacing={6} mb={6}>
{/* Failures by Type */}
<Box
bg="rgba(255, 255, 255, 0.05)"
p={6}
borderRadius="lg"
border="1px solid rgba(255, 255, 255, 0.1)"
>
<Heading size="md" color="white" mb={4}>
Failures by Type
</Heading>
<TableContainer>
<Table variant="simple" colorScheme="whiteAlpha">
<Thead>
<Tr>
<Th color="gray.300">Failure Type</Th>
<Th color="gray.300" isNumeric>
Count
</Th>
</Tr>
</Thead>
<Tbody>
{failures.stats.failuresByType.length > 0 ? (
failures.stats.failuresByType.map((item) => (
<Tr key={item.type}>
<Td color="white">{item.type}</Td>
<Td color="white" isNumeric>
{item.count.toLocaleString()}
</Td>
</Tr>
))
) : (
<Tr>
<Td colSpan={2} color="gray.500" textAlign="center">
No failure data available
</Td>
</Tr>
)}
</Tbody>
</Table>
</TableContainer>
</Box>

{/* Request Themes */}
<Box
bg="rgba(255, 255, 255, 0.05)"
p={6}
borderRadius="lg"
border="1px solid rgba(255, 255, 255, 0.1)"
>
<Heading size="md" color="white" mb={4}>
Request Themes (What People Are Asking)
</Heading>
<TableContainer>
<Table variant="simple" colorScheme="whiteAlpha">
<Thead>
<Tr>
<Th color="gray.300">Theme</Th>
<Th color="gray.300" isNumeric>
Count
</Th>
</Tr>
</Thead>
<Tbody>
{failures.requestThemes.length > 0 ? (
failures.requestThemes.slice(0, 10).map((item) => (
<Tr key={item.theme}>
<Td color="white">{item.theme}</Td>
<Td color="white" isNumeric>
{item.count.toLocaleString()}
</Td>
</Tr>
))
) : (
<Tr>
<Td colSpan={2} color="gray.500" textAlign="center">
No theme data available
</Td>
</Tr>
)}
</Tbody>
</Table>
</TableContainer>
</Box>
</SimpleGrid>

{/* What Platform Can't Do */}
<Box
bg="rgba(255, 255, 255, 0.05)"
p={6}
borderRadius="lg"
border="1px solid rgba(255, 255, 255, 0.1)"
mb={6}
>
<Heading size="md" color="white" mb={4}>
What Platform Cannot Accomplish
</Heading>
<TableContainer>
<Table variant="simple" colorScheme="whiteAlpha">
<Thead>
<Tr>
<Th color="gray.300">User Request</Th>
<Th color="gray.300">Failure Type</Th>
<Th color="gray.300">Summary</Th>
<Th color="gray.300">Date</Th>
</Tr>
</Thead>
<Tbody>
{failures.recentFailures.length > 0 ? (
failures.recentFailures.slice(0, 10).map((failure) => (
<Tr key={failure.id}>
<Td color="white" maxW="200px">
<Text
noOfLines={2}
fontSize="sm"
title={failure.user_prompt}
>
{failure.user_prompt.substring(0, 100)}
{failure.user_prompt.length > 100 ? '...' : ''}
</Text>
</Td>
<Td color="white">
{failure.failure_type || 'Unknown'}
</Td>
<Td color="white" maxW="300px">
<Text
noOfLines={2}
fontSize="sm"
title={failure.failure_summary}
>
{failure.failure_summary
? failure.failure_summary.substring(0, 150)
: 'N/A'}
{failure.failure_summary &&
failure.failure_summary.length > 150
? '...'
: ''}
</Text>
</Td>
<Td color="gray.400" fontSize="sm">
{new Date(
failure.created_at
).toLocaleDateString()}
</Td>
</Tr>
))
) : (
<Tr>
<Td colSpan={4} color="gray.500" textAlign="center">
No failure data available
</Td>
</Tr>
)}
</Tbody>
</Table>
</TableContainer>
</Box>

{/* Tag Distribution */}
{failures.tagDistribution.length > 0 && (
<Box
bg="rgba(255, 255, 255, 0.05)"
p={6}
borderRadius="lg"
border="1px solid rgba(255, 255, 255, 0.1)"
mb={6}
>
<Heading size="md" color="white" mb={4}>
Request Tags Distribution
</Heading>
<TableContainer>
<Table variant="simple" colorScheme="whiteAlpha">
<Thead>
<Tr>
<Th color="gray.300">Tag</Th>
<Th color="gray.300" isNumeric>
Count
</Th>
</Tr>
</Thead>
<Tbody>
{failures.tagDistribution.slice(0, 15).map((item) => (
<Tr key={item.tag}>
<Td color="white">{item.tag}</Td>
<Td color="white" isNumeric>
{item.count.toLocaleString()}
</Td>
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
</Box>
)}
</Box>
</>
)}

{/* Vercel Metrics */}
<Divider borderColor="rgba(255, 255, 255, 0.1)" />
<Box
Expand Down
40 changes: 40 additions & 0 deletions app/migrations/017-create-failure-metrics-table.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
-- Create failure_metrics table to track failed requests and what the platform couldn't accomplish
CREATE TABLE IF NOT EXISTS failure_metrics (
id SERIAL PRIMARY KEY,
job_id VARCHAR(255) NOT NULL,
message_id VARCHAR(255),
user_id VARCHAR(255),
wallet_address VARCHAR(255),
agent_name VARCHAR(255),

-- Request details
user_prompt TEXT NOT NULL,
assistant_response TEXT NOT NULL,

-- Failure analysis
is_failure BOOLEAN DEFAULT false,
failure_type VARCHAR(100), -- e.g., 'agent_not_found', 'capability_limitation', 'error', 'could_not_answer'
failure_reason TEXT, -- Extracted reason from response
failure_summary TEXT, -- Generated summary of what couldn't be done

-- Tagging and themes
detected_tags TEXT[], -- Array of tags/themes detected in the request
request_theme VARCHAR(255), -- Main theme/category of the request

-- Metadata
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

-- Create indexes for efficient querying
CREATE INDEX IF NOT EXISTS idx_failure_metrics_job_id ON failure_metrics(job_id);
CREATE INDEX IF NOT EXISTS idx_failure_metrics_user_id ON failure_metrics(user_id);
CREATE INDEX IF NOT EXISTS idx_failure_metrics_wallet_address ON failure_metrics(wallet_address);
CREATE INDEX IF NOT EXISTS idx_failure_metrics_is_failure ON failure_metrics(is_failure);
CREATE INDEX IF NOT EXISTS idx_failure_metrics_failure_type ON failure_metrics(failure_type);
CREATE INDEX IF NOT EXISTS idx_failure_metrics_created_at ON failure_metrics(created_at);
CREATE INDEX IF NOT EXISTS idx_failure_metrics_request_theme ON failure_metrics(request_theme);

-- Create GIN index for array searches on tags
CREATE INDEX IF NOT EXISTS idx_failure_metrics_tags ON failure_metrics USING GIN(detected_tags);

67 changes: 67 additions & 0 deletions app/pages/api/metrics/failures.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import { NextApiRequest, NextApiResponse } from 'next';
import { FailureMetricsDB } from '@/services/database/db';

export default async function handler(
req: NextApiRequest,
res: NextApiResponse
) {
if (req.method !== 'GET') {
return res.status(405).json({ error: 'Method not allowed' });
}

try {
const stats = await FailureMetricsDB.getFailureStats();
const recentFailures = await FailureMetricsDB.getFailureMetrics({
isFailure: true,
limit: 50,
});

// Get request themes (what people are requesting)
const allMetrics = await FailureMetricsDB.getFailureMetrics({
limit: 1000, // Get enough to analyze themes
});

// Group by theme
const themeCounts: Record<string, number> = {};
allMetrics.forEach((metric) => {
if (metric.request_theme) {
themeCounts[metric.request_theme] =
(themeCounts[metric.request_theme] || 0) + 1;
}
});

const requestThemes = Object.entries(themeCounts)
.map(([theme, count]) => ({ theme, count }))
.sort((a, b) => b.count - a.count)
.slice(0, 20);

// Get tag distribution
const tagCounts: Record<string, number> = {};
allMetrics.forEach((metric) => {
if (metric.detected_tags && Array.isArray(metric.detected_tags)) {
metric.detected_tags.forEach((tag) => {
tagCounts[tag] = (tagCounts[tag] || 0) + 1;
});
}
});

const tagDistribution = Object.entries(tagCounts)
.map(([tag, count]) => ({ tag, count }))
.sort((a, b) => b.count - a.count);

return res.status(200).json({
stats,
recentFailures: recentFailures.slice(0, 20),
requestThemes,
tagDistribution,
timestamp: new Date().toISOString(),
});
} catch (error) {
console.error('[Metrics Failures] Error:', error);
return res.status(500).json({
error:
error instanceof Error ? error.message : 'Internal server error',
});
}
}

Loading