diff --git a/api/models/__tests__/messageFeedback.spec.js b/api/models/__tests__/messageFeedback.spec.js new file mode 100644 index 00000000000..ee5971f72c3 --- /dev/null +++ b/api/models/__tests__/messageFeedback.spec.js @@ -0,0 +1,212 @@ +/** + * Unit tests for message feedback flow. + * + * Tests that feedback (thumbs up / thumbs down) is correctly stored, updated, + * and cleared via the updateMessage function, following the same + * MongoMemoryServer pattern used in api/models/Message.spec.js. + */ + +const mongoose = require('mongoose'); +const { v4: uuidv4 } = require('uuid'); +const { messageSchema } = require('@librechat/data-schemas'); +const { MongoMemoryServer } = require('mongodb-memory-server'); + +const { saveMessage, updateMessage } = require('../Message'); + +// Required to silence the Config/app module that is imported transitively. +jest.mock('~/server/services/Config/app'); + +/** @type {import('mongoose').Model} */ +let Message; + +describe('Message Feedback Flow', () => { + let mongoServer; + let mockReq; + let conversationId; + + beforeAll(async () => { + mongoServer = await MongoMemoryServer.create(); + const mongoUri = mongoServer.getUri(); + Message = mongoose.models.Message || mongoose.model('Message', messageSchema); + await mongoose.connect(mongoUri); + }); + + afterAll(async () => { + await mongoose.disconnect(); + await mongoServer.stop(); + }); + + beforeEach(async () => { + await Message.deleteMany({}); + + conversationId = uuidv4(); + + mockReq = { + user: { id: 'user123' }, + body: {}, + config: { + interfaceConfig: { + temporaryChatRetention: 24, + }, + }, + }; + + // Pre-save a base message that feedback tests will operate on. + await saveMessage(mockReq, { + messageId: 'msg-feedback-test', + conversationId, + text: 'Assistant response', + user: 'user123', + }); + }); + + // --------------------------------------------------------------------------- + // 1. Thumbs up + // --------------------------------------------------------------------------- + it('should store thumbs up feedback on a message', async () => { + const result = await updateMessage(mockReq, { + messageId: 'msg-feedback-test', + feedback: { rating: 'thumbsUp' }, + }); + + expect(result.feedback).toBeDefined(); + expect(result.feedback.rating).toBe('thumbsUp'); + + // Verify persistence in the database. + const dbMsg = await Message.findOne({ + messageId: 'msg-feedback-test', + user: 'user123', + }).lean(); + expect(dbMsg.feedback.rating).toBe('thumbsUp'); + }); + + // --------------------------------------------------------------------------- + // 2. Thumbs down + // --------------------------------------------------------------------------- + it('should store thumbs down feedback on a message', async () => { + const result = await updateMessage(mockReq, { + messageId: 'msg-feedback-test', + feedback: { rating: 'thumbsDown' }, + }); + + expect(result.feedback).toBeDefined(); + expect(result.feedback.rating).toBe('thumbsDown'); + + const dbMsg = await Message.findOne({ + messageId: 'msg-feedback-test', + user: 'user123', + }).lean(); + expect(dbMsg.feedback.rating).toBe('thumbsDown'); + }); + + // --------------------------------------------------------------------------- + // 3. Clearing feedback (setting to null) + // --------------------------------------------------------------------------- + it('should clear feedback when null is provided', async () => { + // First set feedback so there is something to clear. + await updateMessage(mockReq, { + messageId: 'msg-feedback-test', + feedback: { rating: 'thumbsUp' }, + }); + + // Now clear it via the same pattern used in the route handler: + // feedback: feedback || null where feedback is falsy/undefined. + const result = await updateMessage(mockReq, { + messageId: 'msg-feedback-test', + feedback: null, + }); + + // The returned object should have feedback as null / undefined / falsy. + expect(result.feedback == null).toBe(true); + + // Verify the database reflects the cleared state. + const dbMsg = await Message.findOne({ + messageId: 'msg-feedback-test', + user: 'user123', + }).lean(); + expect(dbMsg.feedback == null).toBe(true); + }); + + // --------------------------------------------------------------------------- + // 4. Feedback with a tag + // --------------------------------------------------------------------------- + it('should store a tag alongside the feedback rating', async () => { + const result = await updateMessage(mockReq, { + messageId: 'msg-feedback-test', + feedback: { rating: 'thumbsDown', tag: 'incorrect' }, + }); + + expect(result.feedback).toBeDefined(); + expect(result.feedback.rating).toBe('thumbsDown'); + expect(result.feedback.tag).toBe('incorrect'); + + const dbMsg = await Message.findOne({ + messageId: 'msg-feedback-test', + user: 'user123', + }).lean(); + expect(dbMsg.feedback.tag).toBe('incorrect'); + }); + + // --------------------------------------------------------------------------- + // 5. Feedback with text + // --------------------------------------------------------------------------- + it('should store text alongside the feedback rating', async () => { + const feedbackText = 'This answer was not helpful for my use case.'; + const result = await updateMessage(mockReq, { + messageId: 'msg-feedback-test', + feedback: { rating: 'thumbsDown', text: feedbackText }, + }); + + expect(result.feedback).toBeDefined(); + expect(result.feedback.rating).toBe('thumbsDown'); + expect(result.feedback.text).toBe(feedbackText); + + const dbMsg = await Message.findOne({ + messageId: 'msg-feedback-test', + user: 'user123', + }).lean(); + expect(dbMsg.feedback.text).toBe(feedbackText); + }); + + // --------------------------------------------------------------------------- + // 6. Non-existent message + // --------------------------------------------------------------------------- + it('should throw when submitting feedback to a non-existent message', async () => { + await expect( + updateMessage(mockReq, { + messageId: 'does-not-exist', + feedback: { rating: 'thumbsUp' }, + }), + ).rejects.toThrow('Message not found or user not authorized.'); + }); + + // --------------------------------------------------------------------------- + // 7. Updating existing feedback (thumbsUp -> thumbsDown) + // --------------------------------------------------------------------------- + it('should overwrite existing feedback when a new rating is submitted', async () => { + // Set initial thumbsUp feedback. + await updateMessage(mockReq, { + messageId: 'msg-feedback-test', + feedback: { rating: 'thumbsUp' }, + }); + + // Now switch to thumbsDown with additional context. + const result = await updateMessage(mockReq, { + messageId: 'msg-feedback-test', + feedback: { rating: 'thumbsDown', tag: 'wrong', text: 'Changed my mind.' }, + }); + + expect(result.feedback.rating).toBe('thumbsDown'); + expect(result.feedback.tag).toBe('wrong'); + expect(result.feedback.text).toBe('Changed my mind.'); + + // The database should reflect the latest value only. + const dbMsg = await Message.findOne({ + messageId: 'msg-feedback-test', + user: 'user123', + }).lean(); + expect(dbMsg.feedback.rating).toBe('thumbsDown'); + expect(dbMsg.feedback.tag).toBe('wrong'); + expect(dbMsg.feedback.text).toBe('Changed my mind.'); + }); +}); diff --git a/api/models/__tests__/productFeedback.spec.js b/api/models/__tests__/productFeedback.spec.js new file mode 100644 index 00000000000..ed48317f83c --- /dev/null +++ b/api/models/__tests__/productFeedback.spec.js @@ -0,0 +1,202 @@ +/** + * Unit tests for the product feedback (POST /api/feedback/issues) flow. + * + * Tests that ProductFeedback records are correctly created and persisted in + * MongoDB, following the same MongoMemoryServer pattern used by + * api/models/Message.spec.js and api/models/__tests__/messageFeedback.spec.js. + * + * All tests run against an in-memory MongoDB instance - no real MongoDB or + * librechat.yaml configuration is required. + */ + +'use strict'; + +const mongoose = require('mongoose'); +const { v4: uuidv4 } = require('uuid'); +const { productFeedbackSchema } = require('@librechat/data-schemas'); +const { MongoMemoryServer } = require('mongodb-memory-server'); + +/** @type {import('mongoose').Model} */ +let ProductFeedback; + +/** Minimum valid payload satisfying all required fields in the compiled schema. */ +const makeValidPayload = (overrides = {}) => ({ + request_id: uuidv4(), + user: 'user-abc123', + username: 'testuser', + feedback_reason: 'incorrect', + feedback_title: 'Something was wrong', + ...overrides, +}); + +describe('ProductFeedback Model', () => { + let mongoServer; + + beforeAll(async () => { + mongoServer = await MongoMemoryServer.create(); + const mongoUri = mongoServer.getUri(); + ProductFeedback = mongoose.models.ProductFeedback || mongoose.model('ProductFeedback', productFeedbackSchema); + await mongoose.connect(mongoUri); + }); + + afterAll(async () => { + await mongoose.disconnect(); + await mongoServer.stop(); + }); + + beforeEach(async () => { + await ProductFeedback.deleteMany({}); + }); + + it('should create and persist a ProductFeedback record with required fields', async () => { + const payload = makeValidPayload(); + const record = await ProductFeedback.create(payload); + expect(record._id).toBeDefined(); + const persisted = await ProductFeedback.findById(record._id).lean(); + expect(persisted).not.toBeNull(); + expect(persisted.request_id).toBe(payload.request_id); + expect(persisted.user).toBe(payload.user); + expect(persisted.feedback_reason).toBe(payload.feedback_reason); + expect(persisted.feedback_title).toBe(payload.feedback_title); + }); + + it('should return a record that exposes _id and request_id', async () => { + const requestId = uuidv4(); + const payload = makeValidPayload({ request_id: requestId }); + const record = await ProductFeedback.create(payload); + expect(record._id.toString()).toMatch(/^[a-f0-9]{24}$/i); + expect(record.request_id).toBe(requestId); + }); + + it('should reject a record that is missing feedback_reason', async () => { + const payload = makeValidPayload(); + delete payload.feedback_reason; + await expect(ProductFeedback.create(payload)).rejects.toThrow(); + }); + + it('should reject a record that is missing feedback_title', async () => { + const payload = makeValidPayload(); + delete payload.feedback_title; + await expect(ProductFeedback.create(payload)).rejects.toThrow(); + }); + + it('should reject a record that is missing user', async () => { + const payload = makeValidPayload(); + delete payload.user; + await expect(ProductFeedback.create(payload)).rejects.toThrow(); + }); + + it('should reject a record that is missing request_id', async () => { + const payload = makeValidPayload(); + delete payload.request_id; + await expect(ProductFeedback.create(payload)).rejects.toThrow(); + }); + + it('should reject a record with an invalid feedback_reason enum value', async () => { + const payload = makeValidPayload({ feedback_reason: 'not_a_valid_reason' }); + await expect(ProductFeedback.create(payload)).rejects.toThrow(); + }); + + it('should store user and username exactly as provided', async () => { + const payload = makeValidPayload({ user: 'user-xyz-789', username: 'alice' }); + const record = await ProductFeedback.create(payload); + const persisted = await ProductFeedback.findById(record._id).lean(); + expect(persisted.user).toBe('user-xyz-789'); + expect(persisted.username).toBe('alice'); + }); + + it('should preserve a caller-supplied request_id without modification', async () => { + const customRequestId = 'custom-req-id-' + uuidv4(); + const payload = makeValidPayload({ request_id: customRequestId }); + const record = await ProductFeedback.create(payload); + expect(record.request_id).toBe(customRequestId); + const persisted = await ProductFeedback.findById(record._id).lean(); + expect(persisted.request_id).toBe(customRequestId); + }); + + it('should reject a second record with the same request_id', async () => { + const sharedRequestId = uuidv4(); + await ProductFeedback.create(makeValidPayload({ request_id: sharedRequestId })); + await expect( + ProductFeedback.create(makeValidPayload({ request_id: sharedRequestId })), + ).rejects.toThrow(); + }); + + it('should store all optional fields: conversation, metadata, and contact', async () => { + const conversation = { + conversation_id: uuidv4(), + message_id: uuidv4(), + last_n_messages: [ + { timestamp: new Date().toISOString(), is_user: true, text: 'Hi' }, + { timestamp: new Date().toISOString(), is_user: false, text: 'Hello!' }, + ], + }; + const metadata = { + librechat_version: '0.7.0', + client: 'web', + endpoint: 'openAI', + model: 'gpt-4o', + agent_id: 'agent-001', + }; + const contact = { email: 'tester@example.com' }; + const payload = makeValidPayload({ + feedback_details: 'The answer missed the key point.', + feedback_suggested_fix: 'Include the referenced documentation.', + conversation, + metadata, + contact, + }); + const record = await ProductFeedback.create(payload); + const persisted = await ProductFeedback.findById(record._id).lean(); + expect(persisted.feedback_details).toBe('The answer missed the key point.'); + expect(persisted.feedback_suggested_fix).toBe('Include the referenced documentation.'); + expect(persisted.conversation.conversation_id).toBe(conversation.conversation_id); + expect(persisted.conversation.message_id).toBe(conversation.message_id); + expect(persisted.conversation.last_n_messages).toHaveLength(2); + expect(persisted.conversation.last_n_messages[0].is_user).toBe(true); + expect(persisted.metadata.librechat_version).toBe('0.7.0'); + expect(persisted.metadata.client).toBe('web'); + expect(persisted.metadata.endpoint).toBe('openAI'); + expect(persisted.metadata.model).toBe('gpt-4o'); + expect(persisted.metadata.agent_id).toBe('agent-001'); + expect(persisted.contact.email).toBe('tester@example.com'); + }); + + it('should add createdAt and updatedAt timestamps automatically', async () => { + const before = new Date(); + const record = await ProductFeedback.create(makeValidPayload()); + const after = new Date(); + const persisted = await ProductFeedback.findById(record._id).lean(); + expect(persisted.createdAt).toBeDefined(); + expect(persisted.updatedAt).toBeDefined(); + expect(new Date(persisted.createdAt).getTime()).toBeGreaterThanOrEqual(before.getTime() - 50); + expect(new Date(persisted.createdAt).getTime()).toBeLessThanOrEqual(after.getTime() + 50); + }); + + it.each([ + 'incorrect', + 'unfaithful', + 'safety_or_legal_concern', + 'style_tone_conciseness', + 'other', + ])('should accept feedback_reason = %s', async (reason) => { + const payload = makeValidPayload({ feedback_reason: reason, request_id: uuidv4() }); + const record = await ProductFeedback.create(payload); + expect(record.feedback_reason).toBe(reason); + }); + + it('should allow multiple records from different users in the same collection', async () => { + await ProductFeedback.create( + makeValidPayload({ user: 'user-111', username: 'alice', request_id: uuidv4() }), + ); + await ProductFeedback.create( + makeValidPayload({ user: 'user-222', username: 'bob', request_id: uuidv4() }), + ); + const aliceRecords = await ProductFeedback.find({ user: 'user-111' }).lean(); + const bobRecords = await ProductFeedback.find({ user: 'user-222' }).lean(); + expect(aliceRecords).toHaveLength(1); + expect(aliceRecords[0].username).toBe('alice'); + expect(bobRecords).toHaveLength(1); + expect(bobRecords[0].username).toBe('bob'); + }); +}); diff --git a/scripts/audit-feedback.js b/scripts/audit-feedback.js new file mode 100644 index 00000000000..b6c54ef1820 --- /dev/null +++ b/scripts/audit-feedback.js @@ -0,0 +1,530 @@ +#!/usr/bin/env node +/** + * audit-feedback.js + * + * Audits message feedback consistency in LibreChat's MongoDB. + * + * Compares feedback counts in the messages collection against the + * productfeedbacks collection and flags anomalies (missing rating field, + * invalid rating values, etc.). + * + * Usage: + * node scripts/audit-feedback.js [options] + * + * Options: + * --uri MongoDB connection string (overrides env / .env file) + * --since Only consider messages created on or after this date + * --until Only consider messages created on or before this date + * --json Output results as JSON instead of a formatted table + * --help Show this help text and exit + * + * MongoDB URI resolution order: + * 1. --uri CLI argument + * 2. MONGO_URI environment variable + * 3. MONGO_URI key in .env file at the project root + */ + +'use strict'; + +const path = require('path'); +const fs = require('fs'); + +// --------------------------------------------------------------------------- +// Help +// --------------------------------------------------------------------------- + +const HELP = ` +Usage: node scripts/audit-feedback.js [options] + +Options: + --uri MongoDB connection string + --since Filter messages created on or after this date (ISO 8601) + --until Filter messages created on or before this date (ISO 8601) + --json Output results as JSON + --help Show this help text and exit + +MongoDB URI is resolved from (in order): + 1. --uri CLI argument + 2. MONGO_URI environment variable + 3. MONGO_URI in .env file at the project root + +Examples: + node scripts/audit-feedback.js + node scripts/audit-feedback.js --uri mongodb://localhost:27017/LibreChat + node scripts/audit-feedback.js --since 2024-01-01 --until 2024-12-31 + node scripts/audit-feedback.js --json +`.trim(); + +// --------------------------------------------------------------------------- +// CLI argument parsing +// --------------------------------------------------------------------------- + +function parseArgs(argv) { + const args = argv.slice(2); + const result = { + uri: null, + since: null, + until: null, + json: false, + help: false, + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === '--help' || arg === '-h') { + result.help = true; + } else if (arg === '--json') { + result.json = true; + } else if (arg === '--uri') { + result.uri = args[++i]; + } else if (arg === '--since') { + result.since = args[++i]; + } else if (arg === '--until') { + result.until = args[++i]; + } else { + console.error(`Unknown argument: ${arg}`); + process.exit(1); + } + } + + return result; +} + +// --------------------------------------------------------------------------- +// .env file parser (minimal โ€” no external deps) +// --------------------------------------------------------------------------- + +function readDotEnv(filePath) { + const vars = {}; + if (!fs.existsSync(filePath)) { + return vars; + } + const lines = fs.readFileSync(filePath, 'utf8').split('\n'); + for (const rawLine of lines) { + const line = rawLine.trim(); + // Skip blank lines and comments + if (!line || line.startsWith('#')) { + continue; + } + const eqIdx = line.indexOf('='); + if (eqIdx === -1) { + continue; + } + const key = line.slice(0, eqIdx).trim(); + let value = line.slice(eqIdx + 1).trim(); + // Strip surrounding quotes + if ((value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'"))) { + value = value.slice(1, -1); + } + vars[key] = value; + } + return vars; +} + +// --------------------------------------------------------------------------- +// Resolve MongoDB URI +// --------------------------------------------------------------------------- + +function resolveMongoUri(cliUri) { + if (cliUri) { + return cliUri; + } + if (process.env.MONGO_URI) { + return process.env.MONGO_URI; + } + const projectRoot = path.resolve(__dirname, '..'); + const dotEnvPath = path.join(projectRoot, '.env'); + const dotEnvVars = readDotEnv(dotEnvPath); + if (dotEnvVars.MONGO_URI) { + return dotEnvVars.MONGO_URI; + } + return null; +} + +// --------------------------------------------------------------------------- +// Validate date strings +// --------------------------------------------------------------------------- + +function parseDate(str, label) { + const d = new Date(str); + if (isNaN(d.getTime())) { + console.error(`Invalid ${label} date: "${str}". Use an ISO 8601 date, e.g. 2024-01-01`); + process.exit(1); + } + return d; +} + +// --------------------------------------------------------------------------- +// Main audit logic +// --------------------------------------------------------------------------- + +async function audit({ mongoUri, since, until }) { + // Require mongoose at runtime so the script can be used without bundling + let mongoose; + try { + mongoose = require('mongoose'); + } catch (e) { + console.error('Could not require mongoose. Make sure you run this script from the LibreChat project root with dependencies installed.'); + console.error(e.message); + process.exit(1); + } + + // ------------------------------------------------------------------ + // Connect + // ------------------------------------------------------------------ + await mongoose.connect(mongoUri, { + serverSelectionTimeoutMS: 10000, + connectTimeoutMS: 10000, + }); + + // ------------------------------------------------------------------ + // Define lightweight schemas for read-only access + // + // The feedback sub-schema uses Mixed so we can inspect raw stored data + // regardless of schema enforcement, which is exactly what we need for + // detecting malformed documents. + // ------------------------------------------------------------------ + + const feedbackSubSchema = new mongoose.Schema( + { + rating: mongoose.Schema.Types.Mixed, + tag: mongoose.Schema.Types.Mixed, + text: mongoose.Schema.Types.Mixed, + }, + { _id: false }, + ); + + const messageSchema = new mongoose.Schema( + { + messageId: String, + conversationId: String, + user: String, + isCreatedByUser: Boolean, + feedback: feedbackSubSchema, + createdAt: Date, + updatedAt: Date, + }, + { strict: false, timestamps: false }, + ); + + const productFeedbackSchema = new mongoose.Schema( + { + request_id: String, + user: String, + feedback_reason: String, + createdAt: Date, + }, + { strict: false, timestamps: false }, + ); + + // Use existing models if already registered (idempotency within the session) + const Message = + mongoose.models.AuditMessage || + mongoose.model('AuditMessage', messageSchema, 'messages'); + + const ProductFeedback = + mongoose.models.AuditProductFeedback || + mongoose.model('AuditProductFeedback', productFeedbackSchema, 'productfeedbacks'); + + // ------------------------------------------------------------------ + // Build date filter + // ------------------------------------------------------------------ + + const dateFilter = {}; + if (since) { + dateFilter.$gte = since; + } + if (until) { + dateFilter.$lte = until; + } + const baseFilter = Object.keys(dateFilter).length > 0 ? { createdAt: dateFilter } : {}; + + // ------------------------------------------------------------------ + // Query: messages with feedback field present (including null/empty) + // ------------------------------------------------------------------ + + const feedbackExistsFilter = { + ...baseFilter, + feedback: { $exists: true, $ne: null }, + }; + + // Total messages scanned + const totalScanned = await Message.countDocuments(baseFilter); + + // Messages where feedback field exists + const withFeedbackCount = await Message.countDocuments(feedbackExistsFilter); + + // Aggregate by rating value (valid and invalid) + const ratingAgg = await Message.aggregate([ + { $match: feedbackExistsFilter }, + { + $group: { + _id: '$feedback.rating', + count: { $sum: 1 }, + }, + }, + ]); + + let thumbsUpCount = 0; + let thumbsDownCount = 0; + let unknownRatingCount = 0; + + // rating values as found in the DB (may include unexpected values) + const ratingBreakdown = {}; + + for (const bucket of ratingAgg) { + const ratingVal = bucket._id; + ratingBreakdown[ratingVal === null || ratingVal === undefined ? '(null/undefined)' : String(ratingVal)] = bucket.count; + + if (ratingVal === 'thumbsUp') { + thumbsUpCount += bucket.count; + } else if (ratingVal === 'thumbsDown') { + thumbsDownCount += bucket.count; + } else { + unknownRatingCount += bucket.count; + } + } + + // Messages where feedback exists but rating is missing or not a valid enum value + const malformedFilter = { + ...baseFilter, + feedback: { $exists: true, $ne: null }, + 'feedback.rating': { $not: { $in: ['thumbsUp', 'thumbsDown'] } }, + }; + const malformedCount = await Message.countDocuments(malformedFilter); + + // Sample of malformed docs for inspection + const malformedSamples = malformedCount > 0 + ? await Message.find(malformedFilter, { + messageId: 1, + conversationId: 1, + user: 1, + feedback: 1, + createdAt: 1, + }).limit(5).lean() + : []; + + // ------------------------------------------------------------------ + // Query: ProductFeedback collection + // ------------------------------------------------------------------ + + const productFeedbackFilter = Object.keys(dateFilter).length > 0 + ? { createdAt: dateFilter } + : {}; + + const productFeedbackCount = await ProductFeedback.countDocuments(productFeedbackFilter); + + const productFeedbackByReason = await ProductFeedback.aggregate([ + { $match: productFeedbackFilter }, + { + $group: { + _id: { $ifNull: ['$feedback_reason', '(not set)'] }, + count: { $sum: 1 }, + }, + }, + { $sort: { count: -1 } }, + ]); + + // ------------------------------------------------------------------ + // Detect anomalies + // ------------------------------------------------------------------ + + const anomalies = []; + + if (malformedCount > 0) { + anomalies.push({ + type: 'MALFORMED_FEEDBACK', + description: `${malformedCount} message(s) have a feedback field but are missing a valid rating ('thumbsUp' or 'thumbsDown').`, + count: malformedCount, + }); + } + + if (unknownRatingCount > 0) { + anomalies.push({ + type: 'UNKNOWN_RATING_VALUE', + description: `${unknownRatingCount} message(s) have an unrecognised feedback.rating value.`, + count: unknownRatingCount, + }); + } + + if (withFeedbackCount !== thumbsUpCount + thumbsDownCount + malformedCount) { + // Sanity check: counts should reconcile + anomalies.push({ + type: 'COUNT_MISMATCH', + description: 'Internal count reconciliation failed โ€” total with feedback does not equal thumbsUp + thumbsDown + malformed.', + detail: { + withFeedbackCount, + thumbsUpCount, + thumbsDownCount, + malformedCount, + sum: thumbsUpCount + thumbsDownCount + malformedCount, + }, + }); + } + + // ------------------------------------------------------------------ + // Compose results + // ------------------------------------------------------------------ + + return { + queryFilters: { + since: since ? since.toISOString() : null, + until: until ? until.toISOString() : null, + }, + messages: { + totalScanned, + withFeedback: withFeedbackCount, + thumbsUp: thumbsUpCount, + thumbsDown: thumbsDownCount, + malformed: malformedCount, + ratingBreakdown, + }, + productFeedback: { + total: productFeedbackCount, + byReason: productFeedbackByReason.reduce((acc, b) => { + acc[b._id] = b.count; + return acc; + }, {}), + }, + anomalies, + malformedSamples: malformedSamples.map((doc) => ({ + messageId: doc.messageId, + conversationId: doc.conversationId, + user: doc.user, + feedback: doc.feedback, + createdAt: doc.createdAt, + })), + }; +} + +// --------------------------------------------------------------------------- +// Output helpers +// --------------------------------------------------------------------------- + +function printTable(results) { + const { queryFilters, messages, productFeedback, anomalies, malformedSamples } = results; + + const divider = 'โ”€'.repeat(60); + + console.log('\n' + divider); + console.log(' LibreChat Feedback Audit Report'); + console.log(divider); + + if (queryFilters.since || queryFilters.until) { + const parts = []; + if (queryFilters.since) { + parts.push(`since ${queryFilters.since}`); + } + if (queryFilters.until) { + parts.push(`until ${queryFilters.until}`); + } + console.log(` Date filter : ${parts.join(' and ')}`); + console.log(divider); + } + + console.log('\n[Messages Collection]'); + console.log(` Total messages scanned : ${messages.totalScanned.toLocaleString()}`); + console.log(` Messages with feedback : ${messages.withFeedback.toLocaleString()}`); + console.log(` thumbsUp : ${messages.thumbsUp.toLocaleString()}`); + console.log(` thumbsDown : ${messages.thumbsDown.toLocaleString()}`); + console.log(` Malformed / no rating : ${messages.malformed.toLocaleString()}`); + + if (Object.keys(messages.ratingBreakdown).length > 0) { + console.log('\n Rating value breakdown (raw stored values):'); + for (const [val, count] of Object.entries(messages.ratingBreakdown)) { + console.log(` "${val}" : ${count.toLocaleString()}`); + } + } + + console.log('\n[ProductFeedback Collection]'); + console.log(` Total product feedback : ${productFeedback.total.toLocaleString()}`); + if (Object.keys(productFeedback.byReason).length > 0) { + console.log(' By feedback_reason:'); + for (const [reason, count] of Object.entries(productFeedback.byReason)) { + console.log(` "${reason}" : ${count.toLocaleString()}`); + } + } + + console.log('\n[Anomalies]'); + if (anomalies.length === 0) { + console.log(' No anomalies detected.'); + } else { + for (const anomaly of anomalies) { + console.log(` [${anomaly.type}] ${anomaly.description}`); + if (anomaly.detail) { + console.log(` Detail: ${JSON.stringify(anomaly.detail)}`); + } + } + } + + if (malformedSamples.length > 0) { + console.log('\n[Malformed Feedback Samples (up to 5)]'); + for (const doc of malformedSamples) { + console.log(` messageId : ${doc.messageId}`); + console.log(` conversationId: ${doc.conversationId}`); + console.log(` feedback : ${JSON.stringify(doc.feedback)}`); + console.log(` createdAt : ${doc.createdAt}`); + console.log(' ' + 'ยท'.repeat(40)); + } + } + + console.log('\n' + divider + '\n'); +} + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- + +async function main() { + const opts = parseArgs(process.argv); + + if (opts.help) { + console.log(HELP); + process.exit(0); + } + + const mongoUri = resolveMongoUri(opts.uri); + if (!mongoUri) { + console.error( + 'No MongoDB URI found. Provide one via --uri, the MONGO_URI environment variable, or a .env file in the project root.', + ); + process.exit(1); + } + + const since = opts.since ? parseDate(opts.since, '--since') : null; + const until = opts.until ? parseDate(opts.until, '--until') : null; + + let results; + try { + results = await audit({ mongoUri, since, until }); + } catch (err) { + console.error('Audit failed:', err.message); + process.exit(1); + } finally { + // Always disconnect + try { + const mongoose = require('mongoose'); + await mongoose.disconnect(); + } catch (_) { + // ignore disconnect errors + } + } + + if (opts.json) { + console.log(JSON.stringify(results, null, 2)); + } else { + printTable(results); + } + + // Exit with a non-zero code when anomalies are detected so the script can + // be used in CI pipelines or alerting workflows. + if (results.anomalies.length > 0) { + process.exit(2); + } +} + +main().catch((err) => { + console.error('Unexpected error:', err); + process.exit(1); +});