fix: job history retention

docs/agenstra/deployment/background-jobs.md

@@ -63,6 +63,8 @@ When enabled on the API container (`QUEUE_BULL_BOARD_ENABLED=true`, default in c
 
 Bull Board uses **HTTP Basic authentication** (`QUEUE_BULL_BOARD_USERNAME` / `QUEUE_BULL_BOARD_PASSWORD`). Local compose defaults to `admin` / `bullmq`; override in production. Startup fails in production if the board is enabled without a password.
 
+Completed and failed jobs are **not auto-removed** (`removeOnComplete: false`, `removeOnFail: false`) so run history stays in Bull Board. Treat the **last three runs** and **48 hours** as the minimum retention before any manual cleanup via Bull Board or ops.
+
 Bull Board routes bypass the API **origin allowlist**, **HybridAuthGuard**, and **Keycloak guards** (when `AUTHENTICATION_METHOD=keycloak`) so dashboard actions (retry, delete, clean) are not blocked with `403 Forbidden` when the UI sends browser `Origin` headers or `Authorization: Basic` instead of the API key or OIDC token.
 
 Worker and scheduler containers set `QUEUE_BULL_BOARD_ENABLED=false` so they do not start an HTTP server solely for Bull Board.

libs/domains/shared/backend/util-queue/src/index.ts

@@ -4,6 +4,7 @@ export * from './lib/bull-board-global-prefix';
 export * from './lib/enqueue-unit-job';
 export * from './lib/is-duplicate-job-enqueue-error';
 export * from './lib/job-id.util';
+export * from './lib/job-retention';
 export * from './lib/register-repeatable-coordinator-job';
 export * from './lib/queue-connection.config';
 export * from './lib/queue-role';

libs/domains/shared/backend/util-queue/src/lib/enqueue-unit-job.spec.ts

@@ -1,5 +1,6 @@
 import type { Queue } from 'bullmq';
 
+import { defaultRemoveOnComplete, defaultRemoveOnFail } from './job-retention';
 import { enqueueUnitJob } from './enqueue-unit-job';
 
 describe('enqueueUnitJob', () => {
@@ -18,7 +19,11 @@ describe('enqueueUnitJob', () => {
     expect(add).toHaveBeenCalledWith(
       'billing.subscription.unit',
       { subscriptionId: 'abc' },
-      expect.objectContaining({ jobId: 'billing.subscription.abc' }),
+      expect.objectContaining({
+        jobId: 'billing.subscription.abc',
+        removeOnComplete: defaultRemoveOnComplete,
+        removeOnFail: defaultRemoveOnFail,
+      }),
     );
   });
 

libs/domains/shared/backend/util-queue/src/lib/enqueue-unit-job.ts

@@ -1,6 +1,7 @@
 import type { JobsOptions, Queue } from 'bullmq';
 
 import { isDuplicateJobEnqueueError } from './is-duplicate-job-enqueue-error';
+import { defaultRemoveOnComplete, defaultRemoveOnFail } from './job-retention';
 import { buildJobId } from './job-id.util';
 
 export interface EnqueueUnitJobOptions<T> {
@@ -19,8 +20,8 @@ export async function enqueueUnitJob<T>(options: EnqueueUnitJobOptions<T>): Prom
   try {
     await options.queue.add(options.jobName, options.payload, {
       jobId,
-      removeOnComplete: { age: 3600, count: 1000 },
-      removeOnFail: { age: 86400, count: 5000 },
+      removeOnComplete: defaultRemoveOnComplete,
+      removeOnFail: defaultRemoveOnFail,
       attempts: 3,
       backoff: { type: 'exponential', delay: 5000 },
       ...options.opts,

libs/domains/shared/backend/util-queue/src/lib/job-retention.spec.ts

@@ -0,0 +1,18 @@
+import {
+  BULL_BOARD_JOB_RETENTION_AGE_SECONDS,
+  BULL_BOARD_JOB_RETENTION_COUNT,
+  defaultRemoveOnComplete,
+  defaultRemoveOnFail,
+} from './job-retention';
+
+describe('job retention defaults', () => {
+  it('disables automatic removal of completed and failed jobs', () => {
+    expect(defaultRemoveOnComplete).toBe(false);
+    expect(defaultRemoveOnFail).toBe(false);
+  });
+
+  it('documents the minimum Bull Board visibility policy', () => {
+    expect(BULL_BOARD_JOB_RETENTION_COUNT).toBe(3);
+    expect(BULL_BOARD_JOB_RETENTION_AGE_SECONDS).toBe(48 * 60 * 60);
+  });
+});

libs/domains/shared/backend/util-queue/src/lib/job-retention.ts

@@ -0,0 +1,15 @@
+/** Documented minimum Bull Board visibility before any manual cleanup. */
+export const BULL_BOARD_JOB_RETENTION_COUNT = 3;
+
+/** Documented minimum Bull Board visibility in seconds (48 hours). */
+export const BULL_BOARD_JOB_RETENTION_AGE_SECONDS = 48 * 60 * 60;
+
+/**
+ * Do not auto-remove completed jobs so Bull Board keeps run history.
+ * Jobs should remain visible for at least the last three runs and 48 hours;
+ * automatic trimming is disabled and cleanup is manual via Bull Board or ops.
+ */
+export const defaultRemoveOnComplete = false;
+
+/** Do not auto-remove failed jobs so Bull Board keeps error history. */
+export const defaultRemoveOnFail = false;

libs/domains/shared/backend/util-queue/src/lib/queue.module.ts

@@ -6,6 +6,7 @@ import { DynamicModule, Module } from '@nestjs/common';
 import type { QueueOptions } from 'bullmq';
 
 import { createBullBoardAuthMiddlewareFromEnv } from './bull-board-auth';
+import { defaultRemoveOnComplete, defaultRemoveOnFail } from './job-retention';
 import { shouldEnableBullBoard, shouldRegisterRepeatableJobs, shouldRunQueueWorkers } from './queue-role';
 import {
   readBullBoardPath,
@@ -32,8 +33,8 @@ export class SharedQueueModule {
     const concurrency = options.workerConcurrency ?? readQueueWorkerConcurrency();
 
     const defaultJobOptions: QueueOptions['defaultJobOptions'] = {
-      removeOnComplete: { age: 3600, count: 1000 },
-      removeOnFail: { age: 86400, count: 5000 },
+      removeOnComplete: defaultRemoveOnComplete,
+      removeOnFail: defaultRemoveOnFail,
       attempts: 3,
       backoff: { type: 'exponential', delay: 5000 },
     };

libs/domains/shared/backend/util-queue/src/lib/register-repeatable-coordinator-job.spec.ts

@@ -1,5 +1,6 @@
 import type { Queue } from 'bullmq';
 
+import { defaultRemoveOnComplete, defaultRemoveOnFail } from './job-retention';
 import { registerRepeatableCoordinatorJob } from './register-repeatable-coordinator-job';
 
 describe('registerRepeatableCoordinatorJob', () => {
@@ -30,6 +31,8 @@ describe('registerRepeatableCoordinatorJob', () => {
       expect.objectContaining({
         jobId: 'coordinator.filter-rules-sync',
         repeat: { every: 30_000 },
+        removeOnComplete: defaultRemoveOnComplete,
+        removeOnFail: defaultRemoveOnFail,
       }),
     );
   });
@@ -50,6 +53,13 @@ describe('registerRepeatableCoordinatorJob', () => {
     });
 
     expect(queue.removeRepeatableByKey).not.toHaveBeenCalled();
-    expect(add).toHaveBeenCalledTimes(1);
+    expect(add).toHaveBeenCalledWith(
+      'billing.coordinator',
+      {},
+      expect.objectContaining({
+        removeOnComplete: defaultRemoveOnComplete,
+        removeOnFail: defaultRemoveOnFail,
+      }),
+    );
   });
 });

libs/domains/shared/backend/util-queue/src/lib/register-repeatable-coordinator-job.ts

@@ -1,5 +1,7 @@
 import type { JobsOptions, Queue } from 'bullmq';
 
+import { defaultRemoveOnComplete, defaultRemoveOnFail } from './job-retention';
+
 export interface RegisterRepeatableCoordinatorJobOptions {
   queue: Queue;
   name: string;
@@ -31,8 +33,8 @@ export async function registerRepeatableCoordinatorJob(
     {
       jobId: options.coordinatorJobId,
       repeat: { every: options.everyMs },
-      removeOnComplete: options.removeOnComplete ?? true,
-      removeOnFail: options.removeOnFail ?? 100,
+      removeOnComplete: options.removeOnComplete ?? defaultRemoveOnComplete,
+      removeOnFail: options.removeOnFail ?? defaultRemoveOnFail,
     },
   );
 }