diff --git a/apps/backend-agent-controller/.start-containers.env.example b/apps/backend-agent-controller/.start-containers.env.example index 29c86f25..fb547874 100644 --- a/apps/backend-agent-controller/.start-containers.env.example +++ b/apps/backend-agent-controller/.start-containers.env.example @@ -36,3 +36,17 @@ SMTP_PORT= SMTP_USER= SMTP_PASSWORD= EMAIL_FROM=noreply@localhost + +# Redis / BullMQ (queue provider in docker-compose) +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_PASSWORD= +REDIS_DB=0 +REDIS_KEY_PREFIX=agenstra-controller +REDIS_HOST_PORT=6379 +QUEUE_ROLE=all +QUEUE_WORKER_CONCURRENCY=5 +QUEUE_BULL_BOARD_ENABLED=true +QUEUE_BULL_BOARD_PATH=/admin/queues +QUEUE_BULL_BOARD_USERNAME=admin +QUEUE_BULL_BOARD_PASSWORD=bullmq diff --git a/apps/backend-agent-controller/docker-compose.yaml b/apps/backend-agent-controller/docker-compose.yaml index b80a82d7..32e8f726 100644 --- a/apps/backend-agent-controller/docker-compose.yaml +++ b/apps/backend-agent-controller/docker-compose.yaml @@ -1,3 +1,66 @@ +x-backend-agent-controller-environment: &backend-agent-controller-environment + HOST: ${HOST:-0.0.0.0} + PORT: ${PORT:-3100} + WEBSOCKET_PORT: ${WEBSOCKET_PORT:-8081} + WEBSOCKET_NAMESPACE: ${WEBSOCKET_NAMESPACE:-clients} + WEBSOCKET_CORS_ORIGIN: ${WEBSOCKET_CORS_ORIGIN:-*} + NODE_ENV: ${NODE_ENV:-development} + AUTO_ENRICH_ENABLED_GLOBAL: ${AUTO_ENRICH_ENABLED_GLOBAL:-true} + AUTO_ENRICH_VECTOR_ENABLED: ${AUTO_ENRICH_VECTOR_ENABLED:-true} + AUTO_ENRICH_MAX_SECTIONS: ${AUTO_ENRICH_MAX_SECTIONS:-6} + AUTO_ENRICH_MAX_CHARS: ${AUTO_ENRICH_MAX_CHARS:-12000} + AUTO_ENRICH_VECTOR_TOP_K: ${AUTO_ENRICH_VECTOR_TOP_K:-20} + AUTO_ENRICH_VECTOR_MAX_COSINE_DISTANCE: ${AUTO_ENRICH_VECTOR_MAX_COSINE_DISTANCE:-1} + KNOWLEDGE_EMBEDDINGS_REINDEX_INTERVAL_MS: ${KNOWLEDGE_EMBEDDINGS_REINDEX_INTERVAL_MS:-3600000} + CONTEXT_IMPORT_SCHEDULER_INTERVAL_MS: ${CONTEXT_IMPORT_SCHEDULER_INTERVAL_MS:-120000} + CONTEXT_IMPORT_SCHEDULER_CONFIG_BATCH: ${CONTEXT_IMPORT_SCHEDULER_CONFIG_BATCH:-3} + CONTEXT_IMPORT_ITEM_BUDGET: ${CONTEXT_IMPORT_ITEM_BUDGET:-25} + FILTER_RULES_SYNC_INTERVAL_MS: ${FILTER_RULES_SYNC_INTERVAL_MS:-30000} + FILTER_RULES_SYNC_BATCH_SIZE: ${FILTER_RULES_SYNC_BATCH_SIZE:-10} + AUTONOMOUS_TICKET_SCHEDULER_INTERVAL_MS: ${AUTONOMOUS_TICKET_SCHEDULER_INTERVAL_MS:-60000} + AUTONOMOUS_TICKET_SCHEDULER_BATCH_SIZE: ${AUTONOMOUS_TICKET_SCHEDULER_BATCH_SIZE:-5} + DB_HOST: ${DB_HOST:-postgres} + DB_PORT: ${DB_PORT:-5432} + DB_USERNAME: ${DB_USERNAME:-postgres} + DB_PASSWORD: ${DB_PASSWORD:-postgres} + DB_DATABASE: ${DB_DATABASE:-postgres} + REDIS_HOST: ${REDIS_HOST:-redis} + REDIS_PORT: ${REDIS_PORT:-6379} + REDIS_PASSWORD: ${REDIS_PASSWORD:-} + REDIS_DB: ${REDIS_DB:-0} + REDIS_KEY_PREFIX: ${REDIS_KEY_PREFIX:-agenstra-controller} + QUEUE_WORKER_CONCURRENCY: ${QUEUE_WORKER_CONCURRENCY:-5} + QUEUE_BULL_BOARD_ENABLED: ${QUEUE_BULL_BOARD_ENABLED:-true} + QUEUE_BULL_BOARD_PATH: ${QUEUE_BULL_BOARD_PATH:-/admin/queues} + QUEUE_BULL_BOARD_USERNAME: ${QUEUE_BULL_BOARD_USERNAME:-admin} + QUEUE_BULL_BOARD_PASSWORD: ${QUEUE_BULL_BOARD_PASSWORD:-bullmq} + HETZNER_API_TOKEN: ${HETZNER_API_TOKEN:-} + DIGITALOCEAN_API_TOKEN: ${DIGITALOCEAN_API_TOKEN:-} + AUTHENTICATION_METHOD: ${AUTHENTICATION_METHOD:-api-key} + DISABLE_SIGNUP: ${DISABLE_SIGNUP:-false} + STATIC_API_KEY: ${STATIC_API_KEY:-} + ENCRYPTION_KEY: ${ENCRYPTION_KEY:-} + CORS_ORIGIN: ${CORS_ORIGIN:-} + RATE_LIMIT_ENABLED: ${RATE_LIMIT_ENABLED:-} + RATE_LIMIT_TTL: ${RATE_LIMIT_TTL:-60} + RATE_LIMIT_LIMIT: ${RATE_LIMIT_LIMIT:-100} + KEYCLOAK_SERVER_URL: ${KEYCLOAK_SERVER_URL:-} + KEYCLOAK_AUTH_SERVER_URL: ${KEYCLOAK_AUTH_SERVER_URL:-} + KEYCLOAK_REALM: ${KEYCLOAK_REALM:-} + KEYCLOAK_CLIENT_ID: ${KEYCLOAK_CLIENT_ID:-} + KEYCLOAK_CLIENT_SECRET: ${KEYCLOAK_CLIENT_SECRET:-} + KEYCLOAK_TOKEN_VALIDATION: ${KEYCLOAK_TOKEN_VALIDATION:-online} + JWT_SECRET: ${JWT_SECRET:-} + SMTP_HOST: ${SMTP_HOST:-mailhog} + SMTP_PORT: ${SMTP_PORT:-1025} + SMTP_USER: ${SMTP_USER:-} + SMTP_PASSWORD: ${SMTP_PASSWORD:-} + EMAIL_FROM: ${EMAIL_FROM:-noreply@localhost} + CLIENT_ENDPOINT_ALLOWED_HOSTS: ${CLIENT_ENDPOINT_ALLOWED_HOSTS:-} + CLIENT_ENDPOINT_ALLOW_INSECURE_HTTP: ${CLIENT_ENDPOINT_ALLOW_INSECURE_HTTP:-true} + CLIENT_ENDPOINT_TLS_REJECT_UNAUTHORIZED: ${CLIENT_ENDPOINT_TLS_REJECT_UNAUTHORIZED:-false} + CLIENT_ENDPOINT_ALLOW_INTERNAL_HOST: ${CLIENT_ENDPOINT_ALLOW_INTERNAL_HOST:-true} + services: postgres: image: pgvector/pgvector:pg16 @@ -17,79 +80,74 @@ services: - agent-controller-network restart: unless-stopped + redis: + image: redis:7-alpine + container_name: agent-controller-redis + command: ['redis-server', '--appendonly', 'yes'] + volumes: + - redis_data:/data + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + interval: 10s + timeout: 5s + retries: 5 + ports: + - '${REDIS_HOST_PORT:-6379}:6379' + networks: + - agent-controller-network + restart: unless-stopped + backend-agent-controller: image: ghcr.io/forepath/agenstra-controller-api:latest pull_policy: never container_name: agent-controller-api environment: - # Backend API configuration - HOST: ${HOST:-0.0.0.0} - PORT: ${PORT:-3100} - WEBSOCKET_PORT: ${WEBSOCKET_PORT:-8081} - WEBSOCKET_NAMESPACE: ${WEBSOCKET_NAMESPACE:-clients} - WEBSOCKET_CORS_ORIGIN: ${WEBSOCKET_CORS_ORIGIN:-*} - NODE_ENV: ${NODE_ENV:-development} - # Prompt auto-enrichment (Socket.IO → agent-manager; vector path uses controller env) - AUTO_ENRICH_ENABLED_GLOBAL: ${AUTO_ENRICH_ENABLED_GLOBAL:-true} - AUTO_ENRICH_VECTOR_ENABLED: ${AUTO_ENRICH_VECTOR_ENABLED:-true} - AUTO_ENRICH_MAX_SECTIONS: ${AUTO_ENRICH_MAX_SECTIONS:-6} - AUTO_ENRICH_MAX_CHARS: ${AUTO_ENRICH_MAX_CHARS:-12000} - AUTO_ENRICH_VECTOR_TOP_K: ${AUTO_ENRICH_VECTOR_TOP_K:-20} - # Max pgvector cosine distance (<=>) for knowledge chunks in vector auto-enrichment (0–2; default 1) - AUTO_ENRICH_VECTOR_MAX_COSINE_DISTANCE: ${AUTO_ENRICH_VECTOR_MAX_COSINE_DISTANCE:-1} - # Full knowledge-embedding reindex interval (ms); set <= 0 to disable scheduler - KNOWLEDGE_EMBEDDINGS_REINDEX_INTERVAL_MS: ${KNOWLEDGE_EMBEDDINGS_REINDEX_INTERVAL_MS:-3600000} - # Database configuration - DB_HOST: ${DB_HOST:-postgres} - DB_PORT: ${DB_PORT:-5432} - DB_USERNAME: ${DB_USERNAME:-postgres} - DB_PASSWORD: ${DB_PASSWORD:-postgres} - DB_DATABASE: ${DB_DATABASE:-postgres} - # Environment variables for the provisioning providers - HETZNER_API_TOKEN: ${HETZNER_API_TOKEN:-} - DIGITALOCEAN_API_TOKEN: ${DIGITALOCEAN_API_TOKEN:-} - # Authentication method configuration - AUTHENTICATION_METHOD: ${AUTHENTICATION_METHOD:-api-key} - # Environment variables for disabling signup - DISABLE_SIGNUP: ${DISABLE_SIGNUP:-false} - # Static API key authentication configuration - STATIC_API_KEY: ${STATIC_API_KEY:-} - # Environment variables for encryption - ENCRYPTION_KEY: ${ENCRYPTION_KEY:-} - # CORS configuration (comma-separated list of allowed origins) - # In production: CORS is disabled by default. Set CORS_ORIGIN to allow specific origins. - # In development: CORS allows all origins by default. - CORS_ORIGIN: ${CORS_ORIGIN:-} - # Rate limiting configuration - RATE_LIMIT_ENABLED: ${RATE_LIMIT_ENABLED:-} - RATE_LIMIT_TTL: ${RATE_LIMIT_TTL:-60} - RATE_LIMIT_LIMIT: ${RATE_LIMIT_LIMIT:-100} - # Keycloak configuration - KEYCLOAK_SERVER_URL: ${KEYCLOAK_SERVER_URL:-} - KEYCLOAK_AUTH_SERVER_URL: ${KEYCLOAK_AUTH_SERVER_URL:-} - KEYCLOAK_REALM: ${KEYCLOAK_REALM:-} - KEYCLOAK_CLIENT_ID: ${KEYCLOAK_CLIENT_ID:-} - KEYCLOAK_CLIENT_SECRET: ${KEYCLOAK_CLIENT_SECRET:-} - KEYCLOAK_TOKEN_VALIDATION: ${KEYCLOAK_TOKEN_VALIDATION:-online} - # Environment variables for users authentication (when AUTHENTICATION_METHOD=users) - JWT_SECRET: ${JWT_SECRET:-} - # SMTP / MailHog configuration (for email confirmation and password reset) - SMTP_HOST: ${SMTP_HOST:-mailhog} - SMTP_PORT: ${SMTP_PORT:-1025} - SMTP_USER: ${SMTP_USER:-} - SMTP_PASSWORD: ${SMTP_PASSWORD:-} - EMAIL_FROM: ${EMAIL_FROM:-noreply@localhost} - # Client workspace endpoint configuration - CLIENT_ENDPOINT_ALLOWED_HOSTS: ${CLIENT_ENDPOINT_ALLOWED_HOSTS:-} - CLIENT_ENDPOINT_ALLOW_INSECURE_HTTP: ${CLIENT_ENDPOINT_ALLOW_INSECURE_HTTP:-true} - CLIENT_ENDPOINT_TLS_REJECT_UNAUTHORIZED: ${CLIENT_ENDPOINT_TLS_REJECT_UNAUTHORIZED:-false} - CLIENT_ENDPOINT_ALLOW_INTERNAL_HOST: ${CLIENT_ENDPOINT_ALLOW_INTERNAL_HOST:-true} + <<: *backend-agent-controller-environment + QUEUE_ROLE: ${QUEUE_ROLE:-api} ports: - '${PORT:-3100}:${PORT:-3100}' - '${WEBSOCKET_PORT:-8081}:${WEBSOCKET_PORT:-8081}' depends_on: postgres: condition: service_healthy + redis: + condition: service_healthy + mailhog: + condition: service_started + networks: + - agent-controller-network + restart: unless-stopped + + backend-agent-controller-worker: + image: ghcr.io/forepath/agenstra-controller-api:latest + pull_policy: never + container_name: agent-controller-worker + environment: + <<: *backend-agent-controller-environment + QUEUE_ROLE: worker + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + mailhog: + condition: service_started + networks: + - agent-controller-network + restart: unless-stopped + + backend-agent-controller-scheduler: + image: ghcr.io/forepath/agenstra-controller-api:latest + pull_policy: never + container_name: agent-controller-scheduler + environment: + <<: *backend-agent-controller-environment + QUEUE_ROLE: scheduler + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy mailhog: condition: service_started networks: @@ -108,6 +166,7 @@ services: volumes: postgres_data: + redis_data: networks: agent-controller-network: diff --git a/apps/backend-agent-controller/package.json b/apps/backend-agent-controller/package.json index 383c3bfc..8ccae967 100644 --- a/apps/backend-agent-controller/package.json +++ b/apps/backend-agent-controller/package.json @@ -12,11 +12,18 @@ "@nestjs/platform-socket.io": "11.1.6", "@nestjs/throttler": "6.5.0", "@nestjs/typeorm": "11.0.0", + "@bull-board/api": "7.1.5", + "@bull-board/express": "7.1.5", + "@bull-board/nestjs": "7.1.5", + "@bull-board/ui": "7.1.5", + "@nestjs/bullmq": "11.0.4", "axios": "1.12.2", + "bullmq": "5.76.10", "bcrypt": "6.0.0", "class-transformer": "0.5.1", "class-validator": "0.14.2", "dockerode": "5.0.0", + "ioredis": "5.10.1", "keycloak-connect": "24.0.1", "nest-keycloak-connect": "2.0.0-alpha.2", "nodemailer": "7.0.13", diff --git a/apps/backend-agent-controller/src/app/app.module.ts b/apps/backend-agent-controller/src/app/app.module.ts index 19080881..a3f31473 100644 --- a/apps/backend-agent-controller/src/app/app.module.ts +++ b/apps/backend-agent-controller/src/app/app.module.ts @@ -8,19 +8,22 @@ import { KeycloakUserSyncModule, UsersAuthModule, } from '@forepath/identity/backend'; +import { getTypeOrmOptionsForQueueRole } from '@forepath/shared/backend'; import { Module } from '@nestjs/common'; import { APP_GUARD } from '@nestjs/core'; import { ThrottlerGuard, ThrottlerModule } from '@nestjs/throttler'; import { TypeOrmModule } from '@nestjs/typeorm'; import { KeycloakConnectModule } from 'nest-keycloak-connect'; +import { ControllerQueueModule } from '../queue/controller-queue.module'; import { typeormConfig } from '../typeorm.config'; const authMethod = getAuthenticationMethod(); @Module({ imports: [ - TypeOrmModule.forRoot(typeormConfig), + TypeOrmModule.forRoot(getTypeOrmOptionsForQueueRole(typeormConfig)), + ControllerQueueModule, ThrottlerModule.forRoot(getRateLimitConfig()), ...(authMethod === 'keycloak' ? [KeycloakModule, KeycloakConnectModule.registerAsync({ useExisting: KeycloakService }), KeycloakUserSyncModule] diff --git a/apps/backend-agent-controller/src/bootstrap.ts b/apps/backend-agent-controller/src/bootstrap.ts new file mode 100644 index 00000000..4fc15290 --- /dev/null +++ b/apps/backend-agent-controller/src/bootstrap.ts @@ -0,0 +1,109 @@ +import { assertProductionClientEndpointAllowlistConfigured } from '@forepath/framework/backend/feature-agent-controller'; +import { + CorrelationAwareConsoleLogger, + CorrelationAwareSocketIoAdapter, + createCorrelationIdMiddleware, + registerAxiosCorrelationIdPropagation, +} from '@forepath/framework/backend/util-http-context'; +import { createOriginAllowlistMiddleware } from '@forepath/identity/backend'; +import { assertProductionEncryptionKeyOrExit } from '@forepath/shared/backend'; +import { + assertBullBoardAuthConfigured, + getBullBoardGlobalPrefixExcludes, + getQueueRole, + readBullBoardAuthConfig, + readBullBoardPath, + shouldEnableBullBoard, + runPendingMigrationsIfRoleAllows, + shouldRunApiHttp, +} from '@forepath/shared/backend'; +import { Logger, ValidationPipe } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; +import axios from 'axios'; + +import { AppModule } from './app/app.module'; +import { typeormConfig } from './typeorm.config'; + +export async function bootstrap(): Promise { + assertProductionEncryptionKeyOrExit(new Logger('EncryptionKey')); + assertProductionClientEndpointAllowlistConfigured(new Logger('ClientEndpointAllowlist')); + + const appLogger = new CorrelationAwareConsoleLogger({ json: true, colors: false }); + + Logger.overrideLogger(appLogger); + registerAxiosCorrelationIdPropagation(axios); + + const role = getQueueRole(); + + assertBullBoardAuthConfigured(appLogger); + + const runHttp = shouldRunApiHttp(role) || shouldEnableBullBoard(role); + + if (!runHttp) { + const context = await NestFactory.createApplicationContext(AppModule, { logger: appLogger }); + + Logger.log(`Agent controller queue process started (QUEUE_ROLE=${role})`); + await context.init(); + + return; + } + + const app = await NestFactory.create(AppModule, { logger: appLogger }); + const httpLogger = new Logger('HTTP'); + + app.use( + createCorrelationIdMiddleware({ + log: (message: string) => httpLogger.log(message), + }), + ); + app.use(createOriginAllowlistMiddleware(new Logger('OriginAllowlist'))); + app.useWebSocketAdapter(new CorrelationAwareSocketIoAdapter(app)); + + const isProduction = process.env.NODE_ENV === 'production'; + const corsOrigin = process.env.CORS_ORIGIN; + let origin: string | string[]; + + if (corsOrigin) { + origin = corsOrigin.split(',').map((o) => o.trim()); + } else if (isProduction) { + origin = []; + Logger.warn( + '⚠ī¸ CORS_ORIGIN not set in production - CORS is disabled. Set CORS_ORIGIN environment variable to allow specific origins.', + ); + } else { + origin = '*'; + } + + app.enableCors({ + origin, + credentials: origin !== '*' && Array.isArray(origin) && origin.length > 0, + methods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], + allowedHeaders: ['Content-Type', 'Authorization', 'X-Correlation-Id', 'X-Request-Id'], + exposedHeaders: ['Content-Range', 'X-Content-Range', 'X-Correlation-Id'], + }); + + await runPendingMigrationsIfRoleAllows(app, role, typeormConfig); + + app.useGlobalPipes( + new ValidationPipe({ + whitelist: true, + forbidNonWhitelisted: true, + transform: true, + }), + ); + + const globalPrefix = 'api'; + const bullBoardExcludes = getBullBoardGlobalPrefixExcludes(); + + app.setGlobalPrefix(globalPrefix, bullBoardExcludes.length > 0 ? { exclude: bullBoardExcludes } : undefined); + const port = parseInt(process.env.PORT || '3100', 10); + + await app.listen(port); + Logger.log(`🚀 Application is running on: http://localhost:${port}/${globalPrefix} (QUEUE_ROLE=${role})`); + + if (shouldEnableBullBoard(role)) { + const { username } = readBullBoardAuthConfig(); + + Logger.log(`📊 Bull Board: http://localhost:${port}${readBullBoardPath()} (HTTP Basic, user ${username})`); + } +} diff --git a/apps/backend-agent-controller/src/main.ts b/apps/backend-agent-controller/src/main.ts index 056b163e..c5445cb6 100644 --- a/apps/backend-agent-controller/src/main.ts +++ b/apps/backend-agent-controller/src/main.ts @@ -1,121 +1,4 @@ -/** - * This is not a production server yet! - * This is only a minimal backend to get started. - */ - -import { assertProductionClientEndpointAllowlistConfigured } from '@forepath/framework/backend/feature-agent-controller'; -import { - CorrelationAwareConsoleLogger, - CorrelationAwareSocketIoAdapter, - createCorrelationIdMiddleware, - registerAxiosCorrelationIdPropagation, -} from '@forepath/framework/backend/util-http-context'; -import { createOriginAllowlistMiddleware } from '@forepath/identity/backend'; -import { assertProductionEncryptionKeyOrExit } from '@forepath/shared/backend'; -import { Logger, ValidationPipe } from '@nestjs/common'; -import { NestFactory } from '@nestjs/core'; -import axios from 'axios'; -import { DataSource } from 'typeorm'; - -import { AppModule } from './app/app.module'; -import { typeormConfig } from './typeorm.config'; - -async function bootstrap() { - assertProductionEncryptionKeyOrExit(new Logger('EncryptionKey')); - assertProductionClientEndpointAllowlistConfigured(new Logger('ClientEndpointAllowlist')); - - const appLogger = new CorrelationAwareConsoleLogger({ json: true, colors: false }); - - Logger.overrideLogger(appLogger); - registerAxiosCorrelationIdPropagation(axios); - - const app = await NestFactory.create(AppModule, { - logger: appLogger, - }); - const httpLogger = new Logger('HTTP'); - - app.use( - createCorrelationIdMiddleware({ - log: (message: string) => httpLogger.log(message), - }), - ); - app.use(createOriginAllowlistMiddleware(new Logger('OriginAllowlist'))); - // Configure CORS - // In production: CORS is restricted by default (requires CORS_ORIGIN to be set) - // In development: CORS allows all origins by default (can be restricted via CORS_ORIGIN) - const isProduction = process.env.NODE_ENV === 'production'; - const corsOrigin = process.env.CORS_ORIGIN; - let origin: string | string[]; - - if (corsOrigin) { - // If CORS_ORIGIN is explicitly set, use it (comma-separated list) - origin = corsOrigin.split(',').map((o) => o.trim()); - } else if (isProduction) { - // In production, if CORS_ORIGIN is not set, default to empty array (no CORS) - // This is the most secure default for production - origin = []; - Logger.warn( - '⚠ī¸ CORS_ORIGIN not set in production - CORS is disabled. Set CORS_ORIGIN environment variable to allow specific origins.', - ); - } else { - // In development, allow all origins by default - origin = '*'; - } - - app.enableCors({ - origin, - // credentials can only be true when origin is not '*' - credentials: origin !== '*' && Array.isArray(origin) && origin.length > 0, - methods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], - allowedHeaders: ['Content-Type', 'Authorization', 'X-Correlation-Id', 'X-Request-Id'], - exposedHeaders: ['Content-Range', 'X-Content-Range', 'X-Correlation-Id'], - }); - - if (Array.isArray(origin) && origin.length > 0) { - Logger.log(`🌐 CORS enabled with restricted origins: ${origin.join(', ')}`); - } else if (origin === '*') { - Logger.log('🌐 CORS enabled with origin: * (all origins allowed - development mode)'); - } else { - Logger.log('🌐 CORS disabled (no origins allowed)'); - } - - // Configure WebSocket adapter for Socket.IO - app.useWebSocketAdapter(new CorrelationAwareSocketIoAdapter(app)); - - // Run migrations automatically on startup if synchronize is disabled - // Note: If synchronize: true, schema is auto-synced from entities and migrations won't run - // To use migrations, set synchronize: false in typeorm.config.ts - if (!typeormConfig.synchronize && typeormConfig.migrations?.length) { - const dataSource = app.get(DataSource); - - try { - Logger.log('🔄 Running pending migrations...'); - await dataSource.runMigrations(); - Logger.log('✅ Migrations completed successfully'); - } catch (error) { - Logger.error('❌ Failed to run migrations:', error); - throw error; - } - } else if (typeormConfig.synchronize) { - Logger.log('ℹī¸ Schema synchronization enabled - migrations skipped'); - } - - app.useGlobalPipes( - new ValidationPipe({ - whitelist: true, - forbidNonWhitelisted: true, - transform: true, - }), - ); - - const globalPrefix = 'api'; - - app.setGlobalPrefix(globalPrefix); - const port = parseInt(process.env.PORT || '3100', 10); - - await app.listen(port); - Logger.log(`🚀 Application is running on: http://localhost:${port}/${globalPrefix}`); -} +import { bootstrap } from './bootstrap'; bootstrap().catch((error: unknown) => { const message = error instanceof Error ? error.message : String(error); diff --git a/apps/backend-agent-controller/src/queue/controller-queue-registrar.service.ts b/apps/backend-agent-controller/src/queue/controller-queue-registrar.service.ts new file mode 100644 index 00000000..6374b97c --- /dev/null +++ b/apps/backend-agent-controller/src/queue/controller-queue-registrar.service.ts @@ -0,0 +1,37 @@ +import { shouldRegisterRepeatableJobs } from '@forepath/shared/backend'; +import { InjectQueue } from '@nestjs/bullmq'; +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { Queue } from 'bullmq'; + +import { CONTROLLER_QUEUE_NAME, getControllerRepeatableJobs } from './job-registry'; + +@Injectable() +export class ControllerQueueRegistrarService implements OnModuleInit { + private readonly logger = new Logger(ControllerQueueRegistrarService.name); + + constructor(@InjectQueue(CONTROLLER_QUEUE_NAME) private readonly controllerQueue: Queue) {} + + async onModuleInit(): Promise { + if (!shouldRegisterRepeatableJobs()) { + return; + } + + for (const definition of getControllerRepeatableJobs()) { + if (definition.disabled) { + continue; + } + + await this.controllerQueue.add( + definition.name, + {}, + { + jobId: definition.coordinatorJobId, + repeat: { every: definition.everyMs }, + removeOnComplete: true, + removeOnFail: 100, + }, + ); + this.logger.log(`Registered repeatable job ${definition.name} every ${definition.everyMs}ms`); + } + } +} diff --git a/apps/backend-agent-controller/src/queue/controller-queue.module.ts b/apps/backend-agent-controller/src/queue/controller-queue.module.ts new file mode 100644 index 00000000..e6547f42 --- /dev/null +++ b/apps/backend-agent-controller/src/queue/controller-queue.module.ts @@ -0,0 +1,24 @@ +import { ClientsModule } from '@forepath/framework/backend'; +import { ContextImportModule, FilterRulesModule } from '@forepath/framework/backend/feature-agent-controller'; +import { SharedQueueModule, shouldRegisterRepeatableJobs, shouldRunQueueWorkers } from '@forepath/shared/backend'; +import { Module, forwardRef } from '@nestjs/common'; + +import { ControllerQueueRegistrarService } from './controller-queue-registrar.service'; +import { CONTROLLER_QUEUE_NAME } from './job-registry'; +import { ControllerJobsProcessor } from './processors/controller-jobs.processor'; + +@Module({ + imports: [ + SharedQueueModule.forRoot({ + queueNames: [CONTROLLER_QUEUE_NAME], + }), + forwardRef(() => ClientsModule), + forwardRef(() => FilterRulesModule), + forwardRef(() => ContextImportModule), + ], + providers: [ + ...(shouldRunQueueWorkers() ? [ControllerJobsProcessor] : []), + ...(shouldRegisterRepeatableJobs() ? [ControllerQueueRegistrarService] : []), + ], +}) +export class ControllerQueueModule {} diff --git a/apps/backend-agent-controller/src/queue/job-registry.spec.ts b/apps/backend-agent-controller/src/queue/job-registry.spec.ts new file mode 100644 index 00000000..7c65d93b --- /dev/null +++ b/apps/backend-agent-controller/src/queue/job-registry.spec.ts @@ -0,0 +1,23 @@ +import { ControllerJobName, getControllerRepeatableJobs } from './job-registry'; + +describe('controller job-registry', () => { + it('defines coordinator and unit job names', () => { + expect(ControllerJobName.AUTONOMOUS_TICKET_UNIT).toBe('autonomous-ticket.unit'); + expect(ControllerJobName.FILTER_RULES_RECONCILE).toBe('filter-rules-sync.reconcile'); + }); + + it('getControllerRepeatableJobs includes core coordinators', () => { + const jobs = getControllerRepeatableJobs(); + const names = jobs.map((job) => job.name); + + expect(names).toContain(ControllerJobName.FILTER_RULES_SYNC_COORDINATOR); + expect(names).toContain(ControllerJobName.AUTONOMOUS_TICKET_COORDINATOR); + }); + + it('coordinator job ids are valid for BullMQ (no colons)', () => { + for (const job of getControllerRepeatableJobs()) { + expect(job.coordinatorJobId).not.toContain(':'); + expect(job.coordinatorJobId.startsWith('coordinator.')).toBe(true); + } + }); +}); diff --git a/apps/backend-agent-controller/src/queue/job-registry.ts b/apps/backend-agent-controller/src/queue/job-registry.ts new file mode 100644 index 00000000..b2c3c318 --- /dev/null +++ b/apps/backend-agent-controller/src/queue/job-registry.ts @@ -0,0 +1,92 @@ +import { buildCoordinatorJobId } from '@forepath/shared/backend'; + +/** Central registry for agent-controller BullMQ queues, job names, and coordinator schedules. */ + +export const CONTROLLER_QUEUE_NAME = 'agent-controller'; + +export const ControllerJobName = { + CONTEXT_IMPORT_COORDINATOR: 'context-import.coordinator', + CONTEXT_IMPORT_UNIT: 'context-import.unit', + KNOWLEDGE_EMBEDDING_COORDINATOR: 'knowledge-embedding.coordinator', + KNOWLEDGE_EMBEDDING_UNIT: 'knowledge-embedding.unit', + FILTER_RULES_SYNC_COORDINATOR: 'filter-rules-sync.coordinator', + FILTER_RULES_SYNC_UNIT: 'filter-rules-sync.unit', + FILTER_RULES_RECONCILE: 'filter-rules-sync.reconcile', + AUTONOMOUS_TICKET_COORDINATOR: 'autonomous-ticket.coordinator', + AUTONOMOUS_TICKET_UNIT: 'autonomous-ticket.unit', +} as const; + +export type ControllerJobName = (typeof ControllerJobName)[keyof typeof ControllerJobName]; + +export interface ControllerRepeatableJobDefinition { + name: ControllerJobName; + coordinatorJobId: string; + everyMs: number; + disabled?: boolean; +} + +function parseIntervalMs(envKey: string, fallback: number): number { + const parsed = parseInt(process.env[envKey] ?? String(fallback), 10); + + return Number.isFinite(parsed) ? parsed : fallback; +} + +export function getControllerRepeatableJobs(): ControllerRepeatableJobDefinition[] { + const knowledgeInterval = parseIntervalMs('KNOWLEDGE_EMBEDDINGS_REINDEX_INTERVAL_MS', 3_600_000); + const contextImportInterval = parseIntervalMs('CONTEXT_IMPORT_SCHEDULER_INTERVAL_MS', 120_000); + const jobs: ControllerRepeatableJobDefinition[] = [ + { + name: ControllerJobName.FILTER_RULES_SYNC_COORDINATOR, + coordinatorJobId: buildCoordinatorJobId('filter-rules-sync'), + everyMs: parseIntervalMs('FILTER_RULES_SYNC_INTERVAL_MS', 30_000), + }, + { + name: ControllerJobName.FILTER_RULES_RECONCILE, + coordinatorJobId: buildCoordinatorJobId('filter-rules-reconcile'), + everyMs: parseIntervalMs('FILTER_RULES_SYNC_INTERVAL_MS', 30_000), + }, + { + name: ControllerJobName.AUTONOMOUS_TICKET_COORDINATOR, + coordinatorJobId: buildCoordinatorJobId('autonomous-ticket'), + everyMs: parseIntervalMs('AUTONOMOUS_TICKET_SCHEDULER_INTERVAL_MS', 60_000), + }, + ]; + + if (contextImportInterval > 0) { + jobs.push({ + name: ControllerJobName.CONTEXT_IMPORT_COORDINATOR, + coordinatorJobId: buildCoordinatorJobId('context-import'), + everyMs: contextImportInterval, + }); + } + + if (knowledgeInterval > 0) { + jobs.push({ + name: ControllerJobName.KNOWLEDGE_EMBEDDING_COORDINATOR, + coordinatorJobId: buildCoordinatorJobId('knowledge-embedding'), + everyMs: knowledgeInterval, + }); + } + + return jobs; +} + +export function getContextImportItemBudget(): number { + return parseInt(process.env.CONTEXT_IMPORT_ITEM_BUDGET ?? '25', 10); +} + +export function getContextImportConfigBatch(): number { + return parseInt(process.env.CONTEXT_IMPORT_SCHEDULER_CONFIG_BATCH ?? '3', 10); +} + +export function getFilterRulesSyncBatchSize(): number { + return parseInt(process.env.FILTER_RULES_SYNC_BATCH_SIZE ?? '10', 10); +} + +export function getAutonomousTicketBatchSize(): number { + return parseInt(process.env.AUTONOMOUS_TICKET_SCHEDULER_BATCH_SIZE ?? '5', 10); +} + +export function getKnowledgeEmbeddingPageBatchSize(): number { + return parseInt(process.env.KNOWLEDGE_EMBEDDINGS_PAGE_BATCH_SIZE ?? '50', 10); +} diff --git a/apps/backend-agent-controller/src/queue/processors/controller-jobs.processor.ts b/apps/backend-agent-controller/src/queue/processors/controller-jobs.processor.ts new file mode 100644 index 00000000..b35c99c5 --- /dev/null +++ b/apps/backend-agent-controller/src/queue/processors/controller-jobs.processor.ts @@ -0,0 +1,158 @@ +import { + AutonomousRunOrchestratorService, + ContextImportOrchestratorService, + ExternalImportConfigService, + FilterRulesService, + FilterRulesSyncService, + KnowledgeEmbeddingIndexService, +} from '@forepath/framework/backend'; +import { enqueueUnitJob } from '@forepath/shared/backend'; +import { InjectQueue, Processor, WorkerHost } from '@nestjs/bullmq'; +import { Logger } from '@nestjs/common'; +import { Job, Queue } from 'bullmq'; + +import { + CONTROLLER_QUEUE_NAME, + ControllerJobName, + getAutonomousTicketBatchSize, + getContextImportConfigBatch, + getContextImportItemBudget, + getFilterRulesSyncBatchSize, + getKnowledgeEmbeddingPageBatchSize, +} from '../job-registry'; + +@Processor(CONTROLLER_QUEUE_NAME, { + concurrency: parseInt(process.env.QUEUE_WORKER_CONCURRENCY ?? '5', 10), + lockDuration: 600_000, +}) +export class ControllerJobsProcessor extends WorkerHost { + private readonly logger = new Logger(ControllerJobsProcessor.name); + + constructor( + @InjectQueue(CONTROLLER_QUEUE_NAME) private readonly controllerQueue: Queue, + private readonly contextImportOrchestrator: ContextImportOrchestratorService, + private readonly contextImportConfigService: ExternalImportConfigService, + private readonly knowledgeEmbeddingIndex: KnowledgeEmbeddingIndexService, + private readonly filterRulesSync: FilterRulesSyncService, + private readonly filterRulesService: FilterRulesService, + private readonly autonomousOrchestrator: AutonomousRunOrchestratorService, + ) { + super(); + } + + async process(job: Job): Promise { + switch (job.name) { + case ControllerJobName.CONTEXT_IMPORT_COORDINATOR: + await this.runContextImportCoordinator(); + break; + case ControllerJobName.CONTEXT_IMPORT_UNIT: + await this.contextImportOrchestrator.runConfigById( + (job.data as { configId: string }).configId, + getContextImportItemBudget(), + ); + break; + case ControllerJobName.KNOWLEDGE_EMBEDDING_COORDINATOR: + await this.runKnowledgeEmbeddingCoordinator(); + break; + + case ControllerJobName.KNOWLEDGE_EMBEDDING_UNIT: { + const data = job.data as { clientId: string; nodeId: string; title: string; content: string }; + + await this.knowledgeEmbeddingIndex.reindexPage(data.clientId, data.nodeId, data.title, data.content); + break; + } + + case ControllerJobName.FILTER_RULES_SYNC_COORDINATOR: + await this.runFilterRulesSyncCoordinator(); + break; + case ControllerJobName.FILTER_RULES_SYNC_UNIT: + await this.filterRulesSync.processTargetById((job.data as { targetId: string }).targetId); + break; + case ControllerJobName.FILTER_RULES_RECONCILE: + await this.filterRulesService.reconcileAllGlobalRules(); + break; + case ControllerJobName.AUTONOMOUS_TICKET_COORDINATOR: + await this.runAutonomousTicketCoordinator(); + break; + case ControllerJobName.AUTONOMOUS_TICKET_UNIT: + await this.autonomousOrchestrator.tryStartRunForCandidate( + job.data as { ticket_id: string; client_id: string; agent_id: string }, + ); + break; + default: + this.logger.warn(`Unknown controller job name: ${job.name}`); + } + } + + private async runContextImportCoordinator(): Promise { + const configs = await this.contextImportConfigService.findEnabledForSchedulerBatch(getContextImportConfigBatch()); + + for (const config of configs) { + await enqueueUnitJob({ + queue: this.controllerQueue, + jobName: ControllerJobName.CONTEXT_IMPORT_UNIT, + payload: { configId: config.id }, + jobIdNamespace: 'context-import:config', + jobIdParts: [config.id], + }); + } + } + + private async runKnowledgeEmbeddingCoordinator(): Promise { + const batchSize = getKnowledgeEmbeddingPageBatchSize(); + let offset = 0; + + // eslint-disable-next-line no-constant-condition + while (true) { + const pages = await this.knowledgeEmbeddingIndex.findPageIdsBatch(offset, batchSize); + + if (pages.length === 0) { + break; + } + + for (const page of pages) { + await enqueueUnitJob({ + queue: this.controllerQueue, + jobName: ControllerJobName.KNOWLEDGE_EMBEDDING_UNIT, + payload: page, + jobIdNamespace: 'knowledge:page', + jobIdParts: [page.clientId, page.nodeId], + }); + } + + offset += pages.length; + + if (pages.length < batchSize) { + break; + } + } + } + + private async runFilterRulesSyncCoordinator(): Promise { + const targetIds = await this.filterRulesSync.findPendingTargetIds(getFilterRulesSyncBatchSize()); + + for (const targetId of targetIds) { + await enqueueUnitJob({ + queue: this.controllerQueue, + jobName: ControllerJobName.FILTER_RULES_SYNC_UNIT, + payload: { targetId }, + jobIdNamespace: 'filter-rules:target', + jobIdParts: [targetId], + }); + } + } + + private async runAutonomousTicketCoordinator(): Promise { + const candidates = await this.autonomousOrchestrator.findCandidateIds(getAutonomousTicketBatchSize()); + + for (const candidate of candidates) { + await enqueueUnitJob({ + queue: this.controllerQueue, + jobName: ControllerJobName.AUTONOMOUS_TICKET_UNIT, + payload: candidate, + jobIdNamespace: 'autonomous-ticket', + jobIdParts: [candidate.ticket_id], + }); + } + } +} diff --git a/apps/backend-billing-manager/.start-containers.env.example b/apps/backend-billing-manager/.start-containers.env.example index 014c1bca..089f19a8 100644 --- a/apps/backend-billing-manager/.start-containers.env.example +++ b/apps/backend-billing-manager/.start-containers.env.example @@ -50,8 +50,11 @@ EXPIRATION_SCHEDULER_BATCH_SIZE=100 # Reminder scheduler: sends renewal reminders (default: 3600000 = 1 hour) REMINDER_SCHEDULER_INTERVAL=3600000 REMINDER_SCHEDULER_BATCH_SIZE=100 -BACKORDER_RETRY_INTERVAL=60000 +BACKORDER_RETRY_INTERVAL_MS=60000 BACKORDER_RETRY_BATCH_SIZE=100 +INVOICE_SYNC_SCHEDULER_INTERVAL=60000 +INVOICE_SYNC_SCHEDULER_BATCH_SIZE=100 +OPEN_POSITION_INVOICE_SCHEDULER_INTERVAL=86400000 REMINDER_DAYS=3 # Subscription update scheduler: SSH to provisioned hosts and run docker compose up -d --pull=always (default: 86400000 = 24 hours) SUBSCRIPTION_UPDATE_SCHEDULER_INTERVAL=86400000 @@ -60,3 +63,17 @@ SUBSCRIPTION_UPDATE_SCHEDULER_INTERVAL=86400000 CLOUDFLARE_API_TOKEN= CLOUDFLARE_ZONE_ID= DNS_BASE_DOMAIN=example.com + +# Redis / BullMQ (queue provider in docker-compose) +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_PASSWORD= +REDIS_DB=0 +REDIS_KEY_PREFIX=agenstra-billing +REDIS_HOST_PORT=6380 +QUEUE_ROLE=all +QUEUE_WORKER_CONCURRENCY=5 +QUEUE_BULL_BOARD_ENABLED=true +QUEUE_BULL_BOARD_PATH=/admin/queues +QUEUE_BULL_BOARD_USERNAME=admin +QUEUE_BULL_BOARD_PASSWORD=bullmq diff --git a/apps/backend-billing-manager/docker-compose.yaml b/apps/backend-billing-manager/docker-compose.yaml index 442bb450..e92e0cd7 100644 --- a/apps/backend-billing-manager/docker-compose.yaml +++ b/apps/backend-billing-manager/docker-compose.yaml @@ -1,3 +1,67 @@ +x-backend-billing-manager-environment: &backend-billing-manager-environment + HOST: ${HOST:-0.0.0.0} + PORT: ${PORT:-3200} + WEBSOCKET_PORT: ${WEBSOCKET_PORT:-8082} + WEBSOCKET_NAMESPACE: ${WEBSOCKET_NAMESPACE:-billing} + WEBSOCKET_CORS_ORIGIN: ${WEBSOCKET_CORS_ORIGIN:-*} + NODE_ENV: ${NODE_ENV:-development} + DB_HOST: ${DB_HOST:-postgres} + DB_PORT: ${DB_PORT:-5432} + DB_USERNAME: ${DB_USERNAME:-postgres} + DB_PASSWORD: ${DB_PASSWORD:-postgres} + DB_DATABASE: ${DB_DATABASE:-postgres} + REDIS_HOST: ${REDIS_HOST:-redis} + REDIS_PORT: ${REDIS_PORT:-6379} + REDIS_PASSWORD: ${REDIS_PASSWORD:-} + REDIS_DB: ${REDIS_DB:-0} + REDIS_KEY_PREFIX: ${REDIS_KEY_PREFIX:-agenstra-billing} + QUEUE_WORKER_CONCURRENCY: ${QUEUE_WORKER_CONCURRENCY:-5} + QUEUE_BULL_BOARD_ENABLED: ${QUEUE_BULL_BOARD_ENABLED:-true} + QUEUE_BULL_BOARD_PATH: ${QUEUE_BULL_BOARD_PATH:-/admin/queues} + QUEUE_BULL_BOARD_USERNAME: ${QUEUE_BULL_BOARD_USERNAME:-admin} + QUEUE_BULL_BOARD_PASSWORD: ${QUEUE_BULL_BOARD_PASSWORD:-bullmq} + HETZNER_API_TOKEN: ${HETZNER_API_TOKEN:-} + DIGITALOCEAN_API_TOKEN: ${DIGITALOCEAN_API_TOKEN:-} + AUTHENTICATION_METHOD: ${AUTHENTICATION_METHOD:-api-key} + DISABLE_SIGNUP: ${DISABLE_SIGNUP:-false} + STATIC_API_KEY: ${STATIC_API_KEY:-} + ENCRYPTION_KEY: ${ENCRYPTION_KEY:-} + CORS_ORIGIN: ${CORS_ORIGIN:-} + RATE_LIMIT_ENABLED: ${RATE_LIMIT_ENABLED:-} + RATE_LIMIT_TTL: ${RATE_LIMIT_TTL:-60} + RATE_LIMIT_LIMIT: ${RATE_LIMIT_LIMIT:-100} + KEYCLOAK_SERVER_URL: ${KEYCLOAK_SERVER_URL:-} + KEYCLOAK_AUTH_SERVER_URL: ${KEYCLOAK_AUTH_SERVER_URL:-} + KEYCLOAK_REALM: ${KEYCLOAK_REALM:-} + KEYCLOAK_CLIENT_ID: ${KEYCLOAK_CLIENT_ID:-} + KEYCLOAK_CLIENT_SECRET: ${KEYCLOAK_CLIENT_SECRET:-} + KEYCLOAK_TOKEN_VALIDATION: ${KEYCLOAK_TOKEN_VALIDATION:-online} + JWT_SECRET: ${JWT_SECRET:-} + SMTP_HOST: ${SMTP_HOST:-mailhog} + SMTP_PORT: ${SMTP_PORT:-1025} + SMTP_USER: ${SMTP_USER:-} + SMTP_PASSWORD: ${SMTP_PASSWORD:-} + EMAIL_FROM: ${EMAIL_FROM:-noreply@localhost} + INVOICE_NINJA_BASE_URL: ${INVOICE_NINJA_BASE_URL:-} + INVOICE_NINJA_API_TOKEN: ${INVOICE_NINJA_API_TOKEN:-} + BILLING_SCHEDULER_INTERVAL: ${BILLING_SCHEDULER_INTERVAL:-60000} + BILLING_SCHEDULER_BATCH_SIZE: ${BILLING_SCHEDULER_BATCH_SIZE:-100} + EXPIRATION_SCHEDULER_INTERVAL: ${EXPIRATION_SCHEDULER_INTERVAL:-60000} + EXPIRATION_SCHEDULER_BATCH_SIZE: ${EXPIRATION_SCHEDULER_BATCH_SIZE:-100} + REMINDER_SCHEDULER_INTERVAL: ${REMINDER_SCHEDULER_INTERVAL:-3600000} + REMINDER_SCHEDULER_BATCH_SIZE: ${REMINDER_SCHEDULER_BATCH_SIZE:-100} + REMINDER_DAYS: ${REMINDER_DAYS:-3} + BACKORDER_RETRY_INTERVAL_MS: ${BACKORDER_RETRY_INTERVAL_MS:-60000} + BACKORDER_RETRY_BATCH_SIZE: ${BACKORDER_RETRY_BATCH_SIZE:-100} + INVOICE_SYNC_SCHEDULER_INTERVAL: ${INVOICE_SYNC_SCHEDULER_INTERVAL:-60000} + INVOICE_SYNC_SCHEDULER_BATCH_SIZE: ${INVOICE_SYNC_SCHEDULER_BATCH_SIZE:-100} + OPEN_POSITION_INVOICE_SCHEDULER_INTERVAL: ${OPEN_POSITION_INVOICE_SCHEDULER_INTERVAL:-86400000} + SUBSCRIPTION_UPDATE_SCHEDULER_INTERVAL: ${SUBSCRIPTION_UPDATE_SCHEDULER_INTERVAL:-86400000} + STATUS_POLL_INTERVAL: ${STATUS_POLL_INTERVAL:-15000} + CLOUDFLARE_API_TOKEN: ${CLOUDFLARE_API_TOKEN:-} + CLOUDFLARE_ZONE_ID: ${CLOUDFLARE_ZONE_ID:-} + DNS_BASE_DOMAIN: ${DNS_BASE_DOMAIN:-spirde.com} + services: postgres: image: postgres:16-alpine @@ -17,83 +81,74 @@ services: - billing-network restart: unless-stopped + redis: + image: redis:7-alpine + container_name: billing-manager-redis + command: ['redis-server', '--appendonly', 'yes'] + volumes: + - redis_data:/data + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + interval: 10s + timeout: 5s + retries: 5 + ports: + - '${REDIS_HOST_PORT:-6380}:6379' + networks: + - billing-network + restart: unless-stopped + backend-billing-manager: image: ghcr.io/forepath/agenstra-billing-api:latest pull_policy: never container_name: billing-manager-api environment: - # Backend API configuration - HOST: ${HOST:-0.0.0.0} - PORT: ${PORT:-3200} - WEBSOCKET_PORT: ${WEBSOCKET_PORT:-8082} - WEBSOCKET_NAMESPACE: ${WEBSOCKET_NAMESPACE:-billing} - WEBSOCKET_CORS_ORIGIN: ${WEBSOCKET_CORS_ORIGIN:-*} - NODE_ENV: ${NODE_ENV:-development} - # Database configuration - DB_HOST: ${DB_HOST:-postgres} - DB_PORT: ${DB_PORT:-5432} - DB_USERNAME: ${DB_USERNAME:-postgres} - DB_PASSWORD: ${DB_PASSWORD:-postgres} - DB_DATABASE: ${DB_DATABASE:-postgres} - # Environment variables for the provisioning providers - HETZNER_API_TOKEN: ${HETZNER_API_TOKEN:-} - DIGITALOCEAN_API_TOKEN: ${DIGITALOCEAN_API_TOKEN:-} - # Authentication method configuration - AUTHENTICATION_METHOD: ${AUTHENTICATION_METHOD:-api-key} - # Environment variables for disabling signup - DISABLE_SIGNUP: ${DISABLE_SIGNUP:-false} - # Static API key authentication configuration - STATIC_API_KEY: ${STATIC_API_KEY:-} - # Environment variables for encryption - ENCRYPTION_KEY: ${ENCRYPTION_KEY:-} - # CORS configuration (comma-separated list of allowed origins) - # In production: CORS is disabled by default. Set CORS_ORIGIN to allow specific origins. - # In development: CORS allows all origins by default. - CORS_ORIGIN: ${CORS_ORIGIN:-} - # Rate limiting configuration - RATE_LIMIT_ENABLED: ${RATE_LIMIT_ENABLED:-} - RATE_LIMIT_TTL: ${RATE_LIMIT_TTL:-60} - RATE_LIMIT_LIMIT: ${RATE_LIMIT_LIMIT:-100} - # Keycloak configuration - KEYCLOAK_SERVER_URL: ${KEYCLOAK_SERVER_URL:-} - KEYCLOAK_AUTH_SERVER_URL: ${KEYCLOAK_AUTH_SERVER_URL:-} - KEYCLOAK_REALM: ${KEYCLOAK_REALM:-} - KEYCLOAK_CLIENT_ID: ${KEYCLOAK_CLIENT_ID:-} - KEYCLOAK_CLIENT_SECRET: ${KEYCLOAK_CLIENT_SECRET:-} - KEYCLOAK_TOKEN_VALIDATION: ${KEYCLOAK_TOKEN_VALIDATION:-online} - # Environment variables for users authentication (when AUTHENTICATION_METHOD=users) - JWT_SECRET: ${JWT_SECRET:-} - # SMTP / MailHog configuration (for email confirmation and password reset) - SMTP_HOST: ${SMTP_HOST:-mailhog} - SMTP_PORT: ${SMTP_PORT:-1025} - SMTP_USER: ${SMTP_USER:-} - SMTP_PASSWORD: ${SMTP_PASSWORD:-} - EMAIL_FROM: ${EMAIL_FROM:-noreply@localhost} - # InvoiceNinja configuration - INVOICE_NINJA_BASE_URL: ${INVOICE_NINJA_BASE_URL:-} - INVOICE_NINJA_API_TOKEN: ${INVOICE_NINJA_API_TOKEN:-} - # Scheduler intervals (in milliseconds) - BILLING_SCHEDULER_INTERVAL: ${BILLING_SCHEDULER_INTERVAL:-60000} - BILLING_SCHEDULER_BATCH_SIZE: ${BILLING_SCHEDULER_BATCH_SIZE:-100} - EXPIRATION_SCHEDULER_INTERVAL: ${EXPIRATION_SCHEDULER_INTERVAL:-60000} - EXPIRATION_SCHEDULER_BATCH_SIZE: ${EXPIRATION_SCHEDULER_BATCH_SIZE:-100} - REMINDER_SCHEDULER_INTERVAL: ${REMINDER_SCHEDULER_INTERVAL:-60000} - REMINDER_SCHEDULER_BATCH_SIZE: ${REMINDER_SCHEDULER_BATCH_SIZE:-100} - REMINDER_DAYS: ${REMINDER_DAYS:-3} - BACKORDER_RETRY_INTERVAL: ${BACKORDER_RETRY_INTERVAL:-60000} - BACKORDER_RETRY_BATCH_SIZE: ${BACKORDER_RETRY_BATCH_SIZE:-100} - SUBSCRIPTION_UPDATE_SCHEDULER_INTERVAL: ${SUBSCRIPTION_UPDATE_SCHEDULER_INTERVAL:-86400000} - STATUS_POLL_INTERVAL: ${STATUS_POLL_INTERVAL:-15000} - # DNS settings - CLOUDFLARE_API_TOKEN: ${CLOUDFLARE_API_TOKEN:-} - CLOUDFLARE_ZONE_ID: ${CLOUDFLARE_ZONE_ID:-} - DNS_BASE_DOMAIN: ${DNS_BASE_DOMAIN:-spirde.com} + <<: *backend-billing-manager-environment + QUEUE_ROLE: ${QUEUE_ROLE:-api} ports: - '${PORT:-3200}:${PORT:-3200}' - '${WEBSOCKET_PORT:-8082}:${WEBSOCKET_PORT:-8082}' depends_on: postgres: condition: service_healthy + redis: + condition: service_healthy + mailhog: + condition: service_started + networks: + - billing-network + restart: unless-stopped + + backend-billing-manager-worker: + image: ghcr.io/forepath/agenstra-billing-api:latest + pull_policy: never + container_name: billing-manager-worker + environment: + <<: *backend-billing-manager-environment + QUEUE_ROLE: worker + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + mailhog: + condition: service_started + networks: + - billing-network + restart: unless-stopped + + backend-billing-manager-scheduler: + image: ghcr.io/forepath/agenstra-billing-api:latest + pull_policy: never + container_name: billing-manager-scheduler + environment: + <<: *backend-billing-manager-environment + QUEUE_ROLE: scheduler + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy mailhog: condition: service_started networks: @@ -112,6 +167,7 @@ services: volumes: postgres_data: + redis_data: networks: billing-network: diff --git a/apps/backend-billing-manager/package.json b/apps/backend-billing-manager/package.json index 5968cb66..0183ef01 100644 --- a/apps/backend-billing-manager/package.json +++ b/apps/backend-billing-manager/package.json @@ -12,11 +12,18 @@ "@nestjs/platform-socket.io": "11.1.6", "@nestjs/throttler": "6.5.0", "@nestjs/typeorm": "11.0.0", + "@bull-board/api": "7.1.5", + "@bull-board/express": "7.1.5", + "@bull-board/nestjs": "7.1.5", + "@bull-board/ui": "7.1.5", + "@nestjs/bullmq": "11.0.4", "axios": "1.12.2", + "bullmq": "5.76.10", "bcrypt": "6.0.0", "class-transformer": "0.5.1", "class-validator": "0.14.2", "dockerode": "5.0.0", + "ioredis": "5.10.1", "i18n-iso-countries": "7.14.0", "keycloak-connect": "24.0.1", "nest-keycloak-connect": "2.0.0-alpha.2", diff --git a/apps/backend-billing-manager/src/app/app.module.ts b/apps/backend-billing-manager/src/app/app.module.ts index ac6a03eb..09456f98 100644 --- a/apps/backend-billing-manager/src/app/app.module.ts +++ b/apps/backend-billing-manager/src/app/app.module.ts @@ -11,19 +11,22 @@ import { KeycloakModule, KeycloakService, } from '@forepath/identity/backend'; +import { getTypeOrmOptionsForQueueRole } from '@forepath/shared/backend'; import { Module } from '@nestjs/common'; import { APP_GUARD } from '@nestjs/core'; import { ThrottlerGuard, ThrottlerModule } from '@nestjs/throttler'; import { TypeOrmModule } from '@nestjs/typeorm'; import { KeycloakConnectModule } from 'nest-keycloak-connect'; +import { BillingQueueModule } from '../queue/billing-queue.module'; import { typeormConfig } from '../typeorm.config'; const authMethod = getAuthenticationMethod(); @Module({ imports: [ - TypeOrmModule.forRoot(typeormConfig), + TypeOrmModule.forRoot(getTypeOrmOptionsForQueueRole(typeormConfig)), + BillingQueueModule, ThrottlerModule.forRoot(getRateLimitConfig()), ...(authMethod === 'keycloak' ? [ diff --git a/apps/backend-billing-manager/src/bootstrap.ts b/apps/backend-billing-manager/src/bootstrap.ts new file mode 100644 index 00000000..1149e65c --- /dev/null +++ b/apps/backend-billing-manager/src/bootstrap.ts @@ -0,0 +1,105 @@ +import { + CorrelationAwareConsoleLogger, + CorrelationAwareSocketIoAdapter, + createCorrelationIdMiddleware, + registerAxiosCorrelationIdPropagation, +} from '@forepath/framework/backend/util-http-context'; +import { createOriginAllowlistMiddleware } from '@forepath/identity/backend'; +import { assertProductionEncryptionKeyOrExit } from '@forepath/shared/backend'; +import { + assertBullBoardAuthConfigured, + getBullBoardGlobalPrefixExcludes, + getQueueRole, + readBullBoardAuthConfig, + readBullBoardPath, + shouldEnableBullBoard, + runPendingMigrationsIfRoleAllows, + shouldRunApiHttp, +} from '@forepath/shared/backend'; +import { Logger, ValidationPipe } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; +import axios from 'axios'; + +import { AppModule } from './app/app.module'; +import { typeormConfig } from './typeorm.config'; + +export async function bootstrap(): Promise { + assertProductionEncryptionKeyOrExit(new Logger('EncryptionKey')); + + const appLogger = new CorrelationAwareConsoleLogger({ json: true, colors: false }); + + Logger.overrideLogger(appLogger); + registerAxiosCorrelationIdPropagation(axios); + + const role = getQueueRole(); + + assertBullBoardAuthConfigured(appLogger); + + const runHttp = shouldRunApiHttp(role) || shouldEnableBullBoard(role); + + if (!runHttp) { + const context = await NestFactory.createApplicationContext(AppModule, { logger: appLogger }); + + Logger.log(`Billing queue process started (QUEUE_ROLE=${role})`); + await context.init(); + + return; + } + + const app = await NestFactory.create(AppModule, { logger: appLogger }); + const httpLogger = new Logger('HTTP'); + + app.use( + createCorrelationIdMiddleware({ + log: (message: string) => httpLogger.log(message), + }), + ); + app.use(createOriginAllowlistMiddleware(new Logger('OriginAllowlist'))); + app.useWebSocketAdapter(new CorrelationAwareSocketIoAdapter(app)); + + const isProduction = process.env.NODE_ENV === 'production'; + const corsOrigin = process.env.CORS_ORIGIN; + let origin: string | string[]; + + if (corsOrigin) { + origin = corsOrigin.split(',').map((value) => value.trim()); + } else if (isProduction) { + origin = []; + Logger.warn('⚠ī¸ CORS_ORIGIN not set in production - CORS is disabled. Set CORS_ORIGIN to allow specific origins.'); + } else { + origin = '*'; + } + + app.enableCors({ + origin, + credentials: origin !== '*' && Array.isArray(origin) && origin.length > 0, + methods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], + allowedHeaders: ['Content-Type', 'Authorization', 'X-Correlation-Id', 'X-Request-Id'], + exposedHeaders: ['Content-Range', 'X-Content-Range', 'X-Correlation-Id'], + }); + + await runPendingMigrationsIfRoleAllows(app, role, typeormConfig); + + app.useGlobalPipes( + new ValidationPipe({ + whitelist: true, + forbidNonWhitelisted: true, + transform: true, + }), + ); + + const globalPrefix = 'api'; + const bullBoardExcludes = getBullBoardGlobalPrefixExcludes(); + + app.setGlobalPrefix(globalPrefix, bullBoardExcludes.length > 0 ? { exclude: bullBoardExcludes } : undefined); + const port = parseInt(process.env.PORT || '3200', 10); + + await app.listen(port); + Logger.log(`🚀 Application is running on: http://localhost:${port}/${globalPrefix} (QUEUE_ROLE=${role})`); + + if (shouldEnableBullBoard(role)) { + const { username } = readBullBoardAuthConfig(); + + Logger.log(`📊 Bull Board: http://localhost:${port}${readBullBoardPath()} (HTTP Basic, user ${username})`); + } +} diff --git a/apps/backend-billing-manager/src/main.ts b/apps/backend-billing-manager/src/main.ts index 042d2447..2fb9dfe3 100644 --- a/apps/backend-billing-manager/src/main.ts +++ b/apps/backend-billing-manager/src/main.ts @@ -1,100 +1,3 @@ -import { - CorrelationAwareConsoleLogger, - CorrelationAwareSocketIoAdapter, - createCorrelationIdMiddleware, - registerAxiosCorrelationIdPropagation, -} from '@forepath/framework/backend/util-http-context'; -import { createOriginAllowlistMiddleware } from '@forepath/identity/backend'; -import { assertProductionEncryptionKeyOrExit } from '@forepath/shared/backend'; -import { Logger, ValidationPipe } from '@nestjs/common'; -import { NestFactory } from '@nestjs/core'; -import axios from 'axios'; -import { DataSource } from 'typeorm'; - -import { AppModule } from './app/app.module'; -import { typeormConfig } from './typeorm.config'; - -async function bootstrap() { - assertProductionEncryptionKeyOrExit(new Logger('EncryptionKey')); - - const appLogger = new CorrelationAwareConsoleLogger({ json: true, colors: false }); - - Logger.overrideLogger(appLogger); - registerAxiosCorrelationIdPropagation(axios); - - const app = await NestFactory.create(AppModule, { - logger: appLogger, - }); - const httpLogger = new Logger('HTTP'); - - app.use( - createCorrelationIdMiddleware({ - log: (message: string) => httpLogger.log(message), - }), - ); - app.use(createOriginAllowlistMiddleware(new Logger('OriginAllowlist'))); - - app.useWebSocketAdapter(new CorrelationAwareSocketIoAdapter(app)); - - const isProduction = process.env.NODE_ENV === 'production'; - const corsOrigin = process.env.CORS_ORIGIN; - let origin: string | string[]; - - if (corsOrigin) { - origin = corsOrigin.split(',').map((value) => value.trim()); - } else if (isProduction) { - origin = []; - Logger.warn('⚠ī¸ CORS_ORIGIN not set in production - CORS is disabled. Set CORS_ORIGIN to allow specific origins.'); - } else { - origin = '*'; - } - - app.enableCors({ - origin, - credentials: origin !== '*' && Array.isArray(origin) && origin.length > 0, - methods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], - allowedHeaders: ['Content-Type', 'Authorization', 'X-Correlation-Id', 'X-Request-Id'], - exposedHeaders: ['Content-Range', 'X-Content-Range', 'X-Correlation-Id'], - }); - - if (Array.isArray(origin) && origin.length > 0) { - Logger.log(`🌐 CORS enabled with restricted origins: ${origin.join(', ')}`); - } else if (origin === '*') { - Logger.log('🌐 CORS enabled with origin: * (all origins allowed - development mode)'); - } else { - Logger.log('🌐 CORS disabled (no origins allowed)'); - } - - if (!typeormConfig.synchronize && typeormConfig.migrations?.length) { - const dataSource = app.get(DataSource); - - try { - Logger.log('🔄 Running pending migrations...'); - await dataSource.runMigrations(); - Logger.log('✅ Migrations completed successfully'); - } catch (error) { - Logger.error('❌ Failed to run migrations:', error); - throw error; - } - } else if (typeormConfig.synchronize) { - Logger.log('ℹī¸ Schema synchronization enabled - migrations skipped'); - } - - app.useGlobalPipes( - new ValidationPipe({ - whitelist: true, - forbidNonWhitelisted: true, - transform: true, - }), - ); - - const globalPrefix = 'api'; - - app.setGlobalPrefix(globalPrefix); - const port = parseInt(process.env.PORT || '3200', 10); - - await app.listen(port); - Logger.log(`🚀 Application is running on: http://localhost:${port}/${globalPrefix}`); -} +import { bootstrap } from './bootstrap'; bootstrap(); diff --git a/apps/backend-billing-manager/src/queue/billing-queue-registrar.service.ts b/apps/backend-billing-manager/src/queue/billing-queue-registrar.service.ts new file mode 100644 index 00000000..bf299d7e --- /dev/null +++ b/apps/backend-billing-manager/src/queue/billing-queue-registrar.service.ts @@ -0,0 +1,33 @@ +import { shouldRegisterRepeatableJobs } from '@forepath/shared/backend'; +import { InjectQueue } from '@nestjs/bullmq'; +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { Queue } from 'bullmq'; + +import { BILLING_QUEUE_NAME, getBillingRepeatableJobs } from './job-registry'; + +@Injectable() +export class BillingQueueRegistrarService implements OnModuleInit { + private readonly logger = new Logger(BillingQueueRegistrarService.name); + + constructor(@InjectQueue(BILLING_QUEUE_NAME) private readonly billingQueue: Queue) {} + + async onModuleInit(): Promise { + if (!shouldRegisterRepeatableJobs()) { + return; + } + + for (const definition of getBillingRepeatableJobs()) { + await this.billingQueue.add( + definition.name, + {}, + { + jobId: definition.coordinatorJobId, + repeat: { every: definition.everyMs }, + removeOnComplete: true, + removeOnFail: 100, + }, + ); + this.logger.log(`Registered repeatable job ${definition.name} every ${definition.everyMs}ms`); + } + } +} diff --git a/apps/backend-billing-manager/src/queue/billing-queue.module.ts b/apps/backend-billing-manager/src/queue/billing-queue.module.ts new file mode 100644 index 00000000..625e3e2f --- /dev/null +++ b/apps/backend-billing-manager/src/queue/billing-queue.module.ts @@ -0,0 +1,21 @@ +import { BillingModule } from '@forepath/framework/backend'; +import { SharedQueueModule, shouldRegisterRepeatableJobs, shouldRunQueueWorkers } from '@forepath/shared/backend'; +import { Module } from '@nestjs/common'; + +import { BillingQueueRegistrarService } from './billing-queue-registrar.service'; +import { BILLING_QUEUE_NAME } from './job-registry'; +import { BillingJobsProcessor } from './processors/billing-jobs.processor'; + +@Module({ + imports: [ + SharedQueueModule.forRoot({ + queueNames: [BILLING_QUEUE_NAME], + }), + BillingModule, + ], + providers: [ + ...(shouldRunQueueWorkers() ? [BillingJobsProcessor] : []), + ...(shouldRegisterRepeatableJobs() ? [BillingQueueRegistrarService] : []), + ], +}) +export class BillingQueueModule {} diff --git a/apps/backend-billing-manager/src/queue/job-registry.spec.ts b/apps/backend-billing-manager/src/queue/job-registry.spec.ts new file mode 100644 index 00000000..d3736a27 --- /dev/null +++ b/apps/backend-billing-manager/src/queue/job-registry.spec.ts @@ -0,0 +1,22 @@ +import { BillingJobName, getBillingRepeatableJobs } from './job-registry'; + +describe('billing job-registry', () => { + it('defines coordinator job names', () => { + expect(BillingJobName.SUBSCRIPTION_BILLING_COORDINATOR).toBe('subscription-billing.coordinator'); + expect(BillingJobName.BACKORDER_RETRY_UNIT).toBe('backorder-retry.unit'); + }); + + it('getBillingRepeatableJobs includes seven coordinators', () => { + const jobs = getBillingRepeatableJobs(); + + expect(jobs).toHaveLength(7); + expect(jobs.map((job) => job.name)).toContain(BillingJobName.INVOICE_SYNC_COORDINATOR); + }); + + it('coordinator job ids are valid for BullMQ (no colons)', () => { + for (const job of getBillingRepeatableJobs()) { + expect(job.coordinatorJobId).not.toContain(':'); + expect(job.coordinatorJobId.startsWith('coordinator.')).toBe(true); + } + }); +}); diff --git a/apps/backend-billing-manager/src/queue/job-registry.ts b/apps/backend-billing-manager/src/queue/job-registry.ts new file mode 100644 index 00000000..1181f3fb --- /dev/null +++ b/apps/backend-billing-manager/src/queue/job-registry.ts @@ -0,0 +1,77 @@ +import { buildCoordinatorJobId } from '@forepath/shared/backend'; + +/** Central registry for billing-manager BullMQ queues, job names, and coordinator schedules. */ + +export const BILLING_QUEUE_NAME = 'billing'; + +export const BillingJobName = { + SUBSCRIPTION_BILLING_COORDINATOR: 'subscription-billing.coordinator', + SUBSCRIPTION_BILLING_UNIT: 'subscription-billing.unit', + SUBSCRIPTION_EXPIRATION_COORDINATOR: 'subscription-expiration.coordinator', + SUBSCRIPTION_EXPIRATION_UNIT: 'subscription-expiration.unit', + INVOICE_SYNC_COORDINATOR: 'invoice-sync.coordinator', + INVOICE_SYNC_UNIT: 'invoice-sync.unit', + OPEN_POSITION_INVOICE_COORDINATOR: 'open-position-invoice.coordinator', + OPEN_POSITION_INVOICE_UNIT: 'open-position-invoice.unit', + RENEWAL_REMINDER_COORDINATOR: 'renewal-reminder.coordinator', + RENEWAL_REMINDER_UNIT: 'renewal-reminder.unit', + SUBSCRIPTION_ITEM_UPDATE_COORDINATOR: 'subscription-item-update.coordinator', + SUBSCRIPTION_ITEM_UPDATE_UNIT: 'subscription-item-update.unit', + BACKORDER_RETRY_COORDINATOR: 'backorder-retry.coordinator', + BACKORDER_RETRY_UNIT: 'backorder-retry.unit', +} as const; + +export type BillingJobName = (typeof BillingJobName)[keyof typeof BillingJobName]; + +export interface BillingRepeatableJobDefinition { + name: BillingJobName; + coordinatorJobId: string; + everyMs: number; +} + +function parseIntervalMs(envKey: string, fallback: number): number { + const parsed = parseInt(process.env[envKey] ?? String(fallback), 10); + + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +/** Repeatable coordinator jobs registered on scheduler role startup. */ +export function getBillingRepeatableJobs(): BillingRepeatableJobDefinition[] { + return [ + { + name: BillingJobName.SUBSCRIPTION_BILLING_COORDINATOR, + coordinatorJobId: buildCoordinatorJobId('subscription-billing'), + everyMs: parseIntervalMs('BILLING_SCHEDULER_INTERVAL', 60_000), + }, + { + name: BillingJobName.SUBSCRIPTION_EXPIRATION_COORDINATOR, + coordinatorJobId: buildCoordinatorJobId('subscription-expiration'), + everyMs: parseIntervalMs('EXPIRATION_SCHEDULER_INTERVAL', 60_000), + }, + { + name: BillingJobName.INVOICE_SYNC_COORDINATOR, + coordinatorJobId: buildCoordinatorJobId('invoice-sync'), + everyMs: parseIntervalMs('INVOICE_SYNC_SCHEDULER_INTERVAL', 60_000), + }, + { + name: BillingJobName.OPEN_POSITION_INVOICE_COORDINATOR, + coordinatorJobId: buildCoordinatorJobId('open-position-invoice'), + everyMs: parseIntervalMs('OPEN_POSITION_INVOICE_SCHEDULER_INTERVAL', 86_400_000), + }, + { + name: BillingJobName.RENEWAL_REMINDER_COORDINATOR, + coordinatorJobId: buildCoordinatorJobId('renewal-reminder'), + everyMs: parseIntervalMs('REMINDER_SCHEDULER_INTERVAL', 3_600_000), + }, + { + name: BillingJobName.SUBSCRIPTION_ITEM_UPDATE_COORDINATOR, + coordinatorJobId: buildCoordinatorJobId('subscription-item-update'), + everyMs: parseIntervalMs('SUBSCRIPTION_UPDATE_SCHEDULER_INTERVAL', 86_400_000), + }, + { + name: BillingJobName.BACKORDER_RETRY_COORDINATOR, + coordinatorJobId: buildCoordinatorJobId('backorder-retry'), + everyMs: parseIntervalMs('BACKORDER_RETRY_INTERVAL_MS', 60_000), + }, + ]; +} diff --git a/apps/backend-billing-manager/src/queue/processors/billing-jobs.processor.ts b/apps/backend-billing-manager/src/queue/processors/billing-jobs.processor.ts new file mode 100644 index 00000000..4bab4a49 --- /dev/null +++ b/apps/backend-billing-manager/src/queue/processors/billing-jobs.processor.ts @@ -0,0 +1,201 @@ +import { + BackorderRetryJobHandler, + InvoiceSyncJobHandler, + OpenPositionInvoiceJobHandler, + SubscriptionBillingJobHandler, + SubscriptionExpirationJobHandler, + SubscriptionItemUpdateJobHandler, + SubscriptionRenewalReminderJobHandler, +} from '@forepath/framework/backend'; +import { enqueueUnitJob } from '@forepath/shared/backend'; +import { InjectQueue, Processor, WorkerHost } from '@nestjs/bullmq'; +import { Logger } from '@nestjs/common'; +import { Job, Queue } from 'bullmq'; + +import { BILLING_QUEUE_NAME, BillingJobName } from '../job-registry'; + +@Processor(BILLING_QUEUE_NAME, { concurrency: parseInt(process.env.QUEUE_WORKER_CONCURRENCY ?? '5', 10) }) +export class BillingJobsProcessor extends WorkerHost { + private readonly logger = new Logger(BillingJobsProcessor.name); + + constructor( + @InjectQueue(BILLING_QUEUE_NAME) private readonly billingQueue: Queue, + private readonly subscriptionBilling: SubscriptionBillingJobHandler, + private readonly subscriptionExpiration: SubscriptionExpirationJobHandler, + private readonly invoiceSync: InvoiceSyncJobHandler, + private readonly openPositionInvoice: OpenPositionInvoiceJobHandler, + private readonly renewalReminder: SubscriptionRenewalReminderJobHandler, + private readonly subscriptionItemUpdate: SubscriptionItemUpdateJobHandler, + private readonly backorderRetry: BackorderRetryJobHandler, + ) { + super(); + } + + async process(job: Job): Promise { + switch (job.name) { + case BillingJobName.SUBSCRIPTION_BILLING_COORDINATOR: + await this.runSubscriptionBillingCoordinator(); + break; + case BillingJobName.SUBSCRIPTION_BILLING_UNIT: + await this.subscriptionBilling.processSubscription((job.data as { subscriptionId: string }).subscriptionId); + break; + case BillingJobName.SUBSCRIPTION_EXPIRATION_COORDINATOR: + await this.runSubscriptionExpirationCoordinator(); + break; + case BillingJobName.SUBSCRIPTION_EXPIRATION_UNIT: + await this.subscriptionExpiration.processSubscriptionCancellation( + (job.data as { subscriptionId: string }).subscriptionId, + ); + break; + case BillingJobName.INVOICE_SYNC_COORDINATOR: + await this.runInvoiceSyncCoordinator(); + break; + case BillingJobName.INVOICE_SYNC_UNIT: + await this.invoiceSync.syncInvoiceRef((job.data as { invoiceRefId: string }).invoiceRefId); + break; + case BillingJobName.OPEN_POSITION_INVOICE_COORDINATOR: + await this.runOpenPositionInvoiceCoordinator(); + break; + case BillingJobName.OPEN_POSITION_INVOICE_UNIT: + await this.openPositionInvoice.processUserOpenPositions((job.data as { userId: string }).userId); + break; + case BillingJobName.RENEWAL_REMINDER_COORDINATOR: + await this.runRenewalReminderCoordinator(); + break; + case BillingJobName.RENEWAL_REMINDER_UNIT: + await this.renewalReminder.processReminder(job.data as { subscriptionId: string; periodKey: string }); + break; + case BillingJobName.SUBSCRIPTION_ITEM_UPDATE_COORDINATOR: + await this.runSubscriptionItemUpdateCoordinator(); + break; + case BillingJobName.SUBSCRIPTION_ITEM_UPDATE_UNIT: + await this.subscriptionItemUpdate.updateItem((job.data as { subscriptionItemId: string }).subscriptionItemId); + break; + case BillingJobName.BACKORDER_RETRY_COORDINATOR: + await this.runBackorderRetryCoordinator(); + break; + case BillingJobName.BACKORDER_RETRY_UNIT: + await this.backorderRetry.retryBackorder((job.data as { backorderId: string }).backorderId); + break; + default: + this.logger.warn(`Unknown billing job name: ${job.name}`); + } + } + + private async runSubscriptionBillingCoordinator(): Promise { + const ids = await this.subscriptionBilling.findDueSubscriptionIds(); + + for (const subscriptionId of ids) { + await enqueueUnitJob({ + queue: this.billingQueue, + jobName: BillingJobName.SUBSCRIPTION_BILLING_UNIT, + payload: { subscriptionId }, + jobIdNamespace: 'billing:subscription', + jobIdParts: [subscriptionId], + }); + } + } + + private async runSubscriptionExpirationCoordinator(): Promise { + const ids = await this.subscriptionExpiration.findExpiredSubscriptionIds(); + + for (const subscriptionId of ids) { + await enqueueUnitJob({ + queue: this.billingQueue, + jobName: BillingJobName.SUBSCRIPTION_EXPIRATION_UNIT, + payload: { subscriptionId }, + jobIdNamespace: 'expiration:subscription', + jobIdParts: [subscriptionId], + }); + } + } + + private async runInvoiceSyncCoordinator(): Promise { + let offset = 0; + + // eslint-disable-next-line no-constant-condition + while (true) { + const ids = await this.invoiceSync.findInvoiceRefIdsPage(offset); + + if (ids.length === 0) { + break; + } + + for (const invoiceRefId of ids) { + await enqueueUnitJob({ + queue: this.billingQueue, + jobName: BillingJobName.INVOICE_SYNC_UNIT, + payload: { invoiceRefId }, + jobIdNamespace: 'invoice-sync:ref', + jobIdParts: [invoiceRefId], + }); + } + + offset += ids.length; + + if (ids.length < this.invoiceSync.batchSizeLimit) { + break; + } + } + } + + private async runOpenPositionInvoiceCoordinator(): Promise { + const userIds = await this.openPositionInvoice.findUserIdsForTodayBillingDay(); + + for (const userId of userIds) { + await enqueueUnitJob({ + queue: this.billingQueue, + jobName: BillingJobName.OPEN_POSITION_INVOICE_UNIT, + payload: { userId }, + jobIdNamespace: 'open-position-invoice:user', + jobIdParts: [userId], + }); + } + } + + private async runRenewalReminderCoordinator(): Promise { + if (!this.renewalReminder.isEmailEnabled()) { + return; + } + + const units = await this.renewalReminder.findUpcomingReminderUnits(); + + for (const unit of units) { + await enqueueUnitJob({ + queue: this.billingQueue, + jobName: BillingJobName.RENEWAL_REMINDER_UNIT, + payload: unit, + jobIdNamespace: 'renewal-reminder', + jobIdParts: [unit.periodKey], + }); + } + } + + private async runSubscriptionItemUpdateCoordinator(): Promise { + const ids = await this.subscriptionItemUpdate.findProvisionedItemIds(); + + for (const subscriptionItemId of ids) { + await enqueueUnitJob({ + queue: this.billingQueue, + jobName: BillingJobName.SUBSCRIPTION_ITEM_UPDATE_UNIT, + payload: { subscriptionItemId }, + jobIdNamespace: 'subscription-item-update', + jobIdParts: [subscriptionItemId], + }); + } + } + + private async runBackorderRetryCoordinator(): Promise { + const ids = await this.backorderRetry.findPendingBackorderIds(); + + for (const backorderId of ids) { + await enqueueUnitJob({ + queue: this.billingQueue, + jobName: BillingJobName.BACKORDER_RETRY_UNIT, + payload: { backorderId }, + jobIdNamespace: 'backorder-retry', + jobIdParts: [backorderId], + }); + } + } +} diff --git a/docs/agenstra/deployment/background-jobs.md b/docs/agenstra/deployment/background-jobs.md new file mode 100644 index 00000000..8f340fdb --- /dev/null +++ b/docs/agenstra/deployment/background-jobs.md @@ -0,0 +1,64 @@ +# Background jobs (BullMQ) + +Background work for **backend agent controller** and **backend billing manager** runs through **Redis + BullMQ** instead of in-process `setInterval` loops in the API container. + +## Architecture + +| Role | `QUEUE_ROLE` | HTTP API | Registers repeatable coordinators | Processes unit jobs | +| ------------------------------------ | ------------ | -------- | --------------------------------- | ------------------- | +| API (default in compose API service) | `api` | Yes | No | No | +| Scheduler | `scheduler` | No | Yes | No | +| Worker | `worker` | No | No | Yes | +| Local all-in-one | `all` | Yes | Yes | Yes | + +Each backend stack has its own **Redis** service in Docker Compose. Workers and schedulers use the **same environment variables** as the API (database, tokens, scheduler intervals, etc.). **Database migrations** run only on containers with `QUEUE_ROLE=api` or `QUEUE_ROLE=all` (faster worker/scheduler startup, no concurrent migration runners). + +Job registration (queue names, repeatable intervals, job names) lives in one file per app: + +- `apps/backend-agent-controller/src/queue/job-registry.ts` +- `apps/backend-billing-manager/src/queue/job-registry.ts` + +Coordinators fan out **unit jobs** (one subscription, one ticket, one import config, etc.). BullMQ `jobId` values prevent duplicate active work for the same entity. Custom job IDs use `.` separators and only allowed characters (alphanumeric, `.`, `-`, `_`, `~`) — e.g. `coordinator.filter-rules-sync`, `billing.subscription.`. Colons and slashes are not valid. + +## Redis and queue environment variables + +| Variable | Purpose | Default (local) | +| --------------------------- | -------------------------------------- | ----------------------------------------------- | +| `REDIS_HOST` | Redis hostname | `localhost` / compose service `redis` | +| `REDIS_PORT` | Redis port | `6379` | +| `REDIS_PASSWORD` | Optional password | empty | +| `REDIS_DB` | Redis database index | `0` | +| `REDIS_KEY_PREFIX` | Key namespace | `agenstra-controller` or `agenstra-billing` | +| `QUEUE_ROLE` | `api`, `scheduler`, `worker`, or `all` | `all` (local), `api` in API container | +| `QUEUE_WORKER_CONCURRENCY` | Default worker concurrency | `5` | +| `QUEUE_BULL_BOARD_ENABLED` | Mount Bull Board UI | `true` in dev when role is `all` or `scheduler` | +| `QUEUE_BULL_BOARD_PATH` | Bull Board route | `/admin/queues` | +| `QUEUE_BULL_BOARD_USERNAME` | HTTP Basic username | `admin` | +| `QUEUE_BULL_BOARD_PASSWORD` | HTTP Basic password (required) | `bullmq` in local compose | + +Existing `*_SCHEDULER_INTERVAL*` variables now control **coordinator repeat** intervals (milliseconds). + +## Docker Compose + +Each backend `docker-compose.yaml` defines: + +- `redis` +- `backend-*` (API, `QUEUE_ROLE=api`) +- `backend-*-scheduler` (`QUEUE_ROLE=scheduler`) +- `backend-*-worker` (`QUEUE_ROLE=worker`) + +Billing Redis is published on host port **6380** by default so it can run alongside controller Redis (**6379**). + +## Bull Board + +When enabled on the API container (`QUEUE_BULL_BOARD_ENABLED=true`, default in compose), Bull Board is served at **`QUEUE_BULL_BOARD_PATH`** (default **`/admin/queues`**) on the API port (controller **3100**, billing **3200**). That path is excluded from the Nest global `/api` prefix, so use `http://localhost:3100/admin/queues`, not `/api/admin/queues`. + +Bull Board uses **HTTP Basic authentication** (`QUEUE_BULL_BOARD_USERNAME` / `QUEUE_BULL_BOARD_PASSWORD`). Local compose defaults to `admin` / `bullmq`; override in production. Startup fails in production if the board is enabled without a password. + +Bull Board routes bypass the API **origin allowlist** and **HybridAuthGuard** so dashboard actions (retry, delete, clean) are not blocked with `403 Forbidden` when the UI sends browser `Origin` headers or `Authorization: Basic` instead of the API key. + +## Related documentation + +- [Environment configuration](./environment-configuration.md) +- [Local development](./local-development.md) +- [Docker deployment](./docker-deployment.md) diff --git a/docs/agenstra/deployment/environment-configuration.md b/docs/agenstra/deployment/environment-configuration.md index 173e8796..9cc7d604 100644 --- a/docs/agenstra/deployment/environment-configuration.md +++ b/docs/agenstra/deployment/environment-configuration.md @@ -233,6 +233,26 @@ When `CONFIG` is set, the frontend server fetches and validates the remote JSON - `KEYCLOAK_REALM` - Keycloak realm - `KEYCLOAK_CLIENT_ID` - Keycloak client ID +## Redis and BullMQ (background jobs) + +Used by **backend agent controller** and **backend billing manager**. See [Background jobs](./background-jobs.md). + +| Variable | Description | Default | +| --------------------------- | -------------------------------------- | ------------------------------------------ | +| `REDIS_HOST` | Redis host | `localhost` (compose: `redis`) | +| `REDIS_PORT` | Redis port | `6379` | +| `REDIS_PASSWORD` | Optional password | empty | +| `REDIS_DB` | Redis DB index | `0` | +| `REDIS_KEY_PREFIX` | Key prefix | `agenstra-controller` / `agenstra-billing` | +| `QUEUE_ROLE` | `api`, `scheduler`, `worker`, or `all` | `all` locally; `api` for API container | +| `QUEUE_WORKER_CONCURRENCY` | Worker concurrency | `5` | +| `QUEUE_BULL_BOARD_ENABLED` | Enable Bull Board | `true` in dev for `all`/`scheduler` | +| `QUEUE_BULL_BOARD_PATH` | Bull Board path | `/admin/queues` | +| `QUEUE_BULL_BOARD_USERNAME` | Bull Board HTTP Basic user | `admin` | +| `QUEUE_BULL_BOARD_PASSWORD` | Bull Board HTTP Basic password | required; `bullmq` in local compose | + +Scheduler interval variables (e.g. `BILLING_SCHEDULER_INTERVAL`, `AUTONOMOUS_TICKET_SCHEDULER_INTERVAL_MS`) configure **coordinator** repeat intervals in BullMQ. + ## Environment-Specific Defaults ### Development @@ -254,6 +274,7 @@ When `CONFIG` is set, the frontend server fetches and validates the remote JSON - **[Local Development](./local-development.md)** - Local setup - **[Docker Deployment](./docker-deployment.md)** - Containerized deployment - **[Production Checklist](./production-checklist.md)** - Production deployment +- **[Background jobs](./background-jobs.md)** - BullMQ roles, Redis, and coordinators - **[Atlassian import](../features/atlassian-import.md)** - Import feature, markers, and console entry points --- diff --git a/docs/agenstra/features/atlassian-import.md b/docs/agenstra/features/atlassian-import.md index ae3975eb..fcac2bb0 100644 --- a/docs/agenstra/features/atlassian-import.md +++ b/docs/agenstra/features/atlassian-import.md @@ -27,17 +27,17 @@ Non-admin interactive users receive **403** from the API and cannot open the adm Exact field matrices and validation rules are defined in the OpenAPI DTOs (`CreateExternalImportConfigDto`, `UpdateExternalImportConfigDto`, etc.). -## Scheduler and manual runs +## Queue jobs and manual runs -The **Context import scheduler** runs on an interval in the controller process. Relevant environment variables (defaults shown where applicable): +**Context import** uses BullMQ: a repeatable coordinator enqueues one unit job per enabled config (see [Background jobs](../deployment/background-jobs.md)). Relevant environment variables (defaults shown where applicable): -| Variable | Role | -| --------------------------------------- | ----------------------------------------------------------------------------------------------------------- | -| `CONTEXT_IMPORT_SCHEDULER_INTERVAL_MS` | Interval between ticks in milliseconds (`120000` default). Set to `0` or less to **disable** the scheduler. | -| `CONTEXT_IMPORT_SCHEDULER_CONFIG_BATCH` | Maximum number of enabled configs considered per tick (`3` default). | -| `CONTEXT_IMPORT_ITEM_BUDGET` | Soft cap on work items processed per config per run (`25` default). | +| Variable | Role | +| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | +| `CONTEXT_IMPORT_SCHEDULER_INTERVAL_MS` | Coordinator repeat interval in milliseconds (`120000` default). Set to `0` or less to **disable** the coordinator. | +| `CONTEXT_IMPORT_SCHEDULER_CONFIG_BATCH` | Maximum enabled configs enqueued per coordinator tick (`3` default). | +| `CONTEXT_IMPORT_ITEM_BUDGET` | Soft cap on work items processed per config per unit job (`25` default). | -Admins may trigger **`POST /api/imports/atlassian/configs/{id}/run`** (HTTP 202) to enqueue work for one config without waiting for the next scheduler tick. +Admins may trigger **`POST /api/imports/atlassian/configs/{id}/run`** (HTTP 202) to run one config immediately without waiting for the next coordinator tick. To disable import execution entirely (for example in a staging environment), set **`ATLASSIAN_IMPORT_DISABLED=true`**. The provider then returns a no-op result for import runs while connections and configs remain manageable via the API. diff --git a/docs/agenstra/features/ticket-automation.md b/docs/agenstra/features/ticket-automation.md index d6b0ffa0..c62c3739 100644 --- a/docs/agenstra/features/ticket-automation.md +++ b/docs/agenstra/features/ticket-automation.md @@ -18,16 +18,16 @@ Automation only starts when **all** of the following hold: The controller picks candidates with a SQL query that joins `tickets`, `ticket_automation`, and `client_agent_autonomy` (`enabled = true`). If several agents have autonomy enabled and `allowed_agent_ids` is empty, the same ticket can appear as multiple candidates (one per agent); narrowing `allowed_agent_ids` pins the workload to specific agents. -## Background scheduler +## Background jobs (BullMQ) -The **backend agent controller** runs an in-process scheduler (`AutonomousTicketScheduler`) that wakes on a fixed interval, processes at most **N** tickets per tick, and avoids overlapping ticks if a batch runs longer than the interval. +The **backend agent controller** registers a repeatable **coordinator** job on Redis (BullMQ). Each coordinator tick enqueues at most **N** **unit** jobs (one per ticket candidate). Workers process unit jobs in parallel; BullMQ `jobId` deduplication and DB leases prevent double-starts. -Operator environment variables (see [Environment configuration](../deployment/environment-configuration.md#autonomous-ticket-automation-scheduler)): +Operator environment variables (see [Environment configuration](../deployment/environment-configuration.md) and [Background jobs](../deployment/background-jobs.md)): -- `AUTONOMOUS_TICKET_SCHEDULER_INTERVAL_MS` – Tick interval in milliseconds (default `60000`). -- `AUTONOMOUS_TICKET_SCHEDULER_BATCH_SIZE` – Maximum candidates processed per tick (default `5`). +- `AUTONOMOUS_TICKET_SCHEDULER_INTERVAL_MS` – Coordinator repeat interval in milliseconds (default `60000`). +- `AUTONOMOUS_TICKET_SCHEDULER_BATCH_SIZE` – Maximum candidates enqueued per coordinator tick (default `5`). -There is no separate “start run” HTTP call for this path: eligible tickets are picked up automatically on the next successful scheduler tick. +There is no separate “start run” HTTP call for this path: eligible tickets are picked up when a worker processes their unit job. ## Run phases (high level) diff --git a/libs/domains/framework/backend/feature-agent-controller/docs/filter-rules-sync.mmd b/libs/domains/framework/backend/feature-agent-controller/docs/filter-rules-sync.mmd index 327c7b92..928148ac 100644 --- a/libs/domains/framework/backend/feature-agent-controller/docs/filter-rules-sync.mmd +++ b/libs/domains/framework/backend/feature-agent-controller/docs/filter-rules-sync.mmd @@ -1,5 +1,6 @@ sequenceDiagram - participant Sched as FilterRulesSyncScheduler + participant Coord as FilterRulesSyncCoordinator + participant Worker as FilterRulesSyncWorker participant Sync as FilterRulesSyncService participant DB as SyncTargetsTable participant AM as AgentManager diff --git a/libs/domains/framework/backend/feature-agent-controller/docs/sequence-autonomous-ticket.mmd b/libs/domains/framework/backend/feature-agent-controller/docs/sequence-autonomous-ticket.mmd index 5a5e162c..894996bd 100644 --- a/libs/domains/framework/backend/feature-agent-controller/docs/sequence-autonomous-ticket.mmd +++ b/libs/domains/framework/backend/feature-agent-controller/docs/sequence-autonomous-ticket.mmd @@ -1,5 +1,6 @@ sequenceDiagram - participant Sched as AutonomousTicketScheduler + participant Coord as QueueCoordinator + participant Worker as QueueWorker participant Orch as AutonomousRunOrchestrator participant DB as Postgres participant Vcs as ClientAgentVcsProxy diff --git a/libs/domains/framework/backend/feature-agent-controller/src/index.ts b/libs/domains/framework/backend/feature-agent-controller/src/index.ts index e44a2402..f6f02b25 100644 --- a/libs/domains/framework/backend/feature-agent-controller/src/index.ts +++ b/libs/domains/framework/backend/feature-agent-controller/src/index.ts @@ -118,8 +118,12 @@ export * from './lib/services/autonomous-run-orchestrator.service'; export * from './lib/services/agent-manager-filter-rules-client.service'; export * from './lib/services/filter-rules.service'; export * from './lib/services/filter-rules-sync.service'; -export * from './lib/services/filter-rules-sync.scheduler'; -export * from './lib/services/autonomous-ticket.scheduler'; +export * from './lib/services/autonomous-run-orchestrator.service'; +export * from './lib/services/context-import-orchestrator.service'; +export * from './lib/services/external-import-config.service'; +export * from './lib/modules/clients.module'; +export * from './lib/modules/context-import.module'; +export * from './lib/modules/filter-rules.module'; export * from './lib/services/client-agent-autonomy.service'; export * from './lib/services/client-agent-proxy.service'; export * from './lib/services/auto-context-resolver.service'; diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/clients.module.spec.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/clients.module.spec.ts index 9491c40b..bab64b2d 100644 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/clients.module.spec.ts +++ b/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/clients.module.spec.ts @@ -42,12 +42,10 @@ import { TicketEntity } from '../entities/ticket.entity'; import { UserEnvironmentReadStateEntity } from '../entities/user-environment-read-state.entity'; import { ClientsGateway } from '../gateways/clients.gateway'; import { ClientsRepository } from '../repositories/clients.repository'; -import { AutonomousTicketScheduler } from '../services/autonomous-ticket.scheduler'; import { ClientAgentFileSystemProxyService } from '../services/client-agent-file-system-proxy.service'; import { ClientAgentProxyService } from '../services/client-agent-proxy.service'; import { ClientsService } from '../services/clients.service'; import { ExternalImportSyncMarkerService } from '../services/external-import-sync-marker.service'; -import { KnowledgeEmbeddingIndexScheduler } from '../services/knowledge-embedding-index.scheduler'; import { ClientsModule } from './clients.module'; import { ContextImportModule } from './context-import.module'; @@ -181,19 +179,7 @@ describe('ClientsModule', () => { .overrideProvider(getRepositoryToken(UserEnvironmentReadStateEntity)) .useValue(mockRepository) .overrideProvider(UsersRepository) - .useValue(mockRepository) - .overrideProvider(AutonomousTicketScheduler) - .useValue({ - onModuleInit: jest.fn(), - onModuleDestroy: jest.fn(), - tick: jest.fn().mockResolvedValue(undefined), - }) - .overrideProvider(KnowledgeEmbeddingIndexScheduler) - .useValue({ - onModuleInit: jest.fn(), - onModuleDestroy: jest.fn(), - tick: jest.fn().mockResolvedValue(undefined), - }); + .useValue(mockRepository); // Mock Keycloak providers if auth method is keycloak if (authMethod === 'keycloak') { diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/clients.module.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/clients.module.ts index 5ecd2eaa..a8f5e747 100644 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/clients.module.ts +++ b/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/clients.module.ts @@ -59,7 +59,6 @@ import { AgentConsoleStatusRealtimeService } from '../services/agent-console-sta import { AgentConsoleStatusService } from '../services/agent-console-status.service'; import { AutoContextResolverService } from '../services/auto-context-resolver.service'; import { AutonomousRunOrchestratorService } from '../services/autonomous-run-orchestrator.service'; -import { AutonomousTicketScheduler } from '../services/autonomous-ticket.scheduler'; import { ClientAgentAutonomyService } from '../services/client-agent-autonomy.service'; import { ClientAgentDeploymentsProxyService } from '../services/client-agent-deployments-proxy.service'; import { ClientAgentEnvironmentVariablesProxyService } from '../services/client-agent-environment-variables-proxy.service'; @@ -73,7 +72,6 @@ import { ClientsService } from '../services/clients.service'; import { KnowledgeEmbeddingIndexService } from '../services/embeddings/knowledge-embedding-index.service'; import { LocalEmbeddingProvider } from '../services/embeddings/local-embedding.provider'; import { KnowledgeBoardRealtimeService } from '../services/knowledge-board-realtime.service'; -import { KnowledgeEmbeddingIndexScheduler } from '../services/knowledge-embedding-index.scheduler'; import { KnowledgeTreeService } from '../services/knowledge-tree.service'; import { ProvisioningService } from '../services/provisioning.service'; import { RemoteAgentsSessionService } from '../services/remote-agents-session.service'; @@ -142,13 +140,11 @@ const authMethod = getAuthenticationMethod(); KnowledgeTreeService, AutoContextResolverService, KnowledgeEmbeddingIndexService, - KnowledgeEmbeddingIndexScheduler, LocalEmbeddingProvider, TicketAutomationService, ClientAgentAutonomyService, RemoteAgentsSessionService, AutonomousRunOrchestratorService, - AutonomousTicketScheduler, ClientsRepository, ClientUsersRepository, ClientUsersService, @@ -216,6 +212,8 @@ const authMethod = getAuthenticationMethod(); ProvisioningProviderFactory, TicketsService, KnowledgeTreeService, + KnowledgeEmbeddingIndexService, + AutonomousRunOrchestratorService, ], }) export class ClientsModule {} diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/context-import.module.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/context-import.module.ts index 2050106d..6b958501 100644 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/context-import.module.ts +++ b/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/context-import.module.ts @@ -10,7 +10,6 @@ import { CONTEXT_IMPORT_PROVIDERS } from '../providers/external-import-provider. import { AtlassianImportProvider } from '../providers/import/atlassian-external-import.provider'; import { AtlassianSiteConnectionService } from '../services/atlassian-site-connection.service'; import { ContextImportOrchestratorService } from '../services/context-import-orchestrator.service'; -import { ContextImportScheduler } from '../services/context-import.scheduler'; import { ExternalImportConfigService } from '../services/external-import-config.service'; import { ExternalImportSyncMarkerService } from '../services/external-import-sync-marker.service'; @@ -33,7 +32,6 @@ import { ClientsModule } from './clients.module'; ExternalImportConfigService, ContextImportOrchestratorService, AtlassianImportProvider, - ContextImportScheduler, { provide: CONTEXT_IMPORT_PROVIDERS, useFactory: (factory: ExternalImportProviderFactory, atlassian: AtlassianImportProvider) => { @@ -44,6 +42,6 @@ import { ClientsModule } from './clients.module'; inject: [ExternalImportProviderFactory, AtlassianImportProvider], }, ], - exports: [ExternalImportSyncMarkerService], + exports: [ExternalImportSyncMarkerService, ContextImportOrchestratorService, ExternalImportConfigService], }) export class ContextImportModule {} diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/filter-rules.module.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/filter-rules.module.ts index d809c3b4..3d56befb 100644 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/filter-rules.module.ts +++ b/libs/domains/framework/backend/feature-agent-controller/src/lib/modules/filter-rules.module.ts @@ -7,7 +7,6 @@ import { AgentConsoleRegexFilterRuleClientEntity } from '../entities/agent-conso import { AgentConsoleRegexFilterRuleSyncTargetEntity } from '../entities/agent-console-regex-filter-rule-sync-target.entity'; import { AgentConsoleRegexFilterRuleEntity } from '../entities/agent-console-regex-filter-rule.entity'; import { AgentManagerFilterRulesClientService } from '../services/agent-manager-filter-rules-client.service'; -import { FilterRulesSyncScheduler } from '../services/filter-rules-sync.scheduler'; import { FilterRulesSyncService } from '../services/filter-rules-sync.service'; import { FilterRulesService } from '../services/filter-rules.service'; @@ -29,12 +28,7 @@ import { ClientsModule } from './clients.module'; forwardRef(() => ClientsModule), ], controllers: [FilterRulesController], - providers: [ - AgentManagerFilterRulesClientService, - FilterRulesService, - FilterRulesSyncService, - FilterRulesSyncScheduler, - ], - exports: [FilterRulesService], + providers: [AgentManagerFilterRulesClientService, FilterRulesService, FilterRulesSyncService], + exports: [FilterRulesService, FilterRulesSyncService], }) export class FilterRulesModule {} diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/autonomous-run-orchestrator.service.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/autonomous-run-orchestrator.service.ts index 63c4e6b8..4cb35a60 100644 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/autonomous-run-orchestrator.service.ts +++ b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/autonomous-run-orchestrator.service.ts @@ -85,15 +85,23 @@ export class AutonomousRunOrchestratorService { async processBatch(batchSize: number): Promise { const candidates = await this.findCandidates(batchSize); - for (const c of candidates) { - try { - await this.tryStartRun(c); - } catch (error: unknown) { - const message = error instanceof Error ? error.message : String(error); - const stack = error instanceof Error ? error.stack : undefined; - - this.logger.warn(`Orchestrator skip ticket ${c.ticket_id}: ${message}`, stack); - } + for (const candidate of candidates) { + await this.tryStartRunForCandidate(candidate); + } + } + + async findCandidateIds(batchSize: number): Promise { + return this.findCandidates(batchSize); + } + + async tryStartRunForCandidate(candidate: RunnableCandidate): Promise { + try { + await this.tryStartRun(candidate); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + const stack = error instanceof Error ? error.stack : undefined; + + this.logger.warn(`Orchestrator skip ticket ${candidate.ticket_id}: ${message}`, stack); } } diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/autonomous-ticket.scheduler.spec.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/autonomous-ticket.scheduler.spec.ts deleted file mode 100644 index ab7a556b..00000000 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/autonomous-ticket.scheduler.spec.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { AutonomousRunOrchestratorService } from './autonomous-run-orchestrator.service'; -import { AutonomousTicketScheduler } from './autonomous-ticket.scheduler'; - -describe('AutonomousTicketScheduler', () => { - it('invokes orchestrator on tick', async () => { - const orchestrator = { processBatch: jest.fn().mockResolvedValue(undefined) }; - const module: TestingModule = await Test.createTestingModule({ - providers: [AutonomousTicketScheduler, { provide: AutonomousRunOrchestratorService, useValue: orchestrator }], - }).compile(); - const scheduler = module.get(AutonomousTicketScheduler); - - await scheduler.tick(); - expect(orchestrator.processBatch).toHaveBeenCalled(); - await module.close(); - }); - - it('does not start a second tick while the first is still in flight', async () => { - let releaseBatch: () => void; - const batchGate = new Promise((resolve) => { - releaseBatch = resolve; - }); - const orchestrator = { - processBatch: jest.fn().mockImplementation(() => batchGate), - }; - const module: TestingModule = await Test.createTestingModule({ - providers: [AutonomousTicketScheduler, { provide: AutonomousRunOrchestratorService, useValue: orchestrator }], - }).compile(); - const scheduler = module.get(AutonomousTicketScheduler); - const first = scheduler.tick(); - const second = scheduler.tick(); - - expect(orchestrator.processBatch).toHaveBeenCalledTimes(1); - releaseBatch!(); - await first; - await second; - expect(orchestrator.processBatch).toHaveBeenCalledTimes(1); - await module.close(); - }); - - it('runs another tick after the previous one finished', async () => { - const orchestrator = { processBatch: jest.fn().mockResolvedValue(undefined) }; - const module: TestingModule = await Test.createTestingModule({ - providers: [AutonomousTicketScheduler, { provide: AutonomousRunOrchestratorService, useValue: orchestrator }], - }).compile(); - const scheduler = module.get(AutonomousTicketScheduler); - - await scheduler.tick(); - await scheduler.tick(); - expect(orchestrator.processBatch).toHaveBeenCalledTimes(2); - await module.close(); - }); -}); diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/autonomous-ticket.scheduler.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/autonomous-ticket.scheduler.ts deleted file mode 100644 index 0febe362..00000000 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/autonomous-ticket.scheduler.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; - -import { AutonomousRunOrchestratorService } from './autonomous-run-orchestrator.service'; - -@Injectable() -export class AutonomousTicketScheduler implements OnModuleInit, OnModuleDestroy { - private readonly logger = new Logger(AutonomousTicketScheduler.name); - private intervalHandle?: NodeJS.Timeout; - /** Avoid overlapping ticks when a batch outlasts the interval. */ - private tickInFlight = false; - - private readonly intervalMs = parseInt(process.env.AUTONOMOUS_TICKET_SCHEDULER_INTERVAL_MS ?? '60000', 10); - private readonly batchSize = parseInt(process.env.AUTONOMOUS_TICKET_SCHEDULER_BATCH_SIZE ?? '5', 10); - - constructor(private readonly orchestrator: AutonomousRunOrchestratorService) {} - - onModuleInit(): void { - this.logger.log(`Autonomous ticket scheduler every ${this.intervalMs}ms, batch ${this.batchSize}`); - this.intervalHandle = setInterval(() => { - void this.tick(); - }, this.intervalMs); - } - - onModuleDestroy(): void { - if (this.intervalHandle) { - clearInterval(this.intervalHandle); - } - } - - async tick(): Promise { - if (this.tickInFlight) { - return; - } - - this.tickInFlight = true; - - try { - await this.orchestrator.processBatch(this.batchSize); - } catch (error: unknown) { - const message = error instanceof Error ? error.message : String(error); - const stack = error instanceof Error ? error.stack : undefined; - - this.logger.error(`Autonomous ticket scheduler tick failed: ${message}`, stack); - } finally { - this.tickInFlight = false; - } - } -} diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/context-import.scheduler.spec.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/context-import.scheduler.spec.ts deleted file mode 100644 index 613afc2e..00000000 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/context-import.scheduler.spec.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { ContextImportOrchestratorService } from './context-import-orchestrator.service'; -import { ContextImportScheduler } from './context-import.scheduler'; - -describe('ContextImportScheduler', () => { - const orchestrator = { runSchedulerBatch: jest.fn() }; - - beforeEach(() => { - jest.clearAllMocks(); - orchestrator.runSchedulerBatch.mockResolvedValue(1); - }); - - function createScheduler(): ContextImportScheduler { - return new ContextImportScheduler(orchestrator as unknown as ContextImportOrchestratorService); - } - - it('tick delegates to orchestrator.runSchedulerBatch', async () => { - const scheduler = createScheduler(); - - await scheduler.tick(); - - expect(orchestrator.runSchedulerBatch).toHaveBeenCalled(); - }); - - it('does not overlap ticks while the first is in flight', async () => { - let release: () => void; - const gate = new Promise((resolve) => { - release = resolve; - }); - - orchestrator.runSchedulerBatch.mockImplementation(() => gate); - const scheduler = createScheduler(); - const first = scheduler.tick(); - const second = scheduler.tick(); - - expect(orchestrator.runSchedulerBatch).toHaveBeenCalledTimes(1); - release!(); - await first; - await second; - expect(orchestrator.runSchedulerBatch).toHaveBeenCalledTimes(1); - }); - - it('allows a new tick after the previous tick finished', async () => { - const scheduler = createScheduler(); - - await scheduler.tick(); - await scheduler.tick(); - expect(orchestrator.runSchedulerBatch).toHaveBeenCalledTimes(2); - }); -}); diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/context-import.scheduler.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/context-import.scheduler.ts deleted file mode 100644 index a59bb2b9..00000000 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/context-import.scheduler.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; - -import { ContextImportOrchestratorService } from './context-import-orchestrator.service'; - -@Injectable() -export class ContextImportScheduler implements OnModuleInit, OnModuleDestroy { - private readonly logger = new Logger(ContextImportScheduler.name); - private intervalHandle?: NodeJS.Timeout; - private tickInFlight = false; - - private readonly intervalMs = parseInt(process.env.CONTEXT_IMPORT_SCHEDULER_INTERVAL_MS ?? '120000', 10); - private readonly configBatch = parseInt(process.env.CONTEXT_IMPORT_SCHEDULER_CONFIG_BATCH ?? '3', 10); - private readonly itemBudget = parseInt(process.env.CONTEXT_IMPORT_ITEM_BUDGET ?? '25', 10); - - constructor(private readonly orchestrator: ContextImportOrchestratorService) {} - - onModuleInit(): void { - if (this.intervalMs <= 0) { - this.logger.log('Context import scheduler disabled (CONTEXT_IMPORT_SCHEDULER_INTERVAL_MS <= 0)'); - - return; - } - - this.logger.log( - `Context import scheduler every ${this.intervalMs}ms, config batch ${this.configBatch}, item budget ${this.itemBudget}`, - ); - this.intervalHandle = setInterval(() => { - void this.tick(); - }, this.intervalMs); - } - - onModuleDestroy(): void { - if (this.intervalHandle) { - clearInterval(this.intervalHandle); - } - } - - async tick(): Promise { - if (this.tickInFlight) { - return; - } - - this.tickInFlight = true; - - try { - const processed = await this.orchestrator.runSchedulerBatch(this.configBatch, this.itemBudget); - - if (processed > 0) { - this.logger.debug(`Context import scheduler ran ${processed} config(s)`); - } - } catch (error: unknown) { - const message = error instanceof Error ? error.message : String(error); - const stack = error instanceof Error ? error.stack : undefined; - - this.logger.error(`Context import scheduler tick failed: ${message}`, stack); - } finally { - this.tickInFlight = false; - } - } -} diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/embeddings/knowledge-embedding-index.service.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/embeddings/knowledge-embedding-index.service.ts index dbd564da..42fb5238 100644 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/embeddings/knowledge-embedding-index.service.ts +++ b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/embeddings/knowledge-embedding-index.service.ts @@ -64,6 +64,32 @@ export class KnowledgeEmbeddingIndexService { await this.embeddingRepo.delete({ knowledgeNodeId }); } + async findPageIdsBatch( + offset: number, + limit: number, + clientId?: string, + ): Promise> { + const where = clientId + ? { clientId, nodeType: KnowledgeNodeType.PAGE } + : { + nodeType: KnowledgeNodeType.PAGE, + }; + const pages = await this.knowledgeNodeRepo.find({ + where, + select: ['id', 'clientId', 'title', 'content'], + order: { updatedAt: 'DESC' }, + skip: offset, + take: limit, + }); + + return pages.map((page) => ({ + clientId: page.clientId, + nodeId: page.id, + title: page.title, + content: page.content ?? '', + })); + } + async reindexAllPages(clientId?: string): Promise<{ processed: number }> { const where = clientId ? { clientId, nodeType: KnowledgeNodeType.PAGE } diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.scheduler.spec.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.scheduler.spec.ts deleted file mode 100644 index 02651189..00000000 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.scheduler.spec.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { FilterRulesSyncScheduler } from './filter-rules-sync.scheduler'; -import { FilterRulesSyncService } from './filter-rules-sync.service'; -import { FilterRulesService } from './filter-rules.service'; - -describe('FilterRulesSyncScheduler', () => { - const syncService = { processBatch: jest.fn() }; - const filterRulesService = { reconcileAllGlobalRules: jest.fn() }; - - beforeEach(() => { - jest.clearAllMocks(); - syncService.processBatch.mockResolvedValue(0); - filterRulesService.reconcileAllGlobalRules.mockResolvedValue(undefined); - }); - - function createScheduler(): FilterRulesSyncScheduler { - return new FilterRulesSyncScheduler( - syncService as unknown as FilterRulesSyncService, - filterRulesService as unknown as FilterRulesService, - ); - } - - it('runs processBatch then reconcileAllGlobalRules on tick', async () => { - syncService.processBatch.mockResolvedValue(3); - const scheduler = createScheduler(); - - await scheduler.tick(); - expect(syncService.processBatch).toHaveBeenCalled(); - expect(filterRulesService.reconcileAllGlobalRules).toHaveBeenCalled(); - }); - - it('does not overlap ticks while the first is in flight', async () => { - let release: () => void; - const gate = new Promise((resolve) => { - release = resolve; - }); - - syncService.processBatch.mockImplementation(() => gate); - const scheduler = createScheduler(); - const first = scheduler.tick(); - const second = scheduler.tick(); - - expect(syncService.processBatch).toHaveBeenCalledTimes(1); - release!(); - await first; - await second; - expect(syncService.processBatch).toHaveBeenCalledTimes(1); - }); - - it('allows a new tick after the previous tick finished', async () => { - const scheduler = createScheduler(); - - await scheduler.tick(); - await scheduler.tick(); - expect(syncService.processBatch).toHaveBeenCalledTimes(2); - }); - - it('still clears tickInFlight when processBatch throws', async () => { - syncService.processBatch.mockRejectedValueOnce(new Error('db down')); - const scheduler = createScheduler(); - - await expect(scheduler.tick()).resolves.toBeUndefined(); - syncService.processBatch.mockResolvedValue(0); - await scheduler.tick(); - expect(syncService.processBatch).toHaveBeenCalledTimes(2); - }); -}); diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.scheduler.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.scheduler.ts deleted file mode 100644 index bb697363..00000000 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.scheduler.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; - -import { FilterRulesSyncService } from './filter-rules-sync.service'; -import { FilterRulesService } from './filter-rules.service'; - -@Injectable() -export class FilterRulesSyncScheduler implements OnModuleInit, OnModuleDestroy { - private readonly logger = new Logger(FilterRulesSyncScheduler.name); - private intervalHandle?: NodeJS.Timeout; - private tickInFlight = false; - - private readonly intervalMs = parseInt(process.env.FILTER_RULES_SYNC_INTERVAL_MS ?? '30000', 10); - private readonly batchSize = parseInt(process.env.FILTER_RULES_SYNC_BATCH_SIZE ?? '10', 10); - - constructor( - private readonly syncService: FilterRulesSyncService, - private readonly filterRulesService: FilterRulesService, - ) {} - - onModuleInit(): void { - this.logger.log(`Filter rules sync scheduler every ${this.intervalMs}ms, batch ${this.batchSize}`); - this.intervalHandle = setInterval(() => { - void this.tick(); - }, this.intervalMs); - } - - onModuleDestroy(): void { - if (this.intervalHandle) { - clearInterval(this.intervalHandle); - } - } - - async tick(): Promise { - if (this.tickInFlight) { - return; - } - - this.tickInFlight = true; - - try { - const n = await this.syncService.processBatch(this.batchSize); - - if (n > 0) { - this.logger.debug(`Processed ${n} filter-rule sync targets`); - } - - await this.filterRulesService.reconcileAllGlobalRules(); - } catch (error: unknown) { - const message = error instanceof Error ? error.message : String(error); - const stack = error instanceof Error ? error.stack : undefined; - - this.logger.error(`Filter rules sync tick failed: ${message}`, stack); - } finally { - this.tickInFlight = false; - } - } -} diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.service.spec.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.service.spec.ts index 5a958ebf..a4ecf076 100644 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.service.spec.ts +++ b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.service.spec.ts @@ -10,11 +10,12 @@ import { FilterRulesSyncService } from './filter-rules-sync.service'; describe('FilterRulesSyncService', () => { it('processBatch returns 0 when query empty', async () => { const qb = { - innerJoinAndSelect: jest.fn().mockReturnThis(), + innerJoin: jest.fn().mockReturnThis(), where: jest.fn().mockReturnThis(), andWhere: jest.fn().mockReturnThis(), orderBy: jest.fn().mockReturnThis(), take: jest.fn().mockReturnThis(), + select: jest.fn().mockReturnThis(), getMany: jest.fn().mockResolvedValue([]), }; const targetsRepo = { createQueryBuilder: jest.fn().mockReturnValue(qb) }; diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.service.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.service.ts index 13533686..619f4af6 100644 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.service.ts +++ b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/filter-rules-sync.service.ts @@ -29,10 +29,10 @@ export class FilterRulesSyncService { /** * @returns number of targets processed */ - async processBatch(max: number): Promise { + async findPendingTargetIds(max: number): Promise { const rows = await this.targetsRepo .createQueryBuilder('t') - .innerJoinAndSelect('t.rule', 'rule') + .innerJoin('t.rule', 'rule') .where('t.sync_status IN (:...st)', { st: ['pending', 'failed'] }) .andWhere( new Brackets((qb) => { @@ -43,13 +43,34 @@ export class FilterRulesSyncService { ) .orderBy('t.updatedAt', 'ASC') .take(max) + .select(['t.id', 't.updatedAt']) .getMany(); - for (const t of rows) { - await this.processOne(t); + return rows.map((row) => row.id); + } + + async processTargetById(targetId: string): Promise { + const target = await this.targetsRepo + .createQueryBuilder('t') + .innerJoinAndSelect('t.rule', 'rule') + .where('t.id = :targetId', { targetId }) + .getOne(); + + if (!target) { + return; + } + + await this.processOne(target); + } + + async processBatch(max: number): Promise { + const ids = await this.findPendingTargetIds(max); + + for (const targetId of ids) { + await this.processTargetById(targetId); } - return rows.length; + return ids.length; } private toCreateDto(rule: AgentConsoleRegexFilterRuleEntity): CreateRegexFilterRuleDto { diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/knowledge-embedding-index.scheduler.spec.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/knowledge-embedding-index.scheduler.spec.ts deleted file mode 100644 index a369f739..00000000 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/knowledge-embedding-index.scheduler.spec.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { KnowledgeEmbeddingIndexService } from './embeddings/knowledge-embedding-index.service'; -import { KnowledgeEmbeddingIndexScheduler } from './knowledge-embedding-index.scheduler'; - -describe('KnowledgeEmbeddingIndexScheduler', () => { - const embeddingIndexService = { reindexAllPages: jest.fn() }; - - beforeEach(() => { - jest.clearAllMocks(); - embeddingIndexService.reindexAllPages.mockResolvedValue({ processed: 2 }); - }); - - function createScheduler(): KnowledgeEmbeddingIndexScheduler { - return new KnowledgeEmbeddingIndexScheduler(embeddingIndexService as unknown as KnowledgeEmbeddingIndexService); - } - - it('runs reindexAllPages on tick', async () => { - const scheduler = createScheduler(); - - await scheduler.tick(); - expect(embeddingIndexService.reindexAllPages).toHaveBeenCalledWith(); - }); - - it('runs first reindex from onModuleInit before the interval', async () => { - embeddingIndexService.reindexAllPages.mockResolvedValue({ processed: 0 }); - const scheduler = new KnowledgeEmbeddingIndexScheduler( - embeddingIndexService as unknown as KnowledgeEmbeddingIndexService, - ); - - scheduler.onModuleInit(); - await new Promise((resolve) => setImmediate(() => resolve())); - expect(embeddingIndexService.reindexAllPages).toHaveBeenCalled(); - scheduler.onModuleDestroy(); - }); - - it('does not overlap ticks while the first is in flight', async () => { - let release: () => void; - const gate = new Promise((resolve) => { - release = resolve; - }); - - embeddingIndexService.reindexAllPages.mockImplementation(() => gate); - const scheduler = createScheduler(); - const first = scheduler.tick(); - const second = scheduler.tick(); - - expect(embeddingIndexService.reindexAllPages).toHaveBeenCalledTimes(1); - release!(); - await first; - await second; - expect(embeddingIndexService.reindexAllPages).toHaveBeenCalledTimes(1); - }); - - it('allows a new tick after the previous tick finished', async () => { - const scheduler = createScheduler(); - - await scheduler.tick(); - await scheduler.tick(); - expect(embeddingIndexService.reindexAllPages).toHaveBeenCalledTimes(2); - }); - - it('still clears tickInFlight when reindexAllPages throws', async () => { - embeddingIndexService.reindexAllPages.mockRejectedValueOnce(new Error('db down')); - const scheduler = createScheduler(); - - await expect(scheduler.tick()).resolves.toBeUndefined(); - embeddingIndexService.reindexAllPages.mockResolvedValue({ processed: 0 }); - await scheduler.tick(); - expect(embeddingIndexService.reindexAllPages).toHaveBeenCalledTimes(2); - }); -}); diff --git a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/knowledge-embedding-index.scheduler.ts b/libs/domains/framework/backend/feature-agent-controller/src/lib/services/knowledge-embedding-index.scheduler.ts deleted file mode 100644 index b4e83df8..00000000 --- a/libs/domains/framework/backend/feature-agent-controller/src/lib/services/knowledge-embedding-index.scheduler.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; - -import { KnowledgeEmbeddingIndexService } from './embeddings/knowledge-embedding-index.service'; - -@Injectable() -export class KnowledgeEmbeddingIndexScheduler implements OnModuleInit, OnModuleDestroy { - private readonly logger = new Logger(KnowledgeEmbeddingIndexScheduler.name); - private intervalHandle?: NodeJS.Timeout; - private tickInFlight = false; - - private readonly intervalMs = parseInt(process.env.KNOWLEDGE_EMBEDDINGS_REINDEX_INTERVAL_MS ?? '3600000', 10); - - constructor(private readonly knowledgeEmbeddingIndexService: KnowledgeEmbeddingIndexService) {} - - onModuleInit(): void { - if (this.intervalMs <= 0) { - this.logger.log('Knowledge embedding reindex scheduler disabled (KNOWLEDGE_EMBEDDINGS_REINDEX_INTERVAL_MS <= 0)'); - - return; - } - - this.logger.log(`Knowledge embedding reindex scheduler every ${this.intervalMs}ms (first run at startup)`); - void this.tick(); - this.intervalHandle = setInterval(() => { - void this.tick(); - }, this.intervalMs); - } - - onModuleDestroy(): void { - if (this.intervalHandle) { - clearInterval(this.intervalHandle); - } - } - - async tick(): Promise { - if (this.tickInFlight) { - return; - } - - this.tickInFlight = true; - - try { - const result = await this.knowledgeEmbeddingIndexService.reindexAllPages(); - - if (result.processed > 0) { - this.logger.debug(`Knowledge embedding reindex tick processed ${result.processed} page(s)`); - } - } catch (error: unknown) { - const message = error instanceof Error ? error.message : String(error); - const stack = error instanceof Error ? error.stack : undefined; - - this.logger.error(`Knowledge embedding reindex tick failed: ${message}`, stack); - } finally { - this.tickInFlight = false; - } - } -} diff --git a/libs/domains/framework/backend/feature-billing-manager/src/index.ts b/libs/domains/framework/backend/feature-billing-manager/src/index.ts index e25037b5..38640626 100644 --- a/libs/domains/framework/backend/feature-billing-manager/src/index.ts +++ b/libs/domains/framework/backend/feature-billing-manager/src/index.ts @@ -105,11 +105,13 @@ export * from './lib/services/provisioning.service'; export * from './lib/services/subscription.service'; export * from './lib/services/subscription-item-server.service'; export * from './lib/services/usage.service'; -export * from './lib/services/invoice-sync.scheduler'; -export * from './lib/services/subscription-billing.scheduler'; -export * from './lib/services/subscription-expiration.scheduler'; -export * from './lib/services/subscription-renewal-reminder.scheduler'; -export * from './lib/services/open-position-invoice.scheduler'; +export * from './lib/services/invoice-sync.job-handler'; +export * from './lib/services/subscription-billing.job-handler'; +export * from './lib/services/subscription-expiration.job-handler'; +export * from './lib/services/subscription-renewal-reminder.job-handler'; +export * from './lib/services/open-position-invoice.job-handler'; +export * from './lib/services/subscription-item-update.job-handler'; +export * from './lib/services/backorder-retry.job-handler'; export * from './lib/utils/billing-day.utils'; export * from './lib/utils/config-validation.utils'; export * from './lib/utils/hostname-generator.utils'; diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/billing.module.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/billing.module.ts index 91c31918..1c68d955 100644 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/billing.module.ts +++ b/libs/domains/framework/backend/feature-billing-manager/src/lib/billing.module.ts @@ -48,7 +48,7 @@ import { SubscriptionsRepository } from './repositories/subscriptions.repository import { UsageRecordsRepository } from './repositories/usage-records.repository'; import { UsersBillingDayRepository } from './repositories/users-billing-day.repository'; import { AvailabilityService } from './services/availability.service'; -import { BackorderRetryService } from './services/backorder-retry.service'; +import { BackorderRetryJobHandler } from './services/backorder-retry.job-handler'; import { BackorderService } from './services/backorder.service'; import { BillingScheduleService } from './services/billing-schedule.service'; import { CancellationPolicyService } from './services/cancellation-policy.service'; @@ -59,19 +59,19 @@ import { HetznerProvisioningService } from './services/hetzner-provisioning.serv import { HostnameReservationService } from './services/hostname-reservation.service'; import { InvoiceCreationService } from './services/invoice-creation.service'; import { InvoiceNinjaService } from './services/invoice-ninja.service'; -import { InvoiceSyncScheduler } from './services/invoice-sync.scheduler'; -import { OpenPositionInvoiceScheduler } from './services/open-position-invoice.scheduler'; +import { InvoiceSyncJobHandler } from './services/invoice-sync.job-handler'; +import { OpenPositionInvoiceJobHandler } from './services/open-position-invoice.job-handler'; import { PricingService } from './services/pricing.service'; import { ProviderPricingService } from './services/provider-pricing.service'; import { ProviderRegistryService } from './services/provider-registry.service'; import { ProviderServerTypesService } from './services/provider-server-types.service'; import { ProvisioningService } from './services/provisioning.service'; import { SshExecutorService } from './services/ssh-executor.service'; -import { SubscriptionBillingScheduler } from './services/subscription-billing.scheduler'; -import { SubscriptionExpirationScheduler } from './services/subscription-expiration.scheduler'; +import { SubscriptionBillingJobHandler } from './services/subscription-billing.job-handler'; +import { SubscriptionExpirationJobHandler } from './services/subscription-expiration.job-handler'; import { SubscriptionItemServerService } from './services/subscription-item-server.service'; -import { SubscriptionItemUpdateScheduler } from './services/subscription-item-update.scheduler'; -import { SubscriptionRenewalReminderScheduler } from './services/subscription-renewal-reminder.scheduler'; +import { SubscriptionItemUpdateJobHandler } from './services/subscription-item-update.job-handler'; +import { SubscriptionRenewalReminderJobHandler } from './services/subscription-renewal-reminder.job-handler'; import { SubscriptionService } from './services/subscription.service'; import { UsageService } from './services/usage.service'; @@ -281,7 +281,7 @@ const DIGITALOCEAN_CONFIG_SCHEMA: Record = { providers: [ AvailabilityService, BackorderService, - BackorderRetryService, + BackorderRetryJobHandler, BillingScheduleService, CancellationPolicyService, CloudflareDnsService, @@ -299,11 +299,12 @@ const DIGITALOCEAN_CONFIG_SCHEMA: Record = { SubscriptionService, UsageService, CustomerProfilesService, - InvoiceSyncScheduler, - SubscriptionBillingScheduler, - SubscriptionExpirationScheduler, - SubscriptionRenewalReminderScheduler, - OpenPositionInvoiceScheduler, + InvoiceSyncJobHandler, + SubscriptionBillingJobHandler, + SubscriptionExpirationJobHandler, + SubscriptionRenewalReminderJobHandler, + OpenPositionInvoiceJobHandler, + SubscriptionItemUpdateJobHandler, EmailService, AvailabilitySnapshotsRepository, BackordersRepository, @@ -318,7 +319,6 @@ const DIGITALOCEAN_CONFIG_SCHEMA: Record = { SubscriptionsRepository, UsageRecordsRepository, CustomerProfilesRepository, - SubscriptionItemUpdateScheduler, SshExecutorService, UsersRepository, SocketAuthService, @@ -327,7 +327,7 @@ const DIGITALOCEAN_CONFIG_SCHEMA: Record = { exports: [ AvailabilityService, BackorderService, - BackorderRetryService, + BackorderRetryJobHandler, BillingScheduleService, CancellationPolicyService, CloudflareDnsService, @@ -343,11 +343,12 @@ const DIGITALOCEAN_CONFIG_SCHEMA: Record = { SubscriptionService, UsageService, CustomerProfilesService, - InvoiceSyncScheduler, - SubscriptionBillingScheduler, - SubscriptionExpirationScheduler, - SubscriptionRenewalReminderScheduler, - OpenPositionInvoiceScheduler, + InvoiceSyncJobHandler, + SubscriptionBillingJobHandler, + SubscriptionExpirationJobHandler, + SubscriptionRenewalReminderJobHandler, + OpenPositionInvoiceJobHandler, + SubscriptionItemUpdateJobHandler, EmailService, AvailabilitySnapshotsRepository, BackordersRepository, diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/repositories/invoice-refs.repository.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/repositories/invoice-refs.repository.ts index 09c421c7..389daed2 100644 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/repositories/invoice-refs.repository.ts +++ b/libs/domains/framework/backend/feature-billing-manager/src/lib/repositories/invoice-refs.repository.ts @@ -37,6 +37,10 @@ export class InvoiceRefsRepository { }); } + async findById(id: string): Promise { + return await this.repository.findOne({ where: { id } }); + } + async findBatchForSync(batchSize: number, offset: number): Promise { return await this.repository.find({ order: { createdAt: 'ASC' }, diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/repositories/subscription-items.repository.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/repositories/subscription-items.repository.ts index 6692d8c3..349843ca 100644 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/repositories/subscription-items.repository.ts +++ b/libs/domains/framework/backend/feature-billing-manager/src/lib/repositories/subscription-items.repository.ts @@ -49,6 +49,13 @@ export class SubscriptionItemsRepository { }); } + async findByIdWithRelations(id: string): Promise { + return await this.repository.findOne({ + where: { id }, + relations: ['serviceType', 'subscription'], + }); + } + async findByIdAndSubscriptionId(id: string, subscriptionId: string): Promise { return await this.repository.findOne({ where: { id, subscriptionId }, diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/backorder-retry.job-handler.spec.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/backorder-retry.job-handler.spec.ts new file mode 100644 index 00000000..f5c65a93 --- /dev/null +++ b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/backorder-retry.job-handler.spec.ts @@ -0,0 +1,25 @@ +import { BackorderRetryJobHandler } from './backorder-retry.job-handler'; + +describe('BackorderRetryJobHandler', () => { + const backordersRepository = { + findAllPending: jest.fn(), + } as any; + const backorderService = { + retry: jest.fn(), + } as any; + const handler = new BackorderRetryJobHandler(backordersRepository, backorderService); + + it('findPendingBackorderIds maps repository rows', async () => { + backordersRepository.findAllPending.mockResolvedValue([{ id: 'bo-1' }]); + + await expect(handler.findPendingBackorderIds()).resolves.toEqual(['bo-1']); + }); + + it('retryBackorder delegates to service', async () => { + backorderService.retry.mockResolvedValue(undefined); + + await handler.retryBackorder('bo-1'); + + expect(backorderService.retry).toHaveBeenCalledWith('bo-1'); + }); +}); diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/backorder-retry.job-handler.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/backorder-retry.job-handler.ts new file mode 100644 index 00000000..5d5e9b05 --- /dev/null +++ b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/backorder-retry.job-handler.ts @@ -0,0 +1,27 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { BackordersRepository } from '../repositories/backorders.repository'; + +import { BackorderService } from './backorder.service'; + +@Injectable() +export class BackorderRetryJobHandler { + private readonly logger = new Logger(BackorderRetryJobHandler.name); + private readonly batchSize = parseInt(process.env.BACKORDER_RETRY_BATCH_SIZE ?? '100', 10); + + constructor( + private readonly backordersRepository: BackordersRepository, + private readonly backorderService: BackorderService, + ) {} + + async findPendingBackorderIds(): Promise { + const pending = await this.backordersRepository.findAllPending(this.batchSize, 0); + + return pending.map((backorder) => backorder.id); + } + + async retryBackorder(backorderId: string): Promise { + await this.backorderService.retry(backorderId); + this.logger.log(`Retried backorder ${backorderId}`); + } +} diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/backorder-retry.service.spec.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/backorder-retry.service.spec.ts deleted file mode 100644 index 7eeb52e4..00000000 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/backorder-retry.service.spec.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { BackorderRetryService } from './backorder-retry.service'; - -describe('BackorderRetryService', () => { - const backordersRepository = { - findAllPending: jest.fn(), - } as any; - const backorderService = { - retry: jest.fn(), - } as any; - let service: BackorderRetryService; - - beforeEach(() => { - jest.resetAllMocks(); - service = new BackorderRetryService(backordersRepository, backorderService); - }); - - it('processes pending backorders', async () => { - backordersRepository.findAllPending.mockResolvedValue([{ id: 'b1' }, { id: 'b2' }]); - backorderService.retry.mockResolvedValue({}); - - await service.processPendingBackorders(); - - expect(backorderService.retry).toHaveBeenCalledWith('b1'); - expect(backorderService.retry).toHaveBeenCalledWith('b2'); - }); - - it('handles empty pending backorders', async () => { - backordersRepository.findAllPending.mockResolvedValue([]); - - await service.processPendingBackorders(); - - expect(backorderService.retry).not.toHaveBeenCalled(); - }); - - it('continues processing on individual errors', async () => { - backordersRepository.findAllPending.mockResolvedValue([{ id: 'b1' }, { id: 'b2' }]); - backorderService.retry.mockRejectedValueOnce(new Error('Retry failed')).mockResolvedValueOnce({}); - - await service.processPendingBackorders(); - - expect(backorderService.retry).toHaveBeenCalledTimes(2); - }); -}); diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/backorder-retry.service.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/backorder-retry.service.ts deleted file mode 100644 index c682739e..00000000 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/backorder-retry.service.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; - -import { BackordersRepository } from '../repositories/backorders.repository'; - -import { BackorderService } from './backorder.service'; - -@Injectable() -export class BackorderRetryService implements OnModuleInit, OnModuleDestroy { - private readonly logger = new Logger(BackorderRetryService.name); - private intervalHandle?: NodeJS.Timeout; - - private readonly intervalMs = parseInt(process.env.BACKORDER_RETRY_INTERVAL_MS ?? '60000', 10); - private readonly batchSize = parseInt(process.env.BACKORDER_RETRY_BATCH_SIZE ?? '100', 10); - - constructor( - private readonly backordersRepository: BackordersRepository, - private readonly backorderService: BackorderService, - ) {} - - onModuleInit(): void { - this.logger.log(`Initializing billing scheduler with ${this.intervalMs}ms interval`); - this.intervalHandle = setInterval(() => { - void this.processPendingBackorders(); - }, this.intervalMs); - } - - onModuleDestroy(): void { - if (this.intervalHandle) { - clearInterval(this.intervalHandle); - } - } - - async processPendingBackorders(): Promise { - const pending = await this.backordersRepository.findAllPending(this.batchSize, 0); - - if (pending.length === 0) { - return; - } - - this.logger.log(`Processing ${pending.length} pending backorders`); - - for (const backorder of pending) { - try { - await this.backorderService.retry(backorder.id); - } catch (error) { - this.logger.error(`Failed to retry backorder ${backorder.id}: ${(error as Error).message}`); - } - } - } -} diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/invoice-sync.job-handler.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/invoice-sync.job-handler.ts new file mode 100644 index 00000000..3b852687 --- /dev/null +++ b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/invoice-sync.job-handler.ts @@ -0,0 +1,74 @@ +import { Injectable, Logger, NotFoundException } from '@nestjs/common'; + +import { InvoiceRefEntity } from '../entities/invoice-ref.entity'; +import { InvoiceRefsRepository } from '../repositories/invoice-refs.repository'; + +import { InvoiceNinjaService } from './invoice-ninja.service'; + +@Injectable() +export class InvoiceSyncJobHandler { + private readonly logger = new Logger(InvoiceSyncJobHandler.name); + private readonly batchSize = parseInt(process.env.INVOICE_SYNC_SCHEDULER_BATCH_SIZE ?? '100', 10); + + constructor( + private readonly invoiceRefsRepository: InvoiceRefsRepository, + private readonly invoiceNinjaService: InvoiceNinjaService, + ) {} + + async findInvoiceRefIdsPage(offset: number): Promise { + const refs = await this.invoiceRefsRepository.findBatchForSync(this.batchSize, offset); + + return refs.map((ref) => ref.id); + } + + get batchSizeLimit(): number { + return this.batchSize; + } + + async syncInvoiceRef(invoiceRefId: string): Promise { + const ref = await this.invoiceRefsRepository.findById(invoiceRefId); + + if (!ref) { + throw new NotFoundException(`Invoice ref ${invoiceRefId} not found`); + } + + await this.syncInvoiceRefEntity(ref); + } + + private async syncInvoiceRefEntity(ref: InvoiceRefEntity): Promise { + const details = await this.invoiceNinjaService.getInvoiceDetailsForSync(ref.invoiceNinjaId); + + if (!details) { + return; + } + + const updates: Partial> = {}; + + if (details.status !== undefined && details.status !== ref.status) { + updates.status = details.status; + } + + if (details.invoiceNumber !== undefined && details.invoiceNumber !== ref.invoiceNumber) { + updates.invoiceNumber = details.invoiceNumber; + } + + if (details.balance !== undefined && details.balance !== ref.balance) { + updates.balance = details.balance; + } + + if (details.dueDate !== undefined) { + const same = + ref.dueDate != null && details.dueDate != null && ref.dueDate.getTime() === details.dueDate.getTime(); + + if (!same) { + updates.dueDate = details.dueDate; + } + } + + if (Object.keys(updates).length === 0) { + return; + } + + await this.invoiceRefsRepository.update(ref.id, updates); + } +} diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/invoice-sync.scheduler.spec.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/invoice-sync.scheduler.spec.ts deleted file mode 100644 index 67ac8283..00000000 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/invoice-sync.scheduler.spec.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { InvoiceSyncScheduler } from './invoice-sync.scheduler'; - -describe('InvoiceSyncScheduler', () => { - const invoiceRefsRepository = { - findBatchForSync: jest.fn(), - update: jest.fn(), - } as any; - const invoiceNinjaService = { - getInvoiceDetailsForSync: jest.fn(), - } as any; - const scheduler = new InvoiceSyncScheduler(invoiceRefsRepository, invoiceNinjaService); - - beforeEach(() => { - jest.resetAllMocks(); - }); - - it('syncs invoice status and invoiceNumber when API returns different values', async () => { - const refs = [ - { - id: 'ref-1', - invoiceNinjaId: 'ninja-1', - status: '1', - preAuthUrl: 'https://old.example/inv/old', - invoiceNumber: 'INV-001', - }, - ]; - - invoiceRefsRepository.findBatchForSync.mockResolvedValue(refs); - invoiceNinjaService.getInvoiceDetailsForSync.mockResolvedValue({ - status: '2', - invoiceNumber: 'INV-002', - }); - - await scheduler.syncInvoicesFromInvoiceNinja(); - - expect(invoiceNinjaService.getInvoiceDetailsForSync).toHaveBeenCalledWith('ninja-1'); - expect(invoiceRefsRepository.update).toHaveBeenCalledWith('ref-1', { - status: '2', - invoiceNumber: 'INV-002', - }); - }); - - it('syncs balance when API returns different value', async () => { - const refs = [ - { - id: 'ref-bal', - invoiceNinjaId: 'ninja-bal', - status: '2', - invoiceNumber: 'INV-003', - balance: 50, - }, - ]; - - invoiceRefsRepository.findBatchForSync.mockResolvedValue(refs); - invoiceNinjaService.getInvoiceDetailsForSync.mockResolvedValue({ - status: '2', - invoiceNumber: 'INV-003', - balance: 75.5, - }); - - await scheduler.syncInvoicesFromInvoiceNinja(); - - expect(invoiceRefsRepository.update).toHaveBeenCalledWith('ref-bal', { - balance: 75.5, - }); - }); - - it('updates only when status or invoiceNumber changed', async () => { - const refs = [ - { - id: 'ref-2', - invoiceNinjaId: 'ninja-2', - status: '2', - preAuthUrl: 'https://same.example/link', - invoiceNumber: 'INV-002', - }, - ]; - - invoiceRefsRepository.findBatchForSync.mockResolvedValue(refs); - invoiceNinjaService.getInvoiceDetailsForSync.mockResolvedValue({ - status: '2', - invoiceNumber: 'INV-002', - }); - - await scheduler.syncInvoicesFromInvoiceNinja(); - - expect(invoiceRefsRepository.update).not.toHaveBeenCalled(); - }); - - it('handles empty batch', async () => { - invoiceRefsRepository.findBatchForSync.mockResolvedValue([]); - - await scheduler.syncInvoicesFromInvoiceNinja(); - - expect(invoiceNinjaService.getInvoiceDetailsForSync).not.toHaveBeenCalled(); - expect(invoiceRefsRepository.update).not.toHaveBeenCalled(); - }); - - it('continues processing when getInvoiceDetailsForSync returns null', async () => { - const refs = [ - { - id: 'ref-3', - invoiceNinjaId: 'ninja-missing', - status: '1', - preAuthUrl: 'https://example.com', - invoiceNumber: 'INV-003', - }, - ]; - - invoiceRefsRepository.findBatchForSync.mockResolvedValue(refs); - invoiceNinjaService.getInvoiceDetailsForSync.mockResolvedValue(null); - - await scheduler.syncInvoicesFromInvoiceNinja(); - - expect(invoiceNinjaService.getInvoiceDetailsForSync).toHaveBeenCalledWith('ninja-missing'); - expect(invoiceRefsRepository.update).not.toHaveBeenCalled(); - }); - - it('continues processing on individual ref errors', async () => { - const refs = [ - { - id: 'ref-4', - invoiceNinjaId: 'ninja-4', - status: '1', - preAuthUrl: 'https://example.com', - invoiceNumber: 'INV-004', - }, - { - id: 'ref-5', - invoiceNinjaId: 'ninja-5', - status: '1', - preAuthUrl: 'https://example.com', - invoiceNumber: 'INV-005', - }, - ]; - - invoiceRefsRepository.findBatchForSync.mockResolvedValue(refs); - invoiceNinjaService.getInvoiceDetailsForSync - .mockRejectedValueOnce(new Error('Network error')) - .mockResolvedValueOnce({ status: '3', invoiceNumber: 'INV-005' }); - - await scheduler.syncInvoicesFromInvoiceNinja(); - - expect(invoiceRefsRepository.update).toHaveBeenCalledTimes(1); - expect(invoiceRefsRepository.update).toHaveBeenCalledWith('ref-5', { - status: '3', - }); - }); - - it('processes multiple batches when first batch is full', async () => { - const fullBatch = Array.from({ length: 100 }, (_, i) => ({ - id: `ref-b-${i}`, - invoiceNinjaId: `ninja-b-${i}`, - status: '1', - preAuthUrl: 'https://example.com', - invoiceNumber: `INV-B-${i}`, - })); - const secondBatch = [ - { - id: 'ref-b2', - invoiceNinjaId: 'ninja-b2', - status: '1', - preAuthUrl: 'https://example.com', - invoiceNumber: 'INV-B2', - }, - ]; - - invoiceRefsRepository.findBatchForSync - .mockResolvedValueOnce(fullBatch) - .mockResolvedValueOnce(secondBatch) - .mockResolvedValueOnce([]); - invoiceNinjaService.getInvoiceDetailsForSync.mockResolvedValue({}); - - await scheduler.syncInvoicesFromInvoiceNinja(); - - expect(invoiceRefsRepository.findBatchForSync).toHaveBeenCalledWith(100, 0); - expect(invoiceRefsRepository.findBatchForSync).toHaveBeenCalledWith(100, 100); - expect(invoiceNinjaService.getInvoiceDetailsForSync).toHaveBeenCalledTimes(101); - }); -}); diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/invoice-sync.scheduler.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/invoice-sync.scheduler.ts deleted file mode 100644 index a901b0fc..00000000 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/invoice-sync.scheduler.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; - -import { InvoiceRefEntity } from '../entities/invoice-ref.entity'; -import { InvoiceRefsRepository } from '../repositories/invoice-refs.repository'; - -import { InvoiceNinjaService } from './invoice-ninja.service'; - -@Injectable() -export class InvoiceSyncScheduler implements OnModuleInit, OnModuleDestroy { - private readonly logger = new Logger(InvoiceSyncScheduler.name); - private intervalHandle?: NodeJS.Timeout; - - private readonly intervalMs = parseInt(process.env.INVOICE_SYNC_SCHEDULER_INTERVAL ?? '60000', 10); - private readonly batchSize = parseInt(process.env.INVOICE_SYNC_SCHEDULER_BATCH_SIZE ?? '100', 10); - - constructor( - private readonly invoiceRefsRepository: InvoiceRefsRepository, - private readonly invoiceNinjaService: InvoiceNinjaService, - ) {} - - onModuleInit(): void { - this.logger.log(`Initializing invoice sync scheduler with ${this.intervalMs}ms interval`); - this.intervalHandle = setInterval(() => { - void this.syncInvoicesFromInvoiceNinja(); - }, this.intervalMs); - } - - onModuleDestroy(): void { - if (this.intervalHandle) { - clearInterval(this.intervalHandle); - } - } - - async syncInvoicesFromInvoiceNinja(): Promise { - let offset = 0; - let totalProcessed = 0; - - // eslint-disable-next-line no-constant-condition - while (true) { - const refs = await this.invoiceRefsRepository.findBatchForSync(this.batchSize, offset); - - if (refs.length === 0) { - break; - } - - for (const ref of refs) { - try { - await this.syncOneInvoiceRef(ref); - totalProcessed += 1; - } catch (error) { - this.logger.error( - `Failed to sync invoice ref ${ref.id} (invoiceNinjaId=${ref.invoiceNinjaId}): ${(error as Error).message}`, - ); - } - } - - offset += refs.length; - - if (refs.length < this.batchSize) { - break; - } - } - - if (totalProcessed > 0) { - this.logger.log(`Invoice sync completed: ${totalProcessed} refs processed`); - } - } - - private async syncOneInvoiceRef(ref: InvoiceRefEntity): Promise { - const details = await this.invoiceNinjaService.getInvoiceDetailsForSync(ref.invoiceNinjaId); - - if (!details) { - return; - } - - const updates: Partial> = {}; - - if (details.status !== undefined && details.status !== ref.status) { - updates.status = details.status; - } - - if (details.invoiceNumber !== undefined && details.invoiceNumber !== ref.invoiceNumber) { - updates.invoiceNumber = details.invoiceNumber; - } - - if (details.balance !== undefined && details.balance !== ref.balance) { - updates.balance = details.balance; - } - - if (details.dueDate !== undefined) { - const same = - ref.dueDate != null && details.dueDate != null && ref.dueDate.getTime() === details.dueDate.getTime(); - - if (!same) { - updates.dueDate = details.dueDate; - } - } - - if (Object.keys(updates).length === 0) { - return; - } - - await this.invoiceRefsRepository.update(ref.id, updates); - } -} diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/open-position-invoice.job-handler.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/open-position-invoice.job-handler.ts new file mode 100644 index 00000000..c750827c --- /dev/null +++ b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/open-position-invoice.job-handler.ts @@ -0,0 +1,45 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { OpenPositionsRepository } from '../repositories/open-positions.repository'; +import { UsersBillingDayRepository } from '../repositories/users-billing-day.repository'; +import { getTodayBillingDay } from '../utils/billing-day.utils'; + +import { InvoiceCreationService } from './invoice-creation.service'; + +@Injectable() +export class OpenPositionInvoiceJobHandler { + private readonly logger = new Logger(OpenPositionInvoiceJobHandler.name); + + constructor( + private readonly usersBillingDayRepository: UsersBillingDayRepository, + private readonly openPositionsRepository: OpenPositionsRepository, + private readonly invoiceCreationService: InvoiceCreationService, + ) {} + + async findUserIdsForTodayBillingDay(): Promise { + const todayDay = getTodayBillingDay(); + + return this.usersBillingDayRepository.findUserIdsWithBillingDay(todayDay); + } + + async processUserOpenPositions(userId: string): Promise { + const positions = await this.openPositionsRepository.findUnbilledByUserId(userId); + + if (positions.length === 0) { + return; + } + + try { + const result = await this.invoiceCreationService.createAccumulatedInvoice(userId, positions); + + if (result?.invoiceRefId) { + this.logger.log( + `Created accumulated invoice for user ${userId}, ${positions.length} position(s), ref ${result.invoiceRefId}`, + ); + } + } catch (error) { + this.logger.error(`Failed to create accumulated invoice for user ${userId}: ${(error as Error).message}`); + throw error; + } + } +} diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/open-position-invoice.scheduler.spec.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/open-position-invoice.scheduler.spec.ts deleted file mode 100644 index 034fbce3..00000000 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/open-position-invoice.scheduler.spec.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { getTodayBillingDay } from '../utils/billing-day.utils'; - -import { OpenPositionInvoiceScheduler } from './open-position-invoice.scheduler'; - -jest.mock('../utils/billing-day.utils', () => ({ - getTodayBillingDay: jest.fn(), -})); - -const getTodayBillingDayMock = getTodayBillingDay as jest.MockedFunction; - -describe('OpenPositionInvoiceScheduler', () => { - let usersBillingDayRepository: { findUserIdsWithBillingDay: jest.Mock }; - let openPositionsRepository: { findUnbilledByUserId: jest.Mock; markBilled: jest.Mock }; - let invoiceCreationService: { createAccumulatedInvoice: jest.Mock }; - - beforeEach(() => { - jest.resetAllMocks(); - getTodayBillingDayMock.mockReturnValue(10); - usersBillingDayRepository = { findUserIdsWithBillingDay: jest.fn() }; - openPositionsRepository = { findUnbilledByUserId: jest.fn(), markBilled: jest.fn() }; - invoiceCreationService = { createAccumulatedInvoice: jest.fn() }; - }); - - function createScheduler() { - return new OpenPositionInvoiceScheduler( - usersBillingDayRepository as never, - openPositionsRepository as never, - invoiceCreationService as never, - ); - } - - it('does nothing when no users have billing day today', async () => { - usersBillingDayRepository.findUserIdsWithBillingDay.mockResolvedValue([]); - const scheduler = createScheduler(); - - await scheduler.processBillingDayInvoices(); - - expect(openPositionsRepository.findUnbilledByUserId).not.toHaveBeenCalled(); - expect(invoiceCreationService.createAccumulatedInvoice).not.toHaveBeenCalled(); - }); - - it('creates one accumulated invoice per user with all unbilled positions', async () => { - const positions = [ - { - id: 'pos-1', - subscriptionId: 'sub-1', - userId: 'user-1', - description: 'Subscription 123', - billUntil: new Date('2024-02-01'), - skipIfNoBillableAmount: true, - }, - ]; - - usersBillingDayRepository.findUserIdsWithBillingDay.mockResolvedValue(['user-1']); - openPositionsRepository.findUnbilledByUserId.mockResolvedValue(positions); - invoiceCreationService.createAccumulatedInvoice.mockResolvedValue({ - invoiceRefId: 'ref-1', - }); - - const scheduler = createScheduler(); - - await scheduler.processBillingDayInvoices(); - - expect(usersBillingDayRepository.findUserIdsWithBillingDay).toHaveBeenCalledWith(10); - expect(openPositionsRepository.findUnbilledByUserId).toHaveBeenCalledWith('user-1'); - expect(invoiceCreationService.createAccumulatedInvoice).toHaveBeenCalledTimes(1); - expect(invoiceCreationService.createAccumulatedInvoice).toHaveBeenCalledWith('user-1', positions); - expect(openPositionsRepository.markBilled).not.toHaveBeenCalled(); - }); - - it('does nothing when createAccumulatedInvoice returns undefined (no billable amount)', async () => { - usersBillingDayRepository.findUserIdsWithBillingDay.mockResolvedValue(['user-1']); - openPositionsRepository.findUnbilledByUserId.mockResolvedValue([ - { - id: 'pos-1', - subscriptionId: 'sub-1', - userId: 'user-1', - description: 'Sub', - billUntil: new Date(), - skipIfNoBillableAmount: true, - }, - ]); - invoiceCreationService.createAccumulatedInvoice.mockResolvedValue(undefined); - - const scheduler = createScheduler(); - - await scheduler.processBillingDayInvoices(); - - expect(invoiceCreationService.createAccumulatedInvoice).toHaveBeenCalledWith('user-1', expect.any(Array)); - expect(openPositionsRepository.markBilled).not.toHaveBeenCalled(); - }); - - it('continues with next user when createAccumulatedInvoice throws', async () => { - usersBillingDayRepository.findUserIdsWithBillingDay.mockResolvedValue(['user-1', 'user-2']); - openPositionsRepository.findUnbilledByUserId - .mockResolvedValueOnce([ - { - id: 'pos-1', - subscriptionId: 'sub-1', - userId: 'user-1', - description: 'Sub 1', - billUntil: new Date(), - skipIfNoBillableAmount: true, - }, - ]) - .mockResolvedValueOnce([ - { - id: 'pos-2', - subscriptionId: 'sub-2', - userId: 'user-2', - description: 'Sub 2', - billUntil: new Date(), - skipIfNoBillableAmount: true, - }, - ]); - invoiceCreationService.createAccumulatedInvoice - .mockRejectedValueOnce(new Error('Invoice Ninja error')) - .mockResolvedValueOnce({ invoiceRefId: 'ref-2' }); - - const scheduler = createScheduler(); - - await scheduler.processBillingDayInvoices(); - - expect(invoiceCreationService.createAccumulatedInvoice).toHaveBeenCalledTimes(2); - expect(invoiceCreationService.createAccumulatedInvoice).toHaveBeenNthCalledWith(1, 'user-1', expect.any(Array)); - expect(invoiceCreationService.createAccumulatedInvoice).toHaveBeenNthCalledWith(2, 'user-2', expect.any(Array)); - }); - - it('skips user when findUnbilledByUserId returns empty', async () => { - usersBillingDayRepository.findUserIdsWithBillingDay.mockResolvedValue(['user-1']); - openPositionsRepository.findUnbilledByUserId.mockResolvedValue([]); - - const scheduler = createScheduler(); - - await scheduler.processBillingDayInvoices(); - - expect(invoiceCreationService.createAccumulatedInvoice).not.toHaveBeenCalled(); - }); -}); diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/open-position-invoice.scheduler.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/open-position-invoice.scheduler.ts deleted file mode 100644 index df7f255e..00000000 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/open-position-invoice.scheduler.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; - -import { OpenPositionsRepository } from '../repositories/open-positions.repository'; -import { UsersBillingDayRepository } from '../repositories/users-billing-day.repository'; -import { getTodayBillingDay } from '../utils/billing-day.utils'; - -import { InvoiceCreationService } from './invoice-creation.service'; - -@Injectable() -export class OpenPositionInvoiceScheduler implements OnModuleInit, OnModuleDestroy { - private readonly logger = new Logger(OpenPositionInvoiceScheduler.name); - private intervalHandle?: NodeJS.Timeout; - - private readonly intervalMs = parseInt(process.env.OPEN_POSITION_INVOICE_SCHEDULER_INTERVAL ?? '86400000', 10); - - constructor( - private readonly usersBillingDayRepository: UsersBillingDayRepository, - private readonly openPositionsRepository: OpenPositionsRepository, - private readonly invoiceCreationService: InvoiceCreationService, - ) {} - - onModuleInit(): void { - this.logger.log(`Initializing open-position invoice scheduler with ${this.intervalMs}ms interval`); - this.intervalHandle = setInterval(() => { - void this.processBillingDayInvoices(); - }, this.intervalMs); - } - - onModuleDestroy(): void { - if (this.intervalHandle) { - clearInterval(this.intervalHandle); - } - } - - async processBillingDayInvoices(): Promise { - const todayDay = getTodayBillingDay(); - const userIds = await this.usersBillingDayRepository.findUserIdsWithBillingDay(todayDay); - - if (userIds.length === 0) { - return; - } - - this.logger.log(`Processing billing day ${todayDay} for ${userIds.length} user(s)`); - - for (const userId of userIds) { - try { - await this.processUserOpenPositions(userId); - } catch (error) { - this.logger.error(`Failed to process open positions for user ${userId}: ${(error as Error).message}`); - } - } - } - - private async processUserOpenPositions(userId: string): Promise { - const positions = await this.openPositionsRepository.findUnbilledByUserId(userId); - - if (positions.length === 0) { - return; - } - - try { - const result = await this.invoiceCreationService.createAccumulatedInvoice(userId, positions); - - if (result?.invoiceRefId) { - this.logger.log( - `Created accumulated invoice for user ${userId}, ${positions.length} position(s), ref ${result.invoiceRefId}`, - ); - } - } catch (error) { - this.logger.error(`Failed to create accumulated invoice for user ${userId}: ${(error as Error).message}`); - } - } -} diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-billing.job-handler.spec.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-billing.job-handler.spec.ts new file mode 100644 index 00000000..5d5141cb --- /dev/null +++ b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-billing.job-handler.spec.ts @@ -0,0 +1,58 @@ +import { BillingIntervalType } from '../entities/service-plan.entity'; + +import { BillingScheduleService } from './billing-schedule.service'; +import { SubscriptionBillingJobHandler } from './subscription-billing.job-handler'; + +describe('SubscriptionBillingJobHandler', () => { + const subscriptionsRepository = { + findDueForBilling: jest.fn(), + findByIdOrThrow: jest.fn(), + update: jest.fn(), + } as any; + const servicePlansRepository = { + findByIdOrThrow: jest.fn(), + } as any; + const billingScheduleService = new BillingScheduleService(); + const openPositionsRepository = { + create: jest.fn(), + } as any; + const handler = new SubscriptionBillingJobHandler( + subscriptionsRepository, + servicePlansRepository, + billingScheduleService, + openPositionsRepository, + ); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('findDueSubscriptionIds returns ids from repository', async () => { + subscriptionsRepository.findDueForBilling.mockResolvedValue([{ id: 'sub-1' }]); + + await expect(handler.findDueSubscriptionIds()).resolves.toEqual(['sub-1']); + }); + + it('processSubscription creates open position and updates schedule', async () => { + subscriptionsRepository.findByIdOrThrow.mockResolvedValue({ + id: 'sub-1', + userId: 'user-1', + planId: 'plan-1', + number: '123456', + nextBillingAt: new Date('2026-06-01'), + }); + servicePlansRepository.findByIdOrThrow.mockResolvedValue({ + id: 'plan-1', + billingIntervalType: BillingIntervalType.MONTH, + billingIntervalValue: 1, + billingDayOfMonth: undefined, + }); + openPositionsRepository.create.mockResolvedValue({}); + subscriptionsRepository.update.mockResolvedValue({}); + + await handler.processSubscription('sub-1'); + + expect(openPositionsRepository.create).toHaveBeenCalled(); + expect(subscriptionsRepository.update).toHaveBeenCalled(); + }); +}); diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-billing.scheduler.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-billing.job-handler.ts similarity index 55% rename from libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-billing.scheduler.ts rename to libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-billing.job-handler.ts index c576eb1a..373b0e10 100644 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-billing.scheduler.ts +++ b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-billing.job-handler.ts @@ -1,63 +1,33 @@ -import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { BillingIntervalType } from '../entities/service-plan.entity'; -import { SubscriptionEntity } from '../entities/subscription.entity'; import { OpenPositionsRepository } from '../repositories/open-positions.repository'; import { ServicePlansRepository } from '../repositories/service-plans.repository'; -import { ServiceTypesRepository } from '../repositories/service-types.repository'; import { SubscriptionsRepository } from '../repositories/subscriptions.repository'; import { BillingScheduleService } from './billing-schedule.service'; @Injectable() -export class SubscriptionBillingScheduler implements OnModuleInit, OnModuleDestroy { - private readonly logger = new Logger(SubscriptionBillingScheduler.name); - private intervalHandle?: NodeJS.Timeout; - - private readonly intervalMs = parseInt(process.env.BILLING_SCHEDULER_INTERVAL ?? '60000', 10); +export class SubscriptionBillingJobHandler { + private readonly logger = new Logger(SubscriptionBillingJobHandler.name); private readonly batchSize = parseInt(process.env.BILLING_SCHEDULER_BATCH_SIZE ?? '100', 10); constructor( private readonly subscriptionsRepository: SubscriptionsRepository, private readonly servicePlansRepository: ServicePlansRepository, - private readonly serviceTypesRepository: ServiceTypesRepository, private readonly billingScheduleService: BillingScheduleService, private readonly openPositionsRepository: OpenPositionsRepository, ) {} - onModuleInit(): void { - this.logger.log(`Initializing billing scheduler with ${this.intervalMs}ms interval`); - this.intervalHandle = setInterval(() => { - void this.processDueSubscriptions(); - }, this.intervalMs); - } - - onModuleDestroy(): void { - if (this.intervalHandle) { - clearInterval(this.intervalHandle); - } - } - - async processDueSubscriptions(): Promise { + async findDueSubscriptionIds(): Promise { const now = new Date(); const dueSubscriptions = await this.subscriptionsRepository.findDueForBilling(now, this.batchSize); - if (dueSubscriptions.length === 0) { - return; - } - - this.logger.log(`Processing ${dueSubscriptions.length} subscriptions due for billing`); - - for (const subscription of dueSubscriptions) { - try { - await this.processSubscriptionBilling(subscription); - } catch (error) { - this.logger.error(`Failed to bill subscription ${subscription.id}: ${(error as Error).message}`); - } - } + return dueSubscriptions.map((subscription) => subscription.id); } - private async processSubscriptionBilling(subscription: SubscriptionEntity): Promise { + async processSubscription(subscriptionId: string): Promise { + const subscription = await this.subscriptionsRepository.findByIdOrThrow(subscriptionId); const plan = await this.servicePlansRepository.findByIdOrThrow(subscription.planId); await this.openPositionsRepository.create({ diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-billing.scheduler.spec.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-billing.scheduler.spec.ts deleted file mode 100644 index 15fbcd46..00000000 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-billing.scheduler.spec.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { BillingIntervalType } from '../entities/service-plan.entity'; - -import { BillingScheduleService } from './billing-schedule.service'; -import { SubscriptionBillingScheduler } from './subscription-billing.scheduler'; - -describe('SubscriptionBillingScheduler', () => { - const subscriptionsRepository = { - findDueForBilling: jest.fn(), - update: jest.fn(), - } as any; - const servicePlansRepository = { - findByIdOrThrow: jest.fn(), - } as any; - const serviceTypesRepository = { - findByIdOrThrow: jest.fn(), - } as any; - const billingScheduleService = new BillingScheduleService(); - const openPositionsRepository = { - create: jest.fn(), - } as any; - const scheduler = new SubscriptionBillingScheduler( - subscriptionsRepository, - servicePlansRepository, - serviceTypesRepository, - billingScheduleService, - openPositionsRepository, - ); - - beforeEach(() => { - jest.resetAllMocks(); - }); - - it('processes due subscriptions by creating open position', async () => { - subscriptionsRepository.findDueForBilling.mockResolvedValue([ - { - id: 'sub-1', - userId: 'user-1', - planId: 'plan-1', - status: 'active', - number: '123456', - }, - ]); - servicePlansRepository.findByIdOrThrow.mockResolvedValue({ - id: 'plan-1', - name: 'Basic Plan', - billingIntervalType: BillingIntervalType.MONTH, - billingIntervalValue: 1, - billingDayOfMonth: undefined, - }); - openPositionsRepository.create.mockResolvedValue({}); - - await scheduler.processDueSubscriptions(); - - expect(openPositionsRepository.create).toHaveBeenCalledWith( - expect.objectContaining({ - subscriptionId: 'sub-1', - userId: 'user-1', - description: 'Subscription 123456', - skipIfNoBillableAmount: true, - }), - ); - expect(openPositionsRepository.create.mock.calls[0][0].billUntil).toBeInstanceOf(Date); - expect(subscriptionsRepository.update).toHaveBeenCalled(); - }); - - it('handles empty due subscriptions', async () => { - subscriptionsRepository.findDueForBilling.mockResolvedValue([]); - - await scheduler.processDueSubscriptions(); - - expect(openPositionsRepository.create).not.toHaveBeenCalled(); - }); - - it('continues processing on individual errors', async () => { - subscriptionsRepository.findDueForBilling.mockResolvedValue([ - { id: 'sub-1', userId: 'user-1', planId: 'plan-1', number: '123456' }, - { id: 'sub-2', userId: 'user-2', planId: 'plan-2', number: '654321' }, - ]); - servicePlansRepository.findByIdOrThrow.mockRejectedValueOnce(new Error('Plan not found')).mockResolvedValueOnce({ - id: 'plan-2', - name: 'Plan 2', - billingIntervalType: BillingIntervalType.MONTH, - billingIntervalValue: 1, - }); - openPositionsRepository.create.mockResolvedValue({}); - - await scheduler.processDueSubscriptions(); - - expect(openPositionsRepository.create).toHaveBeenCalledTimes(1); - expect(openPositionsRepository.create).toHaveBeenCalledWith( - expect.objectContaining({ - subscriptionId: 'sub-2', - userId: 'user-2', - description: 'Subscription 654321', - skipIfNoBillableAmount: true, - }), - ); - }); -}); diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-expiration.scheduler.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-expiration.job-handler.ts similarity index 69% rename from libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-expiration.scheduler.ts rename to libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-expiration.job-handler.ts index b507fbc3..b182d3ac 100644 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-expiration.scheduler.ts +++ b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-expiration.job-handler.ts @@ -1,6 +1,6 @@ -import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; -import { SubscriptionEntity, SubscriptionStatus } from '../entities/subscription.entity'; +import { SubscriptionStatus } from '../entities/subscription.entity'; import { OpenPositionsRepository } from '../repositories/open-positions.repository'; import { SubscriptionItemsRepository } from '../repositories/subscription-items.repository'; import { SubscriptionsRepository } from '../repositories/subscriptions.repository'; @@ -10,11 +10,8 @@ import { HostnameReservationService } from './hostname-reservation.service'; import { ProvisioningService } from './provisioning.service'; @Injectable() -export class SubscriptionExpirationScheduler implements OnModuleInit, OnModuleDestroy { - private readonly logger = new Logger(SubscriptionExpirationScheduler.name); - private intervalHandle?: NodeJS.Timeout; - - private readonly intervalMs = parseInt(process.env.EXPIRATION_SCHEDULER_INTERVAL ?? '60000', 10); +export class SubscriptionExpirationJobHandler { + private readonly logger = new Logger(SubscriptionExpirationJobHandler.name); private readonly batchSize = parseInt(process.env.EXPIRATION_SCHEDULER_BATCH_SIZE ?? '100', 10); constructor( @@ -26,39 +23,15 @@ export class SubscriptionExpirationScheduler implements OnModuleInit, OnModuleDe private readonly cloudflareDnsService: CloudflareDnsService, ) {} - onModuleInit(): void { - this.logger.log(`Initializing expiration scheduler with ${this.intervalMs}ms interval`); - this.intervalHandle = setInterval(() => { - void this.processExpiredSubscriptions(); - }, this.intervalMs); - } - - onModuleDestroy(): void { - if (this.intervalHandle) { - clearInterval(this.intervalHandle); - } - } - - async processExpiredSubscriptions(): Promise { + async findExpiredSubscriptionIds(): Promise { const now = new Date(); const expiredSubscriptions = await this.subscriptionsRepository.findDueForCancellation(now, this.batchSize); - if (expiredSubscriptions.length === 0) { - return; - } - - this.logger.log(`Processing ${expiredSubscriptions.length} subscriptions due for cancellation`); - - for (const subscription of expiredSubscriptions) { - try { - await this.processSubscriptionCancellation(subscription); - } catch (error) { - this.logger.error(`Failed to cancel subscription ${subscription.id}: ${(error as Error).message}`); - } - } + return expiredSubscriptions.map((subscription) => subscription.id); } - private async processSubscriptionCancellation(subscription: SubscriptionEntity): Promise { + async processSubscriptionCancellation(subscriptionId: string): Promise { + const subscription = await this.subscriptionsRepository.findByIdOrThrow(subscriptionId); const items = await this.subscriptionItemsRepository.findBySubscription(subscription.id); this.logger.log(`Found ${items.length} items for subscription ${subscription.id}`); diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-expiration.scheduler.spec.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-expiration.scheduler.spec.ts deleted file mode 100644 index 92e54298..00000000 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-expiration.scheduler.spec.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { SubscriptionStatus } from '../entities/subscription.entity'; - -import { SubscriptionExpirationScheduler } from './subscription-expiration.scheduler'; - -describe('SubscriptionExpirationScheduler', () => { - const subscriptionsRepository = { - findDueForCancellation: jest.fn(), - update: jest.fn(), - } as any; - const subscriptionItemsRepository = { - findBySubscription: jest.fn(), - } as any; - const provisioningService = { - deprovision: jest.fn(), - } as any; - const openPositionsRepository = { - create: jest.fn(), - } as any; - const hostnameReservationService = { - releaseHostname: jest.fn().mockResolvedValue(undefined), - } as any; - const cloudflareDnsService = { - deleteRecord: jest.fn().mockResolvedValue(undefined), - } as any; - const scheduler = new SubscriptionExpirationScheduler( - subscriptionsRepository, - subscriptionItemsRepository, - provisioningService, - openPositionsRepository, - hostnameReservationService, - cloudflareDnsService, - ); - - beforeEach(() => { - jest.resetAllMocks(); - }); - - it('processes expired subscriptions by creating open position', async () => { - subscriptionsRepository.findDueForCancellation.mockResolvedValue([ - { id: 'sub-1', userId: 'user-1', status: SubscriptionStatus.PENDING_CANCEL, number: '123456' }, - ]); - subscriptionItemsRepository.findBySubscription.mockResolvedValue([ - { id: 'item-1', providerReference: 'server-123', serviceType: { provider: 'hetzner' } }, - ]); - provisioningService.deprovision.mockResolvedValue(undefined); - openPositionsRepository.create.mockResolvedValue(undefined); - - await scheduler.processExpiredSubscriptions(); - - expect(provisioningService.deprovision).toHaveBeenCalledWith('hetzner', 'server-123'); - expect(openPositionsRepository.create).toHaveBeenCalledWith( - expect.objectContaining({ - subscriptionId: 'sub-1', - userId: 'user-1', - description: expect.stringContaining('Subscription'), - skipIfNoBillableAmount: true, - }), - ); - expect(openPositionsRepository.create.mock.calls[0][0].billUntil).toBeInstanceOf(Date); - expect(subscriptionsRepository.update).toHaveBeenCalledWith('sub-1', { - status: SubscriptionStatus.CANCELED, - }); - }); - - it('handles empty expired subscriptions', async () => { - subscriptionsRepository.findDueForCancellation.mockResolvedValue([]); - - await scheduler.processExpiredSubscriptions(); - - expect(openPositionsRepository.create).not.toHaveBeenCalled(); - expect(subscriptionsRepository.update).not.toHaveBeenCalled(); - }); - - it('continues processing when deprovision fails', async () => { - subscriptionsRepository.findDueForCancellation.mockResolvedValue([ - { id: 'sub-1', status: SubscriptionStatus.PENDING_CANCEL }, - ]); - subscriptionItemsRepository.findBySubscription.mockResolvedValue([ - { id: 'item-1', providerReference: 'server-123', serviceType: { provider: 'hetzner' } }, - ]); - provisioningService.deprovision.mockRejectedValue(new Error('Deprovision failed')); - - await scheduler.processExpiredSubscriptions(); - - expect(subscriptionsRepository.update).toHaveBeenCalledWith('sub-1', { - status: SubscriptionStatus.CANCELED, - }); - }); - - it('skips items without provider reference', async () => { - subscriptionsRepository.findDueForCancellation.mockResolvedValue([ - { id: 'sub-1', status: SubscriptionStatus.PENDING_CANCEL }, - ]); - subscriptionItemsRepository.findBySubscription.mockResolvedValue([ - { id: 'item-1', providerReference: null, serviceType: { provider: 'hetzner' } }, - ]); - - await scheduler.processExpiredSubscriptions(); - - expect(provisioningService.deprovision).not.toHaveBeenCalled(); - expect(subscriptionsRepository.update).toHaveBeenCalledWith('sub-1', { - status: SubscriptionStatus.CANCELED, - }); - }); - - it('skips items without service type', async () => { - subscriptionsRepository.findDueForCancellation.mockResolvedValue([ - { id: 'sub-1', status: SubscriptionStatus.PENDING_CANCEL }, - ]); - subscriptionItemsRepository.findBySubscription.mockResolvedValue([ - { id: 'item-1', providerReference: 'server-123', serviceType: null }, - ]); - - await scheduler.processExpiredSubscriptions(); - - expect(provisioningService.deprovision).not.toHaveBeenCalled(); - expect(subscriptionsRepository.update).toHaveBeenCalledWith('sub-1', { - status: SubscriptionStatus.CANCELED, - }); - }); - - it('removes DNS record and releases hostname when item has hostname', async () => { - subscriptionsRepository.findDueForCancellation.mockResolvedValue([ - { id: 'sub-1', userId: 'user-1', status: SubscriptionStatus.PENDING_CANCEL, number: '123456' }, - ]); - subscriptionItemsRepository.findBySubscription.mockResolvedValue([ - { - id: 'item-1', - hostname: 'awesome-armadillo-abc12', - providerReference: 'server-123', - serviceType: { provider: 'hetzner' }, - }, - ]); - provisioningService.deprovision.mockResolvedValue(undefined); - openPositionsRepository.create.mockResolvedValue(undefined); - - await scheduler.processExpiredSubscriptions(); - - expect(cloudflareDnsService.deleteRecord).toHaveBeenCalledWith('awesome-armadillo-abc12'); - expect(hostnameReservationService.releaseHostname).toHaveBeenCalledWith('item-1'); - expect(provisioningService.deprovision).toHaveBeenCalledWith('hetzner', 'server-123'); - }); -}); diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-item-update.scheduler.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-item-update.job-handler.ts similarity index 53% rename from libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-item-update.scheduler.ts rename to libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-item-update.job-handler.ts index 1fc5ffca..5ad47bbe 100644 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-item-update.scheduler.ts +++ b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-item-update.job-handler.ts @@ -1,4 +1,4 @@ -import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { SubscriptionItemsRepository } from '../repositories/subscription-items.repository'; import { buildAgentControllerUpdateCommand } from '../utils/cloud-init/agent-controller.utils'; @@ -7,19 +7,12 @@ import { buildAgentManagerUpdateCommand } from '../utils/cloud-init/agent-manage import { ProvisioningService } from './provisioning.service'; import { SshExecutorService } from './ssh-executor.service'; -const DEFAULT_INTERVAL_MS = 86400000; // 24 hours const SSH_USER = 'root'; const SSH_PORT = 22; @Injectable() -export class SubscriptionItemUpdateScheduler implements OnModuleInit, OnModuleDestroy { - private readonly logger = new Logger(SubscriptionItemUpdateScheduler.name); - private intervalHandle?: NodeJS.Timeout; - - private readonly intervalMs = parseInt( - process.env.SUBSCRIPTION_UPDATE_SCHEDULER_INTERVAL ?? String(DEFAULT_INTERVAL_MS), - 10, - ); +export class SubscriptionItemUpdateJobHandler { + private readonly logger = new Logger(SubscriptionItemUpdateJobHandler.name); constructor( private readonly subscriptionItemsRepository: SubscriptionItemsRepository, @@ -27,47 +20,19 @@ export class SubscriptionItemUpdateScheduler implements OnModuleInit, OnModuleDe private readonly sshExecutor: SshExecutorService, ) {} - onModuleInit(): void { - this.logger.log(`Initializing subscription item update scheduler with ${this.intervalMs}ms interval`); - this.intervalHandle = setInterval(() => { - void this.runUpdateCycle(); - }, this.intervalMs); - } - - onModuleDestroy(): void { - if (this.intervalHandle) { - clearInterval(this.intervalHandle); - } - } - - async runUpdateCycle(): Promise { + async findProvisionedItemIds(): Promise { const items = await this.subscriptionItemsRepository.findProvisionedWithSshKey(); - if (items.length === 0) { - return; - } + return items.map((item) => item.id); + } - this.logger.log(`Running update cycle for ${items.length} provisioned item(s)`); + async updateItem(subscriptionItemId: string): Promise { + const item = await this.subscriptionItemsRepository.findByIdWithRelations(subscriptionItemId); - for (const item of items) { - try { - await this.updateItem(item); - } catch (error) { - this.logger.error( - `Update failed for subscription item ${item.id} (${item.subscription?.number ?? item.subscriptionId}): ${(error as Error).message}`, - ); - } + if (!item) { + throw new Error(`Subscription item ${subscriptionItemId} not found`); } - } - private async updateItem(item: { - id: string; - subscriptionId: string; - providerReference?: string; - sshPrivateKey?: string; - serviceType?: { provider?: string }; - configSnapshot?: Record; - }): Promise { const provider = item.serviceType?.provider; if (!provider || !item.providerReference || !item.sshPrivateKey) { diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-item-update.scheduler.spec.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-item-update.scheduler.spec.ts deleted file mode 100644 index b712c78a..00000000 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-item-update.scheduler.spec.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { SubscriptionItemsRepository } from '../repositories/subscription-items.repository'; - -import { ProvisioningService } from './provisioning.service'; -import { SshExecutorService } from './ssh-executor.service'; -import { SubscriptionItemUpdateScheduler } from './subscription-item-update.scheduler'; - -describe('SubscriptionItemUpdateScheduler', () => { - let scheduler: SubscriptionItemUpdateScheduler; - let subscriptionItemsRepository: jest.Mocked>; - let provisioningService: jest.Mocked>; - let sshExecutor: jest.Mocked>; - const mockItem = { - id: 'item-1', - subscriptionId: 'sub-1', - providerReference: 'srv-123', - sshPrivateKey: '-----BEGIN OPENSSH PRIVATE KEY-----\n...', - serviceType: { provider: 'hetzner' }, - configSnapshot: { service: 'controller' }, - subscription: { number: 'SUB-001', status: 'active' }, - }; - - beforeEach(() => { - subscriptionItemsRepository = { - findProvisionedWithSshKey: jest.fn(), - }; - provisioningService = { - getServerInfo: jest.fn(), - }; - sshExecutor = { - exec: jest.fn(), - }; - scheduler = new SubscriptionItemUpdateScheduler( - subscriptionItemsRepository as never, - provisioningService as never, - sshExecutor as never, - ); - }); - - it('does nothing when no provisioned items with SSH key', async () => { - subscriptionItemsRepository.findProvisionedWithSshKey.mockResolvedValue([]); - - await scheduler.runUpdateCycle(); - - expect(provisioningService.getServerInfo).not.toHaveBeenCalled(); - expect(sshExecutor.exec).not.toHaveBeenCalled(); - }); - - it('runs update command via SSH for each item with public IP', async () => { - subscriptionItemsRepository.findProvisionedWithSshKey.mockResolvedValue([mockItem as never]); - provisioningService.getServerInfo.mockResolvedValue({ - serverId: 'srv-123', - name: 'test', - publicIp: '1.2.3.4', - status: 'running', - } as never); - sshExecutor.exec.mockResolvedValue({ stdout: '', stderr: '', code: 0 }); - - await scheduler.runUpdateCycle(); - - expect(provisioningService.getServerInfo).toHaveBeenCalledWith('hetzner', 'srv-123'); - expect(sshExecutor.exec).toHaveBeenCalledWith( - '1.2.3.4', - 22, - 'root', - mockItem.sshPrivateKey, - expect.stringContaining('docker compose up -d --pull=always'), - ); - expect(sshExecutor.exec).toHaveBeenCalledWith( - '1.2.3.4', - 22, - 'root', - mockItem.sshPrivateKey, - expect.stringContaining('/opt/agent-controller'), - ); - }); - - it('uses agent-manager update command when service is manager', async () => { - const managerItem = { ...mockItem, configSnapshot: { service: 'manager' } }; - - subscriptionItemsRepository.findProvisionedWithSshKey.mockResolvedValue([managerItem as never]); - provisioningService.getServerInfo.mockResolvedValue({ - publicIp: '1.2.3.4', - } as never); - sshExecutor.exec.mockResolvedValue({ stdout: '', stderr: '', code: 0 }); - - await scheduler.runUpdateCycle(); - - expect(sshExecutor.exec).toHaveBeenCalledWith( - '1.2.3.4', - 22, - 'root', - managerItem.sshPrivateKey, - expect.stringContaining('/opt/agent-manager'), - ); - }); - - it('skips item when getServerInfo returns no public IP', async () => { - subscriptionItemsRepository.findProvisionedWithSshKey.mockResolvedValue([mockItem as never]); - provisioningService.getServerInfo.mockResolvedValue({ - serverId: 'srv-123', - name: 'test', - publicIp: '', - status: 'running', - } as never); - - await scheduler.runUpdateCycle(); - - expect(sshExecutor.exec).not.toHaveBeenCalled(); - }); - - it('skips item when getServerInfo returns null', async () => { - subscriptionItemsRepository.findProvisionedWithSshKey.mockResolvedValue([mockItem as never]); - provisioningService.getServerInfo.mockResolvedValue(null); - - await scheduler.runUpdateCycle(); - - expect(sshExecutor.exec).not.toHaveBeenCalled(); - }); - - it('continues to next item when one fails', async () => { - const item2 = { ...mockItem, id: 'item-2', subscriptionId: 'sub-1', providerReference: 'srv-456' }; - - subscriptionItemsRepository.findProvisionedWithSshKey.mockResolvedValue([mockItem as never, item2 as never]); - provisioningService.getServerInfo - .mockResolvedValueOnce({ publicIp: '1.2.3.4' } as never) - .mockResolvedValueOnce({ publicIp: '5.6.7.8' } as never); - sshExecutor.exec.mockRejectedValueOnce(new Error('SSH connection failed')).mockResolvedValueOnce({ - stdout: '', - stderr: '', - code: 0, - }); - - await scheduler.runUpdateCycle(); - - expect(sshExecutor.exec).toHaveBeenCalledTimes(2); - }); -}); diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-renewal-reminder.scheduler.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-renewal-reminder.job-handler.ts similarity index 52% rename from libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-renewal-reminder.scheduler.ts rename to libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-renewal-reminder.job-handler.ts index 2d7f91b5..7876f23a 100644 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-renewal-reminder.scheduler.ts +++ b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-renewal-reminder.job-handler.ts @@ -1,22 +1,21 @@ import { EmailService } from '@forepath/shared/backend'; -import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; -import { SubscriptionEntity } from '../entities/subscription.entity'; import { CustomerProfilesRepository } from '../repositories/customer-profiles.repository'; import { ServicePlansRepository } from '../repositories/service-plans.repository'; import { SubscriptionsRepository } from '../repositories/subscriptions.repository'; -@Injectable() -export class SubscriptionRenewalReminderScheduler implements OnModuleInit, OnModuleDestroy { - private readonly logger = new Logger(SubscriptionRenewalReminderScheduler.name); - private intervalHandle?: NodeJS.Timeout; +export interface RenewalReminderUnitPayload { + subscriptionId: string; + periodKey: string; +} - private readonly intervalMs = parseInt(process.env.REMINDER_SCHEDULER_INTERVAL ?? '3600000', 10); +@Injectable() +export class SubscriptionRenewalReminderJobHandler { + private readonly logger = new Logger(SubscriptionRenewalReminderJobHandler.name); private readonly reminderDays = parseInt(process.env.REMINDER_DAYS ?? '3', 10); private readonly batchSize = parseInt(process.env.REMINDER_SCHEDULER_BATCH_SIZE ?? '100', 10); - private readonly sentReminders = new Map(); - constructor( private readonly subscriptionsRepository: SubscriptionsRepository, private readonly servicePlansRepository: ServicePlansRepository, @@ -24,26 +23,11 @@ export class SubscriptionRenewalReminderScheduler implements OnModuleInit, OnMod private readonly emailService: EmailService, ) {} - onModuleInit(): void { - this.logger.log( - `Initializing reminder scheduler with ${this.intervalMs}ms interval, reminding ${this.reminderDays} days before renewal`, - ); - this.intervalHandle = setInterval(() => { - void this.processUpcomingRenewals(); - }, this.intervalMs); + isEmailEnabled(): boolean { + return this.emailService.isEnabled(); } - onModuleDestroy(): void { - if (this.intervalHandle) { - clearInterval(this.intervalHandle); - } - } - - async processUpcomingRenewals(): Promise { - if (!this.emailService.isEnabled()) { - return; - } - + async findUpcomingReminderUnits(): Promise { const now = new Date(); const upcomingSubscriptions = await this.subscriptionsRepository.findUpcomingRenewals( this.reminderDays, @@ -51,30 +35,14 @@ export class SubscriptionRenewalReminderScheduler implements OnModuleInit, OnMod this.batchSize, ); - if (upcomingSubscriptions.length === 0) { - return; - } - - this.logger.log(`Processing ${upcomingSubscriptions.length} subscriptions with upcoming renewals`); - - for (const subscription of upcomingSubscriptions) { - try { - await this.processSubscriptionReminder(subscription, now); - } catch (error) { - this.logger.error(`Failed to send reminder for subscription ${subscription.id}: ${(error as Error).message}`); - } - } - - this.cleanupOldReminders(now); + return upcomingSubscriptions.map((subscription) => ({ + subscriptionId: subscription.id, + periodKey: `${subscription.id}:${subscription.currentPeriodEnd?.getTime() ?? 'none'}`, + })); } - private async processSubscriptionReminder(subscription: SubscriptionEntity, now: Date): Promise { - const periodKey = `${subscription.id}:${subscription.currentPeriodEnd?.getTime()}`; - - if (this.sentReminders.has(periodKey)) { - return; - } - + async processReminder(payload: RenewalReminderUnitPayload): Promise { + const subscription = await this.subscriptionsRepository.findByIdOrThrow(payload.subscriptionId); const profile = await this.customerProfilesRepository.findByUserId(subscription.userId); const email = profile?.email; @@ -94,18 +62,7 @@ export class SubscriptionRenewalReminderScheduler implements OnModuleInit, OnMod }); if (sent) { - this.sentReminders.set(periodKey, now); this.logger.log(`Sent renewal reminder for subscription ${subscription.id} to ${email}`); } } - - private cleanupOldReminders(now: Date): void { - const maxAge = 30 * 24 * 60 * 60 * 1000; - - for (const [key, timestamp] of this.sentReminders.entries()) { - if (now.getTime() - timestamp.getTime() > maxAge) { - this.sentReminders.delete(key); - } - } - } } diff --git a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-renewal-reminder.scheduler.spec.ts b/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-renewal-reminder.scheduler.spec.ts deleted file mode 100644 index 57da4c6f..00000000 --- a/libs/domains/framework/backend/feature-billing-manager/src/lib/services/subscription-renewal-reminder.scheduler.spec.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { SubscriptionRenewalReminderScheduler } from './subscription-renewal-reminder.scheduler'; - -describe('SubscriptionRenewalReminderScheduler', () => { - const subscriptionsRepository = { - findUpcomingRenewals: jest.fn(), - } as any; - const servicePlansRepository = { - findByIdOrThrow: jest.fn(), - } as any; - const customerProfilesRepository = { - findByUserId: jest.fn(), - } as any; - const emailService = { - isEnabled: jest.fn(), - send: jest.fn(), - } as any; - let scheduler: SubscriptionRenewalReminderScheduler; - - beforeEach(() => { - jest.resetAllMocks(); - scheduler = new SubscriptionRenewalReminderScheduler( - subscriptionsRepository, - servicePlansRepository, - customerProfilesRepository, - emailService, - ); - }); - - it('sends renewal reminder', async () => { - emailService.isEnabled.mockReturnValue(true); - subscriptionsRepository.findUpcomingRenewals.mockResolvedValue([ - { - id: 'sub-1', - userId: 'user-1', - planId: 'plan-1', - currentPeriodEnd: new Date('2024-02-01'), - nextBillingAt: new Date('2024-02-15'), - }, - ]); - customerProfilesRepository.findByUserId.mockResolvedValue({ - userId: 'user-1', - email: 'user@example.com', - firstName: 'John', - }); - servicePlansRepository.findByIdOrThrow.mockResolvedValue({ - id: 'plan-1', - name: 'Basic Plan', - }); - emailService.send.mockResolvedValue(true); - - await scheduler.processUpcomingRenewals(); - - expect(emailService.send).toHaveBeenCalledWith( - expect.objectContaining({ - to: 'user@example.com', - subject: 'Upcoming subscription renewal: Basic Plan', - }), - ); - }); - - it('skips when email disabled', async () => { - emailService.isEnabled.mockReturnValue(false); - - await scheduler.processUpcomingRenewals(); - - expect(subscriptionsRepository.findUpcomingRenewals).not.toHaveBeenCalled(); - }); - - it('handles empty upcoming renewals', async () => { - emailService.isEnabled.mockReturnValue(true); - subscriptionsRepository.findUpcomingRenewals.mockResolvedValue([]); - - await scheduler.processUpcomingRenewals(); - - expect(emailService.send).not.toHaveBeenCalled(); - }); - - it('skips when no email found for user', async () => { - emailService.isEnabled.mockReturnValue(true); - subscriptionsRepository.findUpcomingRenewals.mockResolvedValue([ - { id: 'sub-1', userId: 'user-1', planId: 'plan-1' }, - ]); - customerProfilesRepository.findByUserId.mockResolvedValue(null); - - await scheduler.processUpcomingRenewals(); - - expect(emailService.send).not.toHaveBeenCalled(); - }); - - it('skips when email profile has no email', async () => { - emailService.isEnabled.mockReturnValue(true); - subscriptionsRepository.findUpcomingRenewals.mockResolvedValue([ - { id: 'sub-1', userId: 'user-1', planId: 'plan-1' }, - ]); - customerProfilesRepository.findByUserId.mockResolvedValue({ - userId: 'user-1', - email: null, - }); - - await scheduler.processUpcomingRenewals(); - - expect(emailService.send).not.toHaveBeenCalled(); - }); - - it('skips duplicate reminder for same period', async () => { - emailService.isEnabled.mockReturnValue(true); - const subscription = { - id: 'sub-1', - userId: 'user-1', - planId: 'plan-1', - currentPeriodEnd: new Date('2024-02-01'), - nextBillingAt: new Date('2024-02-15'), - }; - - subscriptionsRepository.findUpcomingRenewals.mockResolvedValue([subscription, subscription]); - customerProfilesRepository.findByUserId.mockResolvedValue({ - userId: 'user-1', - email: 'user@example.com', - }); - servicePlansRepository.findByIdOrThrow.mockResolvedValue({ id: 'plan-1', name: 'Plan' }); - emailService.send.mockResolvedValue(true); - - await scheduler.processUpcomingRenewals(); - - expect(emailService.send).toHaveBeenCalledTimes(1); - }); -}); diff --git a/libs/domains/identity/backend/util-auth/src/lib/bull-board-request-path.spec.ts b/libs/domains/identity/backend/util-auth/src/lib/bull-board-request-path.spec.ts new file mode 100644 index 00000000..e13533cc --- /dev/null +++ b/libs/domains/identity/backend/util-auth/src/lib/bull-board-request-path.spec.ts @@ -0,0 +1,29 @@ +import { isBullBoardRequestPath } from './bull-board-request-path'; + +describe('isBullBoardRequestPath', () => { + const originalPath = process.env.QUEUE_BULL_BOARD_PATH; + + afterEach(() => { + if (originalPath === undefined) { + delete process.env.QUEUE_BULL_BOARD_PATH; + } else { + process.env.QUEUE_BULL_BOARD_PATH = originalPath; + } + }); + + it('matches default /admin/queues and API subpaths', () => { + delete process.env.QUEUE_BULL_BOARD_PATH; + + expect(isBullBoardRequestPath('/admin/queues')).toBe(true); + expect(isBullBoardRequestPath('/admin/queues/api/queues/agent-controller/jobs/clean')).toBe(true); + expect(isBullBoardRequestPath('/api/health')).toBe(false); + }); + + it('respects QUEUE_BULL_BOARD_PATH', () => { + process.env.QUEUE_BULL_BOARD_PATH = '/ops/queues'; + + expect(isBullBoardRequestPath('/ops/queues')).toBe(true); + expect(isBullBoardRequestPath('/ops/queues/api/queues')).toBe(true); + expect(isBullBoardRequestPath('/admin/queues')).toBe(false); + }); +}); diff --git a/libs/domains/identity/backend/util-auth/src/lib/bull-board-request-path.ts b/libs/domains/identity/backend/util-auth/src/lib/bull-board-request-path.ts new file mode 100644 index 00000000..d4779228 --- /dev/null +++ b/libs/domains/identity/backend/util-auth/src/lib/bull-board-request-path.ts @@ -0,0 +1,11 @@ +/** + * Matches HTTP paths served by Bull Board (uses QUEUE_BULL_BOARD_PATH). + * Used to bypass API guards that conflict with Bull Board's HTTP Basic auth. + */ +export function isBullBoardRequestPath(urlPath: string): boolean { + const boardPath = (process.env.QUEUE_BULL_BOARD_PATH?.trim() || '/admin/queues').replace(/\/+$/, ''); + const base = boardPath.startsWith('/') ? boardPath : `/${boardPath}`; + const path = urlPath.split('?')[0]?.replace(/\/+$/, '') ?? ''; + + return path === base || path.startsWith(`${base}/`); +} diff --git a/libs/domains/identity/backend/util-auth/src/lib/hybrid-auth.guard.spec.ts b/libs/domains/identity/backend/util-auth/src/lib/hybrid-auth.guard.spec.ts index c3591dfc..16ee1603 100644 --- a/libs/domains/identity/backend/util-auth/src/lib/hybrid-auth.guard.spec.ts +++ b/libs/domains/identity/backend/util-auth/src/lib/hybrid-auth.guard.spec.ts @@ -127,6 +127,20 @@ describe('HybridAuthGuard', () => { expect(result).toBe(true); }); + it('should allow Bull Board paths without API key (HTTP Basic on board routes)', () => { + const mockRequest = { + originalUrl: '/admin/queues/api/queues/agent-controller/jobs/clean', + url: '/admin/queues/api/queues/agent-controller/jobs/clean', + headers: { authorization: 'Basic YWRtaW46YnVsbG1x' }, + }; + + mockExecutionContext.switchToHttp = jest.fn().mockReturnValue({ + getRequest: jest.fn().mockReturnValue(mockRequest), + }); + + expect(guard.canActivate(mockExecutionContext)).toBe(true); + }); + it('should throw UnauthorizedException when authorization header is missing', () => { const mockRequest = { headers: {}, diff --git a/libs/domains/identity/backend/util-auth/src/lib/hybrid-auth.guard.ts b/libs/domains/identity/backend/util-auth/src/lib/hybrid-auth.guard.ts index 56a15817..e17e9462 100644 --- a/libs/domains/identity/backend/util-auth/src/lib/hybrid-auth.guard.ts +++ b/libs/domains/identity/backend/util-auth/src/lib/hybrid-auth.guard.ts @@ -2,6 +2,7 @@ import { CanActivate, ExecutionContext, Injectable, UnauthorizedException } from import { APP_GUARD, Reflector } from '@nestjs/core'; import { AuthGuard, ResourceGuard, RoleGuard } from 'nest-keycloak-connect'; +import { isBullBoardRequestPath } from './bull-board-request-path'; import { IS_PUBLIC_KEY } from './decorators/public.decorator'; /** Supported authentication methods. */ @@ -55,6 +56,11 @@ export class HybridAuthGuard implements CanActivate { return true; } + // Bull Board uses its own HTTP Basic auth (QUEUE_BULL_BOARD_*), not API key / Keycloak + if (isBullBoardRequestPath(path)) { + return true; + } + const isPublic = this.reflector.getAllAndOverride(IS_PUBLIC_KEY, [ context.getHandler(), context.getClass(), diff --git a/libs/domains/identity/backend/util-auth/src/lib/origin-allowlist.middleware.spec.ts b/libs/domains/identity/backend/util-auth/src/lib/origin-allowlist.middleware.spec.ts index 5b65bb0d..18819b28 100644 --- a/libs/domains/identity/backend/util-auth/src/lib/origin-allowlist.middleware.spec.ts +++ b/libs/domains/identity/backend/util-auth/src/lib/origin-allowlist.middleware.spec.ts @@ -148,4 +148,18 @@ describe('createOriginAllowlistMiddleware', () => { expect(nextError).toBeUndefined(); }); + + it('skips origin enforcement for Bull Board paths', () => { + process.env.NODE_ENV = 'production'; + process.env.CORS_ORIGIN = 'https://app.example.com'; + delete process.env.QUEUE_BULL_BOARD_PATH; + const middleware = createOriginAllowlistMiddleware(new Logger('test')); + const { nextError } = run(middleware, { + method: 'DELETE', + originalUrl: '/admin/queues/api/queues/agent-controller/jobs/clean', + headers: { origin: 'http://localhost:3100' }, + }); + + expect(nextError).toBeUndefined(); + }); }); diff --git a/libs/domains/identity/backend/util-auth/src/lib/origin-allowlist.middleware.ts b/libs/domains/identity/backend/util-auth/src/lib/origin-allowlist.middleware.ts index f996837b..0d0466af 100644 --- a/libs/domains/identity/backend/util-auth/src/lib/origin-allowlist.middleware.ts +++ b/libs/domains/identity/backend/util-auth/src/lib/origin-allowlist.middleware.ts @@ -1,6 +1,8 @@ import { ForbiddenException, Logger } from '@nestjs/common'; import type { NextFunction, Request, Response } from 'express'; +import { isBullBoardRequestPath } from './bull-board-request-path'; + /** * Comma-separated allowlist parsing (trim, lowercase, drop empties). * Intentionally matches {@link parseAllowedHosts} in `@forepath/shared/shared/util-network-address`; @@ -48,6 +50,14 @@ export function createOriginAllowlistMiddleware( logger: Logger, ): (req: Request, res: Response, next: NextFunction) => void { return (req, _res, next) => { + const requestPath = (req.originalUrl ?? req.url ?? '').split('?')[0] ?? ''; + + if (isBullBoardRequestPath(requestPath)) { + next(); + + return; + } + if (!isOriginAllowlistEnforced()) { next(); diff --git a/libs/domains/shared/backend/index.ts b/libs/domains/shared/backend/index.ts index 6524ce5d..51614023 100644 --- a/libs/domains/shared/backend/index.ts +++ b/libs/domains/shared/backend/index.ts @@ -1,3 +1,4 @@ // shared domain backend exports export * from './util-crypto/src'; export * from './util-email/src'; +export * from './util-queue/src'; diff --git a/libs/domains/shared/backend/util-queue/.eslintrc.json b/libs/domains/shared/backend/util-queue/.eslintrc.json new file mode 100644 index 00000000..b3399565 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../../../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/libs/domains/shared/backend/util-queue/README.md b/libs/domains/shared/backend/util-queue/README.md new file mode 100644 index 00000000..14449596 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/README.md @@ -0,0 +1,7 @@ +# shared-backend-util-queue + +This library was generated with [Nx](https://nx.dev). + +## Running unit tests + +Run `nx test shared-backend-util-queue` to execute the unit tests via [Jest](https://jestjs.io). diff --git a/libs/domains/shared/backend/util-queue/jest.config.cts b/libs/domains/shared/backend/util-queue/jest.config.cts new file mode 100644 index 00000000..8cc0c6dc --- /dev/null +++ b/libs/domains/shared/backend/util-queue/jest.config.cts @@ -0,0 +1,11 @@ +module.exports = { + displayName: 'shared-backend-util-queue', + preset: '../../../../../jest.preset.cjs', + testEnvironment: 'node', + transform: { + '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], + }, + moduleFileExtensions: ['ts', 'js', 'html'], + coverageDirectory: + '../../../../../coverage/libs/domains/shared/backend/util-queue', +}; diff --git a/libs/domains/shared/backend/util-queue/project.json b/libs/domains/shared/backend/util-queue/project.json new file mode 100644 index 00000000..121872d8 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/project.json @@ -0,0 +1,16 @@ +{ + "name": "shared-backend-util-queue", + "$schema": "../../../../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "libs/domains/shared/backend/util-queue/src", + "projectType": "library", + "tags": ["domain:shared", "scope:backend", "type:util"], + "targets": { + "test": { + "executor": "@nx/jest:jest", + "outputs": ["{workspaceRoot}/coverage/{projectRoot}"], + "options": { + "jestConfig": "libs/domains/shared/backend/util-queue/jest.config.cts" + } + } + } +} diff --git a/libs/domains/shared/backend/util-queue/src/index.ts b/libs/domains/shared/backend/util-queue/src/index.ts new file mode 100644 index 00000000..491fd9d8 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/index.ts @@ -0,0 +1,10 @@ +export * from './lib/bull-board-auth'; +export * from './lib/bull-board-auth.middleware'; +export * from './lib/bull-board-global-prefix'; +export * from './lib/enqueue-unit-job'; +export * from './lib/job-id.util'; +export * from './lib/queue-connection.config'; +export * from './lib/queue-role'; +export * from './lib/queue.module'; +export * from './lib/run-pending-migrations'; +export * from './lib/typeorm-options-for-role'; diff --git a/libs/domains/shared/backend/util-queue/src/lib/bull-board-auth.middleware.spec.ts b/libs/domains/shared/backend/util-queue/src/lib/bull-board-auth.middleware.spec.ts new file mode 100644 index 00000000..f61b8ae4 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/bull-board-auth.middleware.spec.ts @@ -0,0 +1,72 @@ +import type { Request, Response } from 'express'; + +import { createBullBoardAuthMiddleware } from './bull-board-auth.middleware'; + +function createMockResponse(): Response & { statusCode: number; body: string; headers: Record } { + const res = { + statusCode: 200, + body: '', + headers: {} as Record, + setHeader(name: string, value: string) { + this.headers[name] = value; + }, + status(code: number) { + this.statusCode = code; + + return this; + }, + send(payload: string) { + this.body = payload; + }, + }; + + return res as Response & typeof res; +} + +describe('createBullBoardAuthMiddleware', () => { + it('rejects when password is not configured', () => { + const middleware = createBullBoardAuthMiddleware({ username: 'admin', password: '' }); + const res = createMockResponse(); + const next = jest.fn(); + + middleware({ headers: {} } as Request, res, next); + + expect(res.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects missing Authorization header', () => { + const middleware = createBullBoardAuthMiddleware({ username: 'admin', password: 'bullmq' }); + const res = createMockResponse(); + const next = jest.fn(); + + middleware({ headers: {} } as Request, res, next); + + expect(res.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects invalid credentials', () => { + const middleware = createBullBoardAuthMiddleware({ username: 'admin', password: 'bullmq' }); + const res = createMockResponse(); + const next = jest.fn(); + const authorization = `Basic ${Buffer.from('admin:wrong').toString('base64')}`; + + middleware({ headers: { authorization } } as Request, res, next); + + expect(res.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('allows valid credentials', () => { + const middleware = createBullBoardAuthMiddleware({ username: 'admin', password: 'bullmq' }); + const res = createMockResponse(); + const next = jest.fn(); + const authorization = `Basic ${Buffer.from('admin:bullmq').toString('base64')}`; + + middleware({ headers: { authorization } } as Request, res, next); + + expect(next).toHaveBeenCalled(); + expect(res.statusCode).toBe(200); + }); +}); diff --git a/libs/domains/shared/backend/util-queue/src/lib/bull-board-auth.middleware.ts b/libs/domains/shared/backend/util-queue/src/lib/bull-board-auth.middleware.ts new file mode 100644 index 00000000..91f72c73 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/bull-board-auth.middleware.ts @@ -0,0 +1,82 @@ +import { timingSafeEqual } from 'node:crypto'; +import type { NextFunction, Request, Response } from 'express'; + +export interface BullBoardAuthCredentials { + username: string; + password: string; +} + +function safeEqualString(left: string, right: string): boolean { + const leftBuffer = Buffer.from(left); + const rightBuffer = Buffer.from(right); + + if (leftBuffer.length !== rightBuffer.length) { + return false; + } + + return timingSafeEqual(leftBuffer, rightBuffer); +} + +function parseBasicAuthHeader(header: string | undefined): { username: string; password: string } | null { + if (!header?.startsWith('Basic ')) { + return null; + } + + const encoded = header.slice('Basic '.length).trim(); + let decoded: string; + + try { + decoded = Buffer.from(encoded, 'base64').toString('utf8'); + } catch { + return null; + } + + const separatorIndex = decoded.indexOf(':'); + + if (separatorIndex < 0) { + return null; + } + + return { + username: decoded.slice(0, separatorIndex), + password: decoded.slice(separatorIndex + 1), + }; +} + +function sendUnauthorized(res: Response): void { + res.setHeader('WWW-Authenticate', 'Basic realm="Bull Board"'); + res.status(401).send('Authentication required'); +} + +/** + * HTTP Basic authentication for Bull Board (Express adapter). + * Rejects all requests when password is not configured (fail closed). + */ +export function createBullBoardAuthMiddleware( + credentials: BullBoardAuthCredentials, +): (req: Request, res: Response, next: NextFunction) => void { + const expectedUsername = credentials.username; + const expectedPassword = credentials.password; + + return (req, res, next) => { + if (!expectedPassword) { + sendUnauthorized(res); + + return; + } + + const provided = parseBasicAuthHeader(req.headers.authorization); + + if ( + !provided || + !safeEqualString(provided.username, expectedUsername) || + !safeEqualString(provided.password, expectedPassword) + ) { + sendUnauthorized(res); + + return; + } + + next(); + }; +} diff --git a/libs/domains/shared/backend/util-queue/src/lib/bull-board-auth.ts b/libs/domains/shared/backend/util-queue/src/lib/bull-board-auth.ts new file mode 100644 index 00000000..cbedb1ae --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/bull-board-auth.ts @@ -0,0 +1,27 @@ +import type { LoggerService } from '@nestjs/common'; + +import { createBullBoardAuthMiddleware } from './bull-board-auth.middleware'; +import { isBullBoardAuthConfigured, readBullBoardAuthConfig } from './queue-connection.config'; +import { shouldEnableBullBoard } from './queue-role'; + +export function createBullBoardAuthMiddlewareFromEnv(): ReturnType { + return createBullBoardAuthMiddleware(readBullBoardAuthConfig()); +} + +/** + * Ensures Bull Board is not exposed without credentials in production. + */ +export function assertBullBoardAuthConfigured(logger?: Pick): void { + if (!shouldEnableBullBoard() || isBullBoardAuthConfigured()) { + return; + } + + const message = + 'QUEUE_BULL_BOARD_PASSWORD must be set when Bull Board is enabled (QUEUE_BULL_BOARD_USERNAME defaults to admin)'; + + if (process.env.NODE_ENV === 'production') { + throw new Error(message); + } + + logger?.warn(`${message}; Bull Board will reject requests until configured`); +} diff --git a/libs/domains/shared/backend/util-queue/src/lib/bull-board-global-prefix.spec.ts b/libs/domains/shared/backend/util-queue/src/lib/bull-board-global-prefix.spec.ts new file mode 100644 index 00000000..770d4d58 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/bull-board-global-prefix.spec.ts @@ -0,0 +1,31 @@ +import { RequestMethod } from '@nestjs/common'; + +import { getBullBoardGlobalPrefixExcludes } from './bull-board-global-prefix'; + +describe('getBullBoardGlobalPrefixExcludes', () => { + const originalEnv = process.env; + + beforeEach(() => { + process.env = { ...originalEnv }; + }); + + afterAll(() => { + process.env = originalEnv; + }); + + it('returns empty when Bull Board is disabled', () => { + process.env.QUEUE_BULL_BOARD_ENABLED = 'false'; + + expect(getBullBoardGlobalPrefixExcludes()).toEqual([]); + }); + + it('excludes configured Bull Board path from global prefix', () => { + process.env.QUEUE_BULL_BOARD_ENABLED = 'true'; + process.env.QUEUE_BULL_BOARD_PATH = '/admin/queues'; + + expect(getBullBoardGlobalPrefixExcludes()).toEqual([ + { path: 'admin/queues', method: RequestMethod.ALL }, + { path: 'admin/queues/*path', method: RequestMethod.ALL }, + ]); + }); +}); diff --git a/libs/domains/shared/backend/util-queue/src/lib/bull-board-global-prefix.ts b/libs/domains/shared/backend/util-queue/src/lib/bull-board-global-prefix.ts new file mode 100644 index 00000000..3b59fe2f --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/bull-board-global-prefix.ts @@ -0,0 +1,22 @@ +import { RequestMethod } from '@nestjs/common'; +import type { RouteInfo } from '@nestjs/common/interfaces'; + +import { readBullBoardPath } from './queue-connection.config'; +import { shouldEnableBullBoard } from './queue-role'; + +/** + * Routes excluded from Nest's global prefix so Bull Board stays at QUEUE_BULL_BOARD_PATH + * (e.g. /admin/queues) instead of /api/admin/queues. + */ +export function getBullBoardGlobalPrefixExcludes(env: NodeJS.ProcessEnv = process.env): RouteInfo[] { + if (!shouldEnableBullBoard()) { + return []; + } + + const route = readBullBoardPath(env).replace(/^\//, ''); + + return [ + { path: route, method: RequestMethod.ALL }, + { path: `${route}/*path`, method: RequestMethod.ALL }, + ]; +} diff --git a/libs/domains/shared/backend/util-queue/src/lib/enqueue-unit-job.ts b/libs/domains/shared/backend/util-queue/src/lib/enqueue-unit-job.ts new file mode 100644 index 00000000..089f9565 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/enqueue-unit-job.ts @@ -0,0 +1,26 @@ +import type { JobsOptions, Queue } from 'bullmq'; + +import { buildJobId } from './job-id.util'; + +export interface EnqueueUnitJobOptions { + queue: Queue; + jobName: string; + payload: T; + jobIdNamespace: string; + jobIdParts: Array; + opts?: Omit; +} + +/** Enqueues a unit job with a stable jobId to prevent duplicate processing. */ +export async function enqueueUnitJob(options: EnqueueUnitJobOptions): Promise { + const jobId = buildJobId(options.jobIdNamespace, ...options.jobIdParts); + + await options.queue.add(options.jobName, options.payload, { + jobId, + removeOnComplete: { age: 3600, count: 1000 }, + removeOnFail: { age: 86400, count: 5000 }, + attempts: 3, + backoff: { type: 'exponential', delay: 5000 }, + ...options.opts, + }); +} diff --git a/libs/domains/shared/backend/util-queue/src/lib/job-id.util.spec.ts b/libs/domains/shared/backend/util-queue/src/lib/job-id.util.spec.ts new file mode 100644 index 00000000..80082f17 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/job-id.util.spec.ts @@ -0,0 +1,33 @@ +import { assertValidBullMqJobId, buildCoordinatorJobId, buildJobId, sanitizeJobIdSegment } from './job-id.util'; + +describe('job-id.util', () => { + it('joins namespace and parts with .', () => { + expect(buildJobId('billing', 'subscription', 'abc-123')).toBe('billing.subscription.abc-123'); + }); + + it('omits empty parts', () => { + expect(buildJobId('sync', undefined, 'id')).toBe('sync.id'); + }); + + it('sanitizes colons and slashes in namespace and parts', () => { + expect(buildJobId('billing:subscription', 'abc-123')).toBe('billing.subscription.abc-123'); + expect(sanitizeJobIdSegment('invoice-sync/ref')).toBe('invoice-sync.ref'); + }); + + it('buildCoordinatorJobId uses allowed characters only', () => { + expect(buildCoordinatorJobId('filter-rules-sync')).toBe('coordinator.filter-rules-sync'); + assertValidBullMqJobId(buildCoordinatorJobId('filter-rules-sync')); + }); + + it('rejects integer-only job ids', () => { + expect(() => assertValidBullMqJobId('12345')).toThrow('must not be an integer'); + }); + + it('rejects job ids containing colons', () => { + expect(() => assertValidBullMqJobId('coordinator:bad')).toThrow("must not contain ':'"); + }); + + it('rejects job ids with disallowed characters', () => { + expect(() => assertValidBullMqJobId('billing/subscription')).toThrow('may only contain'); + }); +}); diff --git a/libs/domains/shared/backend/util-queue/src/lib/job-id.util.ts b/libs/domains/shared/backend/util-queue/src/lib/job-id.util.ts new file mode 100644 index 00000000..1eb29423 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/job-id.util.ts @@ -0,0 +1,62 @@ +/** Allowed in BullMQ custom job IDs: `.` `-` `_` `~` (and alphanumeric). */ +const JOB_ID_SEPARATOR = '.'; + +const ALLOWED_JOB_ID_PATTERN = /^[a-zA-Z0-9._~-]+$/; + +/** Replaces characters outside BullMQ's allowed jobId set. */ +const DISALLOWED_JOB_ID_CHAR = /[^a-zA-Z0-9._~-]/g; + +/** + * Normalizes a segment for BullMQ custom job IDs. + * Colons are reserved by BullMQ for repeatable-job keys and are rejected otherwise. + */ +export function sanitizeJobIdSegment(value: string): string { + return value + .trim() + .replace(/:/g, JOB_ID_SEPARATOR) + .replace(/\//g, JOB_ID_SEPARATOR) + .replace(/\s+/g, '-') + .replace(DISALLOWED_JOB_ID_CHAR, '-') + .replace(/\.{2,}/g, '.') + .replace(/^\.+|\.+$/g, ''); +} + +/** Validates a BullMQ custom jobId (see bullmq Job.validateOptions). */ +export function assertValidBullMqJobId(jobId: string): void { + if (!jobId.length) { + throw new Error('BullMQ jobId must not be empty'); + } + + if (jobId.includes(':')) { + throw new Error(`BullMQ jobId must not contain ':' (got: ${jobId})`); + } + + if (`${parseInt(jobId, 10)}` === jobId) { + throw new Error(`BullMQ jobId must not be an integer (got: ${jobId})`); + } + + if (!ALLOWED_JOB_ID_PATTERN.test(jobId)) { + throw new Error(`BullMQ jobId may only contain alphanumerics and . - _ ~ (got: ${jobId})`); + } +} + +/** Builds a stable BullMQ jobId for deduplication (only one active/waiting job per id). */ +export function buildJobId(namespace: string, ...parts: Array): string { + const segments = [ + sanitizeJobIdSegment(namespace), + ...parts + .filter((part) => part !== undefined && part !== null && String(part).length > 0) + .map((part) => sanitizeJobIdSegment(String(part))), + ].filter((segment) => segment.length > 0); + + const jobId = segments.join(JOB_ID_SEPARATOR); + + assertValidBullMqJobId(jobId); + + return jobId; +} + +/** Stable jobId for repeatable coordinator jobs. */ +export function buildCoordinatorJobId(name: string): string { + return buildJobId('coordinator', name); +} diff --git a/libs/domains/shared/backend/util-queue/src/lib/queue-connection.config.spec.ts b/libs/domains/shared/backend/util-queue/src/lib/queue-connection.config.spec.ts new file mode 100644 index 00000000..a7a7bafe --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/queue-connection.config.spec.ts @@ -0,0 +1,68 @@ +import { + isBullBoardAuthConfigured, + readBullBoardAuthConfig, + readBullBoardPath, + readQueueWorkerConcurrency, + readRedisConnectionConfig, + toBullMqConnection, +} from './queue-connection.config'; + +describe('queue-connection.config', () => { + it('readRedisConnectionConfig uses defaults', () => { + expect(readRedisConnectionConfig({})).toEqual({ + host: 'localhost', + port: 6379, + db: 0, + keyPrefix: 'agenstra', + }); + }); + + it('readRedisConnectionConfig reads custom values', () => { + expect( + readRedisConnectionConfig({ + REDIS_HOST: 'redis', + REDIS_PORT: '6380', + REDIS_PASSWORD: 'secret', + REDIS_DB: '2', + REDIS_KEY_PREFIX: 'billing', + }), + ).toEqual({ + host: 'redis', + port: 6380, + password: 'secret', + db: 2, + keyPrefix: 'billing', + }); + }); + + it('toBullMqConnection maps config', () => { + expect(toBullMqConnection({ host: 'h', port: 1, db: 0, keyPrefix: 'p', password: 'x' })).toEqual({ + host: 'h', + port: 1, + db: 0, + password: 'x', + }); + }); + + it('readQueueWorkerConcurrency', () => { + expect(readQueueWorkerConcurrency({ QUEUE_WORKER_CONCURRENCY: '10' })).toBe(10); + expect(readQueueWorkerConcurrency({ QUEUE_WORKER_CONCURRENCY: '0' })).toBe(5); + }); + + it('readBullBoardPath normalizes', () => { + expect(readBullBoardPath({})).toBe('/admin/queues'); + expect(readBullBoardPath({ QUEUE_BULL_BOARD_PATH: 'queues' })).toBe('/queues'); + }); + + it('readBullBoardAuthConfig reads credentials', () => { + expect(readBullBoardAuthConfig({})).toEqual({ username: 'admin', password: '' }); + expect( + readBullBoardAuthConfig({ + QUEUE_BULL_BOARD_USERNAME: 'ops', + QUEUE_BULL_BOARD_PASSWORD: 'bullmq', + }), + ).toEqual({ username: 'ops', password: 'bullmq' }); + expect(isBullBoardAuthConfigured({ username: 'admin', password: 'bullmq' })).toBe(true); + expect(isBullBoardAuthConfigured({ username: 'admin', password: '' })).toBe(false); + }); +}); diff --git a/libs/domains/shared/backend/util-queue/src/lib/queue-connection.config.ts b/libs/domains/shared/backend/util-queue/src/lib/queue-connection.config.ts new file mode 100644 index 00000000..5167543f --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/queue-connection.config.ts @@ -0,0 +1,59 @@ +import type { QueueOptions } from 'bullmq'; + +export interface RedisConnectionConfig { + host: string; + port: number; + password?: string; + db: number; + keyPrefix: string; +} + +export function readRedisConnectionConfig(env: NodeJS.ProcessEnv = process.env): RedisConnectionConfig { + const password = env.REDIS_PASSWORD?.trim(); + + return { + host: env.REDIS_HOST?.trim() || 'localhost', + port: parseInt(env.REDIS_PORT ?? '6379', 10), + ...(password ? { password } : {}), + db: parseInt(env.REDIS_DB ?? '0', 10), + keyPrefix: env.REDIS_KEY_PREFIX?.trim() || 'agenstra', + }; +} + +export function toBullMqConnection(config: RedisConnectionConfig): QueueOptions['connection'] { + return { + host: config.host, + port: config.port, + db: config.db, + ...(config.password ? { password: config.password } : {}), + }; +} + +export function readQueueWorkerConcurrency(env: NodeJS.ProcessEnv = process.env): number { + const parsed = parseInt(env.QUEUE_WORKER_CONCURRENCY ?? '5', 10); + + return Number.isFinite(parsed) && parsed > 0 ? parsed : 5; +} + +export function readBullBoardPath(env: NodeJS.ProcessEnv = process.env): string { + const path = env.QUEUE_BULL_BOARD_PATH?.trim() || '/admin/queues'; + + return path.startsWith('/') ? path : `/${path}`; +} + +export interface BullBoardAuthConfig { + username: string; + password: string; +} + +/** Defaults: username `admin`; password must be set via env (compose uses `bullmq` locally). */ +export function readBullBoardAuthConfig(env: NodeJS.ProcessEnv = process.env): BullBoardAuthConfig { + return { + username: env.QUEUE_BULL_BOARD_USERNAME?.trim() || 'admin', + password: env.QUEUE_BULL_BOARD_PASSWORD?.trim() ?? '', + }; +} + +export function isBullBoardAuthConfigured(config: BullBoardAuthConfig = readBullBoardAuthConfig()): boolean { + return config.password.length > 0; +} diff --git a/libs/domains/shared/backend/util-queue/src/lib/queue-role.spec.ts b/libs/domains/shared/backend/util-queue/src/lib/queue-role.spec.ts new file mode 100644 index 00000000..d2a5b796 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/queue-role.spec.ts @@ -0,0 +1,52 @@ +import { + getQueueRole, + parseQueueRole, + shouldEnableBullBoard, + shouldRegisterRepeatableJobs, + shouldRunApiHttp, + shouldRunMigrations, + shouldRunQueueWorkers, +} from './queue-role'; + +describe('queue-role', () => { + const env = process.env; + + afterEach(() => { + process.env = { ...env }; + }); + + it('parseQueueRole defaults to all', () => { + expect(parseQueueRole(undefined)).toBe('all'); + }); + + it('parseQueueRole accepts api scheduler worker', () => { + expect(parseQueueRole('api')).toBe('api'); + expect(parseQueueRole('scheduler')).toBe('scheduler'); + expect(parseQueueRole('worker')).toBe('worker'); + }); + + it('parseQueueRole falls back to all for unknown', () => { + expect(parseQueueRole('invalid')).toBe('all'); + }); + + it('role capabilities', () => { + expect(shouldRunApiHttp('api')).toBe(true); + expect(shouldRunApiHttp('worker')).toBe(false); + expect(shouldRegisterRepeatableJobs('scheduler')).toBe(true); + expect(shouldRunQueueWorkers('worker')).toBe(true); + expect(shouldRunMigrations('worker')).toBe(false); + expect(shouldRunMigrations('api')).toBe(true); + }); + + it('getQueueRole reads QUEUE_ROLE', () => { + process.env.QUEUE_ROLE = 'worker'; + expect(getQueueRole()).toBe('worker'); + }); + + it('shouldEnableBullBoard respects env override', () => { + process.env.QUEUE_BULL_BOARD_ENABLED = 'false'; + expect(shouldEnableBullBoard('all')).toBe(false); + process.env.QUEUE_BULL_BOARD_ENABLED = 'true'; + expect(shouldEnableBullBoard('api')).toBe(true); + }); +}); diff --git a/libs/domains/shared/backend/util-queue/src/lib/queue-role.ts b/libs/domains/shared/backend/util-queue/src/lib/queue-role.ts new file mode 100644 index 00000000..7dfa2bf2 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/queue-role.ts @@ -0,0 +1,45 @@ +export type QueueRole = 'api' | 'scheduler' | 'worker' | 'all'; + +const VALID_ROLES: readonly QueueRole[] = ['api', 'scheduler', 'worker', 'all']; + +export function parseQueueRole(raw: string | undefined): QueueRole { + const value = (raw ?? 'all').trim().toLowerCase(); + + if (VALID_ROLES.includes(value as QueueRole)) { + return value as QueueRole; + } + + return 'all'; +} + +export function getQueueRole(): QueueRole { + return parseQueueRole(process.env.QUEUE_ROLE); +} + +export function shouldRunApiHttp(role: QueueRole = getQueueRole()): boolean { + return role === 'api' || role === 'all'; +} + +export function shouldRegisterRepeatableJobs(role: QueueRole = getQueueRole()): boolean { + return role === 'scheduler' || role === 'all'; +} + +export function shouldRunQueueWorkers(role: QueueRole = getQueueRole()): boolean { + return role === 'worker' || role === 'all'; +} + +export function shouldEnableBullBoard(role: QueueRole = getQueueRole()): boolean { + if (process.env.QUEUE_BULL_BOARD_ENABLED === 'false') { + return false; + } + + if (process.env.QUEUE_BULL_BOARD_ENABLED === 'true') { + return true; + } + + return role === 'scheduler' || role === 'all'; +} + +export function shouldRunMigrations(role: QueueRole = getQueueRole()): boolean { + return role === 'api' || role === 'all'; +} diff --git a/libs/domains/shared/backend/util-queue/src/lib/queue.module.ts b/libs/domains/shared/backend/util-queue/src/lib/queue.module.ts new file mode 100644 index 00000000..edfa6c29 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/queue.module.ts @@ -0,0 +1,91 @@ +import { BullMQAdapter } from '@bull-board/api/bullMQAdapter'; +import { BullBoardModule } from '@bull-board/nestjs'; +import { ExpressAdapter } from '@bull-board/express'; +import { BullModule } from '@nestjs/bullmq'; +import { DynamicModule, Module } from '@nestjs/common'; +import type { QueueOptions } from 'bullmq'; + +import { createBullBoardAuthMiddlewareFromEnv } from './bull-board-auth'; +import { shouldEnableBullBoard, shouldRegisterRepeatableJobs, shouldRunQueueWorkers } from './queue-role'; +import { + readBullBoardPath, + readQueueWorkerConcurrency, + readRedisConnectionConfig, + toBullMqConnection, +} from './queue-connection.config'; + +export const QUEUE_CONNECTION = 'QUEUE_CONNECTION'; + +export interface SharedQueueModuleOptions { + /** Queue names to register (processors attach via @Processor in app modules). */ + queueNames: string[]; + /** Default worker concurrency for @Processor classes in this app. */ + workerConcurrency?: number; +} + +@Module({}) +export class SharedQueueModule { + static forRoot(options: SharedQueueModuleOptions): DynamicModule { + const redis = readRedisConnectionConfig(); + const connection = toBullMqConnection(redis); + const prefix = redis.keyPrefix; + const concurrency = options.workerConcurrency ?? readQueueWorkerConcurrency(); + + const defaultJobOptions: QueueOptions['defaultJobOptions'] = { + removeOnComplete: { age: 3600, count: 1000 }, + removeOnFail: { age: 86400, count: 5000 }, + attempts: 3, + backoff: { type: 'exponential', delay: 5000 }, + }; + + const bullImports = [ + BullModule.forRoot({ + connection, + prefix, + defaultJobOptions, + }), + ...options.queueNames.map((name) => + BullModule.registerQueue({ + name, + connection, + prefix, + defaultJobOptions, + }), + ), + ]; + + const imports: DynamicModule['imports'] = [...bullImports]; + const moduleExports: DynamicModule['exports'] = [BullModule]; + + if (shouldEnableBullBoard() && options.queueNames.length > 0) { + imports.push( + BullBoardModule.forRoot({ + route: readBullBoardPath(), + adapter: ExpressAdapter, + middleware: createBullBoardAuthMiddlewareFromEnv(), + }), + BullBoardModule.forFeature( + ...options.queueNames.map((name) => ({ + name, + adapter: BullMQAdapter, + })), + ), + ); + } + + return { + module: SharedQueueModule, + imports, + exports: moduleExports, + global: true, + providers: [ + { + provide: QUEUE_CONNECTION, + useValue: { connection, prefix, concurrency }, + }, + ], + }; + } +} + +export { shouldRegisterRepeatableJobs, shouldRunQueueWorkers }; diff --git a/libs/domains/shared/backend/util-queue/src/lib/run-pending-migrations.ts b/libs/domains/shared/backend/util-queue/src/lib/run-pending-migrations.ts new file mode 100644 index 00000000..7d381796 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/run-pending-migrations.ts @@ -0,0 +1,43 @@ +import { Logger } from '@nestjs/common'; +import type { INestApplication, INestApplicationContext } from '@nestjs/common'; +import type { DataSourceOptions } from 'typeorm'; +import { DataSource } from 'typeorm'; + +import type { QueueRole } from './queue-role'; +import { shouldRunMigrations } from './queue-role'; + +/** + * Runs pending TypeORM migrations only when QUEUE_ROLE is `api` or `all`. + */ +export async function runPendingMigrationsIfRoleAllows( + app: INestApplication | INestApplicationContext, + role: QueueRole, + config: DataSourceOptions, +): Promise { + if (!shouldRunMigrations(role)) { + Logger.log(`Skipping database migrations (QUEUE_ROLE=${role})`, 'Migrations'); + + return; + } + + if (config.synchronize) { + Logger.log('Schema synchronization enabled — migrations skipped', 'Migrations'); + + return; + } + + if (!config.migrations?.length) { + return; + } + + const dataSource = app.get(DataSource); + + try { + Logger.log('Running pending migrations...', 'Migrations'); + await dataSource.runMigrations(); + Logger.log('Migrations completed successfully', 'Migrations'); + } catch (error) { + Logger.error('Failed to run migrations', error, 'Migrations'); + throw error; + } +} diff --git a/libs/domains/shared/backend/util-queue/src/lib/typeorm-options-for-role.spec.ts b/libs/domains/shared/backend/util-queue/src/lib/typeorm-options-for-role.spec.ts new file mode 100644 index 00000000..06123d15 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/typeorm-options-for-role.spec.ts @@ -0,0 +1,18 @@ +import { getTypeOrmOptionsForQueueRole } from './typeorm-options-for-role'; + +describe('getTypeOrmOptionsForQueueRole', () => { + const base = { + type: 'postgres' as const, + migrations: ['apps/example/migrations/*.ts'], + }; + + it('keeps migrations for api and all', () => { + expect(getTypeOrmOptionsForQueueRole(base, 'api').migrations).toEqual(base.migrations); + expect(getTypeOrmOptionsForQueueRole(base, 'all').migrations).toEqual(base.migrations); + }); + + it('clears migrations for worker and scheduler', () => { + expect(getTypeOrmOptionsForQueueRole(base, 'worker').migrations).toEqual([]); + expect(getTypeOrmOptionsForQueueRole(base, 'scheduler').migrations).toEqual([]); + }); +}); diff --git a/libs/domains/shared/backend/util-queue/src/lib/typeorm-options-for-role.ts b/libs/domains/shared/backend/util-queue/src/lib/typeorm-options-for-role.ts new file mode 100644 index 00000000..1bbd55d4 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/src/lib/typeorm-options-for-role.ts @@ -0,0 +1,21 @@ +import type { DataSourceOptions } from 'typeorm'; + +import type { QueueRole } from './queue-role'; +import { getQueueRole, shouldRunMigrations } from './queue-role'; + +/** + * Omits migration definitions for worker/scheduler roles so TypeORM does not load migration files on startup. + */ +export function getTypeOrmOptionsForQueueRole( + config: DataSourceOptions, + role: QueueRole = getQueueRole(), +): DataSourceOptions { + if (shouldRunMigrations(role)) { + return config; + } + + return { + ...config, + migrations: [], + }; +} diff --git a/libs/domains/shared/backend/util-queue/tsconfig.json b/libs/domains/shared/backend/util-queue/tsconfig.json new file mode 100644 index 00000000..9e5fde1c --- /dev/null +++ b/libs/domains/shared/backend/util-queue/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs" + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/libs/domains/shared/backend/util-queue/tsconfig.lib.json b/libs/domains/shared/backend/util-queue/tsconfig.lib.json new file mode 100644 index 00000000..841e950a --- /dev/null +++ b/libs/domains/shared/backend/util-queue/tsconfig.lib.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../../../dist/out-tsc", + "declaration": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"], + "exclude": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.spec.ts", + "src/**/*.test.ts" + ] +} diff --git a/libs/domains/shared/backend/util-queue/tsconfig.spec.json b/libs/domains/shared/backend/util-queue/tsconfig.spec.json new file mode 100644 index 00000000..c714b3c6 --- /dev/null +++ b/libs/domains/shared/backend/util-queue/tsconfig.spec.json @@ -0,0 +1,16 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../../../dist/out-tsc", + "module": "commonjs", + "moduleResolution": "node10", + "types": ["jest", "node"] + }, + "include": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "src/**/*.d.ts" + ] +} diff --git a/package-lock.json b/package-lock.json index 7046d1d2..ab0a48f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,10 +20,15 @@ "@angular/platform-server": "21.2.6", "@angular/router": "21.2.6", "@angular/ssr": "21.2.6", + "@bull-board/api": "7.1.5", + "@bull-board/express": "7.1.5", + "@bull-board/nestjs": "7.1.5", + "@bull-board/ui": "7.1.5", "@fontsource-variable/plus-jakarta-sans": "5.2.8", "@keycloakify/angular": "21.0.5", "@milkdown/crepe": "7.20.0", "@modelcontextprotocol/sdk": "1.18.2", + "@nestjs/bullmq": "11.0.4", "@nestjs/common": "11.1.6", "@nestjs/core": "11.1.6", "@nestjs/jwt": "11.0.2", @@ -47,12 +52,14 @@ "bcrypt": "6.0.0", "bootstrap": "5.3.8", "bootstrap-icons": "1.13.1", + "bullmq": "5.76.10", "class-transformer": "0.5.1", "cookieconsent": "3.1.1", "dockerode": "5.0.0", "express": "4.21.2", "gray-matter": "4.0.3", "i18n-iso-countries": "7.14.0", + "ioredis": "5.10.1", "keycloak-angular": "21.0.0", "keycloak-connect": "24.0.1", "keycloak-js": "26.2.1", @@ -125,7 +132,7 @@ "@swc-node/register": "1.11.1", "@swc/cli": "0.8.1", "@swc/core": "1.15.21", - "@swc/helpers": "0.5.17", + "@swc/helpers": "~0.5.18", "@types/dockerode": "3.3.45", "@types/express": "4.17.23", "@types/jest": "30.0.0", @@ -5080,6 +5087,350 @@ "devOptional": true, "license": "(Apache-2.0 AND BSD-3-Clause)" }, + "node_modules/@bull-board/api": { + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/@bull-board/api/-/api-7.1.5.tgz", + "integrity": "sha512-EW0sbTtGIysu9vipdVpPQeToPqOpPgVZTt+pn1Ut3gbSS/GLWbEgIfFtMmSQDUoSL9WH00RzjgUY5K+43nWh0A==", + "license": "MIT", + "dependencies": { + "redis-info": "^3.1.0" + }, + "peerDependencies": { + "@bull-board/ui": "7.1.5" + } + }, + "node_modules/@bull-board/express": { + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/@bull-board/express/-/express-7.1.5.tgz", + "integrity": "sha512-kp4SzhVjZlykryiQwcOhJjDhiLbBnZoAMoSgEstzqQ0raLw+jERRC6ryJ0MIQO+SO+Jv9EjjxrXCR8O2YSP/eg==", + "license": "MIT", + "dependencies": { + "@bull-board/api": "7.1.5", + "@bull-board/ui": "7.1.5", + "ejs": "^5.0.2", + "express": "^5.2.1" + } + }, + "node_modules/@bull-board/express/node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@bull-board/express/node_modules/body-parser": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", + "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.1", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@bull-board/express/node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/@bull-board/express/node_modules/ejs": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-5.0.2.tgz", + "integrity": "sha512-IpbUaI/CAW86l3f+T8zN0iggSc0LmMZLcIW5eRVStLVNCoTXkE0YlncbbH50fp8Cl6zHIky0sW2uUbhBqGw0Jw==", + "license": "Apache-2.0", + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.12.18" + } + }, + "node_modules/@bull-board/express/node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@bull-board/express/node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@bull-board/express/node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@bull-board/express/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@bull-board/express/node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@bull-board/express/node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@bull-board/express/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@bull-board/express/node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@bull-board/express/node_modules/qs": { + "version": "6.15.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", + "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/@bull-board/express/node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@bull-board/express/node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@bull-board/express/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@bull-board/express/node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@bull-board/express/node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@bull-board/nestjs": { + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/@bull-board/nestjs/-/nestjs-7.1.5.tgz", + "integrity": "sha512-1y+HkjnDaZoSCXJRsiYfBNBVx+PX3I8x3Uv+SSJuSpt2vHifMRwFbChO3XDxeWXetT1eR+yqPVq6ub5eJwNOYQ==", + "license": "MIT", + "peerDependencies": { + "@bull-board/api": "^7.1.5", + "@nestjs/bull-shared": "^10.0.0 || ^11.0.0", + "@nestjs/common": "^9.0.0 || ^10.0.0 || ^11.0.0", + "@nestjs/core": "^9.0.0 || ^10.0.0 || ^11.0.0", + "reflect-metadata": "^0.1.13 || ^0.2.0", + "rxjs": "^7.8.1" + } + }, + "node_modules/@bull-board/ui": { + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/@bull-board/ui/-/ui-7.1.5.tgz", + "integrity": "sha512-2IkatKwNRx/1M9/lAZIptcxS1FPNq6icpp2M46Upwd4olVxs/ujF9Kvs+Ff9ExtIO/OgYfwx7mG2IprGZ+nQCg==", + "license": "MIT", + "dependencies": { + "@bull-board/api": "7.1.5" + } + }, "node_modules/@chevrotain/cst-dts-gen": { "version": "11.0.3", "resolved": "https://registry.npmjs.org/@chevrotain/cst-dts-gen/-/cst-dts-gen-11.0.3.tgz", @@ -9455,7 +9806,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.5.1.tgz", "integrity": "sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw==", - "devOptional": true, "license": "MIT" }, "node_modules/@isaacs/cliui": { @@ -12865,7 +13215,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -12879,7 +13228,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -12893,7 +13241,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -12907,7 +13254,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -12921,7 +13267,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -12935,7 +13280,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -13278,6 +13622,34 @@ "@tybys/wasm-util": "^0.10.1" } }, + "node_modules/@nestjs/bull-shared": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-11.0.4.tgz", + "integrity": "sha512-VBJcDHSAzxQnpcDfA0kt9MTGUD1XZzfByV70su0W0eDCQ9aqIEBlzWRW21tv9FG9dIut22ysgDidshdjlnczLw==", + "license": "MIT", + "dependencies": { + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0" + } + }, + "node_modules/@nestjs/bullmq": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@nestjs/bullmq/-/bullmq-11.0.4.tgz", + "integrity": "sha512-wBzK9raAVG0/6NTMdvLGM4/FQ1lsB35/pYS8L6a0SDgkTiLpd7mAjQ8R692oMx5s7IjvgntaZOuTUrKYLNfIkA==", + "license": "MIT", + "dependencies": { + "@nestjs/bull-shared": "^11.0.4", + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0", + "bullmq": "^3.0.0 || ^4.0.0 || ^5.0.0" + } + }, "node_modules/@nestjs/common": { "version": "11.1.6", "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-11.1.6.tgz", @@ -21103,9 +21475,9 @@ "license": "Apache-2.0" }, "node_modules/@swc/helpers": { - "version": "0.5.17", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.17.tgz", - "integrity": "sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==", + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.21.tgz", + "integrity": "sha512-jI/VAmtdjB/RnI8GTnokyX7Ug8c+g+ffD6QRLa6XQewtnGyukKkKSk3wLTM3b5cjt1jNh9x0jfVlagdN2gDKQg==", "devOptional": true, "license": "Apache-2.0", "dependencies": { @@ -26373,6 +26745,44 @@ "node": ">=10.0.0" } }, + "node_modules/bullmq": { + "version": "5.76.10", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.76.10.tgz", + "integrity": "sha512-LWve7SpQjYSpCP2GEsWmoyzTz2H37L8HRmSTu3YihYsTOr5kJxrfEX6aEV7m6eskEMWXSHZYTMZepX6qNaH6CQ==", + "license": "MIT", + "dependencies": { + "cron-parser": "4.9.0", + "ioredis": "5.10.1", + "msgpackr": "2.0.1", + "node-abort-controller": "3.1.1", + "semver": "7.8.0", + "tslib": "2.8.1" + }, + "engines": { + "node": ">=12.22.0" + } + }, + "node_modules/bullmq/node_modules/msgpackr": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-2.0.1.tgz", + "integrity": "sha512-9J+tqTEsbHqY8YohazYgty7LgerFIWxvMLpUjqETSmjHojtJm2WnX2kK/2a1fLI7CO7ERP1YSEUXMucz4j+yBA==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/bullmq/node_modules/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/bundle-name": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", @@ -27181,7 +27591,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", - "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=0.10.0" @@ -27936,7 +28345,6 @@ "version": "4.9.0", "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", - "dev": true, "license": "MIT", "dependencies": { "luxon": "^3.2.1" @@ -29335,7 +29743,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", - "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=0.10" @@ -34106,7 +34513,6 @@ "version": "5.10.1", "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.1.tgz", "integrity": "sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==", - "devOptional": true, "license": "MIT", "dependencies": { "@ioredis/commands": "1.5.1", @@ -37261,7 +37667,6 @@ "version": "4.18.1", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", - "dev": true, "license": "MIT" }, "node_modules/lodash-es": { @@ -37286,7 +37691,6 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", - "devOptional": true, "license": "MIT" }, "node_modules/lodash.get": { @@ -37307,7 +37711,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", - "devOptional": true, "license": "MIT" }, "node_modules/lodash.isboolean": { @@ -37654,7 +38057,6 @@ "version": "3.7.2", "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -39252,7 +39654,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", - "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -40531,7 +40932,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", - "dev": true, "license": "MIT" }, "node_modules/node-addon-api": { @@ -40680,7 +41080,6 @@ "version": "5.2.2", "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -45637,17 +46036,24 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", - "devOptional": true, "license": "MIT", "engines": { "node": ">=4" } }, + "node_modules/redis-info": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/redis-info/-/redis-info-3.1.0.tgz", + "integrity": "sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg==", + "license": "MIT", + "dependencies": { + "lodash": "^4.17.11" + } + }, "node_modules/redis-parser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", - "devOptional": true, "license": "MIT", "dependencies": { "redis-errors": "^1.0.0" @@ -48429,7 +48835,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", - "devOptional": true, "license": "MIT" }, "node_modules/statuses": { diff --git a/package.json b/package.json index f6bf4178..18ee3649 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "@swc-node/register": "1.11.1", "@swc/cli": "0.8.1", "@swc/core": "1.15.21", - "@swc/helpers": "0.5.17", + "@swc/helpers": "~0.5.18", "@types/dockerode": "3.3.45", "@types/express": "4.17.23", "@types/jest": "30.0.0", @@ -104,10 +104,15 @@ "@angular/platform-server": "21.2.6", "@angular/router": "21.2.6", "@angular/ssr": "21.2.6", + "@bull-board/api": "7.1.5", + "@bull-board/express": "7.1.5", + "@bull-board/nestjs": "7.1.5", + "@bull-board/ui": "7.1.5", "@fontsource-variable/plus-jakarta-sans": "5.2.8", "@keycloakify/angular": "21.0.5", "@milkdown/crepe": "7.20.0", "@modelcontextprotocol/sdk": "1.18.2", + "@nestjs/bullmq": "11.0.4", "@nestjs/common": "11.1.6", "@nestjs/core": "11.1.6", "@nestjs/jwt": "11.0.2", @@ -131,12 +136,14 @@ "bcrypt": "6.0.0", "bootstrap": "5.3.8", "bootstrap-icons": "1.13.1", + "bullmq": "5.76.10", "class-transformer": "0.5.1", "cookieconsent": "3.1.1", "dockerode": "5.0.0", "express": "4.21.2", "gray-matter": "4.0.3", "i18n-iso-countries": "7.14.0", + "ioredis": "5.10.1", "keycloak-angular": "21.0.0", "keycloak-connect": "24.0.1", "keycloak-js": "26.2.1",