diff --git a/config-root.schema.json b/config-root.schema.json index 4e603f659..73c2c5226 100644 --- a/config-root.schema.json +++ b/config-root.schema.json @@ -577,6 +577,16 @@ "type": "number", "description": "Number of aggregation cycles between node storage measurements. 0 to disable. Default: 10" }, + "rawRetentionMs": { + "type": "number", + "minimum": 0, + "description": "Milliseconds to retain raw (pre-aggregation) analytics. Clamped to at least aggregatePeriod to prevent data loss. Default: 3600000 (1 hour)" + }, + "aggregateRetentionMs": { + "type": "number", + "minimum": 0, + "description": "Milliseconds to retain aggregated analytics. 0 disables cleanup (keep forever). Default: 31536000000 (1 year)" + }, "logging": { "$ref": "#/definitions/loggerConfig" } } } diff --git a/resources/analytics/write.ts b/resources/analytics/write.ts index 6fc509a6f..cad92fc14 100644 --- a/resources/analytics/write.ts +++ b/resources/analytics/write.ts @@ -715,11 +715,16 @@ function startScheduledTasks() { nodeStorageInterval = envGet(CONFIG_PARAMS.ANALYTICS_STORAGEINTERVAL) ?? DEFAULT_STORAGE_INTERVAL; const AGGREGATE_PERIOD = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) * 1000; if (AGGREGATE_PERIOD) { + // Clamp raw retention to at least one full aggregation period so raw records + // are never deleted before they can be rolled up. + const rawRetentionMs = Math.max(envGet(CONFIG_PARAMS.ANALYTICS_RAWRETENTIONMS) ?? RAW_EXPIRATION, AGGREGATE_PERIOD); + const aggregateRetentionMs = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATERETENTIONMS) ?? AGGREGATE_EXPIRATION; setInterval( async () => { await aggregation(analyticsDelay, AGGREGATE_PERIOD); - await cleanup(getRawAnalyticsTable(), RAW_EXPIRATION); - await cleanup(getAnalyticsTable(), AGGREGATE_EXPIRATION); + await cleanup(getRawAnalyticsTable(), rawRetentionMs); + // 0 means "keep forever" — skip aggregate cleanup, matching storageInterval: 0 convention + if (aggregateRetentionMs) await cleanup(getAnalyticsTable(), aggregateRetentionMs); }, Math.min(AGGREGATE_PERIOD / 2, 0x7fffffff) ).unref(); diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 26f2b9e55..b7ce6bf0b 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -426,6 +426,8 @@ export const LEGACY_CONFIG_PARAMS = { */ export const CONFIG_PARAMS = { ANALYTICS_AGGREGATEPERIOD: 'analytics_aggregatePeriod', + ANALYTICS_AGGREGATERETENTIONMS: 'analytics_aggregateRetentionMs', + ANALYTICS_RAWRETENTIONMS: 'analytics_rawRetentionMs', ANALYTICS_REPLICATE: 'analytics_replicate', ANALYTICS_STORAGEINTERVAL: 'analytics_storageInterval', AUTHENTICATION_AUTHORIZELOCAL: 'authentication_authorizeLocal',