diff --git a/server/storageReclamation.ts b/server/storageReclamation.ts index e21796e3f..8a3950150 100644 --- a/server/storageReclamation.ts +++ b/server/storageReclamation.ts @@ -1,10 +1,13 @@ +import { readFile } from 'node:fs/promises'; import { statfs } from 'node:fs/promises'; +import { join } from 'node:path'; import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js'; import { logger } from '../utility/logging/logger.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import * as envMgr from '../utility/environment/environmentManager.ts'; import { convertToMS } from '../utility/common_utils.ts'; envMgr.initSync(); + const reclamationHandlers = new Map< string, { priority: number; handler: (priority: number) => Promise | void }[] @@ -12,6 +15,33 @@ const reclamationHandlers = new Map< const RECLAMATION_THRESHOLD = envMgr.get(CONFIG_PARAMS.STORAGE_RECLAMATION_THRESHOLD) ?? 0.4; // 40% remaining free space is the default const RECLAMATION_INTERVAL = convertToMS(envMgr.get(CONFIG_PARAMS.STORAGE_RECLAMATION_INTERVAL)) || 3600000; // 1 hour is the default + +// Written by host-manager every ~90s alongside the instance's hdb root +const QUOTA_STATUS_FILE = 'quota-status.json'; +// Use statfs fallback if the file is older than this (host-manager outage, container start race, etc.) +const QUOTA_STATUS_MAX_AGE_MS = 5 * 60 * 1000; + +export type QuotaStatusData = { + usedBytes: number; + quotaBytes?: number; + updatedAt: number; +}; + +/** + * Reads the quota-status.json file written by host-manager. + * Returns undefined if the file is absent or malformed; does not apply age filtering. + */ +export async function getQuotaStatus(): Promise { + const rootPath = envMgr.get(CONFIG_PARAMS.ROOTPATH); + if (!rootPath) return undefined; + try { + const raw = await readFile(join(rootPath, QUOTA_STATUS_FILE), 'utf8'); + return JSON.parse(raw) as QuotaStatusData; + } catch { + return undefined; + } +} + /** * Register a handler to be called when storage free space is low and reclamation is needed. The callback is called * with the priority of the reclamation, which is the ratio of the threshold to the available space ratio. If space is @@ -39,7 +69,14 @@ export function onStorageReclamation( } } let reclamationTimer: NodeJS.Timeout; + +// If a fresh quota-status.json exists (written by host-manager every ~90s), use quota-based ratio. +// Otherwise fall back to statfs for the registered path. const defaultGetAvailableSpaceRatio = async (path: string): Promise => { + const status = await getQuotaStatus(); + if (status?.quotaBytes && Date.now() - status.updatedAt < QUOTA_STATUS_MAX_AGE_MS) { + return Math.max(0, status.quotaBytes - status.usedBytes) / status.quotaBytes; + } const fsStats = await statfs(path); return fsStats.bavail / fsStats.blocks; }; diff --git a/unitTests/server/storageReclamation.test.js b/unitTests/server/storageReclamation.test.js index 2373ac11b..d0356b6b3 100644 --- a/unitTests/server/storageReclamation.test.js +++ b/unitTests/server/storageReclamation.test.js @@ -1,6 +1,9 @@ 'use strict'; const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); const sinon = require('sinon'); const rewire = require('rewire'); @@ -393,4 +396,110 @@ describe('storageReclamation module', function () { assert.ok(okPathHandler.calledOnce); }); }); + + describe('quota mode', function () { + const QUOTA_100GB = 100 * 1024 * 1024 * 1024; + let tmpDir; + let quotaStatusPath; + let originalRootPath; + + beforeEach(function () { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'harper-quota-test-')); + quotaStatusPath = path.join(tmpDir, 'quota-status.json'); + originalRootPath = env.get('rootPath'); + env.setProperty('rootPath', tmpDir); + }); + + afterEach(function () { + env.setProperty('rootPath', originalRootPath); + try { + fs.rmSync(tmpDir, { recursive: true }); + } catch {} + }); + + describe('getQuotaStatus', function () { + it('returns parsed object when file is present and valid', async function () { + const data = { usedBytes: 50_000_000_000, quotaBytes: QUOTA_100GB, updatedAt: Date.now() }; + fs.writeFileSync(quotaStatusPath, JSON.stringify(data)); + assert.deepEqual(await storageReclamation.getQuotaStatus(), data); + }); + + it('returns undefined when file is absent', async function () { + assert.equal(await storageReclamation.getQuotaStatus(), undefined); + }); + + it('returns undefined when file contains malformed JSON', async function () { + fs.writeFileSync(quotaStatusPath, 'not-valid-json{'); + assert.equal(await storageReclamation.getQuotaStatus(), undefined); + }); + }); + + describe('defaultGetAvailableSpaceRatio', function () { + beforeEach(function () { + storageReclamation.setAvailableSpaceRatioGetter(undefined); // use real default + }); + + it('uses fresh quota-status file and triggers reclamation when headroom is low', async function () { + const usedBytes = 65 * 1024 * 1024 * 1024; // 65 GB → 35% remaining → below 40% threshold + fs.writeFileSync( + quotaStatusPath, + JSON.stringify({ usedBytes, quotaBytes: QUOTA_100GB, updatedAt: Date.now() }) + ); + + const handler = sandbox.stub().returns(Promise.resolve()); + storageReclamation.onStorageReclamation(tmpDir, handler, true); + await storageReclamation.runReclamationHandlers(); + + assert.ok(handler.calledOnce); + assert.ok(handler.firstCall.args[0] > 1); // priority = 0.4 / 0.35 ≈ 1.14 + }); + + it('uses fresh quota-status file and does not trigger when headroom is sufficient', async function () { + const usedBytes = 50 * 1024 * 1024 * 1024; // 50% used → 50% remaining → above threshold + fs.writeFileSync( + quotaStatusPath, + JSON.stringify({ usedBytes, quotaBytes: QUOTA_100GB, updatedAt: Date.now() }) + ); + + const handler = sandbox.stub(); + storageReclamation.onStorageReclamation(tmpDir, handler, true); + await storageReclamation.runReclamationHandlers(); + + assert.ok(handler.notCalled); + }); + + it('clamps ratio to 0 and triggers reclamation when usage exceeds quota', async function () { + const usedBytes = 110 * 1024 * 1024 * 1024; // 10 GB over quota + fs.writeFileSync( + quotaStatusPath, + JSON.stringify({ usedBytes, quotaBytes: QUOTA_100GB, updatedAt: Date.now() }) + ); + + const handler = sandbox.stub().returns(Promise.resolve()); + storageReclamation.onStorageReclamation(tmpDir, handler, true); + await storageReclamation.runReclamationHandlers(); + + // Ratio clamped to 0 → priority = Infinity → handler called + assert.ok(handler.calledOnce); + }); + + it('falls back to statfs when quota-status file is absent', async function () { + const handler = sandbox.stub(); + storageReclamation.onStorageReclamation(tmpDir, handler, true); + await assert.doesNotReject(storageReclamation.runReclamationHandlers()); + }); + + it('falls back to statfs when quota-status file is stale', async function () { + const staleTimestamp = Date.now() - 10 * 60 * 1000; // 10 minutes old + fs.writeFileSync( + quotaStatusPath, + JSON.stringify({ usedBytes: 65 * 1024 * 1024 * 1024, quotaBytes: QUOTA_100GB, updatedAt: staleTimestamp }) + ); + + const handler = sandbox.stub(); + storageReclamation.onStorageReclamation(tmpDir, handler, true); + await assert.doesNotReject(storageReclamation.runReclamationHandlers()); + }); + }); + }); }); diff --git a/unitTests/utility/environment/systemInformation.test.js b/unitTests/utility/environment/systemInformation.test.js index fe8c16b23..5a65e291f 100644 --- a/unitTests/utility/environment/systemInformation.test.js +++ b/unitTests/utility/environment/systemInformation.test.js @@ -176,7 +176,7 @@ const EXPECTED_PROPERTIES = { 'reclaimable', 'writeback', ], - disk: ['io', 'read_write', 'size'], + disk: ['free_space_basis', 'io', 'read_write', 'size'], disk_io: ['rIO', 'wIO', 'tIO'], disk_read_write: ['rx', 'wx', 'tx'], disk_size: ['fs', 'rw', 'type', 'size', 'used', 'use', 'mount', 'available'], @@ -285,7 +285,7 @@ describe('test systemInformation module', () => { env_mgr.setProperty('operationsApi_sysInfo_disk', false); results = await system_information.getDiskInfo(); - assert.deepEqual(results, {}); + assert.deepEqual(Object.keys(results).sort(), ['free_space_basis']); }); it('test getNetworkInfo function', async () => { diff --git a/utility/environment/systemInformation.ts b/utility/environment/systemInformation.ts index 1070a0997..657a8daa9 100644 --- a/utility/environment/systemInformation.ts +++ b/utility/environment/systemInformation.ts @@ -3,6 +3,7 @@ import path from 'node:path'; import si from 'systeminformation'; import logger from '../logging/harper_logger.ts'; import * as hdbTerms from '../hdbTerms.ts'; +import { getQuotaStatus } from '../../server/storageReclamation.ts'; import { lmdbGetTableSize } from '../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.ts'; import { getThreadInfo } from '../../server/threads/manageThreads.js'; import * as env from './environmentManager.ts'; @@ -264,6 +265,10 @@ type DiskInfo = { io?: Pick; read_write?: Pick; size?: si.Systeminformation.FsSizeData[]; + free_space_basis?: 'quota' | 'filesystem'; + quota_size_bytes?: number; + quota_used_bytes?: number; + quota_status_age_seconds?: number; }; /** @@ -272,6 +277,15 @@ type DiskInfo = { */ export async function getDiskInfo(): Promise { const disk: DiskInfo = {}; + const quotaStatus = await getQuotaStatus(); + if (quotaStatus?.quotaBytes) { + disk.free_space_basis = 'quota'; + disk.quota_size_bytes = quotaStatus.quotaBytes; + disk.quota_used_bytes = quotaStatus.usedBytes; + disk.quota_status_age_seconds = Math.floor((Date.now() - quotaStatus.updatedAt) / 1000); + } else { + disk.free_space_basis = 'filesystem'; + } try { if (!env.get(hdbTerms.CONFIG_PARAMS.OPERATIONSAPI_SYSINFO_DISK)) return disk;