Read-only mode

bin/run.js

@@ -24,6 +24,7 @@ const minimist = require('minimist');
 const keys = require('../security/keys.js');
 const { startHTTPThreads } = require('../server/threads/socketRouter.ts');
 const hdbInfoController = require('../dataLayer/hdbInfoController.js');
+const { isReadOnlyMode } = require('../resources/databases.ts');
 const hdbTerms = require('../utility/hdbTerms.ts');
 const { getHdbPid, isProcessRunning } = require('../utility/processManagement/processManagement.js');
 const { PACKAGE_ROOT } = require('../utility/packageUtils');
@@ -75,6 +76,12 @@ async function initialize(calledByInstall = false, calledByMain = false) {
 	// Check to see if HDB is installed, if it isn't we call install.
 	console.log(chalk.magenta('Starting Harper...'));
 
+	// Display read-only mode warning early, before database initialization
+	if (isReadOnlyMode()) {
+		console.log(chalk.yellow('\n*** RUNNING IN READ-ONLY MODE ***'));
+		console.log(chalk.yellow('Database writes are disabled. Analytics collection is disabled.\n'));
+	}
+
 	addUnhandleRejectionListener();
 
 	hdbLogger.suppressLogging?.(() => {
@@ -255,6 +262,10 @@ function startupLog(portResolutions) {
 	const pad = (param) => param.padEnd(padding);
 	let logMsg = '\n';
 
+	if (isReadOnlyMode()) {
+		logMsg += `${pad('Mode:')}${chalk.yellow('READ-ONLY')}\n`;
+	}
+
 	logMsg += `${pad('Hostname:')}${env.get(CONFIG_PARAMS.NODE_HOSTNAME)}\n`;
 
 	logMsg += `${pad('Worker Threads:')}${env.get(CONFIG_PARAMS.THREADS_COUNT)}\n`;

index.ts

@@ -8,6 +8,7 @@ if (!workerThreads.isMainThread) {
 
 // Regular exports (don't require the same initialization as the globals at the end of this file do)
 export { RequestTarget } from './resources/RequestTarget.ts';
+export { flushDatabases } from './resources/databases.ts';
 export { getContext, getResponse, getUser } from './security/jsLoader.ts';
 
 // Type only exports.

resources/analytics/write.ts

@@ -1,6 +1,6 @@
 import { parentPort, threadId } from 'worker_threads';
 import { onMessageByType } from '../../server/threads/manageThreads.js';
-import { getDatabases, table } from '../databases.ts';
+import { getDatabases, table, isReadOnlyMode } from '../databases.ts';
 import type { Databases, Table, Tables } from '../databases.ts';
 import harperLogger from '../../utility/logging/harper_logger.js';
 import { stat, readdir } from 'node:fs/promises';
@@ -37,8 +37,20 @@ interface Action {
 
 let activeActions = new Map<string, Action>();
 let analyticsEnabled = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) > -1;
+let analyticsReadOnlyChecked = false;
 let sendAnalyticsTimeout: NodeJS.Timeout;
 
+// Check read-only mode lazily to avoid circular dependency at module load time
+function checkAnalyticsEnabled(): boolean {
+	if (!analyticsReadOnlyChecked) {
+		analyticsReadOnlyChecked = true;
+		if (isReadOnlyMode()) {
+			analyticsEnabled = false;
+		}
+	}
+	return analyticsEnabled;
+}
+
 export function setAnalyticsEnabled(enabled: boolean) {
 	analyticsEnabled = enabled;
 	clearTimeout(sendAnalyticsTimeout); // reset this
@@ -101,7 +113,7 @@ function recordNewAction(key: string, value: Value, metric?: string, path?: stri
  * @param type
  */
 export function recordAction(value: Value, metric: string, path?: string, method?: string, type?: string) {
-	if (!analyticsEnabled) return;
+	if (!checkAnalyticsEnabled()) return;
 	// TODO: May want to consider nested paths, as they may yield faster hashing of (fixed) strings that hashing concatenated strings
 	let key = metric + (path ? '-' + path : '');
 	if (method !== undefined) key += '-' + method;
@@ -214,6 +226,8 @@ function sendAnalytics() {
 }
 
 export async function recordHostname() {
+	// Skip writes in read-only mode
+	if (isReadOnlyMode()) return;
 	const hostname = server.hostname;
 	log.trace?.('recordHostname server.hostname:', hostname);
 	const nodeId = stableNodeId(hostname);
@@ -244,6 +258,8 @@ function getHostNodeId(hostname: string) {
 }
 
 function storeMetric(table: Table, metric: Metric) {
+	// Skip writes in read-only mode
+	if (isReadOnlyMode()) return;
 	const nodeId = getHostNodeId(server.hostname);
 	const metricValue = {
 		id: [getNextMonotonicTime(), nodeId],
@@ -622,6 +638,8 @@ let lastResourceUsage: ResourceUsage = {
 const rest = () => new Promise(setImmediate);
 
 async function cleanup(AnalyticsTable, expiration) {
+	// Skip writes in read-only mode
+	if (isReadOnlyMode()) return;
 	const end = Date.now() - expiration;
 	for (const key of AnalyticsTable.primaryStore.getKeys({ start: false, end })) {
 		AnalyticsTable.primaryStore.remove(key);
@@ -709,6 +727,8 @@ let totalBytesProcessed = 0;
 const lastUtilizations = new Map();
 const LOG_ANALYTICS = false; // TODO: Make this a config option if we really want this
 function recordAnalytics(message, worker?) {
+	// Skip writes in read-only mode
+	if (isReadOnlyMode()) return;
 	const report = message.report;
 	report.threadId = worker?.threadId || threadId;
 	// Add system information stats as well

resources/auditStore.ts

@@ -11,6 +11,7 @@ import { decodeFromDatabase } from './blob.ts';
 import { onStorageReclamation } from '../server/storageReclamation.ts';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
 import { RocksTransactionLogStore } from './RocksTransactionLogStore.ts';
+import { isReadOnlyMode } from './databases.ts';
 
 /**
  * This module is responsible for the binary representation of audit records in an efficient form.
@@ -145,6 +146,8 @@ export function openAuditStore(rootStore) {
 		}
 	});
 	function scheduleAuditCleanup(newCleanupDelay?: number): Promise<void> {
+		// Skip audit cleanup/purge in read-only mode
+		if (isReadOnlyMode()) return;
 		if (auditStore instanceof RocksTransactionLogStore) {
 			auditStore.rootStore.purgeLogs({
 				before: Date.now() - auditRetention / (1 + cleanupPriority * cleanupPriority),

resources/databases.ts

@@ -33,6 +33,36 @@ import { RocksIndexStore } from './RocksIndexStore.ts';
 import { when } from '../utility/when.ts';
 import { isProcessRunning } from '../utility/processManagement/processManagement.js';
 
+/**
+ * Check if Harper is running in read-only mode.
+ * Read-only mode can be enabled via:
+ * - HARPER_READONLY environment variable (truthy value)
+ * - --readonly CLI flag
+ * - storage.readOnly config setting
+ */
+let _isReadOnlyMode: boolean | undefined;
+export function isReadOnlyMode(): boolean {
+	if (_isReadOnlyMode !== undefined) return _isReadOnlyMode;
+	// Check environment variable
+	const envReadOnly = process.env.HARPER_READONLY;
+	if (envReadOnly && envReadOnly !== '0' && envReadOnly !== 'false') {
+		_isReadOnlyMode = true;
+		return true;
+	}
+	// Check CLI flag (simple argv check)
+	if (process.argv.includes('--readonly') || process.argv.includes('--read-only')) {
+		_isReadOnlyMode = true;
+		return true;
+	}
+	// Check config setting
+	if (envGet(CONFIG_PARAMS.STORAGE_READONLY)) {
+		_isReadOnlyMode = true;
+		return true;
+	}
+	_isReadOnlyMode = false;
+	return false;
+}
+
 function createOpenDBIObject(dupSort = false, isPrimary = false) {
 	return new OpenDBIObject(dupSort, isPrimary);
 }
@@ -114,8 +144,16 @@ const MEMORY_FOR_ROCKS_DB = Math.min(process.constrainedMemory?.() ?? Infinity,
 
 function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSort?: boolean }) {
 	options.disableWAL ??= true;
+	// Apply read-only mode if enabled
+	if (isReadOnlyMode()) {
+		options.readOnly = true;
+	}
 	RocksDatabase.config({ blockCacheSize: MEMORY_FOR_ROCKS_DB });
 	if (!existsSync(path)) {
+		// Don't create directories in read-only mode
+		if (isReadOnlyMode()) {
+			throw new Error(`Database cannot be created in read-only mode: ${path}`);
+		}
 		mkdirSync(path, { recursive: true });
 	}
 	let db: RocksRootDatabase;
@@ -125,8 +163,8 @@ function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSo
 		db = RocksDatabase.open(path, options) as RocksDatabaseEx;
 		// the RocksDB put and remove return promises, which masks thrown errors in non-awaiting calls to put/remove,
 		// making them unsafe to replace LMDB methods, which will synchronously throw errors if there is a problem
-		db.put = db.putSync;
-		db.remove = db.removeSync;
+		db.put = db.putSync as typeof db.put;
+		db.remove = db.removeSync as typeof db.remove;
 		db.encoder.name = options.name;
 	}
 	db.env = {};
@@ -335,7 +373,7 @@ export function readMetaDb(
 	auditPath?: string,
 	isLegacy?: boolean
 ) {
-	const envInit = new OpenEnvironmentObject(path, false);
+	const envInit = new OpenEnvironmentObject(path, isReadOnlyMode());
 	try {
 		let rootStore = lmdbDatabaseEnvs.get(path);
 		if (rootStore) {
@@ -370,7 +408,10 @@ function readRocksMetaDb(path: string, defaultTable?: string, databaseName: stri
 			rootStore = openRocksDatabase(path, { disableWAL: false, enableStats: true }) as RocksDatabaseEx;
 			rocksdbDatabaseEnvs.set(path, rootStore);
 			initStores(path, rootStore, databaseName, defaultTable);
-			replayLogs(rootStore, databases[databaseName]);
+			// Skip transaction log replay in read-only mode
+			if (!isReadOnlyMode()) {
+				replayLogs(rootStore, databases[databaseName]);
+			}
 		}
 		return rootStore;
 	} catch (error) {
@@ -387,7 +428,7 @@ function initStores(
 	auditPath?: string,
 	isLegacy?: boolean
 ) {
-	const envInit = new OpenEnvironmentObject(path, false);
+	const envInit = new OpenEnvironmentObject(path, isReadOnlyMode());
 	const internalDbiInit = createOpenDBIObject(false);
 	let attributesDbi = rootStore.dbisDb;
 	if (!attributesDbi) {
@@ -731,7 +772,7 @@ export function database({ database: databaseName, table: tableName }) {
 		rootStore = lmdbDatabaseEnvs.get(path);
 		if (!rootStore || rootStore.status === 'closed') {
 			// TODO: validate database name
-			const envInit = new OpenEnvironmentObject(path, false);
+			const envInit = new OpenEnvironmentObject(path, isReadOnlyMode());
 			rootStore = open(envInit);
 			lmdbDatabaseEnvs.set(path, rootStore);
 		}
@@ -1312,3 +1353,11 @@ export function getDefaultCompression() {
 	if (STORAGE_COMPRESSION_THRESHOLD) LMDB_COMPRESSION_OPTS['threshold'] = STORAGE_COMPRESSION_THRESHOLD;
 	return LMDB_COMPRESSION && LMDB_COMPRESSION_OPTS;
 }
+
+/**
+ * Force all RocksDB databases to flush to disk.
+ */
+export async function flushDatabases() {
+	// flush all RocksDB databases
+	return Promise.all(Array.from(rocksdbDatabaseEnvs.values()).map((db) => db.flush()));
+}

unitTests/resources/databases.test.js

@@ -0,0 +1,21 @@
+require('../testUtils');
+const assert = require('assert');
+const { setupTestDBPath } = require('../testUtils');
+const { table, flushDatabases } = require('#src/resources/databases');
+const { setMainIsWorker } = require('#js/server/threads/manageThreads');
+
+describe('flushDatabases', () => {
+	before(async function () {
+		setupTestDBPath();
+		setMainIsWorker(true);
+		table({
+			table: 'FlushTest',
+			database: 'test',
+			attributes: [{ name: 'id', isPrimaryKey: true }],
+		});
+	});
+
+	it('flushes all databases without error', async function () {
+		await assert.doesNotReject(() => flushDatabases());
+	});
+});

utility/hdbTerms.ts

@@ -578,6 +578,7 @@ export const CONFIG_PARAMS = {
 	STORAGE_RECLAMATION_INTERVAL: 'storage_reclamation_interval',
 	STORAGE_RECLAMATION_EVICTIONFACTOR: 'storage_reclamation_evictionFactor',
 	STORAGE_ENGINE: 'storage_engine',
+	STORAGE_READONLY: 'storage_readOnly',
 	DATABASES: 'databases',
 	IGNORE_SCRIPTS: 'ignoreScripts',
 	MQTT_NETWORK_PORT: 'mqtt_network_port',