diff --git a/.claude/scheduled_tasks.lock b/.claude/scheduled_tasks.lock new file mode 100644 index 000000000..910d70b00 --- /dev/null +++ b/.claude/scheduled_tasks.lock @@ -0,0 +1 @@ +{"sessionId":"5206baae-267d-4d02-bedd-55b8d6b8cf05","pid":3219805,"procStart":"41160224","acquiredAt":1778799452286} \ No newline at end of file diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index ef5a0f174..f18e96564 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -1,17 +1,19 @@ 'use strict'; -import * as envMgr from '../utility/environment/environmentManager.ts'; -envMgr.initSync(); import * as terms from '../utility/hdbTerms.ts'; import { httpRequest } from '../utility/common_utils.ts'; import * as path from 'path'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as YAML from 'yaml'; import { packageDirectory } from '../components/packageComponent.ts'; import { encode } from 'cbor-x'; -import { getHdbPid } from '../utility/processManagement/processManagement.js'; -import { initConfig, getConfigPath } from '../config/configUtils.js'; - +import { getHdbPid } from '../utility/processManagement/processManagement.ts'; +import { initConfig, getConfigPath } from '../config/configUtils.ts'; +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const OP_ALIASES = { deploy: 'deploy_component', package: 'package_component' }; export { cliOperations, buildRequest }; diff --git a/bin/copyDb.ts b/bin/copyDb.ts index 2166e08be..dca9061a7 100644 --- a/bin/copyDb.ts +++ b/bin/copyDb.ts @@ -1,7 +1,8 @@ import { getDatabases, getDefaultCompression, resetDatabases } from '../resources/databases.ts'; import { open, asBinary } from 'lmdb'; import { join } from 'path'; -import { move, remove } from 'fs-extra'; +import _fs_extra from 'fs-extra'; +const { move, remove } = _fs_extra; import { existsSync, mkdirSync } from 'node:fs'; import { get } from '../utility/environment/environmentManager.ts'; import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.ts'; @@ -10,7 +11,7 @@ import { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } from '../utility/lmdb/terms.ts'; import { CONFIG_PARAMS, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts'; import { AUDIT_STORE_OPTIONS } from '../resources/auditStore.ts'; import { describeSchema } from '../dataLayer/schemaDescribe.ts'; -import { updateConfigValue } from '../config/configUtils.js'; +import { updateConfigValue } from '../config/configUtils.ts'; import * as hdbLogger from '../utility/logging/harper_logger.ts'; import { RocksDatabase, type RocksDatabaseOptions } from '@harperfast/rocksdb-js'; import { RocksIndexStore } from '../resources/RocksIndexStore.ts'; diff --git a/bin/harper.ts b/bin/harper.ts index 3f06a4d0f..479331631 100644 --- a/bin/harper.ts +++ b/bin/harper.ts @@ -4,7 +4,6 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import logger from '../utility/logging/harper_logger.ts'; -import * as cliOperations from './cliOperations.ts'; import { packageJson } from '../utility/packageUtils.js'; import checkNode from '../launchServiceScripts/utility/checkNodeVersion.js'; import * as hdbTerms from '../utility/hdbTerms.ts'; @@ -38,6 +37,36 @@ version - Print the version deploy - Deploy the application locally or remotely with target= `; +/** + * Initialize the environment manager. Call before dynamically importing the + * subcommand module so any module-load reads of `env.get(…)` see a populated + * configuration. Side-effectful initialization is deferred to `onStartup(…)` + * hooks; call `runServerStartup()` once the subcommand module has finished + * loading to drain them. + */ +async function initEnv() { + const env = await import('../utility/environment/environmentManager.ts'); + env.initSync(); +} + +async function runServerStartup() { + const lifecycle = await import('../utility/lifecycle.ts'); + await lifecycle.runStartup(); +} + +/** + * Resolve a default-exported function from a dynamically-imported module that + * may have been loaded from a .ts source (ESM: `mod.default` is the function) + * or a tsc-compiled .js (CJS-wrapped: `mod.default` is the CJS exports object + * and `mod.default.default` is the function). + */ +function getDefaultExport(mod: any): any { + if (typeof mod === 'function') return mod; + if (typeof mod.default === 'function') return mod.default; + if (typeof mod.default?.default === 'function') return mod.default.default; + return mod.default ?? mod; +} + async function harper() { let nodeResults = checkNode(); @@ -61,34 +90,41 @@ async function harper() { switch (service) { case SERVICE_ACTIONS_ENUM.HELP: return HELP; - case SERVICE_ACTIONS_ENUM.START: - return require('./run').launch(); - case SERVICE_ACTIONS_ENUM.INSTALL: - return (require('./install').default || require('./install'))(); - case SERVICE_ACTIONS_ENUM.STOP: - return (require('./stop').default || require('./stop'))().then(() => { + case SERVICE_ACTIONS_ENUM.START: { + await initEnv(); + const mod = await import('./run.ts'); + // runStartup() is drained inside run.ts main() after initialize() + // completes, so server.X singletons and auth's table() resolution + // see a populated env / installed hdbBasePath. + return mod.launch(); + } + case SERVICE_ACTIONS_ENUM.INSTALL: { + return getDefaultExport(await import('./install.ts'))(); + } + case SERVICE_ACTIONS_ENUM.STOP: { + await initEnv(); + return getDefaultExport(await import('./stop.ts'))().then(() => { process.exit(0); }); + } case SERVICE_ACTIONS_ENUM.RESTART: - return require('./restart').restart({}); + return (await import('./restart.ts')).restart({}); case SERVICE_ACTIONS_ENUM.VERSION: return packageJson.version; case SERVICE_ACTIONS_ENUM.UPGRADE: logger.setLogLevel(hdbTerms.LOG_LEVELS.INFO); - // The require is here to better control the flow of imports when this module is called. - return require('./upgrade.js') - .upgrade(null) - .then(() => 'Your instance of Harper is up to date!'); - case SERVICE_ACTIONS_ENUM.STATUS: - return (require('./status').default || require('./status'))(); + return (await import('./upgrade.ts')).upgrade(null).then(() => 'Your instance of Harper is up to date!'); + case SERVICE_ACTIONS_ENUM.STATUS: { + return getDefaultExport(await import('./status.ts'))(); + } case SERVICE_ACTIONS_ENUM.RENEWCERTS: - return require('../security/keys') + return (await import('../security/keys.ts')) .renewSelfSigned() .then(() => 'Successfully renewed self-signed certificates'); case SERVICE_ACTIONS_ENUM.COPYDB: { let sourceDb = process.argv[3]; let targetDbPath = process.argv[4]; - return require('./copyDb').copyDb(sourceDb, targetDbPath); + return (await import('./copyDb.ts')).copyDb(sourceDb, targetDbPath); } case SERVICE_ACTIONS_ENUM.DEV: process.env.DEV_MODE = 'true'; @@ -127,17 +163,27 @@ async function harper() { } } // fall through - case undefined: // run harperdb in the foreground in standard mode - return require('./run').main(); - default: + case undefined: { + // run harperdb in the foreground in standard mode. + // runStartup() is drained inside run.ts main() after initialize(). + await initEnv(); + const mod = await import('./run.ts'); + return mod.main(); + } + default: { + const cliOperations = await import('./cliOperations.ts'); const cliApiOp = cliOperations.buildRequest(); logger.trace('calling cli operations with:', cliApiOp); await cliOperations.cliOperations(cliApiOp); return; + } } } export { harper }; -if (require.main === module) { +// In CJS (dist), `module` is defined and we check the entry. In ESM (typestrip), +// `module` is undefined; this file is only run directly as a CLI, so treat as main. +const isEntry = typeof module === 'undefined' || require.main === module; +if (isEntry) { harper() .then((message) => { if (message) { diff --git a/bin/restart.ts b/bin/restart.ts index f266f430f..154689c39 100644 --- a/bin/restart.ts +++ b/bin/restart.ts @@ -4,17 +4,20 @@ import minimist from 'minimist'; import { isMainThread, parentPort } from 'worker_threads'; import * as hdbTerms from '../utility/hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; -import * as processMan from '../utility/processManagement/processManagement.js'; +import * as processMan from '../utility/processManagement/processManagement.ts'; import { compactOnStart } from './copyDb.ts'; -import { restartWorkers, onMessageByType, shutdownWorkersNow } from '../server/threads/manageThreads.js'; +import { restartWorkers, onMessageByType, shutdownWorkersNow } from '../server/threads/manageThreads.ts'; import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; const { HTTP_STATUS_CODES } = hdbErrors; import * as envMgr from '../utility/environment/environmentManager.ts'; import * as path from 'node:path'; import { unlinkSync } from 'node:fs'; import { getThisNodeName } from '../server/nodeName.ts'; -envMgr.initSync(); - +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const RESTART_RESPONSE = `Restarting Harper. This may take up to ${hdbTerms.RESTART_TIMEOUT_MS / 1000} seconds.`; const INVALID_SERVICE_ERR = 'Invalid service'; @@ -66,7 +69,7 @@ async function restart(req: any) { // and shut down. hdbLogger.debug('Shutdown workers'); await shutdownWorkersNow(); - const { closeServers } = require('../server/threads/threadServer.js'); + const { closeServers } = await import('../server/threads/threadServer.ts'); await closeServers(); await processMan.cleanupChildrenProcesses(false); // remove pid file so it doesn't trip up the launch diff --git a/bin/run.ts b/bin/run.ts index c932d1cf8..cca07bc7b 100644 --- a/bin/run.ts +++ b/bin/run.ts @@ -1,24 +1,27 @@ 'use strict'; import * as env from '../utility/environment/environmentManager.ts'; -env.initSync(); - +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} // This unused restart import is here so that main thread loads ITC event listener defined in restart file. Do not remove. import './restart.ts'; import * as terms from '../utility/hdbTerms.ts'; const { CONFIG_PARAMS } = terms; import hdbLogger from '../utility/logging/harper_logger.ts'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'path'; -import checkJwtTokens from '../utility/install/checkJWTTokensExist.js'; +import checkJwtTokens from '../utility/install/checkJWTTokensExist.ts'; import { install } from '../utility/install/installer.ts'; import chalk from 'chalk'; import { packageJson } from '../utility/packageUtils.js'; import * as hdbUtils from '../utility/common_utils.ts'; import * as installation from '../utility/installation.ts'; -import * as configUtils from '../config/configUtils.js'; +import * as configUtils from '../config/configUtils.ts'; import assignCMDENVVariables from '../utility/assignCmdEnvVariables.ts'; -import * as upgrade from './upgrade.js'; +import * as upgrade from './upgrade.ts'; import { compactOnStart, migrateOnStart } from './copyDb.ts'; import minimist from 'minimist'; import * as keys from '../security/keys.ts'; @@ -27,7 +30,7 @@ import * as hdbInfoController from '../dataLayer/hdbInfoController.ts'; import { isReadOnlyMode } from '../resources/databases.ts'; import { getThisNodeName } from '../server/nodeName.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; -import { getHdbPid, isProcessRunning } from '../utility/processManagement/processManagement.js'; +import { getHdbPid, isProcessRunning } from '../utility/processManagement/processManagement.ts'; import { PACKAGE_ROOT } from '../utility/packageUtils.js'; let pmUtils; @@ -108,7 +111,7 @@ async function initialize(calledByInstall = false, calledByMain = false) { // If HARPER_SET_CONFIG is present, filter out any config keys that are set in it // to prevent individual env vars from overriding explicit runtime configuration - const { filterArgsAgainstRuntimeConfig } = require('../config/harperConfigEnvVars'); + const { filterArgsAgainstRuntimeConfig } = await import('../config/harperConfigEnvVars.ts'); parsedArgs = filterArgsAgainstRuntimeConfig(parsedArgs); if (!hdbUtils.isEmpty(parsedArgs) && !hdbUtils.isEmptyOrZeroLength(Object.keys(parsedArgs))) { @@ -199,6 +202,12 @@ async function main(calledByInstall = false) { } await initialize(calledByInstall, true); + // Drain onStartup hooks after install/initialize so server.X singletons + // (server.http, server.getUser, etc.) and auth.ts's onStartup table() + // call see the populated env / installed hdbBasePath. + const lifecycle = await import('../utility/lifecycle.ts'); + await lifecycle.runStartup(); + if (env.get(terms.CONFIG_PARAMS.STORAGE_COMPACTONSTART)) await compactOnStart(); if (env.get(terms.CONFIG_PARAMS.STORAGE_MIGRATEONSTART)) await migrateOnStart(); @@ -233,7 +242,7 @@ function started() { async function launch(exit = true) { skipExitListeners = !exit; try { - if (pmUtils === undefined) pmUtils = require('../utility/processManagement/processManagement.js'); + if (pmUtils === undefined) pmUtils = await import('../utility/processManagement/processManagement.ts'); hdbLogger.debug('initializing processManagement...'); await initialize(); hdbLogger.debug('Starting new main process'); diff --git a/bin/status.ts b/bin/status.ts index 6d78c5567..d2f80af50 100644 --- a/bin/status.ts +++ b/bin/status.ts @@ -1,6 +1,6 @@ 'use strict'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'path'; import * as YAML from 'yaml'; @@ -9,8 +9,11 @@ import hdbLog from '../utility/logging/harper_logger.ts'; import * as systemInformation from '../utility/environment/systemInformation.ts'; import * as envMgr from '../utility/environment/environmentManager.ts'; import * as installation from '../utility/installation.ts'; -envMgr.initSync(); - +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const STATUSES = { RUNNING: 'running', STOPPED: 'stopped', diff --git a/bin/upgrade.js b/bin/upgrade.ts similarity index 82% rename from bin/upgrade.js rename to bin/upgrade.ts index 152b7b8eb..41b748cfa 100644 --- a/bin/upgrade.js +++ b/bin/upgrade.ts @@ -1,33 +1,28 @@ -'use strict'; - /** * The upgrade module is used to facilitate the upgrade process for existing instances of HDB that pull down a new version * of HDB from NPM that requires a specific upgrade script be run - e.g. there are changes required for the settings.js * config file, a data model change requires a re-indexing script is run, etc. */ -const env = require('../utility/environment/environmentManager.ts'); -env.initSync(); - -const chalk = require('chalk'); -const hdbLogger = require('../utility/logging/harper_logger.ts'); -const hdbTerms = require('../utility/hdbTerms.ts'); -const directivesManager = require('../upgrade/directivesManager.ts'); -const installation = require('../utility/installation.ts'); -const hdbInfoController = require('../dataLayer/hdbInfoController.ts'); -const upgradePrompt = require('../upgrade/upgradePrompt.ts'); -const globalSchema = require('../utility/globalSchema.ts'); -const { packageJson } = require('../utility/packageUtils.js'); -const promisify = require('util').promisify; +import * as env from '../utility/environment/environmentManager.ts'; + +import chalk from 'chalk'; +import _hdbLogger from '../utility/logging/harper_logger.ts'; +const hdbLogger = _hdbLogger; +import * as hdbTerms from '../utility/hdbTerms.ts'; +import * as directivesManager from '../upgrade/directivesManager.ts'; +import * as installation from '../utility/installation.ts'; +import * as hdbInfoController from '../dataLayer/hdbInfoController.ts'; +import * as upgradePrompt from '../upgrade/upgradePrompt.ts'; +import * as globalSchema from '../utility/globalSchema.ts'; +import { packageJson } from '../utility/packageUtils.js'; +import { promisify } from 'util'; const pSchemaToGlobal = promisify(globalSchema.setSchemaDataToGlobal); let pm2Utils; const { UPGRADE_VERSION } = hdbTerms.UPGRADE_JSON_FIELD_NAMES_ENUM; -module.exports = { - upgrade, -}; - +export { upgrade }; /** * Runs the upgrade directives, if needed, for an updated version of Harper. * @@ -39,7 +34,7 @@ async function upgrade(upgradeObj) { // Requiring the processManagement mod will create the .pm2 dir. This code is here to allow install to set // pm2 env vars before that is done. - if (pm2Utils === undefined) pm2Utils = require('../utility/processManagement/processManagement.js'); + if (pm2Utils === undefined) pm2Utils = await import('../utility/processManagement/processManagement.ts'); //We have to make sure HDB is installed before doing anything else const installed = installation.isHdbInstalled(env, hdbLogger); diff --git a/components/Application.ts b/components/Application.ts index c1abcd8cc..9034d1f9a 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1,5 +1,5 @@ import { type Logger } from '../utility/logging/logger.ts'; -import { getConfigObj, getConfigValue, getConfigPath } from '../config/configUtils.js'; +import { getConfigObj, getConfigValue, getConfigPath } from '../config/configUtils.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import logger from '../utility/logging/harper_logger.ts'; diff --git a/components/Component.ts b/components/Component.ts index 7f04e5761..8ea23daf2 100644 --- a/components/Component.ts +++ b/components/Component.ts @@ -1,8 +1,8 @@ -import { resolveBaseURLPath } from './resolveBaseURLPath'; -import { deriveCommonPatternBase } from './deriveCommonPatternBase'; -import { deriveGlobOptions, FastGlobOptions, FilesOption } from './deriveGlobOptions'; -import { scan } from 'micromatch'; - +import { resolveBaseURLPath } from './resolveBaseURLPath.ts'; +import { deriveCommonPatternBase } from './deriveCommonPatternBase.ts'; +import { deriveGlobOptions, type FastGlobOptions, type FilesOption } from './deriveGlobOptions.ts'; +import _micromatch from 'micromatch'; +const { scan } = _micromatch; interface ComponentConfig { files: FilesOption; urlPath?: string; diff --git a/components/ComponentV1.ts b/components/ComponentV1.ts index 4804a271e..f02bd8551 100644 --- a/components/ComponentV1.ts +++ b/components/ComponentV1.ts @@ -3,12 +3,12 @@ import fg from 'fast-glob'; import { Resources } from '../resources/Resources.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; import { resolveBaseURLPath } from './resolveBaseURLPath.ts'; -import { deriveGlobOptions, FastGlobOptions, FilesOption } from './deriveGlobOptions.ts'; +import { deriveGlobOptions, type FastGlobOptions, type FilesOption } from './deriveGlobOptions.ts'; import { basename, join } from 'node:path'; import { readFile } from 'node:fs/promises'; import { deriveURLPath } from './deriveURLPath.ts'; -import { scan } from 'micromatch'; - +import _micromatch from 'micromatch'; +const { scan } = _micromatch; interface ComponentV1Config { files: string | string[] | FilesOption; /** @deprecated */ path?: string; diff --git a/components/EntryHandler.ts b/components/EntryHandler.ts index 77f75ec74..add45507d 100644 --- a/components/EntryHandler.ts +++ b/components/EntryHandler.ts @@ -2,14 +2,14 @@ import { type Logger } from '../utility/logging/logger.ts'; import { loggerWithTag } from '../utility/logging/harper_logger.ts'; import type { Stats } from 'node:fs'; import { EventEmitter, once } from 'node:events'; -import { Component, FileAndURLPathConfig } from './Component.ts'; -import chokidar, { FSWatcher, FSWatcherEventMap } from 'chokidar'; +import { Component, type FileAndURLPathConfig } from './Component.ts'; +import chokidar, { FSWatcher, type FSWatcherEventMap } from 'chokidar'; import { join } from 'node:path'; import { readFile } from 'node:fs/promises'; -import { FilesOption } from './deriveGlobOptions.ts'; +import { type FilesOption } from './deriveGlobOptions.ts'; import { deriveURLPath } from './deriveURLPath.ts'; -import { isMatch } from 'micromatch'; - +import _micromatch from 'micromatch'; +const { isMatch } = _micromatch; export interface BaseEntry { stats?: Stats; urlPath: string; diff --git a/components/OptionsWatcher.ts b/components/OptionsWatcher.ts index 0f02342dd..babbac053 100644 --- a/components/OptionsWatcher.ts +++ b/components/OptionsWatcher.ts @@ -6,8 +6,8 @@ import chokidar, { type FSWatcher } from 'chokidar'; import { readFile } from 'node:fs/promises'; import { isDeepStrictEqual } from 'util'; import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts'; -import { cloneDeep } from 'lodash'; - +import _lodash from 'lodash'; +const { cloneDeep } = _lodash; export interface Config { [key: string]: ConfigValue; } diff --git a/components/PluginModule.ts b/components/PluginModule.ts index 6b0781018..47d107193 100644 --- a/components/PluginModule.ts +++ b/components/PluginModule.ts @@ -1,4 +1,4 @@ -import { Scope } from './Scope'; +import { Scope } from './Scope.ts'; export interface PluginModule { handleApplication: (scope: Scope) => void | Promise; diff --git a/components/Scope.ts b/components/Scope.ts index a2b25d504..c273b430e 100644 --- a/components/Scope.ts +++ b/components/Scope.ts @@ -4,10 +4,10 @@ import { EventEmitter, once } from 'node:events'; import { databaseEventsEmitter } from '../resources/databases.ts'; import { server, type Server } from '../server/Server.ts'; import { EntryHandler, type EntryHandlerEventMap, type onEntryEventHandler } from './EntryHandler.ts'; -import { OptionsWatcher, OptionsWatcherEventMap } from './OptionsWatcher.ts'; +import { OptionsWatcher, type OptionsWatcherEventMap } from './OptionsWatcher.ts'; import { resources, type Resources } from '../resources/Resources.ts'; import type { FileAndURLPathConfig } from './Component.ts'; -import { FilesOption } from './deriveGlobOptions.ts'; +import { type FilesOption } from './deriveGlobOptions.ts'; import { requestRestart } from './requestRestart.ts'; import { ApplicationScope } from './ApplicationScope.ts'; diff --git a/components/componentLoader.ts b/components/componentLoader.ts index a9bc5bde9..b861e917e 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -1,4 +1,5 @@ -import { onMessageByType } from '../server/threads/manageThreads.js'; +import { onMessageByType } from '../server/threads/manageThreads.ts'; +import { onStartup } from '../utility/lifecycle.ts'; import { readdirSync, readFileSync, @@ -25,7 +26,7 @@ import * as staticFiles from '../server/static.ts'; import * as loadEnv from '../resources/loadEnv.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; import * as dataLoader from '../resources/dataLoader.ts'; -import { restartWorkers, getWorkerIndex } from '../server/threads/manageThreads.js'; +import { restartWorkers, getWorkerIndex } from '../server/threads/manageThreads.ts'; import { resetRestartNeeded, subscribeToRestartRequests } from './requestRestart.ts'; import { scopedImport } from '../security/jsLoader.ts'; import { server } from '../server/Server.ts'; @@ -34,7 +35,7 @@ import { table } from '../resources/databases.ts'; import { getHdbBasePath } from '../utility/environment/environmentManager.ts'; import * as auth from '../security/auth.ts'; import * as mqtt from '../server/mqtt.ts'; -import { getConfigObj, getConfigPath } from '../config/configUtils.js'; +import { getConfigObj, getConfigPath } from '../config/configUtils.ts'; import { ErrorResource } from '../resources/ErrorResource.ts'; import { Scope } from './Scope.ts'; import { ApplicationScope } from './ApplicationScope.ts'; @@ -43,7 +44,7 @@ import * as httpComponent from '../server/http.ts'; import { Status } from '../server/status/index.ts'; import { lifecycle as componentLifecycle } from './status/index.ts'; import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts'; -import { PluginModule } from './PluginModule.ts'; +import { type PluginModule } from './PluginModule.ts'; import { getEnvBuiltInComponents } from './Application.ts'; import { pathToFileURL } from 'node:url'; @@ -96,7 +97,7 @@ export const TRUSTED_RESOURCE_PLUGINS: any = { roles, jsResource: jsHandler, get fastifyRoutes() { - return require('../server/fastifyRoutes'); + return _fastifyRoutes; }, login, static: staticFiles, @@ -112,19 +113,23 @@ export const TRUSTED_RESOURCE_PLUGINS: any = { login: ... */ }; -if (isMainThread) { - TRUSTED_RESOURCE_PLUGINS.operationsApi = require('../server/operationsServer'); -} else { - // The HTTP operations API itself only binds in the main thread, but worker threads still - // dispatch operations — most notably, the replication WebSocket handler in workers receives - // inter-node operations like `add_node_back` and calls `server.operation(...)`. That requires - // `server.operation` / `server.registerOperation` to be wired up here too, and the operation - // function map to be initialized, BEFORE component plugins (replication, etc.) load and call - // `server.registerOperation?.({...})` at their module top level. Requiring serverUtilities - // directly (rather than the full operationsServer) avoids binding the fastify HTTP layer in - // workers while still installing the dispatch machinery. - require('../server/serverHelpers/serverUtilities'); -} +let _fastifyRoutes: any; +onStartup(async () => { + _fastifyRoutes = await import('../server/fastifyRoutes.ts'); + if (isMainThread) { + TRUSTED_RESOURCE_PLUGINS.operationsApi = await import('../server/operationsServer.ts'); + } else { + // The HTTP operations API itself only binds in the main thread, but worker threads still + // dispatch operations — most notably, the replication WebSocket handler in workers receives + // inter-node operations like `add_node_back` and calls `server.operation(...)`. That requires + // `server.operation` / `server.registerOperation` to be wired up here too, and the operation + // function map to be initialized, BEFORE component plugins (replication, etc.) load and call + // `server.registerOperation?.({...})` at their module top level. Loading serverUtilities + // directly (rather than the full operationsServer) avoids binding the fastify HTTP layer in + // workers while still installing the dispatch machinery. + await import('../server/serverHelpers/serverUtilities.ts'); + } +}); for (const { name, packageIdentifier } of getEnvBuiltInComponents()) { TRUSTED_RESOURCE_PLUGINS[name] = packageIdentifier; @@ -312,7 +317,19 @@ export async function loadComponent( } else { config = DEFAULT_CONFIG; } + // getConfigObj() can return undefined when the harper config has not yet been + // initialised (e.g. test paths that touch the loader without going through + // bin/harper.ts). Treat that the same as a missing config rather than crashing + // on `config.extensionModule` below. For non-root components, an empty/null + // parse result means an intentionally-empty config file — do NOT fall back to + // DEFAULT_CONFIG, otherwise OptionsWatcher waits forever for plugins that the + // file doesn't actually declare and the worker hangs on scope.ready. + if (isRoot) config ??= DEFAULT_CONFIG; applicationScope.config ??= config; + if (!config) { + // Empty/comment-only config file on a non-root component: nothing to load. + return undefined; + } if (!isRoot) { try { diff --git a/components/operations.js b/components/operations.ts similarity index 90% rename from components/operations.js rename to components/operations.ts index 4256af8b2..d680cc662 100644 --- a/components/operations.js +++ b/components/operations.ts @@ -1,23 +1,24 @@ -'use strict'; - -const path = require('node:path'); -const { isMainThread } = require('node:worker_threads'); -const fs = require('fs-extra'); -const fg = require('fast-glob'); -const normalize = require('normalize-path'); -const validator = require('./operationsValidation.js'); -const log = require('../utility/logging/harper_logger.ts'); -const hdbTerms = require('../utility/hdbTerms.ts'); -const env = require('../utility/environment/environmentManager.ts'); -const configUtils = require('../config/configUtils.js'); -const hdbUtils = require('../utility/common_utils.ts'); -const { handleHDBError, hdbErrors } = require('../utility/errors/hdbError.ts'); +import path from 'node:path'; +import { isMainThread } from 'node:worker_threads'; +import fs from 'fs-extra'; +import fg from 'fast-glob'; +import normalize from 'normalize-path'; +import * as validator from './operationsValidation.ts'; +import log from '../utility/logging/harper_logger.ts'; +import * as hdbTerms from '../utility/hdbTerms.ts'; +import * as env from '../utility/environment/environmentManager.ts'; +import * as configUtils from '../config/configUtils.ts'; +import * as hdbUtils from '../utility/common_utils.ts'; +import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; +import * as serverUtilities from '../server/serverHelpers/serverUtilities.ts'; +import { internal as statusInternal } from './status/index.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; -const manageThreads = require('../server/threads/manageThreads.js'); -const { packageDirectory } = require('../components/packageComponent.ts'); -const { Resources } = require('../resources/Resources.ts'); -const { Application, prepareApplication } = require('./Application.ts'); -const { server } = require('../server/Server.ts'); +import * as manageThreads from '../server/threads/manageThreads.ts'; +import { packageDirectory } from '../components/packageComponent.ts'; +import { Resources } from '../resources/Resources.ts'; +import { Application, prepareApplication as _prepareApplication } from './Application.ts'; +const prepareApplication = _prepareApplication; +import { server } from '../server/Server.ts'; /** * Read the settings.js file and return the @@ -363,8 +364,11 @@ async function deployComponent(req) { // TODO: how can we keep record of the `payload`? Its often too large to stuff into a config file; especially the root config. Maybe we can write it to a file and reference that way? if (req.package) { // Check if trying to overwrite a core component (requires force) - // Lazy-load to avoid circular dependency with componentLoader - const { TRUSTED_RESOURCE_PLUGINS } = require('./componentLoader.ts'); + // Lazy-load to avoid circular dependency with componentLoader. + // (Also keeps componentLoader/dataLoader out of the test-time module graph + // when callers only require this file for unrelated exports — pulling + // dataLoader in transitively defeats dataLoader.test.js's forComponent stub.) + const { TRUSTED_RESOURCE_PLUGINS } = await import('./componentLoader.ts'); if (TRUSTED_RESOURCE_PLUGINS[req.project] && !req.force) { throw handleHDBError( new Error(), @@ -402,8 +406,8 @@ async function deployComponent(req) { const pseudoResources = new Resources(); pseudoResources.isWorker = true; - const componentLoader = require('./componentLoader.ts').default || require('./componentLoader.ts'); let lastError; + const componentLoader = await import('./componentLoader.ts'); componentLoader.setErrorReporter((error) => (lastError = error)); await componentLoader.loadComponent( application.dirPath, @@ -425,7 +429,6 @@ async function deployComponent(req) { manageThreads.restartWorkers('http'); response.message = `Successfully deployed: ${application.name}, restarting Harper`; } else if (rollingRestart) { - const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); const jobResponse = await serverUtilities.executeJob({ operation: 'restart_service', service: 'http', @@ -513,7 +516,6 @@ async function getComponents() { if (sourcePackage) entry.package = sourcePackage; } - const { internal: statusInternal } = require('./status/index.ts'); let consolidatedStatuses; try { @@ -640,16 +642,16 @@ async function dropComponent(req) { return response; } -exports.customFunctionsStatus = customFunctionsStatus; -exports.getCustomFunctions = getCustomFunctions; -exports.getCustomFunction = getCustomFunction; -exports.setCustomFunction = setCustomFunction; -exports.dropCustomFunction = dropCustomFunction; -exports.addComponent = addComponent; -exports.dropCustomFunctionProject = dropCustomFunctionProject; -exports.packageComponent = packageComponent; -exports.deployComponent = deployComponent; -exports.getComponents = getComponents; -exports.getComponentFile = getComponentFile; -exports.setComponentFile = setComponentFile; -exports.dropComponent = dropComponent; +export { customFunctionsStatus }; +export { getCustomFunctions }; +export { getCustomFunction }; +export { setCustomFunction }; +export { dropCustomFunction }; +export { addComponent }; +export { dropCustomFunctionProject }; +export { packageComponent }; +export { deployComponent }; +export { getComponents }; +export { getComponentFile }; +export { setComponentFile }; +export { dropComponent }; diff --git a/components/operationsValidation.js b/components/operationsValidation.ts similarity index 93% rename from components/operationsValidation.js rename to components/operationsValidation.ts index bf87c1045..a92f7a3a6 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.ts @@ -1,19 +1,16 @@ -'use strict'; - -const Joi = require('joi'); -const fs = require('fs-extra'); -const path = require('path'); -const validator = require('../validation/validationWrapper.ts'); -const hdbTerms = require('../utility/hdbTerms.ts'); -const hdbLogger = require('../utility/logging/harper_logger.ts'); -const configUtils = require('../config/configUtils.js'); -const { hdbErrors } = require('../utility/errors/hdbError.ts'); +import Joi from 'joi'; +import fs from 'fs-extra'; +import path from 'path'; +import * as validator from '../validation/validationWrapper.ts'; +import * as hdbTerms from '../utility/hdbTerms.ts'; +import hdbLogger from '../utility/logging/harper_logger.ts'; +import * as configUtils from '../config/configUtils.ts'; +import { hdbErrors } from '../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS } = hdbErrors; - // File name can only be alphanumeric, dash and underscores const PROJECT_FILE_NAME_REGEX = /^[a-zA-Z0-9-_]+$/; -module.exports = { +export { getDropCustomFunctionValidator, setCustomFunctionValidator, addComponentValidator, @@ -24,7 +21,6 @@ module.exports = { getComponentFileValidator, dropComponentFileValidator, }; - /** * Check to see if a project dir exists in the custom functions dir. * @param checkExists - determine if validator returns error if exists or vice versa diff --git a/components/status/crossThread.ts b/components/status/crossThread.ts index 339e46418..758240b32 100644 --- a/components/status/crossThread.ts +++ b/components/status/crossThread.ts @@ -5,8 +5,8 @@ * and aggregating it into a unified view. */ -import { sendItcEvent } from '../../server/threads/itc.js'; -import { getWorkerIndex, onMessageByType, getWorkerCount } from '../../server/threads/manageThreads.js'; +import { sendItcEvent } from '../../server/threads/itc.ts'; +import { getWorkerIndex, onMessageByType, getWorkerCount } from '../../server/threads/manageThreads.ts'; import { ITC_EVENT_TYPES } from '../../utility/hdbTerms.ts'; import { loggerWithTag } from '../../utility/logging/logger.ts'; import { ComponentStatusRegistry } from './ComponentStatusRegistry.ts'; diff --git a/config/RootConfigWatcher.ts b/config/RootConfigWatcher.ts index d51abdae9..da0bddd4f 100644 --- a/config/RootConfigWatcher.ts +++ b/config/RootConfigWatcher.ts @@ -1,6 +1,6 @@ import chokidar, { FSWatcher } from 'chokidar'; import { readFile } from 'node:fs/promises'; -import { getConfigFilePath } from './configUtils.js'; +import { getConfigFilePath } from './configUtils.ts'; import { EventEmitter, once } from 'node:events'; import { parse } from 'yaml'; diff --git a/config/configUtils.js b/config/configUtils.ts similarity index 92% rename from config/configUtils.js rename to config/configUtils.ts index 3b9a98c6f..c3bb9eb83 100644 --- a/config/configUtils.js +++ b/config/configUtils.ts @@ -1,34 +1,40 @@ -'use strict'; - -const hdbTerms = require('../utility/hdbTerms.ts'); -const hdbUtils = require('../utility/common_utils.ts'); -const logger = require('../utility/logging/harper_logger.ts'); -const { configValidator } = require('../validation/configValidator.ts'); -const fs = require('fs-extra'); -const YAML = require('yaml'); -const path = require('path'); -const { threadId } = require('node:worker_threads'); -const { randomBytes } = require('node:crypto'); -const isNumber = require('is-number'); -const PropertiesReader = require('properties-reader'); -const _ = require('lodash'); -const { handleHDBError } = require('../utility/errors/hdbError.ts'); -const { HTTP_STATUS_CODES, HDB_ERROR_MSGS } = require('../utility/errors/commonErrors.ts'); -const { server } = require('../server/Server.ts'); -const { getBackupDirPath } = require('./configHelpers.ts'); -const { PACKAGE_ROOT } = require('../utility/packageUtils'); - +import * as hdbTerms from '../utility/hdbTerms.ts'; +import * as hdbUtils from '../utility/common_utils.ts'; +import _logger from '../utility/logging/harper_logger.ts'; +const logger = _logger; +import { configValidator as _configValidator } from '../validation/configValidator.ts'; +const configValidator = _configValidator; +import _fs from 'fs-extra'; +const fs = _fs; +import _YAML from 'yaml'; +const YAML = _YAML; +import path from 'path'; +import { threadId } from 'node:worker_threads'; +import { randomBytes } from 'node:crypto'; +import isNumber from 'is-number'; +import _PropertiesReader from 'properties-reader'; +const PropertiesReader = _PropertiesReader; +import _ from 'lodash'; +import { handleHDBError } from '../utility/errors/hdbError.ts'; +import { HTTP_STATUS_CODES, HDB_ERROR_MSGS } from '../utility/errors/commonErrors.ts'; +import { server } from '../server/Server.ts'; +import { getBackupDirPath } from './configHelpers.ts'; +import { PACKAGE_ROOT } from '../utility/packageUtils.js'; + +import * as env from '../utility/environment/environmentManager.ts'; +import { applyRuntimeEnvConfig as _applyRuntimeEnvConfig } from './harperConfigEnvVars.ts'; +const applyRuntimeEnvConfig = _applyRuntimeEnvConfig; const { DATABASES_PARAM_CONFIG, CONFIG_PARAMS, CONFIG_PARAM_MAP } = hdbTerms; -const UNINIT_GET_CONFIG_ERR = 'Unable to get config value because config is uninitialized'; -const CONFIG_INIT_MSG = 'Config successfully initialized'; -const BACKUP_ERR = 'Error backing up config file'; -const EMPTY_GET_VALUE = 'Empty parameter sent to getConfigValue'; -const DEFAULT_CONFIG_FILE_PATH = path.join(PACKAGE_ROOT, 'static', hdbTerms.HDB_DEFAULT_CONFIG_FILE); +var UNINIT_GET_CONFIG_ERR = 'Unable to get config value because config is uninitialized'; +var CONFIG_INIT_MSG = 'Config successfully initialized'; +var BACKUP_ERR = 'Error backing up config file'; +var EMPTY_GET_VALUE = 'Empty parameter sent to getConfigValue'; +var DEFAULT_CONFIG_FILE_PATH = path.join(PACKAGE_ROOT, 'static', hdbTerms.HDB_DEFAULT_CONFIG_FILE); -const CONFIGURE_SUCCESS_RESPONSE = +var CONFIGURE_SUCCESS_RESPONSE = 'Configuration successfully set. You must restart Harper for new config settings to take effect.'; -const DEPRECATED_CONFIG = { +var DEPRECATED_CONFIG = { logging_rotation_retain: 'logging.rotation.retain', logging_rotation_rotate: 'logging.rotation.rotate', logging_rotation_rotateinterval: 'logging.rotation.rotateInterval', @@ -37,35 +43,33 @@ const DEPRECATED_CONFIG = { logging_rotation_workerinterval: 'logging.rotation.workerInterval', }; -let flatDefaultConfigObj; -let flatConfigObj; -let configObj; - -exports.createConfigFile = createConfigFile; -exports.getDefaultConfig = getDefaultConfig; -exports.getConfigValue = getConfigValue; -exports.initConfig = initConfig; -exports.flattenConfig = flattenConfig; -exports.updateConfigValue = updateConfigValue; -exports.updateConfigObject = updateConfigObject; -exports.getConfiguration = getConfiguration; -exports.setConfiguration = setConfiguration; -exports.readConfigFile = readConfigFile; -exports.initOldConfig = initOldConfig; -exports.getConfigFromFile = getConfigFromFile; -exports.getConfigFilePath = getConfigFilePath; -exports.addConfig = addConfig; -exports.deleteConfigFromFile = deleteConfigFromFile; -exports.getConfigObj = getConfigObj; -exports.resolvePath = resolvePath; -exports.getFlatConfigObj = getFlatConfigObj; -exports.getConfigPath = getConfigPath; - +var flatDefaultConfigObj; +var flatConfigObj; +var configObj; + +export { createConfigFile }; +export { getDefaultConfig }; +export { getConfigValue }; +export { initConfig }; +export { flattenConfig }; +export { updateConfigValue }; +export { updateConfigObject }; +export { getConfiguration }; +export { setConfiguration }; +export { readConfigFile }; +export { initOldConfig }; +export { getConfigFromFile }; +export { getConfigFilePath }; +export { addConfig }; +export { deleteConfigFromFile }; +export { getConfigObj }; +export { resolvePath }; +export { getFlatConfigObj }; +export { getConfigPath }; function resolvePath(relativePath) { if (relativePath?.startsWith('~/')) { return path.join(hdbUtils.getHomeDir(), relativePath.slice(1)); } - const env = require('../utility/environment/environmentManager.ts'); try { return path.resolve(env.getHdbBasePath(), relativePath); } catch (error) { @@ -79,7 +83,6 @@ function resolvePath(relativePath) { * @param param */ function getConfigPath(param) { - const env = require('../utility/environment/environmentManager.ts'); const value = env.get(param); if (!value || typeof value !== 'string') return value; if (value.startsWith('~/')) { @@ -693,7 +696,7 @@ function backupConfigFile(configPath, hdbRoot) { } } -const PRESERVED_PROPERTIES = ['databases']; +var PRESERVED_PROPERTIES = ['databases']; /** * Flattens the JSON version of Harper config with underscores separating each parent/child key. * @param obj @@ -874,8 +877,6 @@ function applyRuntimeEnvVarConfig(configDoc, configFilePath, options = {}) { // No env vars set, skip entirely (zero overhead) if (!defaultEnvValue && !setEnvValue) return; - const { applyRuntimeEnvConfig } = require('./harperConfigEnvVars.ts'); - // Get rootPath for state file location const rootPath = configDoc.getIn(['rootPath']); if (!rootPath) { diff --git a/config/harperConfigEnvVars.ts b/config/harperConfigEnvVars.ts index 80b0f8433..bf5d3f4b1 100644 --- a/config/harperConfigEnvVars.ts +++ b/config/harperConfigEnvVars.ts @@ -12,20 +12,17 @@ */ import type { Logger } from '../utility/logging/logger.ts'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'node:path'; import * as crypto from 'node:crypto'; -import { cloneDeep } from 'lodash'; +import _lodash from 'lodash'; +const { cloneDeep } = _lodash; import { getBackupDirPath } from './configHelpers.ts'; +import { loggerWithTag } from '../utility/logging/harper_logger.ts'; const STATE_FILE_NAME = '.harper-config-state.json'; -/** - * Get logger instance with tag - lazy loaded to avoid circular dependencies - * and ensure logger is initialized before use - */ function getLogger(): Logger { - const { loggerWithTag } = require('../utility/logging/harper_logger'); return loggerWithTag('env-config'); } diff --git a/dataLayer/SQLSearch.ts b/dataLayer/SQLSearch.ts index 2a2e60be4..e26d53a89 100644 --- a/dataLayer/SQLSearch.ts +++ b/dataLayer/SQLSearch.ts @@ -6,7 +6,7 @@ * process and return results by passing the raw values into the alasql SQL parser */ -import * as _ from 'lodash'; +import _ from 'lodash'; import * as alasql from 'alasql'; alasql.options.cache = false; import alasqlFunctionImporter from '../sqlTranslator/alasqlFunctionImporter.ts'; @@ -14,7 +14,7 @@ import clone from 'clone'; import RecursiveIterator from 'recursive-iterator'; import log from '../utility/logging/harper_logger.ts'; import * as commonUtils from '../utility/common_utils.ts'; -const harperBridge = require('./harperBridge/harperBridge').default; +import harperBridge from './harperBridge/harperBridge.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; import { hdbErrors } from '../utility/errors/hdbError.ts'; import { getDatabases } from '../resources/databases.ts'; diff --git a/dataLayer/bulkLoad.ts b/dataLayer/bulkLoad.ts index 9a5a97d2b..82f5cab82 100644 --- a/dataLayer/bulkLoad.ts +++ b/dataLayer/bulkLoad.ts @@ -1,26 +1,32 @@ import * as insert from './insert.ts'; -import * as validator from '../validation/fileLoadValidator.ts'; -import needle from 'needle'; +import * as _validator from '../validation/fileLoadValidator.ts'; +const validator = _validator; +import _needle from 'needle'; +const needle = _needle; import * as hdbTerms from '../utility/hdbTerms.ts'; import * as hdbUtils from '../utility/common_utils.ts'; -import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; +import { handleHDBError as _handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; +const handleHDBError = _handleHDBError; import { HTTP_STATUS_CODES, HDB_ERROR_MSGS, CHECK_LOGS_WRAPPER } from '../utility/errors/commonErrors.ts'; import logger from '../utility/logging/harper_logger.ts'; import * as papaParse from 'papaparse'; -import * as fs from 'fs-extra'; +import _fs from 'fs-extra'; +const fs = _fs; import * as path from 'path'; -import { chain } from 'stream-chain'; -import StreamArray from 'stream-json/streamers/StreamArray'; -import Batch from 'stream-json/utils/Batch'; -import comp from 'stream-chain/utils/comp'; +import streamChain from 'stream-chain'; +const chain = (streamChain as any).chain ?? streamChain; +import StreamArray from 'stream-json/streamers/StreamArray.js'; +import Batch from 'stream-json/utils/Batch.js'; +import comp from 'stream-chain/utils/comp.js'; import { finished } from 'stream'; import * as env from '../utility/environment/environmentManager.ts'; import * as opFuncCaller from '../utility/OperationFunctionCaller.ts'; import * as AWSConnector from '../utility/AWS/AWSConnector.js'; import { BulkLoadFileObject, BulkLoadDataObject } from './dataObjects/BulkLoadObjects.js'; import PermissionResponseObject from '../security/data_objects/PermissionResponseObject.ts'; -import { verifyBulkLoadAttributePerms } from '../utility/operation_authorization.ts'; +import { verifyBulkLoadAttributePerms as _verifyBulkLoadAttributePerms } from '../utility/operation_authorization.ts'; +const verifyBulkLoadAttributePerms = _verifyBulkLoadAttributePerms; import { databases } from '../resources/databases.ts'; import { coerceType } from '../resources/Table.ts'; diff --git a/dataLayer/dataObjects/UpsertObject.js b/dataLayer/dataObjects/UpsertObject.ts similarity index 76% rename from dataLayer/dataObjects/UpsertObject.js rename to dataLayer/dataObjects/UpsertObject.ts index c318882c5..c3b079bf7 100644 --- a/dataLayer/dataObjects/UpsertObject.js +++ b/dataLayer/dataObjects/UpsertObject.ts @@ -1,6 +1,4 @@ -'use strict'; -const OPERATIONS_ENUM = require('../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import { OPERATIONS_ENUM } from '../../utility/hdbTerms.ts'; /** * object representing an UPSERT operation */ @@ -20,4 +18,4 @@ class UpsertObject { } } -module.exports = UpsertObject; +export default UpsertObject; diff --git a/dataLayer/delete.ts b/dataLayer/delete.ts index 99fa6f273..96c368249 100644 --- a/dataLayer/delete.ts +++ b/dataLayer/delete.ts @@ -9,7 +9,9 @@ import { promisify, callbackify } from 'util'; import * as terms from '../utility/hdbTerms.ts'; import * as globalSchema from '../utility/globalSchema.ts'; const pGlobalSchema = promisify(globalSchema.getTableSchema); -const harperBridge = require('./harperBridge/harperBridge').default; +import _harperBridge from './harperBridge/harperBridge.ts'; +// Lazy access to handle import cycle (insert.ts and harperBridge.ts both import each other transitively). +const harperBridge: any = new Proxy({}, { get: (_, p) => (_harperBridge as any)[p] }); import { DeleteResponseObject } from './DataLayerObjects.ts'; import { handleHDBError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; diff --git a/dataLayer/export.ts b/dataLayer/export.ts index 1eb31c638..cd79f59ac 100644 --- a/dataLayer/export.ts +++ b/dataLayer/export.ts @@ -4,7 +4,7 @@ import * as search from './search.ts'; import * as AWSConnector from '../utility/AWS/AWSConnector.js'; import * as stream from 'stream'; import * as hdbUtils from '../utility/common_utils.ts'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'path'; import hdbLogger from '../utility/logging/harper_logger.ts'; import { promisify } from 'util'; @@ -13,7 +13,8 @@ import { handleHDBError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; import { streamAsJSON } from '../server/serverHelpers/JSONStream.ts'; -let { Upload } = require('@aws-sdk/lib-storage'); +import { Upload as _Upload } from '@aws-sdk/lib-storage'; +const Upload = _Upload; import { toCsvStream } from '../server/serverHelpers/contentTypes.ts'; const VALID_SEARCH_OPERATIONS = ['search_by_value', 'search_by_hash', 'sql', 'search_by_conditions']; diff --git a/dataLayer/getBackup.ts b/dataLayer/getBackup.ts index ed8b339ef..a952135e7 100644 --- a/dataLayer/getBackup.ts +++ b/dataLayer/getBackup.ts @@ -4,7 +4,7 @@ 'use strict'; -const harperBridge = require('./harperBridge/harperBridge').default; +import harperBridge from './harperBridge/harperBridge.ts'; // eslint-disable-next-line no-unused-vars import GetBackupObject from './GetBackupObject.ts'; import * as hdbUtils from '../utility/common_utils.ts'; diff --git a/dataLayer/harperBridge/ResourceBridge.ts b/dataLayer/harperBridge/ResourceBridge.ts index 7a50ddd5b..ca52c43dd 100644 --- a/dataLayer/harperBridge/ResourceBridge.ts +++ b/dataLayer/harperBridge/ResourceBridge.ts @@ -1,7 +1,7 @@ import searchValidator from '../../validation/searchValidator.ts'; import { handleHDBError, ClientError, hdbErrors } from '../../utility/errors/hdbError.ts'; import { table, getDatabases, database, dropDatabase, type Table } from '../../resources/databases.ts'; -import insertUpdateValidate from './bridgeUtility/insertUpdateValidate.js'; +import insertUpdateValidate from './bridgeUtility/insertUpdateValidate.ts'; import SearchObject from '../SearchObject.ts'; import { OPERATIONS_ENUM, @@ -10,7 +10,7 @@ import { READ_AUDIT_LOG_SEARCH_TYPES_ENUM, } from '../../utility/hdbTerms.ts'; import * as signalling from '../../utility/signalling.ts'; -import { SchemaEventMsg } from '../../server/threads/itc.js'; +import { SchemaEventMsg } from '../../server/threads/itc.ts'; import { asyncSetTimeout } from '../../utility/common_utils.ts'; import { transaction } from '../../resources/transaction.ts'; import type { @@ -26,7 +26,7 @@ import { collapseData } from '../../resources/tracked.ts'; import { errorToString } from '../../utility/logging/harper_logger.ts'; import { RocksDatabase } from '@harperfast/rocksdb-js'; import { BridgeMethods } from './BridgeMethods.ts'; -import lmdbGetBackup from './lmdbBridge/lmdbMethods/lmdbGetBackup.js'; +import lmdbGetBackup from './lmdbBridge/lmdbMethods/lmdbGetBackup.ts'; import { DeleteTransactionLogsBeforeResults } from './DeleteTransactionLogsBeforeResults.ts'; import type { Readable } from 'node:stream'; diff --git a/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js b/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.ts similarity index 89% rename from dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js rename to dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.ts index 8fd159da1..84dc49e92 100644 --- a/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +++ b/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.ts @@ -1,11 +1,9 @@ -'use strict'; +import * as hdbUtils from '../../../utility/common_utils.ts'; +import log from '../../../utility/logging/harper_logger.ts'; +import { getDatabases } from '../../../resources/databases.ts'; +import { ClientError } from '../../../utility/errors/hdbError.ts'; -const hdbUtils = require('../../../utility/common_utils.ts'); -const log = require('../../../utility/logging/harper_logger.ts'); -const { getDatabases } = require('../../../resources/databases.ts'); -const { ClientError } = require('../../../utility/errors/hdbError.ts'); - -module.exports = insertUpdateValidate; +export default insertUpdateValidate; //IMPORTANT - This code is the same code as the async validation() function in dataLayer/insert - make sure any changes // below are also made there. This is to resolve a circular dependency. diff --git a/dataLayer/harperBridge/harperBridge.ts b/dataLayer/harperBridge/harperBridge.ts index bb697a182..bf3cb0995 100644 --- a/dataLayer/harperBridge/harperBridge.ts +++ b/dataLayer/harperBridge/harperBridge.ts @@ -1,9 +1,11 @@ 'use strict'; import { ResourceBridge } from './ResourceBridge.ts'; -import * as envMngr from '../../utility/environment/environmentManager.ts'; -envMngr.initSync(); - +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} let harperBridge; // ResourceBridge /** diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.ts similarity index 79% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.ts index 513f6df3b..a9430add3 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.ts @@ -1,22 +1,17 @@ -'use strict'; - -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const writeUtility = require('../../../../utility/lmdb/writeUtility.ts'); -const { getSystemSchemaPath, getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const { validateBySchema } = require('../../../../validation/validationWrapper.ts'); -const Joi = require('joi'); -const LMDBCreateAttributeObject = - require('../lmdbUtility/LMDBCreateAttributeObject.js').default || - require('../lmdbUtility/LMDBCreateAttributeObject.js'); -const returnObject = require('../../bridgeUtility/insertUpdateReturnObj.js'); -const { handleHDBError, hdbErrors, ClientError } = require('../../../../utility/errors/hdbError.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import * as writeUtility from '../../../../utility/lmdb/writeUtility.ts'; +import { getSystemSchemaPath, getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import { validateBySchema } from '../../../../validation/validationWrapper.ts'; +import Joi from 'joi'; +import LMDBCreateAttributeObject from '../lmdbUtility/LMDBCreateAttributeObject.ts'; +import returnObject from '../../bridgeUtility/insertUpdateReturnObj.js'; +import { handleHDBError, hdbErrors, ClientError } from '../../../../utility/errors/hdbError.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; const { HTTP_STATUS_CODES } = hdbErrors; - const ACTION = 'inserted'; -module.exports = lmdbCreateAttribute; +export default lmdbCreateAttribute; /** * First adds the attribute to the system attribute table, then creates the dbi. diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.ts similarity index 65% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.ts index e9a4a085d..6be641802 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.ts @@ -1,19 +1,17 @@ -'use strict'; - -const insertUpdateValidate = require('../../bridgeUtility/insertUpdateValidate.js'); +import insertUpdateValidate from '../../bridgeUtility/insertUpdateValidate.ts'; // eslint-disable-next-line no-unused-vars -const InsertObject = require('../../../InsertObject.ts').default || require('../../../InsertObject.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdbProcessRows = require('../lmdbUtility/lmdbProcessRows.js'); -const lmdbInsertRecords = require('../../../../utility/lmdb/writeUtility.ts').insertRecords; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const logger = require('../../../../utility/logging/harper_logger.ts'); +import InsertObject from '../../../InsertObject.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import lmdbProcessRows from '../lmdbUtility/lmdbProcessRows.ts'; +import { insertRecords as lmdbInsertRecords } from '../../../../utility/lmdb/writeUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import logger from '../../../../utility/logging/harper_logger.ts'; -const lmdbCheckNewAttributes = require('../lmdbUtility/lmdbCheckForNewAttributes.js'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const writeTransaction = require('../lmdbUtility/lmdbWriteTransaction.js'); +import lmdbCheckNewAttributes from '../lmdbUtility/lmdbCheckForNewAttributes.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import writeTransaction from '../lmdbUtility/lmdbWriteTransaction.ts'; -module.exports = lmdbCreateRecords; +export default lmdbCreateRecords; /** * Orchestrates the insertion of data into LMDB and the creation of new attributes/dbis diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.ts similarity index 54% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.ts index 501a7200c..dc3404367 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.ts @@ -1,12 +1,11 @@ -'use strict'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import _lmdbCreateRecords from './lmdbCreateRecords.ts'; +const lmdbCreateRecords = _lmdbCreateRecords; +import InsertObject from '../../../InsertObject.ts'; +import fs from 'fs-extra'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdbCreateRecords = require('./lmdbCreateRecords.js'); -const InsertObject = require('../../../InsertObject.ts').default || require('../../../InsertObject.ts'); -const fs = require('fs-extra'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); - -module.exports = lmdbCreateSchema; +export default lmdbCreateSchema; /** * creates the meta data for the schema diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.ts similarity index 76% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.ts index 1f95a6970..3c7ee53af 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.ts @@ -1,17 +1,14 @@ -'use strict'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import * as _writeUtility from '../../../../utility/lmdb/writeUtility.ts'; +const writeUtility = _writeUtility; +import { getSystemSchemaPath, getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import lmdbCreateAttribute from './lmdbCreateAttribute.ts'; +import LMDBCreateAttributeObject from '../lmdbUtility/LMDBCreateAttributeObject.ts'; +import log from '../../../../utility/logging/harper_logger.ts'; +import createTxnEnvironments from '../lmdbUtility/lmdbCreateTransactionsAuditEnvironment.ts'; -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const writeUtility = require('../../../../utility/lmdb/writeUtility.ts'); -const { getSystemSchemaPath, getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const lmdbCreateAttribute = require('./lmdbCreateAttribute.js'); -const LMDBCreateAttributeObject = - require('../lmdbUtility/LMDBCreateAttributeObject.js').default || - require('../lmdbUtility/LMDBCreateAttributeObject.js'); -const log = require('../../../../utility/logging/harper_logger.ts'); -const createTxnEnvironments = require('../lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js'); - -module.exports = lmdbCreateTable; +export default lmdbCreateTable; /** * Writes new table data to the system tables creates the environment file and creates two datastores to track created and updated diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.ts similarity index 81% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.ts index 0d2f14df3..5d4024876 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.ts @@ -1,20 +1,17 @@ -'use strict'; - -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getTransactionAuditStorePath } = require('../lmdbUtility/initializePaths.js'); +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getTransactionAuditStorePath } from '../lmdbUtility/initializePaths.ts'; // eslint-disable-next-line no-unused-vars -const DeleteBeforeObject = - require('../../../DeleteBeforeObject.ts').default || require('../../../DeleteBeforeObject.ts'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const DeleteAuditLogsBeforeResults = require('./DeleteAuditLogsBeforeResults.js'); -const promisify = require('util').promisify; +import DeleteBeforeObject from '../../../DeleteBeforeObject.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import DeleteAuditLogsBeforeResults from './DeleteAuditLogsBeforeResults.js'; +import { promisify } from 'util'; const pSettimeout = promisify(setTimeout); const BATCH_SIZE = 10000; const SLEEP_TIME_MS = 100; -module.exports = deleteAuditLogsBefore; +export default deleteAuditLogsBefore; /** * diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.ts similarity index 84% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.ts index fc9c872e3..631c963be 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.ts @@ -1,13 +1,11 @@ -'use strict'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import * as deleteUtility from '../../../../utility/lmdb/deleteUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import writeTransaction from '../lmdbUtility/lmdbWriteTransaction.ts'; +import logger from '../../../../utility/logging/harper_logger.ts'; -const hdbUtils = require('../../../../utility/common_utils.ts'); -const deleteUtility = require('../../../../utility/lmdb/deleteUtility.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const writeTransaction = require('../lmdbUtility/lmdbWriteTransaction.js'); -const logger = require('../../../../utility/logging/harper_logger.ts'); - -module.exports = lmdbDeleteRecords; +export default lmdbDeleteRecords; /** * Deletes a full table row at a certain hash. diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.ts similarity index 81% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.ts index de7445992..694b4aa85 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.ts @@ -1,19 +1,16 @@ -'use strict'; - -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const DeleteObject = require('../../../DeleteObject.ts').default || require('../../../DeleteObject.ts'); +import SearchObject from '../../../SearchObject.ts'; +import DeleteObject from '../../../DeleteObject.ts'; // eslint-disable-next-line no-unused-vars -const DropAttributeObject = - require('../../../DropAttributeObject.ts').default || require('../../../DropAttributeObject.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const commonUtils = require('../../../../utility/common_utils.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const systemSchema = require('../../../../json/systemSchema.json'); -const searchByValue = require('./lmdbSearchByValue.js'); -const deleteRecords = require('./lmdbDeleteRecords.js'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); +import DropAttributeObject from '../../../DropAttributeObject.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as commonUtils from '../../../../utility/common_utils.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import * as systemSchema from '../../../../json/systemSchema.json'; +import searchByValue from './lmdbSearchByValue.ts'; +import deleteRecords from './lmdbDeleteRecords.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; -module.exports = lmdbDropAttribute; +export default lmdbDropAttribute; /** * First deletes the attribute/dbi from lmdb then removes its record from system table diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.ts similarity index 73% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.ts index 7191f91f1..cb71462fa 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.ts @@ -1,20 +1,16 @@ -'use strict'; - -const fs = require('fs-extra'); -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const SearchByHashObject = - require('../../../SearchByHashObject.ts').default || require('../../../SearchByHashObject.ts'); -const DeleteObject = require('../../../DeleteObject.ts').default || require('../../../DeleteObject.ts'); -const dropTable = require('./lmdbDropTable.js'); -const deleteRecords = require('./lmdbDeleteRecords.js'); -const getDataByHash = require('./lmdbGetDataByHash.js'); -const searchDataByValue = require('./lmdbSearchByValue.js'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.ts'); +import fs from 'fs-extra'; +import SearchObject from '../../../SearchObject.ts'; +import SearchByHashObject from '../../../SearchByHashObject.ts'; +import DeleteObject from '../../../DeleteObject.ts'; +import dropTable from './lmdbDropTable.ts'; +import deleteRecords from './lmdbDeleteRecords.ts'; +import getDataByHash from './lmdbGetDataByHash.ts'; +import searchDataByValue from './lmdbSearchByValue.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import { handleHDBError, hdbErrors } from '../../../../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; - -module.exports = lmdbDropSchema; +export default lmdbDropSchema; /** * deletes all environment files under the schema folder, deletes all schema/table/attribute meta data from system diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.ts similarity index 82% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.ts index 8573271d1..1fe210b0e 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.ts @@ -1,16 +1,16 @@ -'use strict'; +import SearchObject from '../../../SearchObject.ts'; +import DeleteObject from '../../../DeleteObject.ts'; +import _searchByValue from './lmdbSearchByValue.ts'; +const searchByValue = _searchByValue; +import _deleteRecords from './lmdbDeleteRecords.ts'; +const deleteRecords = _deleteRecords; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getTransactionAuditStorePath, getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import log from '../../../../utility/logging/harper_logger.ts'; -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const DeleteObject = require('../../../DeleteObject.ts').default || require('../../../DeleteObject.ts'); -const searchByValue = require('./lmdbSearchByValue.js'); -const deleteRecords = require('./lmdbDeleteRecords.js'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getTransactionAuditStorePath, getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const log = require('../../../../utility/logging/harper_logger.ts'); - -module.exports = lmdbDropTable; +export default lmdbDropTable; /** * Calls drops the table, all of it's attribute & deletes the environment diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.ts similarity index 76% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.ts index 3ca34e726..d0aa6713c 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.ts @@ -1,13 +1,6 @@ -'use strict'; - -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); - -module.exports = { - flush, - resetReadTxn, -}; - +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +export { flush, resetReadTxn }; /** * This is wrapper for sync/flush to disk * @param schema diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.ts similarity index 87% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.ts index e95a0d651..0cee071b8 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.ts @@ -1,15 +1,13 @@ -'use strict'; +import { Readable } from 'stream'; +import { getDatabases } from '../../../../resources/databases.ts'; +import { readSync, openSync, createReadStream } from 'fs'; +import { open } from 'lmdb'; +import { OpenDBIObject } from '../../../../utility/lmdb/OpenDBIObject.ts'; +import OpenEnvironmentObject from '../../../../utility/lmdb/OpenEnvironmentObject.ts'; +import { AUDIT_STORE_OPTIONS } from '../../../../resources/auditStore.ts'; +import { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } from '../../../../utility/lmdb/terms.ts'; -const { Readable } = require('stream'); -const { getDatabases } = require('../../../../resources/databases.ts'); -const { readSync, openSync, createReadStream } = require('fs'); -const { open } = require('lmdb'); -const { OpenDBIObject } = require('../../../../utility/lmdb/OpenDBIObject.ts'); -const OpenEnvironmentObject = require('../../../../utility/lmdb/OpenEnvironmentObject.ts'); -const { AUDIT_STORE_OPTIONS } = require('../../../../resources/auditStore.ts'); -const { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } = require('../../../../utility/lmdb/terms.ts'); - -module.exports = getBackup; +export default getBackup; const META_SIZE = 32768; const DELAY_ITERATIONS = 100; /** diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.ts similarity index 75% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.ts index aa6d05980..cb0766229 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.ts @@ -1,9 +1,7 @@ -'use strict'; +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import hashSearchInit from '../lmdbUtility/initializeHashSearch.ts'; -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const hashSearchInit = require('../lmdbUtility/initializeHashSearch.js'); - -module.exports = lmdbGetDataByHash; +export default lmdbGetDataByHash; /** * fetches records by their hash values and returns a map of the results diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.ts similarity index 64% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.ts index e5b6abd26..03531c93f 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.ts @@ -1,12 +1,8 @@ -'use strict'; - -const searchValidator = - require('../../../../validation/searchValidator.ts').default || require('../../../../validation/searchValidator.ts'); -const commonUtils = require('../../../../utility/common_utils.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdbSearch = require('../lmdbUtility/lmdbSearch.js'); - -module.exports = lmdbGetDataByValue; +import searchValidator from '../../../../validation/searchValidator.ts'; +import * as commonUtils from '../../../../utility/common_utils.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as lmdbSearch from '../lmdbUtility/lmdbSearch.ts'; +export default lmdbGetDataByValue; /** * gets records by value returns a map of objects diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.ts similarity index 90% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.ts index 30f91654f..d318d3e8f 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.ts @@ -1,15 +1,13 @@ -'use strict'; - -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const { getTransactionAuditStorePath } = require('../lmdbUtility/initializePaths.js'); -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const LMDBTransactionObject = require('../lmdbUtility/LMDBTransactionObject.js'); -const log = require('../../../../utility/logging/harper_logger.ts'); - -module.exports = readAuditLog; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import { getTransactionAuditStorePath } from '../lmdbUtility/initializePaths.ts'; +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import LMDBTransactionObject from '../lmdbUtility/LMDBTransactionObject.js'; +import log from '../../../../utility/logging/harper_logger.ts'; + +export default readAuditLog; /** * function execute the readTransactionLog operation diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.ts similarity index 84% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.ts index 512c78f2f..c333d6c44 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.ts @@ -1,23 +1,17 @@ -'use strict'; - -// eslint-disable-next-line no-unused-vars -const { SearchByConditionsObject, SearchCondition } = - require('../../../SearchByConditionsObject.ts').default || require('../../../SearchByConditionsObject.ts'); -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const searchValidator = - require('../../../../validation/searchValidator.ts').default || require('../../../../validation/searchValidator.ts'); -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const lmdb_search = require('../lmdbUtility/lmdbSearch.js'); -const cursorFunctions = require('../../../../utility/lmdb/searchCursorFunctions.ts'); -const _ = require('lodash'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.ts'); +import SearchObject from '../../../SearchObject.ts'; +import searchValidator from '../../../../validation/searchValidator.ts'; +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as lmdb_search from '../lmdbUtility/lmdbSearch.ts'; +import * as cursorFunctions from '../../../../utility/lmdb/searchCursorFunctions.ts'; +import _ from 'lodash'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { handleHDBError, hdbErrors } from '../../../../utility/errors/hdbError.ts'; const { HTTP_STATUS_CODES } = hdbErrors; const RANGE_ESTIMATE = 100000000; -module.exports = lmdbSearchByConditions; +export default lmdbSearchByConditions; /** * gets records by conditions - returns array of Objects diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.ts similarity index 69% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.ts index 3be46a425..f28aa8f41 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.ts @@ -1,9 +1,7 @@ -'use strict'; +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import hashSearchInit from '../lmdbUtility/initializeHashSearch.ts'; -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const hashSearchInit = require('../lmdbUtility/initializeHashSearch.js'); - -module.exports = lmdbSearchByHash; +export default lmdbSearchByHash; /** * fetches records by their hash values and returns an Array of the results diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.ts similarity index 59% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.ts index 45d0303b1..ebd17bfc2 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.ts @@ -1,14 +1,10 @@ -'use strict'; - // eslint-disable-next-line no-unused-vars -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const searchValidator = - require('../../../../validation/searchValidator.ts').default || require('../../../../validation/searchValidator.ts'); -const commonUtils = require('../../../../utility/common_utils.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdb_search = require('../lmdbUtility/lmdbSearch.js'); - -module.exports = lmdbSearchByValue; +import SearchObject from '../../../SearchObject.ts'; +import searchValidator from '../../../../validation/searchValidator.ts'; +import * as commonUtils from '../../../../utility/common_utils.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as lmdb_search from '../lmdbUtility/lmdbSearch.ts'; +export default lmdbSearchByValue; /** * gets records by value - returns array of Objects diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.ts similarity index 74% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.ts index 9541156ea..d2c885596 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.ts @@ -1,11 +1,6 @@ -'use strict'; - -const { database } = require('../../../../resources/databases.ts'); - -module.exports = { - writeTransaction, -}; +import { database } from '../../../../resources/databases.ts'; +export { writeTransaction }; /** * This is wrapper for write transactions, ensuring that all reads and writes within the callback occur atomically * @param schema diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.ts similarity index 67% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.ts index 2b2dfa8c4..779c05b7d 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.ts @@ -1,16 +1,14 @@ -'use strict'; - -const insertUpdateValidate = require('../../bridgeUtility/insertUpdateValidate.js'); -const lmdbProcessRows = require('../lmdbUtility/lmdbProcessRows.js'); -const lmdbCheckNewAttributes = require('../lmdbUtility/lmdbCheckForNewAttributes.js'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdb_update_records = require('../../../../utility/lmdb/writeUtility.ts').updateRecords; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const writeTransaction = require('../lmdbUtility/lmdbWriteTransaction.js'); -const logger = require('../../../../utility/logging/harper_logger.ts'); - -module.exports = lmdbUpdateRecords; +import insertUpdateValidate from '../../bridgeUtility/insertUpdateValidate.ts'; +import lmdbProcessRows from '../lmdbUtility/lmdbProcessRows.ts'; +import lmdbCheckNewAttributes from '../lmdbUtility/lmdbCheckForNewAttributes.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import { updateRecords as lmdb_update_records } from '../../../../utility/lmdb/writeUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import writeTransaction from '../lmdbUtility/lmdbWriteTransaction.ts'; +import logger from '../../../../utility/logging/harper_logger.ts'; + +export default lmdbUpdateRecords; /** * Orchestrates the update of data in LMDB and the creation of new attributes/dbis diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.ts similarity index 64% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.ts index f84db3a9f..775675284 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.ts @@ -1,21 +1,19 @@ -'use strict'; - // eslint-disable-next-line no-unused-vars -const UpsertObject = - require('../../../dataObjects/UpsertObject.js').default || require('../../../dataObjects/UpsertObject.js'); -const insertUpdateValidate = require('../../bridgeUtility/insertUpdateValidate.js'); -const lmdbProcessRows = require('../lmdbUtility/lmdbProcessRows.js'); -const lmdbCheckNewAttributes = require('../lmdbUtility/lmdbCheckForNewAttributes.js'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdb_upsert_records = require('../../../../utility/lmdb/writeUtility.ts').upsertRecords; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const writeTransaction = require('../lmdbUtility/lmdbWriteTransaction.js'); +import UpsertObject from '../../../dataObjects/UpsertObject.ts'; +import insertUpdateValidate from '../../bridgeUtility/insertUpdateValidate.ts'; +import _lmdbProcessRows from '../lmdbUtility/lmdbProcessRows.ts'; +const lmdbProcessRows = _lmdbProcessRows; +import lmdbCheckNewAttributes from '../lmdbUtility/lmdbCheckForNewAttributes.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import { upsertRecords as lmdb_upsert_records } from '../../../../utility/lmdb/writeUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import writeTransaction from '../lmdbUtility/lmdbWriteTransaction.ts'; -const logger = require('../../../../utility/logging/harper_logger.ts'); -const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.ts'); +import logger from '../../../../utility/logging/harper_logger.ts'; +import { handleHDBError, hdbErrors } from '../../../../utility/errors/hdbError.ts'; -module.exports = lmdbUpsertRecords; +export default lmdbUpsertRecords; /** * Orchestrates the UPSERT of data in LMDB and the creation of new attributes/dbis diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.ts similarity index 75% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.ts index e109a0845..a27399c69 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.ts @@ -1,8 +1,4 @@ -'use strict'; - -const CreateAttributeObject = - require('../../../CreateAttributeObject.ts').default || require('../../../CreateAttributeObject.ts'); - +import CreateAttributeObject from '../../../CreateAttributeObject.ts'; class LMDBCreateAttributeObject extends CreateAttributeObject { /** * @@ -20,4 +16,4 @@ class LMDBCreateAttributeObject extends CreateAttributeObject { } } -module.exports = LMDBCreateAttributeObject; +export default LMDBCreateAttributeObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.ts similarity index 75% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.ts index 5f08bbe76..475bea46e 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.ts @@ -1,7 +1,5 @@ -'use strict'; -const LMDBTransactionObject = require('./LMDBTransactionObject.js'); -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import LMDBTransactionObject from './LMDBTransactionObject.js'; +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; /** * class to define a delete transaction */ @@ -20,4 +18,4 @@ class LMDBDeleteTransactionObject extends LMDBTransactionObject { } } -module.exports = LMDBDeleteTransactionObject; +export default LMDBDeleteTransactionObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.ts similarity index 72% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.ts index 24d5c3271..8ff6ca566 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.ts @@ -1,7 +1,5 @@ -'use strict'; -const LMDBTransactionObject = require('./LMDBTransactionObject.js'); -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import LMDBTransactionObject from './LMDBTransactionObject.js'; +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; /** * class to define an insert transaction */ @@ -19,4 +17,4 @@ class LMDBInsertTransactionObject extends LMDBTransactionObject { } } -module.exports = LMDBInsertTransactionObject; +export default LMDBInsertTransactionObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.ts similarity index 76% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.ts index d10d8e736..e835a42a1 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.ts @@ -1,7 +1,5 @@ -'use strict'; -const LMDBTransactionObject = require('./LMDBTransactionObject.js'); -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import LMDBTransactionObject from './LMDBTransactionObject.js'; +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; /** * class to define an update transaction */ @@ -21,4 +19,4 @@ class LMDBUpdateTransactionObject extends LMDBTransactionObject { } } -module.exports = LMDBUpdateTransactionObject; +export default LMDBUpdateTransactionObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.ts similarity index 76% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.ts index 5e39e3f5b..86189c387 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.ts @@ -1,7 +1,5 @@ -'use strict'; -const LMDBTransactionObject = require('./LMDBTransactionObject.js'); -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import LMDBTransactionObject from './LMDBTransactionObject.js'; +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; /** * class to define an update transaction */ @@ -21,4 +19,4 @@ class LMDBUpsertTransactionObject extends LMDBTransactionObject { } } -module.exports = LMDBUpsertTransactionObject; +export default LMDBUpsertTransactionObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.ts similarity index 52% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.ts index 9f1492513..79f67e72b 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.ts @@ -1,11 +1,8 @@ -'use strict'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import searchValidator from '../../../../validation/searchValidator.ts'; +import { getSchemaPath } from './initializePaths.ts'; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const searchValidator = - require('../../../../validation/searchValidator.ts').default || require('../../../../validation/searchValidator.ts'); -const { getSchemaPath } = require('./initializePaths.js'); - -module.exports = initialize; +export default initialize; /** * diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts similarity index 89% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts index 032f001c0..291512a77 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts @@ -1,14 +1,13 @@ -'use strict'; - -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const env = require('../../../../utility/environment/environmentManager.ts'); -const path = require('path'); -const minimist = require('minimist'); -const fs = require('fs-extra'); -const _ = require('lodash'); -const { getConfigPath } = require('../../../../config/configUtils.js'); -env.initSync(); +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import * as env from '../../../../utility/environment/environmentManager.ts'; +import path from 'path'; +import minimist from 'minimist'; +import fs from 'fs-extra'; +import _ from 'lodash'; +import { getConfigPath } from '../../../../config/configUtils.ts'; +// env.initSync() is called by the CLI entry points (bin/run.ts, bin/cliOperations.ts); +// calling it here at module load would TDZ during ESM cycle evaluation. const { CONFIG_PARAMS, DATABASES_PARAM_CONFIG, SYSTEM_SCHEMA_NAME } = hdbTerms; let BASE_SCHEMA_PATH = undefined; @@ -147,7 +146,7 @@ function resetPaths() { SYSTEM_SCHEMA_PATH = undefined; TRANSACTION_STORE_PATH = undefined; } -module.exports = { +export { getBaseSchemaPath, getSystemSchemaPath, getTransactionAuditStorePath, diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.ts similarity index 79% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.ts index e519e55e9..ecbd04dcc 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.ts @@ -1,17 +1,14 @@ -'use strict'; - -const hUtils = require('../../../../utility/common_utils.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const logger = require('../../../../utility/logging/harper_logger.ts'); -const lmdbCreateAttribute = require('../lmdbMethods/lmdbCreateAttribute.js'); -const LMDBCreateAttributeObject = - require('./LMDBCreateAttributeObject.js').default || require('./LMDBCreateAttributeObject.js'); -const signalling = require('../../../../utility/signalling.ts'); -const { SchemaEventMsg } = require('../../../../server/threads/itc.js'); +import * as hUtils from '../../../../utility/common_utils.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import logger from '../../../../utility/logging/harper_logger.ts'; +import lmdbCreateAttribute from '../lmdbMethods/lmdbCreateAttribute.ts'; +import LMDBCreateAttributeObject from './LMDBCreateAttributeObject.ts'; +import * as signalling from '../../../../utility/signalling.ts'; +import { SchemaEventMsg } from '../../../../server/threads/itc.ts'; const ATTRIBUTE_ALREADY_EXISTS = 'already exists in'; -module.exports = lmdbCheckForNewAttributes; +export default lmdbCheckForNewAttributes; /** * Uses a utility function to check if there are any new attributes that dont exist. Utility function diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.ts similarity index 70% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.ts index 41c4d8c2c..8b723c3c7 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.ts @@ -1,16 +1,11 @@ -'use strict'; - -const fs = require('fs-extra'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getTransactionAuditStorePath } = require('../lmdbUtility/initializePaths.js'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); +import fs from 'fs-extra'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getTransactionAuditStorePath } from '../lmdbUtility/initializePaths.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; // eslint-disable-next-line no-unused-vars -const CreateTableObject = - require('../../../CreateTableObject.ts').default || - require('../../../CreateTableObject.ts').default || - require('../../../CreateTableObject.ts'); +import CreateTableObject from '../../../CreateTableObject.ts'; -module.exports = createTransactionsAuditEnvironment; +export default createTransactionsAuditEnvironment; /** * Creates the environment to hold transactions diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.ts similarity index 83% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.ts index 837a526e8..81994b5f0 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.ts @@ -1,15 +1,12 @@ -'use strict'; - // eslint-disable-next-line no-unused-vars -const InsertObject = require('../../../InsertObject.ts').default || require('../../../InsertObject.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const log = require('../../../../utility/logging/harper_logger.ts'); -const uuid = require('uuid'); -const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.ts'); +import InsertObject from '../../../InsertObject.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import log from '../../../../utility/logging/harper_logger.ts'; +import { v4 as uuidv4 } from 'uuid'; +import { handleHDBError, hdbErrors } from '../../../../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; - -module.exports = processRows; +export default processRows; /** * parses the records and validates the hash value for each row as well as adding updated/created time stamps @@ -70,7 +67,7 @@ function validateAttribute(attribute) { function validateHash(record, hash_attribute, operation) { if (!record.hasOwnProperty(hash_attribute) || hdbUtils.isEmptyOrZeroLength(record[hash_attribute])) { if (operation === hdbTerms.OPERATIONS_ENUM.INSERT || operation === hdbTerms.OPERATIONS_ENUM.UPSERT) { - record[hash_attribute] = uuid.v4(); + record[hash_attribute] = uuidv4(); //return here since the rest of the validations do not apply return; } diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.ts similarity index 94% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.ts index e5af231e8..5420888af 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.ts @@ -1,13 +1,11 @@ -'use strict'; - -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const commonUtils = require('../../../../utility/common_utils.ts'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const systemSchema = require('../../../../json/systemSchema.json'); -const LMDB_ERRORS = require('../../../../utility/errors/commonErrors.ts').LMDB_ERRORS_ENUM; -const { getSchemaPath } = require('./initializePaths.js'); +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import * as commonUtils from '../../../../utility/common_utils.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as systemSchema from '../../../../json/systemSchema.json'; +import { LMDB_ERRORS_ENUM as LMDB_ERRORS } from '../../../../utility/errors/commonErrors.ts'; +import { getSchemaPath } from './initializePaths.ts'; const WILDCARDS = hdbTerms.SEARCH_WILDCARDS; @@ -362,10 +360,10 @@ function createSearchTypeFromSearchObject(searchObject, hash_attribute, returnMa } } -module.exports = { +export { executeSearch, createSearchTypeFromSearchObject, prepSearch, searchByType, - // filterByType, + // filterByType }; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.ts similarity index 76% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.ts index 0889ca63f..c0c8d66e0 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.ts @@ -1,21 +1,18 @@ -'use strict'; +import * as environmentUtil from '../../../../utility/lmdb/environmentUtility.ts'; +import LMDBInsertTransactionObject from './LMDBInsertTransactionObject.ts'; +import LMDBUpdateTransactionObject from './LMDBUpdateTransactionObject.ts'; +import LMDBUpsertTransactionObject from './LMDBUpsertTransactionObject.ts'; +import LMDBDeleteTransactionObject from './LMDBDeleteTransactionObject.ts'; -const environmentUtil = require('../../../../utility/lmdb/environmentUtility.ts'); -const LMDBInsertTransactionObject = require('./LMDBInsertTransactionObject.js'); -const LMDBUpdateTransactionObject = require('./LMDBUpdateTransactionObject.js'); -const LMDBUpsertTransactionObject = require('./LMDBUpsertTransactionObject.js'); -const LMDBDeleteTransactionObject = require('./LMDBDeleteTransactionObject.js'); +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as hdbUtil from '../../../../utility/common_utils.ts'; +import { CONFIG_PARAMS } from '../../../../utility/hdbTerms.ts'; +import * as envMngr from '../../../../utility/environment/environmentManager.ts'; -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const hdbUtil = require('../../../../utility/common_utils.ts'); -const { CONFIG_PARAMS } = require('../../../../utility/hdbTerms.ts'); -const envMngr = require('../../../../utility/environment/environmentManager.ts'); -envMngr.initSync(); +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; +import { getTransactionAuditStorePath } from './initializePaths.ts'; -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; -const { getTransactionAuditStorePath } = require('./initializePaths.js'); - -module.exports = writeTransaction; +export default writeTransaction; /** * diff --git a/dataLayer/insert.ts b/dataLayer/insert.ts index 4efc8331d..07634d5b8 100644 --- a/dataLayer/insert.ts +++ b/dataLayer/insert.ts @@ -7,10 +7,13 @@ * as the update module is meant to be used in more specific circumstances. */ import insertValidator from '../validation/insertValidator.ts'; -import * as hdbUtils from '../utility/common_utils.ts'; +import * as _hdbUtils from '../utility/common_utils.ts'; +const hdbUtils = _hdbUtils; import * as util from 'util'; // Leave this unused signalling import here. Due to circular dependencies we bring it in early to load it before the bridge -const harperBridge = require('./harperBridge/harperBridge').default; +import _harperBridge from './harperBridge/harperBridge.ts'; +// Lazy access to handle import cycle (insert.ts and harperBridge.ts both import each other transitively). +const harperBridge: any = new Proxy({}, { get: (_, p) => (_harperBridge as any)[p] }); import * as globalSchema from '../utility/globalSchema.ts'; import log from '../utility/logging/harper_logger.ts'; import { handleHDBError } from '../utility/errors/hdbError.ts'; diff --git a/dataLayer/readAuditLog.ts b/dataLayer/readAuditLog.ts index b0a4dd217..601b2e3bd 100644 --- a/dataLayer/readAuditLog.ts +++ b/dataLayer/readAuditLog.ts @@ -1,6 +1,8 @@ 'use strict'; -const harperBridge = require('./harperBridge/harperBridge').default; +import _harperBridge from './harperBridge/harperBridge.ts'; +// Lazy access to handle import cycle (insert.ts and harperBridge.ts both import each other transitively). +const harperBridge: any = new Proxy({}, { get: (_, p) => (_harperBridge as any)[p] }); // eslint-disable-next-line no-unused-vars import ReadAuditLogObject from './ReadAuditLogObject.ts'; import * as hdbUtils from '../utility/common_utils.ts'; diff --git a/dataLayer/schema.ts b/dataLayer/schema.ts index 9d3357d5a..d8d6c4dcc 100644 --- a/dataLayer/schema.ts +++ b/dataLayer/schema.ts @@ -1,6 +1,7 @@ 'use strict'; -import * as schemaMetadataValidator from '../validation/schemaMetadataValidator.ts'; +import * as _schemaMetadataValidator from '../validation/schemaMetadataValidator.ts'; +const schemaMetadataValidator = _schemaMetadataValidator; import { validateBySchema } from '../validation/validationWrapper.ts'; import { commonValidators, schemaRegex } from '../validation/common_validators.ts'; import Joi from 'joi'; @@ -9,12 +10,13 @@ import { v4 as uuidV4 } from 'uuid'; import * as signalling from '../utility/signalling.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; import * as util from 'util'; -const harperBridge = require('./harperBridge/harperBridge').default; +import harperBridge from './harperBridge/harperBridge.ts'; import { handleHDBError, ClientError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; -import { SchemaEventMsg } from '../server/threads/itc.js'; -import { getDatabases, dropTableMeta } from '../resources/databases.ts'; +import { SchemaEventMsg } from '../server/threads/itc.ts'; +import { getDatabases as _getDatabases, dropTableMeta } from '../resources/databases.ts'; +const getDatabases = _getDatabases; import { transformReq } from '../utility/common_utils.ts'; import { server } from '../server/Server.ts'; import { cleanupOrphans } from '../resources/blob.ts'; diff --git a/dataLayer/schemaDescribe.ts b/dataLayer/schemaDescribe.ts index 88b260e85..393e6a78e 100644 --- a/dataLayer/schemaDescribe.ts +++ b/dataLayer/schemaDescribe.ts @@ -8,11 +8,14 @@ import * as hdbUtils from '../utility/common_utils.ts'; import { handleHDBError, ClientError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; -import * as envMngr from '../utility/environment/environmentManager.ts'; -envMngr.initSync(); -import { getDatabases } from '../resources/databases.ts'; -import * as fs from 'fs-extra'; - +import { getDatabases as _getDatabases } from '../resources/databases.ts'; +const getDatabases = _getDatabases; +import fs from 'fs-extra'; +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} /** * This method is exposed to the API and internally for system operations. If the op is being made internally, the `opObj` * argument is not passed and, therefore, no permissions are used to filter the final schema metadata results. diff --git a/dataLayer/search.ts b/dataLayer/search.ts index 2edd0f4ee..7fd27b6f0 100644 --- a/dataLayer/search.ts +++ b/dataLayer/search.ts @@ -1,6 +1,6 @@ 'use strict'; -const harperBridge = require('./harperBridge/harperBridge').default; +import harperBridge from './harperBridge/harperBridge.ts'; import { transformReq } from '../utility/common_utils.ts'; export async function searchByConditions(searchObject: any) { diff --git a/dataLayer/transaction.ts b/dataLayer/transaction.ts deleted file mode 100644 index c2a23f91e..000000000 --- a/dataLayer/transaction.ts +++ /dev/null @@ -1,14 +0,0 @@ -'use strict'; - -const harperBridge = require('./harperBridge/harperBridge').default; - -/** - * This is wrapper for write transactions, ensuring that all reads and writes within the callback occur atomically - * @param schema - * @param table - * @param callback - * @returns {Promise} - */ -export function writeTransaction(schema: string, table: string, callback: any) { - return harperBridge.writeTransaction(schema, table, callback); -} diff --git a/dataLayer/update.ts b/dataLayer/update.ts index 974d5bfa6..45c987c24 100644 --- a/dataLayer/update.ts +++ b/dataLayer/update.ts @@ -3,7 +3,8 @@ import * as search from './search.ts'; import * as globalSchema from '../utility/globalSchema.ts'; import logger from '../utility/logging/harper_logger.ts'; -import * as write from './insert.ts'; +import * as _write from './insert.ts'; +const write = _write; import clone from 'clone'; import * as alasql from 'alasql'; import alasqlFunctionImporter from '../sqlTranslator/alasqlFunctionImporter.ts'; diff --git a/index.ts b/index.ts index 9e10f6dc8..faa1edc8f 100644 --- a/index.ts +++ b/index.ts @@ -109,6 +109,6 @@ exports.transaction = undefined; // And finally assign globals to exports. // These values are populated at runtime by `_assignPackageExport()` in their respective modules // (e.g. Resource.ts, databases.ts, Server.ts, etc.) -import { globals } from './server/threads/threadServer.js'; +import { globals } from './server/threads/threadServer.ts'; Object.assign(exports, globals); diff --git a/integrationTests/server/operations-server.test.ts b/integrationTests/server/operations-server.test.ts index 1fedb271d..145188f59 100644 --- a/integrationTests/server/operations-server.test.ts +++ b/integrationTests/server/operations-server.test.ts @@ -11,7 +11,6 @@ import { suite, test, before, after } from 'node:test'; import { ok, strictEqual } from 'node:assert/strict'; import { pack, unpack } from 'msgpackr'; import { encode, decode } from 'cbor-x'; - import { startHarper, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; suite('Operations Server', (ctx: ContextWithHarper) => { diff --git a/launchServiceScripts/launchHarperDB.js b/launchServiceScripts/launchHarperDB.js deleted file mode 100644 index 7bc839255..000000000 --- a/launchServiceScripts/launchHarperDB.js +++ /dev/null @@ -1,3 +0,0 @@ -'use strict'; - -require('../server/operationsServer.ts').hdbServer(); diff --git a/launchServiceScripts/launchHarperDB.ts b/launchServiceScripts/launchHarperDB.ts new file mode 100644 index 000000000..8e032f986 --- /dev/null +++ b/launchServiceScripts/launchHarperDB.ts @@ -0,0 +1 @@ +import('../server/operationsServer.ts').then((m) => m.hdbServer()); diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 79e3a025b..ee351e459 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -1,5 +1,5 @@ import { cleanupUnusedBlobs } from './blob.ts'; -import { Transaction as LMDBTransaction } from 'lmdb'; +import { type Transaction as LMDBTransaction } from 'lmdb'; import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts'; import { ServerError } from '../utility/errors/hdbError.ts'; import * as harperLogger from '../utility/logging/harper_logger.ts'; diff --git a/resources/ErrorResource.ts b/resources/ErrorResource.ts index 07b86001f..09119a9a3 100644 --- a/resources/ErrorResource.ts +++ b/resources/ErrorResource.ts @@ -1,58 +1,86 @@ import { Resource } from './Resource.ts'; import type { Context } from './ResourceInterface.ts'; + /** * ErrorResource is a Resource that throws an error on any request, communicating to the client when attempts are made * to access endpoints/resources that had an internal error in their configuration or setup. This helps ensure that * if there is a problem with a resource, it is immediately apparent and can be fixed. + * + * The class is constructed lazily on first use, not at module load. ErrorResource + * sits inside Resource.ts's own static-graph SCC, so a class-extends declaration + * at module-top would TDZ on `Resource`. The Proxy here lets `new ErrorResource(x)` + * and `ErrorResource.staticMember` work whenever they're called, even when this + * module is loaded directly (unit tests, scripts) without the lifecycle hooks running. */ -export class ErrorResource extends Resource { - error: Error; - constructor(error: Error) { - super(null as any, null); - this.error = error; - } - isError = true; - allowRead(): never { - throw this.error; - } - allowUpdate(): never { - throw this.error; - } - allowCreate(): never { - throw this.error; - } - allowDelete(): never { - throw this.error; - } - getId(): never { - throw this.error; - } - getContext(): Context { - throw this.error; - } - get(): never { - throw this.error; - } - post(): never { - throw this.error; - } - put(): never { - throw this.error; - } - delete(): never { - throw this.error; - } - connect(): never { - throw this.error; - } - getResource() { - // all child paths resolve back to reporting this error - return this; - } - publish(): never { - throw this.error; - } - subscribe(): never { - throw this.error; +let _ErrorResource: any; +function getErrorResource(): any { + if (!_ErrorResource) { + _ErrorResource = class ErrorResource extends Resource { + error: Error; + constructor(error: Error) { + super(null as any, null); + this.error = error; + } + isError = true; + allowRead(): never { + throw this.error; + } + allowUpdate(): never { + throw this.error; + } + allowCreate(): never { + throw this.error; + } + allowDelete(): never { + throw this.error; + } + getId(): never { + throw this.error; + } + getContext(): Context { + throw this.error; + } + get(): never { + throw this.error; + } + post(): never { + throw this.error; + } + put(): never { + throw this.error; + } + delete(): never { + throw this.error; + } + connect(): never { + throw this.error; + } + getResource() { + // all child paths resolve back to reporting this error + return this; + } + publish(): never { + throw this.error; + } + subscribe(): never { + throw this.error; + } + }; } + return _ErrorResource; } + +export const ErrorResource: any = new Proxy(function () {} as any, { + construct(_target, args) { + return Reflect.construct(getErrorResource(), args); + }, + get(_target, prop) { + return getErrorResource()[prop]; + }, + has(_target, prop) { + return prop in getErrorResource(); + }, + getPrototypeOf() { + return getErrorResource().prototype; + }, +}); diff --git a/resources/LMDBTransaction.ts b/resources/LMDBTransaction.ts index 179752182..5dbd8bf5c 100644 --- a/resources/LMDBTransaction.ts +++ b/resources/LMDBTransaction.ts @@ -3,7 +3,7 @@ import { type CommitOptions, type TransactionWrite, type CommitResolution, -} from './DatabaseTransaction'; +} from './DatabaseTransaction.ts'; import { cleanupUnusedBlobs } from './blob.ts'; import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts'; import * as harperLogger from '../utility/logging/harper_logger.ts'; diff --git a/resources/Resource.ts b/resources/Resource.ts index 9b64354d2..1d36c1abb 100644 --- a/resources/Resource.ts +++ b/resources/Resource.ts @@ -1,13 +1,13 @@ import type { User } from '../security/user.ts'; import type { RecordObject } from './RecordEncoder.ts'; import { - ResourceInterface, - SubscriptionRequest, - Id, - Context, - Query, - SourceContext, - RequestTargetOrId, + type ResourceInterface, + type SubscriptionRequest, + type Id, + type Context, + type Query, + type SourceContext, + type RequestTargetOrId, } from './ResourceInterface.ts'; import { randomUUID } from 'crypto'; import { DatabaseTransaction, type Transaction } from './DatabaseTransaction.ts'; diff --git a/resources/Resources.ts b/resources/Resources.ts index 4f121cf22..169068663 100644 --- a/resources/Resources.ts +++ b/resources/Resources.ts @@ -2,6 +2,7 @@ import { transaction } from './transaction.ts'; import logger from '../utility/logging/harper_logger.ts'; import { ServerError } from '../utility/errors/hdbError.ts'; import { server } from '../server/Server.ts'; +import { ErrorResource } from './ErrorResource.ts'; interface ResourceEntry { Resource: any; @@ -44,7 +45,6 @@ export class Resources extends Map { // don't provide anything more descriptive. const error = new ServerError(`Conflicting paths for ${path}`); logger.error(error); - const { ErrorResource } = require('./ErrorResource'); entry.Resource = new ErrorResource(error); } super.set(path, entry); diff --git a/resources/RocksIndexStore.ts b/resources/RocksIndexStore.ts index c223f8409..49ad4c58a 100644 --- a/resources/RocksIndexStore.ts +++ b/resources/RocksIndexStore.ts @@ -5,9 +5,8 @@ import { type StoreRemoveOptions, RocksDatabase, } from '@harperfast/rocksdb-js'; -import { Id } from './ResourceInterface.ts'; +import { type Id } from './ResourceInterface.ts'; import { MAXIMUM_KEY } from 'ordered-binary'; - declare module '@harperfast/rocksdb-js' { interface DBI { getValuesCount(indexedValue: any): number; diff --git a/resources/RocksTransactionLogStore.ts b/resources/RocksTransactionLogStore.ts index 2be45e0aa..9e68e36d8 100644 --- a/resources/RocksTransactionLogStore.ts +++ b/resources/RocksTransactionLogStore.ts @@ -1,7 +1,7 @@ import { TransactionLog, RocksDatabase, shutdown, type TransactionEntry } from '@harperfast/rocksdb-js'; import { ExtendedIterable } from '@harperfast/extended-iterable'; import { getIdOfRemoteNode } from './nodeIdMapping.ts'; -import { Decoder, readAuditEntry, ENTRY_DATAVIEW, AuditRecord, createAuditEntry } from './auditStore.ts'; +import { Decoder, readAuditEntry, ENTRY_DATAVIEW, type AuditRecord, createAuditEntry } from './auditStore.ts'; import { isMainThread } from 'node:worker_threads'; import { EventEmitter } from 'node:events'; import { asBinary } from 'lmdb'; diff --git a/resources/Table.ts b/resources/Table.ts index 274d30835..140b24da6 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -22,7 +22,7 @@ import type { RequestTargetOrId, } from './ResourceInterface.ts'; import type { User } from '../security/user.ts'; -import lmdbProcessRows from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js'; +import lmdbProcessRows from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.ts'; import { Resource, transformForSelect } from './Resource.ts'; import { when, promiseNormalize } from '../utility/when.ts'; import { DatabaseTransaction, ImmediateTransaction, TRANSACTION_STATE } from './DatabaseTransaction.ts'; @@ -30,7 +30,7 @@ import * as envMngr from '../utility/environment/environmentManager.ts'; import { addSubscription } from './transactionBroadcast.ts'; import { handleHDBError, ClientError, ServerError, AccessViolation } from '../utility/errors/hdbError.ts'; import * as signalling from '../utility/signalling.ts'; -import { SchemaEventMsg, UserEventMsg } from '../server/threads/itc.js'; +import { SchemaEventMsg, UserEventMsg } from '../server/threads/itc.ts'; import { databases, table } from './databases.ts'; import { searchByIndex, @@ -44,7 +44,7 @@ import { logger } from '../utility/logging/logger.ts'; import { Addition, assignTrackedAccessors, updateAndFreeze, hasChanges, GenericTrackedObject } from './tracked.ts'; import { transaction, contextStorage } from './transaction.ts'; import { MAXIMUM_KEY, writeKey, compareKeys } from 'ordered-binary'; -import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js'; +import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.ts'; import { HAS_BLOBS, auditRetention, removeAuditEntry } from './auditStore.ts'; import { autoCast, autoCastBooleanStrict } from '../utility/common_utils.ts'; import { @@ -65,12 +65,11 @@ import { RequestTarget } from './RequestTarget.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; import { throttle } from '../server/throttle.ts'; import { RocksDatabase } from '@harperfast/rocksdb-js'; -import { LMDBTransaction, ImmediateTransaction as ImmediateLMDBTransaction } from './LMDBTransaction'; -import { contentTypes } from '../server/serverHelpers/contentTypes'; +import { LMDBTransaction, ImmediateTransaction as ImmediateLMDBTransaction } from './LMDBTransaction.ts'; +import { contentTypes } from '../server/serverHelpers/contentTypes.ts'; const { sortBy } = lodash; const { validateAttribute } = lmdbProcessRows; - export type Attribute = { name: string; type: 'ID' | 'Int' | 'Float' | 'Long' | 'String' | 'Boolean' | 'Date' | 'Bytes' | 'Any' | 'BigInt' | 'Blob' | string; @@ -101,7 +100,11 @@ NULL_WITH_TIMESTAMP[8] = 0xc0; // null const UNCACHEABLE_TIMESTAMP = Infinity; // we use this when dynamic content is accessed that we can't safely cache, and this prevents earlier timestamps from change the "last" modification const RECORD_PRUNING_INTERVAL = 60000; // one minute const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]); -envMngr.initSync(); +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES); const LOCK_TIMEOUT = 10000; export const INVALIDATED = 1; diff --git a/resources/analytics/write.ts b/resources/analytics/write.ts index 6fc509a6f..1bf2d2ca0 100644 --- a/resources/analytics/write.ts +++ b/resources/analytics/write.ts @@ -1,5 +1,5 @@ import { parentPort, threadId } from 'worker_threads'; -import { onMessageByType } from '../../server/threads/manageThreads.js'; +import { onMessageByType } from '../../server/threads/manageThreads.ts'; import { getDatabases, table, isReadOnlyMode } from '../databases.ts'; import type { Databases, Table, Tables } from '../databases.ts'; import harperLogger from '../../utility/logging/harper_logger.ts'; @@ -8,19 +8,18 @@ const { getLogFilePath, forComponent } = harperLogger; import { dirname, join } from 'path'; import { open } from 'fs/promises'; import { getNextMonotonicTime } from '../../utility/lmdb/commonUtility.ts'; -import { get as envGet, getHdbBasePath, initSync } from '../../utility/environment/environmentManager.ts'; +import { get as envGet, getHdbBasePath } from '../../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS } from '../../utility/hdbTerms.ts'; import { server } from '../../server/Server.ts'; import * as fs from 'node:fs'; import { getAnalyticsHostnameTable, nodeIds, stableNodeId } from './hostnames.ts'; import { METRIC } from './metadata.ts'; import { RocksDatabase } from '@harperfast/rocksdb-js'; +import { onStartup } from '../../utility/lifecycle.ts'; const log = forComponent('analytics').conditional; const isBun = typeof globalThis.Bun !== 'undefined'; -initSync(); - type ActionCallback = (action: Action) => void; export type Value = number | boolean | ActionCallback; interface Action { @@ -128,8 +127,6 @@ export function recordAction(value: Value, metric: string, path?: string, method if (!sendAnalyticsTimeout) sendAnalytics(); } -server.recordAnalytics = recordAction; - export function recordActionBinary(value, metric, path?, method?, type?) { recordAction(Boolean(value), metric, path, method, type); } @@ -708,7 +705,7 @@ function getAnalyticsTable() { ); } -if (!parentPort) onMessageByType(ANALYTICS_REPORT_TYPE, recordAnalytics); +if (!parentPort) setImmediate(() => onMessageByType(ANALYTICS_REPORT_TYPE, recordAnalytics)); let scheduledTasksRunning; function startScheduledTasks() { scheduledTasksRunning = true; @@ -862,3 +859,8 @@ function rebalance({ counts, values, totalCount }, resetCounts: boolean) { else counts.set(targetCounts); } */ + +// Wire server singletons during the startup phase +onStartup(() => { + server.recordAnalytics = recordAction; +}); diff --git a/resources/auditStore.ts b/resources/auditStore.ts index e6ff70b0e..5cd3b26ff 100644 --- a/resources/auditStore.ts +++ b/resources/auditStore.ts @@ -1,8 +1,8 @@ import { readKey, writeKey } from 'ordered-binary'; -import { initSync, get as envGet } from '../utility/environment/environmentManager.ts'; +import { get as envGet } from '../utility/environment/environmentManager.ts'; import { AUDIT_STORE_NAME } from '../utility/lmdb/terms.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; -import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js'; +import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.ts'; import { convertToMS } from '../utility/common_utils.ts'; import { PREVIOUS_TIMESTAMP_PLACEHOLDER, LAST_TIMESTAMP_PLACEHOLDER } from './RecordEncoder.ts'; import * as harperLogger from '../utility/logging/harper_logger.ts'; @@ -30,7 +30,6 @@ import { isReadOnlyMode } from './databases.ts'; * username * remaining bytes (optional, not included for deletes/invalidation): the record itself, using the same encoding as its primary store */ -initSync(); export type AuditRecord = { version: number; diff --git a/resources/blob.ts b/resources/blob.ts index a4f036ae7..ddbb689d6 100644 --- a/resources/blob.ts +++ b/resources/blob.ts @@ -33,7 +33,8 @@ import { import type { StatsFs } from 'node:fs'; import { createDeflate, deflate } from 'node:zlib'; import { Readable, pipeline } from 'node:stream'; -import { ensureDirSync } from 'fs-extra'; +import _fs_extra from 'fs-extra'; +const { ensureDirSync } = _fs_extra; import { get as envGet, getHdbBasePath } from '../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import { join, dirname } from 'path'; diff --git a/resources/dataLoader.ts b/resources/dataLoader.ts index e10c803db..937e5276e 100644 --- a/resources/dataLoader.ts +++ b/resources/dataLoader.ts @@ -1,15 +1,31 @@ import { basename, extname } from 'node:path'; import { createHash } from 'node:crypto'; import { parseDocument } from 'yaml'; -import { Databases, databases, table, Tables, tables } from './databases.ts'; -import { getWorkerIndex } from '../server/threads/manageThreads'; +import { type Databases, databases, table, type Tables, tables } from './databases.ts'; +import { getWorkerIndex } from '../server/threads/manageThreads.ts'; import { HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; import { ClientError } from '../utility/errors/hdbError.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; -import { Attribute } from './Table.ts'; -import { FileEntry } from '../components/EntryHandler.ts'; - -const dataLoaderLogger = harperLogger.forComponent('dataLoader'); +import { type Attribute } from './Table.ts'; +import { type FileEntry } from '../components/EntryHandler.ts'; + +// Resolve the logger lazily so unit tests can stub `forComponent` between +// module load and first use (otherwise transitive importers — componentLoader +// pulled in via http.ts / threadServer.ts / operations.ts — capture the real +// logger before dataLoader.test.js installs its stub). +let _dataLoaderLogger: any; +function getDataLoaderLogger() { + if (!_dataLoaderLogger) _dataLoaderLogger = harperLogger.forComponent('dataLoader'); + return _dataLoaderLogger; +} +const dataLoaderLogger = new Proxy( + {}, + { + get(_t, prop) { + return getDataLoaderLogger()[prop]; + }, + } +) as any; /** System table name for storing data loader hashes */ const DATA_LOADER_HASH_TABLE = 'hdb_dataloader_hash'; diff --git a/resources/databases.ts b/resources/databases.ts index 5cd3492de..c6b1a4ebc 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -8,19 +8,19 @@ import { unlink } from 'node:fs/promises'; import { getBaseSchemaPath, getTransactionAuditStoreBasePath, -} from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js'; +} from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts'; import { makeTable } from './Table.ts'; import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.ts'; import { CONFIG_PARAMS, LEGACY_DATABASES_DIR_NAME, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts'; -import { getConfigPath } from '../config/configUtils.js'; +import { getConfigPath } from '../config/configUtils.ts'; import { _assignPackageExport } from '../globals.js'; import { getIndexedValues } from '../utility/lmdb/commonUtility.ts'; import * as signalling from '../utility/signalling.ts'; -import { SchemaEventMsg } from '../server/threads/itc.js'; +import { SchemaEventMsg } from '../server/threads/itc.ts'; import { workerData } from 'worker_threads'; import harperLogger from '../utility/logging/harper_logger.ts'; const { forComponent } = harperLogger; -import * as manageThreads from '../server/threads/manageThreads.js'; +import * as manageThreads from '../server/threads/manageThreads.ts'; import { openAuditStore, readAuditEntry, createAuditEntry, type AuditRecord } from './auditStore.ts'; import { handleLocalTimeForGets } from './RecordEncoder.ts'; import { deleteRootBlobPathsForDB } from './blob.ts'; @@ -31,7 +31,7 @@ import { replayLogs } from './replayLogs.ts'; import { totalmem } from 'node:os'; import { RocksIndexStore } from './RocksIndexStore.ts'; import { when } from '../utility/when.ts'; -import { isProcessRunning } from '../utility/processManagement/processManagement.js'; +import { isProcessRunning } from '../utility/processManagement/processManagement.ts'; /** * Check if Harper is running in read-only mode. @@ -40,7 +40,7 @@ import { isProcessRunning } from '../utility/processManagement/processManagement * - --readonly CLI flag * - storage.readOnly config setting */ -let _isReadOnlyMode: boolean | undefined; +var _isReadOnlyMode: boolean | undefined; export function isReadOnlyMode(): boolean { if (_isReadOnlyMode !== undefined) return _isReadOnlyMode; // Check environment variable @@ -66,15 +66,24 @@ export function isReadOnlyMode(): boolean { function createOpenDBIObject(dupSort = false, isPrimary = false) { return new OpenDBIObject(dupSort, isPrimary); } -const logger = forComponent('storage'); +var logger = forComponent('storage'); -const DEFAULT_DATABASE_NAME = 'data'; -const DEFINED_TABLES = Symbol('defined-tables'); -const DEFAULT_COMPRESSION_THRESHOLD = (envGet(CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages -initSync(); +var DEFAULT_DATABASE_NAME = 'data'; +var DEFINED_TABLES = Symbol('defined-tables'); +var DEFAULT_COMPRESSION_THRESHOLD = (envGet(CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages +// Initialise env on module load to mirror main-branch behaviour. Tests +// (and the api-test setup in particular) reach env.get(...) through +// configUtils without otherwise calling initSync, so without this the +// flat config object stays uninitialised and downstream code paths +// (e.g. installApplications -> getConfigPath(COMPONENTSROOT)) see null. +try { + initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} // I don't know if this is the best place for this, but somewhere we need to specify which tables // replicate by default: -export const NON_REPLICATING_SYSTEM_TABLES = [ +export var NON_REPLICATING_SYSTEM_TABLES = [ 'hdb_temp', 'hdb_certificate', 'hdb_raw_analytics', @@ -143,12 +152,12 @@ export type DatabaseWatcherEventMap = { dropDatabase: [databaseName: string]; }; -export const databaseEventsEmitter = new EventEmitter(); +export var databaseEventsEmitter = new EventEmitter(); -export const tables: Tables = Object.create(null); -export const databases: Databases = Object.create(null); +export var tables: Tables = Object.create(null); +export var databases: Databases = Object.create(null); -const MEMORY_FOR_ROCKS_DB = Math.min(process.constrainedMemory?.() ?? Infinity, totalmem()) * 0.25; // 25% of available memory +var MEMORY_FOR_ROCKS_DB = Math.min(process.constrainedMemory?.() ?? Infinity, totalmem()) * 0.25; // 25% of available memory function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSort?: boolean }) { options.disableWAL ??= true; @@ -179,19 +188,19 @@ function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSo return db; } -const lmdbDatabaseEnvs = new Map(); -const rocksdbDatabaseEnvs = new Map(); +var lmdbDatabaseEnvs = new Map(); +var rocksdbDatabaseEnvs = new Map(); // set the following in both global and exports _assignPackageExport('databases', databases); _assignPackageExport('tables', tables); -const NEXT_TABLE_ID = Symbol.for('next-table-id'); -let loadedDatabases; // indicates if we have loaded databases from the file system yet +var NEXT_TABLE_ID = Symbol.for('next-table-id'); +var loadedDatabases; // indicates if we have loaded databases from the file system yet // This is used to track all the databases that are found when iterating through the file system so that anything that is missing // can be removed: -let definedDatabases: Map>; +var definedDatabases: Map>; /** * This gets the set of tables from the default database ("data"). @@ -1209,8 +1218,8 @@ export function table(tableDefinition: TableDefinition): Tabl } } } -const MAX_OUTSTANDING_INDEXING = 1000; -const MIN_OUTSTANDING_INDEXING = 10; +var MAX_OUTSTANDING_INDEXING = 1000; +var MIN_OUTSTANDING_INDEXING = 10; async function runIndexing(Table, attributes, indicesToRemove) { try { logger.info(`Indexing ${Table.tableName} attributes`, attributes); diff --git a/resources/graphql.ts b/resources/graphql.ts index f70672838..cb1ed2250 100644 --- a/resources/graphql.ts +++ b/resources/graphql.ts @@ -1,7 +1,7 @@ import { dirname } from 'path'; import { Script } from 'node:vm'; import { table } from './databases.ts'; -import { getWorkerIndex } from '../server/threads/manageThreads.js'; +import { getWorkerIndex } from '../server/threads/manageThreads.ts'; import { Resources } from './Resources.ts'; import type { NamedTypeNode, StringValueNode } from 'graphql'; import { once } from 'node:events'; diff --git a/resources/login.ts b/resources/login.ts index 842c6882f..a93c58c5e 100644 --- a/resources/login.ts +++ b/resources/login.ts @@ -1,20 +1,30 @@ import { Resource } from './Resource.ts'; -import { Scope } from '../components/Scope.ts'; +import type { Scope } from '../components/Scope.ts'; + +// Login class is created lazily because login.ts is loaded transitively from +// Resource.ts's own static graph; declaring `class Login extends Resource` at +// module-load would hit Resource's TDZ under that ESM cycle. +let LoginClass: any; +function getLoginClass() { + if (LoginClass) return LoginClass; + // @ts-ignore + LoginClass = class Login extends Resource { + static async get(_id, _body, _request) { + // TODO: Return a login page + } + static async post(_id, body, request) { + const { username, password } = body; + return { + data: await request.login(username, password), + }; + } + }; + return LoginClass; +} + export function handleApplication(scope: Scope) { - scope.resources.set('login', Login); + scope.resources.set('login', getLoginClass()); scope.resources.loginPath = (request) => { return '/login?redirect=' + encodeURIComponent(request.url); }; } -// @ts-ignore -class Login extends Resource { - static async get(_id, _body, _request) { - // TODO: Return a login page - } - static async post(_id, body, request) { - const { username, password } = body; - return { - data: await request.login(username, password), - }; - } -} diff --git a/resources/replayLogs.ts b/resources/replayLogs.ts index 68eec666c..45e0a52c8 100644 --- a/resources/replayLogs.ts +++ b/resources/replayLogs.ts @@ -1,7 +1,7 @@ import { RocksDatabase, Transaction as RocksTransaction } from '@harperfast/rocksdb-js'; import { Resource } from './Resource.ts'; import type { Context } from './ResourceInterface.ts'; -import * as logger from '../utility/logging/harper_logger.js'; +import * as logger from '../utility/logging/harper_logger.ts'; import { DatabaseTransaction } from './DatabaseTransaction.ts'; import { RocksTransactionLogStore } from './RocksTransactionLogStore.ts'; import { isMainThread } from 'node:worker_threads'; diff --git a/resources/roles.ts b/resources/roles.ts index 117ed0f17..23e608b1c 100644 --- a/resources/roles.ts +++ b/resources/roles.ts @@ -1,8 +1,8 @@ import { getDatabases } from './databases.ts'; import { alterRole, addRole } from '../security/role.ts'; import { parseDocument } from 'yaml'; -import { isEqual } from 'lodash'; - +import _lodash from 'lodash'; +const { isEqual } = _lodash; const USERS_NOT_DBS = ['super_user', 'structure_user']; /** diff --git a/resources/search.ts b/resources/search.ts index 6d10fc5b1..f40b3ecd1 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -6,7 +6,7 @@ import { INVALIDATED, EVICTED } from './Table.ts'; import type { DirectCondition, Id } from './ResourceInterface.ts'; import { RequestTarget } from './RequestTarget.ts'; import { lastMetadata } from './RecordEncoder.ts'; -import { recordAction } from './analytics/write'; +import { recordAction } from './analytics/write.ts'; import { RocksDatabase } from '@harperfast/rocksdb-js'; // these are ratios/percentages of overall table size diff --git a/resources/transactionBroadcast.ts b/resources/transactionBroadcast.ts index db0b8d320..e8a2f5749 100644 --- a/resources/transactionBroadcast.ts +++ b/resources/transactionBroadcast.ts @@ -1,4 +1,4 @@ -import { warn } from '../utility/logging/harper_logger.js'; +import { warn } from '../utility/logging/harper_logger.ts'; import { IterableEventQueue } from './IterableEventQueue.ts'; import { keyArrayToString } from './Resources.ts'; import type { Id } from './ResourceInterface.ts'; diff --git a/security/auth.ts b/security/auth.ts index f286bd64e..88fd8dec8 100644 --- a/security/auth.ts +++ b/security/auth.ts @@ -8,45 +8,63 @@ import * as env from '../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS, AUTH_AUDIT_STATUS, AUTH_AUDIT_TYPES } from '../utility/hdbTerms.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; const { forComponent, AuthAuditLog } = harperLogger; -import serverHandlers from '../server/itc/serverHandlers.js'; +import serverHandlers from '../server/itc/serverHandlers.ts'; const { user } = serverHandlers; import { Headers } from '../server/serverHelpers/Headers.ts'; import { convertToMS } from '../utility/common_utils.ts'; import { verifyCertificate } from './certificateVerification/index.ts'; import { serializeMessage } from '../server/serverHelpers/contentTypes.ts'; +import { onStartup } from '../utility/lifecycle.ts'; const authLogger = forComponent('authentication'); const { debug } = authLogger; const authEventLog = authLogger.withTag('auth-event'); -env.initSync(); - -const appsCorsAccesslist = env.get(CONFIG_PARAMS.HTTP_CORSACCESSLIST); -const appsCors = env.get(CONFIG_PARAMS.HTTP_CORS); -const operationsCorsAccesslist = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORSACCESSLIST); -const operationsCors = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORS); +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} +// Config-derived state is populated during startup, after the environment is +// initialized and the module graph is fully linked. Reading config here at +// module-load would either TDZ inside an ESM cycle or pick up stale defaults. +let appsCorsAccesslist: any; +let appsCors: any; +let operationsCorsAccesslist: any; +let operationsCors: any; +let _sessionTable: Table | undefined; +let ENABLE_SESSIONS: boolean = true; +let AUTHORIZE_LOCAL: any = process.env.AUTHENTICATION_AUTHORIZELOCAL ?? process.env.DEV_MODE; +let LOG_AUTH_SUCCESSFUL: boolean = false; +let LOG_AUTH_FAILED: boolean = false; -const _sessionTable = table({ - table: 'hdb_session', - database: 'system', - attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'user' }], -}); function getSessionTable() { return _sessionTable; } -const ENABLE_SESSIONS = env.get(CONFIG_PARAMS.AUTHENTICATION_ENABLESESSIONS) ?? true; -// check the environment for a flag to bypass authentication (for testing) since it doesn't necessarily get set on child threads -let AUTHORIZE_LOCAL = - process.env.AUTHENTICATION_AUTHORIZELOCAL ?? - env.get(CONFIG_PARAMS.AUTHENTICATION_AUTHORIZELOCAL) ?? - process.env.DEV_MODE; -const LOG_AUTH_SUCCESSFUL = env.get(CONFIG_PARAMS.LOGGING_AUDITAUTHEVENTS_LOGSUCCESSFUL) ?? false; -const LOG_AUTH_FAILED = env.get(CONFIG_PARAMS.LOGGING_AUDITAUTHEVENTS_LOGFAILED) ?? false; const DEFAULT_COOKIE_EXPIRES = 'Tue, 01 Oct 8307 19:33:20 GMT'; let authorizationCache = new Map(); -server.onInvalidatedUser(() => { - // TODO: Eventually we probably want to be able to invalidate individual users - authorizationCache = new Map(); + +onStartup(() => { + appsCorsAccesslist = env.get(CONFIG_PARAMS.HTTP_CORSACCESSLIST); + appsCors = env.get(CONFIG_PARAMS.HTTP_CORS); + operationsCorsAccesslist = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORSACCESSLIST); + operationsCors = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORS); + ENABLE_SESSIONS = env.get(CONFIG_PARAMS.AUTHENTICATION_ENABLESESSIONS) ?? true; + AUTHORIZE_LOCAL = + process.env.AUTHENTICATION_AUTHORIZELOCAL ?? + env.get(CONFIG_PARAMS.AUTHENTICATION_AUTHORIZELOCAL) ?? + process.env.DEV_MODE; + LOG_AUTH_SUCCESSFUL = env.get(CONFIG_PARAMS.LOGGING_AUDITAUTHEVENTS_LOGSUCCESSFUL) ?? false; + LOG_AUTH_FAILED = env.get(CONFIG_PARAMS.LOGGING_AUDITAUTHEVENTS_LOGFAILED) ?? false; + _sessionTable = table
({ + table: 'hdb_session', + database: 'system', + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'user' }], + }); + server.onInvalidatedUser(() => { + // TODO: Eventually we probably want to be able to invalidate individual users + authorizationCache = new Map(); + }); }); let bypassUser: any; export function bypassAuth() { diff --git a/security/certificateVerification/certificateVerificationSource.ts b/security/certificateVerification/certificateVerificationSource.ts index 9f304c2c3..2202d1879 100644 --- a/security/certificateVerification/certificateVerificationSource.ts +++ b/security/certificateVerification/certificateVerificationSource.ts @@ -23,62 +23,88 @@ async function loadVerificationFunctions() { } /** - * Certificate Verification Source that can handle both CRL and OCSP + * Certificate Verification Source that can handle both CRL and OCSP. + * + * Late-bound via a Proxy because this module is loaded inside Resource.ts's + * own static-graph SCC (via the auth.ts → server-utilities chain), so a + * class-extends declaration at module-top would TDZ on `Resource`. The Proxy + * defers the `extends Resource` evaluation to first construct/access. */ -export class CertificateVerificationSource extends Resource { - async get(query: Query) { - const id = query.id as string; +let _CertificateVerificationSource: any; +function getCertificateVerificationSource(): any { + if (!_CertificateVerificationSource) { + _CertificateVerificationSource = class CertificateVerificationSource extends Resource { + async get(query: Query) { + const id = query.id as string; - // Get the certificate data from requestContext - const context = this.getContext() as SourceContext; - const requestContext = context?.requestContext; + // Get the certificate data from requestContext + const context = this.getContext() as SourceContext; + const requestContext = context?.requestContext; - if (!requestContext || !requestContext.certPem || !requestContext.issuerPem) { - // Likely a source request for an expired entry - we can't verify without cert and issuer data - return null; - } + if (!requestContext || !requestContext.certPem || !requestContext.issuerPem) { + // Likely a source request for an expired entry - we can't verify without cert and issuer data + return null; + } - const { certPem: certPemStr, issuerPem: issuerPemStr, ocspUrls, config } = requestContext; + const { certPem: certPemStr, issuerPem: issuerPemStr, ocspUrls, config } = requestContext; - // Determine method from cache key - let method: string; - if (id.startsWith('crl:')) { - method = 'crl'; - } else if (id.startsWith('ocsp:')) { - method = 'ocsp'; - } else { - method = 'unknown'; - } + // Determine method from cache key + let method: string; + if (id.startsWith('crl:')) { + method = 'crl'; + } else if (id.startsWith('ocsp:')) { + method = 'ocsp'; + } else { + method = 'unknown'; + } - // Load verification functions - await loadVerificationFunctions(); + // Load verification functions + await loadVerificationFunctions(); - // Perform verification based on method - let result; - let methodConfig; + // Perform verification based on method + let result; + let methodConfig; - if (method === 'crl') { - methodConfig = config.crl; - // Pass distributionPoint as an array if available (for CRL fetch) - const crlUrls = requestContext.distributionPoint ? [requestContext.distributionPoint] : undefined; - result = await performCRLCheck(certPemStr, issuerPemStr, methodConfig, crlUrls); - } else if (method === 'ocsp') { - methodConfig = config.ocsp; - result = await performOCSPCheck(certPemStr, issuerPemStr, methodConfig, ocspUrls); - } else { - throw new Error(`Unsupported verification method: ${method} for ID: ${id}`); - } + if (method === 'crl') { + methodConfig = config.crl; + // Pass distributionPoint as an array if available (for CRL fetch) + const crlUrls = requestContext.distributionPoint ? [requestContext.distributionPoint] : undefined; + result = await performCRLCheck(certPemStr, issuerPemStr, methodConfig, crlUrls); + } else if (method === 'ocsp') { + methodConfig = config.ocsp; + result = await performOCSPCheck(certPemStr, issuerPemStr, methodConfig, ocspUrls); + } else { + throw new Error(`Unsupported verification method: ${method} for ID: ${id}`); + } - // Handle result consistently - const expiresAt = Date.now() + methodConfig.cacheTtl; + // Handle result consistently + const expiresAt = Date.now() + methodConfig.cacheTtl; - return { - certificate_id: id, - status: result.status, - reason: result.reason, - checked_at: Date.now(), - expiresAt, - method, + return { + certificate_id: id, + status: result.status, + reason: result.reason, + checked_at: Date.now(), + expiresAt, + method, + }; + } }; } + return _CertificateVerificationSource; } + +export const CertificateVerificationSource: any = new Proxy(function () {} as any, { + construct(_target, args) { + return Reflect.construct(getCertificateVerificationSource(), args); + }, + get(_target, prop) { + return getCertificateVerificationSource()[prop]; + }, + has(_target, prop) { + return prop in getCertificateVerificationSource(); + }, + getPrototypeOf() { + return getCertificateVerificationSource().prototype; + }, +}); diff --git a/security/certificateVerification/crlVerification.ts b/security/certificateVerification/crlVerification.ts index decef6a2c..6a270599e 100644 --- a/security/certificateVerification/crlVerification.ts +++ b/security/certificateVerification/crlVerification.ts @@ -54,62 +54,89 @@ function getCertificateCacheTable() { } /** - * CRL fetching and validation source + * CRL fetching and validation source. + * + * Late-bound via a Proxy because this module sits inside Resource.ts's + * static-graph SCC, so a class-extends declaration at module-top would TDZ on + * `Resource`. The Proxy defers the `extends Resource` evaluation to first + * construct/access — same pattern as resources/ErrorResource.ts and + * security/certificateVerification/certificateVerificationSource.ts. */ -class CertificateRevocationListSource extends Resource { - async get(id: string) { - const context = this.getContext() as SourceContext; - const requestContext = context?.requestContext; - - if (!requestContext?.distributionPoint || !requestContext?.issuerPem) { - throw new Error(`No CRL data provided for cache key: ${id}`); - } +let _CertificateRevocationListSource: any; +function getCertificateRevocationListSource(): any { + if (!_CertificateRevocationListSource) { + _CertificateRevocationListSource = class CertificateRevocationListSource extends Resource { + async get(id: string) { + const context = this.getContext() as SourceContext; + const requestContext = context?.requestContext; + + if (!requestContext?.distributionPoint || !requestContext?.issuerPem) { + throw new Error(`No CRL data provided for cache key: ${id}`); + } - const { distributionPoint, issuerPem: issuerPemStr, config } = requestContext; + const { distributionPoint, issuerPem: issuerPemStr, config } = requestContext; - try { - const result = await downloadAndParseCRL(distributionPoint, issuerPemStr, config.timeout); + try { + const result = await downloadAndParseCRL(distributionPoint, issuerPemStr, config.timeout); - // Set expiration - use the CRL's nextUpdate time or configured TTL, whichever is sooner - const crlExpiry = result.next_update; - const configExpiry = Date.now() + config.cacheTtl; - const expiresAt = Math.min(crlExpiry, configExpiry); + // Set expiration - use the CRL's nextUpdate time or configured TTL, whichever is sooner + const crlExpiry = result.next_update; + const configExpiry = Date.now() + config.cacheTtl; + const expiresAt = Math.min(crlExpiry, configExpiry); - return { - ...result, - expiresAt, - }; - } catch (error) { - logger.error?.(`CRL fetch error for: ${distributionPoint} - ${error}`); + return { + ...result, + expiresAt, + }; + } catch (error) { + logger.error?.(`CRL fetch error for: ${distributionPoint} - ${error}`); - if (error instanceof CRLSignatureVerificationError) { - throw error; - } + if (error instanceof CRLSignatureVerificationError) { + throw error; + } - // Check failure mode - if (config.failureMode === 'fail-closed') { - // Cache the error for faster recovery - const expiresAt = Date.now() + ERROR_CACHE_TTL; + // Check failure mode + if (config.failureMode === 'fail-closed') { + // Cache the error for faster recovery + const expiresAt = Date.now() + ERROR_CACHE_TTL; + + return { + crl_id: id, + distribution_point: distributionPoint, + issuer_dn: 'unknown', + crl_blob: Buffer.alloc(0), + this_update: Date.now(), + next_update: expiresAt, + signature_valid: false, + expiresAt, + }; + } - return { - crl_id: id, - distribution_point: distributionPoint, - issuer_dn: 'unknown', - crl_blob: Buffer.alloc(0), - this_update: Date.now(), - next_update: expiresAt, - signature_valid: false, - expiresAt, - }; + // Fail open - return null to not cache + logger.warn?.('CRL fetch failed, not caching (fail-open mode)'); + return null; + } } - - // Fail open - return null to not cache - logger.warn?.('CRL fetch failed, not caching (fail-open mode)'); - return null; - } + }; } + return _CertificateRevocationListSource; } +const CertificateRevocationListSource: any = new Proxy(function () {} as any, { + construct(_target, args) { + return Reflect.construct(getCertificateRevocationListSource(), args); + }, + get(_target, prop) { + return getCertificateRevocationListSource()[prop]; + }, + has(_target, prop) { + return prop in getCertificateRevocationListSource(); + }, + getPrototypeOf() { + return getCertificateRevocationListSource().prototype; + }, +}); + // Lazy-load Harper tables let crlCacheTable: ReturnType; let revokedCertificateTable: ReturnType; diff --git a/security/fastifyAuth.ts b/security/fastifyAuth.ts index 004d353ee..adbdd5be3 100644 --- a/security/fastifyAuth.ts +++ b/security/fastifyAuth.ts @@ -8,7 +8,8 @@ import * as util from 'util'; import * as userFunctions from './user.ts'; const cbFindValidateUsers = util.callbackify(userFunctions.findAndValidateUser); import * as hdbTerms from '../utility/hdbTerms.ts'; -import * as tokenAuthentication from './tokenAuthentication.ts'; +import * as _tokenAuthentication from './tokenAuthentication.ts'; +const tokenAuthentication = _tokenAuthentication; import { AccessViolation } from '../utility/errors/hdbError.ts'; import { authentication } from './auth.ts'; diff --git a/security/jsLoader.ts b/security/jsLoader.ts index ed067e37b..9d5d184e4 100644 --- a/security/jsLoader.ts +++ b/security/jsLoader.ts @@ -5,11 +5,18 @@ import { tables, databases } from '../resources/databases.ts'; import { readFile } from 'node:fs/promises'; import { dirname, isAbsolute } from 'node:path'; import { pathToFileURL, fileURLToPath } from 'node:url'; -import { SourceTextModule, SyntheticModule, createContext, runInContext, runInThisContext } from 'node:vm'; +import * as _vm from 'node:vm'; +import { createContext, runInContext, runInThisContext } from 'node:vm'; +// SourceTextModule and SyntheticModule require `--experimental-vm-modules`. Pull +// them off the vm namespace at runtime so the named ESM import doesn't fail when +// the flag isn't passed (e.g. CLI paths that never touch the JS loader). +const { SourceTextModule, SyntheticModule } = _vm as any; +type SourceTextModule = any; +type SyntheticModule = any; import { ApplicationScope } from '../components/ApplicationScope.ts'; import logger from '../utility/logging/harper_logger.ts'; import { createRequire } from 'node:module'; -import * as env from '../utility/environment/environmentManager'; +import * as env from '../utility/environment/environmentManager.ts'; import * as child_process from 'node:child_process'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import { contentTypes } from '../server/serverHelpers/contentTypes.ts'; @@ -26,7 +33,7 @@ import { } from 'node:fs'; import { join } from 'node:path'; import { EventEmitter } from 'node:events'; -import { whenComponentsLoaded } from '../server/threads/threadServer.js'; +import { whenComponentsLoaded } from '../server/threads/threadServer.ts'; type Lockdown = 'none' | 'freeze' | 'ses' | 'freeze-after-load'; const APPLICATIONS_LOCKDOWN: Lockdown = env.get(CONFIG_PARAMS.APPLICATIONS_LOCKDOWN); @@ -722,7 +729,6 @@ const ALLOWED_NODE_BUILTIN_MODULES = env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDB return true; }, }; -const ALLOWED_COMMANDS = new Set(env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDSPAWNCOMMANDS) ?? []); const child_processConstrained: any = { exec: createSpawn(child_process.exec), execFile: createSpawn(child_process.execFile), @@ -878,8 +884,14 @@ function acquirePidFileLock( } function createSpawn(spawnFunction: (...args: any) => child_process.ChildProcess, alwaysAllow?: boolean) { - const basePath = env.getHdbBasePath(); return function (command: string, args?: any, options?: any, callback?: (...args: any[]) => void) { + // Resolved lazily because jsLoader may be evaluated before the environment + // is initialized (e.g. component loading paths in unit tests). Mirror + // defaultConfig.yaml's applications.allowedSpawnCommands as the + // pre-config fallback so the allow-list isn't empty in that window. + const basePath = env.getHdbBasePath(); + const allowedSpawn = env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDSPAWNCOMMANDS) ?? ['npm', 'node']; + const ALLOWED_COMMANDS = new Set(allowedSpawn); if (!ALLOWED_COMMANDS.has(command.split(' ')[0]) && !alwaysAllow) { throw new Error(`Command ${command} is not allowed`); } diff --git a/security/keys.ts b/security/keys.ts index 759ac8fff..62f83cfaf 100644 --- a/security/keys.ts +++ b/security/keys.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import { watch } from 'chokidar'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as forge from 'node-forge'; import * as net from 'net'; import { generateKeyPair as generateKeyPairOrig, X509Certificate, createPrivateKey, randomBytes } from 'node:crypto'; @@ -17,11 +17,12 @@ import * as envManager from '../utility/environment/environmentManager.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; import * as certificatesTerms from '../utility/terms/certificates.js'; -const tls = require('node:tls'); +import tls from 'node:tls'; import { relative, join } from 'node:path'; import assignCmdenvVars from '../utility/assignCmdEnvVariables.ts'; -import * as configUtils from '../config/configUtils.js'; +import * as configUtils from '../config/configUtils.ts'; +import { filterArgsAgainstRuntimeConfig } from '../config/harperConfigEnvVars.ts'; import { table, getDatabases, databases } from '../resources/databases.ts'; const logger = forComponent('tls').conditional; const { CONFIG_PARAMS } = hdbTerms; @@ -31,7 +32,7 @@ import { getThisNodeName, getThisNodeUrl, urlToNodeName, clearThisNodeName } fro export const getPrivateKeys = () => privateKeys; import { readFileSync, statSync } from 'node:fs'; -import { getTicketKeys, onMessageFromWorkers } from '../server/threads/manageThreads.js'; +import { getTicketKeys, onMessageFromWorkers } from '../server/threads/manageThreads.ts'; import { isMainThread } from 'worker_threads'; import { TLSSocket } from 'node:tls'; @@ -58,12 +59,15 @@ export function generateSerialNumber() { return bytes.toString('hex'); } -onMessageFromWorkers(async (message) => { - if (message.type === hdbTerms.ITC_EVENT_TYPES.RESTART) { - envManager.initSync(true); - // This will also call loadCertificates - await reviewSelfSignedCert(); - } +// Defer registration to setImmediate so manageThreads internal state is initialized +setImmediate(() => { + onMessageFromWorkers(async (message) => { + if (message.type === hdbTerms.ITC_EVENT_TYPES.RESTART) { + envManager.initSync(true); + // This will also call loadCertificates + await reviewSelfSignedCert(); + } + }); }); let certificateTable; @@ -652,7 +656,6 @@ export function updateConfigCert() { // Filter out any cert config keys already set by HARPER_SET_CONFIG so we don't overwrite them // with defaults. On first boot, HARPER_SET_CONFIG values are written to the config file during // createConfigFile(), but updateConfigCert() runs afterward without re-applying HARPER_SET_CONFIG. - const { filterArgsAgainstRuntimeConfig } = require('../config/harperConfigEnvVars'); const filteredCerts = filterArgsAgainstRuntimeConfig(newCerts); configUtils.updateConfigValue(undefined, undefined, filteredCerts, false, true); diff --git a/security/permissionsTranslator.js b/security/permissionsTranslator.ts similarity index 97% rename from security/permissionsTranslator.js rename to security/permissionsTranslator.ts index d921d4ec7..6b9a72da0 100644 --- a/security/permissionsTranslator.js +++ b/security/permissionsTranslator.ts @@ -1,15 +1,10 @@ -'use strict'; - -const _ = require('lodash'); -const terms = require('../utility/hdbTerms.ts'); -const { handleHDBError, hdbErrors } = require('../utility/errors/hdbError.ts'); +import _ from 'lodash'; +import * as terms from '../utility/hdbTerms.ts'; +import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; -const logger = require('../utility/logging/harper_logger.ts'); - -module.exports = { - getRolePermissions, -}; +import logger from '../utility/logging/harper_logger.ts'; +export { getRolePermissions }; const rolePermsMap = Object.create(null); const permsTemplateObj = (permsKey) => ({ key: permsKey, perms: {} }); diff --git a/security/role.ts b/security/role.ts index 53eac59ca..0c8ad80d1 100644 --- a/security/role.ts +++ b/security/role.ts @@ -17,7 +17,7 @@ import SearchByHashObject from '../dataLayer/SearchByHashObject.ts'; import { handleHDBError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; -import { UserEventMsg } from '../server/threads/itc.js'; +import { UserEventMsg } from '../server/threads/itc.ts'; function scrubRoleDetails(role) { try { diff --git a/security/tokenAuthentication.ts b/security/tokenAuthentication.ts index 54d12cac8..1de7945e0 100644 --- a/security/tokenAuthentication.ts +++ b/security/tokenAuthentication.ts @@ -18,10 +18,13 @@ import { findAndValidateUser, type User } from './user.ts'; import { update } from '../dataLayer/insert.ts'; import UpdateObject from '../dataLayer/UpdateObject.ts'; import * as signalling from '../utility/signalling.ts'; -import { UserEventMsg } from '../server/threads/itc.js'; +import { UserEventMsg } from '../server/threads/itc.ts'; import * as env from '../utility/environment/environmentManager.ts'; -env.initSync(); - +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} type StringValue = SignOptions['expiresIn']; const OPERATION_TOKEN_TIMEOUT: StringValue = env.get(CONFIG_PARAMS.AUTHENTICATION_OPERATIONTOKENTIMEOUT) || '1d'; const REFRESH_TOKEN_TIMEOUT: StringValue = env.get(CONFIG_PARAMS.AUTHENTICATION_REFRESHTOKENTIMEOUT) || '30d'; diff --git a/security/user.ts b/security/user.ts index 7013dfe52..513785040 100644 --- a/security/user.ts +++ b/security/user.ts @@ -92,11 +92,11 @@ import * as validate from 'validate.js'; import * as logger from '../utility/logging/harper_logger.ts'; import { promisify } from 'util'; import * as env from '../utility/environment/environmentManager.ts'; -import systemSchema from '../json/systemSchema.json'; +import systemSchema from '../json/systemSchema.json' with { type: 'json' }; import { hdbErrors, ClientError } from '../utility/errors/hdbError.ts'; const { HTTP_STATUS_CODES, AUTHENTICATION_ERROR_MSGS, HDB_ERROR_MSGS } = hdbErrors; -const { UserEventMsg } = require('../server/threads/itc.js'); -import * as _ from 'lodash'; +import { UserEventMsg } from '../server/threads/itc.ts'; +import _ from 'lodash'; import * as harperLogger from '../utility/logging/harper_logger.ts'; // Need to use `.js` even for other TS files since TS compiler won't replace requires. @@ -105,14 +105,15 @@ import * as password from '../utility/password.ts'; import { server } from '../server/Server.ts'; import * as terms from '../utility/hdbTerms.ts'; import { expandOperationsPerms } from '../utility/operationPermissions.ts'; +import { onStartup } from '../utility/lifecycle.ts'; -server.getUser = (username: string, password?: string | null): Promise => { +function getUserImpl(username: string, password?: string | null): Promise { return findAndValidateUser(username, password, password != null); -}; +} -server.authenticateUser = (username: string, password?: string | null): Promise => { +function authenticateUserImpl(username: string, password?: string | null): Promise { return findAndValidateUser(username, password); -}; +} const USER_ATTRIBUTE_ALLOWLIST = { username: true, @@ -440,7 +441,7 @@ async function getSuperUser(): Promise { } let invalidateCallbacks = []; -(server as any).invalidateUser = function (user: User | any) { +function invalidateUserImpl(user: User | any) { for (let callback of invalidateCallbacks) { try { callback(user); @@ -448,8 +449,16 @@ let invalidateCallbacks = []; harperLogger.error('Error invalidating user', error); } } -}; +} -server.onInvalidatedUser = function (callback) { +function onInvalidatedUserImpl(callback) { invalidateCallbacks.push(callback); -}; +} + +// Wire server singletons during the startup phase +onStartup(() => { + server.getUser = getUserImpl; + server.authenticateUser = authenticateUserImpl; + server.onInvalidatedUser = onInvalidatedUserImpl; + (server as any).invalidateUser = invalidateUserImpl; +}); diff --git a/server/DurableSubscriptionsSession.ts b/server/DurableSubscriptionsSession.ts index ea8f58241..285ab818d 100644 --- a/server/DurableSubscriptionsSession.ts +++ b/server/DurableSubscriptionsSession.ts @@ -1,14 +1,15 @@ import { table } from '../resources/databases.ts'; +import { onStartup } from '../utility/lifecycle.ts'; import { keyArrayToString, resources } from '../resources/Resources.ts'; import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts'; import { warn, trace } from '../utility/logging/harper_logger.ts'; import { transaction } from '../resources/transaction.ts'; -import { getWorkerIndex } from '../server/threads/manageThreads.js'; -import { whenComponentsLoaded } from '../server/threads/threadServer.js'; +import { getWorkerIndex } from '../server/threads/manageThreads.ts'; +import { whenComponentsLoaded } from '../server/threads/threadServer.ts'; import { server } from '../server/Server.ts'; -import { RequestTarget } from '../resources/RequestTarget'; -import { cloneDeep } from 'lodash'; - +import { RequestTarget } from '../resources/RequestTarget.ts'; +import _lodash from 'lodash'; +const { cloneDeep } = _lodash; const AWAITING_ACKS_HIGH_WATER_MARK = 100; let _DurableSession: any; function getDurableSession() { @@ -49,7 +50,10 @@ function getLastWill() { } return _LastWill; } -if (getWorkerIndex() === 0) { +// Defer to startup so `whenComponentsLoaded` (an `export const` in threadServer.ts) +// is past TDZ — this module is loaded inside threadServer's static-graph SCC. +onStartup(() => { + if (getWorkerIndex() !== 0) return; (async () => { await whenComponentsLoaded; await new Promise((resolve) => setTimeout(resolve, 2000)); @@ -65,7 +69,7 @@ if (getWorkerIndex() === 0) { getLastWill().delete(will.id); } })(); -} +}); /** * This is used for durable sessions, that is sessions in MQTT that are not "clean" sessions (and with QoS >= 1 diff --git a/server/REST.ts b/server/REST.ts index 3791cde78..9f7689640 100644 --- a/server/REST.ts +++ b/server/REST.ts @@ -10,7 +10,7 @@ import { Headers, mergeHeaders } from '../server/serverHelpers/Headers.ts'; import { generateJsonApi } from '../resources/openApi.ts'; import { Request } from '../server/serverHelpers/Request.ts'; -import { RequestTarget } from '../resources/RequestTarget'; +import { RequestTarget } from '../resources/RequestTarget.ts'; const { errorToString } = harperLogger; const etagBytes = new Uint8Array(8); diff --git a/server/Server.ts b/server/Server.ts index 43e8fb938..950669d72 100644 --- a/server/Server.ts +++ b/server/Server.ts @@ -2,7 +2,7 @@ import { Socket } from 'net'; import { _assignPackageExport } from '../globals.js'; import type { Value } from '../resources/analytics/write.ts'; import type { Resources } from '../resources/Resources.ts'; -import { OperationDefinition } from './serverHelpers/serverUtilities.ts'; +import { type OperationDefinition } from './serverHelpers/serverUtilities.ts'; import { Duplex } from 'stream'; import { Request } from './serverHelpers/Request.ts'; diff --git a/server/fastifyRoutes.ts b/server/fastifyRoutes.ts index 1a751bee3..bddef35ac 100644 --- a/server/fastifyRoutes.ts +++ b/server/fastifyRoutes.ts @@ -2,17 +2,17 @@ import { dirname, basename } from 'path'; import { existsSync } from 'fs'; import fastify from 'fastify'; import fastifyCors from '@fastify/cors'; -import requestTimePlugin from './serverHelpers/requestTimePlugin.js'; +import requestTimePlugin from './serverHelpers/requestTimePlugin.ts'; import autoload from '@fastify/autoload'; import * as env from '../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import * as harperLogger from '../utility/logging/harper_logger.ts'; -import * as hdbCore from './fastifyRoutes/plugins/hdbCore.js'; +import * as hdbCore from './fastifyRoutes/plugins/hdbCore.ts'; import * as userSchema from '../security/user.ts'; -import getServerOptions from './fastifyRoutes/helpers/getServerOptions.js'; -import getCORSOptions from './fastifyRoutes/helpers/getCORSOptions.js'; -import getHeaderTimeoutConfig from './fastifyRoutes/helpers/getHeaderTimeoutConfig.js'; -import { serverErrorHandler } from '../server/serverHelpers/serverHandlers.js'; +import getServerOptions from './fastifyRoutes/helpers/getServerOptions.ts'; +import getCORSOptions from './fastifyRoutes/helpers/getCORSOptions.ts'; +import getHeaderTimeoutConfig from './fastifyRoutes/helpers/getHeaderTimeoutConfig.ts'; +import { serverErrorHandler } from '../server/serverHelpers/serverHandlers.ts'; import { registerContentHandlers } from '../server/serverHelpers/contentTypes.ts'; import { server } from './Server.ts'; diff --git a/server/fastifyRoutes/helpers/getCORSOptions.js b/server/fastifyRoutes/helpers/getCORSOptions.ts similarity index 79% rename from server/fastifyRoutes/helpers/getCORSOptions.js rename to server/fastifyRoutes/helpers/getCORSOptions.ts index de4a205b0..222afd8b7 100644 --- a/server/fastifyRoutes/helpers/getCORSOptions.js +++ b/server/fastifyRoutes/helpers/getCORSOptions.ts @@ -1,8 +1,5 @@ -'use strict'; - -const env = require('../../../utility/environment/environmentManager.ts'); -env.initSync(); -const { CONFIG_PARAMS } = require('../../../utility/hdbTerms.ts'); +import * as env from '../../../utility/environment/environmentManager.ts'; +import { CONFIG_PARAMS } from '../../../utility/hdbTerms.ts'; /** * Builds CORS options object to pass to cors plugin when/if it needs to be registered with Fastify @@ -33,4 +30,4 @@ function getCORSOptions() { return corsOptions; } -module.exports = getCORSOptions; +export default getCORSOptions; diff --git a/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js b/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js deleted file mode 100644 index ed4e4400c..000000000 --- a/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +++ /dev/null @@ -1,15 +0,0 @@ -'use strict'; - -const env = require('../../../utility/environment/environmentManager.ts'); -env.initSync(); -const terms = require('../../../utility/hdbTerms.ts'); - -/** - * Returns header timeout value from config file - * @returns {*} - */ -function getHeaderTimeoutConfig() { - return env.get(terms.CONFIG_PARAMS.HTTP_HEADERSTIMEOUT) ?? 60000; -} - -module.exports = getHeaderTimeoutConfig; diff --git a/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.ts b/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.ts new file mode 100644 index 000000000..55bd21cdb --- /dev/null +++ b/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.ts @@ -0,0 +1,12 @@ +import * as env from '../../../utility/environment/environmentManager.ts'; +import * as terms from '../../../utility/hdbTerms.ts'; + +/** + * Returns header timeout value from config file + * @returns {*} + */ +function getHeaderTimeoutConfig() { + return env.get(terms.CONFIG_PARAMS.HTTP_HEADERSTIMEOUT) ?? 60000; +} + +export default getHeaderTimeoutConfig; diff --git a/server/fastifyRoutes/helpers/getServerOptions.js b/server/fastifyRoutes/helpers/getServerOptions.ts similarity index 81% rename from server/fastifyRoutes/helpers/getServerOptions.js rename to server/fastifyRoutes/helpers/getServerOptions.ts index f0ae0226a..6346c50cd 100644 --- a/server/fastifyRoutes/helpers/getServerOptions.js +++ b/server/fastifyRoutes/helpers/getServerOptions.ts @@ -1,8 +1,5 @@ -'use strict'; - -const env = require('../../../utility/environment/environmentManager.ts'); -env.initSync(); -const { CONFIG_PARAMS } = require('../../../utility/hdbTerms.ts'); +import * as env from '../../../utility/environment/environmentManager.ts'; +import { CONFIG_PARAMS } from '../../../utility/hdbTerms.ts'; // eslint-disable-next-line no-magic-numbers const REQ_MAX_BODY_SIZE = 1024 * 1024 * 1024; //this is 1GB in bytes @@ -30,4 +27,4 @@ function getServerOptions(isHttps) { }; } -module.exports = getServerOptions; +export default getServerOptions; diff --git a/server/fastifyRoutes/plugins/hdbCore.js b/server/fastifyRoutes/plugins/hdbCore.ts similarity index 86% rename from server/fastifyRoutes/plugins/hdbCore.js rename to server/fastifyRoutes/plugins/hdbCore.ts index 8b5e7965d..4ea45e51f 100644 --- a/server/fastifyRoutes/plugins/hdbCore.js +++ b/server/fastifyRoutes/plugins/hdbCore.ts @@ -1,12 +1,10 @@ -'use strict'; +import fp from 'fastify-plugin'; -const fp = require('fastify-plugin'); - -const { +import { handlePostRequest, authHandler, reqBodyValidationHandler, -} = require('../../../server/serverHelpers/serverHandlers.js'); +} from '../../../server/serverHelpers/serverHandlers.ts'; /** * Generates a fastify plugin containing three core methods @@ -36,4 +34,4 @@ async function convertAsyncIterators(response) { return response; } -module.exports = fp(hdbCore); +export default fp(hdbCore); diff --git a/server/http.ts b/server/http.ts index 9189e9421..dde6dcee9 100644 --- a/server/http.ts +++ b/server/http.ts @@ -10,8 +10,8 @@ import harperLogger from '../utility/logging/harper_logger.ts'; import { parentPort } from 'node:worker_threads'; import * as env from '../utility/environment/environmentManager.ts'; import * as terms from '../utility/hdbTerms.ts'; -import { getConfigPath } from '../config/configUtils.js'; -import { getTicketKeys, getWorkerIndex } from './threads/manageThreads.js'; +import { getConfigPath } from '../config/configUtils.ts'; +import { getTicketKeys, getWorkerIndex } from './threads/manageThreads.ts'; import { createTLSSelector } from '../security/keys.ts'; import { createSecureServer } from 'node:http2'; import { createServer as createSecureServerHttp1 } from 'node:https'; @@ -23,18 +23,15 @@ import { recordAction, recordActionBinary } from '../resources/analytics/write.t import { Readable, Writable } from 'node:stream'; import { mkdirSync, writeFileSync, unlinkSync, readdirSync } from 'node:fs'; import { join } from 'node:path'; -import { server, type ServerOptions, type HttpOptions, type UpgradeOptions, UpgradeListener } from './Server.ts'; +import { server, type ServerOptions, type HttpOptions, type UpgradeOptions, type UpgradeListener } from './Server.ts'; import { setPortServerMap, SERVERS } from './serverRegistry.ts'; import { getComponentName } from '../components/componentLoader.ts'; import { throttle } from './throttle.ts'; import { makeCallbackChain as buildCallbackChain } from './middlewareChain.ts'; import { WebSocketServer } from 'ws'; +import { onStartup } from '../utility/lifecycle.ts'; const { errorToString } = harperLogger; -server.http = httpServer; -server.request = onRequest; -server.ws = onWebSocket; -server.upgrade = onUpgrade; const websocketServers = {}; const httpServers = {}, httpChain = {}, @@ -1101,3 +1098,11 @@ export function getRequestId() { } return Number(Atomics.add(nextRequestId, 0, 1n)); } + +// Wire server singletons during the startup phase +onStartup(() => { + server.http = httpServer; + server.request = onRequest; + server.ws = onWebSocket; + server.upgrade = onUpgrade; +}); diff --git a/server/itc/serverHandlers.js b/server/itc/serverHandlers.ts similarity index 85% rename from server/itc/serverHandlers.js rename to server/itc/serverHandlers.ts index 8f62881c2..5b5690072 100644 --- a/server/itc/serverHandlers.js +++ b/server/itc/serverHandlers.ts @@ -1,17 +1,15 @@ -'use strict'; - /* global threads */ -const hdbLogger = require('../../utility/logging/harper_logger.ts'); -const hdbTerms = require('../../utility/hdbTerms.ts'); -const cleanLmdbMap = - require('../../utility/lmdb/cleanLMDBMap.ts').default || require('../../utility/lmdb/cleanLMDBMap.ts'); -const userSchema = require('../../security/user.ts'); -const { validateEvent } = require('../threads/itc.js'); -const harperBridge = - require('../../dataLayer/harperBridge/harperBridge.ts').default || - require('../../dataLayer/harperBridge/harperBridge.ts'); -const process = require('process'); -const { resetDatabases } = require('../../resources/databases.ts'); +import hdbLogger from '../../utility/logging/harper_logger.ts'; +import * as hdbTerms from '../../utility/hdbTerms.ts'; +import { internal } from '../../components/status/index.ts'; +import { getWorkerIndex } from '../threads/manageThreads.ts'; +import { sendItcEvent } from '../threads/itc.ts'; +import cleanLmdbMap from '../../utility/lmdb/cleanLMDBMap.ts'; +import * as userSchema from '../../security/user.ts'; +import { validateEvent } from '../threads/itc.ts'; +import harperBridge from '../../dataLayer/harperBridge/harperBridge.ts'; +import process from 'process'; +import { resetDatabases } from '../../resources/databases.ts'; /** * This object/functions are passed to the ITC client instance and dynamically added as event handlers. @@ -116,9 +114,6 @@ async function componentStatusRequestHandler(event) { hdbLogger.trace(`ITC componentStatusRequestHandler received request:`, event); // Get current thread's component status - const { internal } = require('../../components/status/index.ts'); - const { getWorkerIndex } = require('../threads/manageThreads.js'); - const { sendItcEvent } = require('../threads/itc.js'); const componentStatuses = internal.componentStatusRegistry.getAllStatuses(); // Convert Map to array for serialization @@ -157,4 +152,4 @@ async function componentStatusRequestHandler(event) { } } -module.exports = serverItcHandlers; +export default serverItcHandlers; diff --git a/server/jobs/jobProcess.ts b/server/jobs/jobProcess.ts index 2597e6ba4..7388d2040 100644 --- a/server/jobs/jobProcess.ts +++ b/server/jobs/jobProcess.ts @@ -8,8 +8,8 @@ import * as user from '../../security/user.ts'; import * as serverUtils from '../serverHelpers/serverUtilities.ts'; import moment from 'moment'; import * as jobs from './jobs.ts'; -import { cloneDeep } from 'lodash'; - +import _lodash from 'lodash'; +const { cloneDeep } = _lodash; import { pathToFileURL } from 'node:url'; import { join } from 'node:path'; import { getEnvBuiltInComponents } from './../../components/Application.ts'; diff --git a/server/jobs/jobRunner.ts b/server/jobs/jobRunner.ts index dc1d19249..7f4557d51 100644 --- a/server/jobs/jobRunner.ts +++ b/server/jobs/jobRunner.ts @@ -1,7 +1,5 @@ 'use strict'; -import { join } from 'node:path'; - import * as hdbUtil from '../../utility/common_utils.ts'; import * as hdbTerms from '../../utility/hdbTerms.ts'; import moment from 'moment'; @@ -10,11 +8,11 @@ import log from '../../utility/logging/harper_logger.ts'; import * as jobs from './jobs.ts'; import * as hdbExport from '../../dataLayer/export.ts'; import * as hdbDelete from '../../dataLayer/delete.ts'; -import * as threadsStart from '../threads/manageThreads.js'; +import * as threadsStart from '../threads/manageThreads.ts'; import * as transactionLog from '../../utility/logging/transactionLog.ts'; import * as restart from '../../bin/restart.ts'; import { parentPort, isMainThread } from 'worker_threads'; -import { onMessageByType } from '../threads/manageThreads.js'; +import { onMessageByType } from '../threads/manageThreads.ts'; class RunnerMessage { job: any; @@ -133,7 +131,7 @@ async function runJob(runnerMessage: any, operation: any) { async function launchJobThread(job_id: any) { log.trace('launching job thread:', job_id); if (isMainThread) { - threadsStart.startWorker(join(__dirname, './jobProcess.js'), { + threadsStart.startWorker('server/jobs/jobProcess', { autoRestart: false, name: 'job', env: { ...process.env, [hdbTerms.PROCESS_NAME_ENV_PROP]: `JOB-${job_id}` }, @@ -148,7 +146,7 @@ async function launchJobThread(job_id: any) { if (isMainThread) { onMessageByType(hdbTerms.ITC_EVENT_TYPES.START_JOB, async (message) => { try { - threadsStart.startWorker(join(__dirname, './jobProcess.js'), { + threadsStart.startWorker('server/jobs/jobProcess', { autoRestart: false, name: 'job', env: { ...process.env, [hdbTerms.PROCESS_NAME_ENV_PROP]: `JOB-${message.jobId}` }, diff --git a/server/loadRootComponents.js b/server/loadRootComponents.ts similarity index 66% rename from server/loadRootComponents.js rename to server/loadRootComponents.ts index 1c4d676e2..3e2053601 100644 --- a/server/loadRootComponents.js +++ b/server/loadRootComponents.ts @@ -1,11 +1,11 @@ -const { isMainThread } = require('worker_threads'); -const { getTables } = require('../resources/databases.ts'); -const { loadComponentDirectories, loadComponent } = require('../components/componentLoader.ts'); -const { resetResources } = require('../resources/Resources.ts'); -const configUtils = require('../config/configUtils.js'); -const { dirname } = require('path'); -const { loadCertificates } = require('../security/keys.ts'); -const { installApplications } = require('../components/Application.ts'); +import { isMainThread } from 'worker_threads'; +import { getTables } from '../resources/databases.ts'; +import { loadComponentDirectories, loadComponent } from '../components/componentLoader.ts'; +import { resetResources } from '../resources/Resources.ts'; +import * as configUtils from '../config/configUtils.ts'; +import { dirname } from 'path'; +import { loadCertificates } from '../security/keys.ts'; +import { installApplications } from '../components/Application.ts'; let loadedComponents = new Map(); /** @@ -41,4 +41,4 @@ async function loadRootComponents(isWorkerThread = false) { if (allReady.length > 0) await Promise.all(allReady); } -module.exports.loadRootComponents = loadRootComponents; +export { loadRootComponents }; diff --git a/server/mqtt.ts b/server/mqtt.ts index dfed57504..4c8b10186 100644 --- a/server/mqtt.ts +++ b/server/mqtt.ts @@ -1,6 +1,7 @@ // for now we are using mqtt-packet, but we may implement some of this ourselves, particularly packet generation so that // we can implement more efficient progressive buffer allocation. -import { parser as makeParser, generate } from 'mqtt-packet'; +import _mqtt_packet from 'mqtt-packet'; +const { parser: makeParser, generate } = _mqtt_packet; import { getSession, DurableSubscriptionsSession } from './DurableSubscriptionsSession.ts'; import { getSuperUser } from '../security/user.ts'; import { serializeMessage, getDeserializer } from './serverHelpers/contentTypes.ts'; diff --git a/server/nodeName.ts b/server/nodeName.ts index b7a2cc0c0..06423e234 100644 --- a/server/nodeName.ts +++ b/server/nodeName.ts @@ -5,10 +5,14 @@ import * as env from '../utility/environment/environmentManager.ts'; import { logger } from '../utility/logging/logger.ts'; import { server } from './Server.ts'; -Object.defineProperty(server, 'hostname', { - get() { - return getThisNodeName(); - }, +// Defer to next tick so `server` (from Server.ts) is fully evaluated +// even when modules are loaded via an ESM cycle. +setImmediate(() => { + Object.defineProperty(server, 'hostname', { + get() { + return getThisNodeName(); + }, + }); }); let commonNameFromCert: string | undefined; diff --git a/server/operationsServer.ts b/server/operationsServer.ts index 92e58f5e8..41a141ee0 100644 --- a/server/operationsServer.ts +++ b/server/operationsServer.ts @@ -2,14 +2,23 @@ import cluster from 'cluster'; import zlib from 'node:zlib'; import * as env from '../utility/environment/environmentManager.ts'; -env.initSync(); +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} import * as terms from '../utility/hdbTerms.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; -import fastify, { FastifyInstance, FastifyReply, FastifyRequest, FastifyServerOptions } from 'fastify'; +import fastify, { + type FastifyInstance, + type FastifyReply, + type FastifyRequest, + type FastifyServerOptions, +} from 'fastify'; import fastifyCors, { type FastifyCorsOptions } from '@fastify/cors'; import fastifyCompress from '@fastify/compress'; import fastifyStatic from '@fastify/static'; -import requestTimePlugin from './serverHelpers/requestTimePlugin.js'; +import requestTimePlugin from './serverHelpers/requestTimePlugin.ts'; import guidePath from 'path'; import { PACKAGE_ROOT } from '../utility/packageUtils.js'; import * as globalSchema from '../utility/globalSchema.ts'; @@ -22,7 +31,7 @@ import { handlePostRequest, serverErrorHandler, reqBodyValidationHandler, -} from './serverHelpers/serverHandlers.js'; +} from './serverHelpers/serverHandlers.ts'; import { registerBunFastifyInstance } from './http.ts'; import { registerContentHandlers } from './serverHelpers/contentTypes.ts'; import type { OperationFunctionName } from './serverHelpers/serverUtilities.ts'; diff --git a/server/serverHelpers/contentTypes.ts b/server/serverHelpers/contentTypes.ts index 7042cf91f..04818975a 100644 --- a/server/serverHelpers/contentTypes.ts +++ b/server/serverHelpers/contentTypes.ts @@ -13,9 +13,19 @@ import { logger } from '../../utility/logging/logger.ts'; import { Blob } from '../../resources/blob.ts'; // TODO: Only load this if fastify is loaded import fp from 'fastify-plugin'; -const SERIALIZATION_BIGINT = envMgr.get(CONFIG_PARAMS.SERIALIZATION_BIGINT) !== false; -const JSONStringify = SERIALIZATION_BIGINT ? stringify : JSON.stringify; -const JSONParse = SERIALIZATION_BIGINT ? parse : JSON.parse; +// Resolve lazily: reading config at module-load time would TDZ under ESM +// because configUtils.ts is mid-evaluation when this module is imported. +let _serializationBigint: boolean | undefined; +function getSerializationBigint(): boolean { + if (_serializationBigint === undefined) { + _serializationBigint = envMgr.get(CONFIG_PARAMS.SERIALIZATION_BIGINT) !== false; + } + return _serializationBigint; +} +const JSONStringify = ((value: any, ...rest: any[]) => + (getSerializationBigint() ? stringify : JSON.stringify)(value, ...rest)) as typeof JSON.stringify; +const JSONParse = ((text: string, ...rest: any[]) => + (getSerializationBigint() ? parse : JSON.parse)(text, ...rest)) as typeof JSON.parse; const PUBLIC_ENCODE_OPTIONS = { useRecords: false, @@ -36,7 +46,11 @@ const mediaTypes = new Map< >(); export const contentTypes = mediaTypes; -server.contentTypes = contentTypes as any; +// Defer attachment to `server` because under ESM cycles Server.ts may still +// be mid-evaluation when this module is loaded. +setImmediate(() => { + server.contentTypes = contentTypes as any; +}); _assignPackageExport('contentTypes', contentTypes); // TODO: Make these monomorphic for faster access. And use a Map mediaTypes.set('application/json', { @@ -101,29 +115,6 @@ mediaTypes.set('text/yaml', { q: 0.7, }); -const ndjsonHandler = { - serializeStream(data: any) { - if (data?.[Symbol.iterator] || data?.[Symbol.asyncIterator]) { - return Readable.from(transformIterable(data, (msg: any) => JSONStringify(msg) + '\n')); - } - return JSONStringify(data) + '\n'; - }, - serialize(data: any) { - return JSONStringify(data) + '\n'; - }, - deserialize(data: Buffer) { - return data - .toString() - .split('\n') - .map((line) => line.trim()) - .filter(Boolean) - .map(JSONParse); - }, - q: 0.7, -}; -mediaTypes.set('application/x-ndjson', ndjsonHandler); -mediaTypes.set('application/ndjson', ndjsonHandler); - mediaTypes.set('text/event-stream', { // Server-Sent Events (SSE) serializeStream: function (iterable) { @@ -192,6 +183,28 @@ const genericHandler = { }; mediaTypes.set('*/*', genericHandler); mediaTypes.set('', genericHandler); +const ndjsonHandler = { + serializeStream(data: any) { + if (data?.[Symbol.iterator] || data?.[Symbol.asyncIterator]) { + return Readable.from(transformIterable(data, (msg: any) => JSONStringify(msg) + '\n')); + } + return JSONStringify(data) + '\n'; + }, + serialize(data: any) { + return JSONStringify(data) + '\n'; + }, + deserialize(data: Buffer) { + return data + .toString() + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .map(JSONParse); + }, + q: 0.7, +}; +mediaTypes.set('application/x-ndjson', ndjsonHandler); +mediaTypes.set('application/ndjson', ndjsonHandler); // try to JSON parse, but since we don't know for sure, this will return the body // otherwise function tryJSONParse(input) { @@ -345,8 +358,13 @@ export function findBestSerializer(incomingMessage) { return { serializer: bestSerializer, type: bestType, parameters: bestParameters }; } -// about an average TCP packet size (if headers included) -const COMPRESSION_THRESHOLD = envMgr.get(CONFIG_PARAMS.HTTP_COMPRESSIONTHRESHOLD); +// about an average TCP packet size (if headers included). Read lazily — +// reading config at module-load time would fault under ESM cycles where +// the environment manager hasn't finished initializing. +let _compressionThreshold: number | undefined; +function COMPRESSION_THRESHOLD() { + return (_compressionThreshold ??= envMgr.get(CONFIG_PARAMS.HTTP_COMPRESSIONTHRESHOLD)); +} /** * Serialize a response * @param responseData @@ -357,7 +375,7 @@ const COMPRESSION_THRESHOLD = envMgr.get(CONFIG_PARAMS.HTTP_COMPRESSIONTHRESHOLD export function serialize(responseData, request, responseObject) { // TODO: Maybe support other compression encodings; browsers basically universally support brotli, but Node's HTTP // client itself actually (just) supports gzip/deflate - let canCompress = COMPRESSION_THRESHOLD && request.headers.asObject?.['accept-encoding']?.includes('br'); + let canCompress = COMPRESSION_THRESHOLD() && request.headers.asObject?.['accept-encoding']?.includes('br'); let responseBody; if (responseData?.contentType != null && responseData.data != null) { // we use this as a special marker for blobs of data that are explicitly one content type @@ -413,7 +431,7 @@ export function serialize(responseData, request, responseObject) { } responseBody = serializer.serializer.serialize(responseData, responseObject); } - if (canCompress && responseBody?.length > COMPRESSION_THRESHOLD) { + if (canCompress && responseBody?.length > COMPRESSION_THRESHOLD()) { // TODO: Only do this if the size is large and we can cache the result (otherwise use logic above) responseObject.headers.set('Content-Encoding', 'br'); // if we have a single buffer (or string) we compress in a single async call diff --git a/server/serverHelpers/requestTimePlugin.js b/server/serverHelpers/requestTimePlugin.ts similarity index 93% rename from server/serverHelpers/requestTimePlugin.js rename to server/serverHelpers/requestTimePlugin.ts index c369b48ec..97bd8912e 100644 --- a/server/serverHelpers/requestTimePlugin.js +++ b/server/serverHelpers/requestTimePlugin.ts @@ -1,10 +1,10 @@ -const { recordAction, recordActionBinary } = require('../../resources/analytics/write.ts'); -const fp = require('fastify-plugin'); +import { recordAction, recordActionBinary } from '../../resources/analytics/write.ts'; +import fp from 'fastify-plugin'; const ESTIMATED_HEADER_SIZE = 200; // it is very expensive to actually measure HTTP response header size (we change it // ourselves) with an unacceptable performance penalty, so we estimate this part -module.exports = fp( +export default fp( function (fastify, opts, done) { // eslint-disable-next-line require-await fastify.addHook('onResponse', async (request, reply) => { diff --git a/server/serverHelpers/serverHandlers.js b/server/serverHelpers/serverHandlers.ts similarity index 85% rename from server/serverHelpers/serverHandlers.js rename to server/serverHelpers/serverHandlers.ts index 692c9fbc4..2195d9c86 100644 --- a/server/serverHelpers/serverHandlers.js +++ b/server/serverHelpers/serverHandlers.ts @@ -1,20 +1,19 @@ -'use strict'; +import * as terms from '../../utility/hdbTerms.ts'; +import * as hdbUtil from '../../utility/common_utils.ts'; +import harperLogger from '../../utility/logging/harper_logger.ts'; +import { handleHDBError, hdbErrors } from '../../utility/errors/hdbError.ts'; +import { isMainThread } from 'worker_threads'; +import { Readable } from 'stream'; -const terms = require('../../utility/hdbTerms.ts'); -const hdbUtil = require('../../utility/common_utils.ts'); -const harperLogger = require('../../utility/logging/harper_logger.ts'); -const { handleHDBError, hdbErrors } = require('../../utility/errors/hdbError.ts'); -const { isMainThread } = require('worker_threads'); -const { Readable } = require('stream'); +import _os from 'os'; +const os = _os; +import util from 'util'; -const os = require('os'); -const util = require('util'); - -const auth = require('../../security/fastifyAuth.ts'); +import * as auth from '../../security/fastifyAuth.ts'; const pAuthorize = util.promisify(auth.authorize); -const serverUtilities = require('./serverUtilities.ts'); -const { applyImpersonation } = require('../../security/impersonation.ts'); -const { createGzip, constants } = require('zlib'); +import * as serverUtilities from './serverUtilities.ts'; +import { applyImpersonation } from '../../security/impersonation.ts'; +import { createGzip, constants } from 'zlib'; const NO_AUTH_OPERATIONS = [ terms.OPERATIONS_ENUM.CREATE_AUTHENTICATION_TOKENS, @@ -138,7 +137,7 @@ async function handlePostRequest(req, res, _bypassAuth = false) { } } -module.exports = { +export { authHandler, authAndEnsureUserOnRequest, handlePostRequest, diff --git a/server/serverHelpers/serverUtilities.ts b/server/serverHelpers/serverUtilities.ts index a0894103c..cd901f62d 100644 --- a/server/serverHelpers/serverUtilities.ts +++ b/server/serverHelpers/serverUtilities.ts @@ -6,7 +6,7 @@ import * as delete_ from '../../dataLayer/delete.ts'; import readAuditLog from '../../dataLayer/readAuditLog.ts'; import * as user from '../../security/user.ts'; import * as role from '../../security/role.ts'; -import customFunctionOperations from '../../components/operations.js'; +import * as customFunctionOperations from '../../components/operations.ts'; import harperLogger from '../../utility/logging/harper_logger.ts'; import readLog from '../../utility/logging/readLog.ts'; import * as export_ from '../../dataLayer/export.ts'; @@ -23,7 +23,7 @@ import { systemInformation } from '../../utility/environment/systemInformation.t import * as jobRunner from '../jobs/jobRunner.ts'; import * as tokenAuthentication from '../../security/tokenAuthentication.ts'; import * as auth from '../../security/auth.ts'; -import configUtils from '../../config/configUtils.js'; +import * as configUtils from '../../config/configUtils.ts'; import * as transactionLog from '../../utility/logging/transactionLog.ts'; import * as npmUtilities from '../../utility/npmUtilities.ts'; import { _assignPackageExport } from '../../globals.js'; @@ -57,6 +57,7 @@ const GLOBAL_SCHEMA_UPDATE_OPERATIONS_ENUM = { }; import { OperationFunctionObject } from './OperationFunctionObject.ts'; +import { onStartup } from '../../utility/lifecycle.ts'; type ValueOf = T[keyof T]; export type OperationFunctionName = ValueOf; @@ -105,7 +106,6 @@ export async function processLocalTransaction(req: OperationRequest, operationFu const OPERATION_FUNCTION_MAP = initializeOperationFunctionMap(); -server.operation = operation; export type OperationDefinition = { name: string; execute: (operation: any) => any | Promise; @@ -117,9 +117,9 @@ export type OperationDefinition = { * Register an operation function with the server. * @param operationDefinition */ -server.registerOperation = (operationDefinition: OperationDefinition) => { +function registerOperation(operationDefinition: OperationDefinition) { OPERATION_FUNCTION_MAP.set(operationDefinition.name as any, new OperationFunctionObject(operationDefinition.execute)); -}; +} export function chooseOperation(json: OperationRequestBody) { let getOpResult: OperationFunctionObject; @@ -468,3 +468,9 @@ function initializeOperationFunctionMap(): Map { + server.operation = operation; + server.registerOperation = registerOperation; +}); diff --git a/server/static.ts b/server/static.ts index cffe396bd..bb7ffb710 100644 --- a/server/static.ts +++ b/server/static.ts @@ -1,6 +1,6 @@ import { realpathSync, existsSync } from 'node:fs'; import { join } from 'node:path'; -import { Scope } from '../components/Scope'; +import { Scope } from '../components/Scope.ts'; import send from 'send'; /** diff --git a/server/status/index.ts b/server/status/index.ts index 071484944..2f99fc6b7 100644 --- a/server/status/index.ts +++ b/server/status/index.ts @@ -14,7 +14,6 @@ export type { StatusId, StatusRecord, StatusValueMap } from './definitions.ts'; export { STATUS_IDS, DEFAULT_STATUS_ID } from './definitions.ts'; const { HTTP_STATUS_CODES } = hdbErrors; - // For direct function calls, we don't need the operation fields type StatusRequestBody = { id: StatusId; diff --git a/server/storageReclamation.ts b/server/storageReclamation.ts index e21796e3f..fb24e9de6 100644 --- a/server/storageReclamation.ts +++ b/server/storageReclamation.ts @@ -1,10 +1,14 @@ import { statfs } from 'node:fs/promises'; -import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js'; +import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.ts'; import { logger } from '../utility/logging/logger.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import * as envMgr from '../utility/environment/environmentManager.ts'; import { convertToMS } from '../utility/common_utils.ts'; -envMgr.initSync(); +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const reclamationHandlers = new Map< string, { priority: number; handler: (priority: number) => Promise | void }[] diff --git a/server/threads/itc.js b/server/threads/itc.ts similarity index 57% rename from server/threads/itc.js rename to server/threads/itc.ts index e45c0b2f5..f5aafc375 100644 --- a/server/threads/itc.js +++ b/server/threads/itc.ts @@ -1,29 +1,31 @@ -'use strict'; +import * as hdbUtils from '../../utility/common_utils.ts'; +import * as hdbTerms from '../../utility/hdbTerms.ts'; +import { ITC_ERRORS } from '../../utility/errors/commonErrors.ts'; +import { threadId, isMainThread } from 'worker_threads'; +import { onMessageFromWorkers, broadcastWithAcknowledgement } from './manageThreads.ts'; -const hdbUtils = require('../../utility/common_utils.ts'); -const hdbTerms = require('../../utility/hdbTerms.ts'); -const { ITC_ERRORS } = require('../../utility/errors/commonErrors.ts'); -const { threadId, isMainThread } = require('worker_threads'); -const { onMessageFromWorkers, broadcastWithAcknowledgement } = require('./manageThreads.js'); - -module.exports = { - sendItcEvent, - validateEvent, - SchemaEventMsg, - UserEventMsg, -}; +export { sendItcEvent, validateEvent, SchemaEventMsg, UserEventMsg }; let serverItcHandlers; -onMessageFromWorkers(async (event, sender) => { - serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js'); - validateEvent(event); - if (serverItcHandlers[event.type]) { - await serverItcHandlers[event.type](event); - } - if (event.requestId && sender) - sender.postMessage({ - type: 'ack', - id: event.requestId, - }); +// Defer registration so manageThreads.ts is fully evaluated when we read from +// it (ESM cycle would otherwise leave its internal state uninitialized). +setImmediate(() => { + onMessageFromWorkers(async (event, sender) => { + if (!serverItcHandlers) { + const m: any = await import('../itc/serverHandlers.ts'); + // Dynamic import returns the namespace { default: handlersObj }; in + // CJS-dist it can be double-wrapped as { default: { default: ... } }. + serverItcHandlers = m.default?.default ?? m.default ?? m; + } + validateEvent(event); + if (serverItcHandlers[event.type]) { + await serverItcHandlers[event.type](event); + } + if (event.requestId && sender) + sender.postMessage({ + type: 'ack', + id: event.requestId, + }); + }); }); /** diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.ts similarity index 85% rename from server/threads/manageThreads.js rename to server/threads/manageThreads.ts index 8aee58df5..1517d8599 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.ts @@ -1,38 +1,42 @@ -'use strict'; - -const { Worker, MessageChannel, parentPort, isMainThread, threadId, workerData } = require('worker_threads'); -const { join, isAbsolute, extname } = require('path'); -const { server } = require('../Server.ts'); -const { totalmem } = require('os'); -const { setHeapSnapshotNearHeapLimit } = typeof globalThis.Bun !== 'undefined' ? {} : require('v8'); -const hdbTerms = require('../../utility/hdbTerms.ts'); -const envMgr = require('../../utility/environment/environmentManager.ts'); -const harperLogger = require('../../utility/logging/harper_logger.ts'); -const { randomBytes } = require('crypto'); -const { _assignPackageExport } = require('../../globals.js'); -const { PACKAGE_ROOT } = require('../../utility/packageUtils.js'); -const chokidar = require('chokidar'); -const isBun = typeof globalThis.Bun !== 'undefined'; -const MB = 1024 * 1024; -const workers = []; // these are our child workers that we are managing -const connectedPorts = []; // these are all known connected worker ports (siblings, children, parents) -const MAX_UNEXPECTED_RESTARTS = 50; -let threadTerminationTimeout = 10000; // threads, you got 10 seconds to die -const RESTART_TYPE = 'restart'; -const REQUEST_THREAD_INFO = 'request_thread_info'; -const RESOURCE_REPORT = 'resource_report'; -const THREAD_INFO = 'thread_info'; -const ADDED_PORT = 'added-port'; -const ACKNOWLEDGEMENT = 'ack'; -const REMOVE_PORT = 'remove-port'; -const FORCE_EXIT = 'force-exit'; -let getThreadInfo; +import { Worker, MessageChannel, parentPort, isMainThread, threadId, workerData } from 'worker_threads'; +import { onStartup } from '../../utility/lifecycle.ts'; +import { join, isAbsolute, extname } from 'path'; +import { server } from '../Server.ts'; +import { totalmem } from 'os'; +import { resetRestartNeeded } from '../../components/requestRestart.ts'; +import { loadRootComponents } from '../loadRootComponents.ts'; +import { setHeapSnapshotNearHeapLimit as _setHeapSnapshotNearHeapLimit } from 'node:v8'; +var setHeapSnapshotNearHeapLimit = typeof globalThis.Bun !== 'undefined' ? () => {} : _setHeapSnapshotNearHeapLimit; +import * as hdbTerms from '../../utility/hdbTerms.ts'; +import * as envMgr from '../../utility/environment/environmentManager.ts'; +import harperLogger from '../../utility/logging/harper_logger.ts'; +import { randomBytes } from 'crypto'; +import { _assignPackageExport } from '../../globals.js'; +import { RUNTIME_SRC_ROOT, RUNTIME_FILE_EXT } from '../../utility/packageUtils.js'; +import chokidar from 'chokidar'; +var isBun = typeof globalThis.Bun !== 'undefined'; +var MB = 1024 * 1024; +var workers = []; // these are our child workers that we are managing +var connectedPorts = []; // these are all known connected worker ports (siblings, children, parents) +var MAX_UNEXPECTED_RESTARTS = 50; +var threadTerminationTimeout = 10000; // threads, you got 10 seconds to die +var RESTART_TYPE = 'restart'; +var REQUEST_THREAD_INFO = 'request_thread_info'; +var RESOURCE_REPORT = 'resource_report'; +var THREAD_INFO = 'thread_info'; +var ADDED_PORT = 'added-port'; +var ACKNOWLEDGEMENT = 'ack'; +var REMOVE_PORT = 'remove-port'; +var FORCE_EXIT = 'force-exit'; +var getThreadInfo; _assignPackageExport('threads', connectedPorts); -const listenersByType = new Map(); -const messagesQueuedByType = new Map(); +var listenersByType = new Map(); +var messagesQueuedByType = new Map(); +var messageListeners = []; -module.exports = { +export let restartNumber = workerData?.restartNumber || 1; +export { startWorker, restartWorkers, shutdownWorkers, @@ -48,7 +52,6 @@ module.exports = { getTicketKeys, setMainIsWorker, setTerminateTimeout, - restartNumber: workerData?.restartNumber || 1, }; connectedPorts.onMessageByType = onMessageByType; @@ -60,15 +63,16 @@ connectedPorts.sendToThread = function (threadId, message) { return true; } }; -module.exports.whenThreadsStarted = new Promise((resolve) => { - module.exports.threadsHaveStarted = resolve; +export let threadsHaveStarted: (value?: unknown) => void; +export const whenThreadsStarted = new Promise((resolve) => { + threadsHaveStarted = resolve; }); // make sure this is set on all threads, including the main thread (this is no-op // if it was already with the execArgv below) if (envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_HEAPSNAPSHOTNEARLIMIT)) setHeapSnapshotNearHeapLimit(1); -let isMainWorker; +var isMainWorker; function setTerminateTimeout(newTimeout) { threadTerminationTimeout = newTimeout; } @@ -80,33 +84,35 @@ function getWorkerCount() { } function setMainIsWorker(isWorker) { isMainWorker = isWorker; - module.exports.threadsHaveStarted(); + threadsHaveStarted(); } -let workerCount = 1; // should be assigned when workers are created -let ticketKeys; +var workerCount = 1; // should be assigned when workers are created +var ticketKeys; function getTicketKeys() { if (ticketKeys) return ticketKeys; ticketKeys = isMainThread ? randomBytes(48) : workerData.ticketKeys; return ticketKeys; } -Object.defineProperty(server, 'workerIndex', { - get() { - return getWorkerIndex(); - }, -}); -Object.defineProperty(server, 'workerCount', { - get() { - return getWorkerCount(); - }, -}); -if (!parentPort) { - onMessageByType(REQUEST_THREAD_INFO, (message, worker) => { - if (worker) sendThreadInfo(worker); +onStartup(() => { + Object.defineProperty(server, 'workerIndex', { + get() { + return getWorkerIndex(); + }, }); - onMessageByType(RESOURCE_REPORT, (message, worker) => { - if (worker) recordResourceReport(worker, message); + Object.defineProperty(server, 'workerCount', { + get() { + return getWorkerCount(); + }, }); -} + if (!parentPort) { + onMessageByType(REQUEST_THREAD_INFO, (message, worker) => { + if (worker) sendThreadInfo(worker); + }); + onMessageByType(RESOURCE_REPORT, (message, worker) => { + if (worker) recordResourceReport(worker, message); + }); + } +}); // postMessage type listeners that are registered in other ways or can be registered later listenersByType.set(hdbTerms.ITC_EVENT_TYPES.CHILD_STARTED, null); listenersByType.set(hdbTerms.ITC_EVENT_TYPES.SCHEMA, null); @@ -146,7 +152,10 @@ function startWorker(path, options = {}) { portsToSend.push(channel.port2); } - if (!extname(path)) path += '.js'; + // If the caller passed an extensionless module identifier, pick the + // extension that matches the current execution mode (.ts in typestrip + // source mode, .js in dist). + if (!extname(path)) path += RUNTIME_FILE_EXT; const isBun = typeof globalThis.Bun !== 'undefined'; const execArgv = isBun @@ -160,7 +169,7 @@ function startWorker(path, options = {}) { if (!isBun && envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_HEAPSNAPSHOTNEARLIMIT)) execArgv.push('--heapsnapshot-near-heap-limit=1'); - const worker = new Worker(isAbsolute(path) ? path : join(PACKAGE_ROOT, path), { + const worker = new Worker(isAbsolute(path) ? path : join(RUNTIME_SRC_ROOT, path), { resourceLimits: { maxOldGenerationSizeMb: maxOldMemory, maxYoungGenerationSizeMb: maxYoungMemory, @@ -174,7 +183,7 @@ function startWorker(path, options = {}) { workerIndex: options.workerIndex, workerCount: (workerCount = options.threadCount), name: options.name, - restartNumber: module.exports.restartNumber, + restartNumber: restartNumber, ticketKeys: getTicketKeys(), }, transferList: portsToSend, @@ -221,7 +230,7 @@ function startWorker(path, options = {}) { return worker; } -const OVERLAPPING_RESTART_TYPES = [hdbTerms.THREAD_TYPES.HTTP]; +var OVERLAPPING_RESTART_TYPES = [hdbTerms.THREAD_TYPES.HTTP]; /** * Restart all the worker threads @@ -249,15 +258,13 @@ async function restartWorkers( harperLogger.error('Unable to reestablish current working directory', e); } // problematic cyclic dependency, bind late - const { resetRestartNeeded } = require('../../components/requestRestart.ts'); resetRestartNeeded(); // This is here to prevent circular dependencies if (startReplacementThreads) { - const { loadRootComponents } = require('../loadRootComponents.js'); await loadRootComponents(); } - module.exports.restartNumber++; + restartNumber++; if (maxWorkersDown < 1) { // we accept a ratio of workers, and compute absolute maximum being down at a time from the total number of // threads @@ -271,7 +278,7 @@ async function restartWorkers( if ((name && worker.name !== name) || worker.wasShutdown) continue; // filter by type, if specified harperLogger.trace('sending shutdown request to ', worker.threadId); worker.postMessage({ - restartNumber: module.exports.restartNumber, + restartNumber: restartNumber, type: hdbTerms.ITC_EVENT_TYPES.SHUTDOWN, }); worker.wasShutdown = true; @@ -351,11 +358,13 @@ async function shutdownWorkersNow(name) { } } -const messageListeners = []; function onMessageFromWorkers(listener) { + if (!messageListeners) messageListeners = []; messageListeners.push(listener); } function onMessageByType(type, listener) { + if (!listenersByType) listenersByType = new Map(); + if (!messagesQueuedByType) messagesQueuedByType = new Map(); let listeners = listenersByType.get(type); if (!listeners) listenersByType.set(type, (listeners = [])); listeners.push(listener); @@ -369,7 +378,7 @@ function onMessageByType(type, listener) { } } -const MAX_SYNC_BROADCAST = 10; +var MAX_SYNC_BROADCAST = 10; async function broadcast(message, includeSelf) { let count = 0; for (let port of connectedPorts) { @@ -389,8 +398,8 @@ async function broadcast(message, includeSelf) { } } -const awaitingResponses = new Map(); -let nextId = 1; +var awaitingResponses = new Map(); +var nextId = 1; function broadcastWithAcknowledgement(message) { return new Promise((resolve) => { let waitingCount = 0; @@ -463,13 +472,13 @@ function recordResourceReport(worker, message) { worker.resources.updated = Date.now(); } -let monitorListener; +var monitorListener; function setMonitorListener(listener) { monitorListener = listener; } -const MONITORING_INTERVAL = 1000; -let monitoring = false; +var MONITORING_INTERVAL = 1000; +var monitoring = false; function startMonitoring() { if (monitoring) return; monitoring = true; @@ -496,7 +505,7 @@ function startMonitoring() { if (monitorListener) monitorListener(); }, MONITORING_INTERVAL).unref(); } -const REPORTING_INTERVAL = 1000; +var REPORTING_INTERVAL = 1000; if (parentPort && workerData?.addPorts) { addPort(parentPort); @@ -531,8 +540,7 @@ if (parentPort && workerData?.addPorts) { } else { getThreadInfo = getChildWorkerInfo; } -module.exports.getThreadInfo = getThreadInfo; - +export { getThreadInfo }; function removePort(port, deadThreadId) { const idx = connectedPorts.indexOf(port); if (idx === -1) return; @@ -632,11 +640,10 @@ if (isMainThread) { }, 100); }); }; - module.exports.watchDir = watchDir; if (process.env.WATCH_DIR) watchDir(process.env.WATCH_DIR); } else { onMessageByType(hdbTerms.ITC_EVENT_TYPES.SHUTDOWN, async (message) => { - module.exports.restartNumber = message.restartNumber; + (globalThis as any).restartNumber = message.restartNumber; parentPort.unref(); // remove this handle setTimeout(() => { harperLogger.warn('Thread did not voluntarily terminate', threadId); diff --git a/server/threads/socketRouter.ts b/server/threads/socketRouter.ts index e5b2ccc6e..c11e2a7d8 100644 --- a/server/threads/socketRouter.ts +++ b/server/threads/socketRouter.ts @@ -1,10 +1,8 @@ -import { startWorker, setMonitorListener, setMainIsWorker, threadsHaveStarted } from './manageThreads.js'; +import { startWorker, setMonitorListener, setMainIsWorker, threadsHaveStarted } from './manageThreads.ts'; import * as hdbTerms from '../../utility/hdbTerms.ts'; import * as harperLogger from '../../utility/logging/harper_logger.ts'; import { recordHostname } from '../../resources/analytics/write.ts'; import { isMainThread } from 'worker_threads'; -import { join } from 'path'; - const workers = []; const workersReady = []; @@ -27,14 +25,14 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean if (dynamicThreads) { startHTTPWorker(0, 1); } else { - const { loadRootComponents } = require('../loadRootComponents.js'); + const { loadRootComponents } = await import('../loadRootComponents.ts'); if (threadCount === 0) { setMainIsWorker(true); - await require('./threadServer.js').startServers(); + await (await import('./threadServer.ts')).startServers(); return Promise.resolve([]); } await loadRootComponents(); - const { listenOnPorts } = require('./threadServer.js'); + const { listenOnPorts } = await import('./threadServer.ts'); await listenOnPorts(); // Windows does not support SO_REUSEPORT, so only a single HTTP worker is supported. if (process.platform === 'win32') threadCount = 1; @@ -49,7 +47,10 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean } function startHTTPWorker(index, threadCount = 1) { - startWorker(join(__dirname, './threadServer.js'), { + // Worker entry path is resolved by startWorker() against PACKAGE_ROOT, + // which differs between source (`core/server/threads/...`) and dist + // (`core/dist/server/threads/...`). startWorker handles the rewrite. + startWorker('server/threads/threadServer', { name: hdbTerms.THREAD_TYPES.HTTP, workerIndex: index, threadCount, diff --git a/server/threads/threadServer.js b/server/threads/threadServer.ts similarity index 81% rename from server/threads/threadServer.js rename to server/threads/threadServer.ts index 0c4d0783b..61dd0e439 100644 --- a/server/threads/threadServer.js +++ b/server/threads/threadServer.ts @@ -1,30 +1,30 @@ -'use strict'; - -const { isMainThread, parentPort, threadId, workerData } = require('node:worker_threads'); -const { createServer: createSocketServer } = require('node:net'); -const { unlinkSync, existsSync, mkdirSync } = require('fs'); -const { join } = require('path'); +import * as inspector from 'node:inspector'; +import { isMainThread, parentPort, threadId, workerData } from 'node:worker_threads'; +import { createServer as createSocketServer } from 'node:net'; +import { unlinkSync, existsSync, mkdirSync } from 'fs'; +import { join } from 'path'; +import * as loaded from '../loadRootComponents.ts'; let componentsLoadedResolve; -exports.whenComponentsLoaded = new Promise((resolve) => { +export const whenComponentsLoaded = new Promise((resolve) => { componentsLoadedResolve = resolve; }); -const harperLogger = require('../../utility/logging/harper_logger.ts'); -const env = require('../../utility/environment/environmentManager.ts'); -const terms = require('../../utility/hdbTerms.ts'); -const { server } = require('../Server.ts'); -let { createServer: createSecureSocketServer } = require('node:tls'); -const { restartNumber, getWorkerIndex } = require('./manageThreads.js'); -const { isBun } = require('../serverHelpers/Request.ts'); -const { createTLSSelector } = require('../../security/keys.ts'); -const { startupLog } = require('../../bin/run.ts'); -const { SERVERS, setPortServerMap, portServer } = require('../serverRegistry.ts'); -const httpComponent = require('../http.ts'); -const globals = require('../../globals.js'); - +import harperLogger from '../../utility/logging/harper_logger.ts'; +import * as env from '../../utility/environment/environmentManager.ts'; +import * as terms from '../../utility/hdbTerms.ts'; +import { server } from '../Server.ts'; +import { createServer as createSecureSocketServer } from 'node:tls'; +import { restartNumber, getWorkerIndex } from './manageThreads.ts'; +import { isBun } from '../serverHelpers/Request.ts'; +import { createTLSSelector } from '../../security/keys.ts'; +import { startupLog } from '../../bin/run.ts'; +import { SERVERS, setPortServerMap, portServer } from '../serverRegistry.ts'; +import * as httpComponent from '../http.ts'; +import * as globals from '../../globals.js'; +import { onStartup } from '../../utility/lifecycle.ts'; +import { getComponentName } from '../../components/componentLoader.ts'; const debugThreads = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG); const isWindows = process.platform === 'win32'; -server.socket = onSocket; if (!isBun) { if (debugThreads) { @@ -33,7 +33,7 @@ if (!isBun) { port = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_PORT) ?? 9229; const closeInspector = () => { try { - require('inspector').close(); + inspector.close(); } catch (error) { harperLogger.info('Could not close debugger', error); } @@ -51,14 +51,14 @@ if (!isBun) { const host = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_HOST); const waitForDebugger = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_WAITFORDEBUGGER); try { - require('inspector').open(port, host, waitForDebugger); + inspector.open(port, host, waitForDebugger); } catch (error) { harperLogger.trace(`Could not start debugging on port ${port}, you may already be debugging:`, error.message); } } } else if (process.env.DEV_MODE && isMainThread) { try { - require('inspector').open(9229); + inspector.open(9229); } catch (error) { if (restartNumber <= 1) harperLogger.trace('Could not start debugging on port 9229, you may already be debugging:', error.message); @@ -80,12 +80,10 @@ process.on('unhandledRejection', (reason) => { if (reason?.isHandled) return; harperLogger.error('unhandledRejection', reason); }); -env.initSync(); -exports.globals = globals; -exports.listenOnPorts = listenOnPorts; -exports.startServers = startServers; -exports.closeServers = closeServers; - +export { globals }; +export { listenOnPorts }; +export { startServers }; +export { closeServers }; function closeServers() { if (isBun) { // Bun servers use .stop() for graceful shutdown @@ -162,45 +160,45 @@ function startServers() { // ignore any errors with this; just a best effort for now } } - let loaded = require('../loadRootComponents.js') - .loadRootComponents(true) - .then(() => { - parentPort - ?.on('message', (message) => { - if (message.type === terms.ITC_EVENT_TYPES.SHUTDOWN) { - harperLogger.trace('received shutdown request', threadId); - // shutdown (for these threads) means stop listening for incoming requests (finish what we are working) and - // close connections as possible, then let the event loop complete - closeServers().then(() => { - process.exit(0); - }); - // Clean up per-thread UDS socket and metadata files - httpComponent.cleanupUdsFiles(); - if (!isBun && (debugThreads || process.env.DEV_MODE)) { - try { - require('inspector').close(); - } catch (error) { - harperLogger.info('Could not close debugger', error); - } + const loadedPromise = loaded.loadRootComponents(true).then(() => { + parentPort + ?.on('message', (message) => { + if (message.type === terms.ITC_EVENT_TYPES.SHUTDOWN) { + harperLogger.trace('received shutdown request', threadId); + // shutdown (for these threads) means stop listening for incoming requests (finish what we are working) and + // close connections as possible, then let the event loop complete + closeServers().then(() => { + process.exit(0); + }); + // Clean up per-thread UDS socket and metadata files + httpComponent.cleanupUdsFiles(); + if (!isBun && (debugThreads || process.env.DEV_MODE)) { + try { + inspector.close(); + } catch (error) { + harperLogger.info('Could not close debugger', error); } } - }) - .ref(); // use this to keep the thread running until we are ready to shutdown and clean up handles - const listening = listenOnPorts(); + } + }) + .ref(); // use this to keep the thread running until we are ready to shutdown and clean up handles + const listening = listenOnPorts(); - // notify that we are now ready to start receiving requests - Promise.resolve(listening).then(() => { - if (getWorkerIndex() === 0) { - try { - startupLog(portServer); - } catch (err) { - console.error('Error displaying start-up log', err); - } + // notify that we are now ready to start receiving requests. Chained with + // `return` so the outer loadedPromise / whenComponentsLoaded only fulfils + // once HTTP/HTTPS sockets are actually bound — apitests await that. + return Promise.resolve(listening).then(() => { + if (getWorkerIndex() === 0) { + try { + startupLog(portServer); + } catch (err) { + console.error('Error displaying start-up log', err); } - parentPort?.postMessage({ type: terms.ITC_EVENT_TYPES.CHILD_STARTED }); - }); + } + parentPort?.postMessage({ type: terms.ITC_EVENT_TYPES.CHILD_STARTED }); }); - componentsLoadedResolve(loaded); + }); + componentsLoadedResolve(loadedPromise); // Clean up UDS files and force-close Bun server connections on unexpected exit. // Without the stop(true) call, clients holding keep-alive connections to a dead Bun // worker never receive a FIN/RST and hang indefinitely waiting for a response. @@ -217,7 +215,6 @@ function startServers() { } httpComponent.cleanupUdsFiles(); }); - return loaded; } let listening; function listenOnPorts() { @@ -444,7 +441,21 @@ async function listenOnPortsBun() { return Promise.all(listening); } if (!isMainThread && !workerData?.noServerStart) { - startServers(); + // Workers start with an empty environment manager. Run the same init+startup + // sequence as the main entry (bin/harper.ts) before bringing up servers. + // startServers schedules its loadRootComponents().then(...) chain internally + // and notifies the parent via parentPort.postMessage(CHILD_STARTED) once the + // HTTP port is bound — don't await it here, otherwise the worker IIFE blocks + // on the same chain that main is waiting on, deadlocking the startup. + (async () => { + env.initSync(); + const { runStartup } = await import('../../utility/lifecycle.ts'); + await runStartup(); + startServers(); + })().catch((err) => { + harperLogger.fatal('Worker failed to start', err); + process.exit(1); + }); } /** @@ -453,7 +464,6 @@ if (!isMainThread && !workerData?.noServerStart) { * @param options */ function onSocket(listener, options) { - let getComponentName = require('../../components/componentLoader.ts').getComponentName; let socketServer; if (options.securePort) { setPortServerMap(options.securePort, { protocol_name: 'TLS', name: getComponentName() }); @@ -512,3 +522,8 @@ function onSocket(listener, options) { } return socketServer; } + +// Wire server singletons during the startup phase +onStartup(() => { + server.socket = onSocket; +}); diff --git a/sqlTranslator/alasqlFunctionImporter.ts b/sqlTranslator/alasqlFunctionImporter.ts index 14c9d7d4a..9660d22a1 100644 --- a/sqlTranslator/alasqlFunctionImporter.ts +++ b/sqlTranslator/alasqlFunctionImporter.ts @@ -4,9 +4,9 @@ * PUrpose of this is to set up a central module to define and import custom functions into alasql */ -import * as alasqlExtension from '../utility/functions/sql/alaSQLExtension.js'; +import * as alasqlExtension from '../utility/functions/sql/alaSQLExtension.ts'; import * as dateFunctions from '../utility/functions/date/dateFunctions.js'; -import * as geo from '../utility/functions/geo.js'; +import * as geo from '../utility/functions/geo.ts'; //import the custom function, need to define an upper and lower case version of the function so it is parsed properly in alasql export default function (alasql: any) { diff --git a/sqlTranslator/sql_statement_bucket.ts b/sqlTranslator/sql_statement_bucket.ts index 0bd69ef46..3b5b95058 100644 --- a/sqlTranslator/sql_statement_bucket.ts +++ b/sqlTranslator/sql_statement_bucket.ts @@ -6,7 +6,8 @@ import * as alasql from 'alasql'; import RecursiveIterator from 'recursive-iterator'; -const harperLogger = require('../utility/logging/harper_logger').default || require('../utility/logging/harper_logger'); +import _harperLogger from '../utility/logging/harper_logger.ts'; +const harperLogger = _harperLogger; import * as hdbUtils from '../utility/common_utils.ts'; import * as terms from '../utility/hdbTerms.ts'; diff --git a/unitTests/apiTests/setupTestApp.mjs b/unitTests/apiTests/setupTestApp.mjs index 6f38f542e..17398c309 100644 --- a/unitTests/apiTests/setupTestApp.mjs +++ b/unitTests/apiTests/setupTestApp.mjs @@ -35,6 +35,16 @@ function makeString() { let createdRecords; let serverStarted; export async function setupTestApp() { + // Drain onStartup hooks first so server-singleton wiring (server.http, server.getUser, + // server.operation, etc.) is installed before we override server.getUser below. + // Production runs this from bin/harper.ts; the API-test setup needs the same effect. + // Without this, the override below is later clobbered when user.ts's onStartup hook + // fires inside startHTTPThreads, and auth checks fall through to the real getUser. + if (typeof process !== 'undefined' && !serverStarted) { + const { runStartup } = await import('#src/utility/lifecycle'); + await runStartup(); + } + analytics.setAnalyticsEnabled(false); bypassAuth(); bypassAuthMQTT(); @@ -89,6 +99,11 @@ export async function setupTestApp() { } else { const { startHTTPThreads } = await import('#src/server/threads/socketRouter'); serverStarted = await startHTTPThreads(config.threads || 0); + // startServers() schedules its loadRootComponents().then(listenOnPorts) chain + // internally without returning the resolved promise. Block on the shared + // `whenComponentsLoaded` so the test's first axios call sees a bound socket. + const { whenComponentsLoaded } = await import('#src/server/threads/threadServer'); + await whenComponentsLoaded; } try { seed = 0; // reset the seed to make sure we are deterministic here diff --git a/unitTests/config/configUtils-runtimeEnvVars.test.js b/unitTests/config/configUtils-runtimeEnvVars.test.js index 5f30b6f4c..45a4c839c 100644 --- a/unitTests/config/configUtils-runtimeEnvVars.test.js +++ b/unitTests/config/configUtils-runtimeEnvVars.test.js @@ -37,13 +37,8 @@ describe('configUtils - applyRuntimeEnvVarConfig', function () { configUtils.__set__('fs', { writeFileSync: fsWriteFileSyncStub, renameSync: fsRenameSyncStub }); configUtils.__set__('YAML', YAMLStub); - // Mock harperConfigEnvVars module - configUtils.__set__('require', function (modulePath) { - if (modulePath.startsWith('./harperConfigEnvVars')) { - return { applyRuntimeEnvConfig: applyRuntimeEnvConfigStub }; - } - return require(modulePath); - }); + // Stub the top-level-imported applyRuntimeEnvConfig + configUtils.__set__('applyRuntimeEnvConfig', applyRuntimeEnvConfigStub); }); beforeEach(function () { diff --git a/unitTests/resources/create.test.js b/unitTests/resources/create.test.js index 2426ff0d6..3f030aafe 100644 --- a/unitTests/resources/create.test.js +++ b/unitTests/resources/create.test.js @@ -65,7 +65,17 @@ describe('Create records', () => { let id_after = CreateTest.getNewId(); assert(Math.abs(id_before - id_after) > 1000000); }); - after(() => { - test_thread.terminate(); + after(async () => { + // Graceful shutdown — let the worker close its rocksdb handles on its own + // event loop rather than relying on Worker.terminate(), which under + // rocksdb-js triggers a native finalizer crash during the next test's + // process-wide handle setup. + await new Promise((resolve) => { + test_thread.once('exit', resolve); + test_thread.postMessage({ type: 'shutdown' }); + setTimeout(() => { + test_thread.terminate().then(resolve); + }, 1000).unref(); + }); }); }); diff --git a/unitTests/testUtils.js b/unitTests/testUtils.js index 64d956515..30fe155f2 100644 --- a/unitTests/testUtils.js +++ b/unitTests/testUtils.js @@ -12,6 +12,7 @@ const harperBridge = require('#src/dataLayer/harperBridge/harperBridge').default const { isMainThread } = require('node:worker_threads'); const { getDatabases, databases } = require('#src/resources/databases'); const { handleHDBError } = require('#src/utility/errors/hdbError'); +const lifecycle = require('#src/utility/lifecycle'); let envMgrInitSyncStub; @@ -83,6 +84,12 @@ function preTestPrep(testConfigObj) { // Try to change to bin changeProcessToBinDir(); env.initTestEnvironment(testConfigObj); + + // Drain startup hooks so modules that defer side effects via `onStartup(...)` + // (server-singleton wiring, listener registration, config-derived constants) + // see a wired-up state. Production calls this from `bin/harper.ts`; tests + // that go through this helper get the same effect. Idempotent. + void lifecycle.runStartup(); } /** diff --git a/unitTests/utility/logging/readLog.test.js b/unitTests/utility/logging/readLog.test.js index 589ecd5f6..50d16db45 100644 --- a/unitTests/utility/logging/readLog.test.js +++ b/unitTests/utility/logging/readLog.test.js @@ -74,7 +74,7 @@ describe('Test readLog module', () => { }); beforeEach(() => { - getConfigPath_rw = read_log.__set__('configUtils_js_1', { + getConfigPath_rw = read_log.__set__('configUtils_ts_1', { getConfigPath: (key) => { if (key === hdb_terms.HDB_SETTINGS_NAMES.LOG_PATH_KEY) { return TEST_LOG_DIR; diff --git a/upgrade/upgradeUtilities.ts b/upgrade/upgradeUtilities.ts index 9d2151c91..abb686608 100644 --- a/upgrade/upgradeUtilities.ts +++ b/upgrade/upgradeUtilities.ts @@ -1,7 +1,7 @@ 'use strict'; import * as hdbUtil from '../utility/common_utils.ts'; -import * as configUtils from '../config/configUtils.js'; +import * as configUtils from '../config/configUtils.ts'; /** * We need to make sure we are setting empty string for values that are null/undefined/empty string - PropertiesReader diff --git a/utility/common_utils.ts b/utility/common_utils.ts index 5552b71cb..e386a1f77 100644 --- a/utility/common_utils.ts +++ b/utility/common_utils.ts @@ -1,8 +1,8 @@ 'use strict'; import * as path from 'path'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import log from './logging/harper_logger.ts'; -import * as fsExtra from 'fs-extra'; +import fsExtra from 'fs-extra'; import * as os from 'os'; import * as net from 'net'; import RecursiveIterator from 'recursive-iterator'; diff --git a/utility/environment/environmentManager.ts b/utility/environment/environmentManager.ts index cbcc82765..98979d439 100644 --- a/utility/environment/environmentManager.ts +++ b/utility/environment/environmentManager.ts @@ -1,27 +1,27 @@ 'use strict'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'path'; import * as os from 'os'; import PropertiesReader from 'properties-reader'; import log from '../logging/harper_logger.ts'; import * as commonUtils from '../common_utils.ts'; import * as hdbTerms from '../hdbTerms.ts'; -import * as configUtils from '../../config/configUtils.js'; +import * as configUtils from '../../config/configUtils.ts'; import { mkdirSync } from 'node:fs'; -const INIT_ERR = 'Error initializing environment manager'; -const BOOT_PROPS_FILE_PATH = 'BOOT_PROPS_FILE_PATH'; +var INIT_ERR = 'Error initializing environment manager'; +var BOOT_PROPS_FILE_PATH = 'BOOT_PROPS_FILE_PATH'; -let propFileExists = false; +var propFileExists = false; -const installPropsToSave = { +var installPropsToSave = { [hdbTerms.HDB_SETTINGS_NAMES.INSTALL_USER]: true, [hdbTerms.HDB_SETTINGS_NAMES.SETTINGS_PATH_KEY]: true, [hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY]: true, BOOT_PROPS_FILE_PATH: true, }; -let installProps: any = {}; +var installProps: any = {}; export { BOOT_PROPS_FILE_PATH }; /** @@ -30,7 +30,7 @@ export { BOOT_PROPS_FILE_PATH }; * currently known base path here to help with this case. */ export function getHdbBasePath() { - return installProps[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY]; + return installProps?.[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY]; } /** @@ -48,11 +48,18 @@ export function setHdbBasePath(hdbPath: string) { * @returns {*} */ export function get(propName: string): any { - const value = configUtils.getConfigValue(propName); + // Tolerate calls before this module's body has executed (ESM cycle): + // configUtils.ts / installProps may not be initialized yet, in which case + // no config has been read so the value is genuinely unknown. + let value; + try { + value = configUtils.getConfigValue(propName); + } catch (err: any) { + if (err?.name !== 'ReferenceError') throw err; + } if (value === undefined) { - return installProps[propName]; + return installProps?.[propName]; } - return value; } @@ -114,7 +121,13 @@ export function initSync(force: boolean = false) { installProps[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY] = configHdbRoot; } } - } catch (err) { + } catch (err: any) { + // During typestrip ESM evaluation, module-load callers of initSync may + // reach this before configUtils has finished its own top-level evaluation, + // producing a ReferenceError (TDZ) on a module-scope binding. Don't exit + // the process for that — the same module-load chain will retry once + // evaluation completes, or bin/harper.ts will re-call initSync(). + if (err?.name === 'ReferenceError') return; log.error(INIT_ERR); log.error(err); console.error(err); diff --git a/utility/environment/systemInformation.ts b/utility/environment/systemInformation.ts index 7458954f5..18955cc3f 100644 --- a/utility/environment/systemInformation.ts +++ b/utility/environment/systemInformation.ts @@ -4,14 +4,16 @@ import si from 'systeminformation'; import logger from '../logging/harper_logger.ts'; import * as hdbTerms from '../hdbTerms.ts'; import { lmdbGetTableSize } from '../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.ts'; -import { getThreadInfo } from '../../server/threads/manageThreads.js'; +import { getThreadInfo } from '../../server/threads/manageThreads.ts'; import * as env from './environmentManager.ts'; import { getDatabases, type Table } from '../../resources/databases.ts'; import { TableSizeObject } from '../../dataLayer/harperBridge/TableSizeObject.ts'; -import { RocksDatabase, StatsHistogramData } from '@harperfast/rocksdb-js'; - -env.initSync(); - +import { RocksDatabase, type StatsHistogramData } from '@harperfast/rocksdb-js'; +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} //this will hold the system_information which is static to improve performance let systemInformationCache = undefined; diff --git a/utility/functions/geo.js b/utility/functions/geo.ts similarity index 85% rename from utility/functions/geo.js rename to utility/functions/geo.ts index 11df524c2..4e1c27c46 100644 --- a/utility/functions/geo.js +++ b/utility/functions/geo.ts @@ -1,5 +1,3 @@ -'use strict'; - /*** * geo.js * @@ -7,20 +5,20 @@ * turf.js has very robust internal validation as such we offload the validation to turf.js */ -const turfArea = require('@turf/area'); -const turfLength = require('@turf/length'); -const turfCircle = require('@turf/circle'); -const turfDifference = require('@turf/difference'); -const turfDistance = require('@turf/distance'); -const turfBooleanContains = require('@turf/boolean-contains'); -const turfBooleanEqual = require('@turf/boolean-equal'); -const turfBooleanDisjoint = require('@turf/boolean-disjoint'); -const turfHelpers = require('@turf/helpers'); -const hdbTerms = require('../hdbTerms.ts'); -const commonUtils = require('../common_utils.ts'); -const hdbLog = require('../logging/harper_logger.ts'); - -module.exports = { +import turfArea from '@turf/area'; +import turfLength from '@turf/length'; +import turfCircle from '@turf/circle'; +import turfDifference from '@turf/difference'; +import turfDistance from '@turf/distance'; +import turfBooleanContains from '@turf/boolean-contains'; +import turfBooleanEqual from '@turf/boolean-equal'; +import turfBooleanDisjoint from '@turf/boolean-disjoint'; +import * as turfHelpers from '@turf/helpers'; +import * as hdbTerms from '../hdbTerms.ts'; +import * as commonUtils from '../common_utils.ts'; +import hdbLog from '../logging/harper_logger.ts'; + +export { geoArea, geoLength, geoCircle, @@ -32,7 +30,6 @@ module.exports = { geoCrosses, geoConvert, }; - /*** * Takes one or more features and returns the area in square meters * @param geoJSON @@ -47,7 +44,7 @@ function geoArea(geoJSON) { geoJSON = commonUtils.autoCastJSON(geoJSON); } try { - return turfArea.default(geoJSON); + return turfArea(geoJSON); } catch (err) { hdbLog.trace(err, geoJSON); return NaN; @@ -70,7 +67,7 @@ function geoLength(geoJSON, units) { } try { - return turfLength.default(geoJSON, { units: units ? units : 'kilometers' }); + return turfLength(geoJSON, { units: units ? units : 'kilometers' }); } catch (err) { hdbLog.trace(err, geoJSON); return NaN; @@ -98,7 +95,7 @@ function geoCircle(point, radius, units) { } try { - return turfCircle.default(point, radius, { units: units ? units : 'kilometers' }); + return turfCircle(point, radius, { units: units ? units : 'kilometers' }); } catch (err) { hdbLog.trace(err, point, radius); return NaN; @@ -160,7 +157,7 @@ function geoDistance(point1, point2, units) { } try { - return turfDistance.default(point1, point2, { units: units ? units : 'kilometers' }); + return turfDistance(point1, point2, { units: units ? units : 'kilometers' }); } catch (err) { hdbLog.trace(err, point1, point2); return NaN; @@ -239,7 +236,7 @@ function geoContains(geo1, geo2) { } try { - return turfBooleanContains.default(geo1, geo2); + return turfBooleanContains(geo1, geo2); } catch (err) { hdbLog.trace(err, geo1, geo2); return false; @@ -277,7 +274,7 @@ function geoEqual(geo1, geo2) { } try { - return turfBooleanEqual.default(geo1, geo2); + return turfBooleanEqual(geo1, geo2); } catch (err) { hdbLog.trace(err, geo1, geo2); return false; @@ -316,7 +313,7 @@ function geoCrosses(geo1, geo2) { try { //need to do ! as this checks for non-intersections of geometries - return !turfBooleanDisjoint.default(geo1, geo2); + return !turfBooleanDisjoint(geo1, geo2); } catch (err) { hdbLog.trace(err, geo1, geo2); return false; diff --git a/utility/functions/sql/alaSQLExtension.js b/utility/functions/sql/alaSQLExtension.ts similarity index 60% rename from utility/functions/sql/alaSQLExtension.js rename to utility/functions/sql/alaSQLExtension.ts index db3bdaef5..701db0eab 100644 --- a/utility/functions/sql/alaSQLExtension.js +++ b/utility/functions/sql/alaSQLExtension.ts @@ -1,50 +1,26 @@ -'use strict'; - /*** * alaSQLExtension.js * purpose of this module is to hold custom functions for alasql */ -const _ = require('lodash'); -const mathjs = require('mathjs'); -const jsonata = require('jsonata'); -const hdbUtils = require('../../common_utils.ts'); - -module.exports = { - /*** - * distinctArray takes in an array an dedupes its values using lodash. this works on complex as well as simple datatypes - * @param array - * @returns array - */ - distinct_array: (array) => { - if (Array.isArray(array) && array.length > 1) { - return _.uniqWith(array, _.isEqual); - } +import _ from 'lodash'; +import * as mathjs from 'mathjs'; +import jsonata from 'jsonata'; +import * as hdbUtils from '../../common_utils.ts'; +export const distinct_array = (array) => { + if (Array.isArray(array) && array.length > 1) { + return _.uniqWith(array, _.isEqual); + } - return array; - }, - searchJSON, - /*** - * median absolute deviation aggregate function based on http://mathjs.org/docs/reference/functions/mad.html - */ - mad: aggregateFunction.bind(null, mathjs.mad), - /*** - * mean aggregate function based on http://mathjs.org/docs/reference/functions/mean.html - */ - mean: aggregateFunction.bind(null, mathjs.mean), - /*** - * computes the mode of values on http://mathjs.org/docs/reference/functions/mode.html - */ - mode: aggregateFunction.bind(null, mathjs.mode), - /*** - * compute the product based on http://mathjs.org/docs/reference/functions/prod.html - */ - prod: aggregateFunction.bind(null, mathjs.prod), - /*** - * compute the median based on http://mathjs.org/docs/reference/functions/median.html - */ - median: aggregateFunction.bind(null, mathjs.median), + return array; }; +export const mad = aggregateFunction.bind(null, mathjs.mad); +export const mean = aggregateFunction.bind(null, mathjs.mean); +export const mode = aggregateFunction.bind(null, mathjs.mode); +export const prod = aggregateFunction.bind(null, mathjs.prod); +export const median = aggregateFunction.bind(null, mathjs.median); +export { searchJSON }; +export default { distinct_array, searchJSON, mad, mean, mode, prod, median }; /*** * handles the 3 pass loop for aggregates and executes the final calc with the passed in aggregator function diff --git a/utility/globalSchema.ts b/utility/globalSchema.ts index 3a21f225b..e89e13e79 100644 --- a/utility/globalSchema.ts +++ b/utility/globalSchema.ts @@ -1,4 +1,4 @@ -import systemSchema from '../json/systemSchema.json'; +import systemSchema from '../json/systemSchema.json' with { type: 'json' }; import { promisify } from 'util'; import { getDatabases } from '../resources/databases.ts'; diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index ee63fc206..00319bd8e 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -19,7 +19,7 @@ export const HDB_COMPONENT_CONFIG_FILE = 'config.yaml'; /** Name of the Harper Process Script */ export const HDB_PROC_NAME = 'harper.js'; /** Name of the Harper Restart Script */ -export const HDB_RESTART_SCRIPT = 'restartHdb.js'; +export const HDB_RESTART_SCRIPT = 'restartHdb.ts'; /** Harper Process Descriptor */ const HDB_PROC_DESCRIPTOR = 'Harper'; diff --git a/utility/install/checkJWTTokensExist.js b/utility/install/checkJWTTokensExist.ts similarity index 69% rename from utility/install/checkJWTTokensExist.js rename to utility/install/checkJWTTokensExist.ts index ecc437876..95a58265d 100644 --- a/utility/install/checkJWTTokensExist.js +++ b/utility/install/checkJWTTokensExist.ts @@ -1,14 +1,20 @@ -'use strict'; +import * as env from '../../utility/environment/environmentManager.ts'; +import fs from 'fs-extra'; +import path from 'path'; +import * as terms from '../../utility/hdbTerms.ts'; +import crypto from 'crypto'; +import { v4 as uuid } from 'uuid'; -const env = require('../../utility/environment/environmentManager.ts'); -env.initSync(); -const fs = require('fs-extra'); -const path = require('path'); -const terms = require('../../utility/hdbTerms.ts'); -const crypto = require('crypto'); -const uuid = require('uuid').v4; +export default checkJWTTokenExist; -module.exports = checkJWTTokenExist; +// Preserve the CJS-callable shape consumers/tests expect — under tsc CJS emit +// this replaces the `exports` object with the function itself, so +// `require('./checkJWTTokenExist')(...)` works. No-op under typestrip ESM. +declare const module: any; +if (typeof module !== 'undefined') { + module.exports = checkJWTTokenExist; + module.exports.default = checkJWTTokenExist; +} /** * checks that the RSA keys exist for JWT generation, if not we create them */ diff --git a/utility/install/installer.ts b/utility/install/installer.ts index b54a86514..573e775e3 100644 --- a/utility/install/installer.ts +++ b/utility/install/installer.ts @@ -2,8 +2,9 @@ import * as os from 'os'; import inquirer from 'inquirer'; -import * as fs from 'fs-extra'; -import PropertiesReader from 'properties-reader'; +import fs from 'fs-extra'; +import _PropertiesReader from 'properties-reader'; +const PropertiesReader = _PropertiesReader; import chalk from 'chalk'; import * as path from 'path'; let ora; // Will be loaded dynamically as it's an ES module @@ -17,12 +18,15 @@ import * as hdbInfoController from '../../dataLayer/hdbInfoController.ts'; import { packageJson } from '../packageUtils.js'; import * as hdbTerms from '../hdbTerms.ts'; const { CONFIG_PARAMS } = hdbTerms; -import installValidator from '../../validation/installValidator.ts'; -import mountHdb from '../mount_hdb.ts'; -import * as configUtils from '../../config/configUtils.js'; +import _installValidator from '../../validation/installValidator.ts'; +const installValidator = _installValidator; +import _mountHdb from '../mount_hdb.ts'; +const mountHdb = _mountHdb; +import * as configUtils from '../../config/configUtils.ts'; import * as userOps from '../../security/user.ts'; import * as roleOps from '../../security/role.ts'; -import checkJwtTokens from './checkJWTTokensExist.js'; +import _checkJwtTokens from './checkJWTTokensExist.ts'; +const checkJwtTokens = _checkJwtTokens; import * as globalSchema from '../globalSchema.ts'; import { promisify } from 'util'; const pSchemaToGlobal = promisify(globalSchema.setSchemaDataToGlobal); diff --git a/utility/lifecycle.ts b/utility/lifecycle.ts new file mode 100644 index 000000000..d8640a364 --- /dev/null +++ b/utility/lifecycle.ts @@ -0,0 +1,68 @@ +// Startup-phase lifecycle: lets modules declare side-effectful initialization +// (server-singleton wiring, config-derived constants, listener registration) +// without running it at module-load time. The entry point invokes +// `runStartup()` after `env.initSync()` and before the server starts handling +// requests, so all hooks see a fully-linked module graph and an initialized +// environment. +// +// Usage: +// import { onStartup } from '.../utility/lifecycle.ts'; +// onStartup(() => { +// server.recordAnalytics = recordAction; +// }); +// +// Unit tests: any test that exercises code paths depending on these hooks must +// either call `runStartup()` itself in a `before`/`beforeEach`, or import the +// real CLI entry point. `runStartup()` is idempotent — calling it a second +// time is a no-op until `resetStartupForTests()` is called. + +type StartupCallback = () => void | Promise; + +const callbacks: StartupCallback[] = []; +let started = false; +let runningPromise: Promise | null = null; + +/** + * Register a callback to be run during the startup phase. If startup has + * already run, the callback is invoked on the next microtask. + */ +export function onStartup(cb: StartupCallback): void { + if (started) { + Promise.resolve().then(cb); + return; + } + callbacks.push(cb); +} + +/** + * Run all registered startup callbacks in registration order. Idempotent: + * subsequent calls return the same promise as the first invocation. + */ +export function runStartup(): Promise { + if (runningPromise) return runningPromise; + runningPromise = (async () => { + started = true; + // Snapshot in case callbacks register more callbacks (they'll run + // immediately via the microtask path above). + const pending = callbacks.splice(0, callbacks.length); + for (const cb of pending) { + await cb(); + } + })(); + return runningPromise; +} + +/** + * Reset startup state. Intended for unit tests that want to re-run startup + * (e.g. between describe blocks). Production code should never call this. + */ +export function resetStartupForTests(): void { + callbacks.length = 0; + started = false; + runningPromise = null; +} + +/** True once `runStartup()` has begun. */ +export function hasStarted(): boolean { + return started; +} diff --git a/utility/lmdb/OpenDBIObject.ts b/utility/lmdb/OpenDBIObject.ts index 2076bfdab..3d2d7630b 100644 --- a/utility/lmdb/OpenDBIObject.ts +++ b/utility/lmdb/OpenDBIObject.ts @@ -2,8 +2,11 @@ import * as envMngr from '../environment/environmentManager.ts'; import * as terms from '../../utility/hdbTerms.ts'; import { RecordEncoder } from '../../resources/RecordEncoder.ts'; -envMngr.initSync(); - +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const LMDB_CACHING = envMngr.get(terms.CONFIG_PARAMS.STORAGE_CACHING) !== false; /** diff --git a/utility/lmdb/OpenEnvironmentObject.ts b/utility/lmdb/OpenEnvironmentObject.ts index c14583108..5c800983b 100644 --- a/utility/lmdb/OpenEnvironmentObject.ts +++ b/utility/lmdb/OpenEnvironmentObject.ts @@ -7,8 +7,11 @@ const MAX_DBS = 10000; const MAX_READERS = 2048; import * as envMngr from '../environment/environmentManager.ts'; import * as terms from '../../utility/hdbTerms.ts'; -envMngr.initSync(); - +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} export default class OpenEnvironmentObject { [key: string]: any; static MAX_DBS = MAX_DBS; diff --git a/utility/lmdb/environmentUtility.ts b/utility/lmdb/environmentUtility.ts index c602e91fd..caf3a2901 100644 --- a/utility/lmdb/environmentUtility.ts +++ b/utility/lmdb/environmentUtility.ts @@ -1,7 +1,7 @@ 'use strict'; import * as lmdb from 'lmdb'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'path'; import * as common from './commonUtility.ts'; import log from '../logging/harper_logger.ts'; diff --git a/utility/lmdb/writeUtility.ts b/utility/lmdb/writeUtility.ts index bd29bc6ca..e18e08e33 100644 --- a/utility/lmdb/writeUtility.ts +++ b/utility/lmdb/writeUtility.ts @@ -13,8 +13,11 @@ import { v4 as uuidv4 } from 'uuid'; import * as lmdb from 'lmdb'; import { handleHDBError, hdbErrors } from '../errors/hdbError.ts'; import * as envMngr from '../environment/environmentManager.ts'; -envMngr.initSync(); - +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const LMDB_PREFETCH_WRITES = envMngr.get(hdbTerms.CONFIG_PARAMS.STORAGE_PREFETCHWRITES); const CREATED_TIME_ATTRIBUTE_NAME = hdbTerms.TIME_STAMP_NAMES_ENUM.CREATED_TIME; diff --git a/utility/logging/harper_logger.ts b/utility/logging/harper_logger.ts index eb2f8e3b2..05acbe43f 100644 --- a/utility/logging/harper_logger.ts +++ b/utility/logging/harper_logger.ts @@ -1,11 +1,12 @@ 'use strict'; // Note - do not import/use commonUtils.js in this module, it will cause circular dependencies. -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import { workerData, threadId, isMainThread } from 'worker_threads'; import * as pathModule from 'path'; import * as YAML from 'yaml'; -const PropertiesReader = require('properties-reader'); +import _PropertiesReader from 'properties-reader'; +const PropertiesReader = _PropertiesReader; import * as hdbTerms from '../hdbTerms.ts'; import assignCMDENVVariables from '../assignCmdEnvVariables.ts'; import * as os from 'os'; @@ -22,7 +23,6 @@ let nativeStdWrite = process.env.IS_SCRIPTED_SERVICE : (process.stdout as any).nativeWrite || ((process.stdout as any).nativeWrite = process.stdout.write); let fileLoggers = new Map(); const { join } = pathModule; - const MAX_LOG_BUFFER = 10000; const LOG_LEVEL_HIERARCHY = { notify: 7, @@ -141,6 +141,9 @@ function resolveLogPath(configPath: string, rootPath: string) { } async function updateLogSettings() { if (!rootConfig) { + // Lazy-load to avoid a circular dependency at module evaluation time + // (RootConfigWatcher imports configUtils which imports this logger). + const { RootConfigWatcher } = await import('../../config/RootConfigWatcher.ts'); // set up the initial watcher rootConfig = new RootConfigWatcher(); // wait for it to be ready @@ -264,36 +267,41 @@ class HarperLogger extends Console { if (hdbProperties === undefined) initLogSettings(); -module.exports = { - notify, - fatal, - error, - warn, - info, - debug, - trace, - logLevel, - loggerWithTag, - suppressLogging, - initLogSettings, - logCustomLevel, - closeLogFile, - createLogger, - logsAtLevel, - getLogFilePath: () => logFilePath, - forComponent: (name, isExternal) => mainLogger.forComponent(name, isExternal), - setMainLogger, - setLogLevel, - OUTPUTS, - AuthAuditLog, - // for now these functions at least notify us of when the component system is ready so - // we can start using the RootConfigWatcher - start: updateLogSettings, - startOnMainThread: updateLogSettings, - errorToString, - disableStdio, - externalLogger, -}; +// Under tsc CJS emit, expose the same CommonJS shape historical consumers +// (`const logger = require('./harper_logger')`) depend on. Skipped when this +// file is loaded as ESM (Node type-strip), where `module` is undefined. +if (typeof module !== 'undefined') { + module.exports = { + notify, + fatal, + error, + warn, + info, + debug, + trace, + logLevel, + loggerWithTag, + suppressLogging, + initLogSettings, + logCustomLevel, + closeLogFile, + createLogger, + logsAtLevel, + getLogFilePath: () => logFilePath, + forComponent: (name, isExternal) => mainLogger.forComponent(name, isExternal), + setMainLogger, + setLogLevel, + OUTPUTS, + AuthAuditLog, + // for now these functions at least notify us of when the component system is ready so + // we can start using the RootConfigWatcher + start: updateLogSettings, + startOnMainThread: updateLogSettings, + errorToString, + disableStdio, + externalLogger, + }; +} /** * We call this if stdio is not functional @@ -630,15 +638,21 @@ function getFileLogger(path, rotation, isExternalInstance) { setTimeout(() => { logger.rotator?.end(); if (!rotation) return; - const logRotator = require('./logRotator'); - try { - logger.rotator = logRotator({ - logger, - ...rotation, + import('./logRotator.ts') + .then((mod) => { + try { + const logRotator: any = mod.default ?? mod; + logger.rotator = logRotator({ + logger, + ...rotation, + }); + } catch (error) { + logger('Error initializing log rotator', error); + } + }) + .catch((error) => { + logger('Error initializing log rotator', error); }); - } catch (error) { - logger('Error initializing log rotator', error); - } }, 100); } return logger; @@ -921,9 +935,6 @@ export function AuthAuditLog( this.request_method = requestMethod; this.path = path; } -// we have to load this at the end to avoid circular dependencies problems -import { RootConfigWatcher } from '../../config/RootConfigWatcher.ts'; - export const getLogFilePath = () => logFilePath; export const forComponent = (name: string, isExternal?: boolean) => mainLogger.forComponent(name, isExternal); export default { diff --git a/utility/logging/logRotator.ts b/utility/logging/logRotator.ts index a0e24c7df..e3ca19da7 100644 --- a/utility/logging/logRotator.ts +++ b/utility/logging/logRotator.ts @@ -7,7 +7,11 @@ import { pipeline } from 'stream'; const pipe = promisify(pipeline); import * as path from 'path'; import * as envMgr from '../environment/environmentManager.ts'; -envMgr.initSync(); +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} import hdbLogger from './harper_logger.ts'; import { CONFIG_PARAMS } from '../hdbTerms.ts'; import { convertToMS } from '../common_utils.ts'; diff --git a/utility/logging/readLog.ts b/utility/logging/readLog.ts index 5d5fd4b17..63e77ad50 100644 --- a/utility/logging/readLog.ts +++ b/utility/logging/readLog.ts @@ -4,9 +4,9 @@ import * as hdbTerms from '../hdbTerms.ts'; import hdbLogger from './harper_logger.ts'; import validator from '../../validation/readLogValidator.ts'; import * as path from 'path'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import { once } from 'events'; -import { getConfigPath } from '../../config/configUtils.js'; +import { getConfigPath } from '../../config/configUtils.ts'; import { handleHDBError, hdbErrors } from '../errors/hdbError.ts'; import { server } from '../../server/Server.ts'; diff --git a/utility/logging/transactionLog.ts b/utility/logging/transactionLog.ts index 2d9bb66f4..46bbeafec 100644 --- a/utility/logging/transactionLog.ts +++ b/utility/logging/transactionLog.ts @@ -9,7 +9,7 @@ import { readTransactionLogValidator, deleteTransactionLogsBeforeValidator, } from '../../validation/transactionLogValidator.ts'; -const harperBridge = require('../../dataLayer/harperBridge/harperBridge').default; +import harperBridge from '../../dataLayer/harperBridge/harperBridge.ts'; export async function readTransactionLog(req: any) { const validation = readTransactionLogValidator(req); diff --git a/utility/mount_hdb.ts b/utility/mount_hdb.ts index f9355ff04..7dc3bea08 100644 --- a/utility/mount_hdb.ts +++ b/utility/mount_hdb.ts @@ -1,12 +1,14 @@ 'use strict'; -const { mkdirpSync, copySync } = require('fs-extra'); +import fsExtra from 'fs-extra'; +const { mkdirpSync, copySync } = fsExtra; import * as path from 'path'; import * as terms from '../utility/hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; import bridge from '../dataLayer/harperBridge/harperBridge.ts'; -import systemSchema from '../json/systemSchema.json'; -import * as initPaths from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js'; +import systemSchema from '../json/systemSchema.json' with { type: 'json' }; +import CreateTableObject from '../dataLayer/CreateTableObject.ts'; +import * as initPaths from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts'; import { PACKAGE_ROOT } from '../utility/packageUtils.js'; export default async function mountHdb(hdbPath: string) { @@ -28,9 +30,6 @@ export default async function mountHdb(hdbPath: string) { * @returns {Promise} */ async function createTables() { - const CreateTableObject = - require('../dataLayer/CreateTableObject').default || require('../dataLayer/CreateTableObject'); - let tables = Object.keys(systemSchema); for (const tableName of tables) { diff --git a/utility/npmUtilities.ts b/utility/npmUtilities.ts index 876519465..c1310eedc 100644 --- a/utility/npmUtilities.ts +++ b/utility/npmUtilities.ts @@ -6,12 +6,11 @@ import * as path from 'path'; import { handleHDBError, hdbErrors } from './errors/hdbError.ts'; const { HTTP_STATUS_CODES } = hdbErrors; - import * as validator from '../validation/validationWrapper.ts'; import harperLogger from './logging/harper_logger.ts'; import { CONFIG_PARAMS } from './hdbTerms.ts'; -import { getConfigPath } from '../config/configUtils.js'; +import { getConfigPath } from '../config/configUtils.ts'; import { nonInteractiveSpawn } from '../components/Application.ts'; /** diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index e54ba6ed9..1ae35fe72 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -26,12 +26,12 @@ import * as commonUtils from './common_utils.ts'; import * as restart from '../bin/restart.ts'; import * as terms from './hdbTerms.ts'; import { expandOperationsPerms } from './operationPermissions.ts'; -import * as permsTranslator from '../security/permissionsTranslator.js'; +import * as permsTranslator from '../security/permissionsTranslator.ts'; import { systemInformation } from '../utility/environment/systemInformation.ts'; import * as tokenAuthentication from '../security/tokenAuthentication.ts'; import * as auth from '../security/auth.ts'; -import * as configUtils from '../config/configUtils.js'; -import * as functionsOperations from '../components/operations.js'; +import * as configUtils from '../config/configUtils.ts'; +import * as functionsOperations from '../components/operations.ts'; import * as transactionLog from '../utility/logging/transactionLog.ts'; import * as npmUtilities from './npmUtilities.ts'; import * as analytics from '../resources/analytics/read.ts'; @@ -286,12 +286,6 @@ requiredPermissions.set(terms.VALID_SQL_OPS_ENUM.SELECT, new (permission as any) requiredPermissions.set(terms.VALID_SQL_OPS_ENUM.INSERT, new (permission as any)(false, [INSERT_PERM])); requiredPermissions.set(terms.VALID_SQL_OPS_ENUM.UPDATE, new (permission as any)(false, [UPDATE_PERM])); -module.exports = { - verifyPerms, - verifyPermsAST, - verifyBulkLoadAttributePerms, -}; - /** * Verifies permissions and restrictions for a SQL operation based on the user's assigned role. * @param ast - The SQL statement in Syntax Tree form. diff --git a/utility/packageUtils.js b/utility/packageUtils.js index 2279d4397..d5b9b8071 100644 --- a/utility/packageUtils.js +++ b/utility/packageUtils.js @@ -43,4 +43,22 @@ const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8')); */ const PACKAGE_ROOT = dirname(packageJsonPath); -module.exports = { packageJson, PACKAGE_ROOT }; +/** + * The directory that holds source files at runtime: `PACKAGE_ROOT` in + * type-strip mode (where `node bin/harper.ts` runs the .ts sources directly) + * and `PACKAGE_ROOT/dist` in dist mode (where transpiled .js files live). + * + * `__dirname` of this CJS file resolves to either `/utility` + * (source) or `/dist/utility` (dist), so we can detect the mode + * just by looking at this file's own location. + */ +const RUNTIME_SRC_ROOT = __dirname.startsWith(join(PACKAGE_ROOT, 'dist')) ? join(PACKAGE_ROOT, 'dist') : PACKAGE_ROOT; + +/** + * File extension of the running modules: `.ts` in type-strip mode, `.js` in + * dist mode. Use this when constructing file paths for `new Worker(...)` or + * similar APIs that need the on-disk filename. + */ +const RUNTIME_FILE_EXT = RUNTIME_SRC_ROOT === PACKAGE_ROOT ? '.ts' : '.js'; + +module.exports = { packageJson, PACKAGE_ROOT, RUNTIME_SRC_ROOT, RUNTIME_FILE_EXT }; diff --git a/utility/processManagement/processManagement.js b/utility/processManagement/processManagement.ts similarity index 88% rename from utility/processManagement/processManagement.js rename to utility/processManagement/processManagement.ts index 828255c50..4ce1f771f 100644 --- a/utility/processManagement/processManagement.js +++ b/utility/processManagement/processManagement.ts @@ -1,16 +1,14 @@ -'use strict'; +import * as hdbTerms from '../hdbTerms.ts'; +import * as servicesConfig from './servicesConfig.ts'; +import * as envMangr from '../environment/environmentManager.ts'; +import hdbLogger from '../../utility/logging/harper_logger.ts'; +import { onMessageFromWorkers } from '../../server/threads/manageThreads.ts'; +import fs from 'fs'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { execFile, fork } from 'node:child_process'; -const hdbTerms = require('../hdbTerms.ts'); -const servicesConfig = require('./servicesConfig.js'); -const envMangr = require('../environment/environmentManager.ts'); -const hdbLogger = require('../../utility/logging/harper_logger.ts'); -const { onMessageFromWorkers } = require('../../server/threads/manageThreads.js'); -const fs = require('fs'); -const path = require('node:path'); -const { setTimeout: delay } = require('node:timers/promises'); -const { execFile, fork } = require('node:child_process'); - -module.exports = { +export { start, restart, kill, @@ -20,9 +18,11 @@ module.exports = { cleanupChildrenProcesses, expectedRestartOfChildren, }; - -onMessageFromWorkers((message) => { - if (message.type === 'restart') envMangr.initSync(true); +// Defer registration to setImmediate so manageThreads internal state is initialized +setImmediate(() => { + onMessageFromWorkers((message) => { + if (message.type === 'restart') envMangr.initSync(true); + }); }); let childProcesses = []; diff --git a/utility/processManagement/servicesConfig.js b/utility/processManagement/servicesConfig.ts similarity index 80% rename from utility/processManagement/servicesConfig.js rename to utility/processManagement/servicesConfig.ts index 0e269b038..4e98ea652 100644 --- a/utility/processManagement/servicesConfig.js +++ b/utility/processManagement/servicesConfig.ts @@ -1,9 +1,7 @@ -'use strict'; - -const hdbTerms = require('../hdbTerms.ts'); -const path = require('path'); -const { PACKAGE_ROOT } = require('../../utility/packageUtils.js'); -const hdbUtils = require('../common_utils.ts'); +import * as hdbTerms from '../hdbTerms.ts'; +import path from 'path'; +import { PACKAGE_ROOT } from '../../utility/packageUtils.js'; +import * as hdbUtils from '../common_utils.ts'; const SCRIPTS_DIR = path.join(PACKAGE_ROOT, 'utility/scripts'); const RESTART_SCRIPT = path.join(SCRIPTS_DIR, hdbTerms.HDB_RESTART_SCRIPT); @@ -49,8 +47,4 @@ function generateAllServiceConfigs() { }; } -module.exports = { - generateAllServiceConfigs, - generateMainServerConfig, - generateRestart, -}; +export { generateAllServiceConfigs, generateMainServerConfig, generateRestart }; diff --git a/utility/scripts/restartHdb.js b/utility/scripts/restartHdb.ts similarity index 85% rename from utility/scripts/restartHdb.js rename to utility/scripts/restartHdb.ts index 141702fa6..a30baecd9 100644 --- a/utility/scripts/restartHdb.js +++ b/utility/scripts/restartHdb.ts @@ -1,7 +1,5 @@ -'use strict'; - -const pm2Utils = require('../processManagement/processManagement.js'); -const hdbTerms = require('../hdbTerms.ts'); +import * as pm2Utils from '../processManagement/processManagement.ts'; +import * as hdbTerms from '../hdbTerms.ts'; /** * Gets a list of all the running Harper processes and calls reload on each one. diff --git a/utility/signalling.ts b/utility/signalling.ts index 01a5930e4..5a53235d3 100644 --- a/utility/signalling.ts +++ b/utility/signalling.ts @@ -3,15 +3,33 @@ import * as hdbTerms from './hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; import ITCEventObject from '../server/itc/utility/ITCEventObject.js'; +import { sendItcEvent } from '../server/threads/itc.ts'; +import { onStartup } from './lifecycle.ts'; + let serverItcHandlers; -import { sendItcEvent } from '../server/threads/itc.js'; +// Preload server-itc-handlers during the startup phase so signalSchemaChange +// and signalUserChange can stay synchronous (their callers and tests assume so). +onStartup(async () => { + const mod: any = await import('../server/itc/serverHandlers.ts'); + // CJS dist double-wraps default: namespace.default is `exports`, exports.default is the real value. + serverItcHandlers = mod.default?.default ?? mod.default ?? mod; +}); + +function ensureServerItcHandlers() { + // In production the `onStartup` hook above preloads this. In unit tests + // where startup never runs, downstream consumers may call us synchronously; + // if `serverItcHandlers` is still undefined the optional-chained access in + // the caller no-ops, matching the original lazy `require` semantics where + // a failed load was caught and logged. + return serverItcHandlers; +} export function signalSchemaChange(message: any) { try { hdbLogger.debug('signalSchemaChange called with message:', message); - serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + ensureServerItcHandlers(); const itcEventSchema = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, message); - serverItcHandlers.schema(itcEventSchema); + serverItcHandlers?.schema(itcEventSchema); return sendItcEvent(itcEventSchema); } catch (err) { hdbLogger.error(err); @@ -21,9 +39,9 @@ export function signalSchemaChange(message: any) { export function signalUserChange(message: any) { try { hdbLogger.trace('signalUserChange called with message:', message); - serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + ensureServerItcHandlers(); const itcEventUser = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.USER, message); - serverItcHandlers.user(itcEventUser); + serverItcHandlers?.user(itcEventUser); return sendItcEvent(itcEventUser); } catch (err) { hdbLogger.error(err); diff --git a/validation/configValidator.ts b/validation/configValidator.ts index 73bcab9aa..cf977b164 100644 --- a/validation/configValidator.ts +++ b/validation/configValidator.ts @@ -1,6 +1,6 @@ 'use strict'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import Joi from 'joi'; import * as os from 'os'; const { boolean, string, number, array } = Joi.types(); diff --git a/validation/fileLoadValidator.ts b/validation/fileLoadValidator.ts index b4d11e6c3..4ad056f22 100644 --- a/validation/fileLoadValidator.ts +++ b/validation/fileLoadValidator.ts @@ -7,7 +7,6 @@ import joi from 'joi'; const { string } = joi.types(); import { hdbErrors, handleHDBError } from '../utility/errors/hdbError.ts'; const { HTTP_STATUS_CODES } = hdbErrors; - import { commonValidators } from './common_validators.ts'; const isRequiredString = ' is required'; diff --git a/validation/installValidator.ts b/validation/installValidator.ts index 63b6521d6..3251c35ae 100644 --- a/validation/installValidator.ts +++ b/validation/installValidator.ts @@ -2,7 +2,7 @@ import Joi from 'joi'; const { string, number } = Joi.types(); -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as hdbTerms from '../utility/hdbTerms.ts'; import * as path from 'path'; import * as validator from './validationWrapper.ts'; diff --git a/validation/readLogValidator.ts b/validation/readLogValidator.ts index e31d6d1df..5ebd81f2a 100644 --- a/validation/readLogValidator.ts +++ b/validation/readLogValidator.ts @@ -3,9 +3,9 @@ import Joi from 'joi'; import * as validator from './validationWrapper.ts'; import moment from 'moment'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'path'; -import { getConfigPath } from '../config/configUtils.js'; +import { getConfigPath } from '../config/configUtils.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; import { LOG_LEVELS } from '../utility/hdbTerms.ts'; diff --git a/validation/role_validation.ts b/validation/role_validation.ts index a2c65b4b6..3b7b2d1d4 100644 --- a/validation/role_validation.ts +++ b/validation/role_validation.ts @@ -1,11 +1,11 @@ -const validate = require('validate.js'); -const validator = require('./validationWrapper'); +import validate from 'validate.js'; +import * as _validator from './validationWrapper.ts'; +const validator = _validator; import * as terms from '../utility/hdbTerms.ts'; import { validateOperations } from '../utility/operationPermissions.ts'; import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; - const constraintsTemplate = () => ({ role: { presence: true, diff --git a/validation/schemaMetadataValidator.ts b/validation/schemaMetadataValidator.ts index 28d6486e0..266125237 100644 --- a/validation/schemaMetadataValidator.ts +++ b/validation/schemaMetadataValidator.ts @@ -1,6 +1,8 @@ 'use strict'; -export const schemaDescribe = require('../dataLayer/schemaDescribe'); +import * as _schemaDescribe from '../dataLayer/schemaDescribe.ts'; +const schemaDescribe = _schemaDescribe; +export { schemaDescribe }; import { hdbErrors } from '../utility/errors/hdbError.ts'; import { getDatabases } from '../resources/databases.ts'; diff --git a/validation/searchValidator.ts b/validation/searchValidator.ts index 798e6cbed..07c0b9d5f 100644 --- a/validation/searchValidator.ts +++ b/validation/searchValidator.ts @@ -1,4 +1,4 @@ -import * as _ from 'lodash'; +import _ from 'lodash'; import * as validator from './validationWrapper.ts'; import Joi from 'joi'; import * as hdbUtils from '../utility/common_utils.ts'; @@ -6,7 +6,6 @@ import { hdbSchemaTable, checkValidTable, hdbTable, hdbDatabase } from './common import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; import { getDatabases } from '../resources/databases.ts'; const { HTTP_STATUS_CODES } = hdbErrors; - const searchByValueSchema = Joi.object({ database: hdbDatabase, schema: hdbDatabase, diff --git a/validation/validationWrapper.ts b/validation/validationWrapper.ts index c538435d4..b1b42f4e9 100644 --- a/validation/validationWrapper.ts +++ b/validation/validationWrapper.ts @@ -11,7 +11,8 @@ * These are rare enough for it not to be worth creating wrapper functions for those as well. */ -const validate = require('validate.js'); +import _validate from 'validate.js'; +const validate = _validate; //This validator is added here b/c we are still on version 0.11.1 that does not include this build in functionality. When // we do update, we can remove. The reason we have not is related to a breaking change on the "presence" validator rule