From 2129161fd7e1afc93130895b919cdb8eb8ceea22 Mon Sep 17 00:00:00 2001 From: Tucsky Date: Tue, 13 Jan 2026 22:55:30 +0100 Subject: [PATCH 1/6] chore: add eslint --- .prettierrc.js | 10 - eslint.config.mjs | 36 + index.js | 19 +- package-lock.json | 956 ++++++++++++++++- package.json | 8 +- scripts/coinalize/index.js | 170 --- scripts/coinalize/utils.js | 331 ------ scripts/verify-mexc-data.js | 6 +- scripts/web-push/generateVapidKeys.js | 2 +- src/config.js | 12 +- src/exchange.js | 28 +- src/exchanges/binance.js | 2 +- src/exchanges/binance_futures.js | 2 +- src/exchanges/binance_us.js | 2 +- src/exchanges/bitfinex.js | 1 - src/exchanges/bitget.js | 3 +- src/exchanges/bitmart.js | 5 +- src/exchanges/bitmex.js | 7 +- src/exchanges/bitstamp.js | 3 +- src/exchanges/bitunix.js | 2 +- src/exchanges/bybit.js | 5 +- src/exchanges/coinbase.js | 6 +- src/exchanges/cryptocom.js | 9 +- src/exchanges/deribit.js | 2 +- src/exchanges/dydx.js | 4 +- src/exchanges/hitbtc.js | 1 - src/exchanges/huobi.js | 24 +- src/exchanges/kraken.js | 1 - src/exchanges/kucoin.js | 1 - src/exchanges/mexc/index.js | 4 +- src/exchanges/noop.js | 1 - src/exchanges/okex.js | 51 +- src/exchanges/poloniex.js | 1 - src/helper.js | 91 +- src/server.js | 63 +- src/services/alert.js | 4 +- src/services/catalog.js | 2 +- src/services/connections.js | 18 +- src/services/socket.js | 12 +- src/storage/binaries.js | 1416 +++++++++++++++++++++++++ src/storage/files.js | 17 +- src/storage/influx.js | 33 +- 42 files changed, 2570 insertions(+), 801 deletions(-) delete mode 100644 .prettierrc.js create mode 100644 eslint.config.mjs delete mode 100644 scripts/coinalize/index.js delete mode 100644 scripts/coinalize/utils.js create mode 100644 src/storage/binaries.js diff --git a/.prettierrc.js b/.prettierrc.js deleted file mode 100644 index a8d0b37..0000000 --- a/.prettierrc.js +++ /dev/null @@ -1,10 +0,0 @@ -module.exports = { - semi: false, - singleQuote: true, - printWidth: 80, - tabWidth: 2, - useTabs: false, - endOfLine: 'auto', - trailingComma: 'none', - arrowParens: 'avoid' -} diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 0000000..1d989f4 --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,36 @@ +import js from "@eslint/js"; + +export default [ + { + ignores: ["node_modules/**", "dist/**", "coverage/**", "tmp/**", "eslint.config.mjs"], + }, + js.configs.recommended, + { + languageOptions: { + ecmaVersion: "latest", + sourceType: "commonjs", + globals: { + console: "readonly", + process: "readonly", + module: "readonly", + require: "readonly", + exports: "readonly", + __dirname: "readonly", + __filename: "readonly", + Buffer: "readonly", + setInterval: "readonly", + clearInterval: "readonly", + setTimeout: "readonly", + clearTimeout: "readonly", + setImmediate: "readonly", + clearImmediate: "readonly", + }, + }, + rules: { + "no-unused-vars": ["warn", { argsIgnorePattern: "^_", varsIgnorePattern: "^_", caughtErrorsIgnorePattern: "^_" }], + semi: ["error", "never"], + quotes: ["error", "single"], + indent: ["error", 2], + }, + }, +]; diff --git a/index.js b/index.js index 02d5a72..6bffb4b 100644 --- a/index.js +++ b/index.js @@ -8,7 +8,6 @@ const Server = require('./src/server') const alertService = require('./src/services/alert') const { saveConnections } = require('./src/services/connections') const socketService = require('./src/services/socket') -const FilesStorage = require('./src/storage/files') /* Load available exchanges */ @@ -53,34 +52,34 @@ process.on('SIGINT', async function () { if (alertService.enabled) { try { await alertService.persistAlerts() - console.log(`[exit] saved alerts ✓`) + console.log('[exit] saved alerts ✓') } catch (error) { - console.error(`[exit] failed to save alerts`, error.message) + console.error('[exit] failed to save alerts', error.message) } } if (config.persistConnections) { try { await saveConnections(true) - console.log(`[exit] saved connections ✓`) + console.log('[exit] saved connections ✓') } catch (error) { - console.error(`[exit] failed to save connections`, error.message) + console.error('[exit] failed to save connections', error.message) } } try { await server.backupTrades(true) - console.log(`[exit] saved trades ✓`) + console.log('[exit] saved trades ✓') } catch (error) { - console.error(`[exit] failed to save trades`, error.message) + console.error('[exit] failed to save trades', error.message) } } try { await socketService.close() - console.log(`[exit] closed sockets ✓`) + console.log('[exit] closed sockets ✓') } catch (error) { - console.error(`[exit] failed to close sockets`, error.message) + console.error('[exit] failed to close sockets', error.message) } console.log('[init] goodbye') @@ -150,7 +149,7 @@ if (process.env.pmx) { if (recoveredCount) { reply(`${recoveredCount} trades recovered`) } else { - reply(`no trade were recovered`) + reply('no trade were recovered') } } catch (error) { const message = `[${id}.recoverTrades] something went wrong while recovering ${pair}'s missing trades` diff --git a/package-lock.json b/package-lock.json index 77e57a8..a951818 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,7 +28,254 @@ "ws": "^5.0.0" }, "devDependencies": { - "prettier": "3.0.2" + "@eslint/js": "^9.18.0", + "eslint": "^9.18.0" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.1.tgz", + "integrity": "sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^2.1.7", + "debug": "^4.3.1", + "minimatch": "^3.1.2" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/config-array/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@eslint/config-array/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@eslint/config-helpers": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", + "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/core": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", + "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.3.tgz", + "integrity": "sha512-Kr+LPIUVKz2qkx1HAMH8q1q6azbqBAsXJUxBl/ODDuVPX45Z9DfwB8tPjTi6nNZ8BuM3nbJxC5zCAg5elnBUTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.1", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/eslintrc/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@eslint/eslintrc/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@eslint/js": { + "version": "9.39.2", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.2.tgz", + "integrity": "sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + } + }, + "node_modules/@eslint/object-schema": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", + "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", + "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0", + "levn": "^0.4.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", + "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.7", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", + "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.1", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" } }, "node_modules/@opencensus/core": { @@ -402,6 +649,20 @@ "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==" }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/node": { "version": "24.10.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", @@ -423,6 +684,29 @@ "node": ">= 0.6" } }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, "node_modules/agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", @@ -455,6 +739,23 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/amp": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/amp/-/amp-0.3.1.tgz", @@ -727,6 +1028,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/chalk": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", @@ -876,6 +1187,21 @@ "resolved": "https://registry.npmjs.org/croner/-/croner-4.1.97.tgz", "integrity": "sha512-/f6gpQuxDaqXu+1kwQYSckUglPaOrHdbIlBAu0YuW8/Cdb45XwXYNUBXg3r/9Mo6n540Kn/smKcZWko5x99KrQ==" }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/culvert": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/culvert/-/culvert-0.1.2.tgz", @@ -927,6 +1253,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, "node_modules/degenerator": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", @@ -1085,6 +1418,169 @@ "source-map": "~0.6.1" } }, + "node_modules/eslint": { + "version": "9.39.2", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz", + "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.21.1", + "@eslint/config-helpers": "^0.4.2", + "@eslint/core": "^0.17.0", + "@eslint/eslintrc": "^3.3.1", + "@eslint/js": "9.39.2", + "@eslint/plugin-kit": "^0.4.1", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-scope": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/eslint/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/eslint/node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/eslint/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, "node_modules/esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", @@ -1097,6 +1593,32 @@ "node": ">=4" } }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, "node_modules/estraverse": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", @@ -1230,16 +1752,50 @@ "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz", "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==" }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-json-patch": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/fast-json-patch/-/fast-json-patch-3.1.1.tgz", "integrity": "sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==" }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, "node_modules/fclone": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/fclone/-/fclone-1.0.11.tgz", "integrity": "sha512-GDqVQezKzRABdeqflsgMr7ktzgF9CyS+p2oe0jJqUY6izSSbhPIQJDpoU4PtGcD7VPM9xh/dVrTu6z1nwgmEGw==" }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -1276,6 +1832,44 @@ "ms": "2.0.0" } }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", + "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "dev": true, + "license": "ISC" + }, "node_modules/follow-redirects": { "version": "1.15.2", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", @@ -1436,6 +2030,19 @@ "node": ">= 6" } }, + "node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -1596,6 +2203,33 @@ "node": ">=0.10.0" } }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -1695,6 +2329,13 @@ "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, "node_modules/js-git": { "version": "0.7.8", "resolved": "https://registry.npmjs.org/js-git/-/js-git-0.7.8.tgz", @@ -1711,6 +2352,47 @@ "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", "integrity": "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==" }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/js-yaml/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, "node_modules/json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", @@ -1754,6 +2436,16 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, "node_modules/lazy": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/lazy/-/lazy-1.0.11.tgz", @@ -1762,11 +2454,48 @@ "node": ">=0.2.0" } }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, "node_modules/log-driver": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz", @@ -1890,6 +2619,13 @@ "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, "node_modules/needle": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/needle/-/needle-2.4.0.tgz", @@ -1997,6 +2733,56 @@ "wrappy": "1" } }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/pac-proxy-agent": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.1.tgz", @@ -2077,6 +2863,19 @@ "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -2085,6 +2884,16 @@ "node": ">= 0.8" } }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -2093,6 +2902,16 @@ "node": ">=0.10.0" } }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", @@ -2329,19 +3148,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/prettier": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.2.tgz", - "integrity": "sha512-o2YR9qtniXvwEZlOKbveKfDQVyqxbEIWn48Z8m3ZJjBjcCmUy3xZGIv+7AkaeuaTr6yPXJjwv07ZWlsWbEy1rQ==", + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", "dev": true, - "bin": { - "prettier": "bin/prettier.cjs" - }, + "license": "MIT", "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/prettier/prettier?sponsor=1" + "node": ">= 0.8.0" } }, "node_modules/promptly": { @@ -2455,6 +3269,16 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/qs": { "version": "6.11.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", @@ -2563,6 +3387,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/run-series": { "version": "1.1.9", "resolved": "https://registry.npmjs.org/run-series/-/run-series-1.1.9.tgz", @@ -2701,6 +3535,29 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/shimmer": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", @@ -2826,6 +3683,19 @@ "node": ">= 0.8" } }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -2919,6 +3789,19 @@ "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -2961,6 +3844,16 @@ "node": ">= 0.8" } }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, "node_modules/urlsafe-base64": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/urlsafe-base64/-/urlsafe-base64-1.0.0.tgz", @@ -3067,6 +3960,32 @@ "ms": "2.0.0" } }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -3111,6 +4030,19 @@ "json2yaml": "bin/json2yaml", "yaml2json": "bin/yaml2json" } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } } } } diff --git a/package.json b/package.json index c1e2496..95a14bb 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,14 @@ { "name": "aggr-server", - "version": "1.2.0", + "version": "1.2.1", "main": "index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "start": "pm2-runtime ecosystem.config.js", "dev": "pm2-dev ecosystem.config.dev.yaml", "generateVapidKeys": "node scripts/web-push/generateVapidKeys", - "lint:fix": "npx prettier index.js ./src/* --write", + "lint": "eslint .", + "lint:fix": "eslint . --fix", "clean": "rm -rf node-modules && npm clean cache --force", "docker:server:build": "docker-compose build", "docker:server:up": "docker-compose up -d" @@ -46,6 +47,7 @@ "npm": "9.8.0" }, "devDependencies": { - "prettier": "3.0.2" + "@eslint/js": "^9.18.0", + "eslint": "^9.18.0" } } diff --git a/scripts/coinalize/index.js b/scripts/coinalize/index.js deleted file mode 100644 index 96450f5..0000000 --- a/scripts/coinalize/index.js +++ /dev/null @@ -1,170 +0,0 @@ -const config = require('../../src/config') -const { prepareStandalone, getMarkets, getHms, formatAmount, sleep } = require('../../src/helper') -const { parseMarket } = require('../../src/services/catalog') -const { getReqKey, getAllData } = require('./utils') - -console.log('PID: ', process.pid) - -async function program() { - const { exchanges, storage } = await prepareStandalone(false) - - const influxMeasurement = 'trades_' + getHms(config.timeframe) - const influxRP = config.influxRetentionPrefix + getHms(config.timeframe) - - const pairs = getMarkets() - - console.log('Extend historical') - console.log('from', new Date(config.from).toISOString()) - console.log('to', new Date(config.to).toISOString()) - console.log('on', pairs.map((a) => a.market).join(', ')) - - const resampleRange = { - from: Infinity, - to: -Infinity, - markets: [], - points: 0, - } - - for (const pair of pairs) { - const exchange = exchanges.find((exchange) => exchange.id === pair.exchange) - - if (!exchange) { - continue - } - - const market = parseMarket(pair.exchange, pair.pair, pair.exchange === 'POLONIEX') - - if (!market) { - console.warn(`${pair.market}`) - continue - } - - resampleRange.markets.push(market.id) - - if ((market.exchange === 'OKEX' || market.exchange === 'HUOBI') && market.type === 'spot') { - continue - } - - const data = await getAllData(market, config.from, config.to, config.timeframe) - - const points = [] - let totalVbuy = 0 - let totalVsell = 0 - let totalCbuy = 0 - let totalCsell = 0 - let totalLbuy = 0 - let totalLsell = 0 - for (let i = data.length - 1; i >= 0; i--) { - const bar = data[i] - const hl3 = (bar.high + bar.low + bar.close) / 3 - let scaleAmount = true - if ( - pair.exchange === 'BITMEX' || - (pair.exchange === 'OKEX' && market.quote === 'USD') || - (pair.exchange === 'BINANCE_FUTURES' && market.quote === 'USD') || - (pair.exchange === 'HUOBI' && market.quote === 'USD') || - (pair.exchange === 'KRAKEN' && market.quote === 'USD' && market.type === 'perp') || - (pair.exchange === 'DERIBIT' && market.quote === 'USD') || - (pair.exchange === 'BYBIT' && market.quote === 'USD') - ) { - scaleAmount = false - } - - let empty = true - - const fields = {} - - if (bar.vsell || bar.vbuy) { - fields.vsell = bar.vsell * (scaleAmount ? hl3 : 1) - fields.vbuy = bar.vbuy * (scaleAmount ? hl3 : 1) - empty = false - } - - if (pair.exchange === 'OKEX' && market.quote === 'USDT') { - scaleAmount = true - } - - if (typeof bar.close === 'number') { - fields.open = bar.open - fields.high = bar.high - fields.low = bar.low - fields.close = bar.close - empty = false - } - - if (bar.csell || bar.cbuy) { - fields.csell = bar.csell - fields.cbuy = bar.cbuy - empty = false - } - - if (bar.lsell || bar.lbuy) { - fields.lsell = bar.lsell * (scaleAmount ? hl3 : 1) - fields.lbuy = bar.lbuy * (scaleAmount ? hl3 : 1) - empty = false - } else { - fields.lsell = fields.lbuy = 0 - } - - if (!empty) { - totalVbuy += fields.vbuy ? fields.vbuy : 0 - totalVsell += fields.vsell ? fields.vsell : 0 - totalCbuy += fields.cbuy ? fields.cbuy : 0 - totalCsell += fields.csell ? fields.csell : 0 - totalLbuy += fields.lbuy ? fields.lbuy : 0 - totalLsell += fields.lsell ? fields.lsell : 0 - - points.push({ - measurement: influxMeasurement, - tags: { - market: pair.market, - }, - fields: fields, - timestamp: +bar.time, - }) - - resampleRange.from = Math.min(resampleRange.from, bar.time) - resampleRange.to = Math.max(resampleRange.to, bar.time) - } - } - - console.log( - market.id, - points.length + 'points', - '\n-vbuy', - totalVbuy ? formatAmount(totalVbuy) : 'n/a', - '\n-vsell', - totalVsell ? formatAmount(totalVsell) : 'n/a', - '\n-lbuy', - totalLbuy ? formatAmount(totalLbuy) : 'n/a', - '\n-lsell', - totalLsell ? formatAmount(totalLsell) : 'n/a', - '\n-cbuy', - totalCbuy ? formatAmount(totalCbuy) : 'n/a', - '\n-csell', - totalCsell ? formatAmount(totalCsell) : 'n/a' + '\n\n' - ) - - console.log('-> write ' + points.length + ' points') - - if (points.length) { - await storage.writePoints(points, { - precision: 'ms', - retentionPolicy: influxRP, - }) - } - - resampleRange.points += points.length - - console.log(`[${pair.market}] end recovery procedure (${data.length} bars)`) - - await sleep(Math.random() * 3000 + 3000) - } - - if (resampleRange.points) { - await storage.resample(resampleRange, config.timeframe) - console.log(`resampled ${resampleRange.markets.length} market${resampleRange.markets.length > 1 ? 's' : ''} from timeframe ${getHms(config.timeframe)}`) - } -} - -program() diff --git a/scripts/coinalize/utils.js b/scripts/coinalize/utils.js deleted file mode 100644 index 0b8ab90..0000000 --- a/scripts/coinalize/utils.js +++ /dev/null @@ -1,331 +0,0 @@ -const axios = require('axios') -const fs = require('fs') - -const config = require('../../src/config') -const { sleep } = require('../../src/helper') - -const COINALIZE_RESOLUTIONS = [1, 5, 15, 30, 60, 60 * 2, 60 * 4, 60 * 6, 60 * 12, 60 * 24, '1D'] - -const COINALIZE_VENUES = { - BINANCE: 'A', - BITMEX: '0', - BITFINEX: 'F', - BYBIT: '6', - DERIBIT: '2', - HUOBI: '4', - KRAKEN: 'K', - OKEX: '3', - PHEMEX: '7', - DYDX: '8', - BITSTAMP: 'B', - BITTREX: 'T', - COINBASE: 'C', - GEMINI: 'G', - POLONIEX: 'P', -} - -const COINALIZE_REQ_KEY_PATH = 'products/coinalize.key' - -const BARS_PER_REQUEST = 300 - -const baseHeaders = { - "accept": "*/*", - "accept-language": "en-US,en;q=0.9", - "cache-control": "no-cache", - "content-type": "application/json", - "pragma": "no-cache", - "sec-ch-ua": "\"Not.A/Brand\";v=\"8\", \"Chromium\";v=\"114\", \"Google Chrome\";v=\"114\"", - "sec-ch-ua-mobile": "?0", - "sec-ch-ua-platform": "\"Windows\"", - "sec-fetch-dest": "empty", - "sec-fetch-mode": "cors", - "sec-fetch-site": "same-origin", - "x-requested-with": "XMLHttpRequest", - "cookie": "cookies_accepted=1; p_sid=s%3A944WOtc4HxnJXvGuR5KtvuG1yIpHEYeT.r3vU6rPl3k4tjB4xGW472GoQPOjJi%2FbfMIyW0R1bRkM", - "Referer": "https://coinalyze.net/bitcoin/usd/binance/btcusd_perp/price-chart-live/", - "Referrer-Policy": "strict-origin-when-cross-origin" -} - -const baseJSONHeaders = { - ...baseHeaders, - 'content-type': 'application/json', - 'x-requested-with': 'XMLHttpRequest', -} - -let REQ_KEY - -function getCoinalizeMarket(product) { - let symbol = product.local - - if ((product.exchange === 'FTX' && product.type === 'perp') || product.exchange === 'DERIBIT' || product.exchange === 'BYBIT') { - symbol = product.pair - } else if (product.type === 'perp') { - symbol += '_PERP' - } - - if (product.exchange === 'BINANCE' && product.type === 'spot' && product.quote === 'USDT') { - symbol = product.base + 'USD' - } - - if (product.exchange === 'BYBIT' && product.type === 'spot') { - symbol = 's' + symbol.replace('-SPOT', '') - } - - if (product.exchange === 'BITFINEX') { - if (product.local === 'LUNA2USD') { - symbol = symbol.replace('LUNA2', 'LUNA') - } else if (product.local === 'LUNAUSD') { - symbol = symbol.replace('LUNA', 'LUNC') - } - } - - return symbol.replace('LUNA', 'LUNC') + '.' + COINALIZE_VENUES[product.exchange.replace(/_\w+/, '')] -} - -function getCoinalizeResolution(time) { - let resolution - - if (time >= 604800000) { - resolution = time / 604800000 + 'W' - } else if (time >= 86400000) { - resolution = time / 86400000 + 'D' - } else { - resolution = time / 60000 // minutes - } - - if (COINALIZE_RESOLUTIONS.indexOf(resolution) === -1) { - throw new Error('unavailable resolution (' + resolution + ')') - } - - return resolution -} - -function readReqKey() { - return new Promise((resolve) => { - fs.readFile(COINALIZE_REQ_KEY_PATH, 'utf8', (err, keyStamp) => { - if (err) { - if (err.code === 'ENOENT') { - resolve([null, null]) - return - } - - throw new Error(`failed to read coinalize req key (${err.message})`) - } - - resolve(keyStamp.split('|')) - }) - }) -} - -function saveReqKey(key) { - return new Promise((resolve) => { - fs.writeFile(COINALIZE_REQ_KEY_PATH, key + '|' + Date.now(), (err) => { - if (err) { - throw new Error(`failed to save coinalize req key (${err.message})`) - } - - resolve() - }) - }) -} - -async function getReqKey() { - let [key, timestamp] = await readReqKey() - - let renew = config.renew || !key - - if (!renew && Date.now() - timestamp > 1000 * 60 * 2) { - renew = true - } - - if (renew || !key) { - key = await fetchReqKey() - - await saveReqKey(key) - } - - REQ_KEY = key - - return key -} - -function fetchReqKey() { - return axios - .get('https://coinalyze.net/bitcoin/usd/binance/btcusd_perp/price-chart-live/', { - headers: baseHeaders, - }) - .then((response) => response.data) - .then((body) => { - const match = body.match(/window.REQ_KEY\s*=\s*'([a-z0-9-]+)';/) - - if (match && match[1]) { - return match[1] - } else { - throw new Error('invalid req key') - } - }) -} - -module.exports.getAllData = async function (product, from, to, timeframe) { - const coinalizeMarket = getCoinalizeMarket(product) - - const coinalizeResolution = getCoinalizeResolution(timeframe) - console.log('[getAllData]', product.id, coinalizeMarket, coinalizeResolution, 'from', new Date(from).toISOString(), 'to', new Date(to).toISOString()) - const timePerRequest = timeframe * BARS_PER_REQUEST - - const data = [] - - let isFirst = true - - for (let time = to; time >= from; time -= timePerRequest) { - await sleep(Math.random() * 1500 + 1500) - const reqFrom = Math.max(from, time - timePerRequest) - const reqTo = Math.min(to, time) - console.log('- from', new Date(reqFrom).toISOString(), 'to', new Date(reqTo).toISOString(), coinalizeMarket, coinalizeResolution) - const chunk = await getData(product, reqFrom, reqTo, timeframe, isFirst) - - if (chunk.length) { - Array.prototype.push.apply(data, chunk) - } else { - break - } - - isFirst = false - } - - return data -} - -function getBars(product, from, to, timeframe, isFirst, dataset) { - const coinalizeMarket = getCoinalizeMarket(product) - const coinalizeResolution = getCoinalizeResolution(timeframe) - - let source = [coinalizeMarket] - - if (dataset === 'liquidations') { - const [pair, exchangeId] = coinalizeMarket.split('.') - source.push(pair + '_LQB.' + exchangeId) - source.push(pair + '_LQS.' + exchangeId) - } - - source = source.join(',') + (dataset ? '#' + dataset : '') - - const flooredFrom = Math.floor(from / timeframe) * timeframe - const flooredTo = Math.floor(to / timeframe) * timeframe - - // console.log('[FETCH]', `${source} (${new Date(flooredFrom).toISOString()} -> ${new Date(flooredTo).toISOString()})`) - - const data = `{"from":${flooredFrom / 1000},"to":${ - flooredTo / 1000 - },"resolution":"${coinalizeResolution}","symbol":"${source}","firstDataRequest":${ - isFirst ? 'true' : 'false' - },"symbolsForUsdConversion":[],"rk":"${REQ_KEY}"}` - - return axios - .post('https://coinalyze.net/chart/getBars/', data, { - headers: baseJSONHeaders, - }) - .then((response) => { - return response.data - }) - .catch((err) => { - throw err - }) -} - -function getOHLC(product, from, to, timeframe, isFirst) { - return getBars(product, from, to, timeframe, isFirst).then(({ barData }) => { - return barData.reduce((output, dataPoint) => { - output[dataPoint[0] * 1000] = { - open: dataPoint[1], - high: dataPoint[2], - low: dataPoint[3], - close: dataPoint[4], - } - - return output - }, {}) - }) -} - -function getBuySellVolume(product, from, to, timeframe, isFirst) { - return getBars(product, from, to, timeframe, isFirst, 'buy_sell_volume').then(({ barData }) => { - return barData.reduce((output, dataPoint) => { - output[dataPoint[0] * 1000] = { - vbuy: dataPoint[1], - vsell: dataPoint[2], - } - - return output - }, {}) - }) -} - -function getBuySellCount(product, from, to, timeframe, isFirst) { - return getBars(product, from, to, timeframe, isFirst, 'buy_sell_count').then(({ barData }) => { - return barData.reduce((output, dataPoint) => { - output[dataPoint[0] * 1000] = { - cbuy: dataPoint[1], - csell: dataPoint[2], - } - - return output - }, {}) - }) -} - -function getBuySellLiquidations(product, from, to, timeframe, isFirst) { - return getBars(product, from, to, timeframe, isFirst, 'liquidations').then(({ barData }) => { - return Object.keys(barData).reduce((output, timestamp) => { - output[timestamp * 1000] = barData[timestamp].reduce( - (dataPoint, keyValue) => { - if (/_LQB\./.test(keyValue[0])) { - dataPoint.lbuy = keyValue[1] - } else if (/_LQS\./.test(keyValue[0])) { - dataPoint.lsell = keyValue[1] - } - - return dataPoint - }, - { - lbuy: 0, - lsell: 0, - } - ) - - return output - }, {}) - }) -} - -async function getData(product, from, to, timeframe, isFirst) { - if (!REQ_KEY) { - await getReqKey() - } - - const bars = [] - - const ohlcData = await getOHLC(product, from, to, timeframe, isFirst) - await sleep(Math.random() * 100 + 100) - const buySellVolumeData = await getBuySellVolume(product, from, to, timeframe, isFirst) - await sleep(Math.random() * 100 + 100) - const buySellCountData = await getBuySellCount(product, from, to, timeframe, isFirst) - await sleep(Math.random() * 100 + 100) - const buySellLiquidationsData = await getBuySellLiquidations(product, from, to, timeframe, isFirst) - await sleep(Math.random() * 100 + 100) - - const flooredFrom = Math.floor(from / timeframe) * timeframe - const flooredTo = Math.floor(to / timeframe) * timeframe - - for (let time = flooredTo; time >= flooredFrom; time -= timeframe) { - bars.push({ - time, - ...ohlcData[time], - ...buySellVolumeData[time], - ...buySellCountData[time], - ...buySellLiquidationsData[time], - }) - } - - return bars -} diff --git a/scripts/verify-mexc-data.js b/scripts/verify-mexc-data.js index 33661ca..f4f068f 100644 --- a/scripts/verify-mexc-data.js +++ b/scripts/verify-mexc-data.js @@ -239,7 +239,7 @@ function findCompleteCandle(trades, intervalMinutes) { /** * Calculate statistics from API klines */ -function calculateKlineStats(klines, isFutures, symbol) { +function calculateKlineStats(klines, isFutures, _symbol) { let totalVolume = 0 let high = -Infinity let low = Infinity @@ -343,10 +343,10 @@ async function verify() { continue } - console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`) + console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━') console.log(`🕐 ${interval.name.toUpperCase()} CANDLE VERIFICATION`) console.log(` ${new Date(candle.start).toISOString()} → ${new Date(candle.end).toISOString()}`) - console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n`) + console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n') // Calculate local statistics for this candle const localStats = calculateTradeStats(trades, candle.start, candle.end) diff --git a/scripts/web-push/generateVapidKeys.js b/scripts/web-push/generateVapidKeys.js index 8557433..9e1c169 100644 --- a/scripts/web-push/generateVapidKeys.js +++ b/scripts/web-push/generateVapidKeys.js @@ -1,3 +1,3 @@ -const webpush = require('web-push'); +const webpush = require('web-push') const vapidKeys = webpush.generateVAPIDKeys() console.log(`"privateVapidKey": "${vapidKeys.privateKey}",\n"publicVapidKey": "${vapidKeys.publicKey}"`) \ No newline at end of file diff --git a/src/config.js b/src/config.js index c2394f0..f0eb0f5 100644 --- a/src/config.js +++ b/src/config.js @@ -6,7 +6,7 @@ require('dotenv').config({ }) -console.log(`[init] reading config.json...`) +console.log('[init] reading config.json...') /* Default configuration (its not ok to change here!, use config.json.) */ @@ -261,8 +261,8 @@ if (process.env.RUNNING_IN_DOCKER) { */ Object.keys(config).forEach(k => { - config_to_env_key = decamelize(k, '_').toUpperCase() - config_env_value = process.env[config_to_env_key] + const config_to_env_key = decamelize(k, '_').toUpperCase() + const config_env_value = process.env[config_to_env_key] if (config_env_value) { config[k] = config_env_value console.log( @@ -374,17 +374,17 @@ if (config.exchanges && typeof config.exchanges === 'string') { if (!config.storage && config.collect) { console.warn( - `[warning!] server will not persist any of the data it is receiving` + '[warning!] server will not persist any of the data it is receiving' ) } if (!config.collect && !config.api) { - console.warn(`[warning!] server has no purpose`) + console.warn('[warning!] server has no purpose') } if (!config.storage && !config.collect && config.api) { console.warn( - `[warning!] api is enabled but neither storage or collect is enabled (may be useless)` + '[warning!] api is enabled but neither storage or collect is enabled (may be useless)' ) } diff --git a/src/exchange.js b/src/exchange.js index 5e00d24..fa0f82c 100644 --- a/src/exchange.js +++ b/src/exchange.js @@ -228,7 +228,7 @@ class Exchange extends EventEmitter { return api } - createWs(url, pair) { + createWs(url, _pair) { const api = new WebSocket(url) api.id = ID() @@ -424,7 +424,7 @@ class Exchange extends EventEmitter { if (api.readyState !== WebSocket.CLOSED) { if (api._connected.length) { - throw new Error(`cannot unbind api that still has pairs linked to it`) + throw new Error('cannot unbind api that still has pairs linked to it') } console.debug(`[${this.id}.removeWs] close api ${api.id}`) @@ -697,7 +697,7 @@ class Exchange extends EventEmitter { async getProductsAndConnect(pairs, forceRefreshProducts = false) { try { await this.getProducts(forceRefreshProducts) - } catch (error) { + } catch (_error) { this.scheduledOperationsDelays.getProducts = this.schedule( () => { this.getProductsAndConnect(pairs, forceRefreshProducts) @@ -762,7 +762,7 @@ class Exchange extends EventEmitter { if (formatedProducts) { if ( typeof formatedProducts === 'object' && - formatedProducts.hasOwnProperty('products') + Object.prototype.hasOwnProperty.call(formatedProducts, 'products') ) { for (let key in formatedProducts) { this[key] = formatedProducts[key] @@ -783,7 +783,7 @@ class Exchange extends EventEmitter { * Fire when a new websocket connection is created * @param {WebSocket} api WebSocket instance */ - onApiCreated(api) { + onApiCreated(_api) { // should be overrided by exchange class } @@ -791,7 +791,7 @@ class Exchange extends EventEmitter { * Fire when a new websocket connection has been removed * @param {WebSocket} api WebSocket instance */ - onApiRemoved(api) { + onApiRemoved(_api) { // should be overrided by exchange class } @@ -800,7 +800,7 @@ class Exchange extends EventEmitter { * @param {Event} event * @param {WebSocket} api WebSocket instance */ - onMessage(event, api) { + onMessage(_event, _api) { // should be overrided by exchange class } @@ -966,10 +966,10 @@ class Exchange extends EventEmitter { this.keepAliveIntervals[api.id] = setInterval(() => { if (api.readyState === WebSocket.OPEN) { const message = typeof payload === 'function' - ? JSON.stringify(payload()) - : typeof payload === 'string' - ? payload - : JSON.stringify(payload) + ? JSON.stringify(payload()) + : typeof payload === 'string' + ? payload + : JSON.stringify(payload) api.send( message @@ -1085,12 +1085,12 @@ class Exchange extends EventEmitter { * @returns {Promise} */ waitBeforeContinueRecovery(multiplier = 1) { - const delay = config.recoveryRequestDelay * multiplier; + const delay = config.recoveryRequestDelay * multiplier if (!delay) { - return Promise.resolve(); + return Promise.resolve() } - return sleep(delay); + return sleep(delay) } } diff --git a/src/exchanges/binance.js b/src/exchanges/binance.js index 0305afb..e4c8c9a 100644 --- a/src/exchanges/binance.js +++ b/src/exchanges/binance.js @@ -15,7 +15,7 @@ class Binance extends Exchange { PRODUCTS: 'https://data-api.binance.vision/api/v3/exchangeInfo' } - this.url = () => `wss://data-stream.binance.vision:9443/ws` + this.url = () => 'wss://data-stream.binance.vision:9443/ws' } formatProducts(data) { diff --git a/src/exchanges/binance_futures.js b/src/exchanges/binance_futures.js index 97b80e9..92fab51 100644 --- a/src/exchanges/binance_futures.js +++ b/src/exchanges/binance_futures.js @@ -183,7 +183,7 @@ class BinanceFutures extends Exchange { getMissingTrades(range, totalRecovered = 0) { const startTime = range.from let endpoint = `?symbol=${range.pair.toUpperCase()}&startTime=${startTime + 1 - }&endTime=${range.to}&limit=1000` + }&endTime=${range.to}&limit=1000` if (this.dapi[range.pair]) { endpoint = 'https://dapi.binance.com/dapi/v1/aggTrades' + endpoint } else { diff --git a/src/exchanges/binance_us.js b/src/exchanges/binance_us.js index 5c1422e..5c94b6b 100644 --- a/src/exchanges/binance_us.js +++ b/src/exchanges/binance_us.js @@ -13,7 +13,7 @@ class BinanceUs extends Exchange { PRODUCTS: 'https://api.binance.us/api/v3/exchangeInfo' } - this.url = () => `wss://stream.binance.us:9443/ws` + this.url = () => 'wss://stream.binance.us:9443/ws' } formatProducts(data) { diff --git a/src/exchanges/bitfinex.js b/src/exchanges/bitfinex.js index d404cd1..a441828 100644 --- a/src/exchanges/bitfinex.js +++ b/src/exchanges/bitfinex.js @@ -1,6 +1,5 @@ const Exchange = require('../exchange') const axios = require('axios') -const fs = require('fs') const { getHms } = require('../helper') class Bitfinex extends Exchange { diff --git a/src/exchanges/bitget.js b/src/exchanges/bitget.js index 59b0c9d..f24becd 100644 --- a/src/exchanges/bitget.js +++ b/src/exchanges/bitget.js @@ -1,6 +1,5 @@ const Exchange = require('../exchange') const { sleep } = require('../helper') -const WebSocket = require('websocket').w3cwebsocket const SPOT_PAIR_REGEX = /-SPOT$/ @@ -20,7 +19,7 @@ class Bitget extends Exchange { ] } - this.url = pair => { + this.url = _pair => { // V2 API uses a single WebSocket URL for both spot and futures return 'wss://ws.bitget.com/v2/ws/public' } diff --git a/src/exchanges/bitmart.js b/src/exchanges/bitmart.js index 9ceffea..0404013 100644 --- a/src/exchanges/bitmart.js +++ b/src/exchanges/bitmart.js @@ -1,5 +1,4 @@ const Exchange = require('../exchange') -const { sleep, getHms } = require('../helper') const axios = require('axios') const { inflateRaw } = require('pako') @@ -110,7 +109,7 @@ class Bitmart extends Exchange { api.send( JSON.stringify({ - [typeImpl.arg]: `subscribe`, + [typeImpl.arg]: 'subscribe', args: [`${typeImpl.prefix}/trade:${pair}`] }) ) @@ -142,7 +141,7 @@ class Bitmart extends Exchange { api.send( JSON.stringify({ - [typeImpl.arg]: `unsubscribe`, + [typeImpl.arg]: 'unsubscribe', args: [`${typeImpl.prefix}/trade:${pair}`] }) ) diff --git a/src/exchanges/bitmex.js b/src/exchanges/bitmex.js index 5c6dc01..2ed28cc 100644 --- a/src/exchanges/bitmex.js +++ b/src/exchanges/bitmex.js @@ -1,5 +1,4 @@ const Exchange = require('../exchange') -const WebSocket = require('websocket').w3cwebsocket const axios = require('axios') const { getHms } = require('../helper') @@ -19,7 +18,7 @@ class Bitmex extends Exchange { } this.url = () => { - return `wss://www.bitmex.com/realtime` + return 'wss://www.bitmex.com/realtime' } } @@ -33,8 +32,8 @@ class Bitmex extends Exchange { types[product.symbol] = product.isInverse ? 'inverse' : product.isQuanto - ? 'quanto' - : 'linear' + ? 'quanto' + : 'linear' multipliers[product.symbol] = product.multiplier if (types[product.symbol] === 'linear') { diff --git a/src/exchanges/bitstamp.js b/src/exchanges/bitstamp.js index 79ed558..18ebae9 100644 --- a/src/exchanges/bitstamp.js +++ b/src/exchanges/bitstamp.js @@ -1,5 +1,4 @@ const Exchange = require('../exchange') -const WebSocket = require('websocket').w3cwebsocket class Bitstamp extends Exchange { constructor() { @@ -12,7 +11,7 @@ class Bitstamp extends Exchange { } this.url = () => { - return `wss://ws.bitstamp.net` + return 'wss://ws.bitstamp.net' } } diff --git a/src/exchanges/bitunix.js b/src/exchanges/bitunix.js index 337a835..958ee91 100644 --- a/src/exchanges/bitunix.js +++ b/src/exchanges/bitunix.js @@ -1,4 +1,4 @@ -const Exchange = require("../exchange") +const Exchange = require('../exchange') class Bitunix extends Exchange { constructor() { diff --git a/src/exchanges/bybit.js b/src/exchanges/bybit.js index cd311df..5b5beb7 100644 --- a/src/exchanges/bybit.js +++ b/src/exchanges/bybit.js @@ -1,5 +1,4 @@ const Exchange = require('../exchange') -const WebSocket = require('websocket').w3cwebsocket const axios = require('axios') const { getHms } = require('../helper') @@ -79,7 +78,7 @@ class Bybit extends Exchange { api.send( JSON.stringify({ - "op": "subscribe", + 'op': 'subscribe', args: topics }) ) @@ -107,7 +106,7 @@ class Bybit extends Exchange { api.send( JSON.stringify({ - "op": "unsubscribe", + 'op': 'unsubscribe', args: topics }) ) diff --git a/src/exchanges/coinbase.js b/src/exchanges/coinbase.js index 8da0323..6149999 100644 --- a/src/exchanges/coinbase.js +++ b/src/exchanges/coinbase.js @@ -162,9 +162,9 @@ class Coinbase extends Exchange { let endpoint if (isIntx || !range.earliestTradeId) { endpoint = `https://api.coinbase.com/api/v3/brokerage/market/products/${range.pair - }/ticker?limit=100&end=${Math.round(range.to / 1000)}&start=${Math.round( - range.from / 1000 - )}` + }/ticker?limit=100&end=${Math.round(range.to / 1000)}&start=${Math.round( + range.from / 1000 + )}` // If close to current time, wait to allow trades to accumulate if (+new Date() - range.to < 10000) { diff --git a/src/exchanges/cryptocom.js b/src/exchanges/cryptocom.js index 2524bf7..0bdd2f8 100644 --- a/src/exchanges/cryptocom.js +++ b/src/exchanges/cryptocom.js @@ -1,5 +1,4 @@ const Exchange = require('../exchange') -const WebSocket = require('websocket').w3cwebsocket const axios = require('axios') const { getHms } = require('../helper') @@ -15,7 +14,7 @@ class CryptoCom extends Exchange { } async getUrl() { - return `wss://stream.crypto.com/v2/market` + return 'wss://stream.crypto.com/v2/market' } formatProducts(response) { @@ -109,14 +108,14 @@ class CryptoCom extends Exchange { } } - async getMissingTrades(range, totalRecovered = 0, fromTradeId) { + async getMissingTrades(range, totalRecovered = 0, _fromTradeId) { // https://exchange-docs.crypto.com/exchange/v1/rest-ws/index.html#public-get-trades // Retuns maximum 150 trades per request const endpoint = - `https://api.crypto.com/exchange/v1/public/get-trades?` + + 'https://api.crypto.com/exchange/v1/public/get-trades?' + `instrument_name=${range.pair}&` + - `count=1000&` + + 'count=1000&' + `start_ts=${range.from}&` + `end_ts=${range.to}` diff --git a/src/exchanges/deribit.js b/src/exchanges/deribit.js index f04becc..c3dc39e 100644 --- a/src/exchanges/deribit.js +++ b/src/exchanges/deribit.js @@ -15,7 +15,7 @@ class Deribit extends Exchange { ] } - this.url = `wss://www.deribit.com/ws/api/v2` + this.url = 'wss://www.deribit.com/ws/api/v2' } formatProducts(response) { diff --git a/src/exchanges/dydx.js b/src/exchanges/dydx.js index 7b60dd2..a156440 100644 --- a/src/exchanges/dydx.js +++ b/src/exchanges/dydx.js @@ -1,6 +1,4 @@ const Exchange = require('../exchange') -const { sleep } = require('../helper') -const WebSocket = require('websocket').w3cwebsocket class Dydx extends Exchange { constructor() { @@ -13,7 +11,7 @@ class Dydx extends Exchange { } this.url = () => { - return `wss://indexer.dydx.trade/v4/ws` + return 'wss://indexer.dydx.trade/v4/ws' } } diff --git a/src/exchanges/hitbtc.js b/src/exchanges/hitbtc.js index 372fda8..df6cd96 100644 --- a/src/exchanges/hitbtc.js +++ b/src/exchanges/hitbtc.js @@ -1,5 +1,4 @@ const Exchange = require('../exchange') -const WebSocket = require('websocket').w3cwebsocket class HitBtc extends Exchange { constructor() { diff --git a/src/exchanges/huobi.js b/src/exchanges/huobi.js index 7e70c96..3d2e34c 100644 --- a/src/exchanges/huobi.js +++ b/src/exchanges/huobi.js @@ -56,21 +56,21 @@ class Huobi extends Exchange { let pair switch (type) { - case 'spot': - pair = product.symbol - break - case 'futures': - pair = + case 'spot': + pair = product.symbol + break + case 'futures': + pair = product.symbol + '_' + this.contractTypesAliases[product.contract_type] - specs[pair] = product.contract_size - break - case 'swap': - case 'linear': - pair = product.contract_code - specs[pair] = product.contract_size - break + specs[pair] = product.contract_size + break + case 'swap': + case 'linear': + pair = product.contract_code + specs[pair] = product.contract_size + break } types[pair] = type diff --git a/src/exchanges/kraken.js b/src/exchanges/kraken.js index 6c92c1c..51d9569 100644 --- a/src/exchanges/kraken.js +++ b/src/exchanges/kraken.js @@ -1,5 +1,4 @@ const Exchange = require('../exchange') -const WebSocket = require('websocket').w3cwebsocket const axios = require('axios') const { getHms } = require('../helper') diff --git a/src/exchanges/kucoin.js b/src/exchanges/kucoin.js index 6675f95..a17a7d2 100644 --- a/src/exchanges/kucoin.js +++ b/src/exchanges/kucoin.js @@ -1,5 +1,4 @@ const Exchange = require('../exchange') -const WebSocket = require('websocket').w3cwebsocket const axios = require('axios') const { sleep } = require('../helper') diff --git a/src/exchanges/mexc/index.js b/src/exchanges/mexc/index.js index dd73f67..6ebb58e 100644 --- a/src/exchanges/mexc/index.js +++ b/src/exchanges/mexc/index.js @@ -169,7 +169,7 @@ class Mexc extends Exchange { if (spotResult) { return true } - } catch (e) { + } catch (_e) { // Skip non-JSON messages or parse errors } } @@ -200,7 +200,7 @@ class Mexc extends Exchange { if (trades.length > 0) { return this.emitTrades(api.id, trades) } - } catch (e) { + } catch (_e) { // Silent fail for protobuf decode errors return true } diff --git a/src/exchanges/noop.js b/src/exchanges/noop.js index 5f8fb0f..65899ab 100644 --- a/src/exchanges/noop.js +++ b/src/exchanges/noop.js @@ -1,5 +1,4 @@ const Exchange = require('../exchange') -const WebSocket = require('websocket').w3cwebsocket class Noop extends Exchange { constructor(options) { diff --git a/src/exchanges/okex.js b/src/exchanges/okex.js index f421bb6..b680d10 100644 --- a/src/exchanges/okex.js +++ b/src/exchanges/okex.js @@ -1,5 +1,4 @@ const Exchange = require('../exchange') -const WebSocket = require('websocket').w3cwebsocket const axios = require('axios') const { getHms } = require('../helper') @@ -213,8 +212,8 @@ class Okex extends Exchange { // after query param = before // (get the 100 trades preceding endTimestamp) return `${this.endpoints.LIQUIDATIONS}?instId=${range.pair - }&instType=SWAP&uly=${range.pair.replace('-SWAP', '')}&state=filled&after=${range.to - }` + }&instType=SWAP&uly=${range.pair.replace('-SWAP', '')}&state=filled&after=${range.to + }` } /** @@ -261,10 +260,10 @@ class Okex extends Exchange { async getMissingTrades(range, totalRecovered = 0, first = true) { if (this.types[range.pair] !== 'SPOT' && first) { try { - const liquidations = await this.fetchAllLiquidationOrders({ ...range }); + const liquidations = await this.fetchAllLiquidationOrders({ ...range }) console.log( `[${this.id}.recoverMissingTrades] +${liquidations.length} liquidations for ${range.pair}` - ); + ) if (liquidations.length) { this.emitLiquidations( @@ -272,17 +271,17 @@ class Okex extends Exchange { liquidations.map(liquidation => this.formatLiquidation(liquidation, range.pair) ) - ); + ) } } catch (error) { console.error( `[${this.id}] failed to get missing liquidations on ${range.pair}:`, error.message - ); + ) } } - const endpoint = `https://www.okx.com/api/v5/market/history-trades?instId=${range.pair}&type=2&limit=100&after=${range.to}`; + const endpoint = `https://www.okx.com/api/v5/market/history-trades?instId=${range.pair}&type=2&limit=100&after=${range.to}` try { const response = await this.retryWithDelay( @@ -290,7 +289,7 @@ class Okex extends Exchange { 5, // Retry up to 5 times 1, // Start with a multiplier of 1 range - ); + ) if (response.data.data.length) { const trades = response.data.data @@ -299,39 +298,39 @@ class Okex extends Exchange { Number(trade.ts) > range.from && Number(trade.ts) < range.to ) - .map(trade => this.formatTrade(trade)); + .map(trade => this.formatTrade(trade)) if (trades.length) { - this.emitTrades(null, trades); - totalRecovered += trades.length; - range.to = trades[trades.length - 1].timestamp; + this.emitTrades(null, trades) + totalRecovered += trades.length + range.to = trades[trades.length - 1].timestamp } - const remainingMissingTime = range.to - range.from; + const remainingMissingTime = range.to - range.from if (trades.length) { console.log( `[${this.id}.recoverMissingTrades] +${trades.length} ${range.pair } ... but there's more (${getHms(remainingMissingTime)} remaining)` - ); + ) return this.waitBeforeContinueRecovery().then(() => this.getMissingTrades(range, totalRecovered, false) - ); + ) } else { console.log( `[${this.id}.recoverMissingTrades] +${trades.length} ${range.pair } (${getHms(remainingMissingTime)} remaining)` - ); + ) } } - return totalRecovered; + return totalRecovered } catch (err) { console.error( `[${this.id}] failed to get missing trades on ${range.pair} after retries:`, err.message - ); - return totalRecovered; + ) + return totalRecovered } } @@ -345,22 +344,22 @@ class Okex extends Exchange { */ async retryWithDelay(fn, retries, multiplier = 1, range = {}) { try { - return await fn(); + return await fn() } catch (err) { if (retries > 0) { console.warn( `[${this.id}] Retrying with delay (${range.pair || 'unknown pair'}) attempt ${multiplier }...` - ); - await this.waitBeforeContinueRecovery(multiplier); - return this.retryWithDelay(fn, retries - 1, multiplier + 1, range); + ) + await this.waitBeforeContinueRecovery(multiplier) + return this.retryWithDelay(fn, retries - 1, multiplier + 1, range) } console.error( `[${this.id}] Exceeded retry limit for ${range.pair || 'unknown pair' }:`, err.message - ); - throw err; // Propagate the error after exceeding retries + ) + throw err // Propagate the error after exceeding retries } } } diff --git a/src/exchanges/poloniex.js b/src/exchanges/poloniex.js index d4918fe..dd4613c 100644 --- a/src/exchanges/poloniex.js +++ b/src/exchanges/poloniex.js @@ -1,5 +1,4 @@ const Exchange = require('../exchange') -const WebSocket = require('websocket').w3cwebsocket class Poloniex extends Exchange { constructor() { diff --git a/src/helper.js b/src/helper.js index e865d9a..8ef64c6 100644 --- a/src/helper.js +++ b/src/helper.js @@ -1,11 +1,9 @@ const fs = require('fs') const config = require('../src/config') -const { Request } = require('express') - module.exports = { /** - * @param {Request} req + * @param {import('express').Request} req * @returns {string} */ getIp(req) { @@ -191,6 +189,7 @@ module.exports = { }, parseDuration(duration) { duration = duration.toString().trim() + let output if (/d$/i.test(duration)) { output = parseFloat(duration) * 60 * 60 * 24 @@ -231,92 +230,6 @@ module.exports = { } }) }, - async prepareStandalone(onlyNativeRecovery = true) { - if (!config.exchanges || !config.exchanges.length) { - config.exchanges = [] - - fs.readdirSync('./src/exchanges/').forEach(file => { - ;/\.js$/.test(file) && config.exchanges.push(file.replace(/\.js$/, '')) - }) - } - - const exchanges = [] - - for (let i = 0; i < config.exchanges.length; i++) { - const name = config.exchanges[i] - const exchange = new (require('../src/exchanges/' + name))(config) - - if ( - !onlyNativeRecovery || - typeof exchange.getMissingTrades === 'function' - ) { - config.exchanges[i] = exchange.id - - exchanges.push(exchange) - } else { - config.exchanges.splice(i, 1) - i-- - } - } - - if (config.from) { - config.from = module.exports.parseDatetime(config.from) - } else { - throw new Error('from is required') - } - - if (config.to) { - config.to = module.exports.parseDatetime(config.to) - 1 - } else { - config.to = +new Date() - } - - if (isNaN(config.to) || isNaN(config.to)) { - throw new Error('invalid from / to') - } - - if (!config.timeframe) { - throw new Error( - 'you must choose a timeframe / resolution (ex timeframe=1m)' - ) - } - - config.timeframe = module.exports.parseDuration(config.timeframe) - - if (isNaN(config.timeframe)) { - throw new Error('invalid timeframe') - } - - if (onlyNativeRecovery) { - for (const exchange of exchanges) { - await exchange.getProducts() - } - } - - let storage - - for (let name of config.storage) { - if (name !== 'influx') { - continue - } - - console.log(`[storage] using "${name}" storage solution`) - - storage = new (require(`../src/storage/${name}`))(config) - - if (typeof storage.connect === 'function') { - await storage.connect() - } - - break - } - - if (!storage) { - throw new Error('this utility script requires influx storage') - } - - return { exchanges, storage } - }, humanFileSize(size) { var i = Math.floor(Math.log(size) / Math.log(1024)) return ( diff --git a/src/server.js b/src/server.js index 93d6eb6..138e247 100644 --- a/src/server.js +++ b/src/server.js @@ -3,8 +3,6 @@ const fs = require('fs') const { getIp, getHms, - parsePairsFromWsRequest, - groupTrades, formatAmount } = require('./helper') const express = require('express') @@ -46,7 +44,7 @@ class Server extends EventEmitter { this.BANNED_IPS = [] if (config.collect) { - console.log(`\n[server] collect is enabled`) + console.log('\n[server] collect is enabled') console.log(`\tconnect to -> ${this.exchanges.map(a => a.id).join(', ')}`) this.handleExchangesEvents() @@ -128,7 +126,7 @@ class Server extends EventEmitter { this.storages.push(storage) } - console.log(`[storage] all storage ready`) + console.log('[storage] all storage ready') return Promise.all(promises) } @@ -174,7 +172,7 @@ class Server extends EventEmitter { }) .catch(err => { console.error( - `[server] something went wrong while backuping trades...`, + '[server] something went wrong while backuping trades...', err ) }) @@ -255,8 +253,7 @@ class Server extends EventEmitter { if (pairs.length) { console.error( - `[${exchange.id}] api closed unexpectedly (${apiId}, ${ - event.code + `[${exchange.id}] api closed unexpectedly (${apiId}, ${event.code }, ${reason}) (was handling ${pairs.join(',')})` ) @@ -402,7 +399,13 @@ class Server extends EventEmitter { }) } - const storage = this.storages[config.storage.indexOf('influx')] + const storage = this.storages.find(storage => storage.format === 'point') + + if (!storage) { + return res.status(501).json({ + error: 'no compatible storage' + }) + } if (isNaN(from) || isNaN(to)) { return res.status(400).json({ @@ -410,16 +413,14 @@ class Server extends EventEmitter { }) } - if (storage.format === 'point') { - timeframe = parseInt(timeframe) || 1000 * 60 // default to 1m + timeframe = parseInt(timeframe) || 1000 * 60 // default to 1m - length = (to - from) / timeframe + length = (to - from) / timeframe - if (length > config.maxFetchLength) { - return res.status(400).json({ - error: 'too many bars' - }) - } + if (length > config.maxFetchLength) { + return res.status(400).json({ + error: 'too many bars' + }) } if (from > to) { @@ -431,13 +432,13 @@ class Server extends EventEmitter { this.globalUsage.tick += length * markets.length const fetchStartAt = Date.now() - ;(storage + ; (storage ? storage.fetch({ - from, - to, - timeframe, - markets - }) + from, + to, + timeframe, + markets + }) : Promise.resolve([]) ) .then(output => { @@ -449,10 +450,8 @@ class Server extends EventEmitter { if (output.results.length > 10000) { console.log( - `[${user}/${req.get('origin')}] ${getHms(to - from)} (${ - markets.length - } markets, ${getHms(timeframe, true)} tf) -> ${ - +length ? parseInt(length) + ' bars into ' : '' + `[${user}/${req.get('origin')}] ${getHms(to - from)} (${markets.length + } markets, ${getHms(timeframe, true)} tf) -> ${+length ? parseInt(length) + ' bars into ' : '' }${output.results.length} ${storage.format}s, took ${getHms( Date.now() - fetchStartAt )}` @@ -469,7 +468,7 @@ class Server extends EventEmitter { } ) - app.use(function (err, req, res, next) { + app.use(function (err, req, res, _next) { if (err) { console.error(err) @@ -652,7 +651,7 @@ class Server extends EventEmitter { if (!config.configPath) { console.warn( - `[server] couldn't save config because configPath isn't known` + '[server] couldn\'t save config because configPath isn\'t known' ) return Promise.resolve() } @@ -723,7 +722,7 @@ class Server extends EventEmitter { watch() } }) - } catch (error) { + } catch (_error) { const _checkForWatchInterval = setInterval(() => { fs.access(file, fs.constants.F_OK, err => { if (err) { @@ -817,7 +816,7 @@ class Server extends EventEmitter { if (apisToReconnect.length) { dumpConnections(staleConnections) - this.reconnectApis(apisToReconnect, `reconnection threshold reached`) + this.reconnectApis(apisToReconnect, 'reconnection threshold reached') } } @@ -827,7 +826,7 @@ class Server extends EventEmitter { for (const exchange of this.exchanges) { if (recovering[exchange.id]) { console.error( - `Exchange is recovering trades, don't quit while it's doing its thing because all will be lost` + 'Exchange is recovering trades, don\'t quit while it\'s doing its thing because all will be lost' ) output = false break @@ -840,7 +839,7 @@ class Server extends EventEmitter { } this.exitAttempts++ if (this.exitAttempts === 3) { - console.error(`[server] last warning.`) + console.error('[server] last warning.') } else if (this.exitAttempts === 4) { output = true } diff --git a/src/services/alert.js b/src/services/alert.js index 628cba4..adf882d 100644 --- a/src/services/alert.js +++ b/src/services/alert.js @@ -517,7 +517,7 @@ class AlertService extends EventEmitter { queueAlert(alert, market, timestamp, direction) { if (!this.alertEndpoints[alert.endpoint]) { console.error( - `[alert/send] attempted to send alert without matching endpoint`, + '[alert/send] attempted to send alert without matching endpoint', alert ) return @@ -584,7 +584,7 @@ class AlertService extends EventEmitter { async processQueue() { if (!this.alertsQueue.length) { this.isProcessingQueue = false - console.log(`[alert.queue] no more alerts in queue`) + console.log('[alert.queue] no more alerts in queue') return } diff --git a/src/services/catalog.js b/src/services/catalog.js index 71d8850..c30f44d 100644 --- a/src/services/catalog.js +++ b/src/services/catalog.js @@ -37,7 +37,7 @@ const stablecoinPairLookup = new RegExp( '|' )})$|^([A-Z0-9]{3,})[-/:_]?(${reverseStablecoins.join('|')})$` ) -const simplePairLookup = new RegExp(`^([A-Z0-9]{2,})[-/_]?([A-Z0-9]{3,})$`) +const simplePairLookup = new RegExp('^([A-Z0-9]{2,})[-/_]?([A-Z0-9]{3,})$') const reverseStablecoinPairLookup = new RegExp( `(\\w{3,})(${reverseStablecoins.join('|')})$`, diff --git a/src/services/connections.js b/src/services/connections.js index 016d0bc..ef1a582 100644 --- a/src/services/connections.js +++ b/src/services/connections.js @@ -82,7 +82,7 @@ function getConnectionsPersistance() { json = JSON.parse(data) } catch (parseError) { console.error( - `[connections] connection.json is corrupted`, + '[connections] connection.json is corrupted', parseError.message ) json = {} @@ -106,11 +106,11 @@ function cleanConnection(connection) { connection if (!exchange) { - throw new Error(`unknown connection's exchange`) + throw new Error('unknown connection\'s exchange') } if (!pair) { - throw new Error(`unknown connection's pair`) + throw new Error('unknown connection\'s pair') } if (typeof hit === 'undefined') { @@ -245,10 +245,10 @@ module.exports.restoreConnections = async function () { `[connections] couldn't restore ${market}'s connection because ${ persistance[market].timestamp ? `last ping is too old (${getHms( - now - persistance[market].timestamp, - true - )} ago)` - : `last ping is unknown` + now - persistance[market].timestamp, + true + )} ago)` + : 'last ping is unknown' }` ) // connection is to old (too much data to recover) @@ -313,7 +313,7 @@ module.exports.saveConnections = async function (immediate = false) { saveConnectionsTimeout = setTimeout(async () => { saveConnectionsTimeout = null if (await module.exports.saveConnections(true)) { - console.log(`[connections] saved connections`) + console.log('[connections] saved connections') } }, 10000) @@ -411,7 +411,7 @@ module.exports.registerConnection = function (id, exchange, pair) { */ module.exports.updateIndexes = async function (ranges, callback) { for (const index of indexes) { - const open = index.price + const _open = index.price let high = 0 let low = 0 diff --git a/src/services/socket.js b/src/services/socket.js index b6bc24d..511081d 100644 --- a/src/services/socket.js +++ b/src/services/socket.js @@ -2,7 +2,7 @@ const { statSync, unlinkSync } = require('fs') const net = require('net') const EventEmitter = require('events') const config = require('../config') -const { indexes, getActiveConnections } = require('./connections') +const { indexes } = require('./connections') const { v4: uuidv4 } = require('uuid') require('../typedef') @@ -76,14 +76,14 @@ class SocketService extends EventEmitter { this.emit(data.opId, data) }) ) - .on('close', hadError => { + .on('close', _hadError => { // collector never close connection with cluster by itself // console[hadError ? 'error' : 'log'](`[socket/collector] cluster closed`) // schedule reconnection this.reconnectCluster() }) - .on('error', error => { + .on('error', _error => { // the close even destroy the previous strem and may trigger error // reconnect in this situation as well this.reconnectCluster() @@ -155,11 +155,11 @@ class SocketService extends EventEmitter { try { if (statSync(config.influxCollectorsClusterSocketPath)) { console.debug( - `[socket/cluster] unix socket was not closed properly last time` + '[socket/cluster] unix socket was not closed properly last time' ) unlinkSync(config.influxCollectorsClusterSocketPath) } - } catch (error) {} + } catch (_error) { /* ignored */ } this.serverSocket = net.createServer(socket => { console.log('[socket/cluster] collector connected successfully') @@ -219,7 +219,7 @@ class SocketService extends EventEmitter { }) this.serverSocket.on('error', error => { - console.error(`[socket/cluster] server socket error`, error) + console.error('[socket/cluster] server socket error', error) }) this.serverSocket.listen(config.influxCollectorsClusterSocketPath) diff --git a/src/storage/binaries.js b/src/storage/binaries.js new file mode 100644 index 0000000..483f49c --- /dev/null +++ b/src/storage/binaries.js @@ -0,0 +1,1416 @@ +const fs = require('fs') +const path = require('path') +const { ensureDirectoryExists, getHms } = require('../helper') +const config = require('../config') +const alertService = require('../services/alert') +const { updateIndexes } = require('../services/connections') + +/** + * Size in bytes of a single OHLCV record in the binary file. + * Layout (56 bytes total): + * - open: Int32LE (4 bytes, offset 0) + * - high: Int32LE (4 bytes, offset 4) + * - low: Int32LE (4 bytes, offset 8) + * - close: Int32LE (4 bytes, offset 12) + * - vbuy: BigInt64LE (8 bytes, offset 16) + * - vsell: BigInt64LE (8 bytes, offset 24) + * - cbuy: UInt32LE (4 bytes, offset 32) + * - csell: UInt32LE (4 bytes, offset 36) + * - lbuy: BigInt64LE (8 bytes, offset 40) + * - lsell: BigInt64LE (8 bytes, offset 48) + * @constant {number} + */ +const RECORD_SIZE = 56 + +/** + * Scale factor for storing prices as integers. + * Prices are multiplied by this value before storing and divided when reading. + * Example: price 45000.1234 becomes 450001234 (stored as Int32). + * @constant {number} + */ +const PRICE_SCALE = 10000 + +/** + * Scale factor for storing volumes as integers. + * Volumes are multiplied by this value before storing and divided when reading. + * @constant {number} + */ +const VOLUME_SCALE = 1000000 + +/** + * @typedef {Object} BinaryMeta + * @property {string} exchange - Exchange name + * @property {string} symbol - Trading pair symbol + * @property {string} timeframe - Human-readable timeframe label (e.g., "5s", "1m") + * @property {number} timeframeMs - Timeframe in milliseconds + * @property {number} startTs - Timestamp of the first record (bucket time) + * @property {number} endTs - Timestamp just past the last record (startTs + records * timeframeMs) + * @property {number} priceScale - Price scaling factor used + * @property {number} volumeScale - Volume scaling factor used + * @property {number} records - Total number of records in the binary file + * @property {number} lastInputStartTs - Last input bar timestamp processed + */ + +/** + * @typedef {Object} FilePaths + * @property {string} bin - Path to the binary data file + * @property {string} json - Path to the JSON metadata file + */ + +/** + * @typedef {Object} ParsedMarket + * @property {string} exchange - Exchange name + * @property {string} symbol - Trading pair symbol + */ + +/** + * @typedef {Object} WriteResult + * @property {number|null} fromTsWritten - Minimum timestamp written (null if nothing written) + * @property {number|null} toTsWritten - Maximum timestamp written (null if nothing written) + */ + +/** + * @typedef {Object} RecordData + * @property {number} open - Open price + * @property {number} high - High price + * @property {number} low - Low price + * @property {number} close - Close price + * @property {number} vbuy - Buy volume (price * size) + * @property {number} vsell - Sell volume (price * size) + * @property {number} cbuy - Buy trade count + * @property {number} csell - Sell trade count + * @property {number} lbuy - Buy liquidation volume + * @property {number} lsell - Sell liquidation volume + */ + +/** + * Binary storage engine for OHLCV data. + * + * Stores aggregated trade data in dense binary files with fixed-size records. + * Each market (exchange:symbol) has separate files per timeframe. + * + * Key features: + * - Dense indexing: records are stored contiguously, with null bars filling gaps + * - Upsert semantics: can overwrite existing records or append new ones + * - Automatic resampling: base timeframe data is aggregated to higher timeframes + * - Backfill support: older trades can be merged into existing buckets + * + * Data flow: + * 1. Trades arrive via save() → processTrades() aggregates into pendingBars + * 2. On flush interval, closed bars are persisted via upsertBars() + * 3. After base TF write, higher TFs are resampled via resampleToHigherTimeframes() + * + * @class + */ +class BinariesStorage { + /** + * Creates a new BinariesStorage instance. + * Initializes in-memory caches and ensures the data directory exists. + */ + constructor() { + /** @type {string} */ + this.name = this.constructor.name + + /** @type {string} */ + this.format = 'point' + + /** + * In-memory cache of bars not yet persisted (current bucket and recent unflushed). + * Structure: { [market: string]: { [timestamp: number]: Bar } } + * @type {Object>} + */ + this.pendingBars = {} + + /** + * Cache of recently persisted bars for backfill merging. + * Allows late-arriving trades to merge into already-flushed buckets. + * Structure: { [market: string]: { [timestamp: number]: Bar } } + * @type {Object>} + */ + this.archivedBars = {} + + /** + * Base timeframe in milliseconds (smallest granularity stored). + * @type {number} + */ + this.baseTimeframe = config.influxTimeframe + + /** + * All supported timeframes (base + resampled targets). + * @type {number[]} + */ + this.timeframes = [config.influxTimeframe].concat(config.influxResampleTo || []) + + // Ensure the data directory exists + if (!fs.existsSync(config.filesLocation)) { + fs.mkdirSync(config.filesLocation, { recursive: true }) + } + } + + /** + * Removes old entries from the archivedBars cache for a given market. + * Keeps only the last (baseTimeframe * 100) milliseconds worth of buckets. + * This prevents unbounded memory growth while allowing reasonable backfill windows. + * + * @param {string} market - Market identifier (e.g., "COINBASE:BTC-USD") + */ + cleanArchivedBars(market) { + const barsMap = this.archivedBars[market] + if (!barsMap) return + + const keepWindow = this.baseTimeframe * 100 + const now = Date.now() + const cutoff = Math.floor(now / this.baseTimeframe) * this.baseTimeframe - keepWindow + + for (const ts in barsMap) { + if (Number(ts) < cutoff) { + delete barsMap[ts] + } + } + } + + /** + * Reads a single record from the binary file at a specific bucket timestamp. + * Uses positioned read (pread) to read only the needed 56 bytes. + * + * This is used as a disk fallback when processing trades for buckets + * that exist on disk but not in memory caches. + * + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol + * @param {number} timeframe - Timeframe in milliseconds + * @param {number} bucketTs - Bucket timestamp to read + * @returns {Bar|null} Bar data if found and non-null, otherwise null + */ + readSingleRecordAtTs(exchange, symbol, timeframe, bucketTs) { + const paths = this.getFilePath(exchange, symbol, timeframe) + const meta = this.readMeta(paths.json) + if (!meta) return null + if (!fs.existsSync(paths.bin)) return null + + // Check if bucketTs is within the file's range + if (bucketTs < meta.startTs || bucketTs >= meta.endTs) return null + + // Calculate record index and byte offset + const index = Math.floor((bucketTs - meta.startTs) / meta.timeframeMs) + if (index < 0 || index >= meta.records) return null + + const byteOffset = index * RECORD_SIZE + const buffer = Buffer.alloc(RECORD_SIZE) + + let fd + try { + fd = fs.openSync(paths.bin, 'r') + const stat = fs.fstatSync(fd) + if (byteOffset + RECORD_SIZE > stat.size) { + fs.closeSync(fd) + return null + } + // Positioned read - only reads the single record we need + fs.readSync(fd, buffer, 0, RECORD_SIZE, byteOffset) + fs.closeSync(fd) + } catch (_e) { + if (fd !== undefined) { + try { fs.closeSync(fd) } catch (_) { /* ignored */ } + } + return null + } + + const record = this.readRecord(buffer, 0, meta) + + // Check if this is a null record (all OHLC are zero) + if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) { + return null + } + + return { + time: bucketTs, + open: record.open, + high: record.high, + low: record.low, + close: record.close, + vbuy: record.vbuy, + vsell: record.vsell, + cbuy: record.cbuy, + csell: record.csell, + lbuy: record.lbuy, + lsell: record.lsell + } + } + + /** + * Generates file paths for a market's binary and metadata files. + * + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol (will be sanitized) + * @param {number} timeframe - Timeframe in milliseconds + * @returns {FilePaths} Object with bin and json file paths + */ + getFilePath(exchange, symbol, timeframe) { + // Sanitize symbol to create valid filename (replace / and : with -) + const sanitizedSymbol = symbol.replace(/[/:]/g, '-') + const tfLabel = getHms(timeframe) + const dir = path.join(config.filesLocation, exchange, sanitizedSymbol) + return { + bin: path.join(dir, `${tfLabel}.bin`), + json: path.join(dir, `${tfLabel}.json`) + } + } + + /** + * Parses a market string into exchange and symbol components. + * + * @param {string} market - Market identifier (e.g., "COINBASE:BTC-USD") + * @returns {ParsedMarket|null} Parsed components or null if invalid format + */ + parseMarket(market) { + const match = market.match(/([^:]*):(.*)/) + if (!match) return null + return { exchange: match[1], symbol: match[2] } + } + + /** + * Main entry point for saving trades. + * Aggregates trades into pending bars and flushes to disk on interval. + * + * @param {Trade[]} trades - Array of trade objects to process + * @param {boolean} isExiting - If true, forces immediate flush (used on shutdown) + * @returns {Promise} + */ + async save(trades, isExiting) { + if (!trades || !trades.length) { + return Promise.resolve() + } + + // Aggregate trades into pending bars + this.processTrades(trades) + + // Check if it's time to flush (backup interval aligned with resample interval) + const now = Date.now() + const timeBackupFloored = Math.floor(now / config.backupInterval) * config.backupInterval + const timeMinuteFloored = Math.floor(now / config.influxResampleInterval) * config.influxResampleInterval + + if (isExiting || timeBackupFloored === timeMinuteFloored) { + await this.flush(isExiting) + } + } + + /** + * Processes an array of trades and aggregates them into pending bars. + * + * Bar lookup priority (for backfill support): + * 1. pendingBars - already in memory, not yet flushed + * 2. archivedBars - recently flushed, cached for merging + * 3. Disk fallback - read single record from binary file + * 4. Create new empty bar + * + * Merge semantics: + * - "Sticky open": open price is only set when null (first trade wins) + * - "Last write wins close": close updates as trades arrive + * - high/low: extrema of all trades + * - volumes/counts: cumulative sums + * + * @param {Trade[]} trades - Array of trade objects + */ + processTrades(trades) { + /** @type {Object} */ + const ranges = {} + const timeframe = this.baseTimeframe + + for (let i = 0; i < trades.length; i++) { + const trade = trades[i] + const market = trade.exchange + ':' + trade.pair + + // Track price ranges for alert crossover checks (non-liquidation trades only) + if (!trade.liquidation) { + if (!ranges[market]) { + ranges[market] = { low: trade.price, high: trade.price, close: trade.price } + } else { + ranges[market].low = Math.min(ranges[market].low, trade.price) + ranges[market].high = Math.max(ranges[market].high, trade.price) + ranges[market].close = trade.price + } + } + + // Floor trade timestamp to bucket boundary + const tradeFlooredTime = Math.floor(trade.timestamp / timeframe) * timeframe + + // Ensure market maps exist + if (!this.pendingBars[market]) { + this.pendingBars[market] = {} + } + if (!this.archivedBars[market]) { + this.archivedBars[market] = {} + } + + // Priority 1: Check pendingBars + let bar = this.pendingBars[market][tradeFlooredTime] + + // Priority 2: Check archivedBars cache + if (!bar) { + bar = this.archivedBars[market][tradeFlooredTime] + if (bar) { + // Re-add to pendingBars so it can accumulate more trades + this.pendingBars[market][tradeFlooredTime] = bar + } + } + + // Priority 3: Disk fallback (only for base timeframe) + if (!bar) { + // Try disk fallback for base timeframe only + const parsed = this.parseMarket(market) + if (parsed) { + const diskBar = this.readSingleRecordAtTs(parsed.exchange, parsed.symbol, timeframe, tradeFlooredTime) + if (diskBar) { + bar = { + time: tradeFlooredTime, + market: market, + cbuy: diskBar.cbuy, + csell: diskBar.csell, + vbuy: diskBar.vbuy, + vsell: diskBar.vsell, + lbuy: diskBar.lbuy, + lsell: diskBar.lsell, + open: diskBar.open, + high: diskBar.high, + low: diskBar.low, + close: diskBar.close + } + this.archivedBars[market][tradeFlooredTime] = bar + this.pendingBars[market][tradeFlooredTime] = bar + } + } + } + + if (!bar) { + bar = this.pendingBars[market][tradeFlooredTime] = { + time: tradeFlooredTime, + market: market, + cbuy: 0, + csell: 0, + vbuy: 0, + vsell: 0, + lbuy: 0, + lsell: 0, + open: null, + high: null, + low: null, + close: null + } + } + + if (trade.liquidation) { + bar['l' + trade.side] += trade.price * trade.size + } else { + // Sticky open: only set if null + if (bar.open === null) { + bar.open = bar.high = bar.low = bar.close = +trade.price + } else { + // Update high/low extrema and close price + bar.high = Math.max(bar.high, +trade.price) + bar.low = Math.min(bar.low, +trade.price) + bar.close = +trade.price + } + // Increment trade count and volume for the appropriate side + bar['c' + trade.side] += trade.count || 1 + bar['v' + trade.side] += trade.price * trade.size + } + } + + // Update price indexes and check for alert crossovers + updateIndexes(ranges, async (index, high, low, direction) => { + if (alertService && alertService.checkPriceCrossover) { + await alertService.checkPriceCrossover(index, high, low, direction) + } + }) + } + + /** + * Flushes closed pending bars to disk. + * + * A bar is "closed" when its bucket time is before the current bucket threshold. + * Closed bars are: + * 1. Written to disk via upsertBars() + * 2. Copied to archivedBars for future backfill merging + * 3. Removed from pendingBars + * + * After base timeframe write, triggers resampling to higher timeframes. + * + * @param {boolean} isExiting - If true, flushes all bars regardless of close status + * @returns {Promise} + */ + async flush(isExiting) { + const now = Date.now() + const timeframe = this.baseTimeframe + // Current bucket start time - bars before this are considered "closed" + const closedThreshold = Math.floor(now / timeframe) * timeframe + + for (const market in this.pendingBars) { + const barsMap = this.pendingBars[market] + const closedBars = [] + + // Collect closed bars and remove from pending + for (const ts in barsMap) { + const barTime = Number(ts) + const alignedBarTime = Math.floor(barTime / timeframe) * timeframe + + // Bar is closed if its time is before current bucket (or if exiting) + if (isExiting || alignedBarTime < closedThreshold) { + const bar = barsMap[ts] + // Only persist bars that have actual data (non-null close) + if (bar.close !== null) { + bar.time = alignedBarTime + closedBars.push(bar) + } + delete barsMap[ts] + } + } + + if (closedBars.length === 0) { + this.cleanArchivedBars(market) + continue + } + + // Sort by time for consistent write order + closedBars.sort((a, b) => a.time - b.time) + + const parsed = this.parseMarket(market) + if (!parsed) { + this.cleanArchivedBars(market) + continue + } + + // Write to disk (may include overwrites for backfilled buckets) + const writeResult = await this.upsertBars(parsed.exchange, parsed.symbol, timeframe, closedBars) + + // Copy closed bars to archivedBars for future backfill merging + // This allows late-arriving trades to merge into recently flushed buckets + if (!this.archivedBars[market]) { + this.archivedBars[market] = {} + } + for (const bar of closedBars) { + this.archivedBars[market][bar.time] = bar + } + + // Clean old archived bars to prevent unbounded memory growth + this.cleanArchivedBars(market) + + // Resample to higher timeframes if we wrote any data + if (writeResult && writeResult.fromTsWritten !== null) { + await this.resampleToHigherTimeframes(parsed.exchange, parsed.symbol, writeResult.fromTsWritten, writeResult.toTsWritten) + } + } + } + + /** + * Triggers resampling from base timeframe to all configured higher timeframes. + * + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol + * @param {number} fromTs - Start of dirty range (earliest modified bucket) + * @param {number} toTs - End of dirty range (latest modified bucket) + * @returns {Promise} + */ + async resampleToHigherTimeframes(exchange, symbol, fromTs, toTs) { + const baseTimeframe = this.baseTimeframe + const higherTimeframes = this.timeframes.filter(tf => tf > baseTimeframe) + + if (higherTimeframes.length === 0) return + + const basePaths = this.getFilePath(exchange, symbol, baseTimeframe) + const baseMeta = this.readMeta(basePaths.json) + if (!baseMeta || !fs.existsSync(basePaths.bin)) return + + // Resample each higher timeframe + for (const targetTimeframe of higherTimeframes) { + await this.resampleRangeToTimeframe(exchange, symbol, baseMeta, basePaths.bin, fromTs, toTs, targetTimeframe) + } + } + + /** + * Resamples a range of base timeframe data into a higher target timeframe. + * + * Unlike append-only resampling, this supports upsert semantics: + * - Overwrites existing target buckets that were affected by base changes + * - Appends new target buckets as needed + * - Maintains dense indexing (null bars fill gaps) + * + * Aggregation rules: + * - open: first non-null value in bucket (by base bar time order) + * - close: last non-null value in bucket + * - high/low: extrema across all base bars + * - volumes/counts: cumulative sums + * + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol + * @param {BinaryMeta} baseMeta - Metadata for base timeframe file + * @param {string} baseBinPath - Path to base timeframe binary file + * @param {number} fromTs - Start of dirty range (base bucket time) + * @param {number} toTs - End of dirty range (base bucket time) + * @param {number} targetTimeframe - Target timeframe in milliseconds + * @returns {Promise} + */ + async resampleRangeToTimeframe(exchange, symbol, baseMeta, baseBinPath, fromTs, toTs, targetTimeframe) { + const baseTimeframe = baseMeta.timeframeMs + const paths = this.getFilePath(exchange, symbol, targetTimeframe) + await ensureDirectoryExists(paths.bin) + + let meta = this.readMeta(paths.json) + + // Compute dirty target bucket range (inclusive) + // A target bucket is dirty if any of its contributing base buckets were modified + const firstTargetBucket = Math.floor(fromTs / targetTimeframe) * targetTimeframe + const lastTargetBucket = Math.floor(toTs / targetTimeframe) * targetTimeframe + + // Initialize metadata if this is a new file + if (!meta) { + meta = { + exchange: exchange, + symbol: symbol, + timeframe: getHms(targetTimeframe), + timeframeMs: targetTimeframe, + startTs: firstTargetBucket, + endTs: firstTargetBucket, + priceScale: PRICE_SCALE, + volumeScale: VOLUME_SCALE, + records: 0, + lastInputStartTs: firstTargetBucket + } + } + + // Skip buckets before meta.startTs (no prepend allowed) + const startBucket = Math.max(firstTargetBucket, meta.startTs) + if (startBucket > lastTargetBucket) return + + // Calculate base record index range covering the dirty target buckets + // Need to read all base bars that contribute to [startBucket, lastTargetBucket + targetTimeframe) + const baseStartIndex = Math.floor((startBucket - baseMeta.startTs) / baseTimeframe) + const baseEndIndex = Math.ceil((lastTargetBucket + targetTimeframe - baseMeta.startTs) / baseTimeframe) + + // Clamp to valid index range within the base file + const clampedStartIndex = Math.max(0, baseStartIndex) + const clampedEndIndex = Math.min(baseMeta.records, baseEndIndex) + + // Aggregate base bars into target buckets + // Key: target bucket timestamp, Value: aggregated bar data + const aggregated = {} + + if (clampedStartIndex < clampedEndIndex) { + const count = clampedEndIndex - clampedStartIndex + const byteOffset = clampedStartIndex * RECORD_SIZE + const byteLength = count * RECORD_SIZE + + let buffer + try { + const fd = fs.openSync(baseBinPath, 'r') + const stat = fs.fstatSync(fd) + const maxBytes = Math.min(byteLength, stat.size - byteOffset) + if (maxBytes > 0) { + buffer = Buffer.alloc(maxBytes) + fs.readSync(fd, buffer, 0, maxBytes, byteOffset) + } + fs.closeSync(fd) + } catch (_e) { + buffer = null + } + + if (buffer) { + const recordCount = Math.floor(buffer.length / RECORD_SIZE) + + // Process each base record and aggregate into target buckets + for (let i = 0; i < recordCount; i++) { + const record = this.readRecord(buffer, i * RECORD_SIZE, baseMeta) + const barTime = baseMeta.startTs + (clampedStartIndex + i) * baseTimeframe + + // Skip null records (gaps in base data) + if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) continue + + // Determine which target bucket this base bar contributes to + const targetBucketTime = Math.floor(barTime / targetTimeframe) * targetTimeframe + if (targetBucketTime < startBucket || targetBucketTime > lastTargetBucket) continue + + if (!aggregated[targetBucketTime]) { + // First contributing base bar for this target bucket + aggregated[targetBucketTime] = { + time: targetBucketTime, + open: record.open, + high: record.high, + low: record.low, + close: record.close, + vbuy: record.vbuy, + vsell: record.vsell, + cbuy: record.cbuy, + csell: record.csell, + lbuy: record.lbuy, + lsell: record.lsell, + _firstBarTime: barTime // Track earliest base bar for open determination + } + } else { + // Merge subsequent base bars into the aggregate + const agg = aggregated[targetBucketTime] + + // Open comes from the earliest base bar (by time, not arrival order) + if (barTime < agg._firstBarTime) { + agg.open = record.open + agg._firstBarTime = barTime + } + + // High/low are extrema, close comes from the last bar processed + agg.high = Math.max(agg.high, record.high) + agg.low = Math.min(agg.low, record.low) + agg.close = record.close + + // Volumes and counts are summed + agg.vbuy += record.vbuy + agg.vsell += record.vsell + agg.cbuy += record.cbuy + agg.csell += record.csell + agg.lbuy += record.lbuy + agg.lsell += record.lsell + } + } + } + } + + // Build complete list of bars to upsert (dense: one per target bucket in range) + const barsToUpsert = [] + for (let bucketTs = startBucket; bucketTs <= lastTargetBucket; bucketTs += targetTimeframe) { + if (aggregated[bucketTs]) { + barsToUpsert.push(aggregated[bucketTs]) + } else { + // No contributing base bars - write null bar for dense indexing + barsToUpsert.push(this.createNullBar(bucketTs)) + } + } + + if (barsToUpsert.length === 0) return + + // Separate bars into overwrites (existing records) vs appends (new records) + const overwrites = [] + const appends = [] + + for (const bar of barsToUpsert) { + const index = Math.floor((bar.time - meta.startTs) / targetTimeframe) + if (index < 0) continue // No prepend allowed + if (index < meta.records) { + overwrites.push({ index, bar }) + } else { + appends.push({ index, bar }) + } + } + + // Handle overwrites (positioned writes to existing records) + if (overwrites.length > 0) { + let fd = null + try { + // Open in read-write mode for in-place updates + fd = fs.openSync(paths.bin, 'r+') + + // Sort by index for sequential access and batch contiguous writes + overwrites.sort((a, b) => a.index - b.index) + let groupStart = 0 + + // Group contiguous indices for efficient batch writes + while (groupStart < overwrites.length) { + let groupEnd = groupStart + + // Find end of contiguous run + while (groupEnd + 1 < overwrites.length && + overwrites[groupEnd + 1].index === overwrites[groupEnd].index + 1) { + groupEnd++ + } + + // Write contiguous group in single syscall + const groupSize = groupEnd - groupStart + 1 + const buffer = Buffer.alloc(groupSize * RECORD_SIZE) + let bufferOffset = 0 + + for (let i = groupStart; i <= groupEnd; i++) { + this.writeRecord(buffer, bufferOffset, overwrites[i].bar, meta) + bufferOffset += RECORD_SIZE + } + + const fileOffset = overwrites[groupStart].index * RECORD_SIZE + fs.writeSync(fd, buffer, 0, buffer.length, fileOffset) + + groupStart = groupEnd + 1 + } + } finally { + if (fd !== null) { + fs.closeSync(fd) + } + } + } + + // Handle appends (extend file with new records) + if (appends.length > 0) { + appends.sort((a, b) => a.index - b.index) + const lastAppendIndex = appends[appends.length - 1].index + + // Build lookup map for sparse appends + const appendMap = {} + for (const a of appends) { + appendMap[a.index] = a.bar + } + + const BATCH_SIZE = 10000 + let currentIndex = meta.records + let fd = null + + try { + // Open in append mode + fd = fs.openSync(paths.bin, 'a') + + // Write records from current end to last append index (dense, filling gaps with null) + while (currentIndex <= lastAppendIndex) { + const batchRecords = [] + const batchEndIndex = Math.min(currentIndex + BATCH_SIZE, lastAppendIndex + 1) + + while (currentIndex < batchEndIndex) { + if (appendMap[currentIndex]) { + batchRecords.push(appendMap[currentIndex]) + } else { + // Fill gap with null bar + batchRecords.push(this.createNullBar(meta.startTs + currentIndex * targetTimeframe)) + } + currentIndex++ + } + + if (batchRecords.length > 0) { + const buffer = Buffer.alloc(batchRecords.length * RECORD_SIZE) + let offset = 0 + + for (const bar of batchRecords) { + this.writeRecord(buffer, offset, bar, meta) + offset += RECORD_SIZE + } + + fs.writeSync(fd, buffer) + meta.records += batchRecords.length + } + } + } finally { + if (fd !== null) { + fs.closeSync(fd) + } + } + + // Update endTs after extending file + meta.endTs = meta.startTs + meta.records * targetTimeframe + } + + meta.lastInputStartTs = lastTargetBucket + this.writeMeta(paths.json, meta) + } + + /** + * Appends bars to a binary file (append-only, no overwrites). + * + * @deprecated Use upsertBars() instead for backfill support + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol + * @param {number} timeframe - Timeframe in milliseconds + * @param {Bar[]} bars - Array of bars to append + * @returns {Promise} Timestamps of written data range + */ + async appendBars(exchange, symbol, timeframe, bars) { + const paths = this.getFilePath(exchange, symbol, timeframe) + await ensureDirectoryExists(paths.bin) + + let meta = this.readMeta(paths.json) + + // Get time bounds of input bars + const firstBarTime = Math.floor(bars[0].time / timeframe) * timeframe + const lastBarTime = Math.floor(bars[bars.length - 1].time / timeframe) * timeframe + + // Initialize metadata if new file + if (!meta) { + meta = { + exchange: exchange, + symbol: symbol, + timeframe: getHms(timeframe), + timeframeMs: timeframe, + startTs: firstBarTime, + endTs: firstBarTime, + priceScale: PRICE_SCALE, + volumeScale: VOLUME_SCALE, + records: 0, + lastInputStartTs: firstBarTime + } + } + + // Filter to only bars that can be appended (>= endTs, non-null close) + const barsToWriteMap = {} + for (const bar of bars) { + const alignedTime = Math.floor(bar.time / timeframe) * timeframe + if (alignedTime < meta.endTs) continue // Skip existing records + if (bar.close === null) continue // Skip empty bars + barsToWriteMap[alignedTime] = bar + } + + const timestamps = Object.keys(barsToWriteMap).map(Number).sort((a, b) => a - b) + if (timestamps.length === 0) return { fromTsWritten: null, toTsWritten: null } + + const lastNewBarTime = timestamps[timestamps.length - 1] + const fromTsWritten = meta.endTs + + const BATCH_SIZE = 10000 + let currentTs = meta.endTs + let fd = null + + try { + fd = fs.openSync(paths.bin, 'a') + + // Write records from endTs to lastNewBarTime (dense, with gap filling) + while (currentTs <= lastNewBarTime) { + const batchRecords = [] + const batchEnd = Math.min(currentTs + BATCH_SIZE * timeframe, lastNewBarTime + timeframe) + + while (currentTs < batchEnd && currentTs <= lastNewBarTime) { + if (barsToWriteMap[currentTs]) { + batchRecords.push(barsToWriteMap[currentTs]) + } else { + // Fill gap with null bar + batchRecords.push(this.createNullBar(currentTs)) + } + currentTs += timeframe + } + + if (batchRecords.length > 0) { + const buffer = Buffer.alloc(batchRecords.length * RECORD_SIZE) + let offset = 0 + + for (const bar of batchRecords) { + this.writeRecord(buffer, offset, bar, meta) + offset += RECORD_SIZE + } + + fs.writeSync(fd, buffer) + meta.records += batchRecords.length + } + } + } finally { + if (fd !== null) { + fs.closeSync(fd) + } + } + + // Update metadata + meta.endTs = meta.startTs + meta.records * timeframe + meta.lastInputStartTs = lastBarTime + + this.writeMeta(paths.json, meta) + + return { fromTsWritten, toTsWritten: lastNewBarTime } + } + + /** + * Upserts bars to a binary file with support for both overwrites and appends. + * + * This is the primary write method that supports backfill scenarios: + * - Bars with timestamps >= endTs are appended (with gap filling) + * - Bars with timestamps < endTs (but >= startTs) overwrite existing records + * - Bars with timestamps < startTs are ignored (no prepend) + * + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol + * @param {number} timeframe - Timeframe in milliseconds + * @param {Bar[]} bars - Array of bars to upsert + * @returns {Promise} Timestamps of written data range (includes overwrites) + */ + async upsertBars(exchange, symbol, timeframe, bars) { + const paths = this.getFilePath(exchange, symbol, timeframe) + await ensureDirectoryExists(paths.bin) + + let meta = this.readMeta(paths.json) + + // Filter out bars with null close (empty bars) + const validBars = bars.filter(b => b.close !== null) + if (validBars.length === 0) return { fromTsWritten: null, toTsWritten: null } + + // Build map of aligned timestamps to bars + const barsMap = {} + for (const bar of validBars) { + const alignedTime = Math.floor(bar.time / timeframe) * timeframe + barsMap[alignedTime] = bar + } + + const timestamps = Object.keys(barsMap).map(Number).sort((a, b) => a - b) + if (timestamps.length === 0) return { fromTsWritten: null, toTsWritten: null } + + const firstBarTime = timestamps[0] + const _lastBarTime = timestamps[timestamps.length - 1] + + // Initialize metadata if new file + if (!meta) { + meta = { + exchange: exchange, + symbol: symbol, + timeframe: getHms(timeframe), + timeframeMs: timeframe, + startTs: firstBarTime, + endTs: firstBarTime, + priceScale: PRICE_SCALE, + volumeScale: VOLUME_SCALE, + records: 0, + lastInputStartTs: firstBarTime + } + } + + // Filter out bars before startTs (no prepend allowed) + const filteredTimestamps = timestamps.filter(ts => ts >= meta.startTs) + if (filteredTimestamps.length === 0) return { fromTsWritten: null, toTsWritten: null } + + // Track the actual dirty range (for resampling) + const minTs = filteredTimestamps[0] + const maxTs = filteredTimestamps[filteredTimestamps.length - 1] + + // Categorize bars as overwrites or appends based on their index + const overwrites = [] // { index, ts, bar } - existing records to update + const appends = [] // { index, ts, bar } - new records to add + + for (const ts of filteredTimestamps) { + const index = Math.floor((ts - meta.startTs) / timeframe) + if (index < meta.records) { + // Index exists in file - this is an overwrite + overwrites.push({ index, ts, bar: barsMap[ts] }) + } else { + // Index beyond current file - this is an append + appends.push({ index, ts, bar: barsMap[ts] }) + } + } + + // Handle overwrites (positioned writes to existing records) + if (overwrites.length > 0) { + let fd = null + try { + // Open in read-write mode for in-place updates + fd = fs.openSync(paths.bin, 'r+') + + // Sort and group contiguous indices for efficient batch writes + overwrites.sort((a, b) => a.index - b.index) + let groupStart = 0 + + while (groupStart < overwrites.length) { + let groupEnd = groupStart + + // Find end of contiguous run + while (groupEnd + 1 < overwrites.length && + overwrites[groupEnd + 1].index === overwrites[groupEnd].index + 1) { + groupEnd++ + } + + // Write contiguous group in single syscall + const groupSize = groupEnd - groupStart + 1 + const buffer = Buffer.alloc(groupSize * RECORD_SIZE) + let bufferOffset = 0 + + for (let i = groupStart; i <= groupEnd; i++) { + this.writeRecord(buffer, bufferOffset, overwrites[i].bar, meta) + bufferOffset += RECORD_SIZE + } + + const fileOffset = overwrites[groupStart].index * RECORD_SIZE + fs.writeSync(fd, buffer, 0, buffer.length, fileOffset) + + groupStart = groupEnd + 1 + } + } finally { + if (fd !== null) { + fs.closeSync(fd) + } + } + } + + // Handle appends (extend file with new records) + if (appends.length > 0) { + appends.sort((a, b) => a.index - b.index) + const _firstAppendIndex = appends[0].index + const lastAppendIndex = appends[appends.length - 1].index + + // Build lookup map for sparse appends + const appendMap = {} + for (const a of appends) { + appendMap[a.index] = a.bar + } + + const BATCH_SIZE = 10000 + let currentIndex = meta.records + let fd = null + + try { + // Open in append mode + fd = fs.openSync(paths.bin, 'a') + + // Write records from current end to last append index (dense, filling gaps) + while (currentIndex <= lastAppendIndex) { + const batchRecords = [] + const batchEndIndex = Math.min(currentIndex + BATCH_SIZE, lastAppendIndex + 1) + + while (currentIndex < batchEndIndex) { + if (appendMap[currentIndex]) { + batchRecords.push(appendMap[currentIndex]) + } else { + // Fill gap with null bar for dense indexing + batchRecords.push(this.createNullBar(meta.startTs + currentIndex * timeframe)) + } + currentIndex++ + } + + if (batchRecords.length > 0) { + const buffer = Buffer.alloc(batchRecords.length * RECORD_SIZE) + let offset = 0 + + for (const bar of batchRecords) { + this.writeRecord(buffer, offset, bar, meta) + offset += RECORD_SIZE + } + + fs.writeSync(fd, buffer) + meta.records += batchRecords.length + } + } + } finally { + if (fd !== null) { + fs.closeSync(fd) + } + } + + // Update endTs after extending file + meta.endTs = meta.startTs + meta.records * timeframe + } + + meta.lastInputStartTs = maxTs + this.writeMeta(paths.json, meta) + + // Return the full dirty range (includes overwrites, not just appends) + return { fromTsWritten: minTs, toTsWritten: maxTs } + } + + /** + * Creates an empty (null) bar for gap filling. + * Null bars have all OHLC set to null and all volumes/counts set to 0. + * On disk, these are stored as all zeros and skipped when reading. + * + * @param {number} time - Bucket timestamp + * @returns {Object} Null bar object + */ + createNullBar(time) { + return { + time, + open: null, + high: null, + low: null, + close: null, + vbuy: 0, + vsell: 0, + cbuy: 0, + csell: 0, + lbuy: 0, + lsell: 0 + } + } + + /** + * Writes a bar record to a buffer at the specified offset. + * + * Binary layout (56 bytes): + * - Bytes 0-3: open (Int32LE, scaled by priceScale) + * - Bytes 4-7: high (Int32LE, scaled) + * - Bytes 8-11: low (Int32LE, scaled) + * - Bytes 12-15: close (Int32LE, scaled) + * - Bytes 16-23: vbuy (BigInt64LE, scaled by volumeScale) + * - Bytes 24-31: vsell (BigInt64LE, scaled) + * - Bytes 32-35: cbuy (UInt32LE, trade count) + * - Bytes 36-39: csell (UInt32LE, trade count) + * - Bytes 40-47: lbuy (BigInt64LE, liquidation volume, scaled) + * - Bytes 48-55: lsell (BigInt64LE, liquidation volume, scaled) + * + * @param {Buffer} buffer - Target buffer + * @param {number} offset - Byte offset in buffer + * @param {Object} bar - Bar data to write + * @param {BinaryMeta} meta - Metadata with scale factors + */ + writeRecord(buffer, offset, bar, meta) { + // Scale and round prices (null becomes 0) + const open = bar.open !== null ? Math.round(bar.open * meta.priceScale) : 0 + const high = bar.high !== null ? Math.round(bar.high * meta.priceScale) : 0 + const low = bar.low !== null ? Math.round(bar.low * meta.priceScale) : 0 + const close = bar.close !== null ? Math.round(bar.close * meta.priceScale) : 0 + + // Write OHLC + buffer.writeInt32LE(open, offset) + buffer.writeInt32LE(high, offset + 4) + buffer.writeInt32LE(low, offset + 8) + buffer.writeInt32LE(close, offset + 12) + + // Scale and write volumes as BigInt + const vbuy = BigInt(Math.round((bar.vbuy || 0) * meta.volumeScale)) + const vsell = BigInt(Math.round((bar.vsell || 0) * meta.volumeScale)) + buffer.writeBigInt64LE(vbuy, offset + 16) + buffer.writeBigInt64LE(vsell, offset + 24) + + // Write trade counts + buffer.writeUInt32LE(bar.cbuy || 0, offset + 32) + buffer.writeUInt32LE(bar.csell || 0, offset + 36) + + // Scale and write liquidation volumes + const lbuy = BigInt(Math.round((bar.lbuy || 0) * meta.volumeScale)) + const lsell = BigInt(Math.round((bar.lsell || 0) * meta.volumeScale)) + buffer.writeBigInt64LE(lbuy, offset + 40) + buffer.writeBigInt64LE(lsell, offset + 48) + } + + /** + * Reads a bar record from a buffer at the specified offset. + * + * @param {Buffer} buffer - Source buffer + * @param {number} offset - Byte offset in buffer + * @param {BinaryMeta} meta - Metadata with scale factors + * @returns {RecordData} Decoded bar data + */ + readRecord(buffer, offset, meta) { + // Read and descale OHLC + const open = buffer.readInt32LE(offset) / meta.priceScale + const high = buffer.readInt32LE(offset + 4) / meta.priceScale + const low = buffer.readInt32LE(offset + 8) / meta.priceScale + const close = buffer.readInt32LE(offset + 12) / meta.priceScale + + // Read and descale volumes + const vbuy = Number(buffer.readBigInt64LE(offset + 16)) / meta.volumeScale + const vsell = Number(buffer.readBigInt64LE(offset + 24)) / meta.volumeScale + + // Read trade counts + const cbuy = buffer.readUInt32LE(offset + 32) + const csell = buffer.readUInt32LE(offset + 36) + + // Read and descale liquidation volumes + const lbuy = Number(buffer.readBigInt64LE(offset + 40)) / meta.volumeScale + const lsell = Number(buffer.readBigInt64LE(offset + 48)) / meta.volumeScale + + return { open, high, low, close, vbuy, vsell, cbuy, csell, lbuy, lsell } + } + + /** + * Reads metadata JSON file for a market/timeframe. + * + * @param {string} jsonPath - Path to the JSON metadata file + * @returns {BinaryMeta|null} Parsed metadata or null if not found + */ + readMeta(jsonPath) { + try { + if (!fs.existsSync(jsonPath)) return null + const content = fs.readFileSync(jsonPath, 'utf8') + return JSON.parse(content) + } catch (_e) { + return null + } + } + + /** + * Writes metadata JSON file atomically (write to temp, then rename). + * + * @param {string} jsonPath - Path to the JSON metadata file + * @param {BinaryMeta} meta - Metadata to write + */ + writeMeta(jsonPath, meta) { + const tmpPath = jsonPath + '.tmp' + fs.writeFileSync(tmpPath, JSON.stringify(meta, null, 2)) + fs.renameSync(tmpPath, jsonPath) + } + + /** + * Fetches historical OHLCV data for one or more markets. + * + * Returns data in a columnar format suitable for charting. + * Null bars (gaps) are skipped in the response. + * Pending bars near "now" are injected to provide real-time data. + * + * @param {Object} params - Query parameters + * @param {number} params.from - Start timestamp (inclusive) + * @param {number} params.to - End timestamp (exclusive) + * @param {number} [params.timeframe=60000] - Timeframe in milliseconds + * @param {string[]} [params.markets=[]] - Array of market identifiers + * @returns {Promise<{format: string, columns: Object, results: Array[]}>} Query results + */ + fetch({ from, to, timeframe = 60000, markets = [] }) { + // Validate timeframe is supported + const supported = this.timeframes.indexOf(timeframe) !== -1 + if (!supported) { + return Promise.reject(new Error(`Unsupported timeframe: ${getHms(timeframe)}`)) + } + + // Column index mapping for result arrays (matches InfluxDB format) + const columns = { + time: 0, + market: 1, + open: 2, + high: 3, + low: 4, + close: 5, + vbuy: 6, + vsell: 7, + cbuy: 8, + csell: 9, + lbuy: 10, + lsell: 11 + } + + // Pre-allocate with estimated capacity to reduce array resizing + const results = [] + + // Check if we need pending bars (query extends to near-realtime) + const now = Date.now() + const needsPendingBars = to > now - config.influxResampleInterval + + // Read data from disk for each market + for (const market of markets) { + const parsed = this.parseMarket(market) + if (!parsed) continue + + const paths = this.getFilePath(parsed.exchange, parsed.symbol, timeframe) + const meta = this.readMeta(paths.json) + if (!meta) continue + + if (!fs.existsSync(paths.bin)) continue + + // Calculate index range for requested time window + let startIndex = Math.floor((from - meta.startTs) / meta.timeframeMs) + let endIndex = Math.ceil((to - meta.startTs) / meta.timeframeMs) + + // Clamp to valid range + if (startIndex < 0) startIndex = 0 + if (endIndex > meta.records) endIndex = meta.records + if (startIndex >= endIndex) continue + + const count = endIndex - startIndex + const byteOffset = startIndex * RECORD_SIZE + const byteLength = count * RECORD_SIZE + + // Read the range from disk + let buffer + let fd + try { + fd = fs.openSync(paths.bin, 'r') + const stat = fs.fstatSync(fd) + const maxBytes = Math.min(byteLength, stat.size - byteOffset) + if (maxBytes <= 0) { + fs.closeSync(fd) + continue + } + buffer = Buffer.alloc(maxBytes) + fs.readSync(fd, buffer, 0, maxBytes, byteOffset) + fs.closeSync(fd) + } catch (_e) { + if (fd !== undefined) { + try { fs.closeSync(fd) } catch (_) { /* ignored */ } + } + continue + } + + // Decode records and add to results + const recordCount = Math.floor(buffer.length / RECORD_SIZE) + for (let i = 0; i < recordCount; i++) { + const record = this.readRecord(buffer, i * RECORD_SIZE, meta) + + // Skip null bars (gaps) - check before computing barTime for efficiency + if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) continue + + const barTime = meta.startTs + (startIndex + i) * meta.timeframeMs + + // Filter by time range + if (barTime < from || barTime >= to) continue + + // Add as array matching column order (includes market) + // Time is converted to seconds to match InfluxDB format (precision: 's') + results.push([ + Math.floor(barTime / 1000), + market, + record.open, + record.high, + record.low, + record.close, + record.vbuy, + record.vsell, + record.cbuy, + record.csell, + record.lbuy, + record.lsell + ]) + } + } + + // Inject pending bars for near-realtime data + if (needsPendingBars) { + const pendingBars = this.getPendingBars(markets, from, to) + for (let i = 0; i < pendingBars.length; i++) { + const bar = pendingBars[i] + // Time is converted to seconds to match InfluxDB format (precision: 's') + results.push([ + Math.floor(bar.time / 1000), + bar.market, + bar.open, + bar.high, + bar.low, + bar.close, + bar.vbuy, + bar.vsell, + bar.cbuy, + bar.csell, + bar.lbuy, + bar.lsell + ]) + } + } + + // Single sort at the end (by time) + // For large datasets, this is O(n log n) - unavoidable when merging multiple sources + if (markets.length > 1 || needsPendingBars) { + results.sort((a, b) => a[0] - b[0]) + } + + return Promise.resolve({ + format: this.format, + columns, + results + }) + } + + /** + * Gets pending (in-memory, not yet flushed) bars for given markets and time range. + * + * Used by fetch() to inject real-time data for queries near "now". + * Returns base timeframe bars as-is (no resampling - matches InfluxDB behavior). + * + * @param {string[]} markets - Array of market identifiers to include + * @param {number} from - Start timestamp (inclusive) + * @param {number} to - End timestamp (exclusive) + * @returns {Object[]} Array of pending bar objects with market field + */ + getPendingBars(markets, from, to) { + const results = [] + + for (const market of markets) { + const barsMap = this.pendingBars[market] + if (!barsMap) continue + + for (const ts in barsMap) { + const bar = barsMap[ts] + + // Skip empty bars + if (bar.close === null) continue + + const barTime = Number(ts) + + // Filter by requested time range + if (barTime < from || barTime >= to) continue + + results.push(bar) + } + } + + return results + } +} + +module.exports = BinariesStorage diff --git a/src/storage/files.js b/src/storage/files.js index f4ec9ed..8134874 100644 --- a/src/storage/files.js +++ b/src/storage/files.js @@ -3,8 +3,7 @@ const zlib = require('zlib') const { groupTrades, ensureDirectoryExists, - getHms, - humanFileSize + getHms } = require('../helper') const config = require('../config') @@ -306,7 +305,7 @@ class FilesStorage { `[storage/file] insert ${getHms( lastTradeTimestampInsert - firstTradeTimestampInsert )} of trades into ${path} (${ - stat ? 'file exists' : "file doesn't exists" + stat ? 'file exists' : 'file doesn\'t exists' })` ) @@ -398,7 +397,7 @@ class FilesStorage { await this.writeFile(path, tradesFile.join('\n') + '\n') } else { // new file - console.log(`\t create new file`) + console.log('\t create new file') await this.writeFile(path, output[market][fileTimestamp].data) } } @@ -422,14 +421,14 @@ class FilesStorage { return new Promise((resolve, reject) => { fs.readFile(path, (err, data) => { if (err) { - console.error(`[storage/file] failed to read file`, path) + console.error('[storage/file] failed to read file', path) throw err } if (/\.gz$/.test(path)) { zlib.gunzip(data, (err, buffer) => { if (err) { - console.error(`[storage/file] failed to unzip file`, path) + console.error('[storage/file] failed to unzip file', path) return reject(err) } resolve(buffer.toString()) @@ -448,10 +447,10 @@ class FilesStorage { rawPath = rawPath.replace(/\.gz$/, '') } - return new Promise((resolve, reject) => { - fs.writeFile(rawPath, content, (err, data) => { + return new Promise((resolve, _reject) => { + fs.writeFile(rawPath, content, (err, _data) => { if (err) { - console.error(`[storage/file] failed to write file`, path) + console.error('[storage/file] failed to write file', path) throw err } diff --git a/src/storage/influx.js b/src/storage/influx.js index 59ffd5f..a8750b5 100644 --- a/src/storage/influx.js +++ b/src/storage/influx.js @@ -1,6 +1,5 @@ const Influx = require('influx') -const { getHms, sleep, ID, ago } = require('../helper') -const net = require('net') +const { getHms, sleep, ID } = require('../helper') const config = require('../config') const socketService = require('../services/socket') const alertService = require('../services/alert') @@ -43,7 +42,7 @@ class InfluxStorage { } async connect() { - if (/\-/.test(config.influxDatabase)) { + if (/-/.test(config.influxDatabase)) { throw new Error('dashes not allowed inside influxdb database') } @@ -82,8 +81,8 @@ class InfluxStorage { console.error( [ `[storage/influx] Error: ${error.message}... retrying in 1s`, - `Please ensure that the environment variable INFLUX_HOST is correctly set or that InfluxDB is running.`, - `Refer to the README.md file for more instructions.` + 'Please ensure that the environment variable INFLUX_HOST is correctly set or that InfluxDB is running.', + 'Refer to the README.md file for more instructions.' ].join('\n') ) @@ -262,7 +261,7 @@ class InfluxStorage { .map(market => `market = '${market}'`) .join(' OR ')})` - query += `GROUP BY "market" ORDER BY time DESC LIMIT 1` + query += 'GROUP BY "market" ORDER BY time DESC LIMIT 1' this.influx.query(query).then(data => { for (let bar of data) { @@ -581,9 +580,9 @@ class InfluxStorage { if (bar.close !== null) { ;(fields.open = bar.open), - (fields.high = bar.high), - (fields.low = bar.low), - (fields.close = bar.close) + (fields.high = bar.high), + (fields.low = bar.low), + (fields.close = bar.close) } return { @@ -640,10 +639,10 @@ class InfluxStorage { attempt === 1 ? 'st' : attempt === 2 - ? 'nd' - : attempt === 3 - ? 'rd' - : 'th' + ? 'nd' + : attempt === 3 + ? 'rd' + : 'th' } attempt)`, error.message ) @@ -812,10 +811,10 @@ class InfluxStorage { attempt === 1 ? 'st' : attempt === 2 - ? 'nd' - : attempt === 3 - ? 'rd' - : 'th' + ? 'nd' + : attempt === 3 + ? 'rd' + : 'th' } attempt)`, error.message ) From d2e4d75e7973c0d9389c03deb97ca7cae81a0fa9 Mon Sep 17 00:00:00 2001 From: Tucsky Date: Tue, 13 Jan 2026 22:56:57 +0100 Subject: [PATCH 2/6] feat: implement binaries storage - Add io.js for file path generation, record reading/writing, and metadata management. - Introduce resample.js to handle resampling of market data to higher timeframes with upsert semantics. - Create write.js for appending and upserting bars to binary files, including null bar handling for dense indexing. - Implement functions for reading and writing binary records, ensuring efficient file operations and metadata updates. --- src/config.js | 8 +- src/storage/binaries.js | 1416 ----------------------------- src/storage/binaries/README.md | 319 +++++++ src/storage/binaries/constants.js | 99 ++ src/storage/binaries/index.js | 525 +++++++++++ src/storage/binaries/io.js | 223 +++++ src/storage/binaries/resample.js | 312 +++++++ src/storage/binaries/write.js | 316 +++++++ 8 files changed, 1799 insertions(+), 1419 deletions(-) delete mode 100644 src/storage/binaries.js create mode 100644 src/storage/binaries/README.md create mode 100644 src/storage/binaries/constants.js create mode 100644 src/storage/binaries/index.js create mode 100644 src/storage/binaries/io.js create mode 100644 src/storage/binaries/resample.js create mode 100644 src/storage/binaries/write.js diff --git a/src/config.js b/src/config.js index f0eb0f5..b218f9a 100644 --- a/src/config.js +++ b/src/config.js @@ -299,9 +299,11 @@ if (config.storage) { } for (let storage of config.storage) { - const storagePath = path.resolve(__dirname, 'storage/' + storage + '.js') - if (!fs.existsSync(storagePath)) { - throw new Error(`Unknown storage solution "${storagePath}"`) + const storageFilePath = path.resolve(__dirname, 'storage/' + storage + '.js') + const storageDirPath = path.resolve(__dirname, 'storage/' + storage) + // Check for single file (e.g., influx.js) or directory with index.js (e.g., binaries/index.js) + if (!fs.existsSync(storageFilePath) && !fs.statSync(storageDirPath, { throwIfNoEntry: false })?.isDirectory()) { + throw new Error(`Unknown storage solution "${storage}"`) } } } else { diff --git a/src/storage/binaries.js b/src/storage/binaries.js deleted file mode 100644 index 483f49c..0000000 --- a/src/storage/binaries.js +++ /dev/null @@ -1,1416 +0,0 @@ -const fs = require('fs') -const path = require('path') -const { ensureDirectoryExists, getHms } = require('../helper') -const config = require('../config') -const alertService = require('../services/alert') -const { updateIndexes } = require('../services/connections') - -/** - * Size in bytes of a single OHLCV record in the binary file. - * Layout (56 bytes total): - * - open: Int32LE (4 bytes, offset 0) - * - high: Int32LE (4 bytes, offset 4) - * - low: Int32LE (4 bytes, offset 8) - * - close: Int32LE (4 bytes, offset 12) - * - vbuy: BigInt64LE (8 bytes, offset 16) - * - vsell: BigInt64LE (8 bytes, offset 24) - * - cbuy: UInt32LE (4 bytes, offset 32) - * - csell: UInt32LE (4 bytes, offset 36) - * - lbuy: BigInt64LE (8 bytes, offset 40) - * - lsell: BigInt64LE (8 bytes, offset 48) - * @constant {number} - */ -const RECORD_SIZE = 56 - -/** - * Scale factor for storing prices as integers. - * Prices are multiplied by this value before storing and divided when reading. - * Example: price 45000.1234 becomes 450001234 (stored as Int32). - * @constant {number} - */ -const PRICE_SCALE = 10000 - -/** - * Scale factor for storing volumes as integers. - * Volumes are multiplied by this value before storing and divided when reading. - * @constant {number} - */ -const VOLUME_SCALE = 1000000 - -/** - * @typedef {Object} BinaryMeta - * @property {string} exchange - Exchange name - * @property {string} symbol - Trading pair symbol - * @property {string} timeframe - Human-readable timeframe label (e.g., "5s", "1m") - * @property {number} timeframeMs - Timeframe in milliseconds - * @property {number} startTs - Timestamp of the first record (bucket time) - * @property {number} endTs - Timestamp just past the last record (startTs + records * timeframeMs) - * @property {number} priceScale - Price scaling factor used - * @property {number} volumeScale - Volume scaling factor used - * @property {number} records - Total number of records in the binary file - * @property {number} lastInputStartTs - Last input bar timestamp processed - */ - -/** - * @typedef {Object} FilePaths - * @property {string} bin - Path to the binary data file - * @property {string} json - Path to the JSON metadata file - */ - -/** - * @typedef {Object} ParsedMarket - * @property {string} exchange - Exchange name - * @property {string} symbol - Trading pair symbol - */ - -/** - * @typedef {Object} WriteResult - * @property {number|null} fromTsWritten - Minimum timestamp written (null if nothing written) - * @property {number|null} toTsWritten - Maximum timestamp written (null if nothing written) - */ - -/** - * @typedef {Object} RecordData - * @property {number} open - Open price - * @property {number} high - High price - * @property {number} low - Low price - * @property {number} close - Close price - * @property {number} vbuy - Buy volume (price * size) - * @property {number} vsell - Sell volume (price * size) - * @property {number} cbuy - Buy trade count - * @property {number} csell - Sell trade count - * @property {number} lbuy - Buy liquidation volume - * @property {number} lsell - Sell liquidation volume - */ - -/** - * Binary storage engine for OHLCV data. - * - * Stores aggregated trade data in dense binary files with fixed-size records. - * Each market (exchange:symbol) has separate files per timeframe. - * - * Key features: - * - Dense indexing: records are stored contiguously, with null bars filling gaps - * - Upsert semantics: can overwrite existing records or append new ones - * - Automatic resampling: base timeframe data is aggregated to higher timeframes - * - Backfill support: older trades can be merged into existing buckets - * - * Data flow: - * 1. Trades arrive via save() → processTrades() aggregates into pendingBars - * 2. On flush interval, closed bars are persisted via upsertBars() - * 3. After base TF write, higher TFs are resampled via resampleToHigherTimeframes() - * - * @class - */ -class BinariesStorage { - /** - * Creates a new BinariesStorage instance. - * Initializes in-memory caches and ensures the data directory exists. - */ - constructor() { - /** @type {string} */ - this.name = this.constructor.name - - /** @type {string} */ - this.format = 'point' - - /** - * In-memory cache of bars not yet persisted (current bucket and recent unflushed). - * Structure: { [market: string]: { [timestamp: number]: Bar } } - * @type {Object>} - */ - this.pendingBars = {} - - /** - * Cache of recently persisted bars for backfill merging. - * Allows late-arriving trades to merge into already-flushed buckets. - * Structure: { [market: string]: { [timestamp: number]: Bar } } - * @type {Object>} - */ - this.archivedBars = {} - - /** - * Base timeframe in milliseconds (smallest granularity stored). - * @type {number} - */ - this.baseTimeframe = config.influxTimeframe - - /** - * All supported timeframes (base + resampled targets). - * @type {number[]} - */ - this.timeframes = [config.influxTimeframe].concat(config.influxResampleTo || []) - - // Ensure the data directory exists - if (!fs.existsSync(config.filesLocation)) { - fs.mkdirSync(config.filesLocation, { recursive: true }) - } - } - - /** - * Removes old entries from the archivedBars cache for a given market. - * Keeps only the last (baseTimeframe * 100) milliseconds worth of buckets. - * This prevents unbounded memory growth while allowing reasonable backfill windows. - * - * @param {string} market - Market identifier (e.g., "COINBASE:BTC-USD") - */ - cleanArchivedBars(market) { - const barsMap = this.archivedBars[market] - if (!barsMap) return - - const keepWindow = this.baseTimeframe * 100 - const now = Date.now() - const cutoff = Math.floor(now / this.baseTimeframe) * this.baseTimeframe - keepWindow - - for (const ts in barsMap) { - if (Number(ts) < cutoff) { - delete barsMap[ts] - } - } - } - - /** - * Reads a single record from the binary file at a specific bucket timestamp. - * Uses positioned read (pread) to read only the needed 56 bytes. - * - * This is used as a disk fallback when processing trades for buckets - * that exist on disk but not in memory caches. - * - * @param {string} exchange - Exchange name - * @param {string} symbol - Trading pair symbol - * @param {number} timeframe - Timeframe in milliseconds - * @param {number} bucketTs - Bucket timestamp to read - * @returns {Bar|null} Bar data if found and non-null, otherwise null - */ - readSingleRecordAtTs(exchange, symbol, timeframe, bucketTs) { - const paths = this.getFilePath(exchange, symbol, timeframe) - const meta = this.readMeta(paths.json) - if (!meta) return null - if (!fs.existsSync(paths.bin)) return null - - // Check if bucketTs is within the file's range - if (bucketTs < meta.startTs || bucketTs >= meta.endTs) return null - - // Calculate record index and byte offset - const index = Math.floor((bucketTs - meta.startTs) / meta.timeframeMs) - if (index < 0 || index >= meta.records) return null - - const byteOffset = index * RECORD_SIZE - const buffer = Buffer.alloc(RECORD_SIZE) - - let fd - try { - fd = fs.openSync(paths.bin, 'r') - const stat = fs.fstatSync(fd) - if (byteOffset + RECORD_SIZE > stat.size) { - fs.closeSync(fd) - return null - } - // Positioned read - only reads the single record we need - fs.readSync(fd, buffer, 0, RECORD_SIZE, byteOffset) - fs.closeSync(fd) - } catch (_e) { - if (fd !== undefined) { - try { fs.closeSync(fd) } catch (_) { /* ignored */ } - } - return null - } - - const record = this.readRecord(buffer, 0, meta) - - // Check if this is a null record (all OHLC are zero) - if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) { - return null - } - - return { - time: bucketTs, - open: record.open, - high: record.high, - low: record.low, - close: record.close, - vbuy: record.vbuy, - vsell: record.vsell, - cbuy: record.cbuy, - csell: record.csell, - lbuy: record.lbuy, - lsell: record.lsell - } - } - - /** - * Generates file paths for a market's binary and metadata files. - * - * @param {string} exchange - Exchange name - * @param {string} symbol - Trading pair symbol (will be sanitized) - * @param {number} timeframe - Timeframe in milliseconds - * @returns {FilePaths} Object with bin and json file paths - */ - getFilePath(exchange, symbol, timeframe) { - // Sanitize symbol to create valid filename (replace / and : with -) - const sanitizedSymbol = symbol.replace(/[/:]/g, '-') - const tfLabel = getHms(timeframe) - const dir = path.join(config.filesLocation, exchange, sanitizedSymbol) - return { - bin: path.join(dir, `${tfLabel}.bin`), - json: path.join(dir, `${tfLabel}.json`) - } - } - - /** - * Parses a market string into exchange and symbol components. - * - * @param {string} market - Market identifier (e.g., "COINBASE:BTC-USD") - * @returns {ParsedMarket|null} Parsed components or null if invalid format - */ - parseMarket(market) { - const match = market.match(/([^:]*):(.*)/) - if (!match) return null - return { exchange: match[1], symbol: match[2] } - } - - /** - * Main entry point for saving trades. - * Aggregates trades into pending bars and flushes to disk on interval. - * - * @param {Trade[]} trades - Array of trade objects to process - * @param {boolean} isExiting - If true, forces immediate flush (used on shutdown) - * @returns {Promise} - */ - async save(trades, isExiting) { - if (!trades || !trades.length) { - return Promise.resolve() - } - - // Aggregate trades into pending bars - this.processTrades(trades) - - // Check if it's time to flush (backup interval aligned with resample interval) - const now = Date.now() - const timeBackupFloored = Math.floor(now / config.backupInterval) * config.backupInterval - const timeMinuteFloored = Math.floor(now / config.influxResampleInterval) * config.influxResampleInterval - - if (isExiting || timeBackupFloored === timeMinuteFloored) { - await this.flush(isExiting) - } - } - - /** - * Processes an array of trades and aggregates them into pending bars. - * - * Bar lookup priority (for backfill support): - * 1. pendingBars - already in memory, not yet flushed - * 2. archivedBars - recently flushed, cached for merging - * 3. Disk fallback - read single record from binary file - * 4. Create new empty bar - * - * Merge semantics: - * - "Sticky open": open price is only set when null (first trade wins) - * - "Last write wins close": close updates as trades arrive - * - high/low: extrema of all trades - * - volumes/counts: cumulative sums - * - * @param {Trade[]} trades - Array of trade objects - */ - processTrades(trades) { - /** @type {Object} */ - const ranges = {} - const timeframe = this.baseTimeframe - - for (let i = 0; i < trades.length; i++) { - const trade = trades[i] - const market = trade.exchange + ':' + trade.pair - - // Track price ranges for alert crossover checks (non-liquidation trades only) - if (!trade.liquidation) { - if (!ranges[market]) { - ranges[market] = { low: trade.price, high: trade.price, close: trade.price } - } else { - ranges[market].low = Math.min(ranges[market].low, trade.price) - ranges[market].high = Math.max(ranges[market].high, trade.price) - ranges[market].close = trade.price - } - } - - // Floor trade timestamp to bucket boundary - const tradeFlooredTime = Math.floor(trade.timestamp / timeframe) * timeframe - - // Ensure market maps exist - if (!this.pendingBars[market]) { - this.pendingBars[market] = {} - } - if (!this.archivedBars[market]) { - this.archivedBars[market] = {} - } - - // Priority 1: Check pendingBars - let bar = this.pendingBars[market][tradeFlooredTime] - - // Priority 2: Check archivedBars cache - if (!bar) { - bar = this.archivedBars[market][tradeFlooredTime] - if (bar) { - // Re-add to pendingBars so it can accumulate more trades - this.pendingBars[market][tradeFlooredTime] = bar - } - } - - // Priority 3: Disk fallback (only for base timeframe) - if (!bar) { - // Try disk fallback for base timeframe only - const parsed = this.parseMarket(market) - if (parsed) { - const diskBar = this.readSingleRecordAtTs(parsed.exchange, parsed.symbol, timeframe, tradeFlooredTime) - if (diskBar) { - bar = { - time: tradeFlooredTime, - market: market, - cbuy: diskBar.cbuy, - csell: diskBar.csell, - vbuy: diskBar.vbuy, - vsell: diskBar.vsell, - lbuy: diskBar.lbuy, - lsell: diskBar.lsell, - open: diskBar.open, - high: diskBar.high, - low: diskBar.low, - close: diskBar.close - } - this.archivedBars[market][tradeFlooredTime] = bar - this.pendingBars[market][tradeFlooredTime] = bar - } - } - } - - if (!bar) { - bar = this.pendingBars[market][tradeFlooredTime] = { - time: tradeFlooredTime, - market: market, - cbuy: 0, - csell: 0, - vbuy: 0, - vsell: 0, - lbuy: 0, - lsell: 0, - open: null, - high: null, - low: null, - close: null - } - } - - if (trade.liquidation) { - bar['l' + trade.side] += trade.price * trade.size - } else { - // Sticky open: only set if null - if (bar.open === null) { - bar.open = bar.high = bar.low = bar.close = +trade.price - } else { - // Update high/low extrema and close price - bar.high = Math.max(bar.high, +trade.price) - bar.low = Math.min(bar.low, +trade.price) - bar.close = +trade.price - } - // Increment trade count and volume for the appropriate side - bar['c' + trade.side] += trade.count || 1 - bar['v' + trade.side] += trade.price * trade.size - } - } - - // Update price indexes and check for alert crossovers - updateIndexes(ranges, async (index, high, low, direction) => { - if (alertService && alertService.checkPriceCrossover) { - await alertService.checkPriceCrossover(index, high, low, direction) - } - }) - } - - /** - * Flushes closed pending bars to disk. - * - * A bar is "closed" when its bucket time is before the current bucket threshold. - * Closed bars are: - * 1. Written to disk via upsertBars() - * 2. Copied to archivedBars for future backfill merging - * 3. Removed from pendingBars - * - * After base timeframe write, triggers resampling to higher timeframes. - * - * @param {boolean} isExiting - If true, flushes all bars regardless of close status - * @returns {Promise} - */ - async flush(isExiting) { - const now = Date.now() - const timeframe = this.baseTimeframe - // Current bucket start time - bars before this are considered "closed" - const closedThreshold = Math.floor(now / timeframe) * timeframe - - for (const market in this.pendingBars) { - const barsMap = this.pendingBars[market] - const closedBars = [] - - // Collect closed bars and remove from pending - for (const ts in barsMap) { - const barTime = Number(ts) - const alignedBarTime = Math.floor(barTime / timeframe) * timeframe - - // Bar is closed if its time is before current bucket (or if exiting) - if (isExiting || alignedBarTime < closedThreshold) { - const bar = barsMap[ts] - // Only persist bars that have actual data (non-null close) - if (bar.close !== null) { - bar.time = alignedBarTime - closedBars.push(bar) - } - delete barsMap[ts] - } - } - - if (closedBars.length === 0) { - this.cleanArchivedBars(market) - continue - } - - // Sort by time for consistent write order - closedBars.sort((a, b) => a.time - b.time) - - const parsed = this.parseMarket(market) - if (!parsed) { - this.cleanArchivedBars(market) - continue - } - - // Write to disk (may include overwrites for backfilled buckets) - const writeResult = await this.upsertBars(parsed.exchange, parsed.symbol, timeframe, closedBars) - - // Copy closed bars to archivedBars for future backfill merging - // This allows late-arriving trades to merge into recently flushed buckets - if (!this.archivedBars[market]) { - this.archivedBars[market] = {} - } - for (const bar of closedBars) { - this.archivedBars[market][bar.time] = bar - } - - // Clean old archived bars to prevent unbounded memory growth - this.cleanArchivedBars(market) - - // Resample to higher timeframes if we wrote any data - if (writeResult && writeResult.fromTsWritten !== null) { - await this.resampleToHigherTimeframes(parsed.exchange, parsed.symbol, writeResult.fromTsWritten, writeResult.toTsWritten) - } - } - } - - /** - * Triggers resampling from base timeframe to all configured higher timeframes. - * - * @param {string} exchange - Exchange name - * @param {string} symbol - Trading pair symbol - * @param {number} fromTs - Start of dirty range (earliest modified bucket) - * @param {number} toTs - End of dirty range (latest modified bucket) - * @returns {Promise} - */ - async resampleToHigherTimeframes(exchange, symbol, fromTs, toTs) { - const baseTimeframe = this.baseTimeframe - const higherTimeframes = this.timeframes.filter(tf => tf > baseTimeframe) - - if (higherTimeframes.length === 0) return - - const basePaths = this.getFilePath(exchange, symbol, baseTimeframe) - const baseMeta = this.readMeta(basePaths.json) - if (!baseMeta || !fs.existsSync(basePaths.bin)) return - - // Resample each higher timeframe - for (const targetTimeframe of higherTimeframes) { - await this.resampleRangeToTimeframe(exchange, symbol, baseMeta, basePaths.bin, fromTs, toTs, targetTimeframe) - } - } - - /** - * Resamples a range of base timeframe data into a higher target timeframe. - * - * Unlike append-only resampling, this supports upsert semantics: - * - Overwrites existing target buckets that were affected by base changes - * - Appends new target buckets as needed - * - Maintains dense indexing (null bars fill gaps) - * - * Aggregation rules: - * - open: first non-null value in bucket (by base bar time order) - * - close: last non-null value in bucket - * - high/low: extrema across all base bars - * - volumes/counts: cumulative sums - * - * @param {string} exchange - Exchange name - * @param {string} symbol - Trading pair symbol - * @param {BinaryMeta} baseMeta - Metadata for base timeframe file - * @param {string} baseBinPath - Path to base timeframe binary file - * @param {number} fromTs - Start of dirty range (base bucket time) - * @param {number} toTs - End of dirty range (base bucket time) - * @param {number} targetTimeframe - Target timeframe in milliseconds - * @returns {Promise} - */ - async resampleRangeToTimeframe(exchange, symbol, baseMeta, baseBinPath, fromTs, toTs, targetTimeframe) { - const baseTimeframe = baseMeta.timeframeMs - const paths = this.getFilePath(exchange, symbol, targetTimeframe) - await ensureDirectoryExists(paths.bin) - - let meta = this.readMeta(paths.json) - - // Compute dirty target bucket range (inclusive) - // A target bucket is dirty if any of its contributing base buckets were modified - const firstTargetBucket = Math.floor(fromTs / targetTimeframe) * targetTimeframe - const lastTargetBucket = Math.floor(toTs / targetTimeframe) * targetTimeframe - - // Initialize metadata if this is a new file - if (!meta) { - meta = { - exchange: exchange, - symbol: symbol, - timeframe: getHms(targetTimeframe), - timeframeMs: targetTimeframe, - startTs: firstTargetBucket, - endTs: firstTargetBucket, - priceScale: PRICE_SCALE, - volumeScale: VOLUME_SCALE, - records: 0, - lastInputStartTs: firstTargetBucket - } - } - - // Skip buckets before meta.startTs (no prepend allowed) - const startBucket = Math.max(firstTargetBucket, meta.startTs) - if (startBucket > lastTargetBucket) return - - // Calculate base record index range covering the dirty target buckets - // Need to read all base bars that contribute to [startBucket, lastTargetBucket + targetTimeframe) - const baseStartIndex = Math.floor((startBucket - baseMeta.startTs) / baseTimeframe) - const baseEndIndex = Math.ceil((lastTargetBucket + targetTimeframe - baseMeta.startTs) / baseTimeframe) - - // Clamp to valid index range within the base file - const clampedStartIndex = Math.max(0, baseStartIndex) - const clampedEndIndex = Math.min(baseMeta.records, baseEndIndex) - - // Aggregate base bars into target buckets - // Key: target bucket timestamp, Value: aggregated bar data - const aggregated = {} - - if (clampedStartIndex < clampedEndIndex) { - const count = clampedEndIndex - clampedStartIndex - const byteOffset = clampedStartIndex * RECORD_SIZE - const byteLength = count * RECORD_SIZE - - let buffer - try { - const fd = fs.openSync(baseBinPath, 'r') - const stat = fs.fstatSync(fd) - const maxBytes = Math.min(byteLength, stat.size - byteOffset) - if (maxBytes > 0) { - buffer = Buffer.alloc(maxBytes) - fs.readSync(fd, buffer, 0, maxBytes, byteOffset) - } - fs.closeSync(fd) - } catch (_e) { - buffer = null - } - - if (buffer) { - const recordCount = Math.floor(buffer.length / RECORD_SIZE) - - // Process each base record and aggregate into target buckets - for (let i = 0; i < recordCount; i++) { - const record = this.readRecord(buffer, i * RECORD_SIZE, baseMeta) - const barTime = baseMeta.startTs + (clampedStartIndex + i) * baseTimeframe - - // Skip null records (gaps in base data) - if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) continue - - // Determine which target bucket this base bar contributes to - const targetBucketTime = Math.floor(barTime / targetTimeframe) * targetTimeframe - if (targetBucketTime < startBucket || targetBucketTime > lastTargetBucket) continue - - if (!aggregated[targetBucketTime]) { - // First contributing base bar for this target bucket - aggregated[targetBucketTime] = { - time: targetBucketTime, - open: record.open, - high: record.high, - low: record.low, - close: record.close, - vbuy: record.vbuy, - vsell: record.vsell, - cbuy: record.cbuy, - csell: record.csell, - lbuy: record.lbuy, - lsell: record.lsell, - _firstBarTime: barTime // Track earliest base bar for open determination - } - } else { - // Merge subsequent base bars into the aggregate - const agg = aggregated[targetBucketTime] - - // Open comes from the earliest base bar (by time, not arrival order) - if (barTime < agg._firstBarTime) { - agg.open = record.open - agg._firstBarTime = barTime - } - - // High/low are extrema, close comes from the last bar processed - agg.high = Math.max(agg.high, record.high) - agg.low = Math.min(agg.low, record.low) - agg.close = record.close - - // Volumes and counts are summed - agg.vbuy += record.vbuy - agg.vsell += record.vsell - agg.cbuy += record.cbuy - agg.csell += record.csell - agg.lbuy += record.lbuy - agg.lsell += record.lsell - } - } - } - } - - // Build complete list of bars to upsert (dense: one per target bucket in range) - const barsToUpsert = [] - for (let bucketTs = startBucket; bucketTs <= lastTargetBucket; bucketTs += targetTimeframe) { - if (aggregated[bucketTs]) { - barsToUpsert.push(aggregated[bucketTs]) - } else { - // No contributing base bars - write null bar for dense indexing - barsToUpsert.push(this.createNullBar(bucketTs)) - } - } - - if (barsToUpsert.length === 0) return - - // Separate bars into overwrites (existing records) vs appends (new records) - const overwrites = [] - const appends = [] - - for (const bar of barsToUpsert) { - const index = Math.floor((bar.time - meta.startTs) / targetTimeframe) - if (index < 0) continue // No prepend allowed - if (index < meta.records) { - overwrites.push({ index, bar }) - } else { - appends.push({ index, bar }) - } - } - - // Handle overwrites (positioned writes to existing records) - if (overwrites.length > 0) { - let fd = null - try { - // Open in read-write mode for in-place updates - fd = fs.openSync(paths.bin, 'r+') - - // Sort by index for sequential access and batch contiguous writes - overwrites.sort((a, b) => a.index - b.index) - let groupStart = 0 - - // Group contiguous indices for efficient batch writes - while (groupStart < overwrites.length) { - let groupEnd = groupStart - - // Find end of contiguous run - while (groupEnd + 1 < overwrites.length && - overwrites[groupEnd + 1].index === overwrites[groupEnd].index + 1) { - groupEnd++ - } - - // Write contiguous group in single syscall - const groupSize = groupEnd - groupStart + 1 - const buffer = Buffer.alloc(groupSize * RECORD_SIZE) - let bufferOffset = 0 - - for (let i = groupStart; i <= groupEnd; i++) { - this.writeRecord(buffer, bufferOffset, overwrites[i].bar, meta) - bufferOffset += RECORD_SIZE - } - - const fileOffset = overwrites[groupStart].index * RECORD_SIZE - fs.writeSync(fd, buffer, 0, buffer.length, fileOffset) - - groupStart = groupEnd + 1 - } - } finally { - if (fd !== null) { - fs.closeSync(fd) - } - } - } - - // Handle appends (extend file with new records) - if (appends.length > 0) { - appends.sort((a, b) => a.index - b.index) - const lastAppendIndex = appends[appends.length - 1].index - - // Build lookup map for sparse appends - const appendMap = {} - for (const a of appends) { - appendMap[a.index] = a.bar - } - - const BATCH_SIZE = 10000 - let currentIndex = meta.records - let fd = null - - try { - // Open in append mode - fd = fs.openSync(paths.bin, 'a') - - // Write records from current end to last append index (dense, filling gaps with null) - while (currentIndex <= lastAppendIndex) { - const batchRecords = [] - const batchEndIndex = Math.min(currentIndex + BATCH_SIZE, lastAppendIndex + 1) - - while (currentIndex < batchEndIndex) { - if (appendMap[currentIndex]) { - batchRecords.push(appendMap[currentIndex]) - } else { - // Fill gap with null bar - batchRecords.push(this.createNullBar(meta.startTs + currentIndex * targetTimeframe)) - } - currentIndex++ - } - - if (batchRecords.length > 0) { - const buffer = Buffer.alloc(batchRecords.length * RECORD_SIZE) - let offset = 0 - - for (const bar of batchRecords) { - this.writeRecord(buffer, offset, bar, meta) - offset += RECORD_SIZE - } - - fs.writeSync(fd, buffer) - meta.records += batchRecords.length - } - } - } finally { - if (fd !== null) { - fs.closeSync(fd) - } - } - - // Update endTs after extending file - meta.endTs = meta.startTs + meta.records * targetTimeframe - } - - meta.lastInputStartTs = lastTargetBucket - this.writeMeta(paths.json, meta) - } - - /** - * Appends bars to a binary file (append-only, no overwrites). - * - * @deprecated Use upsertBars() instead for backfill support - * @param {string} exchange - Exchange name - * @param {string} symbol - Trading pair symbol - * @param {number} timeframe - Timeframe in milliseconds - * @param {Bar[]} bars - Array of bars to append - * @returns {Promise} Timestamps of written data range - */ - async appendBars(exchange, symbol, timeframe, bars) { - const paths = this.getFilePath(exchange, symbol, timeframe) - await ensureDirectoryExists(paths.bin) - - let meta = this.readMeta(paths.json) - - // Get time bounds of input bars - const firstBarTime = Math.floor(bars[0].time / timeframe) * timeframe - const lastBarTime = Math.floor(bars[bars.length - 1].time / timeframe) * timeframe - - // Initialize metadata if new file - if (!meta) { - meta = { - exchange: exchange, - symbol: symbol, - timeframe: getHms(timeframe), - timeframeMs: timeframe, - startTs: firstBarTime, - endTs: firstBarTime, - priceScale: PRICE_SCALE, - volumeScale: VOLUME_SCALE, - records: 0, - lastInputStartTs: firstBarTime - } - } - - // Filter to only bars that can be appended (>= endTs, non-null close) - const barsToWriteMap = {} - for (const bar of bars) { - const alignedTime = Math.floor(bar.time / timeframe) * timeframe - if (alignedTime < meta.endTs) continue // Skip existing records - if (bar.close === null) continue // Skip empty bars - barsToWriteMap[alignedTime] = bar - } - - const timestamps = Object.keys(barsToWriteMap).map(Number).sort((a, b) => a - b) - if (timestamps.length === 0) return { fromTsWritten: null, toTsWritten: null } - - const lastNewBarTime = timestamps[timestamps.length - 1] - const fromTsWritten = meta.endTs - - const BATCH_SIZE = 10000 - let currentTs = meta.endTs - let fd = null - - try { - fd = fs.openSync(paths.bin, 'a') - - // Write records from endTs to lastNewBarTime (dense, with gap filling) - while (currentTs <= lastNewBarTime) { - const batchRecords = [] - const batchEnd = Math.min(currentTs + BATCH_SIZE * timeframe, lastNewBarTime + timeframe) - - while (currentTs < batchEnd && currentTs <= lastNewBarTime) { - if (barsToWriteMap[currentTs]) { - batchRecords.push(barsToWriteMap[currentTs]) - } else { - // Fill gap with null bar - batchRecords.push(this.createNullBar(currentTs)) - } - currentTs += timeframe - } - - if (batchRecords.length > 0) { - const buffer = Buffer.alloc(batchRecords.length * RECORD_SIZE) - let offset = 0 - - for (const bar of batchRecords) { - this.writeRecord(buffer, offset, bar, meta) - offset += RECORD_SIZE - } - - fs.writeSync(fd, buffer) - meta.records += batchRecords.length - } - } - } finally { - if (fd !== null) { - fs.closeSync(fd) - } - } - - // Update metadata - meta.endTs = meta.startTs + meta.records * timeframe - meta.lastInputStartTs = lastBarTime - - this.writeMeta(paths.json, meta) - - return { fromTsWritten, toTsWritten: lastNewBarTime } - } - - /** - * Upserts bars to a binary file with support for both overwrites and appends. - * - * This is the primary write method that supports backfill scenarios: - * - Bars with timestamps >= endTs are appended (with gap filling) - * - Bars with timestamps < endTs (but >= startTs) overwrite existing records - * - Bars with timestamps < startTs are ignored (no prepend) - * - * @param {string} exchange - Exchange name - * @param {string} symbol - Trading pair symbol - * @param {number} timeframe - Timeframe in milliseconds - * @param {Bar[]} bars - Array of bars to upsert - * @returns {Promise} Timestamps of written data range (includes overwrites) - */ - async upsertBars(exchange, symbol, timeframe, bars) { - const paths = this.getFilePath(exchange, symbol, timeframe) - await ensureDirectoryExists(paths.bin) - - let meta = this.readMeta(paths.json) - - // Filter out bars with null close (empty bars) - const validBars = bars.filter(b => b.close !== null) - if (validBars.length === 0) return { fromTsWritten: null, toTsWritten: null } - - // Build map of aligned timestamps to bars - const barsMap = {} - for (const bar of validBars) { - const alignedTime = Math.floor(bar.time / timeframe) * timeframe - barsMap[alignedTime] = bar - } - - const timestamps = Object.keys(barsMap).map(Number).sort((a, b) => a - b) - if (timestamps.length === 0) return { fromTsWritten: null, toTsWritten: null } - - const firstBarTime = timestamps[0] - const _lastBarTime = timestamps[timestamps.length - 1] - - // Initialize metadata if new file - if (!meta) { - meta = { - exchange: exchange, - symbol: symbol, - timeframe: getHms(timeframe), - timeframeMs: timeframe, - startTs: firstBarTime, - endTs: firstBarTime, - priceScale: PRICE_SCALE, - volumeScale: VOLUME_SCALE, - records: 0, - lastInputStartTs: firstBarTime - } - } - - // Filter out bars before startTs (no prepend allowed) - const filteredTimestamps = timestamps.filter(ts => ts >= meta.startTs) - if (filteredTimestamps.length === 0) return { fromTsWritten: null, toTsWritten: null } - - // Track the actual dirty range (for resampling) - const minTs = filteredTimestamps[0] - const maxTs = filteredTimestamps[filteredTimestamps.length - 1] - - // Categorize bars as overwrites or appends based on their index - const overwrites = [] // { index, ts, bar } - existing records to update - const appends = [] // { index, ts, bar } - new records to add - - for (const ts of filteredTimestamps) { - const index = Math.floor((ts - meta.startTs) / timeframe) - if (index < meta.records) { - // Index exists in file - this is an overwrite - overwrites.push({ index, ts, bar: barsMap[ts] }) - } else { - // Index beyond current file - this is an append - appends.push({ index, ts, bar: barsMap[ts] }) - } - } - - // Handle overwrites (positioned writes to existing records) - if (overwrites.length > 0) { - let fd = null - try { - // Open in read-write mode for in-place updates - fd = fs.openSync(paths.bin, 'r+') - - // Sort and group contiguous indices for efficient batch writes - overwrites.sort((a, b) => a.index - b.index) - let groupStart = 0 - - while (groupStart < overwrites.length) { - let groupEnd = groupStart - - // Find end of contiguous run - while (groupEnd + 1 < overwrites.length && - overwrites[groupEnd + 1].index === overwrites[groupEnd].index + 1) { - groupEnd++ - } - - // Write contiguous group in single syscall - const groupSize = groupEnd - groupStart + 1 - const buffer = Buffer.alloc(groupSize * RECORD_SIZE) - let bufferOffset = 0 - - for (let i = groupStart; i <= groupEnd; i++) { - this.writeRecord(buffer, bufferOffset, overwrites[i].bar, meta) - bufferOffset += RECORD_SIZE - } - - const fileOffset = overwrites[groupStart].index * RECORD_SIZE - fs.writeSync(fd, buffer, 0, buffer.length, fileOffset) - - groupStart = groupEnd + 1 - } - } finally { - if (fd !== null) { - fs.closeSync(fd) - } - } - } - - // Handle appends (extend file with new records) - if (appends.length > 0) { - appends.sort((a, b) => a.index - b.index) - const _firstAppendIndex = appends[0].index - const lastAppendIndex = appends[appends.length - 1].index - - // Build lookup map for sparse appends - const appendMap = {} - for (const a of appends) { - appendMap[a.index] = a.bar - } - - const BATCH_SIZE = 10000 - let currentIndex = meta.records - let fd = null - - try { - // Open in append mode - fd = fs.openSync(paths.bin, 'a') - - // Write records from current end to last append index (dense, filling gaps) - while (currentIndex <= lastAppendIndex) { - const batchRecords = [] - const batchEndIndex = Math.min(currentIndex + BATCH_SIZE, lastAppendIndex + 1) - - while (currentIndex < batchEndIndex) { - if (appendMap[currentIndex]) { - batchRecords.push(appendMap[currentIndex]) - } else { - // Fill gap with null bar for dense indexing - batchRecords.push(this.createNullBar(meta.startTs + currentIndex * timeframe)) - } - currentIndex++ - } - - if (batchRecords.length > 0) { - const buffer = Buffer.alloc(batchRecords.length * RECORD_SIZE) - let offset = 0 - - for (const bar of batchRecords) { - this.writeRecord(buffer, offset, bar, meta) - offset += RECORD_SIZE - } - - fs.writeSync(fd, buffer) - meta.records += batchRecords.length - } - } - } finally { - if (fd !== null) { - fs.closeSync(fd) - } - } - - // Update endTs after extending file - meta.endTs = meta.startTs + meta.records * timeframe - } - - meta.lastInputStartTs = maxTs - this.writeMeta(paths.json, meta) - - // Return the full dirty range (includes overwrites, not just appends) - return { fromTsWritten: minTs, toTsWritten: maxTs } - } - - /** - * Creates an empty (null) bar for gap filling. - * Null bars have all OHLC set to null and all volumes/counts set to 0. - * On disk, these are stored as all zeros and skipped when reading. - * - * @param {number} time - Bucket timestamp - * @returns {Object} Null bar object - */ - createNullBar(time) { - return { - time, - open: null, - high: null, - low: null, - close: null, - vbuy: 0, - vsell: 0, - cbuy: 0, - csell: 0, - lbuy: 0, - lsell: 0 - } - } - - /** - * Writes a bar record to a buffer at the specified offset. - * - * Binary layout (56 bytes): - * - Bytes 0-3: open (Int32LE, scaled by priceScale) - * - Bytes 4-7: high (Int32LE, scaled) - * - Bytes 8-11: low (Int32LE, scaled) - * - Bytes 12-15: close (Int32LE, scaled) - * - Bytes 16-23: vbuy (BigInt64LE, scaled by volumeScale) - * - Bytes 24-31: vsell (BigInt64LE, scaled) - * - Bytes 32-35: cbuy (UInt32LE, trade count) - * - Bytes 36-39: csell (UInt32LE, trade count) - * - Bytes 40-47: lbuy (BigInt64LE, liquidation volume, scaled) - * - Bytes 48-55: lsell (BigInt64LE, liquidation volume, scaled) - * - * @param {Buffer} buffer - Target buffer - * @param {number} offset - Byte offset in buffer - * @param {Object} bar - Bar data to write - * @param {BinaryMeta} meta - Metadata with scale factors - */ - writeRecord(buffer, offset, bar, meta) { - // Scale and round prices (null becomes 0) - const open = bar.open !== null ? Math.round(bar.open * meta.priceScale) : 0 - const high = bar.high !== null ? Math.round(bar.high * meta.priceScale) : 0 - const low = bar.low !== null ? Math.round(bar.low * meta.priceScale) : 0 - const close = bar.close !== null ? Math.round(bar.close * meta.priceScale) : 0 - - // Write OHLC - buffer.writeInt32LE(open, offset) - buffer.writeInt32LE(high, offset + 4) - buffer.writeInt32LE(low, offset + 8) - buffer.writeInt32LE(close, offset + 12) - - // Scale and write volumes as BigInt - const vbuy = BigInt(Math.round((bar.vbuy || 0) * meta.volumeScale)) - const vsell = BigInt(Math.round((bar.vsell || 0) * meta.volumeScale)) - buffer.writeBigInt64LE(vbuy, offset + 16) - buffer.writeBigInt64LE(vsell, offset + 24) - - // Write trade counts - buffer.writeUInt32LE(bar.cbuy || 0, offset + 32) - buffer.writeUInt32LE(bar.csell || 0, offset + 36) - - // Scale and write liquidation volumes - const lbuy = BigInt(Math.round((bar.lbuy || 0) * meta.volumeScale)) - const lsell = BigInt(Math.round((bar.lsell || 0) * meta.volumeScale)) - buffer.writeBigInt64LE(lbuy, offset + 40) - buffer.writeBigInt64LE(lsell, offset + 48) - } - - /** - * Reads a bar record from a buffer at the specified offset. - * - * @param {Buffer} buffer - Source buffer - * @param {number} offset - Byte offset in buffer - * @param {BinaryMeta} meta - Metadata with scale factors - * @returns {RecordData} Decoded bar data - */ - readRecord(buffer, offset, meta) { - // Read and descale OHLC - const open = buffer.readInt32LE(offset) / meta.priceScale - const high = buffer.readInt32LE(offset + 4) / meta.priceScale - const low = buffer.readInt32LE(offset + 8) / meta.priceScale - const close = buffer.readInt32LE(offset + 12) / meta.priceScale - - // Read and descale volumes - const vbuy = Number(buffer.readBigInt64LE(offset + 16)) / meta.volumeScale - const vsell = Number(buffer.readBigInt64LE(offset + 24)) / meta.volumeScale - - // Read trade counts - const cbuy = buffer.readUInt32LE(offset + 32) - const csell = buffer.readUInt32LE(offset + 36) - - // Read and descale liquidation volumes - const lbuy = Number(buffer.readBigInt64LE(offset + 40)) / meta.volumeScale - const lsell = Number(buffer.readBigInt64LE(offset + 48)) / meta.volumeScale - - return { open, high, low, close, vbuy, vsell, cbuy, csell, lbuy, lsell } - } - - /** - * Reads metadata JSON file for a market/timeframe. - * - * @param {string} jsonPath - Path to the JSON metadata file - * @returns {BinaryMeta|null} Parsed metadata or null if not found - */ - readMeta(jsonPath) { - try { - if (!fs.existsSync(jsonPath)) return null - const content = fs.readFileSync(jsonPath, 'utf8') - return JSON.parse(content) - } catch (_e) { - return null - } - } - - /** - * Writes metadata JSON file atomically (write to temp, then rename). - * - * @param {string} jsonPath - Path to the JSON metadata file - * @param {BinaryMeta} meta - Metadata to write - */ - writeMeta(jsonPath, meta) { - const tmpPath = jsonPath + '.tmp' - fs.writeFileSync(tmpPath, JSON.stringify(meta, null, 2)) - fs.renameSync(tmpPath, jsonPath) - } - - /** - * Fetches historical OHLCV data for one or more markets. - * - * Returns data in a columnar format suitable for charting. - * Null bars (gaps) are skipped in the response. - * Pending bars near "now" are injected to provide real-time data. - * - * @param {Object} params - Query parameters - * @param {number} params.from - Start timestamp (inclusive) - * @param {number} params.to - End timestamp (exclusive) - * @param {number} [params.timeframe=60000] - Timeframe in milliseconds - * @param {string[]} [params.markets=[]] - Array of market identifiers - * @returns {Promise<{format: string, columns: Object, results: Array[]}>} Query results - */ - fetch({ from, to, timeframe = 60000, markets = [] }) { - // Validate timeframe is supported - const supported = this.timeframes.indexOf(timeframe) !== -1 - if (!supported) { - return Promise.reject(new Error(`Unsupported timeframe: ${getHms(timeframe)}`)) - } - - // Column index mapping for result arrays (matches InfluxDB format) - const columns = { - time: 0, - market: 1, - open: 2, - high: 3, - low: 4, - close: 5, - vbuy: 6, - vsell: 7, - cbuy: 8, - csell: 9, - lbuy: 10, - lsell: 11 - } - - // Pre-allocate with estimated capacity to reduce array resizing - const results = [] - - // Check if we need pending bars (query extends to near-realtime) - const now = Date.now() - const needsPendingBars = to > now - config.influxResampleInterval - - // Read data from disk for each market - for (const market of markets) { - const parsed = this.parseMarket(market) - if (!parsed) continue - - const paths = this.getFilePath(parsed.exchange, parsed.symbol, timeframe) - const meta = this.readMeta(paths.json) - if (!meta) continue - - if (!fs.existsSync(paths.bin)) continue - - // Calculate index range for requested time window - let startIndex = Math.floor((from - meta.startTs) / meta.timeframeMs) - let endIndex = Math.ceil((to - meta.startTs) / meta.timeframeMs) - - // Clamp to valid range - if (startIndex < 0) startIndex = 0 - if (endIndex > meta.records) endIndex = meta.records - if (startIndex >= endIndex) continue - - const count = endIndex - startIndex - const byteOffset = startIndex * RECORD_SIZE - const byteLength = count * RECORD_SIZE - - // Read the range from disk - let buffer - let fd - try { - fd = fs.openSync(paths.bin, 'r') - const stat = fs.fstatSync(fd) - const maxBytes = Math.min(byteLength, stat.size - byteOffset) - if (maxBytes <= 0) { - fs.closeSync(fd) - continue - } - buffer = Buffer.alloc(maxBytes) - fs.readSync(fd, buffer, 0, maxBytes, byteOffset) - fs.closeSync(fd) - } catch (_e) { - if (fd !== undefined) { - try { fs.closeSync(fd) } catch (_) { /* ignored */ } - } - continue - } - - // Decode records and add to results - const recordCount = Math.floor(buffer.length / RECORD_SIZE) - for (let i = 0; i < recordCount; i++) { - const record = this.readRecord(buffer, i * RECORD_SIZE, meta) - - // Skip null bars (gaps) - check before computing barTime for efficiency - if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) continue - - const barTime = meta.startTs + (startIndex + i) * meta.timeframeMs - - // Filter by time range - if (barTime < from || barTime >= to) continue - - // Add as array matching column order (includes market) - // Time is converted to seconds to match InfluxDB format (precision: 's') - results.push([ - Math.floor(barTime / 1000), - market, - record.open, - record.high, - record.low, - record.close, - record.vbuy, - record.vsell, - record.cbuy, - record.csell, - record.lbuy, - record.lsell - ]) - } - } - - // Inject pending bars for near-realtime data - if (needsPendingBars) { - const pendingBars = this.getPendingBars(markets, from, to) - for (let i = 0; i < pendingBars.length; i++) { - const bar = pendingBars[i] - // Time is converted to seconds to match InfluxDB format (precision: 's') - results.push([ - Math.floor(bar.time / 1000), - bar.market, - bar.open, - bar.high, - bar.low, - bar.close, - bar.vbuy, - bar.vsell, - bar.cbuy, - bar.csell, - bar.lbuy, - bar.lsell - ]) - } - } - - // Single sort at the end (by time) - // For large datasets, this is O(n log n) - unavoidable when merging multiple sources - if (markets.length > 1 || needsPendingBars) { - results.sort((a, b) => a[0] - b[0]) - } - - return Promise.resolve({ - format: this.format, - columns, - results - }) - } - - /** - * Gets pending (in-memory, not yet flushed) bars for given markets and time range. - * - * Used by fetch() to inject real-time data for queries near "now". - * Returns base timeframe bars as-is (no resampling - matches InfluxDB behavior). - * - * @param {string[]} markets - Array of market identifiers to include - * @param {number} from - Start timestamp (inclusive) - * @param {number} to - End timestamp (exclusive) - * @returns {Object[]} Array of pending bar objects with market field - */ - getPendingBars(markets, from, to) { - const results = [] - - for (const market of markets) { - const barsMap = this.pendingBars[market] - if (!barsMap) continue - - for (const ts in barsMap) { - const bar = barsMap[ts] - - // Skip empty bars - if (bar.close === null) continue - - const barTime = Number(ts) - - // Filter by requested time range - if (barTime < from || barTime >= to) continue - - results.push(bar) - } - } - - return results - } -} - -module.exports = BinariesStorage diff --git a/src/storage/binaries/README.md b/src/storage/binaries/README.md new file mode 100644 index 0000000..16316f7 --- /dev/null +++ b/src/storage/binaries/README.md @@ -0,0 +1,319 @@ +# Binaries Storage + +A high-performance binary storage engine for OHLCV (Open, High, Low, Close, Volume) trade data. Designed as a drop-in replacement for InfluxDB storage with significant performance and simplicity benefits. + +## Overview + +Binaries Storage persists aggregated trade data in dense binary files with fixed-size records. Each market (e.g., `COINBASE:BTC-USD`) has separate files for each timeframe (e.g., `10s.bin`, `1m.bin`, `1h.bin`). + +### Key Features + +- **Dense indexing**: Records are stored contiguously with null bars filling gaps +- **Upsert semantics**: Can overwrite existing records or append new ones +- **Automatic resampling**: Base timeframe data is aggregated to higher timeframes +- **Backfill support**: Late-arriving trades merge into existing buckets +- **Arrival-order semantics**: Matches InfluxDB's behavior for consistency + +## File Structure + +``` +data/ +└── COINBASE/ + └── BTC-USD/ + ├── 10s.bin # Binary data (56 bytes per record) + ├── 10s.json # Metadata (startTs, endTs, records, etc.) + ├── 1m.bin + ├── 1m.json + ├── 1h.bin + └── 1h.json +``` + +## Module Structure + +``` +src/storage/binaries/ +├── index.js # Main BinariesStorage class +├── constants.js # Record size, scale factors, typedefs +├── io.js # File I/O operations +├── resample.js # Timeframe resampling logic +└── write.js # Upsert and append operations +``` + +## Data Flow + +### High-Level Architecture + +```mermaid +flowchart TB + subgraph Input + T[Trades from Exchanges] + end + + subgraph Memory ["In-Memory Caches"] + PB[pendingBars
Current unflushed bars] + AB[archivedBars
Recently flushed bars] + end + + subgraph Disk ["Disk Storage"] + BF[Binary Files
*.bin] + MF[Metadata Files
*.json] + end + + subgraph Output + API[fetch API
Historical queries] + end + + T -->|save| PB + PB -->|flush| BF + PB -->|copy on flush| AB + AB -.->|backfill merge| PB + BF -.->|disk fallback| PB + BF -->|read| API + PB -->|inject realtime| API +``` + +### Trade Processing Flow + +```mermaid +flowchart TD + subgraph save["save(trades, isExiting)"] + A[Receive trades] --> B[processTrades] + B --> C{Flush interval?} + C -->|Yes| D[flush] + C -->|No| E[Return] + D --> E + end + + subgraph processTrades["processTrades(trades)"] + P1[For each trade] --> P2[Floor timestamp to bucket] + P2 --> P3{Bar exists in
pendingBars?} + P3 -->|Yes| P7[Use existing bar] + P3 -->|No| P4{Bar exists in
archivedBars?} + P4 -->|Yes| P5[Restore to pendingBars] + P4 -->|No| P6{Bar exists
on disk?} + P6 -->|Yes| P5 + P6 -->|No| P8[Create new bar] + P5 --> P7 + P8 --> P7 + P7 --> P9[Merge trade into bar] + P9 --> P10{More trades?} + P10 -->|Yes| P1 + P10 -->|No| P11[Update price indexes] + end + + subgraph flush["flush(isExiting)"] + F1[For each market] --> F2[Collect closed bars] + F2 --> F3[Sort by time] + F3 --> F4[upsertBars to disk] + F4 --> F5[Copy to archivedBars] + F5 --> F6[cleanArchivedBars] + F6 --> F7[resampleToHigherTimeframes] + F7 --> F8{More markets?} + F8 -->|Yes| F1 + F8 -->|No| F9[Done] + end +``` + +### Upsert Operation + +```mermaid +flowchart LR + subgraph upsertBars["upsertBars(exchange, symbol, timeframe, bars)"] + U1[Filter valid bars] --> U2[Categorize by index] + U2 --> U3{index < records?} + U3 -->|Yes| U4[Overwrites list] + U3 -->|No| U5[Appends list] + U4 --> U6[Batch contiguous writes] + U5 --> U7[Append with gap filling] + U6 --> U8[Update metadata] + U7 --> U8 + end +``` + +### Resampling Flow + +```mermaid +flowchart TD + subgraph resample["resampleToHigherTimeframes"] + R1[Get dirty range
fromTs → toTs] --> R2[For each higher TF] + R2 --> R3[resampleRangeToTimeframe] + R3 --> R4[Read base TF records] + R4 --> R5[Aggregate into target buckets] + R5 --> R6[Upsert target TF file] + R6 --> R7{More timeframes?} + R7 -->|Yes| R2 + R7 -->|No| R8[Done] + end + + subgraph aggregate["Aggregation Rules"] + A1["open = first bar's open (by time)"] + A2["high = max(all highs)"] + A3["low = min(all lows)"] + A4["close = last bar's close"] + A5["volumes = sum(all volumes)"] + A6["counts = sum(all counts)"] + end +``` + +### Fetch Query Flow + +```mermaid +flowchart TD + subgraph fetch["fetch({ from, to, timeframe, markets })"] + Q1[Validate timeframe] --> Q2[For each market] + Q2 --> Q3[Read metadata] + Q3 --> Q4[Calculate index range] + Q4 --> Q5[Read binary records] + Q5 --> Q6[Decode & filter] + Q6 --> Q7[Add to results] + Q7 --> Q8{More markets?} + Q8 -->|Yes| Q2 + Q8 -->|No| Q9{Need pending bars?} + Q9 -->|Yes| Q10[Inject pendingBars] + Q9 -->|No| Q11[Sort by time] + Q10 --> Q11 + Q11 --> Q12[Return results] + end +``` + +## Binary Record Format + +Each record is exactly **56 bytes**: + +| Offset | Size | Type | Field | Description | +|--------|------|------|-------|-------------| +| 0 | 4 | Int32LE | open | Open price × 10,000 | +| 4 | 4 | Int32LE | high | High price × 10,000 | +| 8 | 4 | Int32LE | low | Low price × 10,000 | +| 12 | 4 | Int32LE | close | Close price × 10,000 | +| 16 | 8 | BigInt64LE | vbuy | Buy volume × 1,000,000 | +| 24 | 8 | BigInt64LE | vsell | Sell volume × 1,000,000 | +| 32 | 4 | UInt32LE | cbuy | Buy trade count | +| 36 | 4 | UInt32LE | csell | Sell trade count | +| 40 | 8 | BigInt64LE | lbuy | Buy liquidation volume × 1,000,000 | +| 48 | 8 | BigInt64LE | lsell | Sell liquidation volume × 1,000,000 | + +### Scale Factors + +- **PRICE_SCALE = 10,000**: Prices stored with 4 decimal places precision +- **VOLUME_SCALE = 1,000,000**: Volumes stored with 6 decimal places precision + +### Null Bars + +Gaps in data are represented as "null bars" where all OHLC values are 0. These maintain dense indexing (record position = time offset) but are skipped when reading. + +## Metadata Format + +Each `.json` file contains: + +```json +{ + "exchange": "COINBASE", + "symbol": "BTC-USD", + "timeframe": "10s", + "timeframeMs": 10000, + "startTs": 1704067200000, + "endTs": 1704153600000, + "priceScale": 10000, + "volumeScale": 1000000, + "records": 8640, + "lastInputStartTs": 1704153590000 +} +``` + +## Memory Caches + +### pendingBars + +In-memory buffer of bars not yet written to disk. Structure: + +```javascript +{ + "COINBASE:BTC-USD": { + 1704067200000: { time, open, high, low, close, vbuy, vsell, cbuy, csell, lbuy, lsell }, + 1704067210000: { ... } + } +} +``` + +### archivedBars + +Cache of recently flushed bars (last ~100 buckets). Enables backfill merging when late trades arrive for already-persisted buckets. + +## Merge Semantics + +When multiple trades arrive for the same bucket: + +| Field | Rule | Description | +|-------|------|-------------| +| open | **Sticky** | Once set, never changes (first trade wins) | +| high | Max | Maximum of all trade prices | +| low | Min | Minimum of all trade prices | +| close | **Last write wins** | Updates with each trade | +| volumes | Sum | Cumulative sum | +| counts | Sum | Cumulative sum | + +## Backfill Support + +Handles reconnection scenarios where older trades arrive after newer ones: + +1. **pendingBars lookup**: Check if bucket exists in memory +2. **archivedBars lookup**: Check recently-flushed cache +3. **Disk fallback**: Read single record from binary file +4. **Create new**: Initialize empty bar if bucket doesn't exist + +This ensures trades are never lost, even during exchange reconnections. + +## Configuration + +Relevant config options (from `config.json`): + +```javascript +{ + "storage": "binaries", // Enable binaries storage + "filesLocation": "./data", // Data directory + "influxTimeframe": 10000, // Base timeframe (10s) + "influxResampleTo": [ // Higher timeframes to generate + 15000, 30000, 60000, // 15s, 30s, 1m + 180000, 300000, 900000, // 3m, 5m, 15m + 1800000, 3600000, 7200000, // 30m, 1h, 2h + 14400000, 21600000, 86400000 // 4h, 6h, 1d + ], + "influxResampleInterval": 60000, // Flush/resample interval (1m) + "backupInterval": 60000 // Backup interval (1m) +} +``` + +## API Compatibility + +The `fetch()` method returns data in the same format as InfluxDB storage: + +```javascript +{ + format: "point", + columns: { time: 0, market: 1, open: 2, high: 3, low: 4, close: 5, vbuy: 6, vsell: 7, cbuy: 8, csell: 9, lbuy: 10, lsell: 11 }, + results: [ + [1704067200, "COINBASE:BTC-USD", 42000.5, 42100.0, 41900.0, 42050.0, 1234.56, 987.65, 150, 120, 0, 0], + // ... more rows + ] +} +``` + +**Note**: Time is returned in **seconds** (not milliseconds) to match InfluxDB's `precision: 's'` setting. + +## Performance Characteristics + +| Operation | Complexity | Notes | +|-----------|------------|-------| +| Write (append) | O(n) | Sequential append, batched | +| Write (overwrite) | O(n) | Positioned writes, batched by contiguous indices | +| Read (fetch) | O(n) | Sequential read, memory-mapped friendly | +| Resample | O(n) | Reads base TF, writes target TF | + +### Advantages over InfluxDB + +- **No external service**: Runs in-process, no network overhead +- **Simpler deployment**: Just files on disk +- **Faster writes**: Direct file I/O vs HTTP API +- **Predictable performance**: No query planning overhead +- **Lower memory**: Stream processing vs in-memory aggregation diff --git a/src/storage/binaries/constants.js b/src/storage/binaries/constants.js new file mode 100644 index 0000000..6da8b21 --- /dev/null +++ b/src/storage/binaries/constants.js @@ -0,0 +1,99 @@ +/** + * Size in bytes of a single OHLCV record in the binary file. + * Layout (56 bytes total): + * - open: Int32LE (4 bytes, offset 0) + * - high: Int32LE (4 bytes, offset 4) + * - low: Int32LE (4 bytes, offset 8) + * - close: Int32LE (4 bytes, offset 12) + * - vbuy: BigInt64LE (8 bytes, offset 16) + * - vsell: BigInt64LE (8 bytes, offset 24) + * - cbuy: UInt32LE (4 bytes, offset 32) + * - csell: UInt32LE (4 bytes, offset 36) + * - lbuy: BigInt64LE (8 bytes, offset 40) + * - lsell: BigInt64LE (8 bytes, offset 48) + * @constant {number} + */ +const RECORD_SIZE = 56 + +/** + * Scale factor for storing prices as integers. + * Prices are multiplied by this value before storing and divided when reading. + * Example: price 45000.1234 becomes 450001234 (stored as Int32). + * @constant {number} + */ +const PRICE_SCALE = 10000 + +/** + * Scale factor for storing volumes as integers. + * Volumes are multiplied by this value before storing and divided when reading. + * @constant {number} + */ +const VOLUME_SCALE = 1000000 + +/** + * @typedef {Object} BinaryMeta + * @property {string} exchange - Exchange name + * @property {string} symbol - Trading pair symbol + * @property {string} timeframe - Human-readable timeframe label (e.g., "5s", "1m") + * @property {number} timeframeMs - Timeframe in milliseconds + * @property {number} startTs - Timestamp of the first record (bucket time) + * @property {number} endTs - Timestamp just past the last record (startTs + records * timeframeMs) + * @property {number} priceScale - Price scaling factor used + * @property {number} volumeScale - Volume scaling factor used + * @property {number} records - Total number of records in the binary file + * @property {number} lastInputStartTs - Last input bar timestamp processed + */ + +/** + * @typedef {Object} FilePaths + * @property {string} bin - Path to the binary data file + * @property {string} json - Path to the JSON metadata file + */ + +/** + * @typedef {Object} ParsedMarket + * @property {string} exchange - Exchange name + * @property {string} symbol - Trading pair symbol + */ + +/** + * @typedef {Object} WriteResult + * @property {number|null} fromTsWritten - Minimum timestamp written (null if nothing written) + * @property {number|null} toTsWritten - Maximum timestamp written (null if nothing written) + */ + +/** + * @typedef {Object} RecordData + * @property {number} open - Open price + * @property {number} high - High price + * @property {number} low - Low price + * @property {number} close - Close price + * @property {number} vbuy - Buy volume (price * size) + * @property {number} vsell - Sell volume (price * size) + * @property {number} cbuy - Buy trade count + * @property {number} csell - Sell trade count + * @property {number} lbuy - Buy liquidation volume + * @property {number} lsell - Sell liquidation volume + */ + +/** + * @typedef {Object} Bar + * @property {number} time - Bucket timestamp + * @property {string} [market] - Market identifier + * @property {number|null} open - Open price + * @property {number|null} high - High price + * @property {number|null} low - Low price + * @property {number|null} close - Close price + * @property {number} vbuy - Buy volume + * @property {number} vsell - Sell volume + * @property {number} cbuy - Buy trade count + * @property {number} csell - Sell trade count + * @property {number} lbuy - Buy liquidation volume + * @property {number} lsell - Sell liquidation volume + */ + +module.exports = { + RECORD_SIZE, + PRICE_SCALE, + VOLUME_SCALE +} diff --git a/src/storage/binaries/index.js b/src/storage/binaries/index.js new file mode 100644 index 0000000..e8e19fa --- /dev/null +++ b/src/storage/binaries/index.js @@ -0,0 +1,525 @@ +const fs = require('fs') +const config = require('../../config') +const alertService = require('../../services/alert') +const { updateIndexes } = require('../../services/connections') +const { getHms } = require('../../helper') + +const { RECORD_SIZE } = require('./constants') +const { getFilePath, parseMarket, readRecord, readMeta, readSingleRecordAtTs } = require('./io') +const { upsertBars } = require('./write') +const { resampleToHigherTimeframes } = require('./resample') + +/** + * Binary storage engine for OHLCV data. + * + * Stores aggregated trade data in dense binary files with fixed-size records. + * Each market (exchange:symbol) has separate files per timeframe. + * + * Key features: + * - Dense indexing: records are stored contiguously, with null bars filling gaps + * - Upsert semantics: can overwrite existing records or append new ones + * - Automatic resampling: base timeframe data is aggregated to higher timeframes + * - Backfill support: older trades can be merged into existing buckets + * + * Data flow: + * 1. Trades arrive via save() → processTrades() aggregates into pendingBars + * 2. On flush interval, closed bars are persisted via upsertBars() + * 3. After base TF write, higher TFs are resampled via resampleToHigherTimeframes() + * + * @class + */ +class BinariesStorage { + /** + * Creates a new BinariesStorage instance. + * Initializes in-memory caches and ensures the data directory exists. + */ + constructor() { + /** @type {string} */ + this.name = this.constructor.name + + /** @type {string} */ + this.format = 'point' + + /** + * In-memory cache of bars not yet persisted (current bucket and recent unflushed). + * Structure: { [market: string]: { [timestamp: number]: Bar } } + * @type {Object>} + */ + this.pendingBars = {} + + /** + * Cache of recently persisted bars for backfill merging. + * Allows late-arriving trades to merge into already-flushed buckets. + * Structure: { [market: string]: { [timestamp: number]: Bar } } + * @type {Object>} + */ + this.archivedBars = {} + + /** + * Base timeframe in milliseconds (smallest granularity stored). + * @type {number} + */ + this.baseTimeframe = config.influxTimeframe + + /** + * All supported timeframes (base + resampled targets). + * @type {number[]} + */ + this.timeframes = [config.influxTimeframe].concat(config.influxResampleTo || []) + + // Ensure the data directory exists + if (!fs.existsSync(config.filesLocation)) { + fs.mkdirSync(config.filesLocation, { recursive: true }) + } + } + + /** + * Removes old entries from the archivedBars cache for a given market. + * Keeps only the last (baseTimeframe * 100) milliseconds worth of buckets. + * This prevents unbounded memory growth while allowing reasonable backfill windows. + * + * @param {string} market - Market identifier (e.g., "COINBASE:BTC-USD") + */ + cleanArchivedBars(market) { + const barsMap = this.archivedBars[market] + if (!barsMap) return + + const keepWindow = this.baseTimeframe * 100 + const now = Date.now() + const cutoff = Math.floor(now / this.baseTimeframe) * this.baseTimeframe - keepWindow + + for (const ts in barsMap) { + if (Number(ts) < cutoff) { + delete barsMap[ts] + } + } + } + + /** + * Main entry point for saving trades. + * Aggregates trades into pending bars and flushes to disk on interval. + * + * @param {import('../../typedef').Trade[]} trades - Array of trade objects to process + * @param {boolean} isExiting - If true, forces immediate flush (used on shutdown) + * @returns {Promise} + */ + async save(trades, isExiting) { + if (!trades || !trades.length) { + return Promise.resolve() + } + + // Aggregate trades into pending bars + this.processTrades(trades) + + // Check if it's time to flush (backup interval aligned with resample interval) + const now = Date.now() + const timeBackupFloored = Math.floor(now / config.backupInterval) * config.backupInterval + const timeMinuteFloored = Math.floor(now / config.influxResampleInterval) * config.influxResampleInterval + + if (isExiting || timeBackupFloored === timeMinuteFloored) { + await this.flush(isExiting) + } + } + + /** + * Processes an array of trades and aggregates them into pending bars. + * + * Bar lookup priority (for backfill support): + * 1. pendingBars - already in memory, not yet flushed + * 2. archivedBars - recently flushed, cached for merging + * 3. Disk fallback - read single record from binary file + * 4. Create new empty bar + * + * Merge semantics: + * - "Sticky open": open price is only set when null (first trade wins) + * - "Last write wins close": close updates as trades arrive + * - high/low: extrema of all trades + * - volumes/counts: cumulative sums + * + * @param {import('../../typedef').Trade[]} trades - Array of trade objects + */ + processTrades(trades) { + /** @type {Object} */ + const ranges = {} + const timeframe = this.baseTimeframe + + for (let i = 0; i < trades.length; i++) { + const trade = trades[i] + const market = trade.exchange + ':' + trade.pair + + // Track price ranges for alert crossover checks (non-liquidation trades only) + if (!trade.liquidation) { + if (!ranges[market]) { + ranges[market] = { low: trade.price, high: trade.price, close: trade.price } + } else { + ranges[market].low = Math.min(ranges[market].low, trade.price) + ranges[market].high = Math.max(ranges[market].high, trade.price) + ranges[market].close = trade.price + } + } + + // Floor trade timestamp to bucket boundary + const tradeFlooredTime = Math.floor(trade.timestamp / timeframe) * timeframe + + // Ensure market maps exist + if (!this.pendingBars[market]) { + this.pendingBars[market] = {} + } + if (!this.archivedBars[market]) { + this.archivedBars[market] = {} + } + + // Priority 1: Check pendingBars + let bar = this.pendingBars[market][tradeFlooredTime] + + // Priority 2: Check archivedBars cache + if (!bar) { + bar = this.archivedBars[market][tradeFlooredTime] + if (bar) { + // Re-add to pendingBars so it can accumulate more trades + this.pendingBars[market][tradeFlooredTime] = bar + } + } + + // Priority 3: Disk fallback (only for base timeframe) + if (!bar) { + // Try disk fallback for base timeframe only + const parsed = parseMarket(market) + if (parsed) { + const diskBar = readSingleRecordAtTs(parsed.exchange, parsed.symbol, timeframe, tradeFlooredTime) + if (diskBar) { + bar = { + time: tradeFlooredTime, + market: market, + cbuy: diskBar.cbuy, + csell: diskBar.csell, + vbuy: diskBar.vbuy, + vsell: diskBar.vsell, + lbuy: diskBar.lbuy, + lsell: diskBar.lsell, + open: diskBar.open, + high: diskBar.high, + low: diskBar.low, + close: diskBar.close + } + this.archivedBars[market][tradeFlooredTime] = bar + this.pendingBars[market][tradeFlooredTime] = bar + } + } + } + + if (!bar) { + bar = this.pendingBars[market][tradeFlooredTime] = { + time: tradeFlooredTime, + market: market, + cbuy: 0, + csell: 0, + vbuy: 0, + vsell: 0, + lbuy: 0, + lsell: 0, + open: null, + high: null, + low: null, + close: null + } + } + + if (trade.liquidation) { + bar['l' + trade.side] += trade.price * trade.size + } else { + // Sticky open: only set if null + if (bar.open === null) { + bar.open = bar.high = bar.low = bar.close = +trade.price + } else { + // Update high/low extrema and close price + bar.high = Math.max(bar.high, +trade.price) + bar.low = Math.min(bar.low, +trade.price) + bar.close = +trade.price + } + // Increment trade count and volume for the appropriate side + bar['c' + trade.side] += trade.count || 1 + bar['v' + trade.side] += trade.price * trade.size + } + } + + // Update price indexes and check for alert crossovers + updateIndexes(ranges, async (index, high, low, direction) => { + if (alertService && alertService.checkPriceCrossover) { + await alertService.checkPriceCrossover(index, high, low, direction) + } + }) + } + + /** + * Flushes closed pending bars to disk. + * + * A bar is "closed" when its bucket time is before the current bucket threshold. + * Closed bars are: + * 1. Written to disk via upsertBars() + * 2. Copied to archivedBars for future backfill merging + * 3. Removed from pendingBars + * + * After base timeframe write, triggers resampling to higher timeframes. + * + * @param {boolean} isExiting - If true, flushes all bars regardless of close status + * @returns {Promise} + */ + async flush(isExiting) { + const now = Date.now() + const timeframe = this.baseTimeframe + // Current bucket start time - bars before this are considered "closed" + const closedThreshold = Math.floor(now / timeframe) * timeframe + + for (const market in this.pendingBars) { + const barsMap = this.pendingBars[market] + const closedBars = [] + + // Collect closed bars and remove from pending + for (const ts in barsMap) { + const barTime = Number(ts) + const alignedBarTime = Math.floor(barTime / timeframe) * timeframe + + // Bar is closed if its time is before current bucket (or if exiting) + if (isExiting || alignedBarTime < closedThreshold) { + const bar = barsMap[ts] + // Only persist bars that have actual data (non-null close) + if (bar.close !== null) { + bar.time = alignedBarTime + closedBars.push(bar) + } + delete barsMap[ts] + } + } + + if (closedBars.length === 0) { + this.cleanArchivedBars(market) + continue + } + + // Sort by time for consistent write order + closedBars.sort((a, b) => a.time - b.time) + + const parsed = parseMarket(market) + if (!parsed) { + this.cleanArchivedBars(market) + continue + } + + // Write to disk (may include overwrites for backfilled buckets) + const writeResult = await upsertBars(parsed.exchange, parsed.symbol, timeframe, closedBars) + + // Copy closed bars to archivedBars for future backfill merging + // This allows late-arriving trades to merge into recently flushed buckets + if (!this.archivedBars[market]) { + this.archivedBars[market] = {} + } + for (const bar of closedBars) { + this.archivedBars[market][bar.time] = bar + } + + // Clean old archived bars to prevent unbounded memory growth + this.cleanArchivedBars(market) + + // Resample to higher timeframes if we wrote any data + if (writeResult && writeResult.fromTsWritten !== null) { + await resampleToHigherTimeframes(this, parsed.exchange, parsed.symbol, writeResult.fromTsWritten, writeResult.toTsWritten) + } + } + } + + /** + * Fetches historical OHLCV data for one or more markets. + * + * Returns data in a columnar format suitable for charting. + * Null bars (gaps) are skipped in the response. + * Pending bars near "now" are injected to provide real-time data. + * + * @param {Object} params - Query parameters + * @param {number} params.from - Start timestamp (inclusive) + * @param {number} params.to - End timestamp (exclusive) + * @param {number} [params.timeframe=60000] - Timeframe in milliseconds + * @param {string[]} [params.markets=[]] - Array of market identifiers + * @returns {Promise<{format: string, columns: Object, results: Array[]}>} Query results + */ + fetch({ from, to, timeframe = 60000, markets = [] }) { + // Validate timeframe is supported + const supported = this.timeframes.indexOf(timeframe) !== -1 + if (!supported) { + return Promise.reject(new Error(`Unsupported timeframe: ${getHms(timeframe)}`)) + } + + // Column index mapping for result arrays (matches InfluxDB format) + const columns = { + time: 0, + market: 1, + open: 2, + high: 3, + low: 4, + close: 5, + vbuy: 6, + vsell: 7, + cbuy: 8, + csell: 9, + lbuy: 10, + lsell: 11 + } + + // Pre-allocate with estimated capacity to reduce array resizing + const results = [] + + // Check if we need pending bars (query extends to near-realtime) + const now = Date.now() + const needsPendingBars = to > now - config.influxResampleInterval + + // Read data from disk for each market + for (const market of markets) { + const parsed = parseMarket(market) + if (!parsed) continue + + const paths = getFilePath(parsed.exchange, parsed.symbol, timeframe) + const meta = readMeta(paths.json) + if (!meta) continue + + if (!fs.existsSync(paths.bin)) continue + + // Calculate index range for requested time window + let startIndex = Math.floor((from - meta.startTs) / meta.timeframeMs) + let endIndex = Math.ceil((to - meta.startTs) / meta.timeframeMs) + + // Clamp to valid range + if (startIndex < 0) startIndex = 0 + if (endIndex > meta.records) endIndex = meta.records + if (startIndex >= endIndex) continue + + const count = endIndex - startIndex + const byteOffset = startIndex * RECORD_SIZE + const byteLength = count * RECORD_SIZE + + // Read the range from disk + let buffer + let fd + try { + fd = fs.openSync(paths.bin, 'r') + const stat = fs.fstatSync(fd) + const maxBytes = Math.min(byteLength, stat.size - byteOffset) + if (maxBytes <= 0) { + fs.closeSync(fd) + continue + } + buffer = Buffer.alloc(maxBytes) + fs.readSync(fd, buffer, 0, maxBytes, byteOffset) + fs.closeSync(fd) + } catch (_e) { + if (fd !== undefined) { + try { fs.closeSync(fd) } catch (_) { /* ignored */ } + } + continue + } + + // Decode records and add to results + const recordCount = Math.floor(buffer.length / RECORD_SIZE) + for (let i = 0; i < recordCount; i++) { + const record = readRecord(buffer, i * RECORD_SIZE, meta) + + // Skip null bars (gaps) - check before computing barTime for efficiency + if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) continue + + const barTime = meta.startTs + (startIndex + i) * meta.timeframeMs + + // Filter by time range + if (barTime < from || barTime >= to) continue + + // Add as array matching column order (includes market) + // Time is converted to seconds to match InfluxDB format (precision: 's') + results.push([ + Math.floor(barTime / 1000), + market, + record.open, + record.high, + record.low, + record.close, + record.vbuy, + record.vsell, + record.cbuy, + record.csell, + record.lbuy, + record.lsell + ]) + } + } + + // Inject pending bars for near-realtime data + if (needsPendingBars) { + const pendingBars = this.getPendingBars(markets, from, to) + for (let i = 0; i < pendingBars.length; i++) { + const bar = pendingBars[i] + // Time is converted to seconds to match InfluxDB format (precision: 's') + results.push([ + Math.floor(bar.time / 1000), + bar.market, + bar.open, + bar.high, + bar.low, + bar.close, + bar.vbuy, + bar.vsell, + bar.cbuy, + bar.csell, + bar.lbuy, + bar.lsell + ]) + } + } + + // Single sort at the end (by time) + // For large datasets, this is O(n log n) - unavoidable when merging multiple sources + if (markets.length > 1 || needsPendingBars) { + results.sort((a, b) => a[0] - b[0]) + } + + return Promise.resolve({ + format: this.format, + columns, + results + }) + } + + /** + * Gets pending (in-memory, not yet flushed) bars for given markets and time range. + * + * Used by fetch() to inject real-time data for queries near "now". + * Returns base timeframe bars as-is (no resampling - matches InfluxDB behavior). + * + * @param {string[]} markets - Array of market identifiers to include + * @param {number} from - Start timestamp (inclusive) + * @param {number} to - End timestamp (exclusive) + * @returns {import('./constants').Bar[]} Array of pending bar objects with market field + */ + getPendingBars(markets, from, to) { + const results = [] + + for (const market of markets) { + const barsMap = this.pendingBars[market] + if (!barsMap) continue + + for (const ts in barsMap) { + const bar = barsMap[ts] + + // Skip empty bars + if (bar.close === null) continue + + const barTime = Number(ts) + + // Filter by requested time range + if (barTime < from || barTime >= to) continue + + results.push(bar) + } + } + + return results + } +} + +module.exports = BinariesStorage diff --git a/src/storage/binaries/io.js b/src/storage/binaries/io.js new file mode 100644 index 0000000..c2c091d --- /dev/null +++ b/src/storage/binaries/io.js @@ -0,0 +1,223 @@ +const fs = require('fs') +const path = require('path') +const config = require('../../config') +const { getHms } = require('../../helper') +const { RECORD_SIZE } = require('./constants') + +/** + * Generates file paths for a market's binary and metadata files. + * + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol (will be sanitized) + * @param {number} timeframe - Timeframe in milliseconds + * @returns {import('./constants').FilePaths} Object with bin and json file paths + */ +function getFilePath(exchange, symbol, timeframe) { + // Sanitize symbol to create valid filename (replace / and : with -) + const sanitizedSymbol = symbol.replace(/[/:]/g, '-') + const tfLabel = getHms(timeframe) + const dir = path.join(config.filesLocation, exchange, sanitizedSymbol) + return { + bin: path.join(dir, `${tfLabel}.bin`), + json: path.join(dir, `${tfLabel}.json`) + } +} + +/** + * Parses a market string into exchange and symbol components. + * + * @param {string} market - Market identifier (e.g., "COINBASE:BTC-USD") + * @returns {import('./constants').ParsedMarket|null} Parsed components or null if invalid format + */ +function parseMarket(market) { + const match = market.match(/([^:]*):(.*)/) + if (!match) return null + return { exchange: match[1], symbol: match[2] } +} + +/** + * Writes a bar record to a buffer at the specified offset. + * + * Binary layout (56 bytes): + * - Bytes 0-3: open (Int32LE, scaled by priceScale) + * - Bytes 4-7: high (Int32LE, scaled) + * - Bytes 8-11: low (Int32LE, scaled) + * - Bytes 12-15: close (Int32LE, scaled) + * - Bytes 16-23: vbuy (BigInt64LE, scaled by volumeScale) + * - Bytes 24-31: vsell (BigInt64LE, scaled) + * - Bytes 32-35: cbuy (UInt32LE, trade count) + * - Bytes 36-39: csell (UInt32LE, trade count) + * - Bytes 40-47: lbuy (BigInt64LE, liquidation volume, scaled) + * - Bytes 48-55: lsell (BigInt64LE, liquidation volume, scaled) + * + * @param {Buffer} buffer - Target buffer + * @param {number} offset - Byte offset in buffer + * @param {import('./constants').Bar} bar - Bar data to write + * @param {import('./constants').BinaryMeta} meta - Metadata with scale factors + */ +function writeRecord(buffer, offset, bar, meta) { + // Scale and round prices (null becomes 0) + const open = bar.open !== null ? Math.round(bar.open * meta.priceScale) : 0 + const high = bar.high !== null ? Math.round(bar.high * meta.priceScale) : 0 + const low = bar.low !== null ? Math.round(bar.low * meta.priceScale) : 0 + const close = bar.close !== null ? Math.round(bar.close * meta.priceScale) : 0 + + // Write OHLC + buffer.writeInt32LE(open, offset) + buffer.writeInt32LE(high, offset + 4) + buffer.writeInt32LE(low, offset + 8) + buffer.writeInt32LE(close, offset + 12) + + // Scale and write volumes as BigInt + const vbuy = BigInt(Math.round((bar.vbuy || 0) * meta.volumeScale)) + const vsell = BigInt(Math.round((bar.vsell || 0) * meta.volumeScale)) + buffer.writeBigInt64LE(vbuy, offset + 16) + buffer.writeBigInt64LE(vsell, offset + 24) + + // Write trade counts + buffer.writeUInt32LE(bar.cbuy || 0, offset + 32) + buffer.writeUInt32LE(bar.csell || 0, offset + 36) + + // Scale and write liquidation volumes + const lbuy = BigInt(Math.round((bar.lbuy || 0) * meta.volumeScale)) + const lsell = BigInt(Math.round((bar.lsell || 0) * meta.volumeScale)) + buffer.writeBigInt64LE(lbuy, offset + 40) + buffer.writeBigInt64LE(lsell, offset + 48) +} + +/** + * Reads a bar record from a buffer at the specified offset. + * + * @param {Buffer} buffer - Source buffer + * @param {number} offset - Byte offset in buffer + * @param {import('./constants').BinaryMeta} meta - Metadata with scale factors + * @returns {import('./constants').RecordData} Decoded bar data + */ +function readRecord(buffer, offset, meta) { + // Read and descale OHLC + const open = buffer.readInt32LE(offset) / meta.priceScale + const high = buffer.readInt32LE(offset + 4) / meta.priceScale + const low = buffer.readInt32LE(offset + 8) / meta.priceScale + const close = buffer.readInt32LE(offset + 12) / meta.priceScale + + // Read and descale volumes + const vbuy = Number(buffer.readBigInt64LE(offset + 16)) / meta.volumeScale + const vsell = Number(buffer.readBigInt64LE(offset + 24)) / meta.volumeScale + + // Read trade counts + const cbuy = buffer.readUInt32LE(offset + 32) + const csell = buffer.readUInt32LE(offset + 36) + + // Read and descale liquidation volumes + const lbuy = Number(buffer.readBigInt64LE(offset + 40)) / meta.volumeScale + const lsell = Number(buffer.readBigInt64LE(offset + 48)) / meta.volumeScale + + return { open, high, low, close, vbuy, vsell, cbuy, csell, lbuy, lsell } +} + +/** + * Reads metadata JSON file for a market/timeframe. + * + * @param {string} jsonPath - Path to the JSON metadata file + * @returns {import('./constants').BinaryMeta|null} Parsed metadata or null if not found + */ +function readMeta(jsonPath) { + try { + if (!fs.existsSync(jsonPath)) return null + const content = fs.readFileSync(jsonPath, 'utf8') + return JSON.parse(content) + } catch (_e) { + return null + } +} + +/** + * Writes metadata JSON file atomically (write to temp, then rename). + * + * @param {string} jsonPath - Path to the JSON metadata file + * @param {import('./constants').BinaryMeta} meta - Metadata to write + */ +function writeMeta(jsonPath, meta) { + const tmpPath = jsonPath + '.tmp' + fs.writeFileSync(tmpPath, JSON.stringify(meta, null, 2)) + fs.renameSync(tmpPath, jsonPath) +} + +/** + * Reads a single record from the binary file at a specific bucket timestamp. + * Uses positioned read (pread) to read only the needed 56 bytes. + * + * This is used as a disk fallback when processing trades for buckets + * that exist on disk but not in memory caches. + * + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol + * @param {number} timeframe - Timeframe in milliseconds + * @param {number} bucketTs - Bucket timestamp to read + * @returns {import('./constants').Bar|null} Bar data if found and non-null, otherwise null + */ +function readSingleRecordAtTs(exchange, symbol, timeframe, bucketTs) { + const paths = getFilePath(exchange, symbol, timeframe) + const meta = readMeta(paths.json) + if (!meta) return null + if (!fs.existsSync(paths.bin)) return null + + // Check if bucketTs is within the file's range + if (bucketTs < meta.startTs || bucketTs >= meta.endTs) return null + + // Calculate record index and byte offset + const index = Math.floor((bucketTs - meta.startTs) / meta.timeframeMs) + if (index < 0 || index >= meta.records) return null + + const byteOffset = index * RECORD_SIZE + const buffer = Buffer.alloc(RECORD_SIZE) + + let fd + try { + fd = fs.openSync(paths.bin, 'r') + const stat = fs.fstatSync(fd) + if (byteOffset + RECORD_SIZE > stat.size) { + fs.closeSync(fd) + return null + } + // Positioned read - only reads the single record we need + fs.readSync(fd, buffer, 0, RECORD_SIZE, byteOffset) + fs.closeSync(fd) + } catch (_e) { + if (fd !== undefined) { + try { fs.closeSync(fd) } catch (_) { /* ignored */ } + } + return null + } + + const record = readRecord(buffer, 0, meta) + + // Check if this is a null record (all OHLC are zero) + if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) { + return null + } + + return { + time: bucketTs, + open: record.open, + high: record.high, + low: record.low, + close: record.close, + vbuy: record.vbuy, + vsell: record.vsell, + cbuy: record.cbuy, + csell: record.csell, + lbuy: record.lbuy, + lsell: record.lsell + } +} + +module.exports = { + getFilePath, + parseMarket, + writeRecord, + readRecord, + readMeta, + writeMeta, + readSingleRecordAtTs +} diff --git a/src/storage/binaries/resample.js b/src/storage/binaries/resample.js new file mode 100644 index 0000000..e0cca22 --- /dev/null +++ b/src/storage/binaries/resample.js @@ -0,0 +1,312 @@ +const fs = require('fs') +const { ensureDirectoryExists, getHms } = require('../../helper') +const { RECORD_SIZE, PRICE_SCALE, VOLUME_SCALE } = require('./constants') +const { getFilePath, readRecord, writeRecord, readMeta, writeMeta } = require('./io') +const { createNullBar } = require('./write') + +/** + * Triggers resampling from base timeframe to all configured higher timeframes. + * + * @param {Object} storage - BinariesStorage instance + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol + * @param {number} fromTs - Start of dirty range (earliest modified bucket) + * @param {number} toTs - End of dirty range (latest modified bucket) + * @returns {Promise} + */ +async function resampleToHigherTimeframes(storage, exchange, symbol, fromTs, toTs) { + const baseTimeframe = storage.baseTimeframe + const higherTimeframes = storage.timeframes.filter(tf => tf > baseTimeframe) + + if (higherTimeframes.length === 0) return + + const basePaths = getFilePath(exchange, symbol, baseTimeframe) + const baseMeta = readMeta(basePaths.json) + if (!baseMeta || !fs.existsSync(basePaths.bin)) return + + // Resample each higher timeframe + for (const targetTimeframe of higherTimeframes) { + await resampleRangeToTimeframe(exchange, symbol, baseMeta, basePaths.bin, fromTs, toTs, targetTimeframe) + } +} + +/** + * Resamples a range of base timeframe data into a higher target timeframe. + * + * Unlike append-only resampling, this supports upsert semantics: + * - Overwrites existing target buckets that were affected by base changes + * - Appends new target buckets as needed + * - Maintains dense indexing (null bars fill gaps) + * + * Aggregation rules: + * - open: first non-null value in bucket (by base bar time order) + * - close: last non-null value in bucket + * - high/low: extrema across all base bars + * - volumes/counts: cumulative sums + * + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol + * @param {import('./constants').BinaryMeta} baseMeta - Metadata for base timeframe file + * @param {string} baseBinPath - Path to base timeframe binary file + * @param {number} fromTs - Start of dirty range (base bucket time) + * @param {number} toTs - End of dirty range (base bucket time) + * @param {number} targetTimeframe - Target timeframe in milliseconds + * @returns {Promise} + */ +async function resampleRangeToTimeframe(exchange, symbol, baseMeta, baseBinPath, fromTs, toTs, targetTimeframe) { + const baseTimeframe = baseMeta.timeframeMs + const paths = getFilePath(exchange, symbol, targetTimeframe) + await ensureDirectoryExists(paths.bin) + + let meta = readMeta(paths.json) + + // Compute dirty target bucket range (inclusive) + // A target bucket is dirty if any of its contributing base buckets were modified + const firstTargetBucket = Math.floor(fromTs / targetTimeframe) * targetTimeframe + const lastTargetBucket = Math.floor(toTs / targetTimeframe) * targetTimeframe + + // Initialize metadata if this is a new file + if (!meta) { + meta = { + exchange: exchange, + symbol: symbol, + timeframe: getHms(targetTimeframe), + timeframeMs: targetTimeframe, + startTs: firstTargetBucket, + endTs: firstTargetBucket, + priceScale: PRICE_SCALE, + volumeScale: VOLUME_SCALE, + records: 0, + lastInputStartTs: firstTargetBucket + } + } + + // Skip buckets before meta.startTs (no prepend allowed) + const startBucket = Math.max(firstTargetBucket, meta.startTs) + if (startBucket > lastTargetBucket) return + + // Calculate base record index range covering the dirty target buckets + // Need to read all base bars that contribute to [startBucket, lastTargetBucket + targetTimeframe) + const baseStartIndex = Math.floor((startBucket - baseMeta.startTs) / baseTimeframe) + const baseEndIndex = Math.ceil((lastTargetBucket + targetTimeframe - baseMeta.startTs) / baseTimeframe) + + // Clamp to valid index range within the base file + const clampedStartIndex = Math.max(0, baseStartIndex) + const clampedEndIndex = Math.min(baseMeta.records, baseEndIndex) + + // Aggregate base bars into target buckets + // Key: target bucket timestamp, Value: aggregated bar data + const aggregated = {} + + if (clampedStartIndex < clampedEndIndex) { + const count = clampedEndIndex - clampedStartIndex + const byteOffset = clampedStartIndex * RECORD_SIZE + const byteLength = count * RECORD_SIZE + + let buffer + try { + const fd = fs.openSync(baseBinPath, 'r') + const stat = fs.fstatSync(fd) + const maxBytes = Math.min(byteLength, stat.size - byteOffset) + if (maxBytes > 0) { + buffer = Buffer.alloc(maxBytes) + fs.readSync(fd, buffer, 0, maxBytes, byteOffset) + } + fs.closeSync(fd) + } catch (_e) { + buffer = null + } + + if (buffer) { + const recordCount = Math.floor(buffer.length / RECORD_SIZE) + + // Process each base record and aggregate into target buckets + for (let i = 0; i < recordCount; i++) { + const record = readRecord(buffer, i * RECORD_SIZE, baseMeta) + const barTime = baseMeta.startTs + (clampedStartIndex + i) * baseTimeframe + + // Skip null records (gaps in base data) + if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) continue + + // Determine which target bucket this base bar contributes to + const targetBucketTime = Math.floor(barTime / targetTimeframe) * targetTimeframe + if (targetBucketTime < startBucket || targetBucketTime > lastTargetBucket) continue + + if (!aggregated[targetBucketTime]) { + // First contributing base bar for this target bucket + aggregated[targetBucketTime] = { + time: targetBucketTime, + open: record.open, + high: record.high, + low: record.low, + close: record.close, + vbuy: record.vbuy, + vsell: record.vsell, + cbuy: record.cbuy, + csell: record.csell, + lbuy: record.lbuy, + lsell: record.lsell, + _firstBarTime: barTime // Track earliest base bar for open determination + } + } else { + // Merge subsequent base bars into the aggregate + const agg = aggregated[targetBucketTime] + + // Open comes from the earliest base bar (by time, not arrival order) + if (barTime < agg._firstBarTime) { + agg.open = record.open + agg._firstBarTime = barTime + } + + // High/low are extrema, close comes from the last bar processed + agg.high = Math.max(agg.high, record.high) + agg.low = Math.min(agg.low, record.low) + agg.close = record.close + + // Volumes and counts are summed + agg.vbuy += record.vbuy + agg.vsell += record.vsell + agg.cbuy += record.cbuy + agg.csell += record.csell + agg.lbuy += record.lbuy + agg.lsell += record.lsell + } + } + } + } + + // Build complete list of bars to upsert (dense: one per target bucket in range) + const barsToUpsert = [] + for (let bucketTs = startBucket; bucketTs <= lastTargetBucket; bucketTs += targetTimeframe) { + if (aggregated[bucketTs]) { + barsToUpsert.push(aggregated[bucketTs]) + } else { + // No contributing base bars - write null bar for dense indexing + barsToUpsert.push(createNullBar(bucketTs)) + } + } + + if (barsToUpsert.length === 0) return + + // Separate bars into overwrites (existing records) vs appends (new records) + const overwrites = [] + const appends = [] + + for (const bar of barsToUpsert) { + const index = Math.floor((bar.time - meta.startTs) / targetTimeframe) + if (index < 0) continue // No prepend allowed + if (index < meta.records) { + overwrites.push({ index, bar }) + } else { + appends.push({ index, bar }) + } + } + + // Handle overwrites (positioned writes to existing records) + if (overwrites.length > 0) { + let fd = null + try { + // Open in read-write mode for in-place updates + fd = fs.openSync(paths.bin, 'r+') + + // Sort by index for sequential access and batch contiguous writes + overwrites.sort((a, b) => a.index - b.index) + let groupStart = 0 + + // Group contiguous indices for efficient batch writes + while (groupStart < overwrites.length) { + let groupEnd = groupStart + + // Find end of contiguous run + while (groupEnd + 1 < overwrites.length && + overwrites[groupEnd + 1].index === overwrites[groupEnd].index + 1) { + groupEnd++ + } + + // Write contiguous group in single syscall + const groupSize = groupEnd - groupStart + 1 + const buffer = Buffer.alloc(groupSize * RECORD_SIZE) + let bufferOffset = 0 + + for (let i = groupStart; i <= groupEnd; i++) { + writeRecord(buffer, bufferOffset, overwrites[i].bar, meta) + bufferOffset += RECORD_SIZE + } + + const fileOffset = overwrites[groupStart].index * RECORD_SIZE + fs.writeSync(fd, buffer, 0, buffer.length, fileOffset) + + groupStart = groupEnd + 1 + } + } finally { + if (fd !== null) { + fs.closeSync(fd) + } + } + } + + // Handle appends (extend file with new records) + if (appends.length > 0) { + appends.sort((a, b) => a.index - b.index) + const lastAppendIndex = appends[appends.length - 1].index + + // Build lookup map for sparse appends + const appendMap = {} + for (const a of appends) { + appendMap[a.index] = a.bar + } + + const BATCH_SIZE = 10000 + let currentIndex = meta.records + let fd = null + + try { + // Open in append mode + fd = fs.openSync(paths.bin, 'a') + + // Write records from current end to last append index (dense, filling gaps with null) + while (currentIndex <= lastAppendIndex) { + const batchRecords = [] + const batchEndIndex = Math.min(currentIndex + BATCH_SIZE, lastAppendIndex + 1) + + while (currentIndex < batchEndIndex) { + if (appendMap[currentIndex]) { + batchRecords.push(appendMap[currentIndex]) + } else { + // Fill gap with null bar + batchRecords.push(createNullBar(meta.startTs + currentIndex * targetTimeframe)) + } + currentIndex++ + } + + if (batchRecords.length > 0) { + const buffer = Buffer.alloc(batchRecords.length * RECORD_SIZE) + let offset = 0 + + for (const bar of batchRecords) { + writeRecord(buffer, offset, bar, meta) + offset += RECORD_SIZE + } + + fs.writeSync(fd, buffer) + meta.records += batchRecords.length + } + } + } finally { + if (fd !== null) { + fs.closeSync(fd) + } + } + + // Update endTs after extending file + meta.endTs = meta.startTs + meta.records * targetTimeframe + } + + meta.lastInputStartTs = lastTargetBucket + writeMeta(paths.json, meta) +} + +module.exports = { + resampleToHigherTimeframes, + resampleRangeToTimeframe +} diff --git a/src/storage/binaries/write.js b/src/storage/binaries/write.js new file mode 100644 index 0000000..c4972ae --- /dev/null +++ b/src/storage/binaries/write.js @@ -0,0 +1,316 @@ +const fs = require('fs') +const { ensureDirectoryExists, getHms } = require('../../helper') +const { RECORD_SIZE, PRICE_SCALE, VOLUME_SCALE } = require('./constants') +const { getFilePath, writeRecord, readMeta, writeMeta } = require('./io') + +/** + * Creates an empty (null) bar for gap filling. + * Null bars have all OHLC set to null and all volumes/counts set to 0. + * On disk, these are stored as all zeros and skipped when reading. + * + * @param {number} time - Bucket timestamp + * @returns {import('./constants').Bar} Null bar object + */ +function createNullBar(time) { + return { + time, + open: null, + high: null, + low: null, + close: null, + vbuy: 0, + vsell: 0, + cbuy: 0, + csell: 0, + lbuy: 0, + lsell: 0 + } +} + +/** + * Appends bars to a binary file (append-only, no overwrites). + * + * @deprecated Use upsertBars() instead for backfill support + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol + * @param {number} timeframe - Timeframe in milliseconds + * @param {import('./constants').Bar[]} bars - Array of bars to append + * @returns {Promise} Timestamps of written data range + */ +async function appendBars(exchange, symbol, timeframe, bars) { + const paths = getFilePath(exchange, symbol, timeframe) + await ensureDirectoryExists(paths.bin) + + let meta = readMeta(paths.json) + + // Get time bounds of input bars + const firstBarTime = Math.floor(bars[0].time / timeframe) * timeframe + const lastBarTime = Math.floor(bars[bars.length - 1].time / timeframe) * timeframe + + // Initialize metadata if new file + if (!meta) { + meta = { + exchange: exchange, + symbol: symbol, + timeframe: getHms(timeframe), + timeframeMs: timeframe, + startTs: firstBarTime, + endTs: firstBarTime, + priceScale: PRICE_SCALE, + volumeScale: VOLUME_SCALE, + records: 0, + lastInputStartTs: firstBarTime + } + } + + // Filter to only bars that can be appended (>= endTs, non-null close) + const barsToWriteMap = {} + for (const bar of bars) { + const alignedTime = Math.floor(bar.time / timeframe) * timeframe + if (alignedTime < meta.endTs) continue // Skip existing records + if (bar.close === null) continue // Skip empty bars + barsToWriteMap[alignedTime] = bar + } + + const timestamps = Object.keys(barsToWriteMap).map(Number).sort((a, b) => a - b) + if (timestamps.length === 0) return { fromTsWritten: null, toTsWritten: null } + + const lastNewBarTime = timestamps[timestamps.length - 1] + const fromTsWritten = meta.endTs + + const BATCH_SIZE = 10000 + let currentTs = meta.endTs + let fd = null + + try { + fd = fs.openSync(paths.bin, 'a') + + // Write records from endTs to lastNewBarTime (dense, with gap filling) + while (currentTs <= lastNewBarTime) { + const batchRecords = [] + const batchEnd = Math.min(currentTs + BATCH_SIZE * timeframe, lastNewBarTime + timeframe) + + while (currentTs < batchEnd && currentTs <= lastNewBarTime) { + if (barsToWriteMap[currentTs]) { + batchRecords.push(barsToWriteMap[currentTs]) + } else { + // Fill gap with null bar + batchRecords.push(createNullBar(currentTs)) + } + currentTs += timeframe + } + + if (batchRecords.length > 0) { + const buffer = Buffer.alloc(batchRecords.length * RECORD_SIZE) + let offset = 0 + + for (const bar of batchRecords) { + writeRecord(buffer, offset, bar, meta) + offset += RECORD_SIZE + } + + fs.writeSync(fd, buffer) + meta.records += batchRecords.length + } + } + } finally { + if (fd !== null) { + fs.closeSync(fd) + } + } + + // Update metadata + meta.endTs = meta.startTs + meta.records * timeframe + meta.lastInputStartTs = lastBarTime + + writeMeta(paths.json, meta) + + return { fromTsWritten, toTsWritten: lastNewBarTime } +} + +/** + * Upserts bars to a binary file with support for both overwrites and appends. + * + * This is the primary write method that supports backfill scenarios: + * - Bars with timestamps >= endTs are appended (with gap filling) + * - Bars with timestamps < endTs (but >= startTs) overwrite existing records + * - Bars with timestamps < startTs are ignored (no prepend) + * + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol + * @param {number} timeframe - Timeframe in milliseconds + * @param {import('./constants').Bar[]} bars - Array of bars to upsert + * @returns {Promise} Timestamps of written data range (includes overwrites) + */ +async function upsertBars(exchange, symbol, timeframe, bars) { + const paths = getFilePath(exchange, symbol, timeframe) + await ensureDirectoryExists(paths.bin) + + let meta = readMeta(paths.json) + + // Filter out bars with null close (empty bars) + const validBars = bars.filter(b => b.close !== null) + if (validBars.length === 0) return { fromTsWritten: null, toTsWritten: null } + + // Build map of aligned timestamps to bars + const barsMap = {} + for (const bar of validBars) { + const alignedTime = Math.floor(bar.time / timeframe) * timeframe + barsMap[alignedTime] = bar + } + + const timestamps = Object.keys(barsMap).map(Number).sort((a, b) => a - b) + if (timestamps.length === 0) return { fromTsWritten: null, toTsWritten: null } + + const firstBarTime = timestamps[0] + + // Initialize metadata if new file + if (!meta) { + meta = { + exchange: exchange, + symbol: symbol, + timeframe: getHms(timeframe), + timeframeMs: timeframe, + startTs: firstBarTime, + endTs: firstBarTime, + priceScale: PRICE_SCALE, + volumeScale: VOLUME_SCALE, + records: 0, + lastInputStartTs: firstBarTime + } + } + + // Filter out bars before startTs (no prepend allowed) + const filteredTimestamps = timestamps.filter(ts => ts >= meta.startTs) + if (filteredTimestamps.length === 0) return { fromTsWritten: null, toTsWritten: null } + + // Track the actual dirty range (for resampling) + const minTs = filteredTimestamps[0] + const maxTs = filteredTimestamps[filteredTimestamps.length - 1] + + // Categorize bars as overwrites or appends based on their index + const overwrites = [] // { index, ts, bar } - existing records to update + const appends = [] // { index, ts, bar } - new records to add + + for (const ts of filteredTimestamps) { + const index = Math.floor((ts - meta.startTs) / timeframe) + if (index < meta.records) { + // Index exists in file - this is an overwrite + overwrites.push({ index, ts, bar: barsMap[ts] }) + } else { + // Index beyond current file - this is an append + appends.push({ index, ts, bar: barsMap[ts] }) + } + } + + // Handle overwrites (positioned writes to existing records) + if (overwrites.length > 0) { + let fd = null + try { + // Open in read-write mode for in-place updates + fd = fs.openSync(paths.bin, 'r+') + + // Sort and group contiguous indices for efficient batch writes + overwrites.sort((a, b) => a.index - b.index) + let groupStart = 0 + + while (groupStart < overwrites.length) { + let groupEnd = groupStart + + // Find end of contiguous run + while (groupEnd + 1 < overwrites.length && + overwrites[groupEnd + 1].index === overwrites[groupEnd].index + 1) { + groupEnd++ + } + + // Write contiguous group in single syscall + const groupSize = groupEnd - groupStart + 1 + const buffer = Buffer.alloc(groupSize * RECORD_SIZE) + let bufferOffset = 0 + + for (let i = groupStart; i <= groupEnd; i++) { + writeRecord(buffer, bufferOffset, overwrites[i].bar, meta) + bufferOffset += RECORD_SIZE + } + + const fileOffset = overwrites[groupStart].index * RECORD_SIZE + fs.writeSync(fd, buffer, 0, buffer.length, fileOffset) + + groupStart = groupEnd + 1 + } + } finally { + if (fd !== null) { + fs.closeSync(fd) + } + } + } + + // Handle appends (extend file with new records) + if (appends.length > 0) { + appends.sort((a, b) => a.index - b.index) + const lastAppendIndex = appends[appends.length - 1].index + + // Build lookup map for sparse appends + const appendMap = {} + for (const a of appends) { + appendMap[a.index] = a.bar + } + + const BATCH_SIZE = 10000 + let currentIndex = meta.records + let fd = null + + try { + // Open in append mode + fd = fs.openSync(paths.bin, 'a') + + // Write records from current end to last append index (dense, filling gaps) + while (currentIndex <= lastAppendIndex) { + const batchRecords = [] + const batchEndIndex = Math.min(currentIndex + BATCH_SIZE, lastAppendIndex + 1) + + while (currentIndex < batchEndIndex) { + if (appendMap[currentIndex]) { + batchRecords.push(appendMap[currentIndex]) + } else { + // Fill gap with null bar for dense indexing + batchRecords.push(createNullBar(meta.startTs + currentIndex * timeframe)) + } + currentIndex++ + } + + if (batchRecords.length > 0) { + const buffer = Buffer.alloc(batchRecords.length * RECORD_SIZE) + let offset = 0 + + for (const bar of batchRecords) { + writeRecord(buffer, offset, bar, meta) + offset += RECORD_SIZE + } + + fs.writeSync(fd, buffer) + meta.records += batchRecords.length + } + } + } finally { + if (fd !== null) { + fs.closeSync(fd) + } + } + + // Update endTs after extending file + meta.endTs = meta.startTs + meta.records * timeframe + } + + meta.lastInputStartTs = maxTs + writeMeta(paths.json, meta) + + // Return the full dirty range (includes overwrites, not just appends) + return { fromTsWritten: minTs, toTsWritten: maxTs } +} + +module.exports = { + createNullBar, + appendBars, + upsertBars +} From 813978dc65c820074bf1c8ecdeb091379c928959 Mon Sep 17 00:00:00 2001 From: Tucsky Date: Tue, 13 Jan 2026 22:57:16 +0100 Subject: [PATCH 3/6] feat: enhance error handling and retry logic across exchange modules --- src/exchange.js | 144 ++++++++++++++++++++++++++++++- src/exchanges/binance.js | 8 +- src/exchanges/binance_futures.js | 8 +- src/exchanges/bitfinex.js | 8 +- src/exchanges/bitmart.js | 8 +- src/exchanges/bitmex.js | 8 +- src/exchanges/bybit.js | 8 +- src/exchanges/coinbase.js | 7 +- src/exchanges/cryptocom.js | 24 +++--- src/exchanges/huobi.js | 8 +- src/exchanges/kraken.js | 8 +- src/exchanges/okex.js | 40 ++------- 12 files changed, 206 insertions(+), 73 deletions(-) diff --git a/src/exchange.js b/src/exchange.js index fa0f82c..9af3ff2 100644 --- a/src/exchange.js +++ b/src/exchange.js @@ -615,6 +615,8 @@ class Exchange extends EventEmitter { if (nextRange.pair === range.pair) { // range might have been created while this one was recovering // ensure no dupes + const beforeFrom = nextRange.from + const beforeTo = nextRange.to nextRange.from = Math.max(nextRange.from, originalRange.to) nextRange.to = Math.max(nextRange.to, originalRange.to) @@ -624,7 +626,7 @@ class Exchange extends EventEmitter { ) this.recoveryRanges.splice(i--, 1) continue - } else { + } else if (nextRange.from !== beforeFrom || nextRange.to !== beforeTo) { console.log( `[${this.id}.registerRangeForRecovery] shrinked range (${nextRange.pair}, #${i}: ${new Date(+nextRange.from).toISOString()}, ${new Date(+nextRange.to).toISOString()})` ) @@ -956,6 +958,146 @@ class Exchange extends EventEmitter { return true } + formatErrorForLog(err) { + if (!err) { + return err + } + + if (typeof err !== 'object') { + return { message: String(err) } + } + + const details = { + message: err.message, + name: err.name, + code: err.code, + status: err.status || err.statusCode + } + + if (err.response) { + details.status = err.response.status || details.status + details.statusText = err.response.statusText + details.data = err.response.data + } + + if (err.config) { + details.method = err.config.method + details.params = err.config.params + + const rawUrl = err.config.url || '' + const baseUrl = err.config.baseURL || '' + if (rawUrl) { + details.url = + baseUrl && !/^https?:\/\//i.test(rawUrl) + ? `${baseUrl.replace(/\/$/, '')}/${rawUrl.replace(/^\//, '')}` + : rawUrl + } else if (baseUrl) { + details.url = baseUrl + } + } + + if (!details.url && err.request) { + details.url = err.request.path || err.request.url + details.method = details.method || err.request.method + } + + for (const key of Object.keys(details)) { + if (typeof details[key] === 'undefined') { + delete details[key] + } + } + + return details + } + + getErrorStatus(err) { + if (!err || typeof err !== 'object') { + return undefined + } + + if (err.response && typeof err.response.status !== 'undefined') { + return err.response.status + } + + if (typeof err.status !== 'undefined') { + return err.status + } + + if (typeof err.statusCode !== 'undefined') { + return err.statusCode + } + + return undefined + } + + shouldRetryError(err) { + const status = this.getErrorStatus(err) + if (typeof status === 'number') { + return status === 429 || (status >= 500 && status < 600) + } + + const code = err && err.code + if (!code) { + return false + } + + return [ + 'ECONNRESET', + 'ETIMEDOUT', + 'ECONNABORTED', + 'ENOTFOUND', + 'EAI_AGAIN', + 'ECONNREFUSED', + 'ENETUNREACH', + 'EHOSTUNREACH', + 'EPIPE' + ].includes(code) + } + + async requestWithRetry( + fn, + { maxAttempts = 3, range, label = 'request' } = {} + ) { + let attempt = 1 + + while (true) { + try { + return await fn() + } catch (err) { + if (!this.shouldRetryError(err) || attempt >= maxAttempts) { + throw err + } + + const pairLabel = range && range.pair ? ` on ${range.pair}` : '' + const status = this.getErrorStatus(err) + const meta = { + attempt: attempt + 1, + maxAttempts + } + + if (typeof status !== 'undefined') { + meta.status = status + } + + if (err && err.code) { + meta.code = err.code + } + + if (err && err.message) { + meta.message = err.message + } + + console.warn( + `[${this.id}] retrying ${label}${pairLabel} (${attempt + 1}/${maxAttempts})`, + meta + ) + + await this.waitBeforeContinueRecovery(attempt) + attempt += 1 + } + } + } + startKeepAlive(api, payload = { event: 'ping' }, every = 30000) { if (this.keepAliveIntervals[api.id]) { this.stopKeepAlive(api.id) diff --git a/src/exchanges/binance.js b/src/exchanges/binance.js index e4c8c9a..73f2743 100644 --- a/src/exchanges/binance.js +++ b/src/exchanges/binance.js @@ -103,8 +103,10 @@ class Binance extends Exchange { startTime + 1 }&endTime=${below1HEndTime}&limit=1000` - return axios - .get(endpoint) + return this.requestWithRetry(() => axios.get(endpoint), { + range, + label: 'missing trades' + }) .then(response => { if (response.data.length) { const trades = response.data.map(trade => ({ @@ -143,7 +145,7 @@ class Binance extends Exchange { .catch(err => { console.error( `Failed to get historical trades on ${range.pair}`, - err.message + this.formatErrorForLog(err) ) }) } diff --git a/src/exchanges/binance_futures.js b/src/exchanges/binance_futures.js index 92fab51..243a7eb 100644 --- a/src/exchanges/binance_futures.js +++ b/src/exchanges/binance_futures.js @@ -190,8 +190,10 @@ class BinanceFutures extends Exchange { endpoint = 'https://fapi.binance.com/fapi/v1/aggTrades' + endpoint } - return axios - .get(endpoint) + return this.requestWithRetry(() => axios.get(endpoint), { + range, + label: 'missing trades' + }) .then(response => { if (response.data.length) { const trades = response.data @@ -232,7 +234,7 @@ class BinanceFutures extends Exchange { .catch(err => { console.error( `[${this.id}] failed to get missing trades on ${range.pair}`, - err.message + this.formatErrorForLog(err) ) return totalRecovered diff --git a/src/exchanges/bitfinex.js b/src/exchanges/bitfinex.js index a441828..c526b20 100644 --- a/src/exchanges/bitfinex.js +++ b/src/exchanges/bitfinex.js @@ -194,8 +194,10 @@ class Bitfinex extends Exchange { 't' + range.pair }&start=${startTime + 1}&end=${range.to}&limit=1000&sort=1` - return axios - .get(endpoint) + return this.requestWithRetry(() => axios.get(endpoint), { + range, + label: 'missing trades' + }) .then(response => { if (response.data.length) { const trades = response.data.map(trade => @@ -232,7 +234,7 @@ class Bitfinex extends Exchange { .catch(err => { console.error( `[${this.id}] failed to get missing trades on ${range.pair}`, - err.message + this.formatErrorForLog(err) ) return totalRecovered diff --git a/src/exchanges/bitmart.js b/src/exchanges/bitmart.js index 0404013..be0ca00 100644 --- a/src/exchanges/bitmart.js +++ b/src/exchanges/bitmart.js @@ -232,8 +232,10 @@ class Bitmart extends Exchange { side: trade[4] } } - return axios - .get(endpoint) + return this.requestWithRetry(() => axios.get(endpoint), { + range, + label: 'missing trades' + }) .then(response => { if (response.data.data && response.data.data.length) { const trades = response.data.data.map(trade => @@ -255,7 +257,7 @@ class Bitmart extends Exchange { .catch(err => { console.error( `[${this.id}] failed to get missing trades on ${range.pair}`, - err.message + this.formatErrorForLog(err) ) return totalRecovered diff --git a/src/exchanges/bitmex.js b/src/exchanges/bitmex.js index 2ed28cc..45400a3 100644 --- a/src/exchanges/bitmex.js +++ b/src/exchanges/bitmex.js @@ -152,8 +152,10 @@ class Bitmex extends Exchange { const endpoint = `https://www.bitmex.com/api/v1/trade?symbol=${range.pair}&startTime=${startTimeISO}&endTime=${endTimeISO}&count=1000` - return axios - .get(endpoint) + return this.requestWithRetry(() => axios.get(endpoint), { + range, + label: 'missing trades' + }) .then(response => { if (response.data.length) { const trades = response.data.map(trade => @@ -190,7 +192,7 @@ class Bitmex extends Exchange { .catch(err => { console.error( `[${this.id}] failed to get missing trades on ${range.pair}`, - err.message + this.formatErrorForLog(err) ) }) } diff --git a/src/exchanges/bybit.js b/src/exchanges/bybit.js index 5b5beb7..06cb30f 100644 --- a/src/exchanges/bybit.js +++ b/src/exchanges/bybit.js @@ -179,8 +179,10 @@ class Bybit extends Exchange { const realPair = isSpot ? range.pair.replace(SPOT_PAIR_REGEX, '') : range.pair const endpoint = `${RECENT_TRADE_REST}?category=${this.types[range.pair]}&symbol=${realPair}&limit=${limit}` - return axios - .get(endpoint) + return this.requestWithRetry(() => axios.get(endpoint), { + range, + label: 'missing trades' + }) .then(response => { if (response.data.result.list.length) { const trades = response.data.result.list @@ -223,7 +225,7 @@ class Bybit extends Exchange { .catch(err => { console.error( `[${this.id}] failed to get missing trades on ${range.pair}`, - err.message + this.formatErrorForLog(err) ) return totalRecovered diff --git a/src/exchanges/coinbase.js b/src/exchanges/coinbase.js index 6149999..217976b 100644 --- a/src/exchanges/coinbase.js +++ b/src/exchanges/coinbase.js @@ -175,7 +175,10 @@ class Coinbase extends Exchange { } try { - const response = await axios.get(endpoint) + const response = await this.requestWithRetry(() => axios.get(endpoint), { + range, + label: 'missing trades' + }) const rawData = Array.isArray(response.data) ? response.data : response.data.trades || [] @@ -246,7 +249,7 @@ class Coinbase extends Exchange { } catch (err) { console.error( `[${this.id}] failed to get missing trades on ${range.pair}`, - err.message + this.formatErrorForLog(err) ) return totalRecovered } diff --git a/src/exchanges/cryptocom.js b/src/exchanges/cryptocom.js index 0bdd2f8..1b30690 100644 --- a/src/exchanges/cryptocom.js +++ b/src/exchanges/cryptocom.js @@ -119,9 +119,11 @@ class CryptoCom extends Exchange { `start_ts=${range.from}&` + `end_ts=${range.to}` - return axios - .get(endpoint) - .then(response => { + return this.requestWithRetry(() => axios.get(endpoint), { + range, + label: 'missing trades' + }) + .then(response => { const data = response.data.result.data if (data.length) { @@ -170,14 +172,14 @@ class CryptoCom extends Exchange { return totalRecovered }) - .catch(err => { - console.error( - `[${this.id}] failed to get missing trades on ${range.pair}`, - err.message - ) - - return totalRecovered - }) + .catch(err => { + console.error( + `[${this.id}] failed to get missing trades on ${range.pair}`, + this.formatErrorForLog(err) + ) + + return totalRecovered + }) } } diff --git a/src/exchanges/huobi.js b/src/exchanges/huobi.js index 3d2e34c..395c2d6 100644 --- a/src/exchanges/huobi.js +++ b/src/exchanges/huobi.js @@ -266,8 +266,10 @@ class Huobi extends Exchange { endpoint = `https://api.huobi.pro/market/history/trade?symbol=${range.pair}&size=2000` } - return axios - .get(endpoint) + return this.requestWithRetry(() => axios.get(endpoint), { + range, + label: 'missing trades' + }) .then(response => { if (response.data.data.length) { const trades = response.data.data @@ -300,7 +302,7 @@ class Huobi extends Exchange { .catch(err => { console.error( `[${this.id}] failed to get missing trades on ${range.pair}`, - err.message + this.formatErrorForLog(err) ) return totalRecovered diff --git a/src/exchanges/kraken.js b/src/exchanges/kraken.js index 51d9569..3bffbd2 100644 --- a/src/exchanges/kraken.js +++ b/src/exchanges/kraken.js @@ -210,8 +210,10 @@ class Kraken extends Exchange { }` } - return axios - .get(endpoint) + return this.requestWithRetry(() => axios.get(endpoint), { + range, + label: 'missing trades' + }) .then(response => { let raw @@ -268,7 +270,7 @@ class Kraken extends Exchange { .catch(err => { console.error( `Failed to get historical trades on ${range.pair}`, - err.message + this.formatErrorForLog(err) ) }) } diff --git a/src/exchanges/okex.js b/src/exchanges/okex.js index b680d10..fa241e7 100644 --- a/src/exchanges/okex.js +++ b/src/exchanges/okex.js @@ -284,12 +284,10 @@ class Okex extends Exchange { const endpoint = `https://www.okx.com/api/v5/market/history-trades?instId=${range.pair}&type=2&limit=100&after=${range.to}` try { - const response = await this.retryWithDelay( - () => axios.get(endpoint), - 5, // Retry up to 5 times - 1, // Start with a multiplier of 1 - range - ) + const response = await this.requestWithRetry(() => axios.get(endpoint), { + range, + label: 'missing trades' + }) if (response.data.data.length) { const trades = response.data.data @@ -328,40 +326,12 @@ class Okex extends Exchange { } catch (err) { console.error( `[${this.id}] failed to get missing trades on ${range.pair} after retries:`, - err.message + this.formatErrorForLog(err) ) return totalRecovered } } - /** - * Retry a function with delays between attempts, using backoff - * @param {Function} fn The function to execute - * @param {number} retries Number of retry attempts - * @param {number} multiplier Multiplier for backoff delays (starts at 1) - * @param {Object} range The range object for logging retries (optional) - * @returns {Promise} The result of the function or an error if retries fail - */ - async retryWithDelay(fn, retries, multiplier = 1, range = {}) { - try { - return await fn() - } catch (err) { - if (retries > 0) { - console.warn( - `[${this.id}] Retrying with delay (${range.pair || 'unknown pair'}) attempt ${multiplier - }...` - ) - await this.waitBeforeContinueRecovery(multiplier) - return this.retryWithDelay(fn, retries - 1, multiplier + 1, range) - } - console.error( - `[${this.id}] Exceeded retry limit for ${range.pair || 'unknown pair' - }:`, - err.message - ) - throw err // Propagate the error after exceeding retries - } - } } module.exports = Okex From 2c78ccb7f307211dd87409030f6da7bd88c7be28 Mon Sep 17 00:00:00 2001 From: Tucsky Date: Sun, 18 Jan 2026 12:25:01 +0100 Subject: [PATCH 4/6] Refactor binary storage to support segmented file structure - Updated BinariesStorage to read from segmented files, computing which segments intersect the requested time range. - Introduced new functions for generating file paths for segments and reading/writing segment metadata. - Modified resampling logic to handle segmented base and target timeframe files, ensuring proper aggregation and upsert semantics. - Enhanced upsertBars function to manage bars by segment, allowing for efficient writing and handling of overwrites and appends. - Updated documentation to reflect changes in file structure and functionality. --- package-lock.json | 22 +- scripts/verify-segmented-binaries.js | 390 +++++++++++++++++++++++++++ src/server.js | 2 +- src/storage/binaries/README.md | 240 ++++++++++++----- src/storage/binaries/constants.js | 94 ++++++- src/storage/binaries/index.js | 150 ++++++----- src/storage/binaries/io.js | 86 +++++- src/storage/binaries/resample.js | 336 +++++++++++++---------- src/storage/binaries/write.js | 153 +++++++---- 9 files changed, 1117 insertions(+), 356 deletions(-) create mode 100644 scripts/verify-segmented-binaries.js diff --git a/package-lock.json b/package-lock.json index a951818..639dae6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "aggr-server", - "version": "1.2.0", + "version": "1.2.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "aggr-server", - "version": "1.2.0", + "version": "1.2.1", "license": "ISC", "dependencies": { "axios": "^1.4.0", @@ -690,6 +690,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1001,6 +1002,7 @@ "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.7.tgz", "integrity": "sha512-kukuqc39WOHtdxtw4UScxF/WVnMFVSQVKhtx3AjZJzhd0RGZZldcrfSEbVsWWe6KNH253574cq5F+wpv0G9pJw==", "hasInstallScript": true, + "peer": true, "dependencies": { "node-gyp-build": "^4.3.0" }, @@ -1424,6 +1426,7 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -1936,6 +1939,20 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -3864,6 +3881,7 @@ "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz", "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==", "hasInstallScript": true, + "peer": true, "dependencies": { "node-gyp-build": "^4.3.0" }, diff --git a/scripts/verify-segmented-binaries.js b/scripts/verify-segmented-binaries.js new file mode 100644 index 0000000..716eb5c --- /dev/null +++ b/scripts/verify-segmented-binaries.js @@ -0,0 +1,390 @@ +#!/usr/bin/env node +/** + * Verification script for segmented binaries storage. + * + * Tests: + * 1. Write synthetic bars spanning 2+ segments + * 2. Verify fetch returns correct continuity across segment boundaries + * 3. Verify backfill disk fallback reads from correct segment + * 4. Verify resampling works across segment boundaries + * + * Usage: + * node scripts/verify-segmented-binaries.js + * node scripts/verify-segmented-binaries.js --clean # Clean test data first + */ + +const fs = require('fs') +const path = require('path') + +// Override config for testing +process.env.STORAGE = 'binaries' +process.env.FILES_LOCATION = './data-test' + +const config = require('../src/config') +config.filesLocation = './data-test' +config.influxTimeframe = 10000 // 10s base timeframe +config.influxResampleTo = [60000] // Only 1m for testing +config.binariesSegmentRecords = 100 // Small segments for testing (100 records = 1000s = ~16.6 min) + +const BinariesStorage = require('../src/storage/binaries/index') +const { getSegmentStartTs, getSegmentSpanMs, getSegmentRecords } = require('../src/storage/binaries/constants') +const { getSegmentPaths, readMeta, readSingleRecordAtTs } = require('../src/storage/binaries/io') +const { upsertBars, createNullBar } = require('../src/storage/binaries/write') + +const EXCHANGE = 'TEST' +const SYMBOL = 'BTC-USD' +const MARKET = `${EXCHANGE}:${SYMBOL}` +const TIMEFRAME = config.influxTimeframe + +// Clean test data +function cleanTestData() { + const testDir = path.resolve(config.filesLocation) + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true, force: true }) + console.log(`Cleaned test data directory: ${testDir}`) + } +} + +// Generate synthetic bars +function generateBars(startTs, count, timeframe, priceStart = 50000) { + const bars = [] + for (let i = 0; i < count; i++) { + const time = startTs + i * timeframe + const price = priceStart + Math.random() * 100 - 50 + bars.push({ + time, + open: price, + high: price + 10, + low: price - 10, + close: price + 5, + vbuy: 1000 + i, + vsell: 900 + i, + cbuy: 10 + i, + csell: 8 + i, + lbuy: 0, + lsell: 0 + }) + } + return bars +} + +async function testWriteAcrossSegments() { + console.log('\n=== Test 1: Write bars spanning multiple segments ===') + + const segmentRecords = getSegmentRecords(TIMEFRAME) + const segmentSpanMs = getSegmentSpanMs(TIMEFRAME) + + console.log(`Segment config: ${segmentRecords} records, ${segmentSpanMs}ms span`) + + // Generate bars spanning 2.5 segments + const barCount = Math.floor(segmentRecords * 2.5) + const startTs = Math.floor(Date.now() / TIMEFRAME) * TIMEFRAME - (barCount * TIMEFRAME) + + // Align to segment boundary for cleaner test + const alignedStartTs = getSegmentStartTs(startTs, TIMEFRAME) + + console.log(`Generating ${barCount} bars starting at ${new Date(alignedStartTs).toISOString()}`) + + const bars = generateBars(alignedStartTs, barCount, TIMEFRAME) + + // Write bars + const result = await upsertBars(EXCHANGE, SYMBOL, TIMEFRAME, bars) + console.log(`Write result: fromTs=${result.fromTsWritten}, toTs=${result.toTsWritten}`) + + // Check which segments were created + const firstSegment = getSegmentStartTs(alignedStartTs, TIMEFRAME) + const lastSegment = getSegmentStartTs(alignedStartTs + (barCount - 1) * TIMEFRAME, TIMEFRAME) + + console.log(`Expected segments from ${firstSegment} to ${lastSegment}`) + + let segmentCount = 0 + for (let segmentTs = firstSegment; segmentTs <= lastSegment; segmentTs += segmentSpanMs) { + const paths = getSegmentPaths(EXCHANGE, SYMBOL, TIMEFRAME, segmentTs) + if (fs.existsSync(paths.json)) { + const meta = readMeta(paths.json) + console.log(` Segment ${segmentTs}: ${meta.records} records`) + segmentCount++ + } + } + + console.log(`Created ${segmentCount} segment files`) + + if (segmentCount >= 2) { + console.log('✓ PASS: Multiple segments created') + } else { + console.log('✗ FAIL: Expected at least 2 segments') + return false + } + + return { alignedStartTs, barCount } +} + +async function testFetchAcrossSegments(testData) { + console.log('\n=== Test 2: Fetch data spanning multiple segments ===') + + const storage = new BinariesStorage() + + // Fetch all written data + const from = testData.alignedStartTs + const to = testData.alignedStartTs + testData.barCount * TIMEFRAME + + console.log(`Fetching from ${new Date(from).toISOString()} to ${new Date(to).toISOString()}`) + + const result = await storage.fetch({ + from, + to, + timeframe: TIMEFRAME, + markets: [MARKET] + }) + + console.log(`Fetched ${result.results.length} records`) + + // Verify continuity + const times = result.results.map(r => r[0] * 1000) // Convert back to ms + times.sort((a, b) => a - b) + + let gaps = 0 + for (let i = 1; i < times.length; i++) { + const expectedTime = times[i - 1] + TIMEFRAME + if (times[i] !== expectedTime) { + gaps++ + if (gaps <= 3) { + console.log(` Gap at index ${i}: expected ${expectedTime}, got ${times[i]}`) + } + } + } + + if (gaps === 0) { + console.log('✓ PASS: No gaps in fetched data') + } else { + console.log(`✗ FAIL: Found ${gaps} gaps in fetched data`) + return false + } + + if (result.results.length === testData.barCount) { + console.log('✓ PASS: Correct number of records fetched') + } else { + console.log(`✗ FAIL: Expected ${testData.barCount} records, got ${result.results.length}`) + return false + } + + return true +} + +async function testBackfillDiskFallback(testData) { + console.log('\n=== Test 3: Backfill disk fallback reads from correct segment ===') + + const segmentSpanMs = getSegmentSpanMs(TIMEFRAME) + + // Pick a timestamp in the middle of the second segment + const secondSegmentStart = testData.alignedStartTs + segmentSpanMs + const testBucketTs = secondSegmentStart + 50 * TIMEFRAME // 50 bars into second segment + + console.log(`Reading single record at ${new Date(testBucketTs).toISOString()}`) + console.log(`Expected segment: ${secondSegmentStart}`) + + const bar = readSingleRecordAtTs(EXCHANGE, SYMBOL, TIMEFRAME, testBucketTs) + + if (bar) { + console.log(` Found bar: time=${bar.time}, close=${bar.close.toFixed(2)}`) + if (bar.time === testBucketTs) { + console.log('✓ PASS: Disk fallback returned correct bar') + } else { + console.log(`✗ FAIL: Bar time mismatch: expected ${testBucketTs}, got ${bar.time}`) + return false + } + } else { + console.log('✗ FAIL: Disk fallback returned null') + return false + } + + // Test reading from first segment too + const firstSegmentBucket = testData.alignedStartTs + 10 * TIMEFRAME + const bar2 = readSingleRecordAtTs(EXCHANGE, SYMBOL, TIMEFRAME, firstSegmentBucket) + + if (bar2 && bar2.time === firstSegmentBucket) { + console.log('✓ PASS: Can read from different segments') + } else { + console.log('✗ FAIL: Could not read from first segment') + return false + } + + return true +} + +async function testResamplingAcrossSegments(testData) { + console.log('\n=== Test 4: Resampling works across segment boundaries ===') + + const storage = new BinariesStorage() + const targetTimeframe = 60000 // 1m + + // Trigger resampling by doing a flush simulation + const { resampleToHigherTimeframes } = require('../src/storage/binaries/resample') + + await resampleToHigherTimeframes( + storage, + EXCHANGE, + SYMBOL, + testData.alignedStartTs, + testData.alignedStartTs + (testData.barCount - 1) * TIMEFRAME + ) + + // Fetch resampled data + const from = testData.alignedStartTs + const to = testData.alignedStartTs + testData.barCount * TIMEFRAME + + const result = await storage.fetch({ + from, + to, + timeframe: targetTimeframe, + markets: [MARKET] + }) + + console.log(`Fetched ${result.results.length} resampled 1m records`) + + // Calculate expected 1m bars (each 1m bar = 6 base bars at 10s) + const expectedBars = Math.floor(testData.barCount / 6) + + if (result.results.length >= expectedBars - 2 && result.results.length <= expectedBars + 2) { + console.log('✓ PASS: Resampling produced expected number of bars') + } else { + console.log(`✗ FAIL: Expected ~${expectedBars} bars, got ${result.results.length}`) + return false + } + + // Check for segment files + const targetSegmentSpanMs = getSegmentSpanMs(targetTimeframe) + const firstTargetSegment = getSegmentStartTs(from, targetTimeframe) + const lastTargetSegment = getSegmentStartTs(to - 1, targetTimeframe) + + let targetSegmentCount = 0 + for (let segmentTs = firstTargetSegment; segmentTs <= lastTargetSegment; segmentTs += targetSegmentSpanMs) { + const paths = getSegmentPaths(EXCHANGE, SYMBOL, targetTimeframe, segmentTs) + if (fs.existsSync(paths.json)) { + targetSegmentCount++ + } + } + + console.log(`Created ${targetSegmentCount} resampled segment file(s)`) + + return true +} + +async function testEdgeCases() { + console.log('\n=== Test 5: Edge cases ===') + + const segmentSpanMs = getSegmentSpanMs(TIMEFRAME) + + // Test 5a: Bar at exact segment boundary + console.log('\n5a: Bar at exact segment boundary') + const boundaryTs = Math.floor(Date.now() / segmentSpanMs) * segmentSpanMs + const boundaryBar = generateBars(boundaryTs, 1, TIMEFRAME)[0] + + const result1 = await upsertBars(EXCHANGE, SYMBOL, TIMEFRAME, [boundaryBar]) + + const expectedSegment = getSegmentStartTs(boundaryTs, TIMEFRAME) + if (expectedSegment === boundaryTs) { + console.log('✓ PASS: Boundary bar goes to correct segment') + } + + // Test 5b: Bar at segmentEnd - timeframe (last bar of segment) + console.log('\n5b: Last bar of segment') + const segmentRecords = getSegmentRecords(TIMEFRAME) + const lastBarTs = boundaryTs + (segmentRecords - 1) * TIMEFRAME + const lastBar = generateBars(lastBarTs, 1, TIMEFRAME)[0] + + const result2 = await upsertBars(EXCHANGE, SYMBOL, TIMEFRAME, [lastBar]) + + const lastBarSegment = getSegmentStartTs(lastBarTs, TIMEFRAME) + if (lastBarSegment === boundaryTs) { + console.log('✓ PASS: Last bar of segment goes to correct segment') + } else { + console.log(`✗ FAIL: Expected segment ${boundaryTs}, got ${lastBarSegment}`) + return false + } + + // Test 5c: First bar of next segment + console.log('\n5c: First bar of next segment') + const nextSegmentFirstBarTs = boundaryTs + segmentSpanMs + const nextBar = generateBars(nextSegmentFirstBarTs, 1, TIMEFRAME)[0] + + await upsertBars(EXCHANGE, SYMBOL, TIMEFRAME, [nextBar]) + + const nextBarSegment = getSegmentStartTs(nextSegmentFirstBarTs, TIMEFRAME) + if (nextBarSegment === boundaryTs + segmentSpanMs) { + console.log('✓ PASS: First bar of next segment goes to new segment') + } else { + console.log(`✗ FAIL: Expected segment ${boundaryTs + segmentSpanMs}, got ${nextBarSegment}`) + return false + } + + return true +} + +async function runAllTests() { + console.log('╔════════════════════════════════════════════════════════════╗') + console.log('║ Segmented Binaries Storage Verification ║') + console.log('╚════════════════════════════════════════════════════════════╝') + + // Check for --clean flag + if (process.argv.includes('--clean')) { + cleanTestData() + } + + // Ensure test directory exists + if (!fs.existsSync(config.filesLocation)) { + fs.mkdirSync(config.filesLocation, { recursive: true }) + } + + let passed = 0 + let failed = 0 + + try { + // Test 1: Write across segments + const testData = await testWriteAcrossSegments() + if (testData) { + passed++ + + // Test 2: Fetch across segments + if (await testFetchAcrossSegments(testData)) passed++ + else failed++ + + // Test 3: Backfill disk fallback + if (await testBackfillDiskFallback(testData)) passed++ + else failed++ + + // Test 4: Resampling + if (await testResamplingAcrossSegments(testData)) passed++ + else failed++ + } else { + failed++ + } + + // Test 5: Edge cases + if (await testEdgeCases()) passed++ + else failed++ + + } catch (error) { + console.error('\n✗ Test error:', error) + failed++ + } + + console.log('\n╔════════════════════════════════════════════════════════════╗') + console.log(`║ Results: ${passed} passed, ${failed} failed ║`) + console.log('╚════════════════════════════════════════════════════════════╝') + + if (failed > 0) { + console.log('\nSome tests failed. Check output above for details.') + process.exit(1) + } else { + console.log('\nAll tests passed! Segmented binaries storage is working correctly.') + + // Cleanup + if (!process.argv.includes('--keep')) { + cleanTestData() + console.log('Test data cleaned up. Use --keep to preserve test data.') + } + } +} + +runAllTests() diff --git a/src/server.js b/src/server.js index 138e247..8b97542 100644 --- a/src/server.js +++ b/src/server.js @@ -45,7 +45,7 @@ class Server extends EventEmitter { if (config.collect) { console.log('\n[server] collect is enabled') - console.log(`\tconnect to -> ${this.exchanges.map(a => a.id).join(', ')}`) + // console.log(`\t loading exchanges -> ${this.exchanges.map(a => a.id).join(', ')}`) this.handleExchangesEvents() diff --git a/src/storage/binaries/README.md b/src/storage/binaries/README.md index 16316f7..d52dbc9 100644 --- a/src/storage/binaries/README.md +++ b/src/storage/binaries/README.md @@ -4,28 +4,54 @@ A high-performance binary storage engine for OHLCV (Open, High, Low, Close, Volu ## Overview -Binaries Storage persists aggregated trade data in dense binary files with fixed-size records. Each market (e.g., `COINBASE:BTC-USD`) has separate files for each timeframe (e.g., `10s.bin`, `1m.bin`, `1h.bin`). +Binaries Storage persists aggregated trade data in dense binary files with fixed-size records. Each market (e.g., `COINBASE:BTC-USD`) has separate **segmented files** for each timeframe, enabling bounded file growth and cold-storage archival. ### Key Features +- **Time-segmented files**: Data is split into fixed-size segments to prevent unbounded growth - **Dense indexing**: Records are stored contiguously with null bars filling gaps - **Upsert semantics**: Can overwrite existing records or append new ones - **Automatic resampling**: Base timeframe data is aggregated to higher timeframes - **Backfill support**: Late-arriving trades merge into existing buckets -- **Arrival-order semantics**: Matches InfluxDB's behavior for consistency +- **Cold-storage ready**: Old segments can be moved/archived without rewriting files -## File Structure +## File Structure (Segmented) ``` data/ └── COINBASE/ └── BTC-USD/ - ├── 10s.bin # Binary data (56 bytes per record) - ├── 10s.json # Metadata (startTs, endTs, records, etc.) - ├── 1m.bin - ├── 1m.json - ├── 1h.bin - └── 1h.json + ├── 10s/ # Timeframe directory + │ ├── 1704067200000.bin # Segment file (segmentId = segmentStartTs) + │ ├── 1704067200000.json # Segment metadata + │ ├── 1704108160000.bin # Next segment + │ ├── 1704108160000.json + │ └── ... + ├── 1m/ + │ ├── 1704067200000.bin + │ ├── 1704067200000.json + │ └── ... + └── 1h/ + └── ... +``` + +### Segment ID Format + +The segment ID is the **segment start timestamp in milliseconds** (as a string): +- Deterministic: `segmentStartTs = floor(barTs / segmentSpanMs) * segmentSpanMs` +- Reversible: `segmentStartTs = parseInt(segmentId)` +- File paths: `{exchange}/{symbol}/{timeframe}/{segmentId}.bin|json` + +### Segment Size Calculation + +Each timeframe has a fixed number of records per segment (default: 4096): + +```javascript +segmentRecords = config.binariesSegmentRecords || 4096 +segmentSpanMs = timeframeMs * segmentRecords + +// Example for 10s timeframe: +// segmentSpanMs = 10000 * 4096 = 40,960,000 ms ≈ 11.4 hours ``` ## Module Structure @@ -33,10 +59,10 @@ data/ ``` src/storage/binaries/ ├── index.js # Main BinariesStorage class -├── constants.js # Record size, scale factors, typedefs -├── io.js # File I/O operations -├── resample.js # Timeframe resampling logic -└── write.js # Upsert and append operations +├── constants.js # Record size, scale factors, segment helpers +├── io.js # File I/O operations, segment path helpers +├── resample.js # Timeframe resampling logic (segment-aware) +└── write.js # Upsert operations (segment-aware) ``` ## Data Flow @@ -114,65 +140,63 @@ flowchart TD end ``` -### Upsert Operation +### Upsert Operation (Segmented) ```mermaid flowchart LR subgraph upsertBars["upsertBars(exchange, symbol, timeframe, bars)"] - U1[Filter valid bars] --> U2[Categorize by index] - U2 --> U3{index < records?} - U3 -->|Yes| U4[Overwrites list] - U3 -->|No| U5[Appends list] - U4 --> U6[Batch contiguous writes] - U5 --> U7[Append with gap filling] - U6 --> U8[Update metadata] - U7 --> U8 + U1[Filter valid bars] --> U2[Group by segmentStartTs] + U2 --> U3[For each segment] + U3 --> U4[upsertBarsToSegment] + U4 --> U5{index < records?} + U5 -->|Yes| U6[Overwrites list] + U5 -->|No| U7[Appends list] + U6 --> U8[Batch contiguous writes] + U7 --> U9[Append with gap filling] + U8 --> U10[Update segment metadata] + U9 --> U10 end ``` -### Resampling Flow +### Resampling Flow (Segmented) ```mermaid flowchart TD subgraph resample["resampleToHigherTimeframes"] R1[Get dirty range
fromTs → toTs] --> R2[For each higher TF] R2 --> R3[resampleRangeToTimeframe] - R3 --> R4[Read base TF records] - R4 --> R5[Aggregate into target buckets] - R5 --> R6[Upsert target TF file] - R6 --> R7{More timeframes?} - R7 -->|Yes| R2 - R7 -->|No| R8[Done] - end - - subgraph aggregate["Aggregation Rules"] - A1["open = first bar's open (by time)"] - A2["high = max(all highs)"] - A3["low = min(all lows)"] - A4["close = last bar's close"] - A5["volumes = sum(all volumes)"] - A6["counts = sum(all counts)"] + R3 --> R4[Compute base segment range] + R4 --> R5[Read from all base segments] + R5 --> R6[Aggregate into target buckets] + R6 --> R7[Group by target segment] + R7 --> R8[Upsert each target segment] + R8 --> R9{More timeframes?} + R9 -->|Yes| R2 + R9 -->|No| R10[Done] end ``` -### Fetch Query Flow +### Fetch Query Flow (Segmented) ```mermaid flowchart TD subgraph fetch["fetch({ from, to, timeframe, markets })"] - Q1[Validate timeframe] --> Q2[For each market] - Q2 --> Q3[Read metadata] - Q3 --> Q4[Calculate index range] - Q4 --> Q5[Read binary records] - Q5 --> Q6[Decode & filter] - Q6 --> Q7[Add to results] - Q7 --> Q8{More markets?} - Q8 -->|Yes| Q2 - Q8 -->|No| Q9{Need pending bars?} - Q9 -->|Yes| Q10[Inject pendingBars] - Q9 -->|No| Q11[Sort by time] - Q10 --> Q11 - Q11 --> Q12[Return results] + Q1[Validate timeframe] --> Q2[Compute segment range] + Q2 --> Q3[For each market] + Q3 --> Q4[For each segment in range] + Q4 --> Q5[Read segment metadata] + Q5 --> Q6[Calculate index range within segment] + Q6 --> Q7[Read binary records] + Q7 --> Q8[Decode & filter null bars] + Q8 --> Q9{More segments?} + Q9 -->|Yes| Q4 + Q9 -->|No| Q10{More markets?} + Q10 -->|Yes| Q3 + Q10 -->|No| Q11{Need pending bars?} + Q11 -->|Yes| Q12[Inject pendingBars] + Q11 -->|No| Q13[Sort by time] + Q12 --> Q13 + Q13 --> Q14[Return results] end ``` @@ -180,18 +204,18 @@ flowchart TD Each record is exactly **56 bytes**: -| Offset | Size | Type | Field | Description | -|--------|------|------|-------|-------------| -| 0 | 4 | Int32LE | open | Open price × 10,000 | -| 4 | 4 | Int32LE | high | High price × 10,000 | -| 8 | 4 | Int32LE | low | Low price × 10,000 | -| 12 | 4 | Int32LE | close | Close price × 10,000 | -| 16 | 8 | BigInt64LE | vbuy | Buy volume × 1,000,000 | -| 24 | 8 | BigInt64LE | vsell | Sell volume × 1,000,000 | -| 32 | 4 | UInt32LE | cbuy | Buy trade count | -| 36 | 4 | UInt32LE | csell | Sell trade count | -| 40 | 8 | BigInt64LE | lbuy | Buy liquidation volume × 1,000,000 | -| 48 | 8 | BigInt64LE | lsell | Sell liquidation volume × 1,000,000 | +| Offset | Size | Type | Field | Description | +|--------|------|------------|-------|-------------------------------------| +| 0 | 4 | Int32LE | open | Open price × 10,000 | +| 4 | 4 | Int32LE | high | High price × 10,000 | +| 8 | 4 | Int32LE | low | Low price × 10,000 | +| 12 | 4 | Int32LE | close | Close price × 10,000 | +| 16 | 8 | BigInt64LE | vbuy | Buy volume × 1,000,000 | +| 24 | 8 | BigInt64LE | vsell | Sell volume × 1,000,000 | +| 32 | 4 | UInt32LE | cbuy | Buy trade count | +| 36 | 4 | UInt32LE | csell | Sell trade count | +| 40 | 8 | BigInt64LE | lbuy | Buy liquidation volume × 1,000,000 | +| 48 | 8 | BigInt64LE | lsell | Sell liquidation volume × 1,000,000 | ### Scale Factors @@ -202,9 +226,9 @@ Each record is exactly **56 bytes**: Gaps in data are represented as "null bars" where all OHLC values are 0. These maintain dense indexing (record position = time offset) but are skipped when reading. -## Metadata Format +## Segment Metadata Format -Each `.json` file contains: +Each segment's `.json` file contains: ```json { @@ -212,15 +236,27 @@ Each `.json` file contains: "symbol": "BTC-USD", "timeframe": "10s", "timeframeMs": 10000, - "startTs": 1704067200000, - "endTs": 1704153600000, + "segmentStartTs": 1704067200000, + "segmentEndTs": 1704108160000, + "segmentSpanMs": 40960000, + "segmentRecords": 4096, "priceScale": 10000, "volumeScale": 1000000, - "records": 8640, - "lastInputStartTs": 1704153590000 + "records": 2048, + "lastInputStartTs": 1704087680000 } ``` +### Segment Metadata Fields + +| Field | Description | +|------------------|-------------------------------------------------------------| +| `segmentStartTs` | First bucket timestamp in this segment | +| `segmentEndTs` | `segmentStartTs + segmentSpanMs` (exclusive end) | +| `segmentSpanMs` | Total time span of segment in milliseconds | +| `segmentRecords` | Fixed max records for this segment (default 4096) | +| `records` | Current number of records written (may be < segmentRecords) | + ## Memory Caches ### pendingBars @@ -280,10 +316,30 @@ Relevant config options (from `config.json`): 14400000, 21600000, 86400000 // 4h, 6h, 1d ], "influxResampleInterval": 60000, // Flush/resample interval (1m) - "backupInterval": 60000 // Backup interval (1m) + "backupInterval": 60000, // Backup interval (1m) + + // Segment configuration (optional) + "binariesSegmentRecords": 4096 // Records per segment (default) + + // Or per-timeframe configuration: + // "binariesSegmentRecords": { + // "10s": 8640, // ~24 hours of 10s bars + // "1m": 4320, // ~3 days of 1m bars + // "1h": 4320, // ~180 days of 1h bars + // "1d": 3650 // ~10 years of daily bars + // } } ``` +### Segment Size Examples + +| Timeframe | Default Records | Default Span | +|-----------|-----------------|--------------| +| 10s | 4096 | ~11.4 hours | +| 1m | 4096 | ~2.8 days | +| 1h | 4096 | ~170 days | +| 1d | 4096 | ~11.2 years | + ## API Compatibility The `fetch()` method returns data in the same format as InfluxDB storage: @@ -305,10 +361,10 @@ The `fetch()` method returns data in the same format as InfluxDB storage: | Operation | Complexity | Notes | |-----------|------------|-------| -| Write (append) | O(n) | Sequential append, batched | +| Write (append) | O(n) | Sequential append, batched per segment | | Write (overwrite) | O(n) | Positioned writes, batched by contiguous indices | -| Read (fetch) | O(n) | Sequential read, memory-mapped friendly | -| Resample | O(n) | Reads base TF, writes target TF | +| Read (fetch) | O(n + s) | n=records, s=segments in range | +| Resample | O(n) | Reads base segments, writes target segments | ### Advantages over InfluxDB @@ -317,3 +373,39 @@ The `fetch()` method returns data in the same format as InfluxDB storage: - **Faster writes**: Direct file I/O vs HTTP API - **Predictable performance**: No query planning overhead - **Lower memory**: Stream processing vs in-memory aggregation +- **Bounded growth**: Segments prevent unbounded file sizes + +### Advantages of Segmentation + +- **Cold storage**: Old segments can be moved to cold storage without rewriting +- **Parallel reads**: Multiple segments can theoretically be read in parallel +- **Bounded file size**: Each segment has a maximum size of `segmentRecords * 56` bytes +- **Easy cleanup**: Delete old segments without affecting current data + +## Cross-Segment Queries + +When fetching data that spans multiple segments: + +1. **Segment range computation**: Calculate which segment IDs intersect `[from, to)` + ```javascript + firstSegmentStartTs = floor(from / segmentSpanMs) * segmentSpanMs + lastSegmentStartTs = floor((to - 1) / segmentSpanMs) * segmentSpanMs + ``` + +2. **Index calculation**: For each segment, compute local indices + ```javascript + localIndex = floor((barTs - segmentStartTs) / timeframeMs) + ``` + +3. **Efficient reads**: Only read the required byte ranges from each segment + +4. **Missing segments**: If a segment file doesn't exist, that time range is simply empty + +## Backfill with Segments + +Backfill support works with segments: + +1. **Segment lookup**: Compute which segment the bucket belongs to +2. **Single-record read**: Read only the 56-byte record at the correct offset +3. **Merge in memory**: Combine disk data with incoming trade +4. **Write back**: Upsert the merged bar to the correct segment diff --git a/src/storage/binaries/constants.js b/src/storage/binaries/constants.js index 6da8b21..e692a1a 100644 --- a/src/storage/binaries/constants.js +++ b/src/storage/binaries/constants.js @@ -1,3 +1,5 @@ +const config = require('../../config') + /** * Size in bytes of a single OHLCV record in the binary file. * Layout (56 bytes total): @@ -30,6 +32,90 @@ const PRICE_SCALE = 10000 */ const VOLUME_SCALE = 1000000 +/** + * Default number of records per segment file. + * Each segment stores exactly this many records (dense indexing). + * segmentSpanMs = timeframeMs * SEGMENT_RECORDS + * + * Can be overridden via config.binariesSegmentRecords (number or per-timeframe object) + * @constant {number} + */ +const DEFAULT_SEGMENT_RECORDS = 4096 + +/** + * Get segment records count for a specific timeframe. + * Supports both single value and per-timeframe configuration. + * + * @param {number} timeframeMs - Timeframe in milliseconds + * @returns {number} Number of records per segment + */ +function getSegmentRecords(timeframeMs) { + const configValue = config.binariesSegmentRecords + if (!configValue) return DEFAULT_SEGMENT_RECORDS + if (typeof configValue === 'number') return configValue + if (typeof configValue === 'object') { + // Check for timeframe-specific override (keys can be ms or human-readable) + const msKey = String(timeframeMs) + if (configValue[msKey]) return configValue[msKey] + // Try human-readable key lookup (e.g., "10s", "1m") + const { getHms } = require('../../helper') + const hmsKey = getHms(timeframeMs) + if (configValue[hmsKey]) return configValue[hmsKey] + return DEFAULT_SEGMENT_RECORDS + } + return DEFAULT_SEGMENT_RECORDS +} + +/** + * Compute the segment span in milliseconds for a given timeframe. + * + * @param {number} timeframeMs - Timeframe in milliseconds + * @returns {number} Segment span in milliseconds + */ +function getSegmentSpanMs(timeframeMs) { + return timeframeMs * getSegmentRecords(timeframeMs) +} + +/** + * Compute the segment start timestamp for a given bar timestamp. + * + * @param {number} ts - Bar timestamp in milliseconds + * @param {number} timeframeMs - Timeframe in milliseconds + * @returns {number} Segment start timestamp + */ +function getSegmentStartTs(ts, timeframeMs) { + const segmentSpanMs = getSegmentSpanMs(timeframeMs) + return Math.floor(ts / segmentSpanMs) * segmentSpanMs +} + +/** + * Compute the segment ID (string) for a given bar timestamp. + * The segment ID is the segment start timestamp as a string. + * + * @param {number} ts - Bar timestamp in milliseconds + * @param {number} timeframeMs - Timeframe in milliseconds + * @returns {string} Segment ID + */ +function getSegmentId(ts, timeframeMs) { + return String(getSegmentStartTs(ts, timeframeMs)) +} + +/** + * @typedef {Object} SegmentMeta + * @property {string} exchange - Exchange name + * @property {string} symbol - Trading pair symbol + * @property {string} timeframe - Human-readable timeframe label (e.g., "5s", "1m") + * @property {number} timeframeMs - Timeframe in milliseconds + * @property {number} segmentStartTs - Segment start timestamp (first record time) + * @property {number} segmentEndTs - Segment end timestamp (segmentStartTs + segmentSpanMs) + * @property {number} segmentSpanMs - Total time span of segment in ms + * @property {number} segmentRecords - Fixed number of records in this segment + * @property {number} priceScale - Price scaling factor used + * @property {number} volumeScale - Volume scaling factor used + * @property {number} records - Current number of records written (may be less than segmentRecords) + * @property {number} lastInputStartTs - Last input bar timestamp processed + */ + /** * @typedef {Object} BinaryMeta * @property {string} exchange - Exchange name @@ -42,6 +128,7 @@ const VOLUME_SCALE = 1000000 * @property {number} volumeScale - Volume scaling factor used * @property {number} records - Total number of records in the binary file * @property {number} lastInputStartTs - Last input bar timestamp processed + * @deprecated Use SegmentMeta for segmented storage */ /** @@ -95,5 +182,10 @@ const VOLUME_SCALE = 1000000 module.exports = { RECORD_SIZE, PRICE_SCALE, - VOLUME_SCALE + VOLUME_SCALE, + DEFAULT_SEGMENT_RECORDS, + getSegmentRecords, + getSegmentSpanMs, + getSegmentStartTs, + getSegmentId } diff --git a/src/storage/binaries/index.js b/src/storage/binaries/index.js index e8e19fa..e66780d 100644 --- a/src/storage/binaries/index.js +++ b/src/storage/binaries/index.js @@ -4,8 +4,8 @@ const alertService = require('../../services/alert') const { updateIndexes } = require('../../services/connections') const { getHms } = require('../../helper') -const { RECORD_SIZE } = require('./constants') -const { getFilePath, parseMarket, readRecord, readMeta, readSingleRecordAtTs } = require('./io') +const { RECORD_SIZE, getSegmentStartTs, getSegmentSpanMs } = require('./constants') +const { getSegmentPaths, parseMarket, readRecord, readMeta, readSingleRecordAtTs } = require('./io') const { upsertBars } = require('./write') const { resampleToHigherTimeframes } = require('./resample') @@ -335,6 +335,9 @@ class BinariesStorage { * Null bars (gaps) are skipped in the response. * Pending bars near "now" are injected to provide real-time data. * + * Now reads from segmented files: computes which segments intersect [from, to) + * and reads only the required records from each segment. + * * @param {Object} params - Query parameters * @param {number} params.from - Start timestamp (inclusive) * @param {number} params.to - End timestamp (exclusive) @@ -372,80 +375,93 @@ class BinariesStorage { const now = Date.now() const needsPendingBars = to > now - config.influxResampleInterval + // Compute segment range for the query + const segmentSpanMs = getSegmentSpanMs(timeframe) + const firstSegmentStartTs = getSegmentStartTs(from, timeframe) + const lastSegmentStartTs = getSegmentStartTs(to - 1, timeframe) // to is exclusive + // Read data from disk for each market for (const market of markets) { const parsed = parseMarket(market) if (!parsed) continue - const paths = getFilePath(parsed.exchange, parsed.symbol, timeframe) - const meta = readMeta(paths.json) - if (!meta) continue - - if (!fs.existsSync(paths.bin)) continue - - // Calculate index range for requested time window - let startIndex = Math.floor((from - meta.startTs) / meta.timeframeMs) - let endIndex = Math.ceil((to - meta.startTs) / meta.timeframeMs) - - // Clamp to valid range - if (startIndex < 0) startIndex = 0 - if (endIndex > meta.records) endIndex = meta.records - if (startIndex >= endIndex) continue - - const count = endIndex - startIndex - const byteOffset = startIndex * RECORD_SIZE - const byteLength = count * RECORD_SIZE - - // Read the range from disk - let buffer - let fd - try { - fd = fs.openSync(paths.bin, 'r') - const stat = fs.fstatSync(fd) - const maxBytes = Math.min(byteLength, stat.size - byteOffset) - if (maxBytes <= 0) { + // Iterate through all segments that intersect [from, to) + for (let segmentStartTs = firstSegmentStartTs; segmentStartTs <= lastSegmentStartTs; segmentStartTs += segmentSpanMs) { + const paths = getSegmentPaths(parsed.exchange, parsed.symbol, timeframe, segmentStartTs) + const meta = readMeta(paths.json) + if (!meta) continue + if (!fs.existsSync(paths.bin)) continue + + // Calculate index range within this segment for the requested time window + const segmentEndTs = segmentStartTs + segmentSpanMs + const readFromTs = Math.max(from, segmentStartTs) + const readToTs = Math.min(to, segmentEndTs) + + if (readFromTs >= readToTs) continue + + let startIndex = Math.floor((readFromTs - segmentStartTs) / timeframe) + let endIndex = Math.ceil((readToTs - segmentStartTs) / timeframe) + + // Clamp to valid range + if (startIndex < 0) startIndex = 0 + if (endIndex > meta.records) endIndex = meta.records + if (startIndex >= endIndex) continue + + const count = endIndex - startIndex + const byteOffset = startIndex * RECORD_SIZE + const byteLength = count * RECORD_SIZE + + // Read the range from disk + let buffer + let fd + try { + fd = fs.openSync(paths.bin, 'r') + const stat = fs.fstatSync(fd) + const maxBytes = Math.min(byteLength, stat.size - byteOffset) + if (maxBytes <= 0) { + fs.closeSync(fd) + continue + } + buffer = Buffer.alloc(maxBytes) + fs.readSync(fd, buffer, 0, maxBytes, byteOffset) fs.closeSync(fd) + } catch (_e) { + if (fd !== undefined) { + try { fs.closeSync(fd) } catch (_) { /* ignored */ } + } continue } - buffer = Buffer.alloc(maxBytes) - fs.readSync(fd, buffer, 0, maxBytes, byteOffset) - fs.closeSync(fd) - } catch (_e) { - if (fd !== undefined) { - try { fs.closeSync(fd) } catch (_) { /* ignored */ } - } - continue - } - - // Decode records and add to results - const recordCount = Math.floor(buffer.length / RECORD_SIZE) - for (let i = 0; i < recordCount; i++) { - const record = readRecord(buffer, i * RECORD_SIZE, meta) - // Skip null bars (gaps) - check before computing barTime for efficiency - if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) continue - - const barTime = meta.startTs + (startIndex + i) * meta.timeframeMs - - // Filter by time range - if (barTime < from || barTime >= to) continue - - // Add as array matching column order (includes market) - // Time is converted to seconds to match InfluxDB format (precision: 's') - results.push([ - Math.floor(barTime / 1000), - market, - record.open, - record.high, - record.low, - record.close, - record.vbuy, - record.vsell, - record.cbuy, - record.csell, - record.lbuy, - record.lsell - ]) + // Decode records and add to results + const recordCount = Math.floor(buffer.length / RECORD_SIZE) + for (let i = 0; i < recordCount; i++) { + const record = readRecord(buffer, i * RECORD_SIZE, meta) + + // Skip null bars (gaps) - check before computing barTime for efficiency + if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) continue + + const barTime = segmentStartTs + (startIndex + i) * timeframe + + // Filter by time range + if (barTime < from || barTime >= to) continue + + // Add as array matching column order (includes market) + // Time is converted to seconds to match InfluxDB format (precision: 's') + results.push([ + Math.floor(barTime / 1000), + market, + record.open, + record.high, + record.low, + record.close, + record.vbuy, + record.vsell, + record.cbuy, + record.csell, + record.lbuy, + record.lsell + ]) + } } } diff --git a/src/storage/binaries/io.js b/src/storage/binaries/io.js index c2c091d..afa31b9 100644 --- a/src/storage/binaries/io.js +++ b/src/storage/binaries/io.js @@ -2,11 +2,12 @@ const fs = require('fs') const path = require('path') const config = require('../../config') const { getHms } = require('../../helper') -const { RECORD_SIZE } = require('./constants') +const { RECORD_SIZE, getSegmentStartTs, getSegmentSpanMs, getSegmentRecords } = require('./constants') /** - * Generates file paths for a market's binary and metadata files. + * Generates file paths for a market's binary and metadata files (legacy single-file format). * + * @deprecated Use getSegmentPaths for segmented storage * @param {string} exchange - Exchange name * @param {string} symbol - Trading pair symbol (will be sanitized) * @param {number} timeframe - Timeframe in milliseconds @@ -23,6 +24,43 @@ function getFilePath(exchange, symbol, timeframe) { } } +/** + * Generates file paths for a segment's binary and metadata files. + * + * New directory structure: + * data/{exchange}/{symbol}/{timeframe}/{segmentStartTs}.bin|json + * + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol (will be sanitized) + * @param {number} timeframe - Timeframe in milliseconds + * @param {number} segmentStartTs - Segment start timestamp + * @returns {import('./constants').FilePaths} Object with bin and json file paths + */ +function getSegmentPaths(exchange, symbol, timeframe, segmentStartTs) { + const sanitizedSymbol = symbol.replace(/[/:]/g, '-') + const tfLabel = getHms(timeframe) + const segmentId = String(segmentStartTs) + const dir = path.join(config.filesLocation, exchange, sanitizedSymbol, tfLabel) + return { + bin: path.join(dir, `${segmentId}.bin`), + json: path.join(dir, `${segmentId}.json`) + } +} + +/** + * Gets the timeframe directory path (contains all segments for a timeframe). + * + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol (will be sanitized) + * @param {number} timeframe - Timeframe in milliseconds + * @returns {string} Directory path + */ +function getTimeframeDir(exchange, symbol, timeframe) { + const sanitizedSymbol = symbol.replace(/[/:]/g, '-') + const tfLabel = getHms(timeframe) + return path.join(config.filesLocation, exchange, sanitizedSymbol, tfLabel) +} + /** * Parses a market string into exchange and symbol components. * @@ -144,7 +182,7 @@ function writeMeta(jsonPath, meta) { } /** - * Reads a single record from the binary file at a specific bucket timestamp. + * Reads a single record from a segment file at a specific bucket timestamp. * Uses positioned read (pread) to read only the needed 56 bytes. * * This is used as a disk fallback when processing trades for buckets @@ -157,16 +195,22 @@ function writeMeta(jsonPath, meta) { * @returns {import('./constants').Bar|null} Bar data if found and non-null, otherwise null */ function readSingleRecordAtTs(exchange, symbol, timeframe, bucketTs) { - const paths = getFilePath(exchange, symbol, timeframe) + // Compute which segment this bucket belongs to + const segmentStartTs = getSegmentStartTs(bucketTs, timeframe) + const paths = getSegmentPaths(exchange, symbol, timeframe, segmentStartTs) const meta = readMeta(paths.json) + if (!meta) return null if (!fs.existsSync(paths.bin)) return null - // Check if bucketTs is within the file's range - if (bucketTs < meta.startTs || bucketTs >= meta.endTs) return null + // Compute the segment span and validate bucket is within segment bounds + const segmentSpanMs = getSegmentSpanMs(timeframe) + const segmentEndTs = segmentStartTs + segmentSpanMs + + if (bucketTs < segmentStartTs || bucketTs >= segmentEndTs) return null - // Calculate record index and byte offset - const index = Math.floor((bucketTs - meta.startTs) / meta.timeframeMs) + // Calculate record index within the segment + const index = Math.floor((bucketTs - segmentStartTs) / timeframe) if (index < 0 || index >= meta.records) return null const byteOffset = index * RECORD_SIZE @@ -212,12 +256,36 @@ function readSingleRecordAtTs(exchange, symbol, timeframe, bucketTs) { } } +/** + * Read segment metadata JSON file. + * + * @param {string} jsonPath - Path to the JSON metadata file + * @returns {import('./constants').SegmentMeta|null} Parsed metadata or null if not found + */ +function readSegmentMeta(jsonPath) { + return readMeta(jsonPath) +} + +/** + * Writes segment metadata JSON file atomically (write to temp, then rename). + * + * @param {string} jsonPath - Path to the JSON metadata file + * @param {import('./constants').SegmentMeta} meta - Metadata to write + */ +function writeSegmentMeta(jsonPath, meta) { + writeMeta(jsonPath, meta) +} + module.exports = { getFilePath, + getSegmentPaths, + getTimeframeDir, parseMarket, writeRecord, readRecord, readMeta, writeMeta, - readSingleRecordAtTs + readSingleRecordAtTs, + readSegmentMeta, + writeSegmentMeta } diff --git a/src/storage/binaries/resample.js b/src/storage/binaries/resample.js index e0cca22..ccc1099 100644 --- a/src/storage/binaries/resample.js +++ b/src/storage/binaries/resample.js @@ -1,11 +1,12 @@ const fs = require('fs') const { ensureDirectoryExists, getHms } = require('../../helper') -const { RECORD_SIZE, PRICE_SCALE, VOLUME_SCALE } = require('./constants') -const { getFilePath, readRecord, writeRecord, readMeta, writeMeta } = require('./io') -const { createNullBar } = require('./write') +const { RECORD_SIZE, PRICE_SCALE, VOLUME_SCALE, getSegmentStartTs, getSegmentSpanMs, getSegmentRecords } = require('./constants') +const { getFilePath, getSegmentPaths, readRecord, writeRecord, readMeta, writeMeta } = require('./io') +const { createNullBar, upsertBarsToSegment } = require('./write') /** * Triggers resampling from base timeframe to all configured higher timeframes. + * Now reads from segmented base timeframe files and writes to segmented target files. * * @param {Object} storage - BinariesStorage instance * @param {string} exchange - Exchange name @@ -20,23 +21,15 @@ async function resampleToHigherTimeframes(storage, exchange, symbol, fromTs, toT if (higherTimeframes.length === 0) return - const basePaths = getFilePath(exchange, symbol, baseTimeframe) - const baseMeta = readMeta(basePaths.json) - if (!baseMeta || !fs.existsSync(basePaths.bin)) return - // Resample each higher timeframe for (const targetTimeframe of higherTimeframes) { - await resampleRangeToTimeframe(exchange, symbol, baseMeta, basePaths.bin, fromTs, toTs, targetTimeframe) + await resampleRangeToTimeframe(exchange, symbol, baseTimeframe, fromTs, toTs, targetTimeframe) } } /** * Resamples a range of base timeframe data into a higher target timeframe. - * - * Unlike append-only resampling, this supports upsert semantics: - * - Overwrites existing target buckets that were affected by base changes - * - Appends new target buckets as needed - * - Maintains dense indexing (null bars fill gaps) + * Now reads from segmented base files and writes to segmented target files. * * Aggregation rules: * - open: first non-null value in bucket (by base bar time order) @@ -46,66 +39,57 @@ async function resampleToHigherTimeframes(storage, exchange, symbol, fromTs, toT * * @param {string} exchange - Exchange name * @param {string} symbol - Trading pair symbol - * @param {import('./constants').BinaryMeta} baseMeta - Metadata for base timeframe file - * @param {string} baseBinPath - Path to base timeframe binary file + * @param {number} baseTimeframe - Base timeframe in milliseconds * @param {number} fromTs - Start of dirty range (base bucket time) * @param {number} toTs - End of dirty range (base bucket time) * @param {number} targetTimeframe - Target timeframe in milliseconds * @returns {Promise} */ -async function resampleRangeToTimeframe(exchange, symbol, baseMeta, baseBinPath, fromTs, toTs, targetTimeframe) { - const baseTimeframe = baseMeta.timeframeMs - const paths = getFilePath(exchange, symbol, targetTimeframe) - await ensureDirectoryExists(paths.bin) - - let meta = readMeta(paths.json) - +async function resampleRangeToTimeframe(exchange, symbol, baseTimeframe, fromTs, toTs, targetTimeframe) { // Compute dirty target bucket range (inclusive) - // A target bucket is dirty if any of its contributing base buckets were modified const firstTargetBucket = Math.floor(fromTs / targetTimeframe) * targetTimeframe const lastTargetBucket = Math.floor(toTs / targetTimeframe) * targetTimeframe - // Initialize metadata if this is a new file - if (!meta) { - meta = { - exchange: exchange, - symbol: symbol, - timeframe: getHms(targetTimeframe), - timeframeMs: targetTimeframe, - startTs: firstTargetBucket, - endTs: firstTargetBucket, - priceScale: PRICE_SCALE, - volumeScale: VOLUME_SCALE, - records: 0, - lastInputStartTs: firstTargetBucket - } - } + // Determine which base segments need to be read + // We need to read all base bars that contribute to [firstTargetBucket, lastTargetBucket + targetTimeframe) + const baseReadFrom = firstTargetBucket + const baseReadTo = lastTargetBucket + targetTimeframe - 1 // Last base bar we might need - // Skip buckets before meta.startTs (no prepend allowed) - const startBucket = Math.max(firstTargetBucket, meta.startTs) - if (startBucket > lastTargetBucket) return + // Collect all base segments that intersect this range + const baseSegmentSpanMs = getSegmentSpanMs(baseTimeframe) + const firstBaseSegment = getSegmentStartTs(baseReadFrom, baseTimeframe) + const lastBaseSegment = getSegmentStartTs(baseReadTo, baseTimeframe) - // Calculate base record index range covering the dirty target buckets - // Need to read all base bars that contribute to [startBucket, lastTargetBucket + targetTimeframe) - const baseStartIndex = Math.floor((startBucket - baseMeta.startTs) / baseTimeframe) - const baseEndIndex = Math.ceil((lastTargetBucket + targetTimeframe - baseMeta.startTs) / baseTimeframe) + // Read base bars from all relevant segments + const baseBarsByTime = {} // ts -> record - // Clamp to valid index range within the base file - const clampedStartIndex = Math.max(0, baseStartIndex) - const clampedEndIndex = Math.min(baseMeta.records, baseEndIndex) + for (let segmentStartTs = firstBaseSegment; segmentStartTs <= lastBaseSegment; segmentStartTs += baseSegmentSpanMs) { + const paths = getSegmentPaths(exchange, symbol, baseTimeframe, segmentStartTs) + const meta = readMeta(paths.json) + if (!meta || !fs.existsSync(paths.bin)) continue - // Aggregate base bars into target buckets - // Key: target bucket timestamp, Value: aggregated bar data - const aggregated = {} + // Calculate which records from this segment we need + const segmentEndTs = segmentStartTs + baseSegmentSpanMs + const readFromTs = Math.max(baseReadFrom, segmentStartTs) + const readToTs = Math.min(baseReadTo, segmentEndTs - 1) + + if (readFromTs > readToTs) continue - if (clampedStartIndex < clampedEndIndex) { - const count = clampedEndIndex - clampedStartIndex - const byteOffset = clampedStartIndex * RECORD_SIZE + const startIndex = Math.floor((readFromTs - segmentStartTs) / baseTimeframe) + const endIndex = Math.min( + Math.ceil((readToTs + 1 - segmentStartTs) / baseTimeframe), + meta.records + ) + + if (startIndex >= endIndex) continue + + const count = endIndex - startIndex + const byteOffset = startIndex * RECORD_SIZE const byteLength = count * RECORD_SIZE let buffer try { - const fd = fs.openSync(baseBinPath, 'r') + const fd = fs.openSync(paths.bin, 'r') const stat = fs.fstatSync(fd) const maxBytes = Math.min(byteLength, stat.size - byteOffset) if (maxBytes > 0) { @@ -119,111 +103,187 @@ async function resampleRangeToTimeframe(exchange, symbol, baseMeta, baseBinPath, if (buffer) { const recordCount = Math.floor(buffer.length / RECORD_SIZE) - - // Process each base record and aggregate into target buckets for (let i = 0; i < recordCount; i++) { - const record = readRecord(buffer, i * RECORD_SIZE, baseMeta) - const barTime = baseMeta.startTs + (clampedStartIndex + i) * baseTimeframe + const record = readRecord(buffer, i * RECORD_SIZE, meta) + const barTime = segmentStartTs + (startIndex + i) * baseTimeframe - // Skip null records (gaps in base data) + // Skip null records if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) continue - // Determine which target bucket this base bar contributes to - const targetBucketTime = Math.floor(barTime / targetTimeframe) * targetTimeframe - if (targetBucketTime < startBucket || targetBucketTime > lastTargetBucket) continue - - if (!aggregated[targetBucketTime]) { - // First contributing base bar for this target bucket - aggregated[targetBucketTime] = { - time: targetBucketTime, - open: record.open, - high: record.high, - low: record.low, - close: record.close, - vbuy: record.vbuy, - vsell: record.vsell, - cbuy: record.cbuy, - csell: record.csell, - lbuy: record.lbuy, - lsell: record.lsell, - _firstBarTime: barTime // Track earliest base bar for open determination - } - } else { - // Merge subsequent base bars into the aggregate - const agg = aggregated[targetBucketTime] - - // Open comes from the earliest base bar (by time, not arrival order) - if (barTime < agg._firstBarTime) { - agg.open = record.open - agg._firstBarTime = barTime - } + baseBarsByTime[barTime] = record + } + } + } - // High/low are extrema, close comes from the last bar processed - agg.high = Math.max(agg.high, record.high) - agg.low = Math.min(agg.low, record.low) - agg.close = record.close - - // Volumes and counts are summed - agg.vbuy += record.vbuy - agg.vsell += record.vsell - agg.cbuy += record.cbuy - agg.csell += record.csell - agg.lbuy += record.lbuy - agg.lsell += record.lsell - } + // Aggregate base bars into target buckets + const aggregated = {} + + for (const barTimeStr in baseBarsByTime) { + const barTime = Number(barTimeStr) + const record = baseBarsByTime[barTime] + const targetBucketTime = Math.floor(barTime / targetTimeframe) * targetTimeframe + + if (targetBucketTime < firstTargetBucket || targetBucketTime > lastTargetBucket) continue + + if (!aggregated[targetBucketTime]) { + aggregated[targetBucketTime] = { + time: targetBucketTime, + open: record.open, + high: record.high, + low: record.low, + close: record.close, + vbuy: record.vbuy, + vsell: record.vsell, + cbuy: record.cbuy, + csell: record.csell, + lbuy: record.lbuy, + lsell: record.lsell, + _firstBarTime: barTime + } + } else { + const agg = aggregated[targetBucketTime] + if (barTime < agg._firstBarTime) { + agg.open = record.open + agg._firstBarTime = barTime } + agg.high = Math.max(agg.high, record.high) + agg.low = Math.min(agg.low, record.low) + agg.close = record.close + agg.vbuy += record.vbuy + agg.vsell += record.vsell + agg.cbuy += record.cbuy + agg.csell += record.csell + agg.lbuy += record.lbuy + agg.lsell += record.lsell } } - // Build complete list of bars to upsert (dense: one per target bucket in range) - const barsToUpsert = [] - for (let bucketTs = startBucket; bucketTs <= lastTargetBucket; bucketTs += targetTimeframe) { + // Build list of bars to write (fill gaps with null bars) + const barsToWrite = [] + for (let bucketTs = firstTargetBucket; bucketTs <= lastTargetBucket; bucketTs += targetTimeframe) { if (aggregated[bucketTs]) { - barsToUpsert.push(aggregated[bucketTs]) + const bar = aggregated[bucketTs] + delete bar._firstBarTime // Remove internal tracking field + barsToWrite.push(bar) } else { - // No contributing base bars - write null bar for dense indexing - barsToUpsert.push(createNullBar(bucketTs)) + barsToWrite.push(createNullBar(bucketTs)) } } - if (barsToUpsert.length === 0) return + if (barsToWrite.length === 0) return + + // Group bars by target segment and upsert each segment + const targetSegmentSpanMs = getSegmentSpanMs(targetTimeframe) + const barsBySegment = {} + + for (const bar of barsToWrite) { + const segmentStartTs = getSegmentStartTs(bar.time, targetTimeframe) + if (!barsBySegment[segmentStartTs]) { + barsBySegment[segmentStartTs] = [] + } + barsBySegment[segmentStartTs].push(bar) + } - // Separate bars into overwrites (existing records) vs appends (new records) + // Upsert each target segment + for (const segmentStartTsStr in barsBySegment) { + const segmentStartTs = Number(segmentStartTsStr) + const segmentBars = barsBySegment[segmentStartTs] + + // Filter out null-only bar sets (don't create segment files just for nulls) + const hasNonNull = segmentBars.some(b => b.close !== null) + + // For resampling, we need to write even null bars to maintain consistency + // But only if there's existing data or non-null bars + const paths = getSegmentPaths(exchange, symbol, targetTimeframe, segmentStartTs) + const existingMeta = readMeta(paths.json) + + if (hasNonNull || existingMeta) { + await upsertBarsToSegmentForResample(exchange, symbol, targetTimeframe, segmentStartTs, segmentBars) + } + } +} + +/** + * Upserts bars to a segment for resampling purposes. + * Unlike regular upsert, this handles null bars for dense indexing in resampled data. + * + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol + * @param {number} timeframe - Timeframe in milliseconds + * @param {number} segmentStartTs - Segment start timestamp + * @param {import('./constants').Bar[]} bars - Array of bars (may include null bars) + * @returns {Promise} + */ +async function upsertBarsToSegmentForResample(exchange, symbol, timeframe, segmentStartTs, bars) { + const paths = getSegmentPaths(exchange, symbol, timeframe, segmentStartTs) + await ensureDirectoryExists(paths.bin) + + let meta = readMeta(paths.json) + const segmentRecords = getSegmentRecords(timeframe) + const segmentSpanMs = getSegmentSpanMs(timeframe) + const segmentEndTs = segmentStartTs + segmentSpanMs + + // Initialize segment metadata if new file + if (!meta) { + meta = { + exchange: exchange, + symbol: symbol, + timeframe: getHms(timeframe), + timeframeMs: timeframe, + segmentStartTs: segmentStartTs, + segmentEndTs: segmentEndTs, + segmentSpanMs: segmentSpanMs, + segmentRecords: segmentRecords, + priceScale: PRICE_SCALE, + volumeScale: VOLUME_SCALE, + records: 0, + lastInputStartTs: segmentStartTs + } + } + + // Build map of index to bar + const barsMap = {} + let maxBarTime = segmentStartTs + + for (const bar of bars) { + const alignedTime = Math.floor(bar.time / timeframe) * timeframe + if (alignedTime < segmentStartTs || alignedTime >= segmentEndTs) continue + const index = Math.floor((alignedTime - segmentStartTs) / timeframe) + if (index < 0 || index >= segmentRecords) continue + barsMap[index] = { ...bar, time: alignedTime } + if (alignedTime > maxBarTime) maxBarTime = alignedTime + } + + const indices = Object.keys(barsMap).map(Number).sort((a, b) => a - b) + if (indices.length === 0) return + + // Categorize as overwrites or appends const overwrites = [] const appends = [] - for (const bar of barsToUpsert) { - const index = Math.floor((bar.time - meta.startTs) / targetTimeframe) - if (index < 0) continue // No prepend allowed + for (const index of indices) { if (index < meta.records) { - overwrites.push({ index, bar }) + overwrites.push({ index, bar: barsMap[index] }) } else { - appends.push({ index, bar }) + appends.push({ index, bar: barsMap[index] }) } } - // Handle overwrites (positioned writes to existing records) + // Handle overwrites if (overwrites.length > 0) { let fd = null try { - // Open in read-write mode for in-place updates fd = fs.openSync(paths.bin, 'r+') - - // Sort by index for sequential access and batch contiguous writes overwrites.sort((a, b) => a.index - b.index) let groupStart = 0 - // Group contiguous indices for efficient batch writes while (groupStart < overwrites.length) { let groupEnd = groupStart - - // Find end of contiguous run while (groupEnd + 1 < overwrites.length && overwrites[groupEnd + 1].index === overwrites[groupEnd].index + 1) { groupEnd++ } - // Write contiguous group in single syscall const groupSize = groupEnd - groupStart + 1 const buffer = Buffer.alloc(groupSize * RECORD_SIZE) let bufferOffset = 0 @@ -235,25 +295,23 @@ async function resampleRangeToTimeframe(exchange, symbol, baseMeta, baseBinPath, const fileOffset = overwrites[groupStart].index * RECORD_SIZE fs.writeSync(fd, buffer, 0, buffer.length, fileOffset) - groupStart = groupEnd + 1 } } finally { - if (fd !== null) { - fs.closeSync(fd) - } + if (fd !== null) fs.closeSync(fd) } } - // Handle appends (extend file with new records) + // Handle appends if (appends.length > 0) { appends.sort((a, b) => a.index - b.index) - const lastAppendIndex = appends[appends.length - 1].index + const lastAppendIndex = Math.min(appends[appends.length - 1].index, segmentRecords - 1) - // Build lookup map for sparse appends const appendMap = {} for (const a of appends) { - appendMap[a.index] = a.bar + if (a.index <= lastAppendIndex) { + appendMap[a.index] = a.bar + } } const BATCH_SIZE = 10000 @@ -261,10 +319,8 @@ async function resampleRangeToTimeframe(exchange, symbol, baseMeta, baseBinPath, let fd = null try { - // Open in append mode fd = fs.openSync(paths.bin, 'a') - // Write records from current end to last append index (dense, filling gaps with null) while (currentIndex <= lastAppendIndex) { const batchRecords = [] const batchEndIndex = Math.min(currentIndex + BATCH_SIZE, lastAppendIndex + 1) @@ -273,8 +329,7 @@ async function resampleRangeToTimeframe(exchange, symbol, baseMeta, baseBinPath, if (appendMap[currentIndex]) { batchRecords.push(appendMap[currentIndex]) } else { - // Fill gap with null bar - batchRecords.push(createNullBar(meta.startTs + currentIndex * targetTimeframe)) + batchRecords.push(createNullBar(segmentStartTs + currentIndex * timeframe)) } currentIndex++ } @@ -293,20 +348,15 @@ async function resampleRangeToTimeframe(exchange, symbol, baseMeta, baseBinPath, } } } finally { - if (fd !== null) { - fs.closeSync(fd) - } + if (fd !== null) fs.closeSync(fd) } - - // Update endTs after extending file - meta.endTs = meta.startTs + meta.records * targetTimeframe } - meta.lastInputStartTs = lastTargetBucket + meta.lastInputStartTs = maxBarTime writeMeta(paths.json, meta) } module.exports = { resampleToHigherTimeframes, resampleRangeToTimeframe -} +} \ No newline at end of file diff --git a/src/storage/binaries/write.js b/src/storage/binaries/write.js index c4972ae..15a4c86 100644 --- a/src/storage/binaries/write.js +++ b/src/storage/binaries/write.js @@ -1,7 +1,7 @@ const fs = require('fs') const { ensureDirectoryExists, getHms } = require('../../helper') -const { RECORD_SIZE, PRICE_SCALE, VOLUME_SCALE } = require('./constants') -const { getFilePath, writeRecord, readMeta, writeMeta } = require('./io') +const { RECORD_SIZE, PRICE_SCALE, VOLUME_SCALE, getSegmentStartTs, getSegmentSpanMs, getSegmentRecords } = require('./constants') +const { getFilePath, getSegmentPaths, writeRecord, readMeta, writeMeta } = require('./io') /** * Creates an empty (null) bar for gap filling. @@ -136,6 +136,8 @@ async function appendBars(exchange, symbol, timeframe, bars) { * - Bars with timestamps < endTs (but >= startTs) overwrite existing records * - Bars with timestamps < startTs are ignored (no prepend) * + * Now uses segmented storage: bars are split by segment and written to individual segment files. + * * @param {string} exchange - Exchange name * @param {string} symbol - Trading pair symbol * @param {number} timeframe - Timeframe in milliseconds @@ -143,63 +145,109 @@ async function appendBars(exchange, symbol, timeframe, bars) { * @returns {Promise} Timestamps of written data range (includes overwrites) */ async function upsertBars(exchange, symbol, timeframe, bars) { - const paths = getFilePath(exchange, symbol, timeframe) - await ensureDirectoryExists(paths.bin) - - let meta = readMeta(paths.json) - // Filter out bars with null close (empty bars) const validBars = bars.filter(b => b.close !== null) if (validBars.length === 0) return { fromTsWritten: null, toTsWritten: null } - // Build map of aligned timestamps to bars - const barsMap = {} + // Group bars by segment + const barsBySegment = {} for (const bar of validBars) { const alignedTime = Math.floor(bar.time / timeframe) * timeframe - barsMap[alignedTime] = bar + const segmentStartTs = getSegmentStartTs(alignedTime, timeframe) + if (!barsBySegment[segmentStartTs]) { + barsBySegment[segmentStartTs] = [] + } + barsBySegment[segmentStartTs].push({ ...bar, time: alignedTime }) } - const timestamps = Object.keys(barsMap).map(Number).sort((a, b) => a - b) - if (timestamps.length === 0) return { fromTsWritten: null, toTsWritten: null } + // Track the overall dirty range + let minTs = null + let maxTs = null + + // Upsert each segment + const segmentStartTimestamps = Object.keys(barsBySegment).map(Number).sort((a, b) => a - b) + for (const segmentStartTs of segmentStartTimestamps) { + const segmentBars = barsBySegment[segmentStartTs] + const result = await upsertBarsToSegment(exchange, symbol, timeframe, segmentStartTs, segmentBars) + + if (result.fromTsWritten !== null) { + if (minTs === null || result.fromTsWritten < minTs) minTs = result.fromTsWritten + if (maxTs === null || result.toTsWritten > maxTs) maxTs = result.toTsWritten + } + } - const firstBarTime = timestamps[0] + return { fromTsWritten: minTs, toTsWritten: maxTs } +} - // Initialize metadata if new file +/** + * Upserts bars to a specific segment file. + * + * Segment characteristics: + * - Fixed number of records (segmentRecords) + * - Dense indexing: index = (barTs - segmentStartTs) / timeframe + * - Index must be in range [0, segmentRecords) + * + * @param {string} exchange - Exchange name + * @param {string} symbol - Trading pair symbol + * @param {number} timeframe - Timeframe in milliseconds + * @param {number} segmentStartTs - Segment start timestamp + * @param {import('./constants').Bar[]} bars - Array of bars to upsert (all must belong to this segment) + * @returns {Promise} Timestamps of written data range + */ +async function upsertBarsToSegment(exchange, symbol, timeframe, segmentStartTs, bars) { + const paths = getSegmentPaths(exchange, symbol, timeframe, segmentStartTs) + await ensureDirectoryExists(paths.bin) + + let meta = readMeta(paths.json) + const segmentRecords = getSegmentRecords(timeframe) + const segmentSpanMs = getSegmentSpanMs(timeframe) + const segmentEndTs = segmentStartTs + segmentSpanMs + + // Initialize segment metadata if new file if (!meta) { meta = { exchange: exchange, symbol: symbol, timeframe: getHms(timeframe), timeframeMs: timeframe, - startTs: firstBarTime, - endTs: firstBarTime, + segmentStartTs: segmentStartTs, + segmentEndTs: segmentEndTs, + segmentSpanMs: segmentSpanMs, + segmentRecords: segmentRecords, priceScale: PRICE_SCALE, volumeScale: VOLUME_SCALE, records: 0, - lastInputStartTs: firstBarTime + lastInputStartTs: segmentStartTs } } - // Filter out bars before startTs (no prepend allowed) - const filteredTimestamps = timestamps.filter(ts => ts >= meta.startTs) - if (filteredTimestamps.length === 0) return { fromTsWritten: null, toTsWritten: null } + // Build map of index to bar, filtering out-of-range bars + const barsMap = {} + for (const bar of bars) { + const alignedTime = Math.floor(bar.time / timeframe) * timeframe + // Validate bar is within this segment + if (alignedTime < segmentStartTs || alignedTime >= segmentEndTs) continue + const index = Math.floor((alignedTime - segmentStartTs) / timeframe) + if (index < 0 || index >= segmentRecords) continue + barsMap[index] = { ...bar, time: alignedTime } + } + + const indices = Object.keys(barsMap).map(Number).sort((a, b) => a - b) + if (indices.length === 0) return { fromTsWritten: null, toTsWritten: null } - // Track the actual dirty range (for resampling) - const minTs = filteredTimestamps[0] - const maxTs = filteredTimestamps[filteredTimestamps.length - 1] + // Track dirty range + const minTs = segmentStartTs + indices[0] * timeframe + const maxTs = segmentStartTs + indices[indices.length - 1] * timeframe - // Categorize bars as overwrites or appends based on their index - const overwrites = [] // { index, ts, bar } - existing records to update - const appends = [] // { index, ts, bar } - new records to add + // Categorize as overwrites or appends + const overwrites = [] + const appends = [] - for (const ts of filteredTimestamps) { - const index = Math.floor((ts - meta.startTs) / timeframe) + for (const index of indices) { if (index < meta.records) { - // Index exists in file - this is an overwrite - overwrites.push({ index, ts, bar: barsMap[ts] }) + overwrites.push({ index, bar: barsMap[index] }) } else { - // Index beyond current file - this is an append - appends.push({ index, ts, bar: barsMap[ts] }) + appends.push({ index, bar: barsMap[index] }) } } @@ -207,23 +255,17 @@ async function upsertBars(exchange, symbol, timeframe, bars) { if (overwrites.length > 0) { let fd = null try { - // Open in read-write mode for in-place updates fd = fs.openSync(paths.bin, 'r+') - - // Sort and group contiguous indices for efficient batch writes overwrites.sort((a, b) => a.index - b.index) let groupStart = 0 while (groupStart < overwrites.length) { let groupEnd = groupStart - - // Find end of contiguous run while (groupEnd + 1 < overwrites.length && overwrites[groupEnd + 1].index === overwrites[groupEnd].index + 1) { groupEnd++ } - // Write contiguous group in single syscall const groupSize = groupEnd - groupStart + 1 const buffer = Buffer.alloc(groupSize * RECORD_SIZE) let bufferOffset = 0 @@ -235,25 +277,26 @@ async function upsertBars(exchange, symbol, timeframe, bars) { const fileOffset = overwrites[groupStart].index * RECORD_SIZE fs.writeSync(fd, buffer, 0, buffer.length, fileOffset) - groupStart = groupEnd + 1 } } finally { - if (fd !== null) { - fs.closeSync(fd) - } + if (fd !== null) fs.closeSync(fd) } } - // Handle appends (extend file with new records) + // Handle appends (extend file with new records, filling gaps) if (appends.length > 0) { appends.sort((a, b) => a.index - b.index) const lastAppendIndex = appends[appends.length - 1].index - // Build lookup map for sparse appends + // Cannot append beyond segment boundary + const maxAppendIndex = Math.min(lastAppendIndex, segmentRecords - 1) + const appendMap = {} for (const a of appends) { - appendMap[a.index] = a.bar + if (a.index <= maxAppendIndex) { + appendMap[a.index] = a.bar + } } const BATCH_SIZE = 10000 @@ -261,20 +304,17 @@ async function upsertBars(exchange, symbol, timeframe, bars) { let fd = null try { - // Open in append mode fd = fs.openSync(paths.bin, 'a') - // Write records from current end to last append index (dense, filling gaps) - while (currentIndex <= lastAppendIndex) { + while (currentIndex <= maxAppendIndex) { const batchRecords = [] - const batchEndIndex = Math.min(currentIndex + BATCH_SIZE, lastAppendIndex + 1) + const batchEndIndex = Math.min(currentIndex + BATCH_SIZE, maxAppendIndex + 1) while (currentIndex < batchEndIndex) { if (appendMap[currentIndex]) { batchRecords.push(appendMap[currentIndex]) } else { - // Fill gap with null bar for dense indexing - batchRecords.push(createNullBar(meta.startTs + currentIndex * timeframe)) + batchRecords.push(createNullBar(segmentStartTs + currentIndex * timeframe)) } currentIndex++ } @@ -293,24 +333,19 @@ async function upsertBars(exchange, symbol, timeframe, bars) { } } } finally { - if (fd !== null) { - fs.closeSync(fd) - } + if (fd !== null) fs.closeSync(fd) } - - // Update endTs after extending file - meta.endTs = meta.startTs + meta.records * timeframe } meta.lastInputStartTs = maxTs writeMeta(paths.json, meta) - // Return the full dirty range (includes overwrites, not just appends) return { fromTsWritten: minTs, toTsWritten: maxTs } } module.exports = { createNullBar, appendBars, - upsertBars + upsertBars, + upsertBarsToSegment } From 0a91adf0be141e6701879d87ba869a6832f9e762 Mon Sep 17 00:00:00 2001 From: Tucsky Date: Sun, 18 Jan 2026 12:42:41 +0100 Subject: [PATCH 5/6] feat: update exchange constructors to accept exchange identifiers --- src/exchange.js | 8 +++++++- src/exchanges/aster.js | 3 +-- src/exchanges/binance.js | 4 +--- src/exchanges/binance_futures.js | 5 ++--- src/exchanges/binance_us.js | 3 +-- src/exchanges/bitfinex.js | 3 +-- src/exchanges/bitget.js | 3 +-- src/exchanges/bitmart.js | 7 +------ src/exchanges/bitmex.js | 3 +-- src/exchanges/bitstamp.js | 4 +--- src/exchanges/bitunix.js | 4 +--- src/exchanges/bybit.js | 4 +--- src/exchanges/coinbase.js | 4 +--- src/exchanges/cryptocom.js | 30 ++++++++++++++---------------- src/exchanges/deribit.js | 4 +--- src/exchanges/dydx.js | 4 +--- src/exchanges/gateio.js | 4 ++-- src/exchanges/hitbtc.js | 4 +--- src/exchanges/huobi.js | 4 +--- src/exchanges/hyperliquid.js | 3 +-- src/exchanges/kraken.js | 5 ++--- src/exchanges/kucoin.js | 4 +--- src/exchanges/mexc/index.js | 3 +-- src/exchanges/okex.js | 4 +--- src/exchanges/phemex.js | 3 +-- src/exchanges/poloniex.js | 3 +-- 26 files changed, 48 insertions(+), 82 deletions(-) diff --git a/src/exchange.js b/src/exchange.js index 9af3ff2..dbfef55 100644 --- a/src/exchange.js +++ b/src/exchange.js @@ -17,8 +17,14 @@ const { require('./typedef') class Exchange extends EventEmitter { - constructor() { + /** + * @param {string} id exchange identifier (e.g. 'BINANCE') + */ + constructor(id) { super() + + /** @type {string} */ + this.id = id /** * ping timers diff --git a/src/exchanges/aster.js b/src/exchanges/aster.js index 60000a5..480e98d 100644 --- a/src/exchanges/aster.js +++ b/src/exchanges/aster.js @@ -3,8 +3,7 @@ const Exchange = require('../exchange') class Aster extends Exchange { constructor() { - super() - this.id = 'ASTER' + super('ASTER') this.endpoints = { // Binance-futures style exchangeInfo diff --git a/src/exchanges/binance.js b/src/exchanges/binance.js index 73f2743..adabe04 100644 --- a/src/exchanges/binance.js +++ b/src/exchanges/binance.js @@ -4,9 +4,7 @@ const axios = require('axios') class Binance extends Exchange { constructor() { - super() - - this.id = 'BINANCE' + super('BINANCE') this.lastSubscriptionId = 0 this.maxConnectionsPerApi = 16 this.subscriptions = {} diff --git a/src/exchanges/binance_futures.js b/src/exchanges/binance_futures.js index 243a7eb..a314a49 100644 --- a/src/exchanges/binance_futures.js +++ b/src/exchanges/binance_futures.js @@ -5,9 +5,8 @@ const WebSocket = require('websocket').w3cwebsocket class BinanceFutures extends Exchange { constructor() { - super() - - this.id = 'BINANCE_FUTURES' + super('BINANCE_FUTURES') + this.lastSubscriptionId = 0 this.subscriptions = {} diff --git a/src/exchanges/binance_us.js b/src/exchanges/binance_us.js index 5c94b6b..1f95ac5 100644 --- a/src/exchanges/binance_us.js +++ b/src/exchanges/binance_us.js @@ -3,9 +3,8 @@ const { sleep } = require('../helper') class BinanceUs extends Exchange { constructor() { - super() + super('BINANCE_US') - this.id = 'BINANCE_US' this.lastSubscriptionId = 0 this.subscriptions = {} diff --git a/src/exchanges/bitfinex.js b/src/exchanges/bitfinex.js index c526b20..89f03c9 100644 --- a/src/exchanges/bitfinex.js +++ b/src/exchanges/bitfinex.js @@ -4,9 +4,8 @@ const { getHms } = require('../helper') class Bitfinex extends Exchange { constructor() { - super() + super('BITFINEX') - this.id = 'BITFINEX' this.channels = {} this.prices = {} diff --git a/src/exchanges/bitget.js b/src/exchanges/bitget.js index f24becd..f258971 100644 --- a/src/exchanges/bitget.js +++ b/src/exchanges/bitget.js @@ -5,9 +5,8 @@ const SPOT_PAIR_REGEX = /-SPOT$/ class Bitget extends Exchange { constructor() { - super() + super('BITGET') - this.id = 'BITGET' this.maxConnectionsPerApi = 50 this.endpoints = { diff --git a/src/exchanges/bitmart.js b/src/exchanges/bitmart.js index be0ca00..018e157 100644 --- a/src/exchanges/bitmart.js +++ b/src/exchanges/bitmart.js @@ -12,12 +12,7 @@ const WS_API_FUTURES = 'wss://openapi-ws-v2.bitmart.com/api?protocol=1.1' */ class Bitmart extends Exchange { constructor() { - super() - - /** - * @type {string} - */ - this.id = 'BITMART' + super('BITMART') /** * @type {object} diff --git a/src/exchanges/bitmex.js b/src/exchanges/bitmex.js index 45400a3..55a18c4 100644 --- a/src/exchanges/bitmex.js +++ b/src/exchanges/bitmex.js @@ -4,9 +4,8 @@ const { getHms } = require('../helper') class Bitmex extends Exchange { constructor() { - super() + super('BITMEX') - this.id = 'BITMEX' this.pairCurrencies = {} this.xbtPrice = 48000 this.types = {} diff --git a/src/exchanges/bitstamp.js b/src/exchanges/bitstamp.js index 18ebae9..247b900 100644 --- a/src/exchanges/bitstamp.js +++ b/src/exchanges/bitstamp.js @@ -2,9 +2,7 @@ const Exchange = require('../exchange') class Bitstamp extends Exchange { constructor() { - super() - - this.id = 'BITSTAMP' + super('BITMART') this.endpoints = { PRODUCTS: 'https://www.bitstamp.net/api/v2/trading-pairs-info' diff --git a/src/exchanges/bitunix.js b/src/exchanges/bitunix.js index 958ee91..41af877 100644 --- a/src/exchanges/bitunix.js +++ b/src/exchanges/bitunix.js @@ -2,9 +2,7 @@ const Exchange = require('../exchange') class Bitunix extends Exchange { constructor() { - super() - - this.id = 'BITUNIX' + super('BITUNIX') this.endpoints = { PRODUCTS: 'https://fapi.bitunix.com/api/v1/futures/market/trading_pairs' diff --git a/src/exchanges/bybit.js b/src/exchanges/bybit.js index 06cb30f..2c20e92 100644 --- a/src/exchanges/bybit.js +++ b/src/exchanges/bybit.js @@ -11,9 +11,7 @@ const RECENT_TRADE_REST = 'https://api.bybit.com/v5/market/recent-trade' class Bybit extends Exchange { constructor() { - super() - - this.id = 'BYBIT' + super('BYBIT') this.endpoints = { PRODUCTS: [ diff --git a/src/exchanges/coinbase.js b/src/exchanges/coinbase.js index 217976b..1cbe60c 100644 --- a/src/exchanges/coinbase.js +++ b/src/exchanges/coinbase.js @@ -5,9 +5,7 @@ const INTX_PAIR_REGEX = /-INTX$/ class Coinbase extends Exchange { constructor() { - super() - - this.id = 'COINBASE' + super('COINBASE') this.endpoints = { PRODUCTS: [ diff --git a/src/exchanges/cryptocom.js b/src/exchanges/cryptocom.js index 1b30690..b9ebcbf 100644 --- a/src/exchanges/cryptocom.js +++ b/src/exchanges/cryptocom.js @@ -4,9 +4,7 @@ const { getHms } = require('../helper') class CryptoCom extends Exchange { constructor() { - super() - - this.id = 'CRYPTOCOM' + super('CRYPTOCOM') this.endpoints = { PRODUCTS: 'https://api.crypto.com/exchange/v1/public/get-instruments' @@ -119,11 +117,11 @@ class CryptoCom extends Exchange { `start_ts=${range.from}&` + `end_ts=${range.to}` - return this.requestWithRetry(() => axios.get(endpoint), { - range, - label: 'missing trades' - }) - .then(response => { + return this.requestWithRetry(() => axios.get(endpoint), { + range, + label: 'missing trades' + }) + .then(response => { const data = response.data.result.data if (data.length) { @@ -172,14 +170,14 @@ class CryptoCom extends Exchange { return totalRecovered }) - .catch(err => { - console.error( - `[${this.id}] failed to get missing trades on ${range.pair}`, - this.formatErrorForLog(err) - ) - - return totalRecovered - }) + .catch(err => { + console.error( + `[${this.id}] failed to get missing trades on ${range.pair}`, + this.formatErrorForLog(err) + ) + + return totalRecovered + }) } } diff --git a/src/exchanges/deribit.js b/src/exchanges/deribit.js index c3dc39e..c2ca139 100644 --- a/src/exchanges/deribit.js +++ b/src/exchanges/deribit.js @@ -3,9 +3,7 @@ const Exchange = require('../exchange') class Deribit extends Exchange { constructor() { - super() - - this.id = 'DERIBIT' + super('DERIBIT') this.endpoints = { PRODUCTS: [ diff --git a/src/exchanges/dydx.js b/src/exchanges/dydx.js index a156440..cc6fe04 100644 --- a/src/exchanges/dydx.js +++ b/src/exchanges/dydx.js @@ -2,9 +2,7 @@ const Exchange = require('../exchange') class Dydx extends Exchange { constructor() { - super() - - this.id = 'DYDX' + super('DYDX') this.endpoints = { PRODUCTS: 'https://indexer.dydx.trade/v4/perpetualMarkets' diff --git a/src/exchanges/gateio.js b/src/exchanges/gateio.js index 805a8cf..ba539d4 100644 --- a/src/exchanges/gateio.js +++ b/src/exchanges/gateio.js @@ -5,8 +5,8 @@ const IS_USDT_SETTLEMENT_REGEX = /USDT$/ class Gateio extends Exchange { constructor() { - super() - this.id = 'GATEIO' + super('GATEIO') + this.liquidationApi = {} this.maxConnectionsPerApi = 100 diff --git a/src/exchanges/hitbtc.js b/src/exchanges/hitbtc.js index df6cd96..95a39b0 100644 --- a/src/exchanges/hitbtc.js +++ b/src/exchanges/hitbtc.js @@ -2,9 +2,7 @@ const Exchange = require('../exchange') class HitBtc extends Exchange { constructor() { - super() - - this.id = 'HITBTC' + super('HITBTC') this.endpoints = { PRODUCTS: 'https://api.hitbtc.com/api/2/public/symbol' diff --git a/src/exchanges/huobi.js b/src/exchanges/huobi.js index 395c2d6..0b515bc 100644 --- a/src/exchanges/huobi.js +++ b/src/exchanges/huobi.js @@ -6,9 +6,7 @@ const { getHms } = require('../helper') class Huobi extends Exchange { constructor() { - super() - - this.id = 'HUOBI' + super('HUOBI') this.receivedInitialData = {} this.liquidationOrdersSubscriptions = {} diff --git a/src/exchanges/hyperliquid.js b/src/exchanges/hyperliquid.js index 5ef3d8c..c7f39d5 100644 --- a/src/exchanges/hyperliquid.js +++ b/src/exchanges/hyperliquid.js @@ -4,8 +4,7 @@ const axios = require('axios') class Hyperliquid extends Exchange { constructor() { - super() - this.id = 'HYPERLIQUID' + super('HYPERLIQUID') this.endpoints = { PRODUCTS: 'https://api.hyperliquid.xyz/info' diff --git a/src/exchanges/kraken.js b/src/exchanges/kraken.js index 3bffbd2..2dd6ee7 100644 --- a/src/exchanges/kraken.js +++ b/src/exchanges/kraken.js @@ -4,9 +4,8 @@ const { getHms } = require('../helper') class Kraken extends Exchange { constructor() { - super() - - this.id = 'KRAKEN' + super('KRAKEN') + this.keepAliveIntervals = {} this.endpoints = { diff --git a/src/exchanges/kucoin.js b/src/exchanges/kucoin.js index a17a7d2..06fcbb2 100644 --- a/src/exchanges/kucoin.js +++ b/src/exchanges/kucoin.js @@ -6,9 +6,7 @@ const KUCOIN_TOKEN_EXPIRATION = 1000 * 60 * 5 class Kucoin extends Exchange { constructor() { - super() - - this.id = 'KUCOIN' + super('KUCOIN') this.promiseOfToken = null this.token = null diff --git a/src/exchanges/mexc/index.js b/src/exchanges/mexc/index.js index 6ebb58e..222d875 100644 --- a/src/exchanges/mexc/index.js +++ b/src/exchanges/mexc/index.js @@ -9,9 +9,8 @@ const { class Mexc extends Exchange { constructor() { - super() + super('MEXC') - this.id = 'MEXC' this.maxConnectionsPerApi = 50 this.contractSizes = {} this.inversed = {} diff --git a/src/exchanges/okex.js b/src/exchanges/okex.js index fa241e7..1d09b90 100644 --- a/src/exchanges/okex.js +++ b/src/exchanges/okex.js @@ -4,9 +4,7 @@ const { getHms } = require('../helper') class Okex extends Exchange { constructor() { - super() - - this.id = 'OKEX' + super('OKEX') this.endpoints = { LIQUIDATIONS: 'https://www.okx.com/api/v5/public/liquidation-orders', diff --git a/src/exchanges/phemex.js b/src/exchanges/phemex.js index cb3a136..fd49fd2 100644 --- a/src/exchanges/phemex.js +++ b/src/exchanges/phemex.js @@ -2,9 +2,8 @@ const Exchange = require('../exchange') class Phemex extends Exchange { constructor() { - super() + super('PHEMEX') - this.id = 'PHEMEX' this.isInverse = {} this.isV1 = {} this.priceScales = {} diff --git a/src/exchanges/poloniex.js b/src/exchanges/poloniex.js index dd4613c..344acee 100644 --- a/src/exchanges/poloniex.js +++ b/src/exchanges/poloniex.js @@ -2,9 +2,8 @@ const Exchange = require('../exchange') class Poloniex extends Exchange { constructor() { - super() + super('POLONIEX') - this.id = 'POLONIEX' this.channels = {} this.endpoints = { From e9626932c5374528dc83ace7886e9eaae8a1c2c5 Mon Sep 17 00:00:00 2001 From: Tucsky Date: Sun, 18 Jan 2026 13:44:04 +0100 Subject: [PATCH 6/6] feat: implement performance optimizations with caching and fast null-bar checks --- src/storage/binaries/README.md | 53 ++++++++ src/storage/binaries/index.js | 35 +++-- src/storage/binaries/io.js | 214 ++++++++++++++++++++++++++++--- src/storage/binaries/resample.js | 33 ++--- 4 files changed, 280 insertions(+), 55 deletions(-) diff --git a/src/storage/binaries/README.md b/src/storage/binaries/README.md index d52dbc9..240991b 100644 --- a/src/storage/binaries/README.md +++ b/src/storage/binaries/README.md @@ -357,6 +357,50 @@ The `fetch()` method returns data in the same format as InfluxDB storage: **Note**: Time is returned in **seconds** (not milliseconds) to match InfluxDB's `precision: 's'` setting. +## Performance Optimizations + +### In-Memory Caches + +The storage engine uses two LRU caches to reduce syscall overhead on hot paths: + +#### Metadata Cache + +Caches parsed JSON metadata files with mtime-based validation: + +```javascript +{ + "binariesMetaCacheMax": 5000 // Max cached metadata entries (default) +} +``` + +- **Key**: Absolute path to `.json` file +- **Value**: `{ meta, mtime }` tuple +- **Invalidation**: Re-read if file mtime changes or on explicit invalidation +- **Eviction**: LRU when cache exceeds `binariesMetaCacheMax` + +#### File Descriptor Cache + +Caches open file descriptors for read-mode access: + +```javascript +{ + "binariesFdCacheMax": 256 // Max cached file descriptors (default) +} +``` + +- **Key**: Absolute path to `.bin` file +- **Value**: `{ fd, mode }` tuple +- **Eviction**: LRU with `fs.closeSync()` on eviction +- **Cleanup**: Call `closeAllFds()` on graceful shutdown + +### Fast Null-Bar Check + +Uses `isNullRecord(buffer, offset)` which only reads the first 16 bytes (OHLC int32s) to detect null bars without full decode. + +### Positioned Reads + +Uses `fs.readSync()` with offset parameter (pread) for single-record lookups, avoiding file seek state. + ## Performance Characteristics | Operation | Complexity | Notes | @@ -366,6 +410,15 @@ The `fetch()` method returns data in the same format as InfluxDB storage: | Read (fetch) | O(n + s) | n=records, s=segments in range | | Resample | O(n) | Reads base segments, writes target segments | +### Syscall Reduction + +| Operation | Before | After (with caches) | +|-----------|--------|---------------------| +| Metadata read | `readFileSync` + `JSON.parse` | Cached lookup + mtime check | +| File open | `openSync` per read | Cached FD reuse | +| File stat | `fstatSync` per read | Uses `meta.records` from cache | +| File close | `closeSync` per read | Deferred until eviction | + ### Advantages over InfluxDB - **No external service**: Runs in-process, no network overhead diff --git a/src/storage/binaries/index.js b/src/storage/binaries/index.js index e66780d..18d8dc8 100644 --- a/src/storage/binaries/index.js +++ b/src/storage/binaries/index.js @@ -5,7 +5,7 @@ const { updateIndexes } = require('../../services/connections') const { getHms } = require('../../helper') const { RECORD_SIZE, getSegmentStartTs, getSegmentSpanMs } = require('./constants') -const { getSegmentPaths, parseMarket, readRecord, readMeta, readSingleRecordAtTs } = require('./io') +const { getSegmentPaths, parseMarket, readRecord, isNullRecord, readMeta, readSingleRecordAtTs, openFdCached } = require('./io') const { upsertBars } = require('./write') const { resampleToHigherTimeframes } = require('./resample') @@ -390,7 +390,6 @@ class BinariesStorage { const paths = getSegmentPaths(parsed.exchange, parsed.symbol, timeframe, segmentStartTs) const meta = readMeta(paths.json) if (!meta) continue - if (!fs.existsSync(paths.bin)) continue // Calculate index range within this segment for the requested time window const segmentEndTs = segmentStartTs + segmentSpanMs @@ -402,7 +401,7 @@ class BinariesStorage { let startIndex = Math.floor((readFromTs - segmentStartTs) / timeframe) let endIndex = Math.ceil((readToTs - segmentStartTs) / timeframe) - // Clamp to valid range + // Clamp to valid range (use meta.records, avoids fstatSync) if (startIndex < 0) startIndex = 0 if (endIndex > meta.records) endIndex = meta.records if (startIndex >= endIndex) continue @@ -411,34 +410,32 @@ class BinariesStorage { const byteOffset = startIndex * RECORD_SIZE const byteLength = count * RECORD_SIZE - // Read the range from disk + // Compute max bytes from meta.records (avoids fstatSync) + const maxBytesFromMeta = meta.records * RECORD_SIZE + const maxBytes = Math.min(byteLength, maxBytesFromMeta - byteOffset) + if (maxBytes <= 0) continue + + // Use cached fd (avoids openSync/closeSync per segment) + const fd = openFdCached(paths.bin, 'r') + if (fd === null) continue + let buffer - let fd try { - fd = fs.openSync(paths.bin, 'r') - const stat = fs.fstatSync(fd) - const maxBytes = Math.min(byteLength, stat.size - byteOffset) - if (maxBytes <= 0) { - fs.closeSync(fd) - continue - } buffer = Buffer.alloc(maxBytes) fs.readSync(fd, buffer, 0, maxBytes, byteOffset) - fs.closeSync(fd) } catch (_e) { - if (fd !== undefined) { - try { fs.closeSync(fd) } catch (_) { /* ignored */ } - } continue } // Decode records and add to results const recordCount = Math.floor(buffer.length / RECORD_SIZE) for (let i = 0; i < recordCount; i++) { - const record = readRecord(buffer, i * RECORD_SIZE, meta) + const offset = i * RECORD_SIZE + + // Fast null-bar check (only reads 16 bytes, no decode) + if (isNullRecord(buffer, offset)) continue - // Skip null bars (gaps) - check before computing barTime for efficiency - if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) continue + const record = readRecord(buffer, offset, meta) const barTime = segmentStartTs + (startIndex + i) * timeframe diff --git a/src/storage/binaries/io.js b/src/storage/binaries/io.js index afa31b9..8bcb668 100644 --- a/src/storage/binaries/io.js +++ b/src/storage/binaries/io.js @@ -2,7 +2,121 @@ const fs = require('fs') const path = require('path') const config = require('../../config') const { getHms } = require('../../helper') -const { RECORD_SIZE, getSegmentStartTs, getSegmentSpanMs, getSegmentRecords } = require('./constants') +const { RECORD_SIZE, getSegmentStartTs, getSegmentSpanMs } = require('./constants') + +// ============================================================================ +// Caches for performance optimization +// ============================================================================ + +/** + * Metadata cache to avoid repeated JSON reads. + * Key: jsonPath, Value: { mtimeMs: number, meta: object } + * @type {Map} + */ +const metaCache = new Map() + +/** + * Maximum entries in metadata cache before eviction. + * @type {number} + */ +const META_CACHE_MAX = config.binariesMetaCacheMax || 5000 + +/** + * LRU file descriptor cache for .bin files. + * Key: binPath, Value: fd (number) + * Map maintains insertion order; we use this for LRU eviction. + * @type {Map} + */ +const fdCache = new Map() + +/** + * Maximum open file descriptors to cache. + * @type {number} + */ +const FD_CACHE_MAX = config.binariesFdCacheMax || 256 + +/** + * Opens a file descriptor with LRU caching. + * On cache hit, moves entry to end (most recently used). + * On cache miss, opens file and evicts LRU if needed. + * + * @param {string} binPath - Path to binary file + * @param {string} [flags='r'] - File open flags + * @returns {number|null} File descriptor or null if file doesn't exist + */ +function openFdCached(binPath, flags = 'r') { + // Check cache first + if (fdCache.has(binPath)) { + const fd = fdCache.get(binPath) + // Move to end (most recently used) by deleting and re-inserting + fdCache.delete(binPath) + fdCache.set(binPath, fd) + return fd + } + + // Open new fd + let fd + try { + fd = fs.openSync(binPath, flags) + } catch (e) { + if (e.code === 'ENOENT') return null + throw e + } + + // Evict LRU if at capacity + if (fdCache.size >= FD_CACHE_MAX) { + // First entry is least recently used + const lruKey = fdCache.keys().next().value + const lruFd = fdCache.get(lruKey) + fdCache.delete(lruKey) + try { + fs.closeSync(lruFd) + } catch (_) { /* ignore close errors */ } + } + + fdCache.set(binPath, fd) + return fd +} + +/** + * Closes all cached file descriptors. + * Call this on shutdown to avoid fd leaks. + */ +function closeAllFds() { + for (const [_path, fd] of fdCache) { + try { + fs.closeSync(fd) + } catch (_) { /* ignore */ } + } + fdCache.clear() +} + +/** + * Invalidates a cached file descriptor (e.g., after write). + * @param {string} binPath - Path to invalidate + */ +function invalidateFdCache(binPath) { + if (fdCache.has(binPath)) { + const fd = fdCache.get(binPath) + fdCache.delete(binPath) + try { + fs.closeSync(fd) + } catch (_) { /* ignore */ } + } +} + +/** + * Invalidates metadata cache for a path (call after writes). + * @param {string} jsonPath - Path to invalidate + */ +function invalidateMetaCache(jsonPath) { + metaCache.delete(jsonPath) +} + +// Register cleanup on process exit +process.on('exit', closeAllFds) +process.on('SIGINT', () => { closeAllFds(); process.exit(0) }) +process.on('SIGTERM', () => { closeAllFds(); process.exit(0) }) /** * Generates file paths for a market's binary and metadata files (legacy single-file format). @@ -154,16 +268,68 @@ function readRecord(buffer, offset, meta) { } /** - * Reads metadata JSON file for a market/timeframe. + * Fast null-bar check that only reads the first 16 bytes (OHLC int32s). + * This avoids the cost of reading BigInt64 fields for null records. + * + * @param {Buffer} buffer - Source buffer + * @param {number} offset - Byte offset in buffer + * @returns {boolean} True if this is a null bar (all OHLC are zero) + */ +function isNullRecord(buffer, offset) { + // Check if all 16 bytes of OHLC are zero + // This is faster than reading and comparing 4 separate Int32LE values + return ( + buffer.readInt32LE(offset) === 0 && + buffer.readInt32LE(offset + 4) === 0 && + buffer.readInt32LE(offset + 8) === 0 && + buffer.readInt32LE(offset + 12) === 0 + ) +} + +/** + * Reads metadata JSON file for a market/timeframe with caching. + * Uses mtime-based cache invalidation to avoid stale reads. * * @param {string} jsonPath - Path to the JSON metadata file * @returns {import('./constants').BinaryMeta|null} Parsed metadata or null if not found */ function readMeta(jsonPath) { try { - if (!fs.existsSync(jsonPath)) return null + // Get file stat (also checks existence) + let stat + try { + stat = fs.statSync(jsonPath) + } catch (e) { + if (e.code === 'ENOENT') { + // File doesn't exist, clear any stale cache entry + metaCache.delete(jsonPath) + return null + } + throw e + } + + const mtimeMs = stat.mtimeMs + + // Check cache + const cached = metaCache.get(jsonPath) + if (cached && cached.mtimeMs === mtimeMs) { + return cached.meta + } + + // Read and parse const content = fs.readFileSync(jsonPath, 'utf8') - return JSON.parse(content) + const meta = JSON.parse(content) + + // Evict oldest entries if at capacity (simple FIFO eviction) + if (metaCache.size >= META_CACHE_MAX) { + const firstKey = metaCache.keys().next().value + metaCache.delete(firstKey) + } + + // Cache the result + metaCache.set(jsonPath, { mtimeMs, meta }) + + return meta } catch (_e) { return null } @@ -171,6 +337,7 @@ function readMeta(jsonPath) { /** * Writes metadata JSON file atomically (write to temp, then rename). + * Invalidates the metadata cache for this path. * * @param {string} jsonPath - Path to the JSON metadata file * @param {import('./constants').BinaryMeta} meta - Metadata to write @@ -179,11 +346,14 @@ function writeMeta(jsonPath, meta) { const tmpPath = jsonPath + '.tmp' fs.writeFileSync(tmpPath, JSON.stringify(meta, null, 2)) fs.renameSync(tmpPath, jsonPath) + // Invalidate cache so next read gets fresh data + invalidateMetaCache(jsonPath) } /** * Reads a single record from a segment file at a specific bucket timestamp. * Uses positioned read (pread) to read only the needed 56 bytes. + * Uses cached file descriptor to avoid open/close syscalls on hot paths. * * This is used as a disk fallback when processing trades for buckets * that exist on disk but not in memory caches. @@ -201,7 +371,6 @@ function readSingleRecordAtTs(exchange, symbol, timeframe, bucketTs) { const meta = readMeta(paths.json) if (!meta) return null - if (!fs.existsSync(paths.bin)) return null // Compute the segment span and validate bucket is within segment bounds const segmentSpanMs = getSegmentSpanMs(timeframe) @@ -213,34 +382,31 @@ function readSingleRecordAtTs(exchange, symbol, timeframe, bucketTs) { const index = Math.floor((bucketTs - segmentStartTs) / timeframe) if (index < 0 || index >= meta.records) return null + // Use meta.records to validate bounds (avoids fstatSync) + const maxBytes = meta.records * RECORD_SIZE const byteOffset = index * RECORD_SIZE + if (byteOffset + RECORD_SIZE > maxBytes) return null + const buffer = Buffer.alloc(RECORD_SIZE) - let fd + // Use cached fd (avoids openSync/closeSync per read) + const fd = openFdCached(paths.bin, 'r') + if (fd === null) return null + try { - fd = fs.openSync(paths.bin, 'r') - const stat = fs.fstatSync(fd) - if (byteOffset + RECORD_SIZE > stat.size) { - fs.closeSync(fd) - return null - } // Positioned read - only reads the single record we need fs.readSync(fd, buffer, 0, RECORD_SIZE, byteOffset) - fs.closeSync(fd) } catch (_e) { - if (fd !== undefined) { - try { fs.closeSync(fd) } catch (_) { /* ignored */ } - } return null } - const record = readRecord(buffer, 0, meta) - - // Check if this is a null record (all OHLC are zero) - if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) { + // Fast null-bar check (only reads 16 bytes from buffer, no decode) + if (isNullRecord(buffer, 0)) { return null } + const record = readRecord(buffer, 0, meta) + return { time: bucketTs, open: record.open, @@ -283,9 +449,15 @@ module.exports = { parseMarket, writeRecord, readRecord, + isNullRecord, readMeta, writeMeta, readSingleRecordAtTs, readSegmentMeta, - writeSegmentMeta + writeSegmentMeta, + // Cache management exports + openFdCached, + closeAllFds, + invalidateFdCache, + invalidateMetaCache } diff --git a/src/storage/binaries/resample.js b/src/storage/binaries/resample.js index ccc1099..02ed530 100644 --- a/src/storage/binaries/resample.js +++ b/src/storage/binaries/resample.js @@ -1,8 +1,8 @@ const fs = require('fs') const { ensureDirectoryExists, getHms } = require('../../helper') const { RECORD_SIZE, PRICE_SCALE, VOLUME_SCALE, getSegmentStartTs, getSegmentSpanMs, getSegmentRecords } = require('./constants') -const { getFilePath, getSegmentPaths, readRecord, writeRecord, readMeta, writeMeta } = require('./io') -const { createNullBar, upsertBarsToSegment } = require('./write') +const { getSegmentPaths, readRecord, isNullRecord, writeRecord, readMeta, writeMeta, openFdCached } = require('./io') +const { createNullBar } = require('./write') /** * Triggers resampling from base timeframe to all configured higher timeframes. @@ -66,7 +66,7 @@ async function resampleRangeToTimeframe(exchange, symbol, baseTimeframe, fromTs, for (let segmentStartTs = firstBaseSegment; segmentStartTs <= lastBaseSegment; segmentStartTs += baseSegmentSpanMs) { const paths = getSegmentPaths(exchange, symbol, baseTimeframe, segmentStartTs) const meta = readMeta(paths.json) - if (!meta || !fs.existsSync(paths.bin)) continue + if (!meta) continue // Calculate which records from this segment we need const segmentEndTs = segmentStartTs + baseSegmentSpanMs @@ -87,16 +87,19 @@ async function resampleRangeToTimeframe(exchange, symbol, baseTimeframe, fromTs, const byteOffset = startIndex * RECORD_SIZE const byteLength = count * RECORD_SIZE + // Compute max bytes from meta.records (avoids fstatSync) + const maxBytesFromMeta = meta.records * RECORD_SIZE + const maxBytes = Math.min(byteLength, maxBytesFromMeta - byteOffset) + if (maxBytes <= 0) continue + + // Use cached fd (avoids openSync/closeSync per segment) + const fd = openFdCached(paths.bin, 'r') + if (fd === null) continue + let buffer try { - const fd = fs.openSync(paths.bin, 'r') - const stat = fs.fstatSync(fd) - const maxBytes = Math.min(byteLength, stat.size - byteOffset) - if (maxBytes > 0) { - buffer = Buffer.alloc(maxBytes) - fs.readSync(fd, buffer, 0, maxBytes, byteOffset) - } - fs.closeSync(fd) + buffer = Buffer.alloc(maxBytes) + fs.readSync(fd, buffer, 0, maxBytes, byteOffset) } catch (_e) { buffer = null } @@ -104,12 +107,13 @@ async function resampleRangeToTimeframe(exchange, symbol, baseTimeframe, fromTs, if (buffer) { const recordCount = Math.floor(buffer.length / RECORD_SIZE) for (let i = 0; i < recordCount; i++) { - const record = readRecord(buffer, i * RECORD_SIZE, meta) + const offset = i * RECORD_SIZE const barTime = segmentStartTs + (startIndex + i) * baseTimeframe - // Skip null records - if (record.open === 0 && record.high === 0 && record.low === 0 && record.close === 0) continue + // Fast null-bar check (only reads 16 bytes, no decode) + if (isNullRecord(buffer, offset)) continue + const record = readRecord(buffer, offset, meta) baseBarsByTime[barTime] = record } } @@ -173,7 +177,6 @@ async function resampleRangeToTimeframe(exchange, symbol, baseTimeframe, fromTs, if (barsToWrite.length === 0) return // Group bars by target segment and upsert each segment - const targetSegmentSpanMs = getSegmentSpanMs(targetTimeframe) const barsBySegment = {} for (const bar of barsToWrite) {