diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dd8da6e..4b5ac1d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,6 +23,8 @@ docker compose up npm run dev ``` +> **Note on Authentication:** If you set `API_KEY` in your `.env` file, you must pass `x-api-key` in your request headers when testing locally. Leaving `API_KEY` empty disables authentication for easier development. + 5. Optional smoke check: ```bash @@ -36,6 +38,7 @@ npm run build npm run typecheck npm run lint npm run test +npm run format npm run format:check ``` diff --git a/README.md b/README.md index 5c77857..1ae27e7 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,8 @@ No formal benchmark numbers are published yet. The current focus is correctness, ```bash npm install -docker compose up +cp .env.example .env # adjust values if needed +docker compose up -d # starts Redis in the background npm run dev ``` @@ -207,7 +208,7 @@ Dashboard: http://localhost:4000/dashboard ``` -If you need non-default settings, use the variables documented in [.env.example](./.env.example). +All configuration is done via environment variables. Copy [.env.example](./.env.example) to `.env` and adjust values as needed. ## Authentication @@ -243,43 +244,80 @@ Protected endpoints: `/check-limit`, `/consume`, `/api/dashboard-data` Public endpoints (no key required): `/`, `/health`, `/dashboard`, `/static/*` -## API Example +## API Reference -Check a limit without consuming: +ARCE exposes two main endpoints: + +- **`POST /check-limit`** — evaluates whether the request would be allowed, **without** consuming from the limiter. Use this for read-ahead checks. +- **`POST /consume`** — evaluates and **consumes** from the limiter. Use this for actual enforcement. + +Both endpoints accept the same request body and return the same response shape. + +### Request Fields + +| Field | Required | Default | Description | +| ------------------ | -------- | ------- | -------------------------------------------------------------------------------------------- | +| `algorithm` | ✅ | — | One of `token_bucket`, `sliding_window`, `leaky_bucket` | +| `route` | — | `"/"` | The API route being accessed | +| `method` | — | `"GET"` | HTTP method | +| `userId` | \* | — | User identifier (max 256 chars) | +| `ip` | \* | — | Client IP address (max 64 chars) | +| `identifier` | \* | — | Custom subject key for `custom` scope (max 256 chars) | +| `scope` | — | auto | How to derive the subject: `user`, `ip`, `hybrid` (user+ip), or `custom` (uses `identifier`) | +| `cost` | — | `1` | How many tokens/units this request consumes (1–10) | +| `fingerprint` | — | auto | A string identifying this exact request shape, used for duplicate detection (max 256 chars) | +| `baseLimitPerMinute`| — | `100` | Override the default rate limit for this request (10–10,000) | +| `metadata.userAgent`| — | — | The client's user-agent string, used for missing-UA abuse detection | + +\* At least one of `userId`, `ip`, or `identifier` must be provided. + +### Example: Check a limit ```bash curl -X POST http://localhost:4000/check-limit \ -H "Content-Type: application/json" \ - -d "{\"algorithm\":\"token_bucket\",\"route\":\"/api/orders\",\"method\":\"GET\",\"userId\":\"user-42\",\"ip\":\"203.0.113.8\",\"scope\":\"hybrid\"}" + -H "x-api-key: $ARCE_API_KEY" \ + -d '{"algorithm":"token_bucket","route":"/api/orders","method":"GET","userId":"user-42","ip":"203.0.113.8","scope":"hybrid"}' ``` -Consume from the limiter: +### Example: Consume from the limiter ```bash curl -X POST http://localhost:4000/consume \ -H "Content-Type: application/json" \ - -d "{\"algorithm\":\"sliding_window\",\"route\":\"/api/search\",\"method\":\"GET\",\"ip\":\"203.0.113.8\",\"scope\":\"ip\"}" + -H "x-api-key: $ARCE_API_KEY" \ + -d '{"algorithm":"sliding_window","route":"/api/search","method":"GET","ip":"203.0.113.8","scope":"ip"}' ``` -Example request shape: +### Example Response ```json { - "algorithm": "token_bucket", - "route": "/api/orders", - "method": "GET", - "userId": "user-42", - "ip": "203.0.113.8", - "scope": "hybrid", + "mode": "consume", + "allowed": true, + "blocked": false, + "subject": "ip:203.0.113.8", + "fingerprint": "GET:/api/search", + "algorithm": "sliding_window", "cost": 1, - "fingerprint": "GET:/api/orders?status=open", - "baseLimitPerMinute": 100, - "metadata": { - "userAgent": "curl/8.6.0" - } + "anomalies": [], + "effectivePolicy": { + "tier": "normal", + "effectiveLimitPerMinute": 100, + "riskScore": 0 + }, + "decision": { + "allowed": true, + "remaining": 99, + "retryAfterMs": 0, + "resetAfterMs": 60000 + }, + "evaluatedAt": "2026-04-29T05:30:00.000Z" } ``` +When `allowed` is `false`, the HTTP status is `429`. The `decision.retryAfterMs` field tells the client how long to wait before retrying. + ## SDK Example ```ts @@ -312,8 +350,15 @@ import { ArceClient } from "./src/sdk/client"; import { createArceMiddleware } from "./src/sdk/express-middleware"; const app = express(); -const client = new ArceClient({ baseUrl: "http://localhost:4000" }); +const client = new ArceClient({ + baseUrl: "http://localhost:4000", + headers: { "x-api-key": process.env.ARCE_API_KEY ?? "" } +}); +// All routes below this middleware are rate-limited. +// The middleware calls ARCE's /consume endpoint for each request. +// If the request is denied, the middleware returns 429 with +// x-rate-limit-remaining and x-rate-limit-reset-ms headers. app.use( createArceMiddleware({ client, @@ -326,10 +371,11 @@ app.use( ## Production Integration Notes -- Put ARCE behind your API tier or call it from application middleware. -- Use a stable subject key strategy: user, IP, or hybrid depending on your threat model. -- Keep route fingerprinting intentional. High-cardinality fingerprints reduce the value of duplicate detection. -- Tune baseline limits and block durations per environment. +- **Deployment**: Put ARCE behind your API tier or call it from application middleware. ARCE itself is not designed to be internet-facing — it should be accessible only from your internal network or application pods. +- **Subject strategy**: Use a stable subject key strategy (`user`, `ip`, or `hybrid`) depending on your threat model. `hybrid` (user+IP) is strictest — it tracks the same user from different IPs separately. +- **Fingerprinting**: Keep route fingerprinting intentional. A fingerprint like `GET:/api/orders?status=open` groups similar requests; `GET:/api/orders?status=open&page=3&t=1719000000` would create unique fingerprints for every request, defeating duplicate detection. +- **Tuning**: Start with the defaults (`100 req/min` baseline, `20 req/min` suspicious, `300s` block). Observe the dashboard for false positives before tightening thresholds. +- **Redis**: Use a dedicated Redis instance or a dedicated database number (`redis://localhost:6379/1`) to isolate ARCE state from your application data. The `SERVICE_NAME` prefix prevents key collisions if sharing a cluster. ## Configuration diff --git a/apps/dashboard/README.md b/apps/dashboard/README.md index c4f4d1d..e9bb1b6 100644 --- a/apps/dashboard/README.md +++ b/apps/dashboard/README.md @@ -3,3 +3,6 @@ This directory contains static assets for the ARCE operator dashboard. The dashboard is intentionally lightweight. It visualizes request volume, rate-limited traffic, active blocks, and recent anomaly events without introducing a separate frontend build system. + +**How to view it:** +You do not need to run a separate web server for the dashboard. The main ARCE Express server automatically serves these static assets at the `/dashboard` endpoint (e.g., `http://localhost:4000/dashboard`). diff --git a/configs/runtime.ts b/configs/runtime.ts index 790c607..6d01207 100644 --- a/configs/runtime.ts +++ b/configs/runtime.ts @@ -14,11 +14,22 @@ function readNumber(name: string, fallback: number): number { } export const runtimeConfig = { + isProduction: process.env.NODE_ENV === "production", port: readNumber("PORT", 4000), redisUrl: process.env.REDIS_URL ?? "redis://localhost:6379", serviceName: process.env.SERVICE_NAME ?? "arce", apiKey: process.env.API_KEY ?? "", defaultLimitPerMinute: readNumber("DEFAULT_LIMIT_PER_MINUTE", 100), suspiciousLimitPerMinute: readNumber("SUSPICIOUS_LIMIT_PER_MINUTE", 20), - blockDurationSeconds: readNumber("BLOCK_DURATION_SECONDS", 300) + blockDurationSeconds: readNumber("BLOCK_DURATION_SECONDS", 300), + abuse: { + burstSpikeCount: readNumber("ABUSE_BURST_SPIKE_COUNT", 15), + burstSpikeRatio: readNumber("ABUSE_BURST_SPIKE_RATIO", 3), + duplicateFingerprintCount: readNumber("ABUSE_DUPLICATE_FINGERPRINT_COUNT", 8), + duplicateFingerprintRatio: readNumber("ABUSE_DUPLICATE_FINGERPRINT_RATIO", 0.7), + wideRouteScanCount: readNumber("ABUSE_WIDE_ROUTE_SCAN_COUNT", 12), + wideRouteScanMinuteCount: readNumber("ABUSE_WIDE_ROUTE_SCAN_MINUTE_COUNT", 30), + repeatedDenialsCount: readNumber("ABUSE_REPEATED_DENIALS_COUNT", 5), + missingUserAgentCount: readNumber("ABUSE_MISSING_USER_AGENT_COUNT", 10), + } }; diff --git a/docs/design-decisions.md b/docs/design-decisions.md index 4f6ba5e..a9e584b 100644 --- a/docs/design-decisions.md +++ b/docs/design-decisions.md @@ -22,6 +22,8 @@ Keeping them separate makes each path easier to reason about and change. ARCE is designed to be inspectable and operationally credible. Burst ratios, duplicate request patterns, route fan-out, and repeated denials are understandable signals that an engineer can tune without a training pipeline. +> **How to tune:** The thresholds for these heuristics (e.g., "15 requests in 10s with a 3x multiplier") are currently hardcoded in `src/core/abuse-detector.ts`. If you are deploying ARCE to production, you should review and modify these thresholds in that file to match your application's normal traffic patterns. + ## Why A Minimal Dashboard The dashboard is intentionally thin. Its job is to make the limiter observable during development and demos, not to become a full observability platform. diff --git a/package-lock.json b/package-lock.json index bc7586f..9d67026 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,13 +9,19 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "cors": "^2.8.6", "dotenv": "^17.0.0", "express": "^4.21.2", + "express-rate-limit": "^8.4.1", + "helmet": "^8.1.0", + "pino": "^10.3.1", + "pino-http": "^11.0.0", "redis": "^5.1.1", "zod": "^3.24.2" }, "devDependencies": { "@eslint/js": "^9.27.0", + "@types/cors": "^2.8.19", "@types/express": "^4.17.21", "@types/node": "^24.0.1", "@types/supertest": "^6.0.3", @@ -729,6 +735,11 @@ "@noble/hashes": "^1.1.5" } }, + "node_modules/@pinojs/redact": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", + "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==" + }, "node_modules/@redis/bloom": { "version": "5.12.1", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-5.12.1.tgz", @@ -1156,6 +1167,15 @@ "integrity": "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==", "dev": true }, + "node_modules/@types/cors": { + "version": "2.8.19", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.19.tgz", + "integrity": "sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/deep-eql": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", @@ -1852,6 +1872,14 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", "dev": true }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -2090,6 +2118,22 @@ "integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==", "dev": true }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -2561,6 +2605,23 @@ "url": "https://opencollective.com/express" } }, + "node_modules/express-rate-limit": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.4.1.tgz", + "integrity": "sha512-NGVYwQSAyEQgzxX1iCM978PP9AdO/hW93gMcF6ZwQCm+rFvLsBH6w4xcXWTcliS8La5EPRN3p9wzItqBwJrfNw==", + "dependencies": { + "ip-address": "10.1.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -2737,6 +2798,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-intrinsic": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", @@ -2865,6 +2934,14 @@ "node": ">= 0.4" } }, + "node_modules/helmet": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/helmet/-/helmet-8.1.0.tgz", + "integrity": "sha512-jOiHyAZsmnr8LqoPGmCjYAaiuWwjAPLgY8ZX2XrmHawt99/u1y6RgrZMTeoPfpUbV96HOalYgz1qzkRbw54Pmg==", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/http-errors": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", @@ -2934,6 +3011,14 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/ip-address": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", + "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -3174,6 +3259,14 @@ "node": ">= 0.6" } }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/object-inspect": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", @@ -3185,6 +3278,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -3328,6 +3429,51 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pino": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", + "integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^4.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-http": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/pino-http/-/pino-http-11.0.0.tgz", + "integrity": "sha512-wqg5XIAGRRIWtTk8qPGxkbrfiwEWz1lgedVLvhLALudKXvg1/L2lTFgTGPJ4Z2e3qcRmxoFxDuSdMdMGNM6I1g==", + "dependencies": { + "get-caller-file": "^2.0.5", + "pino": "^10.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz", + "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==" + }, "node_modules/postcss": { "version": "8.5.12", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.12.tgz", @@ -3380,6 +3526,21 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/process-warning": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", + "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ] + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -3415,6 +3576,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -3437,6 +3603,14 @@ "node": ">= 0.8" } }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "engines": { + "node": ">= 12.13.0" + } + }, "node_modules/redis": { "version": "5.12.1", "resolved": "https://registry.npmjs.org/redis/-/redis-5.12.1.tgz", @@ -3533,6 +3707,14 @@ } ] }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -3692,6 +3874,14 @@ "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", "dev": true }, + "node_modules/sonic-boom": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", + "integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -3701,6 +3891,14 @@ "node": ">=0.10.0" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/stackback": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", @@ -3835,6 +4033,17 @@ "node": ">=8" } }, + "node_modules/thread-stream": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.0.0.tgz", + "integrity": "sha512-4iMVL6HAINXWf1ZKZjIPcz5wYaOdPhtO8ATvZ+Xqp3BTdaqtAwQkNmKORqcIo5YkQqGXq5cwfswDwMqqQNrpJA==", + "dependencies": { + "real-require": "^0.2.0" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", diff --git a/package.json b/package.json index c9f55af..1c7d5f3 100644 --- a/package.json +++ b/package.json @@ -33,13 +33,19 @@ "node": ">=20" }, "dependencies": { + "cors": "^2.8.6", "dotenv": "^17.0.0", "express": "^4.21.2", + "express-rate-limit": "^8.4.1", + "helmet": "^8.1.0", + "pino": "^10.3.1", + "pino-http": "^11.0.0", "redis": "^5.1.1", "zod": "^3.24.2" }, "devDependencies": { "@eslint/js": "^9.27.0", + "@types/cors": "^2.8.19", "@types/express": "^4.17.21", "@types/node": "^24.0.1", "@types/supertest": "^6.0.3", diff --git a/src/api/app.ts b/src/api/app.ts index 5c05797..0378953 100644 --- a/src/api/app.ts +++ b/src/api/app.ts @@ -5,6 +5,10 @@ import express, { type Response } from "express"; import path from "node:path"; +import helmet from "helmet"; +import cors from "cors"; +import rateLimit from "express-rate-limit"; +import pinoHttp from "pino-http"; import { ZodError } from "zod"; @@ -32,14 +36,21 @@ export function createServerApp(args: CreateServerAppArgs): Express { app.disable("x-powered-by"); - // Security headers - app.use((_request, response, next) => { - response.setHeader("X-Content-Type-Options", "nosniff"); - response.setHeader("X-Frame-Options", "DENY"); - response.setHeader("X-XSS-Protection", "1; mode=block"); - response.setHeader("Referrer-Policy", "strict-origin-when-cross-origin"); - next(); + // Security and Logging middleware + app.use(helmet()); + app.use(cors()); + app.use(pinoHttp({ + autoLogging: false, + quietReqLogger: true + })); + + // Global rate limiter for the ARCE endpoints themselves + const globalLimiter = rateLimit({ + windowMs: 15 * 60 * 1000, // 15 minutes + limit: 10000, // 10k requests per 15 mins per IP + message: { error: "Too many requests to ARCE itself. Please slow down." } }); + app.use(globalLimiter); app.use(express.json({ limit: "1mb" })); app.use("/static", express.static(staticDir)); @@ -71,7 +82,7 @@ export function createServerApp(args: CreateServerAppArgs): Express { app.use( ( error: unknown, - _request: Request, + request: Request, response: Response, _next: NextFunction ) => { @@ -87,7 +98,7 @@ export function createServerApp(args: CreateServerAppArgs): Express { } if (error instanceof Error) { - console.error("Unhandled error:", error); + request.log.error(error, "Unhandled error during request processing"); } response.status(500).json({ diff --git a/src/api/middleware/auth.ts b/src/api/middleware/auth.ts index 86acb3e..23830cc 100644 --- a/src/api/middleware/auth.ts +++ b/src/api/middleware/auth.ts @@ -1,3 +1,4 @@ +import crypto from "node:crypto"; import type { RequestHandler } from "express"; import { runtimeConfig } from "../../../configs/runtime"; @@ -14,13 +15,32 @@ export function createAuthMiddleware(): RequestHandler { const configuredKey = runtimeConfig.apiKey; if (!configuredKey) { + if (runtimeConfig.isProduction) { + response.status(500).json({ + error: "Internal Server Error: Missing API_KEY in production." + }); + return; + } next(); return; } const provided = request.header("x-api-key"); - if (!provided || provided !== configuredKey) { + if (!provided) { + response.status(401).json({ + error: "Unauthorized. Provide a valid x-api-key header." + }); + return; + } + + const providedBuffer = Buffer.from(provided); + const configuredBuffer = Buffer.from(configuredKey); + + if ( + providedBuffer.length !== configuredBuffer.length || + !crypto.timingSafeEqual(providedBuffer, configuredBuffer) + ) { response.status(401).json({ error: "Unauthorized. Provide a valid x-api-key header." }); diff --git a/src/core/abuse-detector.ts b/src/core/abuse-detector.ts index 2edb3f6..9c93bba 100644 --- a/src/core/abuse-detector.ts +++ b/src/core/abuse-detector.ts @@ -1,3 +1,4 @@ +import { runtimeConfig } from "../../configs/runtime"; import type { AbuseAssessment, AnomalyFlag, BehaviorSnapshot } from "../types"; function push( @@ -20,7 +21,10 @@ export function analyzeBehavior(snapshot: BehaviorSnapshot): AbuseAssessment { ? snapshot.recentTenSecondCount / snapshot.averageTenSecondCount : snapshot.recentTenSecondCount; - if (snapshot.recentTenSecondCount >= 15 && burstRatio >= 3) { + if ( + snapshot.recentTenSecondCount >= runtimeConfig.abuse.burstSpikeCount && + burstRatio >= runtimeConfig.abuse.burstSpikeRatio + ) { riskScore += 35; push( anomalies, @@ -35,8 +39,10 @@ export function analyzeBehavior(snapshot: BehaviorSnapshot): AbuseAssessment { } if ( - snapshot.duplicateFingerprintCount >= 8 && - snapshot.duplicateFingerprintRatio >= 0.7 + snapshot.duplicateFingerprintCount >= + runtimeConfig.abuse.duplicateFingerprintCount && + snapshot.duplicateFingerprintRatio >= + runtimeConfig.abuse.duplicateFingerprintRatio ) { riskScore += 30; push( @@ -51,7 +57,10 @@ export function analyzeBehavior(snapshot: BehaviorSnapshot): AbuseAssessment { ); } - if (snapshot.uniqueRouteCount >= 12 && snapshot.trailingMinuteCount >= 30) { + if ( + snapshot.uniqueRouteCount >= runtimeConfig.abuse.wideRouteScanCount && + snapshot.trailingMinuteCount >= runtimeConfig.abuse.wideRouteScanMinuteCount + ) { riskScore += 25; push( anomalies, @@ -65,7 +74,9 @@ export function analyzeBehavior(snapshot: BehaviorSnapshot): AbuseAssessment { ); } - if (snapshot.deniedLastFiveMinutes >= 5) { + if ( + snapshot.deniedLastFiveMinutes >= runtimeConfig.abuse.repeatedDenialsCount + ) { riskScore += 15; push( anomalies, @@ -80,8 +91,8 @@ export function analyzeBehavior(snapshot: BehaviorSnapshot): AbuseAssessment { } if ( - snapshot.missingUserAgentCount >= 10 && - snapshot.recentTenSecondCount >= 10 + snapshot.missingUserAgentCount >= runtimeConfig.abuse.missingUserAgentCount && + snapshot.recentTenSecondCount >= runtimeConfig.abuse.missingUserAgentCount ) { riskScore += 10; push( diff --git a/src/store/behavior-store.ts b/src/store/behavior-store.ts index cf37e55..28c5bbd 100644 --- a/src/store/behavior-store.ts +++ b/src/store/behavior-store.ts @@ -57,8 +57,6 @@ export class BehaviorStore { private keys(subject: string) { return { requestTenSecond: `${this.prefix}:behavior:req10s:${subject}`, - fingerprints: `${this.prefix}:behavior:fingerprint:${subject}`, - routes: `${this.prefix}:behavior:route:${subject}`, denials: `${this.prefix}:behavior:denials:${subject}`, missingUserAgent: `${this.prefix}:behavior:missing-ua:${subject}`, block: `${this.prefix}:block:${subject}`, @@ -80,13 +78,20 @@ export class BehaviorStore { const fingerprintHash = hashValue(event.fingerprint); const routeHash = hashValue(event.route); + const fingerprintKey = `${this.prefix}:behavior:fp:${subject}:${fingerprintHash}`; + const routeKey = `${this.prefix}:behavior:routes:${subject}:${minuteBucket}`; + const write = this.client.multi(); write.hIncrBy(keys.requestTenSecond, tenSecondBucket, 1); write.pExpire(keys.requestTenSecond, TWENTY_MINUTES_MS); - write.hIncrBy(keys.fingerprints, `${minuteBucket}:${fingerprintHash}`, 1); - write.pExpire(keys.fingerprints, TEN_MINUTES_MS); - write.hIncrBy(keys.routes, `${minuteBucket}:${routeHash}`, 1); - write.pExpire(keys.routes, TEN_MINUTES_MS); + + // Store fingerprint count for this specific hash + write.hIncrBy(fingerprintKey, minuteBucket, 1); + write.pExpire(fingerprintKey, TEN_MINUTES_MS); + + // Track unique routes via HyperLogLog + write.pfAdd(routeKey, routeHash); + write.pExpire(routeKey, TEN_MINUTES_MS); if (!event.userAgent) { write.hIncrBy(keys.missingUserAgent, minuteBucket, 1); @@ -95,18 +100,22 @@ export class BehaviorStore { await write.exec(); + // We only need the current and previous minute buckets for fingerprints + const prevMinuteBucket = (Number(minuteBucket) - 60_000).toString(); + const routePrevKey = `${this.prefix}:behavior:routes:${subject}:${prevMinuteBucket}`; + const [ requestBuckets, - fingerprintBuckets, - routeBuckets, + fingerprintVals, + uniqueRouteCount, denialBuckets, missingUserAgentBuckets, blockReason, blockTtlMs ] = await Promise.all([ this.client.hGetAll(keys.requestTenSecond), - this.client.hGetAll(keys.fingerprints), - this.client.hGetAll(keys.routes), + this.client.hmGet(fingerprintKey, [minuteBucket, prevMinuteBucket]), + this.client.pfCount([routeKey, routePrevKey]), this.client.hGetAll(keys.denials), this.client.hGetAll(keys.missingUserAgent), this.client.get(keys.block), @@ -125,31 +134,12 @@ export class BehaviorStore { requestBuckets, event.timestampMs ); - const duplicateFingerprintCount = Object.entries(fingerprintBuckets).reduce( - (total, [field, value]) => { - const [bucket, storedHash] = field.split(":"); - if ( - storedHash === fingerprintHash && - Number(bucket) >= event.timestampMs - 60_000 - ) { - return total + Number(value); - } + const duplicateFingerprintCount = (Number(fingerprintVals[0]) || 0) + (Number(fingerprintVals[1]) || 0); - return total; - }, - 0 - ); const duplicateFingerprintRatio = trailingMinuteCount > 0 ? duplicateFingerprintCount / trailingMinuteCount : 0; - const uniqueRouteCount = new Set( - Object.keys(routeBuckets) - .filter( - (field) => Number(field.split(":")[0]) >= event.timestampMs - 60_000 - ) - .map((field) => field.split(":")[1]) - ).size; const deniedLastFiveMinutes = sumBuckets( denialBuckets, event.timestampMs - FIVE_MINUTES_MS diff --git a/src/store/rate-limiter-store.ts b/src/store/rate-limiter-store.ts index 33d61bb..25a1fd7 100644 --- a/src/store/rate-limiter-store.ts +++ b/src/store/rate-limiter-store.ts @@ -113,16 +113,23 @@ local request_id = ARGV[7] redis.call("ZREMRANGEBYSCORE", key, "-inf", now - window_ms) -local current = redis.call("ZCARD", key) +local elements = redis.call("ZRANGE", key, 0, -1) +local current = 0 +for _, elem in ipairs(elements) do + -- elem format is "requestId:cost" + local elem_cost = string.match(elem, ":(%d+)$") + if elem_cost then + current = current + tonumber(elem_cost) + end +end + local allowed = 0 local retry_after_ms = 0 if (current + cost) <= limit then allowed = 1 if consume == 1 then - for i = 1, cost do - redis.call("ZADD", key, now, request_id .. ":" .. i) - end + redis.call("ZADD", key, now, request_id .. ":" .. tostring(cost)) current = current + cost end else @@ -132,6 +139,7 @@ else end end + redis.call("PEXPIRE", key, ttl_ms) local remaining = math.max(0, limit - current) diff --git a/src/store/redis.ts b/src/store/redis.ts index 5fa1308..1e6139f 100644 --- a/src/store/redis.ts +++ b/src/store/redis.ts @@ -11,9 +11,10 @@ export async function createRedisConnection(): Promise { reconnectStrategy(retries: number) { if (retries > 10) { console.error( - `Redis reconnection failed after ${retries} attempts. Giving up.` + `Redis reconnection failed after ${retries} attempts. Giving up and crashing process.` ); - return new Error("Redis reconnection limit reached."); + // Exit process so container orchestrator (e.g. Kubernetes/Docker) restarts the service. + process.exit(1); } const delay = Math.min(retries * 200, 5_000); diff --git a/tests/load/README.md b/tests/load/README.md index 8573c6d..5a1a3fd 100644 --- a/tests/load/README.md +++ b/tests/load/README.md @@ -2,10 +2,18 @@ `basic-smoke.js` is a minimal k6 script intended as a starting point for repeatable load validation. +> **Note:** Running this script requires [k6](https://k6.io/docs/get-started/installation/) to be installed on your system. + Example: ```bash k6 run tests/load/basic-smoke.js ``` +If you don't have k6 installed and just want a quick connectivity check, use the zero-dependency smoke test instead: + +```bash +npm run smoke +``` + The script is intentionally lightweight. It exercises the `/consume` path and leaves deeper workload modeling to future contributors.