From 11a330ff8d2b740b35f99c72cd1245b8dba1c48b Mon Sep 17 00:00:00 2001 From: debjit450 Date: Tue, 28 Apr 2026 21:01:23 +0530 Subject: [PATCH 1/6] chore: trigger CI From 040d75f8449ecd4172e7c0039d25f0cceba6e5bc Mon Sep 17 00:00:00 2001 From: debjit450 Date: Tue, 28 Apr 2026 21:03:53 +0530 Subject: [PATCH 2/6] chore: bootstrap CI From a778fc085f767f20e15f056aaa9ca77464b62e5f Mon Sep 17 00:00:00 2001 From: debjit450 Date: Tue, 28 Apr 2026 21:16:13 +0530 Subject: [PATCH 3/6] ci: simplify workflow to single job --- .github/workflows/ci.yml | 34 ++++++++-------------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2828464..ba42d72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,35 +10,17 @@ concurrency: cancel-in-progress: true jobs: - quality: - name: Quality (Node ${{ matrix.node-version }}) + ci: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - node-version: [20, 22] steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - - name: Setup Node.js - uses: actions/setup-node@v4 + - uses: actions/setup-node@v4 with: - node-version: ${{ matrix.node-version }} - cache: npm + node-version: 22 - - name: Install dependencies - run: npm ci - - - name: Build - run: npm run build - - - name: Lint - run: npm run lint - - - name: Check formatting - run: npm run format:check - - - name: Run tests - run: npm test + - run: npm ci + - run: npm run build + - run: npm run lint + - run: npm test \ No newline at end of file From fba55f4b7b30bb2d04f6dd7d761d60c481c6e440 Mon Sep 17 00:00:00 2001 From: debjit450 Date: Wed, 29 Apr 2026 11:24:41 +0530 Subject: [PATCH 4/6] =?UTF-8?q?fix!:=20full=20security=20audit=20remediati?= =?UTF-8?q?on=20=E2=80=94=20auth,=20XSS,=20input=20limits,=20error=20handl?= =?UTF-8?q?ing,=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements all 16 findings from a comprehensive project audit across security, code quality, test coverage, DevOps, and documentation. Security (Critical) ─────────────────── - Add API key authentication middleware on protected endpoints (/check-limit, /consume, /api/dashboard-data). Auth is disabled when API_KEY is unset so local development remains frictionless. - Eliminate XSS in dashboard by replacing all innerHTML with safe DOM construction (createElement + textContent). User-controlled data (subject, reason, anomaly codes) can no longer execute scripts. - Add max-length constraints on userId (256), ip (64), and identifier (256) in Zod schema to prevent Redis memory exhaustion via oversized keys. - Differentiate ZodError (400 + safe validation issues) from internal errors (500 + generic message). Error.message no longer leaks Redis connection strings or stack traces to clients. - Add X-Content-Type-Options, X-Frame-Options, X-XSS-Protection, and Referrer-Policy security headers on all responses. Code Quality ──────────── - Parallelize recordOutcome() and recordDecision() with Promise.all() on the hot path, cutting ~1 Redis round-trip per request. - Add 10-second graceful shutdown timeout. The process now force-exits instead of hanging indefinitely on undrained connections. - Add Redis reconnect strategy with exponential backoff (capped at 5s, max 10 retries) instead of silently logging errors forever. Test Coverage (5 → 62 tests) ──────────────────────────── - abuse-detector.test.ts: 18 tests — every signal threshold boundary, activeBlock override, recommendedBlock cutoff, cumulative scoring - schemas.test.ts: 16 tests — all superRefine rules, max-length enforcement, cost/baseLimitPerMinute ranges, field defaults - identity.test.ts: 11 tests — all 7 resolveSubject() branches, buildFingerprint() paths including whitespace trimming - time.test.ts: 5 tests — bucketStart() with default and custom windows, boundary alignment, zero timestamp - hashing.test.ts: 4 tests — determinism, hex format, length, empty string - express-middleware.test.ts: 3 tests — allow/deny/error-forwarding paths, rate-limit header assertions DevOps & CI ─────────── - Add .dockerignore to exclude node_modules, .git, tests, docs from build context - Dockerfile: run as non-root (USER node), add HEALTHCHECK, use npm ci, copy only dashboard/public instead of entire apps/ dir - CI: add format:check, typecheck, npm audit steps; add Redis 7 service container; test on Node 20 + 22 matrix Documentation ───────────── - Add Authentication section to README with usage example - Add API_KEY to configuration table and .env.example - Update SDK example with x-api-key header - Add format:check to development commands - Add CHANGELOG.md with 1.0.0 and 1.1.0 entries - Fill package.json author field - Reference CHANGELOG from README Files: 10 new, 11 modified Tests: 8 files, 62 passing --- .dockerignore | 18 ++ .env.example | 5 + .github/workflows/ci.yml | 22 ++- CHANGELOG.md | 36 ++++ Dockerfile | 14 +- README.md | 38 +++- apps/dashboard/public/dashboard.js | 191 +++++++++++------- apps/server/main.ts | 12 ++ configs/runtime.ts | 1 + docs/system-diagram.md | 233 ++++++++++++++++++++++ package.json | 2 +- src/api/app.ts | 41 +++- src/api/middleware/auth.ts | 32 +++ src/api/schemas.ts | 6 +- src/core/limiter-service.ts | 14 +- src/store/redis.ts | 18 +- tests/unit/abuse-detector.test.ts | 274 ++++++++++++++++++++++++++ tests/unit/express-middleware.test.ts | 123 ++++++++++++ tests/unit/hashing.test.ts | 29 +++ tests/unit/identity.test.ts | 85 ++++++++ tests/unit/schemas.test.ts | 153 ++++++++++++++ tests/unit/time.test.ts | 33 ++++ 22 files changed, 1281 insertions(+), 99 deletions(-) create mode 100644 .dockerignore create mode 100644 CHANGELOG.md create mode 100644 docs/system-diagram.md create mode 100644 src/api/middleware/auth.ts create mode 100644 tests/unit/abuse-detector.test.ts create mode 100644 tests/unit/express-middleware.test.ts create mode 100644 tests/unit/hashing.test.ts create mode 100644 tests/unit/identity.test.ts create mode 100644 tests/unit/schemas.test.ts create mode 100644 tests/unit/time.test.ts diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..654ccda --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +node_modules +dist +.git +.github +coverage +tests +docs +scripts +*.md +.env +.env.* +!.env.example +.editorconfig +.prettierrc.json +.prettierignore +.gitignore +eslint.config.js +vitest.config.ts diff --git a/.env.example b/.env.example index 7d6a248..21a8b7e 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,11 @@ REDIS_URL=redis://localhost:6379 # Prefix used for Redis keys so multiple environments can share one Redis cluster SERVICE_NAME=arce +# Optional API key for authenticating protected endpoints. +# When set, all /check-limit, /consume, and /api/dashboard-data requests +# must include a matching x-api-key header. When empty, auth is disabled. +API_KEY= + # Baseline limit applied to normal traffic DEFAULT_LIMIT_PER_MINUTE=100 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba42d72..b4e4f71 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,14 +13,32 @@ jobs: ci: runs-on: ubuntu-latest + strategy: + matrix: + node-version: [20, 22] + + services: + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 + steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: ${{ matrix.node-version }} - run: npm ci - run: npm run build + - run: npm run typecheck - run: npm run lint - - run: npm test \ No newline at end of file + - run: npm run format:check + - run: npm test + - run: npm audit --audit-level=moderate diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8a675dc --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,36 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [1.1.0] — 2026-04-29 + +### Security + +- **API key authentication** — protected endpoints (`/check-limit`, `/consume`, `/api/dashboard-data`) now require a valid `x-api-key` header when `API_KEY` is configured. Authentication is disabled when the variable is empty, preserving the local development workflow. +- **Dashboard XSS fix** — replaced all `innerHTML` usage in the dashboard with safe DOM construction (`textContent` and `createElement`). User-controlled data can no longer execute scripts in the operator's browser. +- **Input length limits** — `userId` (max 256), `ip` (max 64), and `identifier` (max 256) fields now enforce maximum length to prevent memory exhaustion via oversized Redis keys. +- **Error handler hardened** — `ZodError` returns 400 with safe validation details. All other errors return a generic 500 without leaking internal messages or stack traces. +- **Security headers** — all responses now include `X-Content-Type-Options`, `X-Frame-Options`, `X-XSS-Protection`, and `Referrer-Policy`. + +### Changed + +- **Parallel I/O on hot path** — `recordOutcome` and `recordDecision` now execute via `Promise.all()` instead of sequential awaits, reducing latency by one Redis round-trip. +- **Graceful shutdown timeout** — the server now force-exits after 10 seconds if `server.close()` hangs on connection draining. +- **Redis reconnect strategy** — the Redis client now retries with exponential backoff (up to 10 attempts) instead of silently logging errors. + +### Added + +- **Comprehensive test suite** — added 50+ tests across 7 new test files: + - Abuse detector edge cases and threshold boundaries + - Schema validation for all `superRefine` rules and field constraints + - Identity utility tests (all `resolveSubject` branches, `buildFingerprint` paths) + - Hashing determinism and format tests + - Time bucketing tests + - Express middleware tests (allow, deny, error forwarding) +- **`.dockerignore`** — excludes dev-only files from the build context. +- **Docker hardening** — non-root user (`USER node`), `HEALTHCHECK` directive, `npm ci` for reproducible installs, only copies necessary runtime assets. +- **CI improvements** — added `format:check`, `typecheck`, `npm audit`, Redis service container, and Node 20 + 22 matrix testing. + +## [1.0.0] — 2026-04-29 + +Initial release. Distributed adaptive traffic control with Redis-backed rate limiting, abuse detection, and operator dashboard. diff --git a/Dockerfile b/Dockerfile index 34d5459..8f6835d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM node:24-alpine AS build WORKDIR /app COPY package*.json ./ -RUN npm install +RUN npm ci COPY tsconfig.json ./ COPY configs ./configs COPY apps ./apps @@ -11,8 +11,16 @@ RUN npm run build FROM node:24-alpine WORKDIR /app COPY package*.json ./ -RUN npm install --omit=dev +RUN npm ci --omit=dev + COPY --from=build /app/dist ./dist -COPY --from=build /app/apps ./apps +COPY --from=build /app/apps/dashboard/public ./apps/dashboard/public + EXPOSE 4000 + +USER node + +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:4000/health || exit 1 + CMD ["node", "dist/apps/server/main.js"] diff --git a/README.md b/README.md index 1bbe555..fa555ef 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ Additional design notes: - [docs/architecture.md](./docs/architecture.md) - [docs/design-decisions.md](./docs/design-decisions.md) +- [docs/system-diagram.md](./docs/system-diagram.md) ## Design Decisions @@ -208,6 +209,21 @@ http://localhost:4000/dashboard If you need non-default settings, use the variables documented in [.env.example](./.env.example). +## Authentication + +When `API_KEY` is set in the environment, ARCE requires all protected endpoints (`/check-limit`, `/consume`, `/api/dashboard-data`) to include a matching `x-api-key` header. + +When `API_KEY` is not set, authentication is disabled so local development remains frictionless. + +```bash +curl -X POST http://localhost:4000/consume \ + -H "Content-Type: application/json" \ + -H "x-api-key: your-secret-key" \ + -d '{"algorithm":"token_bucket","route":"/api/orders","method":"GET","ip":"203.0.113.8","scope":"ip"}' +``` + +Public endpoints (`/`, `/health`, `/dashboard`, `/static/*`) do not require authentication. + ## API Example Check a limit without consuming: @@ -251,7 +267,8 @@ Example request shape: import { ArceClient } from "./src/sdk/client"; const client = new ArceClient({ - baseUrl: "http://localhost:4000" + baseUrl: "http://localhost:4000", + headers: { "x-api-key": process.env.ARCE_API_KEY ?? "" } }); const decision = await client.consume({ @@ -297,14 +314,15 @@ app.use( ## Configuration -| Variable | Default | Purpose | -| ----------------------------- | ------------------------ | ---------------------------------------------------- | -| `PORT` | `4000` | HTTP port for the ARCE server | -| `REDIS_URL` | `redis://localhost:6379` | Redis connection string | -| `SERVICE_NAME` | `arce` | Prefix for Redis keys | -| `DEFAULT_LIMIT_PER_MINUTE` | `100` | Baseline limit for normal traffic | -| `SUSPICIOUS_LIMIT_PER_MINUTE` | `20` | Lower bound for suspicious traffic | -| `BLOCK_DURATION_SECONDS` | `300` | Temporary block duration for critical abuse patterns | +| Variable | Default | Purpose | +| ----------------------------- | ------------------------ | ---------------------------------------------------------------- | +| `PORT` | `4000` | HTTP port for the ARCE server | +| `REDIS_URL` | `redis://localhost:6379` | Redis connection string | +| `SERVICE_NAME` | `arce` | Prefix for Redis keys | +| `API_KEY` | _(empty, auth disabled)_ | API key for protecting `/check-limit`, `/consume`, and dashboard | +| `DEFAULT_LIMIT_PER_MINUTE` | `100` | Baseline limit for normal traffic | +| `SUSPICIOUS_LIMIT_PER_MINUTE` | `20` | Lower bound for suspicious traffic | +| `BLOCK_DURATION_SECONDS` | `300` | Temporary block duration for critical abuse patterns | ## Development @@ -312,11 +330,13 @@ app.use( npm run build npm run typecheck npm run lint +npm run format:check npm run test npm run smoke ``` See [CONTRIBUTING.md](./CONTRIBUTING.md) for contribution and PR expectations. +See [CHANGELOG.md](./CHANGELOG.md) for a history of changes. ## Roadmap diff --git a/apps/dashboard/public/dashboard.js b/apps/dashboard/public/dashboard.js index cee1ca6..47fcdb0 100644 --- a/apps/dashboard/public/dashboard.js +++ b/apps/dashboard/public/dashboard.js @@ -9,6 +9,26 @@ function formatTime(isoString) { }); } +function el(tag, className, children) { + const element = document.createElement(tag); + + if (className) { + element.className = className; + } + + if (typeof children === "string") { + element.textContent = children; + } else if (Array.isArray(children)) { + children.forEach(function (child) { + element.appendChild(child); + }); + } else if (children instanceof Node) { + element.appendChild(children); + } + + return element; +} + function renderTotals(data) { document.getElementById("requests-total").textContent = formatNumber( data.totals.requests @@ -26,97 +46,130 @@ function renderTotals(data) { data.totals.anomalies ); document.getElementById("last-updated").textContent = - `Updated ${new Date().toLocaleTimeString()}`; + "Updated " + new Date().toLocaleTimeString(); } function renderChart(series) { - const container = document.getElementById("traffic-chart"); - const maxRequests = Math.max(...series.map((point) => point.requests), 1); - - container.innerHTML = series - .map((point) => { - const requestHeight = Math.max( - 4, - Math.round((point.requests / maxRequests) * 170) - ); - const blockedHeight = - point.blocked > 0 - ? Math.max(3, Math.round((point.blocked / maxRequests) * 36)) - : 0; - - return ` -
-
-
-
${formatTime(point.timestamp)}
-
- `; - }) - .join(""); + var container = document.getElementById("traffic-chart"); + var maxRequests = Math.max.apply( + null, + series + .map(function (point) { + return point.requests; + }) + .concat([1]) + ); + + container.innerHTML = ""; + + series.forEach(function (point) { + var requestHeight = Math.max( + 4, + Math.round((point.requests / maxRequests) * 170) + ); + var blockedHeight = + point.blocked > 0 + ? Math.max(3, Math.round((point.blocked / maxRequests) * 36)) + : 0; + + var barBlocked = el("div", "bar-blocked"); + barBlocked.style.height = blockedHeight + "px"; + + var barFill = el("div", "bar-fill"); + barFill.style.height = requestHeight + "px"; + + var barLabel = el("div", "bar-label", formatTime(point.timestamp)); + + var bar = el("div", "bar", [barBlocked, barFill, barLabel]); + bar.title = + formatTime(point.timestamp) + + " | requests=" + + point.requests + + ", blocked=" + + point.blocked + + ", anomalies=" + + point.anomalies; + + container.appendChild(bar); + }); } function renderBlocks(blocks) { - const container = document.getElementById("active-blocks"); + var container = document.getElementById("active-blocks"); + container.innerHTML = ""; if (!blocks.length) { - container.innerHTML = - '
No active blocks right now.
'; + container.appendChild(el("div", "empty", "No active blocks right now.")); return; } - container.innerHTML = blocks - .map( - (block) => ` -
- ${block.subject} -

${block.reason}

-
- TTL ${(block.ttlMs / 1000).toFixed(0)}s - Expires ${formatTime(block.expiresAt)} -
-
- ` - ) - .join(""); + blocks.forEach(function (block) { + var subjectEl = el("strong", null, block.subject); + var reasonEl = el("p", null, block.reason); + var ttlBadge = el( + "span", + "badge", + "TTL " + (block.ttlMs / 1000).toFixed(0) + "s" + ); + var expiresBadge = el( + "span", + "badge", + "Expires " + formatTime(block.expiresAt) + ); + var badgeRow = el("div", "badge-row", [ttlBadge, expiresBadge]); + var article = el("article", "list-item", [subjectEl, reasonEl, badgeRow]); + + container.appendChild(article); + }); } function renderAnomalies(events) { - const container = document.getElementById("recent-anomalies"); + var container = document.getElementById("recent-anomalies"); + container.innerHTML = ""; if (!events.length) { - container.innerHTML = '
No flagged anomalies yet.
'; + container.appendChild(el("div", "empty", "No flagged anomalies yet.")); return; } - container.innerHTML = events - .map((event) => { - const badges = event.anomalies - .map((anomaly) => `${anomaly.code}`) - .join(""); - - return ` -
- ${event.subject} -

${event.reasons.join(" ")}

-
- Tier ${event.tier} - Risk ${event.riskScore} - ${formatTime(event.timestamp)} - ${badges} -
-
- `; - }) - .join(""); + events.forEach(function (event) { + var subjectEl = el("strong", null, event.subject); + var reasonEl = el("p", null, event.reasons.join(" ")); + var tierBadge = el("span", "badge", "Tier " + event.tier); + var riskBadge = el("span", "badge", "Risk " + event.riskScore); + var timeBadge = el("span", "badge", formatTime(event.timestamp)); + + var badges = [tierBadge, riskBadge, timeBadge]; + + event.anomalies.forEach(function (anomaly) { + badges.push(el("span", "badge", anomaly.code)); + }); + + var badgeRow = el("div", "badge-row", badges); + var article = el("article", "list-item", [subjectEl, reasonEl, badgeRow]); + + container.appendChild(article); + }); } async function refresh() { - const response = await fetch("/api/dashboard-data"); - const data = await response.json(); - renderTotals(data); - renderChart(data.recentSeries); - renderBlocks(data.activeBlocks); - renderAnomalies(data.recentAnomalies); + try { + var response = await fetch("/api/dashboard-data"); + + if (!response.ok) { + document.getElementById("last-updated").textContent = + "Error: " + response.status; + return; + } + + var data = await response.json(); + renderTotals(data); + renderChart(data.recentSeries); + renderBlocks(data.activeBlocks); + renderAnomalies(data.recentAnomalies); + } catch { + document.getElementById("last-updated").textContent = "Connection error"; + } } refresh(); diff --git a/apps/server/main.ts b/apps/server/main.ts index bd93e37..6f74403 100644 --- a/apps/server/main.ts +++ b/apps/server/main.ts @@ -8,6 +8,8 @@ import { MetricsStore } from "../../src/store/metrics-store"; import { createRedisConnection } from "../../src/store/redis"; import { RedisRateLimiterStore } from "../../src/store/rate-limiter-store"; +const SHUTDOWN_TIMEOUT_MS = 10_000; + async function bootstrap(): Promise { const redis = await createRedisConnection(); const prefix = runtimeConfig.serviceName; @@ -36,8 +38,18 @@ async function bootstrap(): Promise { }); const shutdown = (): void => { + console.log("Shutting down gracefully..."); + + const forceExit = setTimeout(() => { + console.error("Graceful shutdown timed out. Forcing exit."); + process.exit(1); + }, SHUTDOWN_TIMEOUT_MS); + + forceExit.unref(); + server.close(async () => { await redis.quit(); + clearTimeout(forceExit); process.exit(0); }); }; diff --git a/configs/runtime.ts b/configs/runtime.ts index 1fa4d3d..790c607 100644 --- a/configs/runtime.ts +++ b/configs/runtime.ts @@ -17,6 +17,7 @@ export const runtimeConfig = { port: readNumber("PORT", 4000), redisUrl: process.env.REDIS_URL ?? "redis://localhost:6379", serviceName: process.env.SERVICE_NAME ?? "arce", + apiKey: process.env.API_KEY ?? "", defaultLimitPerMinute: readNumber("DEFAULT_LIMIT_PER_MINUTE", 100), suspiciousLimitPerMinute: readNumber("SUSPICIOUS_LIMIT_PER_MINUTE", 20), blockDurationSeconds: readNumber("BLOCK_DURATION_SECONDS", 300) diff --git a/docs/system-diagram.md b/docs/system-diagram.md new file mode 100644 index 0000000..f922973 --- /dev/null +++ b/docs/system-diagram.md @@ -0,0 +1,233 @@ +# ARCE System Diagram + +This document translates the current codebase into diagrams that are useful for design reviews, demos, and onboarding. + +It is based on the implementation in: + +- `apps/server/main.ts` +- `src/api/*` +- `src/core/*` +- `src/store/*` +- `src/sdk/*` +- `apps/dashboard/public/*` +- `configs/*` +- `.github/workflows/ci.yml` + +## 1. End-to-End System Overview + +```mermaid +flowchart TB + subgraph Actors["External Actors"] + ApiClients["API clients"] + HostApps["Node/Express apps using ARCE"] + Operators["Operators using the dashboard"] + Developers["Developers and CI"] + end + + subgraph Integration["Integration Surface"] + SDK["ArceClient
src/sdk/client.ts"] + Middleware["createArceMiddleware
src/sdk/express-middleware.ts"] + end + + subgraph Runtime["ARCE Server Runtime"] + Bootstrap["Bootstrap
apps/server/main.ts"] + App["Express app
src/api/app.ts"] + Root["GET /"] + Health["GET /health"] + Check["POST /check-limit"] + Consume["POST /consume"] + Dashboard["GET /dashboard"] + DashboardData["GET /api/dashboard-data"] + StaticAssets["Dashboard static files
apps/dashboard/public/*"] + Schema["Zod request validation
src/api/schemas.ts"] + Service["LimiterService"] + Detector["AbuseDetector
analyzeBehavior()"] + Policy["AdaptivePolicy
buildEffectivePolicy()"] + BehaviorStore["BehaviorStore"] + LimiterStore["RedisRateLimiterStore"] + MetricsStore["MetricsStore"] + Snapshot["getDashboardSnapshot()"] + end + + subgraph RedisLayer["Redis Shared State"] + RedisServer["Redis 7 server"] + LimitState["Limiter state
arce:limit:*"] + BehaviorCounters["Behavior counters
req10s / fingerprint / route / missing-ua"] + DenialHistory["Denial history
arce:behavior:denials:*"] + BlockState["Block TTLs and indexes
arce:block:* / arce:blocks*"] + MetricState["Metric totals and minute buckets
arce:metrics:*"] + RecentAnomalyState["Recent anomaly feed"] + end + + subgraph Tooling["Configuration, Delivery, and Validation"] + Env[".env + configs/runtime.ts"] + Compose["docker-compose.yml"] + Docker["Dockerfile"] + Tests["Vitest / Supertest / smoke tests"] + CI["GitHub Actions CI"] + Dist["Compiled dist/* output"] + end + + ApiClients -->|direct HTTP| App + ApiClients -->|Node integration| SDK + HostApps --> Middleware + Middleware -->|build payload + consume| SDK + SDK -->|POST JSON| App + + Operators -->|open page| Dashboard + Operators -->|poll data| DashboardData + + Developers -->|npm run dev / start| Bootstrap + Developers -->|npm test / smoke| Tests + CI -->|npm ci, build, lint, test| Tests + Docker --> Dist + Dist --> Bootstrap + Compose --> RedisServer + Env --> Bootstrap + + Bootstrap --> App + Bootstrap --> BehaviorStore + Bootstrap --> LimiterStore + Bootstrap --> MetricsStore + Bootstrap --> RedisServer + + App --> Root + App --> Health + App --> Check + App --> Consume + App --> Dashboard + App --> DashboardData + Dashboard --> StaticAssets + + Check --> Schema + Consume --> Schema + Schema --> Service + + Service --> BehaviorStore + BehaviorStore --> RedisServer + RedisServer --> BehaviorCounters + RedisServer --> DenialHistory + RedisServer --> BlockState + + Service --> Detector + Detector --> Policy + Policy -->|tier and limit| Service + Policy -->|blocked tier| BehaviorStore + + Service --> LimiterStore + LimiterStore -->|Lua enforcement| RedisServer + RedisServer --> LimitState + + Service --> MetricsStore + MetricsStore --> RedisServer + RedisServer --> MetricState + RedisServer --> RecentAnomalyState + + DashboardData --> Snapshot + Snapshot --> MetricsStore + Snapshot --> BehaviorStore +``` + +## 2. Request Lifecycle + +This sequence shows the hot path for `POST /consume`. + +```mermaid +sequenceDiagram + participant Caller as Caller + participant Client as ArceClient or direct HTTP client + participant API as limit handler + participant Schema as Zod schema + participant Service as LimiterService + participant Behavior as BehaviorStore + participant Redis as Redis + participant Detector as AbuseDetector + participant Policy as AdaptivePolicy + participant Limiter as RedisRateLimiterStore + participant Metrics as MetricsStore + + Caller->>Client: send subject + route + algorithm + Client->>API: POST /consume + API->>Schema: validate request body + Schema-->>API: normalized payload + API->>Service: evaluate(payload, true) + + Service->>Behavior: observe(subject, event) + Behavior->>Redis: update counters and read recent state + Redis-->>Behavior: behavior snapshot + Behavior-->>Service: recent behavior + + Service->>Detector: analyzeBehavior(snapshot) + Detector-->>Service: risk score + anomalies + Service->>Policy: buildEffectivePolicy(...) + Policy-->>Service: tier + effective limit + + alt active block already exists + Service->>Behavior: recordOutcome(denied) + else blocked tier recommended + Service->>Behavior: registerBlock(subject, reason, ttl) + Service->>Behavior: recordOutcome(denied) + else enforce adaptive limit + Service->>Limiter: evaluate(subject, algorithm, policy, cost, consume) + Limiter->>Redis: execute Lua script atomically + Redis-->>Limiter: allowed, remaining, retry, reset + Limiter-->>Service: limiter decision + Service->>Behavior: recordOutcome(allowed or denied) + end + + Service->>Metrics: recordDecision(result) + Metrics->>Redis: update totals, minute buckets, anomaly feed + Service-->>API: EnforcementResult + API-->>Caller: 200 or 429 JSON +``` + +## 3. Redis Responsibility Map + +This diagram shows how Redis is partitioned by responsibility instead of treating every key as one blob of rate-limiter state. + +```mermaid +flowchart LR + BehaviorStore["BehaviorStore"] --> Req10s["arce:behavior:req10s:[subject]
10-second volume buckets"] + BehaviorStore --> Fingerprints["arce:behavior:fingerprint:[subject]
per-minute duplicate fingerprints"] + BehaviorStore --> Routes["arce:behavior:route:[subject]
per-minute route fan-out"] + BehaviorStore --> Denials["arce:behavior:denials:[subject]
recent denials"] + BehaviorStore --> MissingUa["arce:behavior:missing-ua:[subject]
missing user-agent counts"] + BehaviorStore --> LiveBlock["arce:block:[subject]
temporary block TTL"] + BehaviorStore --> BlockIndex["arce:blocks and arce:blocks:meta
active block index and metadata"] + + LimiterStore["RedisRateLimiterStore"] --> Token["token bucket state
tokens + updatedAt"] + LimiterStore --> Leaky["leaky bucket state
level + updatedAt"] + LimiterStore --> Sliding["sliding window state
sorted set of request timestamps"] + + MetricsStore["MetricsStore"] --> Totals["arce:metrics:totals
global counters"] + MetricsStore --> Series["arce:metrics:req / blocked / rate-limited / anomalies
minute series"] + MetricsStore --> Recent["arce:metrics:recent-anomalies
dashboard event feed"] +``` + +## 4. Build, Run, and Verify Path + +```mermaid +flowchart LR + Source["TypeScript source
apps / src / configs"] --> Build["npm run build
tsc"] + Build --> Dist["dist/* compiled JavaScript"] + Dist --> Start["node dist/apps/server/main.js"] + Start --> Runtime["ARCE server process"] + Redis["redis:7 via docker-compose"] --> Runtime + Env[".env settings"] --> Runtime + + Dev["Developer"] -->|npm run dev| Runtime + Dev -->|npm test| TestSuite["Vitest + Supertest"] + Dev -->|npm run smoke| Smoke["scripts/smoke-test.mjs"] + + CI[".github/workflows/ci.yml"] --> Install["npm ci"] + Install --> CIBuild["npm run build"] + CIBuild --> Lint["npm run lint"] + Lint --> CITest["npm test"] +``` + +## Reading Guide + +- Use diagram 1 when you need the full system picture. +- Use diagram 2 when you need to explain exactly how one rate-limit decision is produced. +- Use diagram 3 when you need to reason about Redis data ownership and lifecycle. +- Use diagram 4 when you need to explain developer workflow, packaging, and CI. diff --git a/package.json b/package.json index 18045d6..c9f55af 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "traffic-control", "express" ], - "author": "", + "author": "ARCE contributors", "license": "MIT", "engines": { "node": ">=20" diff --git a/src/api/app.ts b/src/api/app.ts index bcbd244..5c05797 100644 --- a/src/api/app.ts +++ b/src/api/app.ts @@ -6,6 +6,8 @@ import express, { } from "express"; import path from "node:path"; +import { ZodError } from "zod"; + import { createDashboardDataHandler, createDashboardPageHandler @@ -13,6 +15,7 @@ import { import { createHealthHandler } from "./handlers/health"; import { createLimitHandler } from "./handlers/limit"; import { createRootHandler } from "./handlers/root"; +import { createAuthMiddleware } from "./middleware/auth"; import { LimiterService } from "../core/limiter-service"; import type { RedisClient } from "../store/redis"; @@ -25,20 +28,40 @@ interface CreateServerAppArgs { export function createServerApp(args: CreateServerAppArgs): Express { const app = express(); const staticDir = path.resolve(args.dashboardPublicDir); + const auth = createAuthMiddleware(); app.disable("x-powered-by"); + + // Security headers + app.use((_request, response, next) => { + response.setHeader("X-Content-Type-Options", "nosniff"); + response.setHeader("X-Frame-Options", "DENY"); + response.setHeader("X-XSS-Protection", "1; mode=block"); + response.setHeader("Referrer-Policy", "strict-origin-when-cross-origin"); + next(); + }); + app.use(express.json({ limit: "1mb" })); app.use("/static", express.static(staticDir)); + // Public endpoints app.get("/", createRootHandler()); app.get("/health", createHealthHandler(args.redis)); - app.post("/check-limit", createLimitHandler(args.limiterService, false)); - app.post("/consume", createLimitHandler(args.limiterService, true)); + app.get("/dashboard", createDashboardPageHandler(staticDir)); + + // Protected endpoints + app.post( + "/check-limit", + auth, + createLimitHandler(args.limiterService, false) + ); + app.post("/consume", auth, createLimitHandler(args.limiterService, true)); app.get( "/api/dashboard-data", + auth, createDashboardDataHandler(args.limiterService) ); - app.get("/dashboard", createDashboardPageHandler(staticDir)); + app.use((_request, response) => { response.status(404).json({ error: "Route not found" @@ -52,13 +75,21 @@ export function createServerApp(args: CreateServerAppArgs): Express { response: Response, _next: NextFunction ) => { - if (error instanceof Error) { + if (error instanceof ZodError) { response.status(400).json({ - error: error.message + error: "Validation failed", + issues: error.issues.map((issue) => ({ + path: issue.path.join("."), + message: issue.message + })) }); return; } + if (error instanceof Error) { + console.error("Unhandled error:", error); + } + response.status(500).json({ error: "Unexpected server error" }); diff --git a/src/api/middleware/auth.ts b/src/api/middleware/auth.ts new file mode 100644 index 0000000..86acb3e --- /dev/null +++ b/src/api/middleware/auth.ts @@ -0,0 +1,32 @@ +import type { RequestHandler } from "express"; + +import { runtimeConfig } from "../../../configs/runtime"; + +/** + * API key authentication middleware. + * + * When `API_KEY` is set in the environment, all protected endpoints require + * a matching `x-api-key` header. When `API_KEY` is not set, authentication + * is disabled so local development remains frictionless. + */ +export function createAuthMiddleware(): RequestHandler { + return (request, response, next) => { + const configuredKey = runtimeConfig.apiKey; + + if (!configuredKey) { + next(); + return; + } + + const provided = request.header("x-api-key"); + + if (!provided || provided !== configuredKey) { + response.status(401).json({ + error: "Unauthorized. Provide a valid x-api-key header." + }); + return; + } + + next(); + }; +} diff --git a/src/api/schemas.ts b/src/api/schemas.ts index 8e671fd..0f8ec20 100644 --- a/src/api/schemas.ts +++ b/src/api/schemas.ts @@ -5,9 +5,9 @@ export const limitRequestSchema = z algorithm: z.enum(["token_bucket", "sliding_window", "leaky_bucket"]), route: z.string().trim().min(1).default("/"), method: z.string().trim().min(1).default("GET"), - userId: z.string().trim().min(1).optional(), - ip: z.string().trim().min(1).optional(), - identifier: z.string().trim().min(1).optional(), + userId: z.string().trim().min(1).max(256).optional(), + ip: z.string().trim().min(1).max(64).optional(), + identifier: z.string().trim().min(1).max(256).optional(), scope: z.enum(["user", "ip", "hybrid", "custom"]).optional(), fingerprint: z.string().trim().min(1).max(256).optional(), cost: z.number().int().min(1).max(10).default(1), diff --git a/src/core/limiter-service.ts b/src/core/limiter-service.ts index 2020992..afe375d 100644 --- a/src/core/limiter-service.ts +++ b/src/core/limiter-service.ts @@ -124,12 +124,14 @@ export class LimiterService { }; } - await this.behaviorStore.recordOutcome(subject, { - allowed: result.allowed, - blocked: result.blocked, - timestampMs: nowMs - }); - await this.metricsStore.recordDecision(result); + await Promise.all([ + this.behaviorStore.recordOutcome(subject, { + allowed: result.allowed, + blocked: result.blocked, + timestampMs: nowMs + }), + this.metricsStore.recordDecision(result) + ]); return result; } diff --git a/src/store/redis.ts b/src/store/redis.ts index 77c4457..5fa1308 100644 --- a/src/store/redis.ts +++ b/src/store/redis.ts @@ -6,7 +6,23 @@ export type RedisClient = ReturnType; export async function createRedisConnection(): Promise { const client = createClient({ - url: runtimeConfig.redisUrl + url: runtimeConfig.redisUrl, + socket: { + reconnectStrategy(retries: number) { + if (retries > 10) { + console.error( + `Redis reconnection failed after ${retries} attempts. Giving up.` + ); + return new Error("Redis reconnection limit reached."); + } + + const delay = Math.min(retries * 200, 5_000); + console.warn( + `Redis connection lost. Retrying in ${delay}ms (attempt ${retries})...` + ); + return delay; + } + } }); client.on("error", (error) => { diff --git a/tests/unit/abuse-detector.test.ts b/tests/unit/abuse-detector.test.ts new file mode 100644 index 0000000..99888c5 --- /dev/null +++ b/tests/unit/abuse-detector.test.ts @@ -0,0 +1,274 @@ +import { describe, expect, it } from "vitest"; + +import { analyzeBehavior } from "../../src/core/abuse-detector"; +import type { BehaviorSnapshot } from "../../src/types"; + +function baseline(): BehaviorSnapshot { + return { + recentTenSecondCount: 0, + trailingMinuteCount: 0, + averageTenSecondCount: 0, + duplicateFingerprintCount: 0, + duplicateFingerprintRatio: 0, + uniqueRouteCount: 0, + deniedLastFiveMinutes: 0, + missingUserAgentCount: 0, + activeBlock: null + }; +} + +describe("analyzeBehavior", () => { + describe("burst spike detection", () => { + it("does not flag when below thresholds", () => { + const snapshot = { + ...baseline(), + recentTenSecondCount: 14, + averageTenSecondCount: 5 + }; + const result = analyzeBehavior(snapshot); + expect( + result.anomalies.find((a) => a.code === "burst_spike") + ).toBeUndefined(); + }); + + it("does not flag high count with low ratio", () => { + const snapshot = { + ...baseline(), + recentTenSecondCount: 15, + averageTenSecondCount: 10 + }; + const result = analyzeBehavior(snapshot); + expect( + result.anomalies.find((a) => a.code === "burst_spike") + ).toBeUndefined(); + }); + + it("flags when count >= 15 and ratio >= 3", () => { + const snapshot = { + ...baseline(), + recentTenSecondCount: 15, + averageTenSecondCount: 5 + }; + const result = analyzeBehavior(snapshot); + const anomaly = result.anomalies.find((a) => a.code === "burst_spike"); + expect(anomaly).toBeDefined(); + expect(anomaly!.severity).toBe("high"); + expect(result.riskScore).toBe(35); + }); + + it("uses raw count as ratio when average is zero", () => { + const snapshot = { + ...baseline(), + recentTenSecondCount: 15, + averageTenSecondCount: 0 + }; + const result = analyzeBehavior(snapshot); + expect( + result.anomalies.find((a) => a.code === "burst_spike") + ).toBeDefined(); + }); + }); + + describe("repeated identical requests", () => { + it("does not flag when below thresholds", () => { + const snapshot = { + ...baseline(), + duplicateFingerprintCount: 7, + duplicateFingerprintRatio: 0.8 + }; + const result = analyzeBehavior(snapshot); + expect( + result.anomalies.find((a) => a.code === "repeated_identical_requests") + ).toBeUndefined(); + }); + + it("does not flag high count with low ratio", () => { + const snapshot = { + ...baseline(), + duplicateFingerprintCount: 10, + duplicateFingerprintRatio: 0.6 + }; + const result = analyzeBehavior(snapshot); + expect( + result.anomalies.find((a) => a.code === "repeated_identical_requests") + ).toBeUndefined(); + }); + + it("flags when count >= 8 and ratio >= 0.7", () => { + const snapshot = { + ...baseline(), + duplicateFingerprintCount: 8, + duplicateFingerprintRatio: 0.7 + }; + const result = analyzeBehavior(snapshot); + const anomaly = result.anomalies.find( + (a) => a.code === "repeated_identical_requests" + ); + expect(anomaly).toBeDefined(); + expect(anomaly!.severity).toBe("high"); + expect(result.riskScore).toBe(30); + }); + }); + + describe("wide route scan", () => { + it("does not flag few routes", () => { + const snapshot = { + ...baseline(), + uniqueRouteCount: 11, + trailingMinuteCount: 40 + }; + const result = analyzeBehavior(snapshot); + expect( + result.anomalies.find((a) => a.code === "wide_route_scan") + ).toBeUndefined(); + }); + + it("does not flag many routes with low traffic", () => { + const snapshot = { + ...baseline(), + uniqueRouteCount: 15, + trailingMinuteCount: 20 + }; + const result = analyzeBehavior(snapshot); + expect( + result.anomalies.find((a) => a.code === "wide_route_scan") + ).toBeUndefined(); + }); + + it("flags when routes >= 12 and trailing count >= 30", () => { + const snapshot = { + ...baseline(), + uniqueRouteCount: 12, + trailingMinuteCount: 30 + }; + const result = analyzeBehavior(snapshot); + const anomaly = result.anomalies.find( + (a) => a.code === "wide_route_scan" + ); + expect(anomaly).toBeDefined(); + expect(anomaly!.severity).toBe("medium"); + expect(result.riskScore).toBe(25); + }); + }); + + describe("repeated denials", () => { + it("does not flag below threshold", () => { + const snapshot = { ...baseline(), deniedLastFiveMinutes: 4 }; + const result = analyzeBehavior(snapshot); + expect( + result.anomalies.find((a) => a.code === "repeated_denials") + ).toBeUndefined(); + }); + + it("flags when denied >= 5", () => { + const snapshot = { ...baseline(), deniedLastFiveMinutes: 5 }; + const result = analyzeBehavior(snapshot); + const anomaly = result.anomalies.find( + (a) => a.code === "repeated_denials" + ); + expect(anomaly).toBeDefined(); + expect(anomaly!.severity).toBe("medium"); + expect(result.riskScore).toBe(15); + }); + }); + + describe("missing user agent", () => { + it("does not flag low volume", () => { + const snapshot = { + ...baseline(), + missingUserAgentCount: 15, + recentTenSecondCount: 9 + }; + const result = analyzeBehavior(snapshot); + expect( + result.anomalies.find((a) => a.code === "missing_user_agent") + ).toBeUndefined(); + }); + + it("flags when count >= 10 and recent traffic >= 10", () => { + const snapshot = { + ...baseline(), + missingUserAgentCount: 10, + recentTenSecondCount: 10 + }; + const result = analyzeBehavior(snapshot); + const anomaly = result.anomalies.find( + (a) => a.code === "missing_user_agent" + ); + expect(anomaly).toBeDefined(); + expect(anomaly!.severity).toBe("low"); + expect(result.riskScore).toBe(10); + }); + }); + + describe("active block override", () => { + it("overrides risk score to at least 90 when an active block exists", () => { + const snapshot = { + ...baseline(), + activeBlock: { + reason: "test block", + ttlMs: 60_000, + expiresAtMs: Date.now() + 60_000 + } + }; + const result = analyzeBehavior(snapshot); + expect(result.riskScore).toBeGreaterThanOrEqual(90); + expect(result.reasons).toContain( + "An active temporary block is already in effect." + ); + }); + }); + + describe("recommended block", () => { + it("does not recommend block below 75", () => { + const snapshot = { + ...baseline(), + recentTenSecondCount: 15, + averageTenSecondCount: 5, + duplicateFingerprintCount: 8, + duplicateFingerprintRatio: 0.7 + }; + const result = analyzeBehavior(snapshot); + // 35 + 30 = 65 + expect(result.riskScore).toBe(65); + expect(result.recommendedBlock).toBe(false); + }); + + it("recommends block at 75 or above", () => { + const snapshot = { + ...baseline(), + recentTenSecondCount: 15, + averageTenSecondCount: 5, + duplicateFingerprintCount: 8, + duplicateFingerprintRatio: 0.7, + uniqueRouteCount: 12, + trailingMinuteCount: 30 + }; + const result = analyzeBehavior(snapshot); + // 35 + 30 + 25 = 90 + expect(result.riskScore).toBe(90); + expect(result.recommendedBlock).toBe(true); + }); + }); + + describe("cumulative scoring", () => { + it("accumulates risk from multiple signals", () => { + const snapshot = { + ...baseline(), + recentTenSecondCount: 20, + averageTenSecondCount: 5, + duplicateFingerprintCount: 10, + duplicateFingerprintRatio: 0.8, + uniqueRouteCount: 14, + trailingMinuteCount: 40, + deniedLastFiveMinutes: 8, + missingUserAgentCount: 12 + }; + const result = analyzeBehavior(snapshot); + // 35 + 30 + 25 + 15 + 10 = 115 + expect(result.riskScore).toBe(115); + expect(result.anomalies).toHaveLength(5); + expect(result.recommendedBlock).toBe(true); + }); + }); +}); diff --git a/tests/unit/express-middleware.test.ts b/tests/unit/express-middleware.test.ts new file mode 100644 index 0000000..c0c3a5f --- /dev/null +++ b/tests/unit/express-middleware.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it, vi } from "vitest"; + +import { createArceMiddleware } from "../../src/sdk/express-middleware"; +import type { ArceClient } from "../../src/sdk/client"; +import type { EnforcementResult } from "../../src/types"; +import type { Request, Response } from "express"; + +function mockResult(allowed: boolean): EnforcementResult { + return { + mode: "consume", + allowed, + blocked: !allowed, + subject: "ip:10.0.0.1", + fingerprint: "GET:/test", + algorithm: "token_bucket", + cost: 1, + anomalies: [], + behavior: { + recentTenSecondCount: 1, + trailingMinuteCount: 1, + averageTenSecondCount: 0, + duplicateFingerprintCount: 0, + duplicateFingerprintRatio: 0, + uniqueRouteCount: 1, + deniedLastFiveMinutes: 0, + missingUserAgentCount: 0, + activeBlock: null + }, + effectivePolicy: { + algorithm: "token_bucket", + tier: "normal", + baseLimitPerMinute: 100, + effectiveLimitPerMinute: 100, + capacity: 100, + windowMs: 60_000, + refillTokensPerMs: 100 / 60_000, + leakRatePerMs: 100 / 60_000, + riskScore: 0, + reasons: [], + blockDurationSeconds: 0 + }, + decision: { allowed, remaining: 99, retryAfterMs: 0, resetAfterMs: 60_000 }, + evaluatedAt: new Date().toISOString() + }; +} + +function mockRequest(overrides: Partial = {}): Request { + return { + originalUrl: "/test", + method: "GET", + ip: "10.0.0.1", + headers: { "user-agent": "test-agent" }, + header: vi.fn().mockReturnValue(undefined), + socket: { remoteAddress: "10.0.0.1" }, + ...overrides + } as unknown as Request; +} + +function mockResponse(): Response & { + _status: number; + _json: unknown; + _headers: Record; +} { + const res = { + _status: 200, + _json: null, + _headers: {} as Record, + status(code: number) { + res._status = code; + return res; + }, + json(data: unknown) { + res._json = data; + return res; + }, + setHeader(key: string, value: string) { + res._headers[key] = value; + return res; + } + }; + return res as never; +} + +describe("createArceMiddleware", () => { + it("calls next() on allowed request", async () => { + const consume = vi.fn().mockResolvedValue(mockResult(true)); + const client = { consume } as unknown as ArceClient; + const middleware = createArceMiddleware({ client }); + const req = mockRequest(); + const res = mockResponse(); + const next = vi.fn(); + + await (middleware as (...args: unknown[]) => Promise)(req, res, next); + expect(next).toHaveBeenCalled(); + expect(res._headers["x-rate-limit-remaining"]).toBe("99"); + }); + + it("returns 429 on denied request", async () => { + const consume = vi.fn().mockResolvedValue(mockResult(false)); + const client = { consume } as unknown as ArceClient; + const middleware = createArceMiddleware({ client }); + const req = mockRequest(); + const res = mockResponse(); + const next = vi.fn(); + + await (middleware as (...args: unknown[]) => Promise)(req, res, next); + expect(next).not.toHaveBeenCalled(); + expect(res._status).toBe(429); + }); + + it("forwards errors to next()", async () => { + const error = new Error("connection failed"); + const consume = vi.fn().mockRejectedValue(error); + const client = { consume } as unknown as ArceClient; + const middleware = createArceMiddleware({ client }); + const req = mockRequest(); + const res = mockResponse(); + const next = vi.fn(); + + await (middleware as (...args: unknown[]) => Promise)(req, res, next); + expect(next).toHaveBeenCalledWith(error); + }); +}); diff --git a/tests/unit/hashing.test.ts b/tests/unit/hashing.test.ts new file mode 100644 index 0000000..7916e96 --- /dev/null +++ b/tests/unit/hashing.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; + +import { hashValue } from "../../src/utils/hashing"; + +describe("hashValue", () => { + it("returns a 16-character hex string", () => { + const result = hashValue("test-input"); + expect(result).toHaveLength(16); + expect(result).toMatch(/^[0-9a-f]{16}$/); + }); + + it("returns deterministic output for the same input", () => { + const first = hashValue("hello"); + const second = hashValue("hello"); + expect(first).toBe(second); + }); + + it("returns different output for different inputs", () => { + const a = hashValue("input-a"); + const b = hashValue("input-b"); + expect(a).not.toBe(b); + }); + + it("handles empty string", () => { + const result = hashValue(""); + expect(result).toHaveLength(16); + expect(result).toMatch(/^[0-9a-f]{16}$/); + }); +}); diff --git a/tests/unit/identity.test.ts b/tests/unit/identity.test.ts new file mode 100644 index 0000000..1068a97 --- /dev/null +++ b/tests/unit/identity.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; + +import { resolveSubject, buildFingerprint } from "../../src/utils/identity"; +import type { LimitRequestPayload } from "../../src/types"; + +function basePayload( + overrides: Partial = {} +): LimitRequestPayload { + return { + algorithm: "token_bucket", + route: "/test", + method: "GET", + ...overrides + }; +} + +describe("resolveSubject", () => { + it("returns hybrid key when scope is hybrid", () => { + const result = resolveSubject( + basePayload({ scope: "hybrid", userId: "u1", ip: "1.2.3.4" }) + ); + expect(result).toBe("hybrid:user:u1:ip:1.2.3.4"); + }); + + it("returns user key when scope is user", () => { + const result = resolveSubject(basePayload({ scope: "user", userId: "u1" })); + expect(result).toBe("user:u1"); + }); + + it("returns ip key when scope is ip", () => { + const result = resolveSubject(basePayload({ scope: "ip", ip: "10.0.0.1" })); + expect(result).toBe("ip:10.0.0.1"); + }); + + it("returns custom key when identifier is provided", () => { + const result = resolveSubject(basePayload({ identifier: "device-abc" })); + expect(result).toBe("custom:device-abc"); + }); + + it("falls back to userId when no scope is set", () => { + const result = resolveSubject(basePayload({ userId: "u1" })); + expect(result).toBe("user:u1"); + }); + + it("falls back to ip when no scope and no userId", () => { + const result = resolveSubject(basePayload({ ip: "10.0.0.1" })); + expect(result).toBe("ip:10.0.0.1"); + }); + + it("throws when no identifier can be derived", () => { + expect(() => resolveSubject(basePayload())).toThrow( + "Unable to derive a subject key" + ); + }); +}); + +describe("buildFingerprint", () => { + it("uses the provided fingerprint when available", () => { + const result = buildFingerprint( + basePayload({ fingerprint: "GET:/api?q=1" }) + ); + expect(result).toBe("GET:/api?q=1"); + }); + + it("trims whitespace from provided fingerprint", () => { + const result = buildFingerprint( + basePayload({ fingerprint: " GET:/api " }) + ); + expect(result).toBe("GET:/api"); + }); + + it("builds a default fingerprint from method and route", () => { + const result = buildFingerprint( + basePayload({ method: "POST", route: "/orders" }) + ); + expect(result).toBe("POST:/orders"); + }); + + it("uppercases the method in default fingerprint", () => { + const result = buildFingerprint( + basePayload({ method: "post", route: "/orders" }) + ); + expect(result).toBe("POST:/orders"); + }); +}); diff --git a/tests/unit/schemas.test.ts b/tests/unit/schemas.test.ts new file mode 100644 index 0000000..16ad6d5 --- /dev/null +++ b/tests/unit/schemas.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it } from "vitest"; + +import { limitRequestSchema } from "../../src/api/schemas"; + +describe("limitRequestSchema", () => { + it("accepts minimal valid payload", () => { + const r = limitRequestSchema.safeParse({ + algorithm: "token_bucket", + ip: "10.0.0.1", + scope: "ip" + }); + expect(r.success).toBe(true); + }); + + it("rejects missing algorithm", () => { + expect(limitRequestSchema.safeParse({ ip: "10.0.0.1" }).success).toBe( + false + ); + }); + + it("rejects invalid algorithm", () => { + expect( + limitRequestSchema.safeParse({ algorithm: "invalid", ip: "1.2.3.4" }) + .success + ).toBe(false); + }); + + it("rejects when no identifier provided", () => { + const r = limitRequestSchema.safeParse({ algorithm: "token_bucket" }); + expect(r.success).toBe(false); + }); + + it("rejects hybrid without userId", () => { + const r = limitRequestSchema.safeParse({ + algorithm: "token_bucket", + ip: "1.2.3.4", + scope: "hybrid" + }); + expect(r.success).toBe(false); + }); + + it("rejects hybrid without ip", () => { + const r = limitRequestSchema.safeParse({ + algorithm: "token_bucket", + userId: "u1", + scope: "hybrid" + }); + expect(r.success).toBe(false); + }); + + it("accepts hybrid with both", () => { + const r = limitRequestSchema.safeParse({ + algorithm: "token_bucket", + userId: "u1", + ip: "1.2.3.4", + scope: "hybrid" + }); + expect(r.success).toBe(true); + }); + + it("rejects user scope without userId", () => { + const r = limitRequestSchema.safeParse({ + algorithm: "token_bucket", + ip: "1.2.3.4", + scope: "user" + }); + expect(r.success).toBe(false); + }); + + it("rejects ip scope without ip", () => { + const r = limitRequestSchema.safeParse({ + algorithm: "token_bucket", + userId: "u1", + scope: "ip" + }); + expect(r.success).toBe(false); + }); + + it("rejects custom scope without identifier", () => { + const r = limitRequestSchema.safeParse({ + algorithm: "token_bucket", + ip: "1.2.3.4", + scope: "custom" + }); + expect(r.success).toBe(false); + }); + + it("accepts custom scope with identifier", () => { + const r = limitRequestSchema.safeParse({ + algorithm: "token_bucket", + identifier: "d1", + scope: "custom" + }); + expect(r.success).toBe(true); + }); + + it("enforces max length on userId", () => { + const r = limitRequestSchema.safeParse({ + algorithm: "token_bucket", + userId: "x".repeat(257), + scope: "user" + }); + expect(r.success).toBe(false); + }); + + it("enforces max length on ip", () => { + const r = limitRequestSchema.safeParse({ + algorithm: "token_bucket", + ip: "x".repeat(65), + scope: "ip" + }); + expect(r.success).toBe(false); + }); + + it("enforces max length on identifier", () => { + const r = limitRequestSchema.safeParse({ + algorithm: "token_bucket", + identifier: "x".repeat(257), + scope: "custom" + }); + expect(r.success).toBe(false); + }); + + it("enforces cost range", () => { + expect( + limitRequestSchema.safeParse({ + algorithm: "token_bucket", + ip: "1.2.3.4", + cost: 0 + }).success + ).toBe(false); + expect( + limitRequestSchema.safeParse({ + algorithm: "token_bucket", + ip: "1.2.3.4", + cost: 11 + }).success + ).toBe(false); + }); + + it("defaults cost, route, and method", () => { + const r = limitRequestSchema.safeParse({ + algorithm: "token_bucket", + ip: "1.2.3.4" + }); + expect(r.success).toBe(true); + if (r.success) { + expect(r.data.cost).toBe(1); + expect(r.data.route).toBe("/"); + expect(r.data.method).toBe("GET"); + } + }); +}); diff --git a/tests/unit/time.test.ts b/tests/unit/time.test.ts new file mode 100644 index 0000000..683e82e --- /dev/null +++ b/tests/unit/time.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; + +import { bucketStart } from "../../src/utils/time"; + +describe("bucketStart", () => { + it("floors to the nearest minute by default", () => { + // 2026-01-01T00:01:30.500Z → 2026-01-01T00:01:00.000Z + const ts = new Date("2026-01-01T00:01:30.500Z").getTime(); + const result = bucketStart(ts); + expect(result).toBe(new Date("2026-01-01T00:01:00.000Z").getTime()); + }); + + it("returns the same value when already on a boundary", () => { + const ts = new Date("2026-01-01T00:02:00.000Z").getTime(); + expect(bucketStart(ts)).toBe(ts); + }); + + it("supports custom window sizes", () => { + const ts = new Date("2026-01-01T00:00:15.000Z").getTime(); + const result = bucketStart(ts, 10_000); + expect(result).toBe(new Date("2026-01-01T00:00:10.000Z").getTime()); + }); + + it("handles zero timestamp", () => { + expect(bucketStart(0)).toBe(0); + }); + + it("handles 10-second windows correctly", () => { + const ts = new Date("2026-01-01T00:00:37.000Z").getTime(); + const result = bucketStart(ts, 10_000); + expect(result).toBe(new Date("2026-01-01T00:00:30.000Z").getTime()); + }); +}); From 1800ce7f4700755192d06c0019b741e209b235ce Mon Sep 17 00:00:00 2001 From: debjit450 Date: Wed, 29 Apr 2026 11:35:32 +0530 Subject: [PATCH 5/6] update readme --- README.md | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fa555ef..ef024b7 100644 --- a/README.md +++ b/README.md @@ -211,18 +211,37 @@ If you need non-default settings, use the variables documented in [.env.example] ## Authentication -When `API_KEY` is set in the environment, ARCE requires all protected endpoints (`/check-limit`, `/consume`, `/api/dashboard-data`) to include a matching `x-api-key` header. +ARCE uses a shared API key for protecting its endpoints. The operator generates their own key and configures it via the `API_KEY` environment variable — there is no built-in key issuance system. -When `API_KEY` is not set, authentication is disabled so local development remains frictionless. +**Setup:** + +1. Generate a key using any method you prefer: + +```bash +# Example: generate a random 32-byte hex key +node -e "console.log(require('crypto').randomBytes(32).toString('hex'))" +``` + +2. Set it in your `.env` file: + +```env +API_KEY=your-generated-key-here +``` + +3. Pass the same key in the `x-api-key` header on every request to a protected endpoint: ```bash curl -X POST http://localhost:4000/consume \ -H "Content-Type: application/json" \ - -H "x-api-key: your-secret-key" \ + -H "x-api-key: your-generated-key-here" \ -d '{"algorithm":"token_bucket","route":"/api/orders","method":"GET","ip":"203.0.113.8","scope":"ip"}' ``` -Public endpoints (`/`, `/health`, `/dashboard`, `/static/*`) do not require authentication. +When `API_KEY` is not set, authentication is disabled so local development remains frictionless. + +Protected endpoints: `/check-limit`, `/consume`, `/api/dashboard-data` + +Public endpoints (no key required): `/`, `/health`, `/dashboard`, `/static/*` ## API Example From a2267c31bdffa3091bd2d3877002e9eede176dab Mon Sep 17 00:00:00 2001 From: debjit450 Date: Wed, 29 Apr 2026 13:15:20 +0530 Subject: [PATCH 6/6] chore(audit): harden security, optimize performance, and refactor architecture Security Enhancements: - Enforce fail-close authentication by requiring an API_KEY in production, preventing unintended open access. - Defeat potential timing attacks by replacing standard string inequality with `crypto.timingSafeEqual` in the auth middleware. - Protect the ARCE API itself from DoS by implementing a global `express-rate-limit`. - Standardise baseline security by integrating `helmet` and `cors` middleware. Performance & Scalability: - Mitigate a severe Redis memory exhaustion DoS vector in BehaviorStore by migrating route cardinality tracking to HyperLogLog (PFADD/PFCOUNT), shifting from unbounded O(N) memory consumption to O(1). - Optimize the SLIDING_WINDOW_LUA script by eliminating an O(N) loop during insertion, now storing the combined `requestId:cost` payload to aggregate costs during evaluation. - Fix a zombie container bug by forcing `process.exit(1)` when Redis exceeds its maximum reconnect attempts, allowing container orchestrators to correctly restart the process. Architecture & Maintainability: - Extract all magic number thresholds from the abuse detection heuristics, centralizing them in `runtimeConfig` to allow dynamic tuning via environment variables. - Introduce structured JSON logging via `pino-http` to improve production observability and debugging. --- CONTRIBUTING.md | 3 + README.md | 94 ++++++++++---- apps/dashboard/README.md | 3 + configs/runtime.ts | 13 +- docs/design-decisions.md | 2 + package-lock.json | 209 ++++++++++++++++++++++++++++++++ package.json | 6 + src/api/app.ts | 29 +++-- src/api/middleware/auth.ts | 22 +++- src/core/abuse-detector.ts | 25 ++-- src/store/behavior-store.ts | 50 +++----- src/store/rate-limiter-store.ts | 16 ++- src/store/redis.ts | 5 +- tests/load/README.md | 8 ++ 14 files changed, 407 insertions(+), 78 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dd8da6e..4b5ac1d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,6 +23,8 @@ docker compose up npm run dev ``` +> **Note on Authentication:** If you set `API_KEY` in your `.env` file, you must pass `x-api-key` in your request headers when testing locally. Leaving `API_KEY` empty disables authentication for easier development. + 5. Optional smoke check: ```bash @@ -36,6 +38,7 @@ npm run build npm run typecheck npm run lint npm run test +npm run format npm run format:check ``` diff --git a/README.md b/README.md index ef024b7..1a32178 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,8 @@ No formal benchmark numbers are published yet. The current focus is correctness, ```bash npm install -docker compose up +cp .env.example .env # adjust values if needed +docker compose up -d # starts Redis in the background npm run dev ``` @@ -207,7 +208,7 @@ Dashboard: http://localhost:4000/dashboard ``` -If you need non-default settings, use the variables documented in [.env.example](./.env.example). +All configuration is done via environment variables. Copy [.env.example](./.env.example) to `.env` and adjust values as needed. ## Authentication @@ -243,43 +244,80 @@ Protected endpoints: `/check-limit`, `/consume`, `/api/dashboard-data` Public endpoints (no key required): `/`, `/health`, `/dashboard`, `/static/*` -## API Example +## API Reference -Check a limit without consuming: +ARCE exposes two main endpoints: + +- **`POST /check-limit`** — evaluates whether the request would be allowed, **without** consuming from the limiter. Use this for read-ahead checks. +- **`POST /consume`** — evaluates and **consumes** from the limiter. Use this for actual enforcement. + +Both endpoints accept the same request body and return the same response shape. + +### Request Fields + +| Field | Required | Default | Description | +| ------------------ | -------- | ------- | -------------------------------------------------------------------------------------------- | +| `algorithm` | ✅ | — | One of `token_bucket`, `sliding_window`, `leaky_bucket` | +| `route` | — | `"/"` | The API route being accessed | +| `method` | — | `"GET"` | HTTP method | +| `userId` | \* | — | User identifier (max 256 chars) | +| `ip` | \* | — | Client IP address (max 64 chars) | +| `identifier` | \* | — | Custom subject key for `custom` scope (max 256 chars) | +| `scope` | — | auto | How to derive the subject: `user`, `ip`, `hybrid` (user+ip), or `custom` (uses `identifier`) | +| `cost` | — | `1` | How many tokens/units this request consumes (1–10) | +| `fingerprint` | — | auto | A string identifying this exact request shape, used for duplicate detection (max 256 chars) | +| `baseLimitPerMinute`| — | `100` | Override the default rate limit for this request (10–10,000) | +| `metadata.userAgent`| — | — | The client's user-agent string, used for missing-UA abuse detection | + +\* At least one of `userId`, `ip`, or `identifier` must be provided. + +### Example: Check a limit ```bash curl -X POST http://localhost:4000/check-limit \ -H "Content-Type: application/json" \ - -d "{\"algorithm\":\"token_bucket\",\"route\":\"/api/orders\",\"method\":\"GET\",\"userId\":\"user-42\",\"ip\":\"203.0.113.8\",\"scope\":\"hybrid\"}" + -H "x-api-key: $ARCE_API_KEY" \ + -d '{"algorithm":"token_bucket","route":"/api/orders","method":"GET","userId":"user-42","ip":"203.0.113.8","scope":"hybrid"}' ``` -Consume from the limiter: +### Example: Consume from the limiter ```bash curl -X POST http://localhost:4000/consume \ -H "Content-Type: application/json" \ - -d "{\"algorithm\":\"sliding_window\",\"route\":\"/api/search\",\"method\":\"GET\",\"ip\":\"203.0.113.8\",\"scope\":\"ip\"}" + -H "x-api-key: $ARCE_API_KEY" \ + -d '{"algorithm":"sliding_window","route":"/api/search","method":"GET","ip":"203.0.113.8","scope":"ip"}' ``` -Example request shape: +### Example Response ```json { - "algorithm": "token_bucket", - "route": "/api/orders", - "method": "GET", - "userId": "user-42", - "ip": "203.0.113.8", - "scope": "hybrid", + "mode": "consume", + "allowed": true, + "blocked": false, + "subject": "ip:203.0.113.8", + "fingerprint": "GET:/api/search", + "algorithm": "sliding_window", "cost": 1, - "fingerprint": "GET:/api/orders?status=open", - "baseLimitPerMinute": 100, - "metadata": { - "userAgent": "curl/8.6.0" - } + "anomalies": [], + "effectivePolicy": { + "tier": "normal", + "effectiveLimitPerMinute": 100, + "riskScore": 0 + }, + "decision": { + "allowed": true, + "remaining": 99, + "retryAfterMs": 0, + "resetAfterMs": 60000 + }, + "evaluatedAt": "2026-04-29T05:30:00.000Z" } ``` +When `allowed` is `false`, the HTTP status is `429`. The `decision.retryAfterMs` field tells the client how long to wait before retrying. + ## SDK Example ```ts @@ -312,8 +350,15 @@ import { ArceClient } from "./src/sdk/client"; import { createArceMiddleware } from "./src/sdk/express-middleware"; const app = express(); -const client = new ArceClient({ baseUrl: "http://localhost:4000" }); +const client = new ArceClient({ + baseUrl: "http://localhost:4000", + headers: { "x-api-key": process.env.ARCE_API_KEY ?? "" } +}); +// All routes below this middleware are rate-limited. +// The middleware calls ARCE's /consume endpoint for each request. +// If the request is denied, the middleware returns 429 with +// x-rate-limit-remaining and x-rate-limit-reset-ms headers. app.use( createArceMiddleware({ client, @@ -326,10 +371,11 @@ app.use( ## Production Integration Notes -- Put ARCE behind your API tier or call it from application middleware. -- Use a stable subject key strategy: user, IP, or hybrid depending on your threat model. -- Keep route fingerprinting intentional. High-cardinality fingerprints reduce the value of duplicate detection. -- Tune baseline limits and block durations per environment. +- **Deployment**: Put ARCE behind your API tier or call it from application middleware. ARCE itself is not designed to be internet-facing — it should be accessible only from your internal network or application pods. +- **Subject strategy**: Use a stable subject key strategy (`user`, `ip`, or `hybrid`) depending on your threat model. `hybrid` (user+IP) is strictest — it tracks the same user from different IPs separately. +- **Fingerprinting**: Keep route fingerprinting intentional. A fingerprint like `GET:/api/orders?status=open` groups similar requests; `GET:/api/orders?status=open&page=3&t=1719000000` would create unique fingerprints for every request, defeating duplicate detection. +- **Tuning**: Start with the defaults (`100 req/min` baseline, `20 req/min` suspicious, `300s` block). Observe the dashboard for false positives before tightening thresholds. +- **Redis**: Use a dedicated Redis instance or a dedicated database number (`redis://localhost:6379/1`) to isolate ARCE state from your application data. The `SERVICE_NAME` prefix prevents key collisions if sharing a cluster. ## Configuration diff --git a/apps/dashboard/README.md b/apps/dashboard/README.md index c4f4d1d..e9bb1b6 100644 --- a/apps/dashboard/README.md +++ b/apps/dashboard/README.md @@ -3,3 +3,6 @@ This directory contains static assets for the ARCE operator dashboard. The dashboard is intentionally lightweight. It visualizes request volume, rate-limited traffic, active blocks, and recent anomaly events without introducing a separate frontend build system. + +**How to view it:** +You do not need to run a separate web server for the dashboard. The main ARCE Express server automatically serves these static assets at the `/dashboard` endpoint (e.g., `http://localhost:4000/dashboard`). diff --git a/configs/runtime.ts b/configs/runtime.ts index 790c607..6d01207 100644 --- a/configs/runtime.ts +++ b/configs/runtime.ts @@ -14,11 +14,22 @@ function readNumber(name: string, fallback: number): number { } export const runtimeConfig = { + isProduction: process.env.NODE_ENV === "production", port: readNumber("PORT", 4000), redisUrl: process.env.REDIS_URL ?? "redis://localhost:6379", serviceName: process.env.SERVICE_NAME ?? "arce", apiKey: process.env.API_KEY ?? "", defaultLimitPerMinute: readNumber("DEFAULT_LIMIT_PER_MINUTE", 100), suspiciousLimitPerMinute: readNumber("SUSPICIOUS_LIMIT_PER_MINUTE", 20), - blockDurationSeconds: readNumber("BLOCK_DURATION_SECONDS", 300) + blockDurationSeconds: readNumber("BLOCK_DURATION_SECONDS", 300), + abuse: { + burstSpikeCount: readNumber("ABUSE_BURST_SPIKE_COUNT", 15), + burstSpikeRatio: readNumber("ABUSE_BURST_SPIKE_RATIO", 3), + duplicateFingerprintCount: readNumber("ABUSE_DUPLICATE_FINGERPRINT_COUNT", 8), + duplicateFingerprintRatio: readNumber("ABUSE_DUPLICATE_FINGERPRINT_RATIO", 0.7), + wideRouteScanCount: readNumber("ABUSE_WIDE_ROUTE_SCAN_COUNT", 12), + wideRouteScanMinuteCount: readNumber("ABUSE_WIDE_ROUTE_SCAN_MINUTE_COUNT", 30), + repeatedDenialsCount: readNumber("ABUSE_REPEATED_DENIALS_COUNT", 5), + missingUserAgentCount: readNumber("ABUSE_MISSING_USER_AGENT_COUNT", 10), + } }; diff --git a/docs/design-decisions.md b/docs/design-decisions.md index 4f6ba5e..a9e584b 100644 --- a/docs/design-decisions.md +++ b/docs/design-decisions.md @@ -22,6 +22,8 @@ Keeping them separate makes each path easier to reason about and change. ARCE is designed to be inspectable and operationally credible. Burst ratios, duplicate request patterns, route fan-out, and repeated denials are understandable signals that an engineer can tune without a training pipeline. +> **How to tune:** The thresholds for these heuristics (e.g., "15 requests in 10s with a 3x multiplier") are currently hardcoded in `src/core/abuse-detector.ts`. If you are deploying ARCE to production, you should review and modify these thresholds in that file to match your application's normal traffic patterns. + ## Why A Minimal Dashboard The dashboard is intentionally thin. Its job is to make the limiter observable during development and demos, not to become a full observability platform. diff --git a/package-lock.json b/package-lock.json index bc7586f..9d67026 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,13 +9,19 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "cors": "^2.8.6", "dotenv": "^17.0.0", "express": "^4.21.2", + "express-rate-limit": "^8.4.1", + "helmet": "^8.1.0", + "pino": "^10.3.1", + "pino-http": "^11.0.0", "redis": "^5.1.1", "zod": "^3.24.2" }, "devDependencies": { "@eslint/js": "^9.27.0", + "@types/cors": "^2.8.19", "@types/express": "^4.17.21", "@types/node": "^24.0.1", "@types/supertest": "^6.0.3", @@ -729,6 +735,11 @@ "@noble/hashes": "^1.1.5" } }, + "node_modules/@pinojs/redact": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", + "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==" + }, "node_modules/@redis/bloom": { "version": "5.12.1", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-5.12.1.tgz", @@ -1156,6 +1167,15 @@ "integrity": "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==", "dev": true }, + "node_modules/@types/cors": { + "version": "2.8.19", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.19.tgz", + "integrity": "sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/deep-eql": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", @@ -1852,6 +1872,14 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", "dev": true }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -2090,6 +2118,22 @@ "integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==", "dev": true }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -2561,6 +2605,23 @@ "url": "https://opencollective.com/express" } }, + "node_modules/express-rate-limit": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.4.1.tgz", + "integrity": "sha512-NGVYwQSAyEQgzxX1iCM978PP9AdO/hW93gMcF6ZwQCm+rFvLsBH6w4xcXWTcliS8La5EPRN3p9wzItqBwJrfNw==", + "dependencies": { + "ip-address": "10.1.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -2737,6 +2798,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-intrinsic": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", @@ -2865,6 +2934,14 @@ "node": ">= 0.4" } }, + "node_modules/helmet": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/helmet/-/helmet-8.1.0.tgz", + "integrity": "sha512-jOiHyAZsmnr8LqoPGmCjYAaiuWwjAPLgY8ZX2XrmHawt99/u1y6RgrZMTeoPfpUbV96HOalYgz1qzkRbw54Pmg==", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/http-errors": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", @@ -2934,6 +3011,14 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/ip-address": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", + "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -3174,6 +3259,14 @@ "node": ">= 0.6" } }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/object-inspect": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", @@ -3185,6 +3278,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -3328,6 +3429,51 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pino": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", + "integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^4.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-http": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/pino-http/-/pino-http-11.0.0.tgz", + "integrity": "sha512-wqg5XIAGRRIWtTk8qPGxkbrfiwEWz1lgedVLvhLALudKXvg1/L2lTFgTGPJ4Z2e3qcRmxoFxDuSdMdMGNM6I1g==", + "dependencies": { + "get-caller-file": "^2.0.5", + "pino": "^10.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz", + "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==" + }, "node_modules/postcss": { "version": "8.5.12", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.12.tgz", @@ -3380,6 +3526,21 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/process-warning": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", + "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ] + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -3415,6 +3576,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -3437,6 +3603,14 @@ "node": ">= 0.8" } }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "engines": { + "node": ">= 12.13.0" + } + }, "node_modules/redis": { "version": "5.12.1", "resolved": "https://registry.npmjs.org/redis/-/redis-5.12.1.tgz", @@ -3533,6 +3707,14 @@ } ] }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -3692,6 +3874,14 @@ "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", "dev": true }, + "node_modules/sonic-boom": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", + "integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -3701,6 +3891,14 @@ "node": ">=0.10.0" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/stackback": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", @@ -3835,6 +4033,17 @@ "node": ">=8" } }, + "node_modules/thread-stream": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.0.0.tgz", + "integrity": "sha512-4iMVL6HAINXWf1ZKZjIPcz5wYaOdPhtO8ATvZ+Xqp3BTdaqtAwQkNmKORqcIo5YkQqGXq5cwfswDwMqqQNrpJA==", + "dependencies": { + "real-require": "^0.2.0" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", diff --git a/package.json b/package.json index c9f55af..1c7d5f3 100644 --- a/package.json +++ b/package.json @@ -33,13 +33,19 @@ "node": ">=20" }, "dependencies": { + "cors": "^2.8.6", "dotenv": "^17.0.0", "express": "^4.21.2", + "express-rate-limit": "^8.4.1", + "helmet": "^8.1.0", + "pino": "^10.3.1", + "pino-http": "^11.0.0", "redis": "^5.1.1", "zod": "^3.24.2" }, "devDependencies": { "@eslint/js": "^9.27.0", + "@types/cors": "^2.8.19", "@types/express": "^4.17.21", "@types/node": "^24.0.1", "@types/supertest": "^6.0.3", diff --git a/src/api/app.ts b/src/api/app.ts index 5c05797..0378953 100644 --- a/src/api/app.ts +++ b/src/api/app.ts @@ -5,6 +5,10 @@ import express, { type Response } from "express"; import path from "node:path"; +import helmet from "helmet"; +import cors from "cors"; +import rateLimit from "express-rate-limit"; +import pinoHttp from "pino-http"; import { ZodError } from "zod"; @@ -32,14 +36,21 @@ export function createServerApp(args: CreateServerAppArgs): Express { app.disable("x-powered-by"); - // Security headers - app.use((_request, response, next) => { - response.setHeader("X-Content-Type-Options", "nosniff"); - response.setHeader("X-Frame-Options", "DENY"); - response.setHeader("X-XSS-Protection", "1; mode=block"); - response.setHeader("Referrer-Policy", "strict-origin-when-cross-origin"); - next(); + // Security and Logging middleware + app.use(helmet()); + app.use(cors()); + app.use(pinoHttp({ + autoLogging: false, + quietReqLogger: true + })); + + // Global rate limiter for the ARCE endpoints themselves + const globalLimiter = rateLimit({ + windowMs: 15 * 60 * 1000, // 15 minutes + limit: 10000, // 10k requests per 15 mins per IP + message: { error: "Too many requests to ARCE itself. Please slow down." } }); + app.use(globalLimiter); app.use(express.json({ limit: "1mb" })); app.use("/static", express.static(staticDir)); @@ -71,7 +82,7 @@ export function createServerApp(args: CreateServerAppArgs): Express { app.use( ( error: unknown, - _request: Request, + request: Request, response: Response, _next: NextFunction ) => { @@ -87,7 +98,7 @@ export function createServerApp(args: CreateServerAppArgs): Express { } if (error instanceof Error) { - console.error("Unhandled error:", error); + request.log.error(error, "Unhandled error during request processing"); } response.status(500).json({ diff --git a/src/api/middleware/auth.ts b/src/api/middleware/auth.ts index 86acb3e..23830cc 100644 --- a/src/api/middleware/auth.ts +++ b/src/api/middleware/auth.ts @@ -1,3 +1,4 @@ +import crypto from "node:crypto"; import type { RequestHandler } from "express"; import { runtimeConfig } from "../../../configs/runtime"; @@ -14,13 +15,32 @@ export function createAuthMiddleware(): RequestHandler { const configuredKey = runtimeConfig.apiKey; if (!configuredKey) { + if (runtimeConfig.isProduction) { + response.status(500).json({ + error: "Internal Server Error: Missing API_KEY in production." + }); + return; + } next(); return; } const provided = request.header("x-api-key"); - if (!provided || provided !== configuredKey) { + if (!provided) { + response.status(401).json({ + error: "Unauthorized. Provide a valid x-api-key header." + }); + return; + } + + const providedBuffer = Buffer.from(provided); + const configuredBuffer = Buffer.from(configuredKey); + + if ( + providedBuffer.length !== configuredBuffer.length || + !crypto.timingSafeEqual(providedBuffer, configuredBuffer) + ) { response.status(401).json({ error: "Unauthorized. Provide a valid x-api-key header." }); diff --git a/src/core/abuse-detector.ts b/src/core/abuse-detector.ts index 2edb3f6..9c93bba 100644 --- a/src/core/abuse-detector.ts +++ b/src/core/abuse-detector.ts @@ -1,3 +1,4 @@ +import { runtimeConfig } from "../../configs/runtime"; import type { AbuseAssessment, AnomalyFlag, BehaviorSnapshot } from "../types"; function push( @@ -20,7 +21,10 @@ export function analyzeBehavior(snapshot: BehaviorSnapshot): AbuseAssessment { ? snapshot.recentTenSecondCount / snapshot.averageTenSecondCount : snapshot.recentTenSecondCount; - if (snapshot.recentTenSecondCount >= 15 && burstRatio >= 3) { + if ( + snapshot.recentTenSecondCount >= runtimeConfig.abuse.burstSpikeCount && + burstRatio >= runtimeConfig.abuse.burstSpikeRatio + ) { riskScore += 35; push( anomalies, @@ -35,8 +39,10 @@ export function analyzeBehavior(snapshot: BehaviorSnapshot): AbuseAssessment { } if ( - snapshot.duplicateFingerprintCount >= 8 && - snapshot.duplicateFingerprintRatio >= 0.7 + snapshot.duplicateFingerprintCount >= + runtimeConfig.abuse.duplicateFingerprintCount && + snapshot.duplicateFingerprintRatio >= + runtimeConfig.abuse.duplicateFingerprintRatio ) { riskScore += 30; push( @@ -51,7 +57,10 @@ export function analyzeBehavior(snapshot: BehaviorSnapshot): AbuseAssessment { ); } - if (snapshot.uniqueRouteCount >= 12 && snapshot.trailingMinuteCount >= 30) { + if ( + snapshot.uniqueRouteCount >= runtimeConfig.abuse.wideRouteScanCount && + snapshot.trailingMinuteCount >= runtimeConfig.abuse.wideRouteScanMinuteCount + ) { riskScore += 25; push( anomalies, @@ -65,7 +74,9 @@ export function analyzeBehavior(snapshot: BehaviorSnapshot): AbuseAssessment { ); } - if (snapshot.deniedLastFiveMinutes >= 5) { + if ( + snapshot.deniedLastFiveMinutes >= runtimeConfig.abuse.repeatedDenialsCount + ) { riskScore += 15; push( anomalies, @@ -80,8 +91,8 @@ export function analyzeBehavior(snapshot: BehaviorSnapshot): AbuseAssessment { } if ( - snapshot.missingUserAgentCount >= 10 && - snapshot.recentTenSecondCount >= 10 + snapshot.missingUserAgentCount >= runtimeConfig.abuse.missingUserAgentCount && + snapshot.recentTenSecondCount >= runtimeConfig.abuse.missingUserAgentCount ) { riskScore += 10; push( diff --git a/src/store/behavior-store.ts b/src/store/behavior-store.ts index cf37e55..28c5bbd 100644 --- a/src/store/behavior-store.ts +++ b/src/store/behavior-store.ts @@ -57,8 +57,6 @@ export class BehaviorStore { private keys(subject: string) { return { requestTenSecond: `${this.prefix}:behavior:req10s:${subject}`, - fingerprints: `${this.prefix}:behavior:fingerprint:${subject}`, - routes: `${this.prefix}:behavior:route:${subject}`, denials: `${this.prefix}:behavior:denials:${subject}`, missingUserAgent: `${this.prefix}:behavior:missing-ua:${subject}`, block: `${this.prefix}:block:${subject}`, @@ -80,13 +78,20 @@ export class BehaviorStore { const fingerprintHash = hashValue(event.fingerprint); const routeHash = hashValue(event.route); + const fingerprintKey = `${this.prefix}:behavior:fp:${subject}:${fingerprintHash}`; + const routeKey = `${this.prefix}:behavior:routes:${subject}:${minuteBucket}`; + const write = this.client.multi(); write.hIncrBy(keys.requestTenSecond, tenSecondBucket, 1); write.pExpire(keys.requestTenSecond, TWENTY_MINUTES_MS); - write.hIncrBy(keys.fingerprints, `${minuteBucket}:${fingerprintHash}`, 1); - write.pExpire(keys.fingerprints, TEN_MINUTES_MS); - write.hIncrBy(keys.routes, `${minuteBucket}:${routeHash}`, 1); - write.pExpire(keys.routes, TEN_MINUTES_MS); + + // Store fingerprint count for this specific hash + write.hIncrBy(fingerprintKey, minuteBucket, 1); + write.pExpire(fingerprintKey, TEN_MINUTES_MS); + + // Track unique routes via HyperLogLog + write.pfAdd(routeKey, routeHash); + write.pExpire(routeKey, TEN_MINUTES_MS); if (!event.userAgent) { write.hIncrBy(keys.missingUserAgent, minuteBucket, 1); @@ -95,18 +100,22 @@ export class BehaviorStore { await write.exec(); + // We only need the current and previous minute buckets for fingerprints + const prevMinuteBucket = (Number(minuteBucket) - 60_000).toString(); + const routePrevKey = `${this.prefix}:behavior:routes:${subject}:${prevMinuteBucket}`; + const [ requestBuckets, - fingerprintBuckets, - routeBuckets, + fingerprintVals, + uniqueRouteCount, denialBuckets, missingUserAgentBuckets, blockReason, blockTtlMs ] = await Promise.all([ this.client.hGetAll(keys.requestTenSecond), - this.client.hGetAll(keys.fingerprints), - this.client.hGetAll(keys.routes), + this.client.hmGet(fingerprintKey, [minuteBucket, prevMinuteBucket]), + this.client.pfCount([routeKey, routePrevKey]), this.client.hGetAll(keys.denials), this.client.hGetAll(keys.missingUserAgent), this.client.get(keys.block), @@ -125,31 +134,12 @@ export class BehaviorStore { requestBuckets, event.timestampMs ); - const duplicateFingerprintCount = Object.entries(fingerprintBuckets).reduce( - (total, [field, value]) => { - const [bucket, storedHash] = field.split(":"); - if ( - storedHash === fingerprintHash && - Number(bucket) >= event.timestampMs - 60_000 - ) { - return total + Number(value); - } + const duplicateFingerprintCount = (Number(fingerprintVals[0]) || 0) + (Number(fingerprintVals[1]) || 0); - return total; - }, - 0 - ); const duplicateFingerprintRatio = trailingMinuteCount > 0 ? duplicateFingerprintCount / trailingMinuteCount : 0; - const uniqueRouteCount = new Set( - Object.keys(routeBuckets) - .filter( - (field) => Number(field.split(":")[0]) >= event.timestampMs - 60_000 - ) - .map((field) => field.split(":")[1]) - ).size; const deniedLastFiveMinutes = sumBuckets( denialBuckets, event.timestampMs - FIVE_MINUTES_MS diff --git a/src/store/rate-limiter-store.ts b/src/store/rate-limiter-store.ts index 33d61bb..25a1fd7 100644 --- a/src/store/rate-limiter-store.ts +++ b/src/store/rate-limiter-store.ts @@ -113,16 +113,23 @@ local request_id = ARGV[7] redis.call("ZREMRANGEBYSCORE", key, "-inf", now - window_ms) -local current = redis.call("ZCARD", key) +local elements = redis.call("ZRANGE", key, 0, -1) +local current = 0 +for _, elem in ipairs(elements) do + -- elem format is "requestId:cost" + local elem_cost = string.match(elem, ":(%d+)$") + if elem_cost then + current = current + tonumber(elem_cost) + end +end + local allowed = 0 local retry_after_ms = 0 if (current + cost) <= limit then allowed = 1 if consume == 1 then - for i = 1, cost do - redis.call("ZADD", key, now, request_id .. ":" .. i) - end + redis.call("ZADD", key, now, request_id .. ":" .. tostring(cost)) current = current + cost end else @@ -132,6 +139,7 @@ else end end + redis.call("PEXPIRE", key, ttl_ms) local remaining = math.max(0, limit - current) diff --git a/src/store/redis.ts b/src/store/redis.ts index 5fa1308..1e6139f 100644 --- a/src/store/redis.ts +++ b/src/store/redis.ts @@ -11,9 +11,10 @@ export async function createRedisConnection(): Promise { reconnectStrategy(retries: number) { if (retries > 10) { console.error( - `Redis reconnection failed after ${retries} attempts. Giving up.` + `Redis reconnection failed after ${retries} attempts. Giving up and crashing process.` ); - return new Error("Redis reconnection limit reached."); + // Exit process so container orchestrator (e.g. Kubernetes/Docker) restarts the service. + process.exit(1); } const delay = Math.min(retries * 200, 5_000); diff --git a/tests/load/README.md b/tests/load/README.md index 8573c6d..5a1a3fd 100644 --- a/tests/load/README.md +++ b/tests/load/README.md @@ -2,10 +2,18 @@ `basic-smoke.js` is a minimal k6 script intended as a starting point for repeatable load validation. +> **Note:** Running this script requires [k6](https://k6.io/docs/get-started/installation/) to be installed on your system. + Example: ```bash k6 run tests/load/basic-smoke.js ``` +If you don't have k6 installed and just want a quick connectivity check, use the zero-dependency smoke test instead: + +```bash +npm run smoke +``` + The script is intentionally lightweight. It exercises the `/consume` path and leaves deeper workload modeling to future contributors.