From 970062ebd664ddb1611a2513bd2828dd2d77867a Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sat, 25 Jul 2026 15:55:44 +0100 Subject: [PATCH] feat(metrics): fixed-cardinality active_users gauge + restored dashboard panel Brings back "Active users (last 1 h)" without the per-user series that blew the Grafana free tier: the middleware stamps user-id -> last-seen in memory and _ActiveUsersCollector emits active_users{window="1h"|"24h"} at scrape time - 2 series total regardless of user count, and no user identifiers ever become metric labels. Trade-offs vs the removed user_page_views_total: count resets on deploy (in-memory) and there is no per-user breakdown. Co-Authored-By: Claude Fable 5 --- backend/server/app.py | 8 ++++++ backend/server/metrics.py | 42 ++++++++++++++++++++++++++++ ops/grafana/dashboards/frontend.json | 15 ++++++++++ tests/server/test_metrics.py | 34 ++++++++++++++++++++++ 4 files changed, 99 insertions(+) diff --git a/backend/server/app.py b/backend/server/app.py index 9e1cd7fe..eb6f46f1 100644 --- a/backend/server/app.py +++ b/backend/server/app.py @@ -99,6 +99,7 @@ async def get_response(self, path: str, scope): # type: ignore[override] _register_db_collector, check_metrics_auth, normalize_http_labels, + record_user_seen, should_track_path, ) from backend.server.server_context import ServerContextMiddleware @@ -263,6 +264,13 @@ async def dispatch(self, request: Request, call_next): # type: ignore[override] if response.status_code >= 500: APP_ERRORS.labels(source=label_path).inc() + # Feed the fixed-cardinality active_users gauge — the user id + # stays in app memory, never becomes a metric label. + if request.method == "GET" and response.status_code < 400: + user = request.session.get("user") + if user: + record_user_seen(str(user.get("id") or user.get("username") or "?")) + return response diff --git a/backend/server/metrics.py b/backend/server/metrics.py index 24341386..e345174c 100644 --- a/backend/server/metrics.py +++ b/backend/server/metrics.py @@ -16,6 +16,7 @@ import os import re import sqlite3 +import time from prometheus_client import ( REGISTRY, @@ -59,6 +60,46 @@ buckets=(0.01, 0.05, 0.1, 0.25, 1.0, 2.5), ) +# ── Active users ────────────────────────────────────────────────────────────── +# Bounded replacement for the removed user_page_views_total (which cost +# username × path series — 12.9k at peak). The distinct-user count is computed +# app-side from in-memory last-seen timestamps and exported as ONE series per +# window by _ActiveUsersCollector, so cardinality is fixed regardless of how +# many users exist. Resets on deploy (in-memory), like every other gauge here. + +_ACTIVE_WINDOWS: dict[str, float] = {"1h": 3600.0, "24h": 86400.0} +_user_last_seen: dict[str, float] = {} + + +def record_user_seen(user_id: str) -> None: + """Stamp an authenticated user as active now (called from the metrics + middleware). Dict ops are atomic under the GIL; scrape-side reads snapshot.""" + now = time.time() + _user_last_seen[user_id] = now + # Prune only if the dict somehow grows far past the real user count so a + # long-lived process can't accumulate unboundedly. + if len(_user_last_seen) > 2048: + cutoff = now - max(_ACTIVE_WINDOWS.values()) + for key in [k for k, ts in _user_last_seen.items() if ts < cutoff]: + _user_last_seen.pop(key, None) + + +class _ActiveUsersCollector(Collector): + """Emit active_users{window=} at scrape time from the last-seen map.""" + + def collect(self): # type: ignore[override] + g = GaugeMetricFamily( + "active_users", + "Distinct authenticated users seen within the trailing window", + labels=["window"], + ) + now = time.time() + stamps = list(_user_last_seen.values()) + for label, span in _ACTIVE_WINDOWS.items(): + g.add_metric([label], float(sum(1 for ts in stamps if now - ts <= span))) + yield g + + # ── Cache metrics ───────────────────────────────────────────────────────────── # Labels: cache = character | guild | claim @@ -328,6 +369,7 @@ def _register_db_collector() -> None: REGISTRY.register(_DBCollector()) REGISTRY.register(_DBFileSizeCollector()) REGISTRY.register(_CensusHealthCollector()) + REGISTRY.register(_ActiveUsersCollector()) _db_collector_registered = True diff --git a/ops/grafana/dashboards/frontend.json b/ops/grafana/dashboards/frontend.json index c8ffdafb..2385d6b8 100644 --- a/ops/grafana/dashboards/frontend.json +++ b/ops/grafana/dashboards/frontend.json @@ -105,6 +105,21 @@ "legendFormat": "p95", "refId": "A" }] }, + { + "id": 4, "type": "stat", "title": "Active users (last 1 h)", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Distinct authenticated users seen in the last hour. Computed app-side (active_users gauge, one series per window) — never re-add per-user metric labels.", + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "colorMode": "value", "graphMode": "none", "textMode": "auto" + }, + "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] } }, "overrides": [] }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "active_users{job=\"eq2companion\",window=\"1h\"}", + "legendFormat": "users", "refId": "A" }] + }, + { "id": 5, "type": "timeseries", "title": "HTTP requests / sec by status code", "gridPos": { "h": 9, "w": 12, "x": 0, "y": 5 }, diff --git a/tests/server/test_metrics.py b/tests/server/test_metrics.py index 2c94d91d..b743a2e7 100644 --- a/tests/server/test_metrics.py +++ b/tests/server/test_metrics.py @@ -41,3 +41,37 @@ def test_should_track_path_skips_static_mounts(): for skipped in ("/assets/app.js", "/class-icons/13.png", "/spell-icons/1.png", "/metrics"): assert not metrics.should_track_path(skipped) assert metrics.should_track_path("/api/character/Foo") + + +def _active_counts() -> dict[str, float]: + (family,) = metrics._ActiveUsersCollector().collect() + return {sample.labels["window"]: sample.value for sample in family.samples} + + +def test_active_users_counts_per_window(monkeypatch): + """Distinct-user counts are computed app-side from last-seen stamps — + fixed 2-series cardinality regardless of user count (the whole point of + replacing user_page_views_total).""" + monkeypatch.setattr(metrics, "_user_last_seen", {}) + now = 1_800_000_000.0 + monkeypatch.setattr(metrics.time, "time", lambda: now) + + assert _active_counts() == {"1h": 0.0, "24h": 0.0} + + metrics.record_user_seen("111") + metrics.record_user_seen("222") + metrics.record_user_seen("111") # repeat visits stay one user + metrics._user_last_seen["333"] = now - 7200 # 2h ago: out of 1h, in 24h + metrics._user_last_seen["444"] = now - 100_000 # out of both windows + + assert _active_counts() == {"1h": 2.0, "24h": 3.0} + + +def test_record_user_seen_prunes_stale_entries(monkeypatch): + monkeypatch.setattr(metrics, "_user_last_seen", {}) + now = 1_800_000_000.0 + monkeypatch.setattr(metrics.time, "time", lambda: now) + for i in range(2049): + metrics._user_last_seen[str(i)] = now - 100_000 # all stale + metrics.record_user_seen("fresh") + assert metrics._user_last_seen == {"fresh": now}