From d302ca2499656f866600f0945d33e31165e975f1 Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sat, 25 Jul 2026 12:54:05 +0100 Subject: [PATCH 1/2] feat(stats): server Stats page, character Lifetime panel + live stat Explorer Server-wide statistics from the census character.stat aggregate family plus a live per-character stat explorer: - GET /api/stats/server: population, class distribution, lifetime totals, named leaderboards (7 boards; two-key hit records via range-filter + client-side sort), class averages w/ global comparison. Never builds inline - cold cache answers 202 {building} and kicks a lock-deduped background build; startup prewarm; failed build -> 60s cooldown -> 503. - GET /api/stats/character/{name}: lifetime panel on the character sheet (kills/deaths/K:D/biggest hits with ability names via spells.db crc lookup, crafts, rare harvests + class-average comparison line). - GET /api/stats/explore?stat=&cls=: live top-20 by whitelisted stat - combat snapshots (stats.combat.*/health/power), progression scalars (quests/collections/achievements/AA points spent, projected narrowly), lifetime statistics.*. Class scope via numeric type.classid (string class filters silently return empty). 15-min cache, stale-beats-error. - /stats page: Server Records + Explorer tabs, 202 polling, transient network failures retry instead of dead-ending. - CensusClient: stat aggregate/leader/range/statistics/worldid methods. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 1 + backend/census/client.py | 104 ++++ backend/server/api/stats.py | 530 ++++++++++++++++++ backend/server/app.py | 4 + backend/server/cache.py | 3 +- frontend/src/App.tsx | 3 + frontend/src/components/MobileNav.tsx | 1 + .../src/pages/CharacterPage.gearsets.test.tsx | 35 +- frontend/src/pages/CharacterPage.tsx | 42 ++ frontend/src/pages/StatsPage.test.tsx | 199 +++++++ frontend/src/pages/StatsPage.tsx | 370 ++++++++++++ tests/server/test_stats.py | 357 ++++++++++++ 12 files changed, 1647 insertions(+), 2 deletions(-) create mode 100644 backend/server/api/stats.py create mode 100644 frontend/src/pages/StatsPage.test.tsx create mode 100644 frontend/src/pages/StatsPage.tsx create mode 100644 tests/server/test_stats.py diff --git a/CLAUDE.md b/CLAUDE.md index eeec02a0..d67166d6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,6 +36,7 @@ A web companion site for EverQuest 2 (TLE), with a Discord bot for spot checks ( | `backend/server/api/characters.py` | GET /api/characters/search — character name search | | `backend/server/api/guild_officer.py` | Officer claim-review endpoints; imports _officer_chars, _roster_rank_map from guild.py | | `backend/server/api/item_watch.py` | Item watch endpoints; imports _officer_chars, _roster_rank_map from guild.py | +| `backend/server/api/stats.py` | Per-server Stats page from the census `character.stat` aggregate family (global/world/world×class rows, 12 lifetime stats as max/sum/avg, recomputed daily by census). `GET /api/stats/server` = population, class distribution, totals, named leaderboards (single-value stats census-sort; K/D sorts `.value` with a 1000-kill floor; the two-key hit records can't census-sort — range-filter near the aggregate max + client-side sort). `GET /api/stats/character/{name}` = lifetime panel w/ ability names via spells.db `find_by_crc` (crc 4294967295 = sentinel, skip). classid = classes.db `icon_id` (Templar=13). 6h SWR module cache per world; leader queries retry (census drops back-to-back sorted queries). Never builds inline: cold cache → 202 `{"status":"building"}` + background build (lock-deduped, frontend polls 5s); startup prewarm via `prewarm_server_stats()` in app.py lifespan; failed build → 60s cooldown → 503. `GET /api/stats/explore?stat=&cls=` = live top-20 by whitelisted stat (`_EXPLORE_STATS`: combat snapshots under `stats.combat.*`/health/power + progression scalars quests.complete / collections.complete / achievements.points/.completed / alternateadvancements.spentpoints, projected narrowly — never `c:show=achievements`, it drags the 500+-item list + lifetime `statistics.*`), optional class scope — census needs numeric `type.classid` (string `type.class` filters silently return empty); 15-min cache, stale-beats-error. | | `backend/server/census_health.py` | Site-wide Census availability signal: background poll every 5 min; `is_down()`/`get_state()` read by the read/refresh paths. | | `backend/server/census_events.py` | In-process async pub/sub backing the SSE stream (single-process only). | | `backend/server/census_refresh.py` | Background refresh orchestration (throttle 15 min / in-flight dedupe / skip-when-down); merges into census_store, updates cache, publishes SSE. `_merge_roster` best-known join. | diff --git a/backend/census/client.py b/backend/census/client.py index 81fe4533..1ddabbeb 100644 --- a/backend/census/client.py +++ b/backend/census/client.py @@ -452,6 +452,110 @@ async def get_gear_sets(self, character_id: str | int) -> list[GearSet] | None: sets.append(GearSet(name=name, equipment=equipment)) return sets + # ── Lifetime statistics (the character.stat aggregate family) ──────────── + + @staticmethod + def _list_of(data: dict) -> list: + """Census wraps rows in a '_list' key — grab it by suffix.""" + for key, value in data.items(): + if key.endswith("_list") and isinstance(value, list): + return value + return [] + + async def get_stat_aggregates(self) -> dict | None: + """The three character.stat aggregate collections, refreshed daily by + Census: per-class global rows (id=classid or 'all'), per-world rows + (id=worldid), and the world×class matrix (id='worldid.classid', + paginated at 100/request). Returns + ``{"global": [...], "world": [...], "world_class": [...]}`` or None + when any fetch fails (partial aggregates would mislead).""" + # The matrix is ~1095 rows paginated at 100/request — fire the whole + # page fan (plus global+world) concurrently instead of serially; the + # build drops from ~10s to one census round-trip. + page_starts = list(range(0, 1300, 100)) + results = await asyncio.gather( + self._census_get("character.stat.global/", {"c:limit": "40"}), + self._census_get("character.stat.world/", {"c:limit": "80"}), + *[self._census_get("character.stat/", {"c:limit": "100", "c:start": str(s)}) for s in page_starts], + ) + g, w, *pages = results + if g is None or w is None: + return None + world_class: list = [] + for p in pages: + if p is None: + return None + world_class.extend(self._list_of(p)) + return {"global": self._list_of(g), "world": self._list_of(w), "world_class": world_class} + + async def get_stat_leaders( + self, + world: str, + sort_path: str, + limit: int = 10, + extra_filter: dict[str, str] | None = None, + show: str = "statistics", + ) -> list[dict] | None: + """Characters on ``world`` ordered by a census field, name+class+ + value shape. ``sort_path`` is the census sort key (single-value stats + sort on 'statistics.kills'; ratio-style need '...value'). ``show`` + selects the payload subtree ('statistics' for lifetime stats, + 'stats' for combat snapshots). None on census failure.""" + params = { + "locationdata.world": world, + "c:sort": f"{sort_path}:-1", + "c:show": f"name.first,type.class,type.level,{show}", + "c:limit": str(limit), + **(extra_filter or {}), + } + data = await self._census_get("character/", params, timeout_s=20) + return self._list_of(data) if data is not None else None + + async def get_stat_range( + self, world: str, stat_value_path: str, minimum: float, limit: int = 200 + ) -> list[dict] | None: + """Characters on ``world`` whose statistic exceeds ``minimum`` — + unsorted (census can't sort two-key stat objects like max_melee_hit; + the caller sorts client-side). None on census failure.""" + params = { + "locationdata.world": world, + stat_value_path: f"]{minimum:.0f}", + "c:show": "name.first,type.class,type.level,statistics", + "c:limit": str(limit), + } + data = await self._census_get("character/", params, timeout_s=20) + return self._list_of(data) if data is not None else None + + async def get_character_statistics(self, name: str, world: str) -> dict | None: + """One character's lifetime ``statistics`` object (kills, deaths, + biggest hits w/ ability crcs, crafts, rare harvests). None when the + character is unknown or census fails.""" + params = { + "name.first": name, + "locationdata.world": world, + "c:show": "name.first,type.class,statistics", + "c:limit": "1", + } + data = await self._census_get("character/", params) + if data is None: + return None + rows = self._list_of(data) + return rows[0] if rows else None + + async def get_worldid(self, world: str) -> int | None: + """Resolve a world name to its numeric census id (the key the + character.stat aggregates use) by probing one character's + locationdata. None when the world has no census characters.""" + params = {"locationdata.world": world, "c:show": "locationdata.worldid", "c:limit": "1"} + data = await self._census_get("character/", params) + if data is None: + return None + rows = self._list_of(data) + try: + return int(rows[0]["locationdata"]["worldid"]) if rows else None + except (KeyError, TypeError, ValueError): + return None + async def get_character_aas(self, name: str, world: str) -> CharacterAAs | None: params = { "name.first": name, diff --git a/backend/server/api/stats.py b/backend/server/api/stats.py new file mode 100644 index 00000000..e914d555 --- /dev/null +++ b/backend/server/api/stats.py @@ -0,0 +1,530 @@ +"""Server statistics — the census character.stat aggregate family surfaced +as a per-server Stats page + per-character lifetime panel. + +Data shape (probed 2026-07-25): + - character.stat.global: one row per classid + 'all', game-wide. + - character.stat.world: one row per worldid. + - character.stat: one row per 'worldid.classid'. + Each row: 12 lifetime statistics as {max, sum, avg} + count + ts, + recomputed daily by census (~09:45 UTC). + +Named leaderboards come from sorted/filtered character queries: + - single-value stats sort server-side ('statistics.kills:-1'), + - K/D sorts on '...ratio.value' with a kills floor (pure ratio farming + from a handful of kills would otherwise own the board), + - the two hit records can't be census-sorted (two-key objects), so we + range-filter near the aggregate max and sort client-side. + +Everything is cached per world for STATS_TTL_S (census only recomputes +daily) with stale-while-revalidate semantics, so page views cost zero +census once warm. +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from typing import Any + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import JSONResponse +from pydantic import BaseModel + +from backend.core.log_safety import scrub +from backend.eq2db.classes import catalogue as classes_db +from backend.eq2db.spells import catalogue as spells_db +from backend.server.auth_deps import require_user_session as _require_user +from backend.server.cache import lifetime_cache +from backend.server.core.census_lifecycle import shared_census_client +from backend.server.core.validation import validate_character_name +from backend.server.limiter import limiter +from backend.server.server_context import current_world + +_log = logging.getLogger(__name__) + +router = APIRouter(tags=["stats"]) + +STATS_TTL_S = 6 * 3600 # census recomputes daily; refresh at most every 6 h +LEADER_LIMIT = 10 +LEADER_BUILD_DEADLINE_S = 60 # hard cap on the leaderboard build phase +KD_KILLS_FLOOR = 1000 # K/D board: require this many kills so ratio farming can't own it + +# The aggregate stat keys we surface (aggregate-only ones like +# quests.complete can't be resolved to named characters — census rejects +# per-character sorts on them — so they appear in totals/averages only). +_TOTAL_STATS = ("kills", "deaths", "quests.complete", "collections.complete", "items_crafted", "rare_harvests") +_AVG_STATS = ("kills", "deaths", "kills_deaths_ratio", "quests.complete", "achievements.points") + +# Leaderboards: (key, census sort path, extra filter) — None sort = range mode. +_LEADER_SORTS: list[tuple[str, str | None, dict[str, str] | None]] = [ + ("kills", "statistics.kills", None), + ("deaths", "statistics.deaths", None), + ("kills_deaths_ratio", "statistics.kills_deaths_ratio.value", {"statistics.kills.value": f"]{KD_KILLS_FLOOR}"}), + ("items_crafted", "statistics.items_crafted", None), + ("rare_harvests", "statistics.rare_harvests", None), + ("max_melee_hit", None, None), # range mode + ("max_magic_hit", None, None), # range mode +] + + +class StatAggregate(BaseModel): + max: float = 0 + avg: float = 0 + sum: float = 0 + + +class ClassStatsRow(BaseModel): + classid: int + name: str + count: int + avg: dict[str, float] = {} # stat key → world avg for this class + global_avg: dict[str, float] = {} # same stats, all-worlds average + + +class LeaderEntry(BaseModel): + name: str + cls: str | None = None + level: int | None = None + value: float + + +class ServerStatsResponse(BaseModel): + world: str + ts: int # census aggregate compute time + fetched_at: int + population: int + totals: dict[str, float] = {} + records: dict[str, float] = {} # anonymous aggregate maxes + averages: dict[str, float] = {} + global_averages: dict[str, float] = {} + classes: list[ClassStatsRow] = [] + leaders: dict[str, list[LeaderEntry]] = {} + + +class LifetimeStatResponse(BaseModel): + character_name: str + kills: int | None = None + deaths: int | None = None + kills_deaths_ratio: float | None = None + max_melee_hit: int | None = None + max_melee_ability: str | None = None + max_magic_hit: int | None = None + max_magic_ability: str | None = None + items_crafted: int | None = None + rare_harvests: int | None = None + # Context: this world's per-class averages for "vs average" chips. + class_avg_kills: float | None = None + class_avg_kd: float | None = None + + +def _classid_names() -> dict[int, str]: + """classes.db's icon_id column IS the census classid scheme (Templar=13, + Mystic=19 — verified against live character type.classid).""" + return {int(row["icon_id"]): row["name"] for row in classes_db.list_all() if row.get("icon_id") is not None} + + +def _agg(value: dict, stat: str) -> StatAggregate: + raw = value.get(stat) or {} + return StatAggregate(max=raw.get("max") or 0, avg=raw.get("avg") or 0, sum=raw.get("sum") or 0) + + +def _leader_entries(rows: list[dict], stat: str, limit: int = LEADER_LIMIT) -> list[LeaderEntry]: + out = [] + for ch in rows: + st = (ch.get("statistics") or {}).get(stat) or {} + value = st.get("value") + if value is None: + continue + out.append( + LeaderEntry( + name=(ch.get("name") or {}).get("first") or "?", + cls=(ch.get("type") or {}).get("class"), + level=(ch.get("type") or {}).get("level"), + value=float(value), + ) + ) + out.sort(key=lambda e: -e.value) + return out[:limit] + + +async def _build_server_stats(world: str) -> ServerStatsResponse | None: + """One full refresh: 3 aggregate collections + ~7 leader queries.""" + started = time.monotonic() + async with shared_census_client() as client: + worldid = await client.get_worldid(world) + if worldid is None: + return None + aggregates = await client.get_stat_aggregates() + if aggregates is None: + return None + + world_row = next((r["value"] for r in aggregates["world"] if str(r.get("id")) == str(worldid)), None) + if world_row is None: + return None + global_all = next((r["value"] for r in aggregates["global"] if r.get("id") == "all"), {}) + global_by_class = {str(r.get("id")): r["value"] for r in aggregates["global"] if r.get("id") != "all"} + world_by_class = { + str(r.get("id")).split(".", 1)[1]: r["value"] + for r in aggregates["world_class"] + if str(r.get("id")).startswith(f"{worldid}.") + } + + names = _classid_names() + classes = [] + for cid_str, value in world_by_class.items(): + try: + cid = int(cid_str) + except ValueError: + continue + classes.append( + ClassStatsRow( + classid=cid, + name=names.get(cid, f"Class {cid}"), + count=int(value.get("count") or 0), + avg={s: _agg(value, s).avg for s in _AVG_STATS}, + global_avg={s: _agg(global_by_class.get(cid_str, {}), s).avg for s in _AVG_STATS}, + ) + ) + classes.sort(key=lambda c: -c.count) + + # Leaderboards — sorted queries for single-value stats, range mode for + # the two-key hit records. Individual failures degrade to an absent + # board rather than failing the page. + # Leaderboard queries run sequentially (census drops back-to-back + # sorted queries when parallelised) but under a HARD deadline: a + # flaky census must never hang the page — boards that didn't make + # the cut are simply absent until the next 6-hourly refresh. + leaders: dict[str, list[LeaderEntry]] = {} + deadline = time.monotonic() + LEADER_BUILD_DEADLINE_S + for stat, sort_path, extra in _LEADER_SORTS: + if time.monotonic() > deadline: + _log.warning("[stats] leader build deadline hit on %s — %d boards done", world, len(leaders)) + break + try: + rows = None + for attempt in range(2): + if attempt: + await asyncio.sleep(1.5) + if sort_path is not None: + rows = await client.get_stat_leaders(world, sort_path, LEADER_LIMIT, extra) + else: + record = _agg(world_row, stat).max + if record <= 0: + break + # Adaptive floor: near the record first, widening until + # the board fills (worlds concentrate at the top). + for fraction in (0.5, 0.1, 0.01): + rows = await client.get_stat_range(world, f"statistics.{stat}.value", record * fraction) + if rows is not None and len(rows) >= LEADER_LIMIT: + break + if rows or time.monotonic() > deadline: + break + if not rows: + _log.warning("[stats] leader query failed for %s on %s", stat, world) + continue + entries = _leader_entries(rows, stat) + if entries: + leaders[stat] = entries + except Exception as exc: + _log.warning("[stats] leader board %s failed on %s: %s", stat, world, exc) + + _log.info( + "[stats] built %s stats in %.1fs (%d classes, %d leader boards)", + world, + time.monotonic() - started, + len(classes), + len(leaders), + ) + return ServerStatsResponse( + world=world, + ts=int(world_row.get("ts") or 0), + fetched_at=int(time.time()), + population=int(world_row.get("count") or 0), + totals={s: _agg(world_row, s).sum for s in _TOTAL_STATS}, + records={ + s: _agg(world_row, s).max + for s in ( + "kills", + "deaths", + "max_melee_hit", + "max_magic_hit", + "quests.complete", + "items_crafted", + "rare_harvests", + ) + }, + averages={s: _agg(world_row, s).avg for s in _AVG_STATS}, + global_averages={s: _agg(global_all, s).avg for s in _AVG_STATS}, + classes=classes, + leaders=leaders, + ) + + +# world → (fetched_at_monotonic, payload). Builds take ~1-2 min against a +# slow census (sorted leader queries), so a request NEVER builds inline: +# cold hits kick a background build and return 202; the page polls. +_stats_cache: dict[str, tuple[float, ServerStatsResponse]] = {} +_stats_locks: dict[str, asyncio.Lock] = {} +_last_build_failure: dict[str, float] = {} +_BUILD_FAILURE_COOLDOWN_S = 60 + + +def _lock_for(world: str) -> asyncio.Lock: + return _stats_locks.setdefault(world, asyncio.Lock()) + + +async def _refresh_server_stats(world: str) -> ServerStatsResponse | None: + async with _lock_for(world): + cached = _stats_cache.get(world) + if cached and time.monotonic() - cached[0] < STATS_TTL_S: + return cached[1] # someone else refreshed while we waited + built = await _build_server_stats(world) + if built is not None: + _stats_cache[world] = (time.monotonic(), built) + _last_build_failure.pop(world, None) + else: + _last_build_failure[world] = time.monotonic() + return built + + +async def prewarm_server_stats() -> None: + """Startup task: build every registered server's stats in the background + (sequentially — census throttles parallel sorted queries) so the first + page view after boot is a cache hit, not a minute-long build.""" + from backend.server.core.executor import run_sync + from backend.server.db.servers import store as servers_store + + try: + servers = await run_sync(servers_store.list_servers_sync) + except Exception as exc: + _log.warning("[stats] prewarm skipped — server registry unavailable: %s", exc) + return + for srv in servers: + try: + await _refresh_server_stats(srv["world"]) + except Exception as exc: + _log.warning("[stats] prewarm failed for %s: %s", srv.get("world"), exc) + + +@router.get("/stats/server", response_model=None) +@limiter.limit("30/minute") +async def get_server_stats(request: Request) -> ServerStatsResponse | JSONResponse: + """Warm cache → the payload (with SWR background refresh when stale). + Cold → kick a background build and 202; the frontend polls. A recently + failed build → 503 so the poll loop doesn't spin forever.""" + _require_user(request) + world = current_world() + + cached = _stats_cache.get(world) + if cached: + if time.monotonic() - cached[0] >= STATS_TTL_S: + asyncio.create_task(_refresh_server_stats(world)) + return cached[1] + + failed_at = _last_build_failure.get(world) + if failed_at is not None and time.monotonic() - failed_at < _BUILD_FAILURE_COOLDOWN_S: + raise HTTPException( + status_code=503, detail="Server statistics unavailable (Census unreachable). Try again shortly." + ) + + asyncio.create_task(_refresh_server_stats(world)) # lock-deduped + return JSONResponse(status_code=202, content={"status": "building"}) + + +# --------------------------------------------------------------------------- +# Explorer — "show me the highest s" +# --------------------------------------------------------------------------- +# Whitelisted stat catalogue: key → (census sort path, value extraction path, +# extra filter). Combat stats are point-in-time census snapshots +# (stats.combat.*); lifetime stats live under statistics.*. Class filtering +# uses type.classid (numeric — census silently ignores type.class strings). + +_EXPLORE_STATS: dict[str, tuple[str, tuple[str, ...], dict[str, str] | None]] = { + # Combat snapshot + "ability_mod": ("stats.combat.abilitymod", ("stats", "combat", "abilitymod"), None), + "potency": ("stats.combat.basemodifier", ("stats", "combat", "basemodifier"), None), + "crit_bonus": ("stats.combat.critbonus", ("stats", "combat", "critbonus"), None), + "crit_chance": ("stats.combat.critchance", ("stats", "combat", "critchance"), None), + "multi_attack": ("stats.combat.doubleattackchance", ("stats", "combat", "doubleattackchance"), None), + "dps": ("stats.combat.dps", ("stats", "combat", "dps"), None), + "attack_speed": ("stats.combat.attackspeed", ("stats", "combat", "attackspeed"), None), + "flurry": ("stats.combat.flurry", ("stats", "combat", "flurry"), None), + "block_chance": ("stats.combat.blockchance", ("stats", "combat", "blockchance"), None), + "strikethrough": ("stats.combat.strikethrough", ("stats", "combat", "strikethrough"), None), + "max_health": ("stats.health.max", ("stats", "health", "max"), None), + "max_power": ("stats.power.max", ("stats", "power", "max"), None), + # Progression (top-level scalar fields — sortable, unlike the statistics.* + # family's aggregate-only quests/collections/achievements buckets) + "quests_completed": ("quests.complete", ("quests", "complete"), None), + "collections_completed": ("collections.complete", ("collections", "complete"), None), + "achievement_points": ("achievements.points", ("achievements", "points"), None), + "achievements_completed": ("achievements.completed", ("achievements", "completed"), None), + "aa_points": ("alternateadvancements.spentpoints", ("alternateadvancements", "spentpoints"), None), + # Lifetime + "kills": ("statistics.kills", ("statistics", "kills", "value"), None), + "deaths": ("statistics.deaths", ("statistics", "deaths", "value"), None), + "kd_ratio": ( + "statistics.kills_deaths_ratio.value", + ("statistics", "kills_deaths_ratio", "value"), + {"statistics.kills.value": f"]{KD_KILLS_FLOOR}"}, + ), + "items_crafted": ("statistics.items_crafted", ("statistics", "items_crafted", "value"), None), + "rare_harvests": ("statistics.rare_harvests", ("statistics", "rare_harvests", "value"), None), +} + +EXPLORE_LIMIT = 20 +_EXPLORE_TTL_S = 900 # combat snapshots move slowly; 15 min keeps it fun-fresh +_explore_cache: dict[str, tuple[float, ExploreResponse]] = {} + + +class ExploreResponse(BaseModel): + stat: str + cls: str | None = None + entries: list[LeaderEntry] = [] + + +def _classid_for_name(cls: str) -> int | None: + for cid, name in _classid_names().items(): + if name.lower() == cls.lower(): + return cid + return None + + +def _walk(row: dict, path: tuple[str, ...]) -> Any: + value: Any = row + for key in path: + if not isinstance(value, dict): + return None + value = value.get(key) + return value + + +@router.get("/stats/explore", response_model=ExploreResponse) +@limiter.limit("30/minute") +async def explore_stats(request: Request, stat: str, cls: str | None = None) -> ExploreResponse: + """Live top-N by any whitelisted stat, optionally scoped to one class — + 'show me the highest Ability Mod Templars'.""" + _require_user(request) + entry = _EXPLORE_STATS.get(stat) + if entry is None: + raise HTTPException(status_code=400, detail=f"Unknown stat: {stat}") + sort_path, value_path, extra = entry + + classid = None + if cls: + classid = _classid_for_name(cls) + if classid is None: + raise HTTPException(status_code=400, detail=f"Unknown class: {cls}") + + world = current_world() + cache_key = f"{world.lower()}:{stat}:{classid or 'all'}" + cached = _explore_cache.get(cache_key) + if cached and time.monotonic() - cached[0] < _EXPLORE_TTL_S: + return cached[1] + + filters = dict(extra or {}) + if classid is not None: + filters["type.classid"] = str(classid) + # stats/statistics project the whole subtree (cheap); progression fields + # project the exact scalar — c:show=achievements would drag the full + # 500+-item achievement_list along with it. + show = value_path[0] if value_path[0] in ("stats", "statistics") else ".".join(value_path) + async with shared_census_client() as client: + rows = await client.get_stat_leaders(world, sort_path, EXPLORE_LIMIT, filters, show=show) + if rows is None: + if cached: + return cached[1] # stale beats an error page + raise HTTPException(status_code=503, detail="Census is not answering — try again shortly.") + + entries = [] + for ch in rows: + value = _walk(ch, value_path) + if value is None: + continue + entries.append( + LeaderEntry( + name=(ch.get("name") or {}).get("first") or "?", + cls=(ch.get("type") or {}).get("class"), + level=(ch.get("type") or {}).get("level"), + value=float(value), + ) + ) + entries.sort(key=lambda e: -e.value) + + resp = ExploreResponse(stat=stat, cls=cls, entries=entries[:EXPLORE_LIMIT]) + _explore_cache[cache_key] = (time.monotonic(), resp) + return resp + + +def _resolve_ability(crc: Any) -> str | None: + """Best-effort crc → ability name via spells.db (coverage is partial — + sentinel/unknown crcs simply render unnamed).""" + try: + crc_int = int(crc) + except (TypeError, ValueError): + return None + if crc_int <= 0 or crc_int >= 2**32 - 1: # 4294967295 = census sentinel + return None + row = spells_db.find_by_crc(crc_int) + return row.get("name") if row else None + + +@router.get("/stats/character/{name}", response_model=LifetimeStatResponse) +@limiter.limit("30/minute") +async def get_character_lifetime(request: Request, name: str) -> LifetimeStatResponse: + """A character's lifetime statistics + their class's world averages for + the sheet's "vs average" context.""" + _require_user(request) + sanitised = validate_character_name(name) + if sanitised is None: + raise HTTPException(status_code=400, detail="Character name is invalid.") + name = sanitised + world = current_world() + cache_key = f"lifetime:{name.lower()}:{world.lower()}" + + cached, is_stale = lifetime_cache.get_stale(cache_key) + if cached is not None and not is_stale: + return cached + + async with shared_census_client() as client: + ch = await client.get_character_statistics(name, world) + if ch is None: + if cached is not None: + return cached # stale beats nothing when census flakes + raise HTTPException(status_code=404, detail=f"No lifetime statistics for '{name}'.") + + st = ch.get("statistics") or {} + cls = (ch.get("type") or {}).get("class") + + def _val(stat: str) -> int | None: + v = (st.get(stat) or {}).get("value") + return int(v) if v is not None else None + + # Class context from the (SWR-cached) server aggregates — best-effort. + class_avg_kills = class_avg_kd = None + server_stats = _stats_cache.get(world, (0, None))[1] + if server_stats and cls: + row = next((c for c in server_stats.classes if c.name == cls), None) + if row: + class_avg_kills = row.avg.get("kills") + class_avg_kd = row.avg.get("kills_deaths_ratio") + + kd = (st.get("kills_deaths_ratio") or {}).get("value") + resp = LifetimeStatResponse( + character_name=(ch.get("name") or {}).get("first") or name, + kills=_val("kills"), + deaths=_val("deaths"), + kills_deaths_ratio=round(float(kd), 1) if kd is not None else None, + max_melee_hit=_val("max_melee_hit"), + max_melee_ability=_resolve_ability((st.get("max_melee_hit") or {}).get("weapon")), + max_magic_hit=_val("max_magic_hit"), + max_magic_ability=_resolve_ability((st.get("max_magic_hit") or {}).get("spell")), + items_crafted=_val("items_crafted"), + rare_harvests=_val("rare_harvests"), + class_avg_kills=class_avg_kills, + class_avg_kd=class_avg_kd, + ) + lifetime_cache.set(cache_key, resp) + _log.debug("[stats] lifetime fetched for %s@%s", scrub(name), world) + return resp diff --git a/backend/server/app.py b/backend/server/app.py index 3cb32de7..1a983ab7 100644 --- a/backend/server/app.py +++ b/backend/server/app.py @@ -78,6 +78,8 @@ async def get_response(self, path: str, scope): # type: ignore[override] from backend.server.api.recipes import router as recipes_router from backend.server.api.role_requests import router as role_requests_router from backend.server.api.server import router as server_router +from backend.server.api.stats import prewarm_server_stats +from backend.server.api.stats import router as stats_router from backend.server.api.supporters import router as supporters_router from backend.server.api.zones import router as zones_router from backend.server.api.zones_admin import router as zones_admin_router @@ -486,6 +488,7 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]: tasks: list[asyncio.Task] = [ asyncio.create_task(prewarm_character_cache(), name="prewarm-character-cache"), + asyncio.create_task(prewarm_server_stats(), name="prewarm-server-stats"), asyncio.create_task(_cache_sweep_loop(), name="cache-sweep-loop"), asyncio.create_task(census_health.poll_loop(), name="census-health-poll"), asyncio.create_task(_parse_cleanup_loop(), name="parse-cleanup-loop"), @@ -627,6 +630,7 @@ async def _parse_cleanup_loop() -> None: role_requests_router, census_router, server_router, + stats_router, supporters_router, ] for _r in _ROUTERS: diff --git a/backend/server/cache.py b/backend/server/cache.py index 9e3fb923..79edde85 100644 --- a/backend/server/cache.py +++ b/backend/server/cache.py @@ -15,7 +15,7 @@ _log = logging.getLogger(__name__) -CacheName = Literal["character", "guild", "claim", "aa", "gear-sets", "rankings", "favorites"] +CacheName = Literal["character", "guild", "claim", "aa", "gear-sets", "lifetime", "rankings", "favorites"] class TTLCache: @@ -211,3 +211,4 @@ def delete(self, key: str) -> None: aa_cache: TTLCache = TTLCache(name="aa", maxsize=200) gear_sets_cache: TTLCache = TTLCache(name="gear-sets", maxsize=200) favorite_count_cache: TTLCache = TTLCache(name="favorites", maxsize=1000) +lifetime_cache: TTLCache = TTLCache(name="lifetime", maxsize=300) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 1c0ff08d..36bafc14 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -26,6 +26,7 @@ const ParsesPage = lazy(() => import('./pages/ParsesPage')) const RaidZonePage = lazy(() => import('./pages/RaidZonePage')) const RaidZonesPage = lazy(() => import('./pages/RaidZonesPage')) const ComparePage = lazy(() => import('./pages/ComparePage')) +const StatsPage = lazy(() => import('./pages/StatsPage')) const AAPlanSharePage = lazy(() => import('./pages/AAPlanSharePage')) import { useAuth } from './hooks/useAuth' import { CensusStreamProvider } from './hooks/useCensusStream' @@ -138,6 +139,7 @@ function NavLinks() { + ) } @@ -302,6 +304,7 @@ function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/frontend/src/components/MobileNav.tsx b/frontend/src/components/MobileNav.tsx index 94663c99..f5c85e10 100644 --- a/frontend/src/components/MobileNav.tsx +++ b/frontend/src/components/MobileNav.tsx @@ -19,6 +19,7 @@ const ITEMS: NavSpec[] = [ { to: '/raids', label: 'Raids', also: '/raids/' }, { to: '/parses', label: 'Parses', also: '/parse/' }, { to: '/rankings', label: 'Rankings' }, + { to: '/stats', label: 'Stats' }, ] export function MobileNav() { diff --git a/frontend/src/pages/CharacterPage.gearsets.test.tsx b/frontend/src/pages/CharacterPage.gearsets.test.tsx index af6f865f..3843a9b9 100644 --- a/frontend/src/pages/CharacterPage.gearsets.test.tsx +++ b/frontend/src/pages/CharacterPage.gearsets.test.tsx @@ -92,7 +92,7 @@ const TANK_SET: GearSet = { } const DPS_SET: GearSet = { name: 'DPS', ilvl: 58, stat_deltas: {}, equipment: [slot('Head', 'Deeps Helm')] } -function stubFetch(opts: { char: Character; sets?: GearSet[]; items?: Record }) { +function stubFetch(opts: { char: Character; sets?: GearSet[]; items?: Record; lifetime?: Record }) { vi.stubGlobal('fetch', vi.fn(async (url: string) => { const ok = (body: unknown) => ({ ok: true, status: 200, json: async () => body }) if (url.includes('/gear-sets')) { @@ -105,6 +105,11 @@ function stubFetch(opts: { char: Character; sets?: GearSet[]; items?: Record) => ({ ...over, }) +describe('CharacterPage lifetime stats', () => { + it('renders the Lifetime panel with ability-named record hits', async () => { + stubFetch({ + char: mkChar('Lifey'), + lifetime: { + character_name: 'Lifey', kills: 17286, deaths: 140, kills_deaths_ratio: 123.5, + max_melee_hit: 125009, max_melee_ability: 'Frenzy II', + max_magic_hit: 66938, max_magic_ability: null, + items_crafted: 2243, rare_harvests: 0, + class_avg_kills: 6843.5, class_avg_kd: 160.2, + }, + }) + renderAt('/character/Lifey') + expect(await screen.findByText('Lifetime')).toBeInTheDocument() + expect(screen.getByText('17,286')).toBeInTheDocument() + expect(screen.getByText('125,009 (Frenzy II)')).toBeInTheDocument() + expect(screen.getByText('66,938')).toBeInTheDocument() // unnamed crc → bare value + expect(screen.getByText(/Class average on this server: 6,844 kills/)).toBeInTheDocument() + }) + + it('hides the panel when the character has no lifetime data', async () => { + stubFetch({ char: mkChar('Nolife') }) + renderAt('/character/Nolife') + expect(await screen.findByText('Current Helm')).toBeInTheDocument() + expect(screen.queryByText('Lifetime')).not.toBeInTheDocument() + }) +}) + describe('CharacterPage set bonuses', () => { it('lists worn sets with piece counts and marks active tiers', async () => { const bonuses = [ diff --git a/frontend/src/pages/CharacterPage.tsx b/frontend/src/pages/CharacterPage.tsx index 4c5af20f..2e1994ea 100644 --- a/frontend/src/pages/CharacterPage.tsx +++ b/frontend/src/pages/CharacterPage.tsx @@ -502,6 +502,7 @@ function CharacterView({ char, maxLevel, ratingConfig }: { char: Character; maxL deltas={activeSet ? activeSet.stat_deltas : null} onStatHover={setHoveredStat} onStatLeave={() => setHoveredStat(null)} /> + {/* Right: paperdoll */} @@ -575,6 +576,47 @@ function CharacterView({ char, maxLevel, ratingConfig }: { char: Character; maxL ) } +// ── Lifetime statistics (census character statistics object) ───────────────── + +interface LifetimeStats { + character_name: string + kills: number | null + deaths: number | null + kills_deaths_ratio: number | null + max_melee_hit: number | null + max_melee_ability: string | null + max_magic_hit: number | null + max_magic_ability: string | null + items_crafted: number | null + rare_harvests: number | null + class_avg_kills: number | null + class_avg_kd: number | null +} + +function LifetimePanel({ charName }: { charName: string }) { + const { data } = useFetch(`/api/stats/character/${encodeURIComponent(charName)}`) + if (!data || data.kills == null) return null // no lifetime data → no panel + const hit = (value: number | null, ability: string | null) => + value == null ? null : ability ? `${value.toLocaleString()} (${ability})` : value.toLocaleString() + return ( + + + + + + + + + {data.class_avg_kills != null && ( +
+ Class average on this server: {Math.round(data.class_avg_kills).toLocaleString()} kills + {data.class_avg_kd != null ? ` · ${data.class_avg_kd.toFixed(1)} K/D` : ''} +
+ )} + + ) +} + // ── Set bonuses (between the paperdoll and consumables) ─────────────────────── interface WornSet { diff --git a/frontend/src/pages/StatsPage.test.tsx b/frontend/src/pages/StatsPage.test.tsx new file mode 100644 index 00000000..28bbc67f --- /dev/null +++ b/frontend/src/pages/StatsPage.test.tsx @@ -0,0 +1,199 @@ +/** + * StatsPage render tests — totals cards, class-distribution bars, named + * leaderboards with character links, class averages table. + */ +import { describe, it, expect, beforeEach, vi } from 'vitest' +import { render, screen, fireEvent } from '@testing-library/react' +import { MemoryRouter } from 'react-router-dom' + +import StatsPage from './StatsPage' + +vi.mock('../hooks/useServer', () => ({ + useServer: () => ({ displayName: 'Wuoshi' }), +})) + +// useClasses caches [] module-wide after its first fetch, so stub the hook +// directly — the page only needs `classes` (explorer dropdown) + `colourFor`. +vi.mock('../useClasses', () => ({ + useClasses: () => ({ + classes: [{ name: 'Templar' }, { name: 'Necromancer' }], + byName: new Map(), + colourFor: () => 'var(--text)', + iconUrlFor: () => null, + }), +})) + +const STATS = { + world: 'Wuoshi', + ts: 1784972773, + fetched_at: 1784980000, + population: 21343, + totals: { + kills: 146061046, + deaths: 637644, + 'quests.complete': 1358803, + 'collections.complete': 78657, + items_crafted: 40225247, + rare_harvests: 891495, + }, + records: { kills: 465696, max_melee_hit: 268591 }, + averages: { kills: 6843.5 }, + global_averages: { kills: 9000 }, + classes: [ + { classid: 33, name: 'Necromancer', count: 2305, avg: { kills: 13216, kills_deaths_ratio: 300 }, global_avg: { kills_deaths_ratio: 250 } }, + { classid: 13, name: 'Templar', count: 900, avg: { kills: 6000, kills_deaths_ratio: 120 }, global_avg: { kills_deaths_ratio: 110 } }, + ], + leaders: { + kills: [ + { name: 'Kaipai', cls: 'Necromancer', level: 70, value: 465696 }, + { name: 'Touxin', cls: 'Fury', level: 70, value: 459917 }, + ], + max_melee_hit: [{ name: 'Dema', cls: 'Guardian', level: 70, value: 268591 }], + }, +} + +function stubFetch(body: unknown = STATS, ok = true) { + vi.stubGlobal('fetch', vi.fn(async (url: string) => ({ + ok: url.includes('/api/stats/server') ? ok : true, + status: ok ? 200 : 503, + json: async () => (url.includes('/api/classes') ? [] : body), + })) as unknown as typeof fetch) +} + +beforeEach(() => { + vi.restoreAllMocks() +}) + +describe('StatsPage', () => { + it('renders population, totals, class bars, leaderboards and the averages table', async () => { + stubFetch() + render() + + expect(await screen.findByText(/21,343 characters known to Census/)).toBeInTheDocument() + // Totals card + expect(screen.getByText('146,061,046')).toBeInTheDocument() + expect(screen.getByText('NPCs slain')).toBeInTheDocument() + // Class distribution (name appears in the bar row AND the table) + expect(screen.getAllByText('Necromancer').length).toBeGreaterThanOrEqual(2) + expect(screen.getByText(/2,305 · 10.8%/)).toBeInTheDocument() + // Leaderboards: names link to character pages + const kaipai = screen.getByRole('link', { name: 'Kaipai' }) + expect(kaipai).toHaveAttribute('href', '/character/Kaipai') + expect(screen.getByText('Most Kills')).toBeInTheDocument() + expect(screen.getByText('Biggest Melee Hit')).toBeInTheDocument() + // Boards with no data don't render + expect(screen.queryByText('Rare Harvests')).not.toBeInTheDocument() + // Averages table shows the global comparison column + expect(screen.getByText('Global avg K/D')).toBeInTheDocument() + }) + + it('surfaces the 503 message when stats are unavailable', async () => { + stubFetch({ detail: 'Server statistics unavailable (Census unreachable). Try again shortly.' }, false) + render() + expect(await screen.findByText(/Census unreachable/)).toBeInTheDocument() + }) + + it('explorer tab fetches whitelisted stats and refetches on class change', async () => { + const calls: string[] = [] + vi.stubGlobal('fetch', vi.fn(async (url: string) => { + calls.push(url) + if (url.includes('/api/stats/explore')) { + return { + ok: true, + status: 200, + json: async () => ({ + stat: 'ability_mod', + cls: url.includes('cls=') ? 'Templar' : null, + entries: url.includes('cls=') + ? [{ name: 'Menludiir', cls: 'Templar', level: 70, value: 1868 }] + : [ + { name: 'Menludiir', cls: 'Templar', level: 70, value: 1868 }, + { name: 'Sihtric', cls: 'Wizard', level: 70, value: 1700 }, + ], + }), + } + } + return { ok: true, status: 200, json: async () => STATS } + }) as unknown as typeof fetch) + + render() + fireEvent.click(screen.getByRole('button', { name: 'Explorer' })) + + // Default stat (Ability Mod, all classes) loads and links to characters. + expect(await screen.findByRole('link', { name: 'Menludiir' })).toHaveAttribute('href', '/character/Menludiir') + expect(screen.getByText('Sihtric')).toBeInTheDocument() + expect(calls.some(u => u.includes('/api/stats/explore?stat=ability_mod') && !u.includes('cls='))).toBe(true) + + // Narrowing to a class refetches with cls= and re-renders. + fireEvent.change(screen.getByLabelText('Class'), { target: { value: 'Templar' } }) + await vi.waitFor(() => { + if (!calls.some(u => u.includes('stat=ability_mod') && u.includes('cls=Templar'))) throw new Error('not yet') + }) + // The table hides while the refetch is in flight — wait for loading to + // clear so we're asserting against the narrowed result set. + await vi.waitFor(() => { + if (screen.queryByText('Asking Census…')) throw new Error('still loading') + if (screen.queryByText('Sihtric')) throw new Error('not yet') + }) + expect(screen.getByRole('link', { name: 'Menludiir' })).toBeInTheDocument() + }) + + it('network failure → retries instead of dead-ending, recovers when the server returns', async () => { + vi.useFakeTimers() + try { + let call = 0 + vi.stubGlobal('fetch', vi.fn(async (url: string) => { + if (url.includes('/api/classes')) return { ok: true, status: 200, json: async () => [] } + call += 1 + // First hit: the server is mid-restart — connection-level failure. + if (call === 1) throw new TypeError('NetworkError when attempting to fetch resource.') + return { ok: true, status: 200, json: async () => STATS } + }) as unknown as typeof fetch) + + render() + // The failure surfaces as a soft retry message, not the red error. + expect(await vi.waitFor(() => { + const el = screen.queryByText(/Lost contact with the server/) + if (!el) throw new Error('not yet') + return el + })).toBeInTheDocument() + expect(screen.queryByText(/NetworkError/)).not.toBeInTheDocument() + // The 5s retry refetches and real data replaces the message. + await vi.advanceTimersByTimeAsync(5100) + await vi.waitFor(() => { + if (!screen.queryByText(/21,343 characters known to Census/)) throw new Error('not yet') + }) + } finally { + vi.useRealTimers() + } + }) + + it('202 building → shows the crunching message and polls until data lands', async () => { + vi.useFakeTimers() + try { + let call = 0 + vi.stubGlobal('fetch', vi.fn(async (url: string) => { + if (url.includes('/api/classes')) return { ok: true, status: 200, json: async () => [] } + call += 1 + return call === 1 + ? { ok: true, status: 202, json: async () => ({ status: 'building' }) } + : { ok: true, status: 200, json: async () => STATS } + }) as unknown as typeof fetch) + + render() + // First response: 202 → the building copy shows. + expect(await vi.waitFor(() => { + const el = screen.queryByText(/Crunching server statistics/) + if (!el) throw new Error('not yet') + return el + })).toBeInTheDocument() + // The 5s poll refetches and real data replaces the message. + await vi.advanceTimersByTimeAsync(5100) + await vi.waitFor(() => { + if (!screen.queryByText(/21,343 characters known to Census/)) throw new Error('not yet') + }) + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/frontend/src/pages/StatsPage.tsx b/frontend/src/pages/StatsPage.tsx new file mode 100644 index 00000000..0b630072 --- /dev/null +++ b/frontend/src/pages/StatsPage.tsx @@ -0,0 +1,370 @@ +/** + * StatsPage — per-server lifetime statistics from the census character.stat + * aggregate family: population + class distribution, server lifetime totals, + * and named leaderboards per statistic. Data refreshes server-side every + * 6 h (census recomputes the aggregates daily). + */ +import { useEffect, useState } from 'react' +import { Link } from 'react-router-dom' + +import Breadcrumb from '../components/Breadcrumb' +import { Card, SectionLabel } from '../components/ui' +import { TabButton } from '../components/ui/TabButton' +import { fmtLocalDate } from '../formatters' +import { useFetch } from '../hooks/useFetch' +import { useServer } from '../hooks/useServer' +import { useClasses } from '../useClasses' + +interface ClassStatsRow { + classid: number + name: string + count: number + avg: Record + global_avg: Record +} + +interface LeaderEntry { + name: string + cls: string | null + level: number | null + value: number +} + +interface ServerStats { + world: string + ts: number + fetched_at: number + population: number + totals: Record + records: Record + averages: Record + global_averages: Record + classes: ClassStatsRow[] + leaders: Record +} + +const LEADER_BOARDS: { key: string; title: string; hint?: string }[] = [ + { key: 'kills', title: 'Most Kills' }, + { key: 'deaths', title: 'Most Deaths' }, + { key: 'kills_deaths_ratio', title: 'Best K/D Ratio', hint: 'min 1,000 kills' }, + { key: 'max_melee_hit', title: 'Biggest Melee Hit' }, + { key: 'max_magic_hit', title: 'Biggest Magic Hit' }, + { key: 'items_crafted', title: 'Items Crafted' }, + { key: 'rare_harvests', title: 'Rare Harvests' }, +] + +const TOTAL_CARDS: { key: string; label: string }[] = [ + { key: 'kills', label: 'NPCs slain' }, + { key: 'deaths', label: 'Deaths suffered' }, + { key: 'quests.complete', label: 'Quests completed' }, + { key: 'collections.complete', label: 'Collections finished' }, + { key: 'items_crafted', label: 'Items crafted' }, + { key: 'rare_harvests', label: 'Rares harvested' }, +] + +const fmt = (n: number | undefined | null, digits = 0): string => + n == null ? '—' : n.toLocaleString(undefined, { maximumFractionDigits: digits }) + +// ── Explorer ────────────────────────────────────────────────────────────────── + +const EXPLORE_STATS: { key: string; label: string; group: string; digits?: number }[] = [ + { key: 'ability_mod', label: 'Ability Mod', group: 'Combat' }, + { key: 'potency', label: 'Potency', group: 'Combat', digits: 1 }, + { key: 'crit_bonus', label: 'Crit Bonus', group: 'Combat', digits: 1 }, + { key: 'crit_chance', label: 'Crit Chance', group: 'Combat', digits: 1 }, + { key: 'multi_attack', label: 'Multi Attack', group: 'Combat', digits: 1 }, + { key: 'dps', label: 'DPS Mod', group: 'Combat', digits: 1 }, + { key: 'attack_speed', label: 'Attack Speed', group: 'Combat', digits: 1 }, + { key: 'flurry', label: 'Flurry', group: 'Combat', digits: 1 }, + { key: 'block_chance', label: 'Block Chance', group: 'Combat', digits: 1 }, + { key: 'strikethrough', label: 'Strikethrough', group: 'Combat', digits: 1 }, + { key: 'max_health', label: 'Max Health', group: 'Combat' }, + { key: 'max_power', label: 'Max Power', group: 'Combat' }, + { key: 'quests_completed', label: 'Quests Completed', group: 'Progression' }, + { key: 'collections_completed', label: 'Collections Completed', group: 'Progression' }, + { key: 'achievement_points', label: 'Achievement Points', group: 'Progression' }, + { key: 'achievements_completed', label: 'Achievements Completed', group: 'Progression' }, + { key: 'aa_points', label: 'AA Points Spent', group: 'Progression' }, + { key: 'kills', label: 'Kills', group: 'Lifetime' }, + { key: 'deaths', label: 'Deaths', group: 'Lifetime' }, + { key: 'kd_ratio', label: 'K/D Ratio (min 1,000 kills)', group: 'Lifetime', digits: 1 }, + { key: 'items_crafted', label: 'Items Crafted', group: 'Lifetime' }, + { key: 'rare_harvests', label: 'Rare Harvests', group: 'Lifetime' }, +] + +interface ExploreResponse { + stat: string + cls: string | null + entries: LeaderEntry[] +} + +function ExplorerTab({ colourFor }: { colourFor: (cls: string | null | undefined, fallback: string) => string }) { + const { classes } = useClasses() + const [stat, setStat] = useState('ability_mod') + const [cls, setCls] = useState('') + + const url = `/api/stats/explore?stat=${encodeURIComponent(stat)}${cls ? `&cls=${encodeURIComponent(cls)}` : ''}` + const { data, loading, error } = useFetch(url) + const statDef = EXPLORE_STATS.find(s => s.key === stat) + + const groups = [...new Set(EXPLORE_STATS.map(s => s.group))] + return ( +
+
+ + +
+ + {loading &&

Asking Census…

} + {error &&

{error}

} + + {data && !loading && ( + + + + + + + + + + + + {data.entries.length === 0 && ( + + )} + {data.entries.map((e, i) => ( + + + + + + + ))} + +
#CharacterClass{statDef?.label ?? stat}
No characters matched.
{i + 1} + + {e.name} + + {e.cls ?? '—'}{e.level ? ` (${e.level})` : ''}{fmt(e.value, statDef?.digits ?? 0)}
+ + )} +

+ Combat stats are Census snapshots from each character's last index — close to live, not real-time. +

+
+ ) +} + +export default function StatsPage() { + const server = useServer() + const { colourFor } = useClasses() + // A cold server answers 202 {status:'building'} while it assembles the + // stats in the background (~1-2 min of census queries) — poll until real + // data lands. Normally the startup prewarm means this never fires. + const { data: raw, loading, error, statusCode, refetch } = useFetch( + '/api/stats/server', + ) + const building = statusCode === 202 + const data = raw && 'population' in raw ? raw : null + + // A connection-level failure (statusCode never set — server restarting, + // e.g. a Railway deploy or a dev-server reload) is transient: ride it out + // like the 202 state instead of dead-ending on "NetworkError…". + const MAX_NET_RETRIES = 4 + const [netRetries, setNetRetries] = useState(0) + const netFailure = error !== null && statusCode === null + const reconnecting = netFailure && netRetries < MAX_NET_RETRIES + + useEffect(() => { + if (statusCode !== null && netRetries !== 0) setNetRetries(0) + }, [statusCode, netRetries]) + + useEffect(() => { + if (!building && !reconnecting) return + const t = setTimeout(() => { + if (reconnecting) setNetRetries(r => r + 1) + refetch() + }, 5000) + return () => clearTimeout(t) + }, [building, reconnecting, raw, refetch]) + + const [tab, setTab] = useState<'server' | 'explore'>('server') + const maxClassCount = data ? Math.max(1, ...data.classes.map(c => c.count)) : 1 + + return ( +
+ +
+

+ {server?.displayName ?? data?.world ?? 'Server'} Statistics +

+ {data && ( + + {fmt(data.population)} characters known to Census · aggregates from {fmtLocalDate(data.ts)} + + )} +
+

+ Lifetime statistics across every character Census has seen on this server. +

+ + {/* Tabs: the aggregate view vs the live stat explorer */} +
+ setTab('server')}>Server Records + setTab('explore')}>Explorer +
+ + {tab === 'explore' && } + + {tab === 'server' && (loading || building || reconnecting) && ( +

+ {building + ? 'Crunching server statistics — the first build after a restart takes a minute or two…' + : reconnecting || netRetries > 0 + ? 'Lost contact with the server — retrying…' + : 'Loading server statistics…'} +

+ )} + {tab === 'server' && error && !reconnecting &&

{error}

} + + {tab === 'server' && data && ( + <> + {/* ── Lifetime totals ── */} + Server Lifetime Totals +
+ {TOTAL_CARDS.map(t => ( + +
+ {fmt(data.totals[t.key])} +
+
{t.label}
+ + ))} +
+ + {/* ── Class distribution ── */} + Class Population + +
+ {data.classes.map(c => ( +
+ + {c.name} + +
+
+
+ + {fmt(c.count)} · {((c.count / Math.max(1, data.population)) * 100).toFixed(1)}% + +
+ ))} +
+ + + {/* ── Leaderboards ── */} + Server Records +
+ {LEADER_BOARDS.filter(b => data.leaders[b.key]?.length).map(board => ( + +
+ + {board.title} + + {board.hint && {board.hint}} +
+
    + {data.leaders[board.key].map((e, i) => ( +
  1. + + {i + 1}. + + + {e.name} + + {fmt(e.value, 1)} +
    2. + ))} +
+ + ))} +
+ + {/* ── Class averages ── */} + Class Averages + + + + + + + + + + + + + {data.classes.map(c => ( + + + + + + + + ))} + +
ClassCharactersAvg killsAvg K/D + Global avg K/D +
{c.name}{fmt(c.count)}{fmt(c.avg['kills'])}{fmt(c.avg['kills_deaths_ratio'], 1)} + {fmt(c.global_avg['kills_deaths_ratio'], 1)} +
+ + +

+ Counts cover every character Census has indexed on this server (including inactive ones). Aggregates are + recomputed daily by Daybreak; leaderboards refresh with them. +

+ + )} +
+ ) +} diff --git a/tests/server/test_stats.py b/tests/server/test_stats.py new file mode 100644 index 00000000..7dff20b6 --- /dev/null +++ b/tests/server/test_stats.py @@ -0,0 +1,357 @@ +"""Tests for the server statistics API (census character.stat family). + +The census client is mocked at the census_lifecycle layer (same pattern as +test_gear_sets); classes.db and spells.db assertions run against the real +committed catalogues so classid→name and crc→ability mappings stay honest. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from httpx import ASGITransport, AsyncClient + +from backend.server.api import stats as stats_mod +from backend.server.cache import lifetime_cache +from tests.fixtures.users import make_fake_require_user, make_fake_user + +_fake_user = make_fake_require_user(make_fake_user(id="123")) + +WORLDID = 618 + + +def _bucket(count: int, kills_avg: float = 5000.0, kills_max: float = 100000.0) -> dict: + return { + "count": count, + "ts": 1784972773, + "kills": {"max": kills_max, "sum": kills_avg * count, "avg": kills_avg}, + "deaths": {"max": 3000, "sum": 30_000, "avg": 30.0}, + "kills_deaths_ratio": {"max": 20000, "sum": 5000, "avg": 160.0}, + "max_melee_hit": {"max": 268591, "sum": 1, "avg": 4000.0}, + "max_magic_hit": {"max": 12929854, "sum": 1, "avg": 9000.0}, + "quests.complete": {"max": 2221, "sum": 1_358_803, "avg": 63.7}, + "collections.complete": {"max": 500, "sum": 78_657, "avg": 3.7}, + "achievements.points": {"max": 2605, "sum": 100, "avg": 331.4}, + "items_crafted": {"max": 4_978_221, "sum": 40_225_247, "avg": 1884.7}, + "rare_harvests": {"max": 18780, "sum": 891_495, "avg": 41.8}, + } + + +def _char(name: str, cls: str, stat: str, value: float) -> dict: + return {"name": {"first": name}, "type": {"class": cls, "level": 70}, "statistics": {stat: {"value": value}}} + + +def _mock_client() -> MagicMock: + client = MagicMock() + client.get_worldid = AsyncMock(return_value=WORLDID) + client.get_stat_aggregates = AsyncMock( + return_value={ + "global": [ + {"id": "all", "value": _bucket(500_000, kills_avg=9000)}, + {"id": "13", "value": _bucket(20_000, kills_avg=7000)}, # Templar, game-wide + ], + "world": [{"id": str(WORLDID), "value": _bucket(21_343, kills_avg=6843.5, kills_max=465_696)}], + "world_class": [ + {"id": f"{WORLDID}.13", "value": _bucket(900, kills_avg=6000)}, # Templar + {"id": f"{WORLDID}.19", "value": _bucket(1100, kills_avg=8000)}, # Mystic + {"id": "999.13", "value": _bucket(5)}, # other world — must be ignored + ], + } + ) + client.get_stat_leaders = AsyncMock( + return_value=[_char("Kaipai", "Necromancer", "kills", 465_696), _char("Touxin", "Fury", "kills", 459_917)] + ) + # Range mode returns UNSORTED rows — the builder must sort client-side. + client.get_stat_range = AsyncMock( + side_effect=lambda world, path, minimum, limit=200: [ + _char("Biffels", "Berserker", path.split(".")[1], 260_007), + _char("Dema", "Guardian", path.split(".")[1], 268_591), + ] + ) + client.get_character_statistics = AsyncMock(return_value=None) + return client + + +@pytest.fixture(autouse=True) +def _fresh_caches(): + stats_mod._stats_cache.clear() + stats_mod._stats_locks.clear() + stats_mod._last_build_failure.clear() + stats_mod._explore_cache.clear() + lifetime_cache._store.clear() + yield + stats_mod._stats_cache.clear() + stats_mod._stats_locks.clear() + stats_mod._last_build_failure.clear() + stats_mod._explore_cache.clear() + lifetime_cache._store.clear() + + +def _census(client: MagicMock): + mock_cc = MagicMock(return_value=client) + return ( + patch("backend.server.core.census_lifecycle._clients", {}), + patch("backend.server.core.census_lifecycle.CensusClient", mock_cc), + ) + + +# --------------------------------------------------------------------------- +# Payload builder +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_build_server_stats_shapes_payload(): + client = _mock_client() + p1, p2 = _census(client) + with p1, p2: + stats = await stats_mod._build_server_stats("Wuoshi") + + assert stats is not None + assert stats.population == 21_343 + assert stats.totals["kills"] == pytest.approx(6843.5 * 21_343) + assert stats.records["max_melee_hit"] == 268_591 + + # classid → name via the committed classes.db (13=Templar, 19=Mystic), + # sorted by population desc; the other-world row is excluded. + assert [(c.name, c.count) for c in stats.classes] == [("Mystic", 1100), ("Templar", 900)] + assert stats.classes[1].global_avg["kills"] == 7000 + + # Sorted boards come straight through; range boards are client-sorted. + assert stats.leaders["kills"][0].name == "Kaipai" + assert [e.name for e in stats.leaders["max_melee_hit"][:2]] == ["Dema", "Biffels"] + + +@pytest.mark.asyncio +async def test_build_returns_none_when_census_fails(): + client = _mock_client() + client.get_stat_aggregates = AsyncMock(return_value=None) + p1, p2 = _census(client) + with p1, p2: + assert await stats_mod._build_server_stats("Wuoshi") is None + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_server_stats_endpoint_202_when_cold_then_serves(app): + """A cold cache never builds inline: the endpoint kicks a background + build and answers 202; once the cache is warm it serves 200 without + re-fetching the aggregates.""" + client = _mock_client() + p1, p2 = _census(client) + with p1, p2, patch("backend.server.api.stats._require_user", _fake_user): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as http: + cold = await http.get("/api/stats/server") + assert cold.status_code == 202 + assert cold.json() == {"status": "building"} + # Let the background build (kicked by the request) finish. + await stats_mod._refresh_server_stats("Varsoon") + warm = await http.get("/api/stats/server") + again = await http.get("/api/stats/server") + + assert warm.status_code == 200 + assert warm.json()["population"] == 21_343 + assert again.status_code == 200 + # Aggregates fetched once (bg task + explicit refresh dedupe on the lock). + assert client.get_stat_aggregates.await_count == 1 + + +@pytest.mark.asyncio +async def test_server_stats_503_after_recent_failed_build(app): + client = _mock_client() + client.get_worldid = AsyncMock(return_value=None) + p1, p2 = _census(client) + with p1, p2, patch("backend.server.api.stats._require_user", _fake_user): + # A failed build stamps the cooldown → the endpoint 503s instead of + # keeping the frontend polling forever. + await stats_mod._refresh_server_stats("Varsoon") + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as http: + r = await http.get("/api/stats/server") + assert r.status_code == 503 + + +_REAL_SPELLS_DB = __import__("pathlib").Path("data/spells/spells.db") + + +@pytest.mark.asyncio +@pytest.mark.skipif(not _REAL_SPELLS_DB.exists(), reason="local spells.db not present (gitignored artifact)") +async def test_character_lifetime_resolves_ability_names(app, monkeypatch): + """crc→ability via the local spells.db: 1729734970 = Frenzy II + (real-data assertion — the id came from a live census record hit). + conftest points the spells catalogue at an empty test db; re-point it + at the real artifact for this test only.""" + from backend.eq2db.spells import catalogue as spells_catalogue + + monkeypatch.setattr(spells_catalogue, "path", _REAL_SPELLS_DB) + client = _mock_client() + client.get_character_statistics = AsyncMock( + return_value={ + "name": {"first": "Badbang"}, + "type": {"class": "Berserker"}, + "statistics": { + "kills": {"value": 17286}, + "deaths": {"value": 140}, + "kills_deaths_ratio": {"value": 123.47142}, + "max_melee_hit": {"weapon": 1729734970, "value": 125009}, + "max_magic_hit": {"spell": 4294967295, "value": 66938}, # sentinel → unnamed + "items_crafted": {"value": 2243}, + "rare_harvests": {"value": 0}, + }, + } + ) + p1, p2 = _census(client) + with p1, p2, patch("backend.server.api.stats._require_user", _fake_user): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as http: + r = await http.get("/api/stats/character/Badbang") + + assert r.status_code == 200 + body = r.json() + assert body["kills"] == 17286 + assert body["kills_deaths_ratio"] == 123.5 + assert body["max_melee_ability"] == "Frenzy II" + assert body["max_magic_ability"] is None # sentinel crc stays unnamed + + +@pytest.mark.asyncio +async def test_character_lifetime_404_and_400(app): + client = _mock_client() + p1, p2 = _census(client) + with p1, p2, patch("backend.server.api.stats._require_user", _fake_user): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as http: + missing = await http.get("/api/stats/character/Ghostchar") + invalid = await http.get("/api/stats/character/bad name!!") + assert missing.status_code == 404 + assert invalid.status_code == 400 + + +# --------------------------------------------------------------------------- +# Explorer +# --------------------------------------------------------------------------- + + +def _combat_char(name: str, cls: str, ability_mod: float) -> dict: + return { + "name": {"first": name}, + "type": {"class": cls, "level": 70}, + "stats": {"combat": {"abilitymod": ability_mod}}, + } + + +@pytest.mark.asyncio +async def test_explore_rejects_unknown_stat_and_class(app): + client = _mock_client() + p1, p2 = _census(client) + with p1, p2, patch("backend.server.api.stats._require_user", _fake_user): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as http: + bad_stat = await http.get("/api/stats/explore?stat=hax") + bad_cls = await http.get("/api/stats/explore?stat=ability_mod&cls=Wibble") + assert bad_stat.status_code == 400 + assert bad_cls.status_code == 400 + client.get_stat_leaders.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_explore_class_filter_and_client_sort(app): + """cls=Templar must reach census as type.classid=13 (string names are + silently ignored by census), combat stats query the `stats` subtree, and + rows are re-sorted client-side.""" + client = _mock_client() + client.get_stat_leaders = AsyncMock( + return_value=[_combat_char("Lowmod", "Templar", 900), _combat_char("Menludiir", "Templar", 1868)] + ) + p1, p2 = _census(client) + with p1, p2, patch("backend.server.api.stats._require_user", _fake_user): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as http: + r = await http.get("/api/stats/explore?stat=ability_mod&cls=Templar") + + assert r.status_code == 200 + body = r.json() + assert [e["name"] for e in body["entries"]] == ["Menludiir", "Lowmod"] + assert body["entries"][0]["value"] == 1868 + + args, kwargs = client.get_stat_leaders.await_args + assert args[1] == "stats.combat.abilitymod" + assert args[3] == {"type.classid": "13"} # committed classes.db: Templar icon_id 13 + assert kwargs["show"] == "stats" + + +@pytest.mark.asyncio +async def test_explore_progression_stat_uses_narrow_projection(app): + """Progression stats live at top-level scalar paths (quests.complete, + achievements.points, alternateadvancements.spentpoints) and must project + the exact scalar — c:show=achievements would drag the 500+-item + achievement_list into every row.""" + client = _mock_client() + client.get_stat_leaders = AsyncMock( + return_value=[ + {"name": {"first": "Talormane"}, "type": {"class": "Warden", "level": 70}, "quests": {"complete": 2232}} + ] + ) + p1, p2 = _census(client) + with p1, p2, patch("backend.server.api.stats._require_user", _fake_user): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as http: + r = await http.get("/api/stats/explore?stat=quests_completed") + + assert r.status_code == 200 + assert r.json()["entries"][0] == {"name": "Talormane", "cls": "Warden", "level": 70, "value": 2232} + args, kwargs = client.get_stat_leaders.await_args + assert args[1] == "quests.complete" + assert kwargs["show"] == "quests.complete" + + +@pytest.mark.asyncio +async def test_explore_kd_floor_and_statistics_show(app): + client = _mock_client() + client.get_stat_leaders = AsyncMock(return_value=[_char("Badbang", "Berserker", "kills_deaths_ratio", 123.5)]) + p1, p2 = _census(client) + with p1, p2, patch("backend.server.api.stats._require_user", _fake_user): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as http: + r = await http.get("/api/stats/explore?stat=kd_ratio") + + assert r.status_code == 200 + args, kwargs = client.get_stat_leaders.await_args + assert args[3] == {"statistics.kills.value": f"]{stats_mod.KD_KILLS_FLOOR}"} + assert kwargs["show"] == "statistics" + + +@pytest.mark.asyncio +async def test_explore_cache_hit_skips_census(app): + client = _mock_client() + client.get_stat_leaders = AsyncMock(return_value=[_combat_char("Menludiir", "Templar", 1868)]) + p1, p2 = _census(client) + with p1, p2, patch("backend.server.api.stats._require_user", _fake_user): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as http: + first = await http.get("/api/stats/explore?stat=ability_mod&cls=Templar") + second = await http.get("/api/stats/explore?stat=ability_mod&cls=Templar") + + assert first.status_code == second.status_code == 200 + assert client.get_stat_leaders.await_count == 1 + + +@pytest.mark.asyncio +async def test_explore_stale_beats_error_and_503_when_cold(app): + client = _mock_client() + client.get_stat_leaders = AsyncMock(return_value=[_combat_char("Menludiir", "Templar", 1868)]) + p1, p2 = _census(client) + with p1, p2, patch("backend.server.api.stats._require_user", _fake_user): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as http: + warm = await http.get("/api/stats/explore?stat=ability_mod") + assert warm.status_code == 200 + + # Expire the cache entry, then break census: stale data still serves. + key, (_, payload) = next(iter(stats_mod._explore_cache.items())) + stats_mod._explore_cache[key] = (-(10**9), payload) + client.get_stat_leaders = AsyncMock(return_value=None) + stale = await http.get("/api/stats/explore?stat=ability_mod") + assert stale.status_code == 200 + assert stale.json()["entries"][0]["name"] == "Menludiir" + + # No cache at all + census down → 503. + stats_mod._explore_cache.clear() + cold = await http.get("/api/stats/explore?stat=ability_mod") + assert cold.status_code == 503 From b69da11f5a8995dc013d1568d839f35bbd09a56a Mon Sep 17 00:00:00 2001 From: VortexUK Date: Sat, 25 Jul 2026 13:01:29 +0100 Subject: [PATCH 2/2] ci(security): allowlist mechanism for npm audit, ignore react-router RSC CSRF GHSA-qwww-vcr4-c8h2 (published 2026-07-24) flags react-router >=7.12.0 <8.3.0 for an RSC-mode CSRF bypass. Not applicable here - the app is a client-only SPA (BrowserRouter, no RSC/server actions) - and no 7.x patch exists; the fix is the react-router 8 major. npm audit has no native ignore flag, so the CI step now mirrors the .pip-audit-ignore pattern: frontend/.npm-audit-ignore lists GHSA IDs with rationale; only unallowlisted high/critical advisories fail the build. A missing/errored audit report still fails loudly. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 41 ++++++++++++++++++++++++++++++++++---- frontend/.npm-audit-ignore | 10 ++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 frontend/.npm-audit-ignore diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d36c3b88..64c7eeb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,7 +131,40 @@ jobs: - name: npm audit (frontend production deps) working-directory: frontend - # --audit-level=high → exit 1 only on high or critical; moderate/low - # are reported but don't fail the build. --omit=dev skips devDeps - # (vitest, ESLint, etc.) — same rationale as pip-audit's --no-dev. - run: npm audit --audit-level=high --omit=dev + # Same policy as the bare `npm audit --audit-level=high --omit=dev` + # this replaces: only high/critical advisories fail the build, and + # devDeps are skipped (same rationale as pip-audit's --no-dev). npm + # audit has no native ignore flag, so this mirrors .pip-audit-ignore: + # frontend/.npm-audit-ignore lists GHSA IDs (first token per line, + # rest is rationale) for accepted risks / not-applicable advisories. + run: | + npm audit --json --omit=dev > /tmp/npm-audit.json || true + # A registry/network failure must fail loudly, not pass silently. + jq -e '.vulnerabilities != null' /tmp/npm-audit.json > /dev/null || { + echo "::error::npm audit did not produce a report" + cat /tmp/npm-audit.json + exit 1 + } + jq -r '[.vulnerabilities[]?.via[]? | objects + | select(.severity == "high" or .severity == "critical") + | (.url // empty)] | unique | .[] | sub(".*/"; "")' \ + /tmp/npm-audit.json > /tmp/npm-audit-ids + : > /tmp/npm-audit-ignored + if [ -f .npm-audit-ignore ]; then + grep -vE '^[[:space:]]*(#|$)' .npm-audit-ignore | awk '{print $1}' > /tmp/npm-audit-ignored || true + fi + FAIL=0 + while IFS= read -r id; do + [ -z "$id" ] && continue + if grep -qx "$id" /tmp/npm-audit-ignored; then + echo "allowlisted by .npm-audit-ignore: $id" + else + echo "::error::high/critical advisory not allowlisted: https://github.com/advisories/$id" + FAIL=1 + fi + done < /tmp/npm-audit-ids + if [ "$FAIL" = "1" ]; then + npm audit --audit-level=high --omit=dev || true + exit 1 + fi + echo "npm audit: no unallowlisted high/critical advisories" diff --git a/frontend/.npm-audit-ignore b/frontend/.npm-audit-ignore new file mode 100644 index 00000000..eede30bc --- /dev/null +++ b/frontend/.npm-audit-ignore @@ -0,0 +1,10 @@ +# npm audit ignore list for the CI security job (.github/workflows/ci.yml). +# +# Format: one GHSA ID per line (first whitespace-delimited token); the rest +# of the line is free for a rationale. Blank lines and lines starting with +# `#` are ignored. Same convention as ../.pip-audit-ignore. +# +# Prefer fixing the advisory by bumping the dep. Use this file only for +# accepted-risk items or advisories that don't apply to how we ship. + +GHSA-qwww-vcr4-c8h2 react-router RSC-mode CSRF (actions run before 400). Not applicable: this app is a client-only SPA (BrowserRouter, no RSC / server actions). No 7.x patch exists — fixed only in react-router 8.3.0 (major). Remove this line when we take the v8 upgrade.