feat(statapi): add /wait0 stats endpoint and refresh metrics

- add authenticated GET /wait0 and /wait0/ control endpoint with stats:read scope
- add 5s snapshot-cached stats payload for cache, memory, sitemap, and refresh durations
- instrument revalidation durations and expose min/avg/max as refresh_duration_ms
- extend cache metadata snapshots for efficient aggregate calculations
- update debug config, docs, architecture notes, and tests for new endpoint behavior

Author: mamchurovskyy

AGENTS.md

@@ -63,6 +63,11 @@ wait0 is an ultra-fast cache-first HTTP reverse proxy written in Go that serves
 | API Endpoints | docs/api-endpoints.md | Endpoint and response reference |
 | Docker Hub notes | DOCKERHUB.md | Alias to README for Docker Hub presentation |
 
+## AI Context Files
+| File | Description |
+|------|-------------|
+| .ai-factory/ARCHITECTURE.md | Architecture decisions and guidelines |
+
 ## Build & Development Commands
 This project uses a `Makefile` for build automation.
 

README.md

@@ -84,6 +84,17 @@ curl -i \
 Returns `202 Accepted` and processes invalidation asynchronously.
 Authorization is scope-based (`invalidation:write`) to support least-privilege tokens and future API growth without changing token model.
 
+## Stats API Example
+
+```bash
+curl -i \
+  "http://localhost:8082/wait0" \
+  -H "Authorization: Bearer ${WAIT0_STATS_TOKEN}"
+```
+
+Returns `200 OK` with cache/memory/refresh/sitemap metrics snapshot.
+Authorization is scope-based (`stats:read`).
+
 ## Documentation
 
 | Guide | Description |

debug/wait0.yaml

@@ -19,9 +19,10 @@ server:
 auth:
   tokens:
     - id: debug-backoffice
-      token: debug-invalidation-token
+      token: debug-token
       scopes:
         - invalidation:write
+        - stats:read
 
 urlsDiscover:
   initalDelay: '20s'

docs/api-endpoints.md

@@ -10,6 +10,7 @@ Complete HTTP endpoint reference for `wait0`.
 
 - A reverse-proxy data path for regular client requests.
 - A control endpoint for asynchronous cache invalidation.
+- A control endpoint for read-only runtime/cache statistics.
 
 Base URL examples:
 
@@ -62,7 +63,101 @@ curl -i "http://localhost:8082/"
 Typical first request: `X-Wait0: miss`.
 Subsequent request: `X-Wait0: hit`.
 
-## 2) Invalidation API
+## 2) Stats API
+
+## Route
+
+- `GET /wait0`
+- `GET /wait0/`
+
+## Auth
+
+- `Authorization: Bearer <token>` required.
+- Token must map to scope: `stats:read`.
+
+## Behavior
+
+- Returns a cached metrics snapshot (recomputed at most once every 5 seconds).
+- Designed for dashboard/backoffice polling with bounded server overhead.
+- `refresh_duration_ms` is calculated from observed revalidation execution durations (min/avg/max), not cache entry age.
+- Duration aggregates are process-lifetime metrics (since current process start).
+
+## Successful response
+
+Status: `200 OK`
+
+```json
+{
+  "generated_at": "2026-03-05T10:00:00Z",
+  "snapshot_ttl_seconds": 5,
+  "cache": {
+    "urls_total": 123,
+    "responses_size_bytes_total": 456789,
+    "response_size_bytes": {
+      "min": 128,
+      "avg": 1024,
+      "max": 4096
+    }
+  },
+  "memory": {
+    "rss_bytes": 12345678,
+    "go_alloc_bytes": 2345678
+  },
+  "refresh_duration_ms": {
+    "min": 19,
+    "avg": 66,
+    "max": 119
+  },
+  "sitemap": {
+    "discovered_urls": 80,
+    "crawled_urls": 60,
+    "crawl_percentage": 75
+  }
+}
+```
+
+## Response field reference
+
+The table below explains each field in the stats payload, including what it means and how it is computed.
+
+| Field | Type | Meaning | Calculation | Update behavior / notes |
+|------|------|---------|-------------|-------------------------|
+| `generated_at` | RFC3339Nano string | UTC timestamp when this snapshot was generated. | `time.Now().UTC()` at snapshot build time. | New value only when snapshot is recomputed. |
+| `snapshot_ttl_seconds` | integer | Snapshot cache TTL used by `/wait0`. | Fixed constant `5`. | Endpoint may return identical payload for calls within this TTL. |
+| `cache.urls_total` | integer | Total number of unique cached keys currently known to wait0. Includes active + inactive entries. | Unique union of RAM keys and disk keys. | Recomputed per snapshot. |
+| `cache.responses_size_bytes_total` | integer (bytes) | Total logical size of cached responses for all unique keys. | Sum over unique keys of per-entry logical size (`headers + body` bytes). | Recomputed per snapshot. |
+| `cache.response_size_bytes.min` | integer (bytes) | Smallest logical response size among unique cached keys. | Min of per-key logical response size. | Recomputed per snapshot; `0` when no keys. |
+| `cache.response_size_bytes.avg` | integer (bytes) | Average logical response size among unique cached keys. | `responses_size_bytes_total / urls_total` (integer division). | Recomputed per snapshot; `0` when no keys. |
+| `cache.response_size_bytes.max` | integer (bytes) | Largest logical response size among unique cached keys. | Max of per-key logical response size. | Recomputed per snapshot; `0` when no keys. |
+| `memory.rss_bytes` | integer (bytes) | Current process resident memory (RSS) as seen by OS probes. | `ProcessRSSBytes()`; `0` when unavailable on platform/runtime. | Recomputed per snapshot. |
+| `memory.go_alloc_bytes` | integer (bytes) | Current heap bytes allocated by Go runtime. | `runtime.ReadMemStats(&ms); ms.Alloc`. | Recomputed per snapshot. |
+| `refresh_duration_ms.min` | integer (ms) | Fastest observed revalidation execution time. | Min of observed `revalidation.Once(...)` durations, converted to milliseconds. | Process-lifetime aggregate since current process start. |
+| `refresh_duration_ms.avg` | integer (ms) | Average observed revalidation execution time. | Sum of all observed revalidation durations / count, converted to ms (integer division). | Process-lifetime aggregate since current process start. |
+| `refresh_duration_ms.max` | integer (ms) | Slowest observed revalidation execution time. | Max of observed `revalidation.Once(...)` durations, converted to milliseconds. | Process-lifetime aggregate since current process start. |
+| `sitemap.discovered_urls` | integer | Number of unique cached keys whose discovery source is sitemap. | Count of unique keys where `discovered_by == "sitemap"` (case-insensitive). | Recomputed per snapshot. |
+| `sitemap.crawled_urls` | integer | Number of sitemap-discovered keys that are currently active (not inactive seed entries). | Count of sitemap keys where `inactive == false`. | Recomputed per snapshot. |
+| `sitemap.crawl_percentage` | float | Share of sitemap-discovered keys currently crawled/active. | `crawled_urls * 100 / discovered_urls`; `0` if `discovered_urls == 0`. | Recomputed per snapshot. |
+
+### Additional interpretation notes
+
+- Snapshot caching: `/wait0` returns cached stats for up to `snapshot_ttl_seconds`; polling faster than TTL will often return unchanged values.
+- Lifetime vs point-in-time:
+  - `refresh_duration_ms.*` is lifetime cumulative for this process (does not reset per warmup batch).
+  - `cache.*`, `memory.*`, `sitemap.*` are point-in-time values at snapshot generation.
+- Duplicate keys across RAM and disk are deduplicated as one logical cached URL in all `cache.*` and `sitemap.*` counts.
+- Size units:
+  - `*_bytes` fields are raw bytes.
+  - `refresh_duration_ms` is milliseconds.
+
+## Error responses
+
+| HTTP | Body `error` | Cause |
+|------|--------------|-------|
+| `401` | `unauthorized` | Missing/invalid bearer token |
+| `403` | `forbidden` | Token exists but lacks `stats:read` scope |
+| `405` | `method not allowed` | Non-GET request |
+
+## 3) Invalidation API
 
 ## Route
 

docs/for-developers.md

@@ -104,6 +104,8 @@ Validation rules: all numeric values above must be `> 0`.
 
 For invalidation API, at least one token must have scope `invalidation:write` when invalidation is enabled.
 
+For stats API (`GET /wait0`), tokens need scope `stats:read`.
+
 ## `rules[]`
 
 | Field | Required | Notes |

internal/wait0/cache/disk.go

@@ -13,8 +13,12 @@ import (
 )
 
 type diskMeta struct {
-	Size       int64
-	LastAccess int64
+	Size         int64
+	LastAccess   int64
+	StatsSize    int64
+	Inactive     bool
+	DiscoveredBy string
+	LastRefresh  int64
 }
 
 type diskOp struct {
@@ -75,6 +79,30 @@ func (d *Disk) SnapshotAccessTimes() map[string]int64 {
 	return out
 }
 
+func (d *Disk) MetaSnapshot() map[string]EntryMeta {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	out := make(map[string]EntryMeta, len(d.index))
+	for k, m := range d.index {
+		lastRefresh := m.LastRefresh
+		if lastRefresh <= 0 {
+			// Backward compatibility for metadata written before LastRefresh existed.
+			lastRefresh = m.LastAccess * int64(time.Second)
+		}
+		size := m.StatsSize
+		if size <= 0 {
+			size = m.Size
+		}
+		out[k] = EntryMeta{
+			Size:                size,
+			Inactive:            m.Inactive,
+			DiscoveredBy:        m.DiscoveredBy,
+			LastRefreshUnixNano: lastRefresh,
+		}
+	}
+	return out
+}
+
 func (d *Disk) TotalSize() int64 {
 	d.mu.Lock()
 	defer d.mu.Unlock()
@@ -207,6 +235,11 @@ func (d *Disk) applyPutOrTouch(key string, ent *Entry) {
 			return
 		}
 		size := int64(len(b))
+		statsSize := EntryLogicalSize(*ent)
+		lastRefresh := ent.RevalidatedAt
+		if lastRefresh <= 0 && ent.StoredAt > 0 {
+			lastRefresh = ent.StoredAt * int64(time.Second)
+		}
 
 		d.mu.Lock()
 		old := d.index[key]
@@ -215,6 +248,10 @@ func (d *Disk) applyPutOrTouch(key string, ent *Entry) {
 		}
 		meta.Size = size
 		meta.LastAccess = now
+		meta.StatsSize = statsSize
+		meta.Inactive = ent.Inactive
+		meta.DiscoveredBy = ent.DiscoveredBy
+		meta.LastRefresh = lastRefresh
 		d.index[key] = meta
 		d.totalSize += size
 		total := d.totalSize

internal/wait0/cache/ram.go

@@ -13,6 +13,7 @@ type ramItem struct {
 	key        string
 	ent        Entry
 	size       int64
+	statsSize  int64
 	lastAccess int64
 	prev       *ramItem
 	next       *ramItem
@@ -91,6 +92,7 @@ func (c *RAM) Put(key string, ent Entry, disk *Disk, overflowLog Logger) {
 		return
 	}
 	sz := int64(len(b))
+	statsSize := EntryLogicalSize(ent)
 
 	if c.maxBytes > 0 && sz > c.maxBytes {
 		if disk != nil {
@@ -107,6 +109,7 @@ func (c *RAM) Put(key string, ent Entry, disk *Disk, overflowLog Logger) {
 		c.total -= it.size
 		it.ent = ent
 		it.size = sz
+		it.statsSize = statsSize
 		it.lastAccess = now
 		c.total += sz
 		c.moveToFront(it)
@@ -126,7 +129,7 @@ func (c *RAM) Put(key string, ent Entry, disk *Disk, overflowLog Logger) {
 		}
 	}
 
-	it := &ramItem{key: key, ent: ent, size: sz, lastAccess: now}
+	it := &ramItem{key: key, ent: ent, size: sz, statsSize: statsSize, lastAccess: now}
 	c.items[key] = it
 	c.addToFront(it)
 	c.total += sz
@@ -142,6 +145,26 @@ func (c *RAM) SnapshotAccessTimes() map[string]int64 {
 	return out
 }
 
+func (c *RAM) MetaSnapshot() map[string]EntryMeta {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	out := make(map[string]EntryMeta, len(c.items))
+	for k, it := range c.items {
+		lastRefresh := it.ent.RevalidatedAt
+		if lastRefresh <= 0 && it.ent.StoredAt > 0 {
+			lastRefresh = it.ent.StoredAt * int64(time.Second)
+		}
+		out[k] = EntryMeta{
+			Size:                it.statsSize,
+			Inactive:            it.ent.Inactive,
+			DiscoveredBy:        it.ent.DiscoveredBy,
+			LastRefreshUnixNano: lastRefresh,
+			StoredAtUnix:        it.ent.StoredAt,
+		}
+	}
+	return out
+}
+
 func (c *RAM) SetLastAccessForTest(key string, ts int64) bool {
 	c.mu.Lock()
 	defer c.mu.Unlock()

internal/wait0/cache/types.go

@@ -15,3 +15,29 @@ type Entry struct {
 	RevalidatedAt int64
 	RevalidatedBy string
 }
+
+type EntryMeta struct {
+	// Size is logical response size in bytes (headers + body).
+	Size int64
+
+	Inactive     bool
+	DiscoveredBy string
+
+	// LastRefreshUnixNano is unix nanos timestamp of latest refresh.
+	// May be zero for legacy entries.
+	LastRefreshUnixNano int64
+
+	// StoredAtUnix is unix seconds timestamp.
+	StoredAtUnix int64
+}
+
+func EntryLogicalSize(ent Entry) int64 {
+	total := int64(len(ent.Body))
+	for k, vals := range ent.Header {
+		total += int64(len(k))
+		for _, v := range vals {
+			total += int64(len(v))
+		}
+	}
+	return total
+}

internal/wait0/cache_disk.go

@@ -22,6 +22,10 @@ func (d *diskCache) SnapshotAccessTimes() map[string]int64 {
 	return d.inner.SnapshotAccessTimes()
 }
 
+func (d *diskCache) MetaSnapshot() map[string]cache.EntryMeta {
+	return d.inner.MetaSnapshot()
+}
+
 func (d *diskCache) TotalSize() int64 {
 	return d.inner.TotalSize()
 }

internal/wait0/cache_ram.go

@@ -50,6 +50,10 @@ func (c *ramCache) SnapshotAccessTimes() map[string]int64 {
 	return c.inner.SnapshotAccessTimes()
 }
 
+func (c *ramCache) MetaSnapshot() map[string]cache.EntryMeta {
+	return c.inner.MetaSnapshot()
+}
+
 func (c *ramCache) setLastAccessForTest(key string, ts int64) bool {
 	return c.inner.SetLastAccessForTest(key, ts)
 }