Skip to content

Commit b236e4c

Browse files
authored
telemetry: add runtime, benchmark, and sync infrastructure (#82)
* feat: add telemetry and benchmark infrastructure * ci: use available zig toolchain for bench workflow * ci: install zig directly in bench workflow * ci: fix zig archive URL in bench workflow * ci: write base benchmark artifact to workspace * ci: capture benchmark JSON explicitly * ci: run benchmark helper from PR workspace * ci: bootstrap benchmark comparison until main emits json
1 parent 5c5929a commit b236e4c

13 files changed

Lines changed: 1188 additions & 379 deletions

File tree

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
name: bench-regression
2+
3+
on:
4+
pull_request:
5+
6+
permissions:
7+
contents: read
8+
pull-requests: write
9+
10+
jobs:
11+
bench:
12+
runs-on: ubuntu-latest
13+
steps:
14+
- uses: actions/checkout@v4
15+
with:
16+
fetch-depth: 0
17+
18+
- name: Install Zig
19+
run: |
20+
curl -L https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz -o zig.tar.xz
21+
tar -xf zig.tar.xz
22+
echo "$PWD/zig-x86_64-linux-0.15.2" >> "$GITHUB_PATH"
23+
24+
- name: Benchmark head
25+
run: python3 scripts/run-bench-json.py bench-head.json
26+
27+
- name: Benchmark base
28+
run: |
29+
git fetch origin "${{ github.event.pull_request.base.ref }}" --depth=1
30+
git worktree add ../codedb-base FETCH_HEAD
31+
cd ../codedb-base
32+
if python3 "$GITHUB_WORKSPACE/scripts/run-bench-json.py" "$GITHUB_WORKSPACE/bench-base.json"; then
33+
echo "have_base_json=true" >> "$GITHUB_ENV"
34+
else
35+
echo "have_base_json=false" >> "$GITHUB_ENV"
36+
fi
37+
38+
- name: Compare
39+
if: env.have_base_json == 'true'
40+
run: |
41+
python3 scripts/compare-bench.py bench-base.json bench-head.json --threshold-pct 10 --markdown-out bench-report.md
42+
43+
- name: Bootstrap report
44+
if: env.have_base_json != 'true'
45+
run: |
46+
cat > bench-report.md <<'EOF'
47+
## Benchmark Regression Report
48+
49+
Skipped strict comparison because the base branch does not yet emit machine-readable benchmark JSON.
50+
This PR introduces the JSON benchmark format that future PRs will compare against.
51+
EOF
52+
53+
- name: Upload artifacts
54+
uses: actions/upload-artifact@v4
55+
with:
56+
name: bench-results
57+
path: |
58+
bench-base.json
59+
bench-head.json
60+
bench-report.md
61+
62+
- name: Comment PR
63+
if: github.event_name == 'pull_request'
64+
uses: actions/github-script@v7
65+
with:
66+
script: |
67+
const fs = require('fs');
68+
const body = fs.readFileSync('bench-report.md', 'utf8');
69+
await github.rest.issues.createComment({
70+
owner: context.repo.owner,
71+
repo: context.repo.repo,
72+
issue_number: context.issue.number,
73+
body,
74+
});

README.md

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -292,14 +292,19 @@ All threads share a `shutdown: atomic.Value(bool)` for graceful termination.
292292

293293
## 🔒 Data & Privacy
294294

295-
codedb is **fully local** — no telemetry, no analytics, no network calls. Nothing leaves your machine.
295+
codedb keeps runtime data local by default. Telemetry, when enabled, is written to `~/.codedb/telemetry.ndjson` on the same machine and is not uploaded automatically.
296296

297297
| Location | Contents | Purpose |
298298
|----------|----------|---------|
299299
| `~/.codedb/projects/<hash>/` | Trigram index, frequency table, data log | Persistent index cache |
300+
| `~/.codedb/telemetry.ndjson` | Aggregate tool calls and startup stats | Local telemetry log |
300301
| `./codedb.snapshot` | File tree, outlines, content, frequency table | Portable snapshot for instant MCP startup |
301302

302-
**Not stored:** No source code is sent anywhere. No network requests. No usage analytics. Sensitive files auto-excluded (`.env*`, `credentials.json`, `secrets.*`, `.pem`, `.key`, SSH keys, AWS configs).
303+
**Not stored:** No source code is sent anywhere. No file contents, file paths, or search queries are collected in telemetry. Sensitive files auto-excluded (`.env*`, `credentials.json`, `secrets.*`, `.pem`, `.key`, SSH keys, AWS configs).
304+
305+
To disable the local telemetry log entirely, set `CODEDB_NO_TELEMETRY=1`.
306+
307+
To sync the local NDJSON file into Postgres for analysis or dashboards, use [`scripts/sync-telemetry.py`](./scripts/sync-telemetry.py) with the schema in [`docs/telemetry/postgres-schema.sql`](./docs/telemetry/postgres-schema.sql). The data flow is documented in [`docs/telemetry.md`](./docs/telemetry.md).
303308

304309
```bash
305310
rm -rf ~/.codedb/ # clear all cached indexes

build.zig

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ pub fn build(b: *std.Build) void {
2121
}),
2222
});
2323

24-
2524
// ── mcp-zig dependency ──
2625
const mcp_dep = b.dependency("mcp_zig", .{});
2726
exe.root_module.addImport("mcp", mcp_dep.module("mcp"));
@@ -82,6 +81,8 @@ pub fn build(b: *std.Build) void {
8281
}),
8382
});
8483
const bench_run = b.addRunArtifact(bench);
84+
bench.root_module.addImport("mcp", mcp_dep.module("mcp"));
85+
if (b.args) |args| bench_run.addArgs(args);
8586
const bench_step = b.step("bench", "Run benchmarks");
8687
bench_step.dependOn(&bench_run.step);
8788
// Make module available so dependents don't need to wire it up manually

docs/telemetry.md

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Telemetry Data Flow
2+
3+
codedb writes local telemetry to `~/.codedb/telemetry.ndjson` unless `CODEDB_NO_TELEMETRY=1` is set. The file is append-only and stays on disk until an operator syncs it.
4+
5+
The current on-disk format is compact:
6+
7+
- `ts` or `timestamp_ms`
8+
- `ev` or `event_type`
9+
- `tool`, `ns` / `latency_ns`, `err` / `error`, `bytes` / `response_bytes`
10+
- `files` / `file_count`, `lines` / `total_lines`
11+
- optional `languages`, `index_size_bytes`, `startup_time_ms`, `version`, `platform`
12+
13+
`scripts/sync-telemetry.py` normalizes those fields and loads them into Postgres with `COPY`.
14+
15+
## Postgres schema
16+
17+
Use [`docs/telemetry/postgres-schema.sql`](./telemetry/postgres-schema.sql) to create the destination table and indexes.
18+
19+
## Sync
20+
21+
```bash
22+
python3 scripts/sync-telemetry.py --dsn "$DATABASE_URL"
23+
```
24+
25+
For a preview without touching Postgres:
26+
27+
```bash
28+
python3 scripts/sync-telemetry.py --dry-run
29+
```
30+
31+
The sync path stores aggregate usage and performance data only. It does not capture file contents, file paths, or search queries.

docs/telemetry/postgres-schema.sql

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
CREATE TABLE IF NOT EXISTS codedb_events (
2+
id BIGSERIAL PRIMARY KEY,
3+
timestamp_ms BIGINT NOT NULL,
4+
event_type TEXT NOT NULL,
5+
tool TEXT,
6+
latency_ns BIGINT,
7+
error BOOLEAN,
8+
response_bytes INTEGER,
9+
file_count INTEGER,
10+
total_lines BIGINT,
11+
languages TEXT[],
12+
index_size_bytes BIGINT,
13+
startup_time_ms BIGINT,
14+
version TEXT,
15+
platform TEXT,
16+
ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
17+
);
18+
19+
CREATE INDEX IF NOT EXISTS idx_codedb_events_timestamp_ms
20+
ON codedb_events(timestamp_ms);
21+
22+
CREATE INDEX IF NOT EXISTS idx_codedb_events_tool
23+
ON codedb_events(tool)
24+
WHERE tool IS NOT NULL;

scripts/compare-bench.py

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/usr/bin/env python3
2+
from __future__ import annotations
3+
4+
import argparse
5+
import json
6+
import sys
7+
from pathlib import Path
8+
9+
10+
def parse_args() -> argparse.Namespace:
11+
parser = argparse.ArgumentParser(description="Compare codedb benchmark JSON results.")
12+
parser.add_argument("base", help="baseline benchmark JSON")
13+
parser.add_argument("head", help="candidate benchmark JSON")
14+
parser.add_argument("--threshold-pct", type=float, default=10.0, help="maximum allowed latency regression percentage")
15+
parser.add_argument("--markdown-out", help="write markdown report to this path")
16+
return parser.parse_args()
17+
18+
19+
def load_tools(path: str) -> dict[str, dict]:
20+
data = json.loads(Path(path).read_text(encoding="utf-8"))
21+
return {tool["tool"]: tool for tool in data["tools"]}
22+
23+
24+
def pct_change(base_ns: int, head_ns: int) -> float:
25+
if base_ns == 0:
26+
return 0.0
27+
return ((head_ns - base_ns) / base_ns) * 100.0
28+
29+
30+
def render_markdown(rows: list[tuple[str, int, int, float]], threshold_pct: float) -> str:
31+
lines = [
32+
"## Benchmark Regression Report",
33+
"",
34+
f"Threshold: {threshold_pct:.2f}%",
35+
"",
36+
"| Tool | Base (ns) | Head (ns) | Delta | Status |",
37+
"| --- | ---: | ---: | ---: | --- |",
38+
]
39+
for tool, base_ns, head_ns, delta in rows:
40+
status = "FAIL" if delta > threshold_pct else "OK"
41+
lines.append(f"| `{tool}` | {base_ns} | {head_ns} | {delta:+.2f}% | {status} |")
42+
return "\n".join(lines) + "\n"
43+
44+
45+
def main() -> int:
46+
args = parse_args()
47+
base = load_tools(args.base)
48+
head = load_tools(args.head)
49+
50+
missing = sorted(set(base) ^ set(head))
51+
if missing:
52+
print(f"error: tool mismatch: {', '.join(missing)}", file=sys.stderr)
53+
return 1
54+
55+
rows: list[tuple[str, int, int, float]] = []
56+
failures: list[str] = []
57+
58+
for tool in sorted(base):
59+
base_ns = int(base[tool]["avg_latency_ns"])
60+
head_ns = int(head[tool]["avg_latency_ns"])
61+
delta = pct_change(base_ns, head_ns)
62+
rows.append((tool, base_ns, head_ns, delta))
63+
if delta > args.threshold_pct:
64+
failures.append(f"{tool} regressed by {delta:.2f}%")
65+
66+
report = render_markdown(rows, args.threshold_pct)
67+
sys.stdout.write(report)
68+
69+
if args.markdown_out:
70+
Path(args.markdown_out).write_text(report, encoding="utf-8")
71+
72+
if failures:
73+
for failure in failures:
74+
print(failure, file=sys.stderr)
75+
return 1
76+
return 0
77+
78+
79+
if __name__ == "__main__":
80+
raise SystemExit(main())

scripts/run-bench-json.py

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/usr/bin/env python3
2+
from __future__ import annotations
3+
4+
import argparse
5+
import pathlib
6+
import subprocess
7+
import sys
8+
9+
10+
def parse_args() -> argparse.Namespace:
11+
parser = argparse.ArgumentParser(description="Run `zig build bench -- --json` and persist the JSON payload.")
12+
parser.add_argument("output", help="output JSON file")
13+
return parser.parse_args()
14+
15+
16+
def extract_json(stdout: str, stderr: str) -> str:
17+
text = stdout.strip()
18+
if text.startswith("{") and text.endswith("}"):
19+
return text + "\n"
20+
21+
for stream in (stdout, stderr):
22+
for line in reversed(stream.splitlines()):
23+
line = line.strip()
24+
if line.startswith("{") and line.endswith("}"):
25+
return line + "\n"
26+
raise RuntimeError("benchmark command did not emit JSON")
27+
28+
29+
def main() -> int:
30+
args = parse_args()
31+
proc = subprocess.run(
32+
["zig", "build", "bench", "--", "--json"],
33+
capture_output=True,
34+
text=True,
35+
check=True,
36+
)
37+
if proc.stderr:
38+
sys.stderr.write(proc.stderr)
39+
payload = extract_json(proc.stdout, proc.stderr)
40+
pathlib.Path(args.output).write_text(payload, encoding="utf-8")
41+
return 0
42+
43+
44+
if __name__ == "__main__":
45+
raise SystemExit(main())

0 commit comments

Comments
 (0)