Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
f41089e
feat(audit): /audit dashboard with archetype classifier, scoring, and…
SiddarthAA May 27, 2026
ba6bcab
docs(audit): CHANGELOG entry + /audit dashboard section in dashboard.mdx
SiddarthAA May 27, 2026
9a0b22b
feat(audit): persona variant catalog + scroll/poster/install-CTA polish
SiddarthAA May 29, 2026
1b38daf
feat(auth): email-OTP login for CLI + dashboard, wired to failproof-a…
SiddarthAA May 31, 2026
934080b
docs(auth): add docs/cli/auth.mdx and env-vars entry for FAILPROOF_AP…
SiddarthAA May 31, 2026
34bf971
feat(ui): unify dashboard around audit pixel-craft system; fix nav st…
SiddarthAA May 31, 2026
500e97a
update global css
SiddarthAA May 31, 2026
4e0f805
feat(ui): bigger type, score+share card, persistent reminder, re-audi…
SiddarthAA May 31, 2026
1e6ccff
docs(auth): document ~/.failproofai/next-audit.json + reminder endpoint
SiddarthAA May 31, 2026
187ee90
ui fixes
SiddarthAA Jun 1, 2026
31e17cf
ui fixes
SiddarthAA Jun 1, 2026
356bd17
ui fixes
SiddarthAA Jun 1, 2026
1884dda
feat(cli): update auth cli, rename commands
SiddarthAA Jun 1, 2026
bd623b2
feat(posthog) : add posthog telemetry, update cli docs, cleanup old t…
SiddarthAA Jun 1, 2026
75eefec
feat(telemetry): instrument auth/reminder routes and wire reminder sc…
SiddarthAA Jun 3, 2026
91df2e9
docs(changelog): note auth/reminder telemetry + scheduler wiring
SiddarthAA Jun 3, 2026
4bcd6eb
- app/globals.css — added .section-h-dot + pulse keyframes (green dot…
SiddarthAA Jun 4, 2026
507f3c9
fix(audit+auth): hardening sweep across dashboard + CLI
SiddarthAA Jun 4, 2026
7c2b961
test(audit+auth): cover archetypes classifier, findings, strengths, a…
SiddarthAA Jun 4, 2026
7fa1de9
feat(telemetry): identity-link on CLI auth + policy add/remove failur…
SiddarthAA Jun 4, 2026
685937b
docs(changelog): note identity-link CLI emit + policy add/remove fail…
SiddarthAA Jun 4, 2026
78c326e
feat(telemetry): close five funnel gaps in audit-page events
SiddarthAA Jun 5, 2026
d93392e
feat(auth): default api-server base URL to https://api.befailproof.ai
SiddarthAA Jun 5, 2026
34b6c99
docs(cli): note new https://api.befailproof.ai default in env-vars index
SiddarthAA Jun 5, 2026
d8703b7
docs(changelog): credit environment-variables.mdx in api-server URL e…
SiddarthAA Jun 5, 2026
45ad9ee
Merge remote-tracking branch 'origin/main' into stable
NiveditJain Jun 7, 2026
162b414
fix(audit+auth): apply CodeRabbit suggestions across auth + audit das…
NiveditJain Jun 7, 2026
11bf8e7
docs(changelog): document CodeRabbit hardening pass
NiveditJain Jun 7, 2026
57d1092
fix(audit+auth): max-effort review + new CodeRabbit pass
NiveditJain Jun 7, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 25 additions & 8 deletions CHANGELOG.md

Large diffs are not rendered by default.

115 changes: 115 additions & 0 deletions __tests__/audit/archetypes.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
// @vitest-environment node
import { describe, it, expect } from "vitest";
import { ARCHETYPES, classifyAgent, pickArchetypeVariant } from "../../src/audit/archetypes";
import type { AuditCount, AuditResult } from "../../src/audit/types";

function mkRow(name: string, hits: number, opts: Partial<AuditCount> = {}): AuditCount {
return {
name,
source: "builtin",
category: "test",
severity: "warn",
hits,
projects: 1,
examples: [],
displayTitle: name,
impact: "",
enabledInConfig: false,
installHint: "",
...opts,
};
}

function mkResult(rows: AuditCount[]): AuditResult {
return {
version: 2,
scannedAt: "2026-06-01T00:00:00.000Z",
scope: { cli: ["claude"], projects: "all", since: null },
transcripts: { scanned: 0, skipped: 0, errors: 0, durationMs: 0 },
results: rows,
totals: { hits: rows.reduce((s, r) => s + r.hits, 0), projectsWithHits: 0 },
projectsScanned: [],
eventsScanned: 0,
enabledBuiltinNames: [],
};
}

describe("classifyAgent", () => {
it("returns precision when there is no signal at all", () => {
const cls = classifyAgent(mkResult([]));
expect(cls.archetype).toBe("precision");
expect(cls.secondary).toBe(ARCHETYPES.precision.secondary);
expect(cls.totalSignal).toBe(0);
});

it("returns precision when every row is zero hits", () => {
const cls = classifyAgent(mkResult([mkRow("failproofai/block-rm-rf", 0)]));
expect(cls.archetype).toBe("precision");
});

it("returns goldfish for broad spread (≥5 archetypes, top-3 share < 60%)", () => {
// Hand-built spread: 8 archetypes hit roughly evenly so top-3 ≤ 60%.
const cls = classifyAgent(mkResult([
mkRow("failproofai/block-rm-rf", 5), // cowboy x2.0 = 10
mkRow("failproofai/block-read-outside-cwd", 8), // explorer x1.2 = 9.6
mkRow("failproofai/warn-large-file-write", 9), // ghost x1.0 = 9
mkRow("redundant-cd-cwd", 9, { source: "audit-detector" }), // optimist x1.0 = 9
mkRow("failproofai/warn-repeated-tool-calls", 6), // hammer x1.5 = 9
mkRow("failproofai/reread-after-edit", 11), // architect x0.8 = 8.8
]));
expect(cls.archetype).toBe("goldfish");
// Secondary should be the strongest signal so the UI can hint at it.
expect(cls.secondary).toBeDefined();
});

it("promotes secondary when ≥40% of primary", () => {
const cls = classifyAgent(mkResult([
mkRow("failproofai/block-rm-rf", 5), // cowboy x2.0 = 10
mkRow("failproofai/block-env-files", 6), // explorer x1.5 = 9 (>= 40% of 10)
]));
expect(cls.archetype).toBe("cowboy");
expect(cls.secondary).toBe("explorer");
});

it("falls back to authored secondary when runner-up < 40% of primary", () => {
const cls = classifyAgent(mkResult([
mkRow("failproofai/block-rm-rf", 10), // cowboy x2.0 = 20
mkRow("failproofai/block-env-files", 1), // explorer x1.5 = 1.5 (< 40% of 20)
]));
expect(cls.archetype).toBe("cowboy");
expect(cls.secondary).toBe(ARCHETYPES.cowboy.secondary);
});

it("ignores rows whose policy name doesn't map to a signal", () => {
const cls = classifyAgent(mkResult([
mkRow("failproofai/some-future-unmapped-policy", 50),
]));
// No mapped signal → still treated as the clean baseline.
expect(cls.archetype).toBe("precision");
});

it("weights detector hits by hits × weight", () => {
const cls = classifyAgent(mkResult([
mkRow("sleep-polling-loop", 5, { source: "audit-detector" }), // hammer x1.2 = 6
]));
expect(cls.archetype).toBe("hammer");
expect(cls.weights.hammer).toBe(6);
});
});

describe("pickArchetypeVariant", () => {
it("returns the same variant for the same seed", () => {
const a = pickArchetypeVariant("optimist", "my-project");
const b = pickArchetypeVariant("optimist", "my-project");
expect(a).toEqual(b);
});

it("can return different variants for different seeds", () => {
const variants = new Set(
["a", "b", "c", "d", "e", "f"].map((s) => pickArchetypeVariant("optimist", s).tagline),
);
// Out of 6 seeds we expect at least 2 distinct taglines — the picker
// would otherwise be effectively constant.
expect(variants.size).toBeGreaterThan(1);
});
});
95 changes: 95 additions & 0 deletions __tests__/audit/dashboard-cache.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
// @vitest-environment node
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync, statSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
readDashboardCache,
writeDashboardCache,
isCacheStale,
} from "../../src/audit/dashboard-cache";
import type { AuditResult } from "../../src/audit/types";

const FAKE_RESULT: AuditResult = {
version: 2,
scannedAt: "2026-05-26T00:00:00.000Z",
scope: { cli: ["claude"], projects: "all", since: null },
transcripts: { scanned: 5, skipped: 0, errors: 0, durationMs: 100 },
results: [],
totals: { hits: 0, projectsWithHits: 0 },
projectsScanned: ["/home/u/a", "/home/u/b"],
eventsScanned: 42,
enabledBuiltinNames: ["block-failproofai-commands"],
};

describe("dashboard cache", () => {
let tmpHome: string;
let originalHome: string | undefined;

beforeEach(() => {
// Redirect homedir() to a tmp directory by overriding HOME — os.homedir()
// reads it on every call on POSIX, so the dashboard-cache module sees
// our tmp path without needing module mocks.
tmpHome = mkdtempSync(join(tmpdir(), "fpa-audit-cache-test-"));
originalHome = process.env.HOME;
process.env.HOME = tmpHome;
});

afterEach(() => {
if (originalHome === undefined) delete process.env.HOME;
else process.env.HOME = originalHome;
try { rmSync(tmpHome, { recursive: true, force: true }); } catch { /* ignore */ }
});

it("returns null when no cache file exists", () => {
expect(readDashboardCache()).toBeNull();
});

it("round-trips a written entry", () => {
writeDashboardCache({ since: "7d" }, FAKE_RESULT);
const entry = readDashboardCache();
expect(entry).not.toBeNull();
expect(entry?.params).toEqual({ since: "7d" });
expect(entry?.result.transcripts.scanned).toBe(5);
expect(entry?.result.projectsScanned).toEqual(["/home/u/a", "/home/u/b"]);
expect(typeof entry?.cachedAt).toBe("string");
});

it("writes mode 0600 on the file", () => {
writeDashboardCache({}, FAKE_RESULT);
const cachePath = join(tmpHome, ".failproofai", "audit-dashboard.json");
expect(existsSync(cachePath)).toBe(true);
const mode = statSync(cachePath).mode & 0o777;
// Some filesystems (FAT, etc.) can't honor mode bits perfectly — just
// assert no world-readable bit is set.
expect(mode & 0o004).toBe(0);
});

it("returns null for a corrupt JSON cache file", () => {
const dir = join(tmpHome, ".failproofai");
mkdirSync(dir, { recursive: true });
writeFileSync(join(dir, "audit-dashboard.json"), "{ not json", "utf-8");
expect(readDashboardCache()).toBeNull();
});

it("returns null when shape is wrong", () => {
const dir = join(tmpHome, ".failproofai");
mkdirSync(dir, { recursive: true });
writeFileSync(join(dir, "audit-dashboard.json"), JSON.stringify({ foo: 1 }), "utf-8");
expect(readDashboardCache()).toBeNull();
});

it("isCacheStale returns true past the threshold", () => {
const old = new Date(Date.now() - 60 * 60_000).toISOString(); // 1 hour ago
expect(isCacheStale(old, 30)).toBe(true);
});

it("isCacheStale returns false within the threshold", () => {
const recent = new Date(Date.now() - 10 * 60_000).toISOString(); // 10 min ago
expect(isCacheStale(recent, 30)).toBe(false);
});

it("isCacheStale treats unparseable timestamps as stale", () => {
expect(isCacheStale("not-a-date")).toBe(true);
});
});
119 changes: 119 additions & 0 deletions __tests__/audit/findings.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
// @vitest-environment node
import { describe, it, expect } from "vitest";
import { deriveFindings } from "../../src/audit/findings";
import type { AuditCount, AuditResult } from "../../src/audit/types";

function mkRow(name: string, hits: number, opts: Partial<AuditCount> = {}): AuditCount {
return {
name,
source: "builtin",
category: "test",
severity: "warn",
hits,
projects: 1,
examples: [],
displayTitle: name,
impact: "",
enabledInConfig: false,
installHint: "",
...opts,
};
}

function mkResult(rows: AuditCount[], extras: Partial<AuditResult> = {}): AuditResult {
return {
version: 2,
scannedAt: "2026-06-01T00:00:00.000Z",
scope: { cli: ["claude"], projects: "all", since: null },
transcripts: { scanned: 0, skipped: 0, errors: 0, durationMs: 0 },
results: rows,
totals: { hits: rows.reduce((s, r) => s + r.hits, 0), projectsWithHits: 0 },
projectsScanned: [],
eventsScanned: 0,
enabledBuiltinNames: [],
...extras,
};
}

describe("deriveFindings", () => {
it("ranks by hits desc and drops zero-hit rows", () => {
const cards = deriveFindings(mkResult([
mkRow("failproofai/block-rm-rf", 3),
mkRow("failproofai/block-sudo", 0), // dropped
mkRow("failproofai/block-curl-pipe-sh", 9),
]));
expect(cards.map((c) => c.sourceSlug)).toEqual([
"block-curl-pipe-sh",
"block-rm-rf",
]);
expect(cards[0].num).toBe("01");
expect(cards[1].num).toBe("02");
});

it("remaps a detector to its prescribed-fix policy slug", () => {
const [card] = deriveFindings(mkResult([
mkRow("redundant-cd-cwd", 4, { source: "audit-detector" }),
]));
expect(card.sourceSlug).toBe("redundant-cd-cwd");
expect(card.policy).toBe("warn-repeated-tool-calls");
expect(card.fix.slug).toBe("warn-repeated-tool-calls");
expect(card.fix.install).toContain("warn-repeated-tool-calls");
});

it("attaches `alsoCoveredBy` when the detector mapping carries an extra policy", () => {
const [card] = deriveFindings(mkResult([
mkRow("prefer-write-over-heredoc", 2, { source: "audit-detector" }),
]));
expect(card.fix.alsoCoveredBy).toBe("block-secrets-write");
});

it("marks the fix as already-enabled when the policy is in the enabled set", () => {
const cards = deriveFindings(mkResult(
[mkRow("redundant-cd-cwd", 4, { source: "audit-detector" })],
{ enabledBuiltinNames: ["warn-repeated-tool-calls"] },
));
expect(cards[0].alreadyEnabled).toBe(true);
});

it("marks already-enabled when a builtin row reports enabledInConfig", () => {
const [card] = deriveFindings(mkResult([
mkRow("failproofai/block-rm-rf", 1, { enabledInConfig: true }),
]));
expect(card.alreadyEnabled).toBe(true);
});

it("falls back to displayTitle/impact copy when no hand-written copy exists", () => {
const [card] = deriveFindings(mkResult([
mkRow("failproofai/some-unknown-policy", 2, {
displayTitle: "Some unknown policy",
impact: "explains the impact",
}),
]));
expect(card.body).toBe("explains the impact");
expect(card.cost).toBe("explains the impact");
});

it("injects a placeholder evidence entry when no examples were captured", () => {
const [card] = deriveFindings(mkResult([
mkRow("failproofai/block-rm-rf", 1, { examples: [] }),
]));
expect(card.evidence).toHaveLength(1);
expect(card.evidence[0].kind).toBe("comment");
});

it("renders a relative-time lastSeen", () => {
// 2h ago
const iso = new Date(Date.now() - 2 * 60 * 60_000).toISOString();
const [card] = deriveFindings(mkResult([
mkRow("failproofai/block-rm-rf", 1, { lastSeen: iso }),
]));
expect(card.lastSeen).toMatch(/^\d+h ago$/);
});

it("returns em-dash when lastSeen is missing", () => {
const [card] = deriveFindings(mkResult([
mkRow("failproofai/block-rm-rf", 1),
]));
expect(card.lastSeen).toBe("—");
});
});
53 changes: 52 additions & 1 deletion __tests__/audit/replay.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
// @vitest-environment node
import { describe, it, expect, beforeEach } from "vitest";
import { resetReplay, replayEvent } from "../../src/audit/replay";
import { resetReplay, replayEvent, initReplay, restoreReplay } from "../../src/audit/replay";
import {
clearPolicies,
getAllPolicies,
registerPolicy,
} from "../../src/hooks/policy-registry";
import { allow } from "../../src/hooks/policy-helpers";
import type { NormalizedToolEvent } from "../../src/audit/types";

function bash(command: string): NormalizedToolEvent {
Expand Down Expand Up @@ -50,3 +56,48 @@ describe("replay engine", () => {
expect(hits.some((h) => h.eventType === "PostToolUse")).toBe(true);
});
});

describe("replay registry snapshot/restore", () => {
beforeEach(() => {
resetReplay();
clearPolicies();
});

it("restoreReplay puts back the pre-init registry", () => {
registerPolicy(
"test/custom-marker",
"test policy",
async () => allow(),
{ events: ["PreToolUse"] },
);
const before = getAllPolicies().map((p) => p.name).sort();
expect(before).toContain("test/custom-marker");

initReplay();
const duringInit = getAllPolicies().map((p) => p.name);
expect(duringInit).not.toContain("test/custom-marker");
expect(duringInit.length).toBeGreaterThan(10); // builtins are loaded

restoreReplay();
const after = getAllPolicies().map((p) => p.name).sort();
expect(after).toEqual(before);
});

it("restoreReplay is idempotent when called twice", () => {
registerPolicy(
"test/another-marker",
"test policy",
async () => allow(),
{ events: ["PreToolUse"] },
);
initReplay();
restoreReplay();
restoreReplay(); // second call should be a no-op
expect(getAllPolicies().map((p) => p.name)).toContain("test/another-marker");
});

it("restoreReplay before initReplay is a no-op", () => {
expect(() => restoreReplay()).not.toThrow();
expect(getAllPolicies()).toEqual([]);
});
});
Loading