diff --git a/CHANGELOG.md b/CHANGELOG.md index c34f87d..6822b5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,59 @@ All notable changes to fs-cortex will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [4.1.0] — 2026-07-04 + +**Audit hardening.** Fixes and hygiene from a multi-agent read-only audit of the +live system (laws, instincts, proposals, storage, hooks, reflexes, injection +overhead). Adversarial verification kept 34 findings and refuted 6; three more +were rejected here as misdiagnoses (see Notes). + +### Fixed +- **SessionStart injected only the first line of each law** (`session-start.py` + `load_laws`): now reads the full law text (cap 1000 chars) so multi-line + checklists and exclusions actually reach context. (P1) +- **Truncated laws restored**: five laws that ended mid-sentence with `…` + rewritten to complete text from their backing instinct action; + `meta-broad-trigger-instinct-noise` moved out of `laws/` (engine + meta-diagnosis, not an operating law) — active law count 15 → 14. (P1) +- **Rejected proposals re-entering the backlog** (`session-learner.js`): + the tombstone gate now matches by trigger (not just id) against + `proposals-history.jsonl`; the live backlog was purged of 8 already-rejected + entries. (P1) +- **`python3-bypass-write-tool` reflex re-enabled** — a high-severity safety + reflex was disabled, leaving a known Write/Edit bypass unguarded. (P1) +- **`bash-cat-use-read` reflex noise** (`core/reflexes.default.json`): condition + and `evaluator.anti_pattern` hardened with a negative lookahead so piped and + heredoc `cat` no longer false-positive. (P1) +- **`cross-day-tracker.jsonl` never pruned** (`storage-rotation.js`): a file past + 1.5× its threshold now prunes regardless of the 24h rotation gate. (P1) + +### Added +- `storage-rotation.js`: stale `.lock` and `.bak`/`.backup` cleanup (>30d), + `impact.archive/` retention (keep 5), and `log/timeline.jsonl` rotation + (keep last 1000, archive the rest). (P2) +- `distill_engine.py`: `prune_instinct_tracking()` (drops tracking entries with + no backing instinct) and `reap_stale_nudge_state()` (decays stale/saturated + nudge entries by 0.10), both wired into the maintain pipeline and exposed as + `prune-tracking` / `reap-nudges` CLI subcommands. (P1/P2) +- `session-learner.js`: duplicate-trigger dedup for proposals, `$HOME`→`~` + normalization of hardcoded paths in `action`, and head+tail sampling instead + of a hard 200-char cut. (P2) +- `injector-engine.js`: session token budget raised 8000 → 12000 with a visible + suppression warning (was silent, debug-gated), plus optional per-reflex + `domains` filtering (no-op when the field is absent). (P2) + +### Notes +- **Audit recommendations rejected as misdiagnoses:** (1) moving five + `scope: global` instincts into project dirs — conflated provenance + (`project_id`) with injection targeting (`scope`); they are trigger-gated + cross-project patterns and stay global. (2) Lowering `NUDGE_MAX_CONF` + 0.99 → 0.80 — would make law auto-promotion's `conf >= 0.95` unreachable via + nudging; the real staleness problem is handled by `reap_stale_nudge_state`. + (3) Tightening per-domain instinct dedup — the proposed gap>0.10 exception + produced inconsistent behavior (kept a low-confidence second instinct when the + gap was large); reverted to the original max-2-per-domain-if-both-≥0.85 rule. + ## [4.0.0] — 2026-07-02 **"Signal-first, zero-decision."** Full design rationale in `docs/DESIGN-V4.md` diff --git a/core/reflexes.default.json b/core/reflexes.default.json index eded701..a60e6d7 100644 --- a/core/reflexes.default.json +++ b/core/reflexes.default.json @@ -140,7 +140,7 @@ { "id": "bash-cat-use-read", "matcher": "Bash", - "condition": "(?:^|[;&|]\\s*)(cat|head|tail)\\s+(-[0-9]+\\s+)?[\\.\\/~]?\\S*\\.(py|js|jsx|ts|tsx|md|json|jsonl|ya?ml|sh|html|css|toml|cfg|ini|conf|sql|env)\\b", + "condition": "(?:^|[;&|]\\s*)(cat|head|tail)\\s+(-[0-9]+\\s+)?[\\.\\/~]?\\S*\\.(py|js|jsx|ts|tsx|md|json|jsonl|ya?ml|sh|html|css|toml|cfg|ini|conf|sql|env)\\b(?!\\s*(\\||<<))", "action": "Use Read tool (with limit/offset) instead of cat/head/tail for source files.", "severity": "medium", "enabled": true, @@ -152,7 +152,7 @@ "type": "tool-substitution", "expected_tool": "Read", "anti_tool": "Bash", - "anti_pattern": "(?:^|[;&|]\\s*)(cat|head|tail)\\s+(-[0-9]+\\s+)?[\\.\\/~]?\\S*\\.(py|js|jsx|ts|tsx|md|json|jsonl|ya?ml|sh|html|css|toml|cfg|ini|conf|sql|env)\\b", + "anti_pattern": "(?:^|[;&|]\\s*)(cat|head|tail)\\s+(-[0-9]+\\s+)?[\\.\\/~]?\\S*\\.(py|js|jsx|ts|tsx|md|json|jsonl|ya?ml|sh|html|css|toml|cfg|ini|conf|sql|env)\\b(?!\\s*(\\||<<))", "window": 3 } }, diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 8472252..aa3afd2 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -1,4 +1,4 @@ -# fs-cortex v4.0.0 — Feature Reference +# fs-cortex v4.1.0 — Feature Reference > Complete inventory of all features, commands, hooks, modules, and capabilities. > Last updated: 2026-07-02 diff --git a/hooks/lib/distill_engine.py b/hooks/lib/distill_engine.py index d6d7eec..38b4150 100644 --- a/hooks/lib/distill_engine.py +++ b/hooks/lib/distill_engine.py @@ -80,11 +80,27 @@ # injector-engine.js writes here on every PreToolUse match — dedup'd, cap 20 # entries per instinct. INSTINCT_TRACKING_FILE = CORTEX_DIR / "instinct-tracking.json" +# Written by impact_log.py's apply_outcome_nudges (schema v2, see that +# module's _load_nudge_state docstring). Read/pruned here too (reap_stale_ +# nudge_state) via its own path constant rather than importing impact_log, +# to keep this module's write surface self-contained. +NUDGE_STATE_FILE = CORTEX_DIR / "nudge-state.json" RATE_LIMIT_HOURS = 24 DECAY_PER_30_DAYS = 0.05 DECAY_PERIOD_DAYS = 30 ARCHIVE_THRESHOLD = 0.10 +# nudge-state.json reaper (P2 audit 2026-07-04): a saturated instinct +# (last_direction == "saturated", i.e. it hit NUDGE_MAX_CONF and stopped +# receiving nudges) with no fresh outcome cohort for NUDGE_STALE_DAYS is +# stuck at ceiling confidence forever, since apply_outcome_nudges only +# advances it when a new cohort arrives. reap_stale_nudge_state() applies +# a one-off decay so it re-enters the normal decay/nudge cycle instead of +# being cemented at 0.99. Entries whose last_event_ts itself is older than +# NUDGE_STALE_DAYS (stale regardless of saturation) get the same treatment. +NUDGE_STALE_DAYS = 60 +NUDGE_SATURATED_STALE_DAYS = 7 +NUDGE_REAP_DECAY = 0.10 LAW_THRESHOLD_CONF = 0.95 LAW_SUSTAINED_DAYS = 14 LAW_MIN_PROJECTS = 3 # v4 (2026-07-02, DESIGN-V4.md §3): restored 1 → 3. @@ -2332,6 +2348,163 @@ def _write_marker() -> None: MARKER_FILE.write_text(_dt.datetime.now(_dt.timezone.utc).isoformat(), encoding="utf-8") +def prune_instinct_tracking(dry_run: bool = False) -> dict: + """Drop instinct-tracking.json entries whose instinct no longer exists. + + injector-engine.js appends to this file on every PreToolUse match but + never removes an entry when the backing YAML is archived, promoted to + a law, or manually deleted — so the file grows with dead tracking data + forever (P2 audit 2026-07-04). An id counts as "alive" if an active + (non-archived) instinct YAML with that id exists in global/ or any + projects/*/instincts/ — same definition _all_instinct_paths() already + uses everywhere else in this module. + + Returns {"before": int, "after": int, "pruned": int}. + """ + tracking = _load_instinct_tracking() + if not tracking: + return {"before": 0, "after": 0, "pruned": 0} + + alive_ids = {p.stem for p in _all_instinct_paths()} + before = len(tracking) + kept = {iid: rec for iid, rec in tracking.items() if iid in alive_ids} + pruned = before - len(kept) + + if pruned and not dry_run: + _atomic_write(INSTINCT_TRACKING_FILE, json.dumps(kept, indent=2, ensure_ascii=False) + "\n") + _log_knowledge( + "pruned-tracking", "-", + f"removed {pruned} dead instinct-tracking.json entries (no backing yaml)", + source="cx-auto-distill", + ) + + return {"before": before, "after": len(kept), "pruned": pruned} + + +def reap_stale_nudge_state(dry_run: bool = False) -> dict: + """Un-cement saturated instincts stuck at NUDGE_MAX_CONF (0.99) forever. + + apply_outcome_nudges() (hooks/lib/impact_log.py) only advances an iid's + nudge-state entry when a NEW outcome cohort arrives; a high-confidence + instinct that stops generating fresh outcome feedback stays saturated + at conf 0.99 with a frozen `last_direction: "saturated"` marker + indefinitely (P2 audit 2026-07-04). This reaper, meant to be invoked + from maintain alongside the rest of the auto-distill pipeline, applies + a one-off -NUDGE_REAP_DECAY (0.10) to the backing instinct YAML's + confidence and resets the nudge-state entry's direction so the + instinct re-enters the normal decay/nudge cycle instead of being + cemented, for any entry where: + - last_event_ts is older than NUDGE_STALE_DAYS (60d), regardless of + saturation, OR + - last_direction == "saturated" AND last_nudge_ts is older than + NUDGE_SATURATED_STALE_DAYS (7d) + + Reads/writes ~/.claude/cortex/nudge-state.json directly (own path + constant, no import of impact_log) to keep this module's write + surface self-contained; the schema (version 2, `iids: {: {...}}`) + is impact_log.py's, treated here as a stable on-disk contract. + + Returns {"before": int, "reaped": int} where "before" is the number of + entries considered saturated/stale-eligible before reaping. + """ + if not NUDGE_STATE_FILE.exists(): + return {"before": 0, "reaped": 0} + try: + state = json.loads(NUDGE_STATE_FILE.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {"before": 0, "reaped": 0} + if not isinstance(state, dict): + return {"before": 0, "reaped": 0} + iids_state = state.get("iids") + if not isinstance(iids_state, dict) or not iids_state: + return {"before": 0, "reaped": 0} + + now = _dt.datetime.now(_dt.timezone.utc) + stale_cutoff = now - _dt.timedelta(days=NUDGE_STALE_DAYS) + saturated_cutoff = now - _dt.timedelta(days=NUDGE_SATURATED_STALE_DAYS) + + def _parse_ts(raw: str) -> _dt.datetime | None: + try: + return _dt.datetime.strptime(raw[:19], "%Y-%m-%dT%H:%M:%S").replace( + tzinfo=_dt.timezone.utc + ) + except (ValueError, TypeError): + return None + + eligible: list[str] = [] + for iid, rec in iids_state.items(): + if not isinstance(rec, dict): + continue + last_event = _parse_ts(str(rec.get("last_event_ts", ""))) + last_nudge = _parse_ts(str(rec.get("last_nudge_ts", ""))) + is_stale_event = last_event is not None and last_event < stale_cutoff + is_stale_saturated = ( + str(rec.get("last_direction", "")) == "saturated" + and last_nudge is not None + and last_nudge < saturated_cutoff + ) + if is_stale_event or is_stale_saturated: + eligible.append(iid) + + before = len(eligible) + if not eligible: + return {"before": 0, "reaped": 0} + + reaped = 0 + if dry_run: + return {"before": before, "reaped": 0} + + # Build an id → path index once, reusing the same discovery already + # used for tracking/decay elsewhere in this module. + path_by_id = {p.stem: p for p in _all_instinct_paths()} + today_iso = now.isoformat() + + for iid in eligible: + path = path_by_id.get(iid) + if path is None: + # No backing YAML (already archived/promoted) — just drop the + # stale nudge-state entry, nothing to decay. + iids_state.pop(iid, None) + reaped += 1 + continue + result = _read_instinct(path) + if result is None: + continue + fields, text = result + conf = fields.get("confidence") + try: + conf = float(conf) + except (TypeError, ValueError): + continue + new_conf = max(0.0, round(conf - NUDGE_REAP_DECAY, 4)) + new_text = re.sub( + r'^(confidence\s*:\s*)["\']?[\d.]+["\']?\s*$', + lambda m: f"confidence: {new_conf:.4f}", + text, + count=1, + flags=re.MULTILINE, + ) + _atomic_write(path, new_text) + iids_state[iid] = { + **iids_state.get(iid, {}), + "last_direction": "reaped", + "last_nudge_ts": today_iso, + "conf_at_last_nudge": new_conf, + } + _log_knowledge( + "reaped-nudge", iid, + f"conf {conf:.4f} → {new_conf:.4f} (stale nudge-state, -{NUDGE_REAP_DECAY:.2f})", + source="cx-auto-distill", + ) + reaped += 1 + + if reaped: + _save = json.dumps(state, indent=2, ensure_ascii=False) + "\n" + _atomic_write(NUDGE_STATE_FILE, _save) + + return {"before": before, "reaped": reaped} + + def _prune_cross_day_tracker(): """Prune entries older than 365 days from cross-day-tracker.jsonl.""" tracker_path = CORTEX_DIR / "cross-day-tracker.jsonl" @@ -2450,6 +2623,12 @@ def run_auto_distill(dry_run: bool = False) -> dict: prune_result = _prune_cross_day_tracker() if prune_result["pruned"] > 0: print(f"Pruned {prune_result['pruned']} cross-day-tracker entries (>365d)") + tracking_prune_result = prune_instinct_tracking() + if tracking_prune_result["pruned"] > 0: + print(f"Pruned {tracking_prune_result['pruned']} dead instinct-tracking.json entries") + reap_result = reap_stale_nudge_state() + if reap_result["reaped"] > 0: + print(f"Reaped {reap_result['reaped']} stale/saturated nudge-state entries") _write_marker() return { @@ -2758,6 +2937,21 @@ def _cmd_decay(dry_run: bool) -> None: print(f" {c['id']}: {c['old_conf']:.4f} → {c['new_conf']:.4f} ({c['days_unused']}d unused)") +def _cmd_prune_tracking(dry_run: bool) -> None: + result = prune_instinct_tracking(dry_run=dry_run) + prefix = "[DRY-RUN] " if dry_run else "" + print(f"{prefix}instinct-tracking.json: {result['before']} → {result['after']} " + f"({result['pruned']} pruned, no backing instinct)") + + +def _cmd_reap_nudges(dry_run: bool) -> None: + result = reap_stale_nudge_state(dry_run=dry_run) + prefix = "[DRY-RUN] " if dry_run else "" + print(f"{prefix}nudge-state.json: {result['before']} stale/saturated entr" + f"{'y' if result['before'] == 1 else 'ies'} found, {result['reaped']} reaped " + f"(-{NUDGE_REAP_DECAY:.2f} conf, direction reset)") + + def _cmd_promote(dry_run: bool) -> None: promoted, candidates = auto_promote_to_law(dry_run=dry_run) prefix = "[DRY-RUN] " if dry_run else "" @@ -3059,6 +3253,18 @@ def main(argv: list[str] | None = None) -> int: p_promote = sub.add_parser("promote", help="Check promotion candidates") p_promote.add_argument("--dry-run", action="store_true") + p_prune_tracking = sub.add_parser( + "prune-tracking", + help="Remove instinct-tracking.json entries with no backing instinct YAML", + ) + p_prune_tracking.add_argument("--dry-run", action="store_true") + + p_reap_nudges = sub.add_parser( + "reap-nudges", + help="Decay stale/saturated nudge-state.json entries so they re-enter the nudge cycle", + ) + p_reap_nudges.add_argument("--dry-run", action="store_true") + p_backfill = sub.add_parser("backfill", help="Recover legacy session_id/session fields and rebuild tracking sessions (preview by default; --apply writes)") p_backfill.add_argument("--apply", action="store_true", help="Write changes (atomically + cross-runtime locked since v3.35.0 / issue #49)") @@ -3085,6 +3291,10 @@ def main(argv: list[str] | None = None) -> int: _cmd_decay(args.dry_run) elif args.cmd == "promote": _cmd_promote(args.dry_run) + elif args.cmd == "prune-tracking": + _cmd_prune_tracking(args.dry_run) + elif args.cmd == "reap-nudges": + _cmd_reap_nudges(args.dry_run) elif args.cmd == "backfill": _cmd_backfill(args.apply) elif args.cmd == "status": diff --git a/hooks/lib/injector-engine.js b/hooks/lib/injector-engine.js index 5727708..cd5fb02 100644 --- a/hooks/lib/injector-engine.js +++ b/hooks/lib/injector-engine.js @@ -141,6 +141,20 @@ function main() { try { injectedCounts = JSON.parse(fs.readFileSync(INJECTED_COUNTS_FILE, "utf8")) || {}; } catch {} const suppressed = []; // {id} — repeats + budget drops, logged as suppress events + // Project + domain detection hoisted above reflex matching (was section 2b) + // so reflexes can domain-filter too (audit P2 reflex-no-domain-filter). + // detectProjectDomains/_scanDir/etc. are function declarations further + // below in this same function body — hoisted, safe to call here. Their + // const dependencies (DOMAIN_CACHE_FILE etc.) are NOT hoisted, so those + // are hoisted up here too (still declared once, no duplicate below). + const DOMAIN_CACHE_FILE = path.join(CORTEX_DIR, ".project-domains-cache"); + const DOMAIN_CACHE_TTL_MS = 5 * 60 * 1000; + const WORKSPACE_MARKERS = ["pnpm-workspace.yaml", "turbo.json", "nx.json", "lerna.json", "rush.json", "workspace.json"]; + const SKIP_DIRS = new Set(["node_modules", ".git", ".venv", "venv", "target", "dist", "build", ".next", ".nuxt", ".turbo", ".cache", "__pycache__"]); + + const { id: projectId, root: projectRoot } = detectProject(cwd); + const projectDomains = detectProjectDomains(projectRoot); + // ── 1. Load and match reflexes ─────────────────────────────────────── const reflexesFile = process.env._CX_REFLEXES_FILE; @@ -151,6 +165,14 @@ function main() { const MAX_REFLEXES = memoryConfig.max_reflexes_per_injection || 2; for (const r of reflexes) { if (!r.enabled) continue; + // v3.37.x (audit P2 reflex-no-domain-filter): optional scope/domains + // field on a reflex — skip when it doesn't match the detected + // project domain(s). No-op (passes through) when the field is + // absent, so existing reflexes without it keep firing globally. + if (r.domains && Array.isArray(r.domains) && r.domains.length > 0) { + const matchesDomain = r.domains.some(d => projectDomains.has(d)); + if (!matchesDomain) continue; + } if (!r.matcher || !safeRegexTest(r.matcher, toolName, { tag: `reflex:${r.id}:matcher` })) continue; if (r.condition && !safeRegexTest(r.condition, toolInputStr, { tag: `reflex:${r.id}:condition` })) continue; if ((injectedCounts["reflex:" + r.id] || 0) >= MAX_REPEAT_INJECTIONS) { @@ -174,7 +196,6 @@ function main() { instinctFiles.push(...listYamlFiles(globalDir)); } - const { id: projectId, root: projectRoot } = detectProject(cwd); if (projectId) { const projectDir = path.join(CORTEX_DIR, "projects", projectId, "instincts"); instinctFiles.push(...listYamlFiles(projectDir)); @@ -191,11 +212,8 @@ function main() { // 4. Cached 5 min in ~/.claude/cortex/.project-domains-cache (JSON). // // Always short-circuits at depth 3 to avoid scanning node_modules / .venv. - - const DOMAIN_CACHE_FILE = path.join(CORTEX_DIR, ".project-domains-cache"); - const DOMAIN_CACHE_TTL_MS = 5 * 60 * 1000; - const WORKSPACE_MARKERS = ["pnpm-workspace.yaml", "turbo.json", "nx.json", "lerna.json", "rush.json", "workspace.json"]; - const SKIP_DIRS = new Set(["node_modules", ".git", ".venv", "venv", "target", "dist", "build", ".next", ".nuxt", ".turbo", ".cache", "__pycache__"]); + // DOMAIN_CACHE_FILE / DOMAIN_CACHE_TTL_MS / WORKSPACE_MARKERS / SKIP_DIRS + // are declared earlier in this function (hoisted above reflex matching). function _inspectPackageJson(file, domains) { try { @@ -296,7 +314,7 @@ function main() { return domains; } - const projectDomains = detectProjectDomains(projectRoot); + // projectDomains already computed above (hoisted ahead of reflex matching). // ── 3. Parse, filter, match instincts ──────────────────────────────── @@ -514,7 +532,11 @@ function main() { // ── 4. Token budget cap ────────────────────────────────────────────── const SESSION_BUDGET_FILE = path.join(CORTEX_DIR, ".session-token-budget"); - const MAX_SESSION_TOKENS = 8000; + // v3.37.x (audit P2 budget-too-low): 8000 was tripping mid-session on + // ordinary work, silently dropping instincts with only a debug-gated log + // line as evidence. Raised to 12000 and paired with a visible warning + // (below) so the suppression is no longer invisible outside CORTEX_DEBUG. + const MAX_SESSION_TOKENS = 12000; let sessionTokens = 0; try { sessionTokens = parseInt(fs.readFileSync(SESSION_BUDGET_FILE, "utf8").trim(), 10) || 0; } catch {} @@ -522,14 +544,16 @@ function main() { const reflexTokens = matchedReflexes.reduce((sum, r) => sum + Math.ceil(r.action.length / 4) + 15, 0); // v3.37.0 — degrade instead of flush. The old killswitch zeroed the whole - // instinct batch silently once the session crossed 8000 tokens, which is - // one of the reasons "Cortex stopped learning": late-session matches never - // reached the context. matchedInstincts is confidence-desc, so pop() + // instinct batch silently once the session crossed the token budget, which + // is one of the reasons "Cortex stopped learning": late-session matches + // never reached the context. matchedInstincts is confidence-desc, so pop() // drops the weakest first. + let budgetDroppedCount = 0; while (matchedInstincts.length > 0 && sessionTokens + reflexTokens + instinctTokens > MAX_SESSION_TOKENS) { const droppedInst = matchedInstincts.pop(); suppressed.push({ id: droppedInst.id }); + budgetDroppedCount++; instinctTokens -= Math.ceil(droppedInst.action.length / 4) + 20; if (process.env.CORTEX_DEBUG) { process.stderr.write("[cortex:injector] token budget (" + sessionTokens + "/" + MAX_SESSION_TOKENS + "): dropped " + droppedInst.id + "\n"); @@ -637,7 +661,7 @@ function main() { // ── 5. Build output ────────────────────────────────────────────────── - if (matchedReflexes.length === 0 && matchedInstincts.length === 0) { + if (matchedReflexes.length === 0 && matchedInstincts.length === 0 && budgetDroppedCount === 0) { process.exit(0); } @@ -651,6 +675,12 @@ function main() { lines.push("[instinct:" + inst.id + "] " + inst.action + " (conf:" + inst.confidence.toFixed(2) + ")"); } + // v3.37.x (audit P2 budget-too-low): make budget suppression visible + // outside CORTEX_DEBUG — it used to be a silent .json log entry only. + if (budgetDroppedCount > 0) { + lines.push("[CORTEX] presupuesto de tokens superado (" + (sessionTokens + reflexTokens + instinctTokens) + "/" + MAX_SESSION_TOKENS + ") — " + budgetDroppedCount + " instinct(s) suprimido(s)"); + } + const output = { hookSpecificOutput: { hookEventName: "PreToolUse", diff --git a/hooks/lib/storage-rotation.js b/hooks/lib/storage-rotation.js index 46e2256..dbfee75 100644 --- a/hooks/lib/storage-rotation.js +++ b/hooks/lib/storage-rotation.js @@ -38,6 +38,12 @@ const KNOWLEDGE_ROTATE_MB = parseFloat(process.env.CORTEX_KNOWLEDGE_ROTATE_MB || // survives. const DAILY_KEEP_FILES = Math.max(1, parseInt(process.env.CORTEX_DAILY_KEEP_FILES || '60', 10) || 60); const FIREONCE_MAX_DAYS = Math.max(1, parseInt(process.env.CORTEX_FIREONCE_MAX_DAYS || '30', 10) || 30); +// v3.37.0 (audit 2026-07-04) — impact.archive/ (rotated impact.jsonl chunks) +// and log/timeline.jsonl had no prune path: 105MB archive dir, 5.1MB/79k-line +// timeline observed. +const IMPACT_ARCHIVE_KEEP_FILES = Math.max(1, parseInt(process.env.CORTEX_IMPACT_ARCHIVE_KEEP_FILES || '5', 10) || 5); +const TIMELINE_ROTATE_MB = parseFloat(process.env.CORTEX_TIMELINE_ROTATE_MB || '2'); +const TIMELINE_KEEP_LINES = Math.max(1, parseInt(process.env.CORTEX_TIMELINE_KEEP_LINES || '1000', 10) || 1000); const MARKER_NAME = '.last-storage-rotate'; const DAY_MS = 24 * 3600 * 1000; @@ -129,12 +135,86 @@ function _pruneFireOnceMarkers(log) { if (removed > 0) log(`Storage rotation: .fire-once pruned ${removed} marker(s) older than ${FIREONCE_MAX_DAYS}d`); } +// v3.37.0 (audit 2026-07-04) — stale lock files (writer crashed/killed before +// release) and .bak/.backup snapshots left by ad-hoc edits. Walk CORTEX_DIR +// one level plus known subdirectory groups (projects/*) rather than a full +// recursive scan, matching this file's existing flat-tree assumptions. +const STALE_FILE_MAX_DAYS = 30; + +function _walkFilesShallow(dir, depth) { + let out = []; + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch (_) { return out; } + for (const e of entries) { + const full = path.join(dir, e.name); + if (e.isFile()) { out.push(full); continue; } + if (e.isDirectory() && depth > 0) { out = out.concat(_walkFilesShallow(full, depth - 1)); } + } + return out; +} + +function _pruneStaleLocks(log) { + const cutoff = Date.now() - STALE_FILE_MAX_DAYS * DAY_MS; + let removed = 0; + for (const full of _walkFilesShallow(CORTEX_DIR, 3)) { + if (!full.endsWith('.lock')) continue; + try { + if (fs.statSync(full).mtimeMs < cutoff) { fs.unlinkSync(full); removed++; } + } catch (_) {} + } + if (removed > 0) log(`Storage rotation: stale locks pruned ${removed} .lock file(s) older than ${STALE_FILE_MAX_DAYS}d`); +} + +function _pruneBackupFiles(log) { + const cutoff = Date.now() - STALE_FILE_MAX_DAYS * DAY_MS; + let removed = 0; + for (const full of _walkFilesShallow(CORTEX_DIR, 3)) { + if (!/\.(bak|backup)$/.test(full)) continue; + try { + if (fs.statSync(full).mtimeMs < cutoff) { fs.unlinkSync(full); removed++; } + } catch (_) {} + } + if (removed > 0) log(`Storage rotation: .bak/.backup pruned ${removed} file(s) older than ${STALE_FILE_MAX_DAYS}d`); +} + +// Keep the last `keepLines` JSONL entries, archiving the rest (rename-rotate +// style: write the trimmed tail to a temp file, rename over the live file, +// archive the original next to it). No dependency on external tools. +function _rotateJsonlKeepLast(file, archiveDir, keepLines, maxMb, log, label) { + if (fileMb(file) < maxMb) return; + let lines; + try { lines = fs.readFileSync(file, 'utf8').split('\n'); } catch (_) { return; } + if (lines.length && lines[lines.length - 1] === '') lines.pop(); + if (lines.length <= keepLines) return; + try { fs.mkdirSync(archiveDir, { recursive: true, mode: 0o700 }); } catch (_) {} + const base = path.basename(file); + const dest = path.join(archiveDir, `${base}.${_stamp()}-${process.pid}`); + try { + fs.renameSync(file, dest); + const tail = lines.slice(-keepLines).join('\n') + '\n'; + fs.writeFileSync(file, tail, { mode: 0o600 }); + log(`Storage rotation: ${label} kept last ${keepLines} of ${lines.length} lines, rest archived → ${path.basename(archiveDir)}/`); + } catch (e) { + log(`Storage rotation: ${label} rotate error: ${e.message}`); + } +} + function maybeRotateStorage(log) { log = log || (() => {}); const marker = path.join(CORTEX_DIR, MARKER_NAME); try { - if (Date.now() - fs.statSync(marker).mtimeMs < DAY_MS) { - return { ran: false }; + const gateAgeMs = Date.now() - fs.statSync(marker).mtimeMs; + if (gateAgeMs < DAY_MS) { + // P1-7 (audit 2026-07-04): a file that already blew past its own + // threshold by 1.5x must not wait out the full 24h gate — sessions + // spaced >24h apart let it grow unbounded. Early-trigger on size alone. + const trackerMbNow = (() => { + try { return fileMb(require(path.join(__dirname, 'cross-day-tracker')).TRACKER_PATH); } catch (_) { return 0; } + })(); + if (trackerMbNow < TRACKER_PRUNE_MB * 1.5) { + return { ran: false }; + } + log(`Storage rotation: gate bypassed — cross-day-tracker.jsonl ${trackerMbNow.toFixed(1)}MB ≥ ${(TRACKER_PRUNE_MB * 1.5).toFixed(1)}MB (1.5x threshold)`); } } catch (_) { /* no marker yet — proceed */ } @@ -164,6 +244,13 @@ function maybeRotateStorage(log) { log(`Storage rotation: impact rotate error: ${e.message}`); } } + // impact.archive/ itself is unbounded (each rotate adds a file, never + // removed) — keep only the newest IMPACT_ARCHIVE_KEEP_FILES. + try { + _pruneDirByCount(path.join(CORTEX_DIR, 'impact.archive'), IMPACT_ARCHIVE_KEEP_FILES, log, 'impact.archive'); + } catch (e) { + log(`Storage rotation: impact.archive prune error: ${e.message}`); + } // cross-day-tracker.jsonl — prune >365d + same-day duplicate compaction try { @@ -208,6 +295,21 @@ function maybeRotateStorage(log) { try { _pruneFireOnceMarkers(log); } catch (e) { log(`Storage rotation: fire-once prune error: ${e.message}`); } + try { _pruneStaleLocks(log); } catch (e) { + log(`Storage rotation: stale lock prune error: ${e.message}`); + } + try { _pruneBackupFiles(log); } catch (e) { + log(`Storage rotation: backup prune error: ${e.message}`); + } + try { + _rotateJsonlKeepLast( + path.join(CORTEX_DIR, 'log', 'timeline.jsonl'), + path.join(CORTEX_DIR, 'log', 'timeline.archive'), + TIMELINE_KEEP_LINES, TIMELINE_ROTATE_MB, log, 'log/timeline.jsonl' + ); + } catch (e) { + log(`Storage rotation: timeline rotate error: ${e.message}`); + } try { fs.writeFileSync(marker, new Date().toISOString() + '\n', { mode: 0o600 }); } catch (_) {} return result; @@ -216,4 +318,5 @@ function maybeRotateStorage(log) { module.exports = { maybeRotateStorage, IMPACT_ROTATE_MB, TRACKER_PRUNE_MB, MARKER_NAME, HISTORY_ROTATE_MB, KNOWLEDGE_ROTATE_MB, DAILY_KEEP_FILES, FIREONCE_MAX_DAYS, + IMPACT_ARCHIVE_KEEP_FILES, TIMELINE_ROTATE_MB, TIMELINE_KEEP_LINES, }; diff --git a/hooks/session-learner.js b/hooks/session-learner.js index 685a7cf..345b7f2 100755 --- a/hooks/session-learner.js +++ b/hooks/session-learner.js @@ -241,7 +241,16 @@ function resolveProjectAndObservations(stdinData) { // Sanitize text used in proposal actions against prompt injection function sanitizeProposalAction(text) { const BLOCKED = /\b(ignore|forget|override|disregard|bypass|system\s*:|you\s+are|all\s+previous|new\s+instructions|do\s+not\s+follow)\b/gi; - return String(text) + // v4 (audit P2) — collapse this machine's $HOME prefix to `~` so an action + // like "cd /Users/fer/github/LinkedIn/app && ..." doesn't hardcode an + // operator-specific absolute path into a proposal meant to be reusable + // across machines/projects. Only the known HOME prefix is touched; a full + // repo-root→{PROJECT_ROOT} placeholder is out of scope here (no reliable + // way to know the project root from this string alone). + const homeNormalized = HOME + ? String(text).split(HOME).join('~') + : String(text); + return homeNormalized .replace(/[\x00-\x1f\x7f]/g, '') .replace(BLOCKED, '[BLOCKED]') .slice(0, 300); @@ -271,6 +280,23 @@ function isLowQualityAction(action) { return false; } +// v4 (audit P2) — sample_input/sample_output used to be a hard cut to 200 +// chars, which for multi-line commands/output kept only the beginning and +// silently dropped the tail where the actual failure/result often lives. +// Keep the first line plus the last 3 lines (Sinapsis-style head+tail); if +// the content is short (<=200 chars) or single-line, return it unchanged. +const SAMPLE_MAX_CHARS = 200; +function headTailSample(text) { + const s = String(text || ''); + if (s.length <= SAMPLE_MAX_CHARS) return s; + const lines = s.split('\n'); + if (lines.length === 1) return s.slice(0, SAMPLE_MAX_CHARS); + const head = lines[0]; + const tail = lines.slice(-3); + const combined = [head, '…', ...tail].join('\n'); + return combined; +} + // Guards against err_msg that is actually a header/listing line rather than // a real error (belt-and-braces on top of observe.py's ERROR_PATTERNS // guards per DESIGN-V4.md §1: grep `===== file =====` headers, `ls -l` @@ -497,8 +523,8 @@ function detectErrorResolutions(observations) { project_id: projectSpecific ? projectIdOf(obs) : null, project_name: projectSpecific ? resolveProjectName(projectIdOf(obs)) : null, // Sinapsis-style evidence samples so /cx-validate shows context. - sample_input: failingInput.slice(0, 200), - sample_output: String(obs.output || '').slice(0, 200), + sample_input: headTailSample(failingInput), + sample_output: headTailSample(obs.output), err_msg: String(obs.err_msg).slice(0, 200), _incident: { sid: obs._resolvedSession || obs.sid || null, @@ -1362,20 +1388,25 @@ const TOMBSTONE_REJECTERS = new Set(['cx-validate', 'cx-auto-validate', 'cx-clea function loadRejectedTombstones() { const ids = new Set(); + // v4 (audit P1-6) — id alone missed re-detections of the same rejected + // pattern under a fresh hash (new incident, same trigger regex). Index + // rejected triggers too so writeProposals can tombstone on either match. + const triggers = new Set(); try { const lines = fs.readFileSync(HISTORY_PATH, 'utf8').split('\n'); for (const line of lines) { if (!line.trim()) continue; try { const p = JSON.parse(line); - if (p && p.status === 'rejected' && p.id && + if (p && p.status === 'rejected' && TOMBSTONE_REJECTERS.has(p.rejected_by === undefined ? 'cx-validate' : p.rejected_by)) { - ids.add(p.id); + if (p.id) ids.add(p.id); + if (p.trigger) triggers.add(p.trigger); } } catch (_) { /* malformed line — skip */ } } } catch (_) { /* no history yet */ } - return ids; + return { ids, triggers }; } function writeProposals(newProposals) { @@ -1399,16 +1430,39 @@ function writeProposals(newProposals) { } // v3.37.1 — drop proposals whose id was already rejected (tombstone). - const tombstones = loadRejectedTombstones(); - if (tombstones.size > 0) { - const resurrected = newProposals.filter((p) => tombstones.has(p.id)); + // v4 (audit P1-6) — also tombstone on identical trigger, so a rejected + // pattern re-detected under a fresh id/hash doesn't resurrect. + const { ids: tombstoneIds, triggers: tombstoneTriggers } = loadRejectedTombstones(); + if (tombstoneIds.size > 0 || tombstoneTriggers.size > 0) { + const isTombstoned = (p) => tombstoneIds.has(p.id) || (p.trigger && tombstoneTriggers.has(p.trigger)); + const resurrected = newProposals.filter(isTombstoned); if (resurrected.length > 0) { log(`Tombstone gate dropped ${resurrected.length} previously-rejected proposal(s): ${resurrected.map((p) => p.id).slice(0, 10).join(', ')}`); - newProposals = newProposals.filter((p) => !tombstones.has(p.id)); + newProposals = newProposals.filter((p) => !isTombstoned(p)); } } if (newProposals.length === 0) return; + // v4 (audit P2) — dedup exact-trigger duplicates within this batch before + // merging with existing pending proposals, keeping the highest-confidence + // survivor (ties broken by first-seen order). + { + const byTrigger = new Map(); + for (const p of newProposals) { + if (!p.trigger) continue; + const prev = byTrigger.get(p.trigger); + if (!prev || (p.confidence || 0) > (prev.confidence || 0)) byTrigger.set(p.trigger, p); + } + const withTrigger = newProposals.filter((p) => p.trigger); + const withoutTrigger = newProposals.filter((p) => !p.trigger); + const keptTriggerProposals = new Set(byTrigger.values()); + const droppedCount = withTrigger.length - keptTriggerProposals.size; + if (droppedCount > 0) { + log(`Trigger-dedup dropped ${droppedCount} duplicate-trigger proposal(s) in this batch`); + } + newProposals = [...withoutTrigger, ...keptTriggerProposals]; + } + // v3.29.5 §F5 — one-shot migration: split historical accepted+rejected // entries to proposals-history.jsonl so proposals.json only carries the // live working set (pending + held) going forward. Idempotent — guarded @@ -1437,7 +1491,27 @@ function writeProposals(newProposals) { } byId.set(p.id, p); } - const deduped = Array.from(byId.values()); + let deduped = Array.from(byId.values()); + + // v4 (audit P2) — dedup pending proposals against the existing pending set + // by identical trigger (not just id): two detector runs on the same + // pattern can mint different hashes for the same regex trigger, and the + // per-id dedup above lets both survive as separate pending entries. Keep + // only the highest-confidence pending proposal per trigger; non-pending + // (held/accepted/rejected) entries are left untouched. + { + const pendingByTrigger = new Map(); + const nonPending = []; + for (const p of deduped) { + if (p.status !== 'pending' || !p.trigger) { + nonPending.push(p); + continue; + } + const prev = pendingByTrigger.get(p.trigger); + if (!prev || (p.confidence || 0) > (prev.confidence || 0)) pendingByTrigger.set(p.trigger, p); + } + deduped = [...nonPending, ...pendingByTrigger.values()]; + } // v3.29.5 §F5 — route terminal-state proposals (accepted, rejected) to // history.jsonl and persist only pending + held to proposals.json. diff --git a/hooks/session-start.py b/hooks/session-start.py index b9ad356..e791848 100644 --- a/hooks/session-start.py +++ b/hooks/session-start.py @@ -56,9 +56,13 @@ def load_laws(): laws = [] for f in law_files: try: - first_line = f.read_text().split('\n')[0].strip() - if first_line: - laws.append(first_line) + full_text = f.read_text()[:1000].strip() + if full_text: + # Indent continuation lines so each law renders as one + # readable list item under the '- {law}' join downstream, + # instead of breaking the bullet list structure. + indented = full_text.replace('\n', '\n ') + laws.append(indented) except Exception: pass return laws diff --git a/install.ps1 b/install.ps1 index eea5f0e..53b546c 100644 --- a/install.ps1 +++ b/install.ps1 @@ -20,7 +20,7 @@ $CommandsDir = Join-Path $ClaudeDir "commands" $HooksDir = Join-Path (Join-Path $ClaudeDir "hooks") "cortex" $SettingsFile = Join-Path $ClaudeDir "settings.json" $ClaudeMd = Join-Path $ClaudeDir "CLAUDE.md" -$NewVersion = "4.0.0" +$NewVersion = "4.1.0" # v3.25.1 — explicit downgrade flag (parity with install.sh). # A behind-remote repo would silently rewind hooks otherwise. diff --git a/install.sh b/install.sh index 16635ac..3d40674 100755 --- a/install.sh +++ b/install.sh @@ -21,7 +21,7 @@ COMMANDS_DIR="$CLAUDE_DIR/commands" HOOKS_DIR="$CLAUDE_DIR/hooks/cortex" SETTINGS_FILE="$CLAUDE_DIR/settings.json" CLAUDE_MD="$CLAUDE_DIR/CLAUDE.md" -NEW_VERSION="4.0.0" +NEW_VERSION="4.1.0" # v3.25.1 — explicit downgrade flag. The installer is a copy-not-merge of # hooks/commands, so running an older `install.sh` over a newer install diff --git a/tests/test_injector.sh b/tests/test_injector.sh index a9b1366..44b1d42 100755 --- a/tests/test_injector.sh +++ b/tests/test_injector.sh @@ -213,8 +213,9 @@ domain: workflow --- YAML # Pre-fill the session budget so only ONE instinct fits. Pre-v3.37.0 the -# killswitch zeroed the whole batch here. -printf '7920' > "$S14/cortex/.session-token-budget" +# killswitch zeroed the whole batch here. Value tracks MAX_SESSION_TOKENS +# (12000 since v4.1.0) minus a one-instinct headroom. +printf '11920' > "$S14/cortex/.session-token-budget" printf '{"tool_name": "Bash", "tool_input": {"command": "npm install"}, "cwd": "%s", "session_id": "t14"}\n' "$S14/project" > "$S14/input.json" out14=$(CORTEX_DIR="$S14/cortex" _CX_CORTEX_DIR="$S14/cortex" _CX_INPUT_FILE="$S14/input.json" \ _CX_GLOBAL_INSTINCTS_DIR="$S14/cortex/instincts/global" node "$ENGINE" 2>/dev/null || true)