Skip to content

fix(learner): eliminate false-positive proposals (structural error detection + by-design file exclusions + trigram filter) #29

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
juanparisma wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(learner): eliminate false-positive proposals (structural error detection + by-design file exclusions + trigram filter) #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 34 additions & 12 deletions core/_session-learner.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#!/bin/bash
# Session Learner - Sinapsis v4.3.3
# Stop hook: five detectors —
# 1. error-fix pairs (error → same tool success within 5 events)
# 2. user-corrections (same file edited 2+ times within 10 events)
# 3. workflow-chains (tool trigram repeated 2+ times)
# 1. error-fix pairs (validated error → same tool success within 5 events)
# 2. user-corrections (same file edited 2+ times within 10 events; by-design files excluded)
# 3. workflow-chains (tool trigram repeated 5+ times, >=2 distinct tools, >=1 non-generic)
# 4. repetitions (same error pattern across 3+ sessions — cross-session memory)
# 5. agent-patterns (subagent tool sequences captured from Agent tool calls)
# Also writes context.md per project.
Expand Down Expand Up @@ -58,6 +58,15 @@ try {

if (lines.length < 3) process.exit(0);

// v4.8.1: validate is_error against hard failure markers. Observations written by
// observers < v4.8.1 flagged is_error on any output whose CONTENT mentioned "error"
// (e.g. a successful Read of source code). Do not trust the flag alone.
const HARD_ERR_RE = /(tool_use_error|Permission denied|command not found|No such file or directory|InputValidationError|String to replace not found|Traceback \(most recent call last\)|fatal: |npm ERR!|\bEPERM\b|\bENOENT\b|\bEACCES\b|UnicodeEncodeError|exit code [1-9]|syntax error|was blocked|Web search error)/;
function isRealError(l) {
if (!l || !l.is_error) return false;
return HARD_ERR_RE.test((l.err_msg || "") + "\n" + (l.output || "").slice(0, 600));
}

// ── JOB 1: Write project context.md (ALWAYS — not just when proposals exist) ──
const projectDir = path.dirname(obsFile);
const projectHash = path.basename(projectDir);
Expand Down Expand Up @@ -132,9 +141,9 @@ try {
// Error patterns count (for proposals hint)
let errorCount = 0;
for (let i = 0; i < lines.length - 1; i++) {
if (!lines[i].is_error) continue;
if (!isRealError(lines[i])) continue;
for (let j = i+1; j < Math.min(i+6, lines.length); j++) {
if (lines[j].tool === lines[i].tool && !lines[j].is_error) {
if (lines[j].tool === lines[i].tool && !isRealError(lines[j])) {
errorCount++;
break;
}
Expand Down Expand Up @@ -257,16 +266,17 @@ try {
const proposedIds = new Set(proposals.proposals.map(p => p.id));
const found = [];

// PATTERN 1: error → same tool success within 5 events (uses is_error flag from observe_v3)
// PATTERN 1: error → same tool success within 5 events (uses is_error flag from observe_v3,
// re-validated against hard failure markers — see isRealError)
// Dedup: one proposal per tool per day
for (let i = 0; i < lines.length - 1; i++) {
if (!lines[i].is_error) continue;
if (!isRealError(lines[i])) continue;

const toolId = "fix-" + lines[i].tool.toLowerCase().replace(/[^a-z]/g, "");
if (existing.has(toolId) || proposedIds.has(toolId)) continue;

for (let j = i+1; j < Math.min(i+6, lines.length); j++) {
if (lines[j].tool === lines[i].tool && !lines[j].is_error) {
if (lines[j].tool === lines[i].tool && !isRealError(lines[j])) {
found.push({
type: "error_resolution",
id: toolId,
Expand All @@ -290,11 +300,15 @@ const editEvents = lines
.map((l, idx) => ({ ...l, _idx: idx }))
.filter(l => l.event === "tool_complete" && (l.tool === "Edit" || l.tool === "Write"));

// v4.8.1: files that are re-edited BY DESIGN (journals, control panels, wiki pages
// appended item-by-item during /eod dual save) are not correction signals.
const BYDESIGN_RE = /(?:^|[\\\/])(hot\.md|brief\.md|working-memory\.md|MEMORY\.md|CLAUDE\.md|\d{4}-\d{2}-\d{2}\.md)$|[\\\/](cerebro|briefs|_daily-summaries|daily-summaries)[\\\/]/i;

const correctedFiles = {};
for (let i = 0; i < editEvents.length - 1; i++) {
let fileA = "";
try { const inp = JSON.parse(editEvents[i].input || "{}"); fileA = inp.file_path || ""; } catch(e) {}
if (!fileA) continue;
if (!fileA || BYDESIGN_RE.test(fileA)) continue;

for (let j = i + 1; j < editEvents.length; j++) {
if (editEvents[j]._idx - editEvents[i]._idx > 10) break; // window of 10 events
Expand Down Expand Up @@ -330,6 +344,11 @@ const toolSeq = lines
.filter(l => l.event === "tool_complete")
.map(l => l.tool);

// v4.8.1: generic-tool trigrams (Bash>Bash>Bash x380...) are coding activity, not
// workflows — one session produced 154 junk proposals. Only propose trigrams that
// include a NON-generic tool (MCP, custom), have >= 2 distinct tools, and repeat 5+.
const GENERIC_TOOLS = new Set(["Bash","Read","Edit","Write","Grep","Glob","ToolSearch","AskUserQuestion","Skill","Task","TodoWrite","WebSearch","WebFetch","Agent","Monitor","ScheduleWakeup","NotebookEdit","TaskCreate","TaskUpdate"]);

if (toolSeq.length >= 6) {
const trigramCounts = {};
for (let i = 0; i <= toolSeq.length - 3; i++) {
Expand All @@ -338,8 +357,10 @@ if (toolSeq.length >= 6) {
}

for (const [seq, count] of Object.entries(trigramCounts)) {
if (count < 2) continue;
if (count < 5) continue;
const parts = seq.split(">");
if (new Set(parts).size < 2) continue; // homogeneous = noise
if (parts.every(p => GENERIC_TOOLS.has(p))) continue; // all-generic = coding activity
const wfId = "workflow-" + parts.map(p => p.toLowerCase().replace(/[^a-z]/g, "")).join("-");
if (existing.has(wfId) || proposedIds.has(wfId)) continue;
found.push({
Expand Down Expand Up @@ -403,8 +424,9 @@ for (const ae of agentEvents) {
// \u0027 is single-quote: avoids closing the bash single-quoted node -e block
const agentTypeMatch = output.match(/subagent_type[=:]?\s*["\u0027]?(\w+)/i);
const agentType = agentTypeMatch ? agentTypeMatch[1] : "general";
// If agent output contains error keywords, propose a pattern
const hasError = /\berror\b|\bfailed\b|\bexception\b/i.test(output);
// v4.8.1: substring keywords flagged any agent whose report MENTIONED errors;
// require a validated hard failure marker instead.
const hasError = isRealError(ae);
if (hasError) {
const agId = "agent-error-" + agentType.toLowerCase();
if (existing.has(agId) || proposedIds.has(agId)) continue;
Expand Down
68 changes: 52 additions & 16 deletions skills/sinapsis-learning/hooks/observe_v3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
"""Sinapsis Observer v3 - Single-invocation Python script.
Appends one JSONL observation per tool use to homunculus/projects/{hash}/observations.jsonl
Scrubs secrets from input/output before writing.
Sets is_error=True when output contains error keywords (used by session-learner)."""
Sets is_error=True only on structural harness failures or hard failure markers
in execution (Bash) output — never on keywords inside read file content (v4.8.1)."""

import json, sys, os, re, hashlib, stat
try:
Expand Down Expand Up @@ -140,23 +141,58 @@ def scrub(val):
observation["output"] = scrub(output_str)
# Also capture input for tool_complete (enables full context analysis)
observation["input"] = scrub(input_str)
# Flag errors — session-learner uses this to detect error→resolution patterns
# Use word boundaries to avoid false positives like "0 errors found"
error_patterns = [
r"\berror[:\s]", r"\bfailed\b", r"\bexception\b",
r"\btraceback\b", r"\berrno\b", r"\bEPERM\b", r"\bENOENT\b",
r"exit code [1-9]", r"command not found",
]
output_lower = output_str.lower()
if any(re.search(pat, output_lower) for pat in error_patterns):
observation["is_error"] = True
# Extract first error line for downstream analysis
for line in output_str.split('\n'):
line_lower = line.strip().lower()
if any(re.search(pat, line_lower) for pat in error_patterns):
observation["err_msg"] = scrub(line.strip()[:500])
# Flag errors — session-learner uses this to detect error→resolution patterns.
# v4.8.1 fix: substring matching on the whole output flagged successful Reads
# whose CONTENT mentioned "error" (e.g. source code with `throw new Error`).
# Now: (a) structural error signals from the harness for every tool,
# (b) hard failure markers only for execution tools (Bash), whose output is
# a run result, not arbitrary file content.
is_error = False
err_line = ""

# (a) Structural: harness-reported failure shapes (any tool)
if isinstance(tool_output, dict) and (
tool_output.get("is_error") or tool_output.get("error")
):
is_error = True
err_line = str(tool_output.get("error") or tool_output.get("is_error"))[:500]
elif "tool_use_error" in output_str:
is_error = True
elif re.match(r'^\s*"?(Error|InputValidationError)\b', output_str):
# Failed calls return an error string; successful content tools return
# JSON like {"type": "text", "file": ...} — start-of-output only.
is_error = True

# (b) Hard markers for execution output (Bash) — never for content-bearing
# tools (Read/Grep/Glob/WebFetch/...), whose output is data, not a verdict.
if not is_error and tool_name == "Bash":
hard_markers = [
r"Permission denied", r"command not found", r"No such file or directory",
r"Traceback \(most recent call last\)", r"\bfatal: ", r"npm ERR!",
r"\bEPERM\b", r"\bENOENT\b", r"\bEACCES\b", r"UnicodeEncodeError",
r"exit code [1-9]", r"syntax error", r"was blocked",
]
for pat in hard_markers:
if re.search(pat, output_str):
is_error = True
break

if is_error:
observation["is_error"] = True
if not err_line:
# Extract first line carrying a failure marker (best effort)
fail_re = re.compile(
r"(tool_use_error|Permission denied|command not found|No such file"
r"|Traceback|fatal: |npm ERR!|EPERM|ENOENT|EACCES|UnicodeEncodeError"
r"|exit code [1-9]|syntax error|was blocked|^\s*\"?Error)"
)
for line in output_str.split('\n'):
if fail_re.search(line):
err_line = line.strip()[:500]
break
if err_line:
observation["err_msg"] = scrub(err_line)

obs_file = os.path.join(project_dir, "observations.jsonl")

# Auto-archive if file exceeds 10MB (with lock to prevent concurrent rotation)
Expand Down
Loading