diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ab63ca3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,79 @@ +# --- Operator-only docs / machine-local config --- +CLAUDE.md +.mcp.json +.aiwg/ +.claude/ +.transcripts/ + +# --- Secrets / credentials --- +project/data/secrets.enc +project/data/secrets.enc.tmp +project/secrets/ +project/config.json +project/config.json.bak + +# --- Model weights / large checkpoints --- +project/.checkpoints/ + +# --- Runtime caches and embeddings --- +project/_embedding_cache.safetensors +project/_embedding_cache_meta.json +project/_embedding_cache.json +project/embeddings/ +project/agent/embeddings/ +project/sdol/embeddings/ + +# --- Caches and venv --- +project/venv/ +project/__pycache__/ +**/__pycache__/ +*.pyc +*.pyo + +# --- Install stamp files --- +.deps_v* +.datascience_v* +.imagebind_v* +.chatterbox_patch_v* +.whisperx_v* + +# --- Runtime outputs (per leOS git etiquette) --- +project/data/ +# Exception: SKOS concept hierarchy is seed config, not runtime state. +!project/data/skos_hierarchy.json +project/knowledgebase/ +project/media_library/files/ +project/media_library/derived/ +project/media_library/records/ +project/media_library/thumbnails/ +project/voice/data/ +project/projects/ +project/agent/data/ +project/bots/data/ +project/subsystems/evaluations/ +project/_media-processing/ +project/infra/decision_trees/audit_log.jsonl +project/_system-maintenance/*.log +project/_system-maintenance/*.err +project/_system-maintenance/*_report.json +project/_system-maintenance/*_report.csv +project/_system-maintenance/files/ +project/_system-maintenance/history/ +project/_system-maintenance/postmortems/ +project/_system-maintenance/skills/ +project/_system-maintenance/project.json + +# --- Topic-centroid cache backups (regenerable from SEED_TOPICS) --- +project/data/topic_centroids.json +project/data/topic_centroids.json.old +project/data/topic_centroids.json.before-* + +# --- Editor / OS noise --- +.vscode/ +.idea/ +.DS_Store +Thumbs.db +*.swp + +# --- ComfyUI bundled install (run.bat clones on first boot) --- +project/external/ diff --git a/project/README.md b/project/README.md index de08a21..2ebb70b 100644 --- a/project/README.md +++ b/project/README.md @@ -707,8 +707,8 @@ work at every tier transition. Before any classified user query reaches the LLM, the substrate- gather package (`bots/substrate_gather/`) runs first. This is the -mechanism that turns "what is Xavier" from a 30-second LLM round trip -into a 100ms KB lookup when an article on Xavier already exists. +mechanism that turns "what is X" from a 30-second LLM round trip +into a 100ms KB lookup when an article on X already exists. The companion brain classifies queries into six categories: **status_check**, **codebase_question**, **system_question**, diff --git a/project/__init__.py b/project/__init__.py index 952747d..c8db76d 100644 --- a/project/__init__.py +++ b/project/__init__.py @@ -403,10 +403,10 @@ def _format_dict_brief(d, max_keys=8, depth=0, max_depth=1): runaway nesting on deeply-structured API payloads. For dexscreener-style data this means {baseToken: {symbol: - XAVIER, name: Xavier}} renders as: + ALPHA, name: Alpha}} renders as: - baseToken: - - symbol: XAVIER - - name: Xavier + - symbol: ALPHA + - name: Alpha instead of "baseToken: (dict, 2 items)". """ if not d: diff --git a/project/_system-maintenance/batch_all.py b/project/_system-maintenance/batch_all.py new file mode 100644 index 0000000..9b3459b --- /dev/null +++ b/project/_system-maintenance/batch_all.py @@ -0,0 +1,302 @@ +"""batch_all.py — Pipeline every unique podcast episode through download +(if needed) + RETRANSCRIBE_RECORD + MEDIA_REVIEW. + +Scope discovery: + * Walks `media_library/records/media_*.json`. + * Deduplicates by `source_url` — many shows were ingested multiple + times producing duplicate JSON stubs that share the same audio. + One representative record per URL survives discovery (the one + that already has an mp3 wins; otherwise the lexicographically + first record_id). + * Skips records that already have a fresh `analysis.review` unless + `--force` is passed. + +For each survivor: + 1. If the local mp3 is missing, download it from `source_url` into + `media_library/files/{record_id}.mp3` (streamed, 120s timeout). + 2. POST RETRANSCRIBE_RECORD — runs faster-whisper + pyannote + diarization + chained RESOLVE_SPEAKERS. + 3. POST MEDIA_REVIEW — 7-stage staged review pipeline. + +Resilient: on any per-record HTTP error or non-ok response, the +record is logged as a failure and the loop continues. + +Operator-side checkpoints fire after every 10 successful records: +counts new KB articles / monologues / thinking_logs since batch +start, runs SEARCH probes against the KB, and queries REFLEX_STATS +to confirm leOS is using the ingested knowledge. + +Usage: + python -m _system-maintenance.batch_all # all eligible + python -m _system-maintenance.batch_all --force # re-process + python -m _system-maintenance.batch_all --limit 5 + python -m _system-maintenance.batch_all --no-download # only records that already have mp3 + python -m _system-maintenance.batch_all --download-only # fetch mp3s, skip pipeline +""" + +import argparse +import collections +import json +import os +import sys +import time +import urllib.request +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +RECORDS_DIR = PROJECT_ROOT / "media_library" / "records" +FILES_DIR = PROJECT_ROOT / "media_library" / "files" +KB_DIR = PROJECT_ROOT / "knowledgebase" +MONO_DIR = PROJECT_ROOT / "voice" / "data" / "monologues" +THOUGHT_DIR = PROJECT_ROOT / "data" / "thinking_logs" +LEOS_URL = "http://localhost:5000" + +CHECKPOINT_EVERY = 10 + + +def post(instruction, args, timeout=1800): + payload = json.dumps({"instruction": instruction, "args": args or {}}).encode("utf-8") + req = urllib.request.Request( + f"{LEOS_URL}/kernel/execute", + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def discover(force): + """Return list of (record_id, title, source_url, has_mp3, mp3_size_mb). + + Dedup by source_url: many records share a URL because TOOL_PODCAST + has been re-run on the same feeds. Pick the record that already + owns an mp3 (avoid re-download) over a stub-only sibling. + """ + by_url = collections.defaultdict(list) + for p in sorted(RECORDS_DIR.glob("media_*.json")): + try: + d = json.loads(p.read_text(encoding="utf-8")) + except Exception: + continue + rid = p.stem + url = (d.get("source_url") or "").strip() + if not url: + continue + title = (d.get("title") or "")[:100] + analysis = d.get("analysis") or {} + review = (analysis.get("review") or {}) + kb_id = review.get("kb_article_id") + mp3 = FILES_DIR / f"{rid}.mp3" + by_url[url].append({ + "rid": rid, + "title": title, + "url": url, + "has_mp3": mp3.exists(), + "mp3_size_mb": round(mp3.stat().st_size / 1e6, 1) if mp3.exists() else 0, + "has_review": bool(kb_id), + }) + + out = [] + for url, candidates in by_url.items(): + # Prefer a record that already has the mp3 — saves re-download. + candidates.sort(key=lambda c: (not c["has_mp3"], c["rid"])) + primary = candidates[0] + if not force and primary["has_review"] and primary["has_mp3"]: + continue # already pipelined fully + out.append(primary) + + out.sort(key=lambda c: (not c["has_mp3"], c["rid"])) # mp3-having first + return out + + +def download_audio(url, dest_path, timeout=120): + """Stream-download `url` to `dest_path` (mp3). Returns size in + bytes on success, raises on failure. Uses requests if available + for streaming + auto-retry; otherwise falls back to urllib.""" + try: + import requests + with requests.get(url, stream=True, timeout=timeout, headers={ + "User-Agent": "Mozilla/5.0 (compatible; leOS-batch/1.0)", + }) as r: + r.raise_for_status() + tmp = str(dest_path) + ".part" + size = 0 + with open(tmp, "wb") as f: + for chunk in r.iter_content(chunk_size=64 * 1024): + if chunk: + f.write(chunk) + size += len(chunk) + os.replace(tmp, dest_path) + return size + except ImportError: + # Fallback: urllib (blocking, no streaming chunks) + req = urllib.request.Request(url, headers={ + "User-Agent": "Mozilla/5.0 (compatible; leOS-batch/1.0)", + }) + tmp = str(dest_path) + ".part" + with urllib.request.urlopen(req, timeout=timeout) as resp, open(tmp, "wb") as f: + data = resp.read() + f.write(data) + os.replace(tmp, dest_path) + return len(data) + + +def file_count_since(dirpath, t0): + if not dirpath.exists(): + return 0 + return sum(1 for p in dirpath.iterdir() if p.is_file() and p.stat().st_mtime > t0) + + +def checkpoint_review(checkpoint_idx, t_start): + """Sanity check that leOS is actually using the ingested knowledge. + Counts new artifacts since batch start, runs a few sample searches + against the KB. Output is descriptive, not an assertion.""" + print(f"\n=== CHECKPOINT #{checkpoint_idx} (every {CHECKPOINT_EVERY} records) ===", flush=True) + new_kb = file_count_since(KB_DIR, t_start) + new_mono = file_count_since(MONO_DIR, t_start) + new_think = file_count_since(THOUGHT_DIR, t_start) + print(f" artifacts since batch start:", flush=True) + print(f" KB articles: +{new_kb}", flush=True) + print(f" monologues: +{new_mono}", flush=True) + print(f" thinking_logs: +{new_think}", flush=True) + + queries = [ + "consciousness mortality death", + "health relationships well-being", + "podcast host interview", + ] + print(f" KB search probes:", flush=True) + for q in queries: + try: + r = post("SEARCH", {"partition": "text_store", "query": q, "k": 3}) + results = r.get("results") or [] + top_sim = (results[0].get("similarity") if results else 0) or 0 + print(f" {q!r}: {len(results)} hits top_sim={top_sim:.3f}", + flush=True) + except Exception as e: + print(f" {q!r}: search error: {e}", flush=True) + + try: + r = post("REFLEX_STATS", {}) + print(f" reflex arc: attempts={r.get('attempts',0)} fires={r.get('fires',0)} " + f"hit_rate={r.get('hit_rate',0):.2f}", flush=True) + except Exception: + pass + print("=== checkpoint complete ===\n", flush=True) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--force", action="store_true", + help="re-process records that already have a review") + p.add_argument("--limit", type=int, help="cap number of records") + p.add_argument("--no-download", action="store_true", + help="only process records that already have an mp3") + p.add_argument("--download-only", action="store_true", + help="fetch missing mp3s and stop (skip retranscribe+review)") + args = p.parse_args() + + candidates = discover(args.force) + + have_mp3 = sum(1 for c in candidates if c["has_mp3"]) + need_dl = len(candidates) - have_mp3 + + if args.no_download: + candidates = [c for c in candidates if c["has_mp3"]] + + if args.limit: + candidates = candidates[: args.limit] + + if not candidates: + print("nothing to do", flush=True) + return + + print(f"=== batch_all: {len(candidates)} unique episodes to process ===", flush=True) + print(f" have mp3: {have_mp3}, need download: {need_dl}", flush=True) + print(f" checkpoint every {CHECKPOINT_EVERY} records", flush=True) + t_start = time.time() + successes = 0 + failures = 0 + downloaded = 0 + + for i, c in enumerate(candidates, 1): + elapsed = int(time.time() - t_start) + print(f"\n[{i}/{len(candidates)}] elapsed={elapsed//60}m{elapsed%60}s {c['rid']}", + flush=True) + print(f" {c['title']}", flush=True) + rec_t0 = time.time() + + # Stage 0: download mp3 if missing + if not c["has_mp3"]: + mp3_path = FILES_DIR / f"{c['rid']}.mp3" + try: + size = download_audio(c["url"], mp3_path) + downloaded += 1 + print(f" [download] {size/1e6:.1f} MB in {time.time()-rec_t0:.0f}s", + flush=True) + except Exception as e: + print(f" [download] FAILED: {e}", flush=True) + failures += 1 + continue + + if args.download_only: + successes += 1 + continue + + # Stage A: retranscribe + diarize + resolve speakers + try: + rt = post("RETRANSCRIBE_RECORD", + {"record_id": c["rid"], "wipe_review": True}) + except Exception as e: + print(f" [retrans] HTTP error: {e}", flush=True) + failures += 1 + continue + if not rt.get("ok"): + print(f" [retrans] failed: {rt.get('error')}", flush=True) + failures += 1 + continue + sr = rt.get("speakers_resolved") or [] + # Show every named speaker with role + evidence so co-hosts + # and resolved guests both surface (not just the first). + named = [ + f"{s.get('name')}({s.get('role','?')[:1]}/{s.get('evidence','')})" + for s in sr if s.get("name") + ] + named_str = ", ".join(named) if named else "none" + print(f" [retrans] segs={rt.get('segments_count')} " + f"speakers={rt.get('speakers_detected')} " + f"resolved=[{named_str}]", + flush=True) + + # Stage B: review + try: + rv = post("MEDIA_REVIEW", {"record_id": c["rid"]}) + except Exception as e: + print(f" [review] HTTP error: {e}", flush=True) + failures += 1 + continue + if not rv.get("ok"): + print(f" [review] failed: {rv.get('error')}", flush=True) + failures += 1 + continue + rev = rv.get("review") or {} + print(f" [review] kb={rev.get('kb_article_id')} " + f"facts={len(rev.get('facts') or [])} " + f"claims={len(rev.get('claims') or [])} " + f"themes={len(rev.get('themes') or [])} " + f"({time.time()-rec_t0:.0f}s total)", + flush=True) + successes += 1 + + if successes % CHECKPOINT_EVERY == 0: + checkpoint_review(successes // CHECKPOINT_EVERY, t_start) + + total_min = int((time.time() - t_start) / 60) + print(f"\n=== DONE: {successes} succeeded, {failures} failed, " + f"{downloaded} downloaded, {total_min}m total ===", + flush=True) + + +if __name__ == "__main__": + main() diff --git a/project/_system-maintenance/dedup_records.py b/project/_system-maintenance/dedup_records.py new file mode 100644 index 0000000..51a1c93 --- /dev/null +++ b/project/_system-maintenance/dedup_records.py @@ -0,0 +1,155 @@ +"""dedup_records.py — collapse duplicate media records by source_url. + +Many shows were ingested through TOOL_PODCAST multiple times, leaving +multiple JSON stubs that share the same audio source. This script +keeps one record per source_url and removes the rest. + +For each URL with N>1 records: + 1. Keep the record that already has the most work invested. + Priority: has_review > has_mp3 > lexicographically first id. + 2. If the kept record lacks an mp3 but a sibling has one, MOVE the + mp3 into the kept record's slot. We don't lose audio. + 3. Delete the sibling JSON files. + +The script is destructive but only removes record JSONs that share +their source_url with a survivor (and have nothing else to lose). +Pre-flight verification refuses to delete a record that has a review +unless the survivor also has one — guards against accidental +work-loss. + +Usage: + python -m _system-maintenance.dedup_records --dry-run # preview + python -m _system-maintenance.dedup_records # execute +""" + +import argparse +import collections +import json +import os +import sys +import shutil +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +RECORDS_DIR = PROJECT_ROOT / "media_library" / "records" +FILES_DIR = PROJECT_ROOT / "media_library" / "files" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--dry-run", action="store_true", + help="report what would be removed without touching disk") + args = ap.parse_args() + + by_url = collections.defaultdict(list) + no_url = [] + + for p in sorted(RECORDS_DIR.glob("media_*.json")): + try: + d = json.loads(p.read_text(encoding="utf-8")) + except Exception as e: + print(f" skip unreadable {p.name}: {e}", flush=True) + continue + rid = p.stem + url = (d.get("source_url") or "").strip() + if not url: + no_url.append((rid, p)) + continue + has_mp3 = (FILES_DIR / f"{rid}.mp3").exists() + has_review = bool(((d.get("analysis") or {}).get("review") or {}).get("kb_article_id")) + by_url[url].append({ + "rid": rid, + "path": p, + "has_mp3": has_mp3, + "has_review": has_review, + }) + + print(f"unique URLs: {len(by_url)}", flush=True) + print(f"records without source_url (left untouched): {len(no_url)}", flush=True) + + keep_set, drop_set, mp3_rescues, refused = [], [], [], [] + + for url, candidates in by_url.items(): + if len(candidates) == 1: + keep_set.append(candidates[0]) + continue + # Priority sort: review > mp3 > rid + candidates.sort(key=lambda c: (not c["has_review"], not c["has_mp3"], c["rid"])) + kept = candidates[0] + keep_set.append(kept) + for sib in candidates[1:]: + # Safety: refuse to drop a record with a review if the + # survivor doesn't have one. Should be impossible after + # the priority sort, but enforce. + if sib["has_review"] and not kept["has_review"]: + refused.append((url, kept["rid"], sib["rid"])) + continue + # Rescue mp3: if survivor lacks audio but sibling has it, + # move the mp3 into the survivor's slot. + if sib["has_mp3"] and not kept["has_mp3"]: + mp3_rescues.append({ + "url": url, + "from_rid": sib["rid"], + "to_rid": kept["rid"], + }) + drop_set.append(sib) + + print(f"keep: {len(keep_set)}", flush=True) + print(f"drop: {len(drop_set)}", flush=True) + print(f"mp3 rescues (sibling has audio, survivor doesn't): {len(mp3_rescues)}", flush=True) + print(f"refused dedups (would lose review): {len(refused)}", flush=True) + + if refused: + print("\nREFUSED — these need manual review:", flush=True) + for url, kept_rid, sib_rid in refused[:10]: + print(f" url={url[:80]} keep={kept_rid} drop={sib_rid}", flush=True) + + if args.dry_run: + print("\n(dry run — no changes)", flush=True) + return + + # ---- Execute --------------------------------------------------- + # 1. Rescue mp3s + rescued = 0 + for r in mp3_rescues: + src = FILES_DIR / f"{r['from_rid']}.mp3" + dst = FILES_DIR / f"{r['to_rid']}.mp3" + if not src.exists(): + continue + if dst.exists(): + # Survivor got an mp3 between dry-run and now — leave src. + continue + try: + shutil.move(str(src), str(dst)) + rescued += 1 + except Exception as e: + print(f" rescue failed {src.name} -> {dst.name}: {e}", flush=True) + + # 2. Delete drop record JSONs (and any leftover mp3 of dropped sibling) + deleted = 0 + leftover_mp3 = 0 + for sib in drop_set: + # Sibling mp3 not rescued? It's a dup of the kept one (or + # we rescued elsewhere). Remove it to free disk. + sib_mp3 = FILES_DIR / f"{sib['rid']}.mp3" + if sib_mp3.exists(): + try: + sib_mp3.unlink() + leftover_mp3 += 1 + except Exception: + pass + try: + sib["path"].unlink() + deleted += 1 + except Exception as e: + print(f" delete failed {sib['path'].name}: {e}", flush=True) + + print(f"\ndone:", flush=True) + print(f" mp3 files rescued (renamed to survivor): {rescued}", flush=True) + print(f" duplicate mp3 files removed: {leftover_mp3}", flush=True) + print(f" drop record JSONs deleted: {deleted}", flush=True) + print(f" surviving records: {len(keep_set)}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/project/_system-maintenance/probe_corpus.py b/project/_system-maintenance/probe_corpus.py new file mode 100644 index 0000000..3093e7e --- /dev/null +++ b/project/_system-maintenance/probe_corpus.py @@ -0,0 +1,247 @@ +"""probe_corpus.py — Probe whether ingested podcast knowledge is +actually useful through normal leOS interaction surfaces. + +Sections: + A. KB search on natural-language questions — does retrieval surface + the right articles, with what similarity? + B. Cross-episode hops — given an article, does graph linking + produce semantically meaningful neighbors? + C. Agent chat — pose a question and let the agent answer using + the KB as context. Did it cite/use the ingested material? + D. Auto-tag relevance — do the auto-tags assigned during review + match what the article actually discusses? + +This is a one-shot diagnostic, not a benchmark — output is human- +readable so we can judge whether the system is actually using +what was ingested. +""" + +import json +import sys +import time +import urllib.request + +LEOS_URL = "http://127.0.0.1:5000" + + +def post(instruction, args, timeout=600): + payload = json.dumps({"instruction": instruction, "args": args or {}}).encode("utf-8") + req = urllib.request.Request( + f"{LEOS_URL}/kernel/execute", + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def hr(title=""): + print() + print("=" * 78) + if title: + print(f" {title}") + print("=" * 78) + + +# -------------------------------------------------------------------- +# A. KB search on natural-language questions +# -------------------------------------------------------------------- + +QUERIES = [ + # Direct topic queries — should hit the matching review article + ("bears garbage dump tranquilization conservation", + "Ologies — Ursinology"), + ("language phonology accent dialect", + "Ologies — Phonology"), + ("colon cancer screening colonoscopy", + "Ologies — Coloproctology"), + ("history of innovation connections technology", + "Ologies — Syndesiology"), + ("brain supplement social cognition", + "Here We Are — Tailored Brain"), + ("evolution menopause grandmothers", + "Mind Under Matter — Grandmas"), + ("interview podcast comedy religion", + "Here We Are — Comedy Sex God"), + ("psychedelic comedian guest", + "(several MUM episodes)"), + # Cross-show synthesis — knowledge that spans multiple reviews + ("how does diet relate to mental health and cognition?", + "(should hit BioME / Tailored Brain / others)"), + ("what makes a podcast host successful at interviews?", + "(meta)"), +] + + +def section_a_kb_search(): + """Probe both retrieval paths: + * SEARCH on text_store (raw kernel partition search) — JSON shape. + * KB_SEARCH (TOOL_KB_SEARCH wrapper) — formatted text shape. + """ + hr("A. KB search on natural-language questions") + for q, expected in QUERIES: + print(f"\nQ: {q!r}") + print(f" expected: {expected}") + # 1) Partition SEARCH — JSON, scored + try: + # Pass source_agent="review" to suppress leOS system-doc bleed + # on podcast topical queries. Omit (or pass "all") for + # cross-source synthesis searches. + r = post("SEARCH", {"partition": "text_store", "query": q, "k": 5, + "source_agent": "review"}, timeout=20) + results = r.get("results") or [] + if results: + print(f" [SEARCH text_store]") + for hit in results[:5]: + sim = hit.get("similarity") or hit.get("score") or 0 + rid = hit.get("id") or hit.get("article_id") or "?" + title_or_snippet = ( + hit.get("title") + or (hit.get("content") or "")[:70].replace("\n", " ") + or rid + ) + print(f" {sim:.3f} {rid} {title_or_snippet}") + else: + print(f" [SEARCH text_store] --> NO HITS") + except Exception as e: + print(f" [SEARCH] error: {e}") + + # 2) KB_SEARCH — preferred path for UI / agent tools. + # Now uses RRF fusion (BM25 + nomic + qwen) under the hood. + try: + r = post("KB_SEARCH", {"query": q, "depth": 0}, timeout=30) + # TOOL_KB_SEARCH returns formatted text in `output` not JSON + # results. Surface the first ~200 chars so we can see if + # it actually found anything. + out = (r.get("output") or "").strip() + if out: + preview = out[:240].replace("\n", " | ") + print(f" [KB_SEARCH] {preview}") + else: + print(f" [KB_SEARCH] --> empty output") + except Exception as e: + print(f" [KB_SEARCH] error: {e}") + + +# -------------------------------------------------------------------- +# B. Cross-episode hops via auto-link / KG +# -------------------------------------------------------------------- + +ANCHOR_ARTICLES = [ + "kb_3a6bbe13", # Ursinology (BEARS) + "kb_a5abc93f", # Phonology + "kb_8e7fe081", # Coloproctology + "kb_6212b837", # Tailored Brain +] + + +def section_b_kg_hops(): + hr("B. Cross-episode hops — graph expansion from anchor articles") + for kb_id in ANCHOR_ARTICLES: + print(f"\nFROM {kb_id}") + try: + r = post("KB_GRAPH_EXPAND", {"article_id": kb_id, "max_hops": 1, "max_neighbors": 6}, + timeout=15) + neighbors = r.get("neighbors") or r.get("expanded") or [] + if not neighbors: + # Fallback: read article.related directly + import pathlib + p = pathlib.Path(__file__).resolve().parent.parent / "knowledgebase" / f"{kb_id}.json" + if p.exists(): + art = json.loads(p.read_text(encoding="utf-8")) + related = art.get("related") or [] + print(f" KB_GRAPH_EXPAND empty; article.related has {len(related)} ids") + neighbors = [{"article_id": rid} for rid in related[:6]] + for nb in neighbors[:6]: + rid = nb.get("article_id") or nb.get("id") or "?" + w = nb.get("weight") or nb.get("similarity") or "" + t = nb.get("title") or "" + if not t: + # Look it up + try: + import pathlib + p = pathlib.Path(__file__).resolve().parent.parent / "knowledgebase" / f"{rid}.json" + if p.exists(): + t = (json.loads(p.read_text(encoding="utf-8"))).get("title", "") + except Exception: + pass + print(f" -> {rid} {w} {t[:70]}") + except Exception as e: + print(f" error: {e}") + + +# -------------------------------------------------------------------- +# C. Agent chat — substantive Q&A against the corpus +# -------------------------------------------------------------------- + +CHAT_QUESTIONS = [ + "Based on what you've learned from podcast episodes I've ingested, what do you know about bear conservation?", + "What did Pete Holmes say about Comedy Sex God?", + "Summarize what you know about menopause and human evolution.", +] + + +def section_c_agent_chat(): + hr("C. Agent chat — substantive Q&A using ingested knowledge") + for q in CHAT_QUESTIONS: + print(f"\nQ: {q}") + t0 = time.time() + try: + # ASSIST is the lightweight (intern) model — fast, no tool calls + r = post("ASSIST", { + "prompt": q, + "context": { + "type": "explain", + "max_tokens": 400, + "temperature": 0.5, + }, + }, timeout=120) + dt = time.time() - t0 + ans = r.get("response") or r.get("text") or "(empty)" + print(f" [intern, {dt:.0f}s]") + print(f" {ans[:1200]}") + except Exception as e: + print(f" error: {e}") + + +# -------------------------------------------------------------------- +# D. Auto-tag check — do the assigned tags match the content? +# -------------------------------------------------------------------- + +def section_d_tag_relevance(): + hr("D. Auto-tag relevance — sample of recently-ingested articles") + import pathlib + kb = pathlib.Path(__file__).resolve().parent.parent / "knowledgebase" + sample = sorted(kb.glob("kb_*.json"), key=lambda x: x.stat().st_mtime, reverse=True)[:8] + for p in sample: + try: + d = json.loads(p.read_text(encoding="utf-8")) + except Exception: + continue + if d.get("source_agent") != "review": + continue + title = (d.get("title") or "")[:80] + topics = d.get("topics") or [] + keywords = d.get("keywords") or [] + print(f"\n{p.stem} {title}") + print(f" topics: {topics}") + if keywords: + print(f" keywords: {keywords[:8]}") + + +# -------------------------------------------------------------------- + +def main(): + print("=" * 78) + print(" leOS corpus probe — does ingested podcast knowledge work?") + print("=" * 78) + section_a_kb_search() + section_b_kg_hops() + section_d_tag_relevance() + section_c_agent_chat() + print() + + +if __name__ == "__main__": + main() diff --git a/project/_system-maintenance/repopulate.py b/project/_system-maintenance/repopulate.py new file mode 100644 index 0000000..1c53cac --- /dev/null +++ b/project/_system-maintenance/repopulate.py @@ -0,0 +1,85 @@ +"""repopulate.py — Re-run MEDIA_REVIEW on every podcast record that has +a transcript, regenerating clean structured KB articles. + +Runs sequentially against the live leOS server. Safe to interrupt; just +re-run to resume (records that already have a fresh kb_article_id will +get overwritten — the reviewer doesn't dedupe today, but the new article +is identical structure, only the kb_id differs). +""" + +import json +import time +import urllib.request +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +RECORDS_DIR = PROJECT_ROOT / "media_library" / "records" +LEOS_URL = "http://localhost:5000" + + +def post(instruction, args, timeout=900): + payload = json.dumps({"instruction": instruction, "args": args or {}}).encode("utf-8") + req = urllib.request.Request( + f"{LEOS_URL}/kernel/execute", + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def main(): + candidates = [] + for p in sorted(RECORDS_DIR.glob("media_*.json")): + try: + d = json.loads(p.read_text()) + except Exception: + continue + a = d.get("analysis") or {} + if not a.get("transcript"): + continue + candidates.append({"rid": p.stem, "title": (d.get("title") or "")[:80]}) + + print(f"{len(candidates)} records with transcripts", flush=True) + t_start = time.time() + completed = 0 + failed = 0 + for i, c in enumerate(candidates, 1): + elapsed = int(time.time() - t_start) + print(f"\n[{i}/{len(candidates)}] elapsed={elapsed}s {c['rid']}", flush=True) + print(f" {c['title']}", flush=True) + t0 = time.time() + try: + r = post("MEDIA_REVIEW", {"record_id": c["rid"]}, timeout=900) + except Exception as e: + print(f" ERROR (HTTP): {e}", flush=True) + failed += 1 + continue + if not r.get("ok"): + print(f" ERROR: {r.get('error')}", flush=True) + failed += 1 + continue + dt = time.time() - t0 + review = r.get("review") or {} + kb_id = review.get("kb_article_id") + n_facts = len(review.get("facts") or []) + n_claims = len(review.get("claims") or []) + n_themes = len(review.get("themes") or []) + n_links = len(review.get("linked_episodes") or []) + n_tags = len(review.get("tags") or []) + summary_len = len(review.get("summary") or "") + print(f" ok in {dt:.1f}s kb={kb_id} " + f"facts={n_facts} themes={n_themes} claims={n_claims} " + f"summary={summary_len}c links={n_links} tags={n_tags}", + flush=True) + if kb_id: + completed += 1 + + total_elapsed = int(time.time() - t_start) + print(f"\n=== DONE: {completed} repopulated, {failed} failed, " + f"{total_elapsed//60}m{total_elapsed%60}s total ===", flush=True) + + +if __name__ == "__main__": + main() diff --git a/project/_system-maintenance/retag_relink.py b/project/_system-maintenance/retag_relink.py new file mode 100644 index 0000000..c14d34f --- /dev/null +++ b/project/_system-maintenance/retag_relink.py @@ -0,0 +1,353 @@ +"""retag_relink.py — Apply the new auto-tag + auto-link logic to +already-ingested KB articles without re-running MEDIA_REVIEW. + +Pass 1 (re-tag): builds topic centroids by calling EMBED on each +topic phrase (the new richer phrases from topic_vocabulary.py), +embeds each article's content, scores against centroids using the +new threshold (0.50) + relative-margin (0.85) filter, and writes +the new ``topics`` list back to the article JSON. + +Pass 2 (re-link): for each article, calls KB_GRAPH_AUTOLINK to +recompute ``related[]`` with the new threshold (0.62) and cap (8). + +Usage: + python -m _system-maintenance.retag_relink # both passes + python -m _system-maintenance.retag_relink --tags-only + python -m _system-maintenance.retag_relink --links-only + python -m _system-maintenance.retag_relink --limit 5 +""" + +import argparse +import json +import math +import pathlib +import sys +import urllib.request + +LEOS_URL = "http://127.0.0.1:5000" +PROJECT_ROOT = pathlib.Path(__file__).resolve().parent.parent +KB_DIR = PROJECT_ROOT / "knowledgebase" + + +def post(instruction, args, timeout=120): + payload = json.dumps({"instruction": instruction, "args": args or {}}).encode("utf-8") + req = urllib.request.Request( + f"{LEOS_URL}/kernel/execute", + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def embed(text): + r = post("EMBED", {"content": text[:6000], "model": "nomic_text"}, timeout=30) + return r.get("vector") or r.get("embedding") or r.get("vec") + + +def cosine(a, b): + s = 0.0 + norm_a = norm_b = 0.0 + for x, y in zip(a, b): + s += x * y + norm_a += x * x + norm_b += y * y + if norm_a == 0 or norm_b == 0: + return 0.0 + return s / (math.sqrt(norm_a) * math.sqrt(norm_b)) + + +def load_topic_phrases(): + """Read SEED_TOPICS from topic_vocabulary.py without importing + the embeddings-bound module — we just need the (label, phrase) + list.""" + sys.path.insert(0, str(PROJECT_ROOT / "knowledge")) + # Import-free read: parse SEED_TOPICS via exec in a sandbox. + src = (PROJECT_ROOT / "knowledge" / "topic_vocabulary.py").read_text(encoding="utf-8") + ns = {} + # Extract just the SEED_TOPICS literal — it's a list of tuples + # at module scope. Easiest robust path: exec the slice between + # ``SEED_TOPICS = [`` and the closing ``]``. + start = src.index("SEED_TOPICS = [") + end = src.index("\n]", start) + 2 + exec(src[start:end], ns) + seeds = ns["SEED_TOPICS"] + out = [] + for entry in seeds: + if isinstance(entry, str): + out.append((entry, entry)) + else: + out.append((entry[0], entry[1])) + return out + + +def build_centroids(): + """Embed each topic phrase via the kernel EMBED instruction.""" + topics = load_topic_phrases() + centroids = {} + print(f"Embedding {len(topics)} topic phrases via kernel EMBED...", flush=True) + for label, phrase in topics: + try: + v = embed(phrase) + if v: + centroids[label] = v + except Exception as e: + print(f" failed {label!r}: {e}", flush=True) + print(f"Built {len(centroids)} centroids.", flush=True) + return centroids + + +def suggest_tags(vec, centroids, threshold=0.50, max_tags=5, + relative_margin=0.85): + scored = [] + for name, c in centroids.items(): + s = cosine(vec, c) + if s >= threshold: + scored.append((name, s)) + if not scored: + return [] + scored.sort(key=lambda kv: -kv[1]) + top = scored[0][1] + cutoff = top * relative_margin + return [(n, s) for n, s in scored if s >= cutoff][:max_tags] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--limit", type=int) + ap.add_argument("--tags-only", action="store_true") + ap.add_argument("--links-only", action="store_true") + args = ap.parse_args() + + candidates = [] + for p in sorted(KB_DIR.glob("kb_*.json")): + try: + d = json.loads(p.read_text(encoding="utf-8")) + except Exception: + continue + if d.get("source_agent") != "review": + continue + candidates.append((p, d)) + + if args.limit: + candidates = candidates[: args.limit] + + if not candidates: + print("nothing to do", flush=True) + return + + print(f"=== retag_relink: {len(candidates)} review articles ===", flush=True) + fix_tags = not args.links_only + fix_links = not args.tags_only + + if fix_tags: + centroids = build_centroids() + if not centroids: + print("ERROR: could not build centroids", flush=True) + sys.exit(1) + + print(f"\n--- pass 1: re-tag ---", flush=True) + for i, (p, d) in enumerate(candidates, 1): + kb_id = p.stem + title = (d.get("title") or "")[:70] + old_topics = d.get("topics") or [] + content = (d.get("summary") or "") + "\n" + (d.get("content") or "") + content = content.strip() + if not content: + print(f" [{i}/{len(candidates)}] {kb_id} — no content, skip", flush=True) + continue + try: + vec = embed(content) + except Exception as e: + print(f" [{i}/{len(candidates)}] {kb_id} — embed error: {e}", flush=True) + continue + if not vec: + print(f" [{i}/{len(candidates)}] {kb_id} — empty vec", flush=True) + continue + new = suggest_tags(vec, centroids, threshold=0.50, max_tags=5, + relative_margin=0.85) + if not new: + # No tags meet the new threshold. CLEAR the old + # topics — better to have an empty list than to keep + # stale meta-attractors like "interviewing and + # storytelling" that the new vocabulary doesn't even + # contain. Empty topics still allow SKOS hierarchy + # ancestors to be inferred for narrow free-text tags + # the LLM stages may add later. + if old_topics: + d["topics"] = [] + try: + p.write_text(json.dumps(d, ensure_ascii=False, indent=2), + encoding="utf-8") + except Exception as e: + print(f" [{i}/{len(candidates)}] write failed: {e}", + flush=True) + print(f" [{i}/{len(candidates)}] {kb_id} {title}", flush=True) + print(f" old: {old_topics[:3]}", flush=True) + print(f" new: (cleared — no tags above 0.50)", flush=True) + continue + d["topics"] = [t[0] for t in new] + try: + p.write_text(json.dumps(d, ensure_ascii=False, indent=2), + encoding="utf-8") + except Exception as e: + print(f" [{i}/{len(candidates)}] write failed: {e}", flush=True) + continue + print(f" [{i}/{len(candidates)}] {kb_id} {title}", flush=True) + print(f" old: {', '.join(old_topics[:3])}", flush=True) + print(f" new: {', '.join(f'{n}({s:.2f})' for n,s in new)}", flush=True) + + if fix_links: + print(f"\n--- pass 2: SKOS-aware tag-based re-link ---", flush=True) + # Load the SKOS hierarchy so we score on EXPANDED concept sets + # (article tagged 'bears' also matches via 'animals and pets', + # 'life sciences' ancestors). Falls back to pure-overlap when + # the hierarchy isn't loadable. + skos = None + try: + sys.path.insert(0, str(PROJECT_ROOT / "knowledge")) + from skos import get_hierarchy + skos = get_hierarchy() + print(f" SKOS loaded: {skos.stats()}", flush=True) + except Exception as e: + print(f" SKOS load failed ({e}); using flat tag overlap", flush=True) + + # Reload candidates so we have the FRESH topics we just wrote. + fresh = [] + for p, _ in candidates: + try: + d = json.loads(p.read_text(encoding="utf-8")) + fresh.append((p, d)) + except Exception: + continue + + # Pre-expand each article's tag set to include ancestors. + # Stored under ``_expanded_tags`` for quick reuse below. + for _, d in fresh: + tags = list(d.get("topics") or []) + if skos: + d["_expanded_tags"] = skos.expand(tags, with_ancestors=True) + else: + d["_expanded_tags"] = set(tags) + + # Inverted index: expanded_tag -> set(article_id) — picks up + # the SKOS ancestors so Bears can find Manatees via shared + # 'animals and pets' even though neither has it as a direct tag. + tag_to_articles = {} + for p, d in fresh: + for t in d["_expanded_tags"]: + tag_to_articles.setdefault(t, set()).add(p.stem) + + # Cache embeddings so we only call EMBED once per article. + emb_cache = {} + def _get_emb(p, d): + if p.stem in emb_cache: + return emb_cache[p.stem] + content = (d.get("summary") or "")[:2000] + try: + v = embed(content) + except Exception: + v = None + emb_cache[p.stem] = v + return v + + MAX_RELATED = 8 + for i, (p, d) in enumerate(fresh, 1): + kb_id = p.stem + title = (d.get("title") or "")[:70] + my_tags_direct = set(d.get("topics") or []) + my_tags_expanded = d["_expanded_tags"] + + # Candidates: any article sharing at least one expanded tag. + candidate_ids = set() + for t in my_tags_expanded: + candidate_ids |= tag_to_articles.get(t, set()) + candidate_ids.discard(kb_id) + + if not candidate_ids: + d["related"] = [] + d.pop("_expanded_tags", None) + p.write_text(json.dumps(d, ensure_ascii=False, indent=2), + encoding="utf-8") + print(f" [{i}/{len(fresh)}] {kb_id} {title}", flush=True) + print(f" related: 0 (no shared tags or ancestors)", flush=True) + continue + + # Score each candidate. Direct shared tags weight more + # than ancestor-only matches; embedding cosine breaks ties. + # composite = direct_shared * 1.0 + ancestor_shared * 0.5 + cos * 0.3 + my_emb = _get_emb(p, d) + scored = [] + for other_id in candidate_ids: + other_path = KB_DIR / f"{other_id}.json" + if not other_path.exists(): + continue + # Look up other's expanded tags via the in-memory list. + other_d = next((x[1] for x in fresh if x[0].stem == other_id), None) + if other_d is None: + continue + other_direct = set(other_d.get("topics") or []) + other_expanded = other_d.get("_expanded_tags") or other_direct + + direct_shared = len(my_tags_direct & other_direct) + ancestor_shared = len(my_tags_expanded & other_expanded) - direct_shared + if direct_shared == 0 and ancestor_shared == 0: + continue + + cos_score = 0.0 + if my_emb: + other_emb = _get_emb(other_path, other_d) + if other_emb: + cos_score = cosine(my_emb, other_emb) + composite = direct_shared + 0.5 * ancestor_shared + 0.3 * cos_score + scored.append((other_id, direct_shared, ancestor_shared, cos_score, composite)) + + scored.sort(key=lambda x: -x[4]) + top = scored[:MAX_RELATED] + d["related"] = [t[0] for t in top] + d.pop("_expanded_tags", None) + p.write_text(json.dumps(d, ensure_ascii=False, indent=2), + encoding="utf-8") + + print(f" [{i}/{len(fresh)}] {kb_id} {title}", flush=True) + for other_id, direct_shared, ancestor_shared, cos_score, _ in top[:5]: + op = KB_DIR / f"{other_id}.json" + ot = "" + if op.exists(): + try: + ot = (json.loads(op.read_text(encoding="utf-8")).get("title") or "")[:50] + except Exception: + pass + print(f" direct={direct_shared} anc={ancestor_shared} " + f"cos={cos_score:.2f} {other_id} {ot}", flush=True) + # Strip the temporary ``_expanded_tags`` from any unsaved entries + for _, d in fresh: + d.pop("_expanded_tags", None) + + # Mirror the freshly-written ``related[]`` into the typed + # LinkStore so data/links.jsonl gets populated from the + # in-place retag (without going through auto_link_entry). + try: + sys.path.insert(0, str(PROJECT_ROOT / "knowledge")) + from links import get_link_store, KIND_RELATED + ls = get_link_store() + mig = 0 + for p, d in fresh: + kb_id = p.stem + rel = d.get("related") or [] + # replace_outgoing clears any stale typed edges for + # this article first, then writes the new set. + # Score = 1.0 - (rank/N) so first item ranks highest. + n = len(rel) or 1 + targets = [(rid, max(0.0, 1.0 - i / n)) for i, rid in enumerate(rel)] + ls.replace_outgoing(kb_id, KIND_RELATED, targets, + metadata={"source": "retag_relink"}) + mig += len(targets) + print(f" typed edges written: {mig} ({ls.stats()})", flush=True) + except Exception as e: + print(f" typed edge mirror failed: {e}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/project/_system-maintenance/test_review.py b/project/_system-maintenance/test_review.py new file mode 100644 index 0000000..d1f526b --- /dev/null +++ b/project/_system-maintenance/test_review.py @@ -0,0 +1,197 @@ +"""test_review.py — Smoke test for the staged media review pipeline. + +Runs MEDIA_REVIEW against a single known-good record and asserts that +each stage produced non-trivial output. Prints a PASS/FAIL line per +assertion. Exit status: 0 if all pass, 1 if any fail. + +Usage (run from project/ root, leOS server up, Ollama serving): + python -m _system-maintenance.test_review # auto-pick a record + python -m _system-maintenance.test_review media_xxx # specific record id +""" + +import argparse +import json +import sys +import time +import urllib.request +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +RECORDS_DIR = PROJECT_ROOT / "media_library" / "records" +KB_DIR = PROJECT_ROOT / "knowledgebase" +LEOS_URL = "http://localhost:5000" + + +def post(instruction, args, timeout=900): + payload = json.dumps({"instruction": instruction, "args": args or {}}).encode("utf-8") + req = urllib.request.Request( + f"{LEOS_URL}/kernel/execute", + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def pick_record(): + """Return the record id of the shortest transcript-bearing record. + Shortest because the smoke test should be fast.""" + best = None + for p in sorted(RECORDS_DIR.glob("media_*.json")): + try: + d = json.loads(p.read_text()) + except Exception: + continue + a = d.get("analysis") or {} + tx = a.get("transcript") or "" + if not tx: + continue + if best is None or len(tx) < best[1]: + best = (p.stem, len(tx), d.get("title", "")) + return best + + +def assert_pass(label, ok, detail=""): + mark = "PASS" if ok else "FAIL" + print(f" [{mark}] {label}{(': ' + detail) if detail else ''}", flush=True) + return ok + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("record_id", nargs="?") + args = p.parse_args() + + if args.record_id: + rid = args.record_id + title = "(specified)" + else: + picked = pick_record() + if picked is None: + print("FAIL: no records with transcripts available") + sys.exit(1) + rid, _, title = picked + + print(f"=== test_review on {rid} ({title[:60]}) ===", flush=True) + + # Reachability check + try: + s = post("STATUS", {}) + if not s.get("ok"): + print(f"FAIL: STATUS not ok: {s}") + sys.exit(1) + except Exception as e: + print(f"FAIL: cannot reach leOS at {LEOS_URL}: {e}") + sys.exit(1) + + print("Running MEDIA_REVIEW...", flush=True) + t0 = time.time() + try: + r = post("MEDIA_REVIEW", {"record_id": rid}, timeout=900) + except Exception as e: + print(f"FAIL: MEDIA_REVIEW HTTP error: {e}") + sys.exit(1) + dt = time.time() - t0 + + if not r.get("ok"): + print(f"FAIL: MEDIA_REVIEW returned not-ok: {r}") + sys.exit(1) + + review = r.get("review") or {} + if not review: + print(f"FAIL: review payload empty") + sys.exit(1) + + print(f" completed in {dt:.1f}s\n", flush=True) + print("Stage assertions:") + + results = [] + results.append(assert_pass( + "stage 0 (segments)", + len(review.get("segments") or []) >= 1, + f"{len(review.get('segments') or [])} segments", + )) + results.append(assert_pass( + "stage 1+2 (summary)", + len(review.get("summary") or "") >= 50, + f"{len(review.get('summary') or '')} chars", + )) + results.append(assert_pass( + "stage 1+2 (themes)", + len(review.get("themes") or []) >= 1, + f"{len(review.get('themes') or [])} themes", + )) + results.append(assert_pass( + "stage 1+2 (facts)", + len(review.get("facts") or []) >= 1, + f"{len(review.get('facts') or [])} facts", + )) + results.append(assert_pass( + "stage 1+2 (entities)", + len(review.get("entities") or []) >= 1, + f"{len(review.get('entities') or [])} entities", + )) + results.append(assert_pass( + "stage 1+2 (claims)", + len(review.get("claims") or []) >= 1, + f"{len(review.get('claims') or [])} claims", + )) + results.append(assert_pass( + "stage 3 (RAPTOR)", + len(review.get("raptor") or []) >= 1, + f"{len(review.get('raptor') or [])} nodes, " + f"{len(review.get('raptor_roots') or [])} roots", + )) + tags = review.get("tags") or [] + top_score = max((t.get("score") or 0.0) for t in tags) if tags else 0.0 + results.append(assert_pass( + "stage 4 (auto-tag domain match)", + top_score >= 0.6, + f"top score {top_score:.3f}, {len(tags)} tags", + )) + # Stage 5 linked_episodes can legitimately be empty if corpus is small + results.append(assert_pass( + "stage 5 (linking ran)", + "linked_episodes" in review, + f"{len(review.get('linked_episodes') or [])} links", + )) + results.append(assert_pass( + "stage 6 (knowledge graph)", + len(review.get("knowledge_graph") or []) >= 1, + f"{len(review.get('knowledge_graph') or [])} triples", + )) + kb_id = review.get("kb_article_id") + results.append(assert_pass( + "stage 7 (KB save)", + bool(kb_id), + kb_id or "no kb_article_id", + )) + if kb_id: + kb_path = KB_DIR / f"{kb_id}.json" + results.append(assert_pass( + "stage 7 (KB file on disk)", + kb_path.exists(), + str(kb_path), + )) + if kb_path.exists(): + try: + article = json.loads(kb_path.read_text()) + results.append(assert_pass( + "stage 7 (article structured)", + bool(article.get("content")) and rid in (article.get("media_refs") or []), + f"content {len(article.get('content','')):,} chars, " + f"media_refs links back: " + f"{rid in (article.get('media_refs') or [])}", + )) + except Exception as e: + results.append(assert_pass("stage 7 (article parseable)", False, str(e))) + + passed = sum(1 for r in results if r) + total = len(results) + print(f"\n=== {passed}/{total} assertions passed ===") + sys.exit(0 if passed == total else 1) + + +if __name__ == "__main__": + main() diff --git a/project/agent/agent_chat_api_chat.py b/project/agent/agent_chat_api_chat.py index 986839a..8b10711 100644 --- a/project/agent/agent_chat_api_chat.py +++ b/project/agent/agent_chat_api_chat.py @@ -749,7 +749,7 @@ def _handle_intern_routing( # embedding model: the brain's Step 1 embed + Step 2.5 # KB hybrid search would just barely outrun the cap on # hot models, but cold-model first-query timed out - # reliably. See the 2026-04-25 Xavier regression where + # reliably. See the token-triage regression (2026-04-25) where # substrate_gather had a strong KB hit available but # never got to use it because triage gave up at 25s. # - Now 90s — generous enough that the cold path @@ -880,7 +880,7 @@ def _run_triage(): # to the needs_agent dialog at the bottom of the function, # which routes EVERY remaining message to main_agent. # - # That was the 2026-04-25 Xavier regression part 2: + # That was the token-triage regression (2026-04-25) part 2: # - substrate produced format_combined conf=0.80 in # 643ms (KB content + dexscreener + geckoterminal) # - brain wire-in set substrate_short_circuited=True @@ -1082,7 +1082,7 @@ def _run_triage(): # machinery, which would launch the intern in a # fresh tool-loop and overwrite substrate's render. # - # Without this gate the 2026-04-25 Xavier regression + # Without this gate the token-triage regression (2026-04-25) # happens: substrate surfaces format_combined at # conf=0.80 in 654ms, brain returns with response set, # was_hedged_late evaluates True (because voronoi_ratio @@ -1126,7 +1126,7 @@ def _run_triage(): # below would route EVERY force_relay=True to main_agent, # including research queries the brain's router had # already pre-analyzed for intern-tier dispatch. That's - # the Moose regression from 2026-04-24: "generate a + # the research-relay regression from 2026-04-24: "generate a # report about a solana token: " classified as # research, brain's _handle_research returned # handoff_with_prefill, force_relay=True was set -- and diff --git a/project/agent/agent_fabricator.py b/project/agent/agent_fabricator.py index 5cfdbb9..af29929 100644 --- a/project/agent/agent_fabricator.py +++ b/project/agent/agent_fabricator.py @@ -1,14 +1,14 @@ """ agent_fabricator.py - Dynamic Agent Fabrication (Phase D4) -Adapted from the old agent_fabricator.py. The original used preset JSON -files and a qwen-agents tool-list format. This version: +Constructs an agent configuration (system prompt, tool selection, +temperature, max rounds, confidence) for a given task: - - Uses prompt_focus.py for system prompt composition (fragment library) - - Uses tool_relevance.score_tools() for tool selection from AGENT_TOOLS - - Keeps the blueprint library (JSONL for append-only storage) - - Keeps temperature calibration (keyword heuristic + displacement stats) - - Keeps confidence scoring (simplified — no preset similarity needed) + - prompt_focus.py for system prompt composition (fragment library) + - tool_relevance.score_tools() for tool selection from AGENT_TOOLS + - Blueprint library (JSONL, append-only) for reuse across similar tasks + - Temperature calibration (keyword heuristic + displacement stats) + - Confidence scoring from tool coverage and relevance quality Output format: { @@ -439,3 +439,119 @@ def fabricate_agent(task_description, kernel=None, role="agent", _save_blueprint(blueprint_entry) return config + + +# =================================================================== +# spawn_from_script — one-shot scripted bot spawner +# =================================================================== +# +# Wraps BOT_CREATE + BOT_START with a manifest-driven config so the +# caller can hand off a list of {instruction, args} pairs and get back +# a running bot id. Pairs with bots/bot_runner_perceive._perceive_manifest +# and bots/bot_runner_act._act_agent_session. See those for the full +# execution lifecycle. +# +# Cap: refuses to spawn when more than MAX_CONCURRENT_SCRIPTED bots are +# already running scripted work — single-GPU machines cannot +# usefully fan out beyond a small handful of agent sessions. +# =================================================================== + +MAX_CONCURRENT_SCRIPTED = 5 + + +def spawn_from_script(script_id, steps, kernel, + output_scope=None, created_by="agent_fabricator", + description=None): + """Spawn a one-shot bot that executes ``steps`` (a list of + ``{instruction, args}`` dicts) in order. + + The bot self-terminates after the manifest finishes; results land + on the observation ledger and (optionally) on ``output_scope`` as + a SCOPE_NOTE summary. + + Args: + script_id: Short identifier for this job (used in the bot + name and observation notes). + steps: List of ``{"instruction": str, "args": dict}`` + entries to execute in order. + kernel: Live kernel instance. + output_scope: Optional scope_id to attach a completion note. + created_by: Attribution for the bot record. + description: Optional human-readable description; defaults to + ``"Scripted job: {script_id} (N steps)"``. + + Returns: + dict ``{ok, bot_id, message}``. + """ + if not script_id: + return {"ok": False, "bot_id": None, + "message": "script_id required"} + if not isinstance(steps, list) or not steps: + return {"ok": False, "bot_id": None, + "message": "steps must be a non-empty list"} + + # Concurrency cap: single-GPU machine, agent_session expensive. + try: + list_result = kernel.execute("BOT_LIST", {"status": "running"}) + running = list_result.get("bots") if isinstance(list_result, dict) else [] + scripted_running = [ + b for b in (running or []) + if (b.get("config", {}).get("perceive", {}).get("type") + == "perceive_manifest") + ] + if len(scripted_running) >= MAX_CONCURRENT_SCRIPTED: + return { + "ok": False, "bot_id": None, + "message": ( + f"refused: {len(scripted_running)} scripted bots " + f"already running (cap {MAX_CONCURRENT_SCRIPTED})" + ), + } + except Exception: + pass # if BOT_LIST fails, fall through and let BOT_CREATE decide + + bot_name = f"script_{script_id}_{int(time.time()) % 100000}" + config = { + "perceive": { + "type": "perceive_manifest", + "script_id": script_id, + "steps": steps, + "output_scope": output_scope or "", + }, + "evaluate": { + "type": "evaluate_always", + }, + "act": { + "type": "act_agent_session", + "output_scope": output_scope or "", + }, + "schedule": { + # interval=0 means "fire on next scheduler tick"; the bot + # self-stops after one cycle so it won't re-fire. + "interval_seconds": 0, + "max_cycles": 1, + }, + "description": ( + description + or f"Scripted job: {script_id} ({len(steps)} steps)" + ), + } + + create_r = kernel.execute("BOT_CREATE", { + "name": bot_name, + "config": config, + "created_by": created_by, + }) + if not isinstance(create_r, dict) or not create_r.get("ok"): + return { + "ok": False, "bot_id": None, + "message": (create_r or {}).get("error", "BOT_CREATE failed"), + } + bot_id = create_r.get("bot_id", "") + + start_r = kernel.execute("BOT_START", {"bot_id": bot_id}) + return { + "ok": bool(isinstance(start_r, dict) and start_r.get("ok")), + "bot_id": bot_id, + "message": f"Scripted bot {bot_id} started: {script_id}", + } diff --git a/project/agent/leos_orchestrator.py b/project/agent/leos_orchestrator.py index 754d35c..872e186 100644 --- a/project/agent/leos_orchestrator.py +++ b/project/agent/leos_orchestrator.py @@ -1,35 +1,14 @@ """ -leos_orchestrator.py — Simplified orchestration for leOS +leos_orchestrator.py — Orchestration entry point for leOS -Replaces the old 8600-line orchestrator.py + 1800-line agent_factory.py -that coordinated multiple qwen-agent Assistant instances through a -Director/Worker pattern. - -WHY THIS IS SIMPLER: - The old system had separate agents (Director, DevOps, Librarian, Coder, - Researcher) each with a subset of tools. The Director created a plan, - delegated steps to workers, reviewed their output, and replanned. This - required 8600 lines of orchestration because every hand-off between - agents needed context marshalling, plan tracking, loop detection, - course correction, timeout management, and a blizzard of callbacks. - - leOS has ONE agent with 14 tools. The tools ARE the specialists. - The agent plans, executes, and reviews in a single session. The - tool-calling loop in agent_session.py handles the generate → tool_call - → execute → feed-back cycle automatically. - - Result: ~300 lines instead of ~10,400. +leOS runs a single agent with 14 tools instead of a multi-agent +Director/Worker hierarchy. The tools are the specialists; the +tool-calling loop in agent_session.py handles generate → tool_call +→ execute → feed-back automatically. INTERFACES PROVIDED: run_directed() — Drop-in replacement called by task_scheduler.py build_agents() — Returns an agent session (task_scheduler expects this) - -MODULES THAT CALL THESE: - task_scheduler.py line 891-892: from agent_factory import build_agents - from orchestrator import run_directed - idle_maintenance.py (triggers Director sessions via scheduler) - -Both of those modules can now import from here instead. """ import os @@ -46,8 +25,6 @@ # =================================================================== # Config utilities (used by state.py and others) # =================================================================== -# These were originally in agent_factory.py. state.py imports them -# at module level, so they need to be available early. def load_global_config(config_path=None): """Load the global config.json file. @@ -461,12 +438,10 @@ def run_directed(agents, histories, user_message, except ImportError: fabricate_agent = None - # Consecutive-stall guard (added 2026-04-14). Mirror of - # the pattern used in task_orchestrator.run_orchestrated. - # If two subtasks in a row stall (forced-stop with zero - # tool calls), abort the whole chain — continuing would - # chain more stalls and burn another ~10 min each. The - # counter resets on any non-stall subtask. + # Consecutive-stall guard. If two subtasks in a row stall + # (forced-stop with zero tool calls), abort the whole chain — + # continuing chains more stalls at ~10 min each. + # Resets on any non-stall subtask. consecutive_subtask_stalls = 0 for _st_idx, subtask in enumerate(subtasks): @@ -690,14 +665,11 @@ def run_directed(agents, histories, user_message, except Exception: pass - # -------------------------------------------------- - # Consecutive-stall bail-out (2026-04-14) - # -------------------------------------------------- - # Inspect the most-recent result to see if this - # subtask exited on stall (zero tool calls in the - # checkin window). AgentSession.run() demotes - # ok to False and carries stop_cause forward. On - # a stall we increment; on anything else we reset. + # Consecutive-stall bail-out. + # If this subtask exited on stall (zero tool calls + # in the checkin window), increment; otherwise reset. + # AgentSession.run() demotes ok to False and carries + # stop_cause forward on a stall. # Two stalls in a row aborts the chain. _last = all_results[-1] if all_results else {} if _last.get("stop_cause") == "stall": @@ -972,8 +944,22 @@ def _spawn_bot_subtask(subtask, kernel): # Build a minimal bot config. The perceive step reads the manifest # from the KB if one was provided; otherwise it uses the description. perceive_config = { - "type": "text", - "description": description, + # ``perceive_manifest`` + ``act_agent_session`` is the + # registered, validated component pair for one-shot scripted + # bots. The previous "type=text"/"type=agent" pairing was a + # silent BOT_CREATE validation failure (neither type lives in + # KNOWN_COMPONENTS) — fixed here. + "type": "perceive_manifest", + "script_id": subtask.get("id") or "subtask", + "steps": manifest or [ + # Default fallback: single ESCALATE step with the description + # as the prompt. Preserves the prior intent of "let the + # primary agent figure out how to respond" without needing + # a registered "agent" act type. + {"instruction": "ESCALATE", + "args": {"prompt": description}}, + ], + "output_scope": subtask.get("scope_id", "") or "", } if manifest: perceive_config["manifest"] = manifest @@ -981,12 +967,11 @@ def _spawn_bot_subtask(subtask, kernel): bot_config = { "perceive": perceive_config, "evaluate": { - "type": "always", - "threshold": 0.0, + "type": "evaluate_always", }, "act": { - "type": "agent", - "instruction": description, + "type": "act_agent_session", + "output_scope": subtask.get("scope_id", "") or "", }, "schedule": { "interval_seconds": 0, # run once, then stop diff --git a/project/agent/task_orchestrator.py b/project/agent/task_orchestrator.py index a30964a..03d4968 100644 --- a/project/agent/task_orchestrator.py +++ b/project/agent/task_orchestrator.py @@ -48,14 +48,13 @@ # ───────────────────────────────────────────────────────────────────── -# Plan 22 — retry escalation: read user_correction notes for a scope +# User-correction reader for planner prompts # ───────────────────────────────────────────────────────────────────── -# Hard cap on how much user-correction text we feed into a single -# planner prompt. The intern is a 0.6-0.8B model; rambling feedback -# easily destabilises its output (per plan 22 Risk 2). 1000 chars per -# entry, 3 entries max — newest first — gives the planner the most -# recent corrective signal without overwhelming the prompt. +# Hard cap on user-correction text fed into a single planner prompt. +# The intern is a 0.6-0.8B model: rambling feedback destabilises its +# output. 1000 chars × 3 entries (newest first) gives the most +# recent corrective signal without overwhelming the context. # _FOLLOWUP_PATTERNS _FOLLOWUP_PATTERNS = { @@ -762,6 +761,29 @@ def run_orchestrated(agent_session, messages, scope_id, system_prompt, # stall; the threshold below (2) aborts the plan to return control. consecutive_stalls = 0 + # ── Hard wallclock cap ───────────────────────────────────────── + # The orchestrator has no absolute time ceiling by design — long + # research tasks can legitimately run for hours. But token + # reports and similar API-bounded workflows are different: the + # data is either available in seconds or it isn't there at all. + # A run that hits the cap without producing complete steps has + # entered the force_replan death spiral; more time won't recover. + # + # Configurable via config.json ``orchestrator_hard_cap_minutes`` + # (default 0 = disabled, preserves old behaviour). Recommended: + # set to 30 for token-report style workflows. When the cap + # fires, the loop breaks and the partial-result assembler runs + # — the user gets whatever was actually gathered, not a + # template fabricated from an empty context. + _hard_cap_s = 0 + try: + from state import CONFIG + _cap_min = int((CONFIG or {}).get("orchestrator_hard_cap_minutes", 0)) + if _cap_min > 0: + _hard_cap_s = _cap_min * 60 + except Exception: + pass + # ── Plan 22: pre-fetch user_correction notes ────────────────── # When this orchestrator run is the result of a user clicking # "Try again" on a previous response, retry feedback is written @@ -802,6 +824,35 @@ def run_orchestrated(agent_session, messages, scope_id, system_prompt, "at step %d/%d", step_num, len(plan_steps)) break + # ── Hard wallclock cap check ── + # Fires before each step so the cap is exact to within one + # step boundary. Falls through to the same partial-result + # assembler that cancellation uses. See ``_hard_cap_s`` + # initialisation above for the rationale. + if _hard_cap_s > 0: + try: + _elapsed_so_far = _time.perf_counter() - start_time + except Exception: + _elapsed_so_far = 0.0 + if _elapsed_so_far > _hard_cap_s: + logger.warning( + "Orchestrator hard cap fired at %.0fs (cap=%ds, " + "step=%d/%d) — returning partial results", + _elapsed_so_far, _hard_cap_s, + step_num, len(plan_steps), + ) + try: + _emit_thought( + kernel, + f"Hard time cap reached at " + f"{int(_elapsed_so_far / 60)}m — returning what " + f"was gathered (step {step_num}/{len(plan_steps)})", + "hard_cap", 0.95, drift_type="redshift", + ) + except Exception: + pass # thought emission is diagnostic, never blocks + break + # ── Early completion check (issue #2 fix) ── # Before doing more work, check if the accumulated answers # already satisfy the user's original request. Most "simple" @@ -1632,7 +1683,7 @@ def run_orchestrated(agent_session, messages, scope_id, system_prompt, # itself (the intern/agent confabulates when its tool loop has no # new tool calls to report on). That summary must NEVER become the # user-facing final answer because it LOOKS like a report but - # contains no actual findings. See the Xavier: Renegade Angel bug: + # contains no actual findings. Documented failure mode: # after a plan aborted at step 4 with two consecutive stalls, the # stall-summary — a template full of phrases like "market # capitalization tracked" and "whale concentration analyzed" — diff --git a/project/agent/tool_bridge.py b/project/agent/tool_bridge.py index 43c7c99..30bc482 100644 --- a/project/agent/tool_bridge.py +++ b/project/agent/tool_bridge.py @@ -84,12 +84,33 @@ def _call_basetool(module_name, class_name, params): start = time.perf_counter() try: - # BaseTool.call() can accept dict or JSON string + # BaseTool.call() expects a JSON string (calls json5.loads + # internally), but /kernel/execute and tests pass dicts. + # Converting here avoids 40 redundant per-subclass dict checks + # and the silent "Error: Could not parse" failure mode. + if isinstance(params, dict): + params = json.dumps(params) + result = tool.call(params) elapsed = (time.perf_counter() - start) * 1000 # Result might be a string (formatted output) or dict if isinstance(result, str): + # Many BaseTool subclasses return json.dumps(...) for + # structured output. Parse and merge so callers can read + # fields (results, count, status, …) directly off ``res``. + stripped = result.lstrip() + if stripped.startswith("{"): + try: + parsed = json.loads(result) + if isinstance(parsed, dict): + parsed["ok"] = parsed.get("ok", parsed.get("status") != "error") + parsed["elapsed_ms"] = elapsed + # Preserve raw string for tools/UI that want it + parsed.setdefault("output", result) + return parsed + except Exception: + pass return {"ok": True, "output": result, "elapsed_ms": elapsed} elif isinstance(result, dict): result["ok"] = result.get("ok", True) @@ -122,6 +143,17 @@ def _call_run_func(module_name, params, base_dir=None): elapsed = (time.perf_counter() - start) * 1000 if isinstance(result, str): + stripped = result.lstrip() + if stripped.startswith("{"): + try: + parsed = json.loads(result) + if isinstance(parsed, dict): + parsed["ok"] = parsed.get("ok", parsed.get("status") != "error") + parsed["elapsed_ms"] = elapsed + parsed.setdefault("output", result) + return parsed + except Exception: + pass return {"ok": True, "output": result, "elapsed_ms": elapsed} elif isinstance(result, dict): result["ok"] = result.get("ok", True) @@ -352,6 +384,8 @@ def handler(self, args): # ---- KB search (was orphaned) ---- "TOOL_KB_SEARCH": ("kb_search", "KBSearchTool"), + # ---- KB nearest-K via cached embedding matrix ---- + "TOOL_KB_NEAREST": ("kb_nearest", "KBNearestTool"), # ---- Reference lookup (was orphaned) ---- "TOOL_REF_LOOKUP": ("ref_lookup", "RefLookupTool"), @@ -364,6 +398,7 @@ def handler(self, args): "TOOL_MEDIA_FETCH": ("media_fetch", "MediaFetch"), "TOOL_MEDIA_SEARCH": ("media_search", "MediaSearch"), "TOOL_VIDEO_INFO": ("video_info", "VideoInfoTool"), + "TOOL_PODCAST": ("podcast", "Podcast"), # ---- Web3/Crypto extras (was orphaned) ---- "TOOL_PRICE_HISTORY": ("price_history", "PriceHistoryTool"), diff --git a/project/agent/tool_registry.py b/project/agent/tool_registry.py index c626ad4..eded1f1 100644 --- a/project/agent/tool_registry.py +++ b/project/agent/tool_registry.py @@ -261,6 +261,13 @@ "group": None, "desc": "Media library: list all stored media, search by description, " "ingest new media from a URL or local path."}, + {"name": "podcast", "domain": "media", + "group": None, + "desc": "Podcast tool: search Apple's public podcast directory by name, " + "fetch and parse RSS feeds into clean episode lists, and ingest " + "selected episodes (all / latest N / index / range / name match) " + "into the media library — Whisper transcription and embedding " + "happen automatically through the existing audio pipeline."}, # ── Data & charts ───────────────────────────────────────────── {"name": "chart_create", "domain": "data", diff --git a/project/bots/ambient_reactor.py b/project/bots/ambient_reactor.py index 45b7598..97335cb 100644 --- a/project/bots/ambient_reactor.py +++ b/project/bots/ambient_reactor.py @@ -438,6 +438,12 @@ def start(self) -> None: self._on_signal, subscriber_id="ambient_reactor", serial=True, + # Default max_queue=10 fills during the boot burst of + # kernel.instruction signals (which the handler then + # filters out internally — but they consume queue + # slots before being filtered). 500 absorbs even a + # heavy warmup without dropping admin signals. + max_queue=500, ) self._subscribed = True except Exception as e: diff --git a/project/bots/bot_factory.py b/project/bots/bot_factory.py index cda97ff..9cfb521 100644 --- a/project/bots/bot_factory.py +++ b/project/bots/bot_factory.py @@ -455,11 +455,12 @@ def save_template(name, template_data): KNOWN_COMPONENTS = { "perceive": { - "perceive_web", # fetch a URL and extract text - "perceive_rss", # parse an RSS/Atom feed - "perceive_file", # read a local file for changes - "perceive_api", # call a JSON API endpoint - "perceive_dir", # scan a directory for new/changed files + "perceive_web", # fetch a URL and extract text + "perceive_rss", # parse an RSS/Atom feed + "perceive_file", # read a local file for changes + "perceive_api", # call a JSON API endpoint + "perceive_dir", # scan a directory for new/changed files + "perceive_manifest", # load a script manifest (list of {instruction,args}) }, "evaluate": { "evaluate_threshold", # compare a numeric field to a value @@ -469,12 +470,13 @@ def save_template(name, template_data): "evaluate_always", # always trigger (useful for logging bots) }, "act": { - "act_record", # write observation to the observation ledger - "act_alert", # send an alert (inbox message or log) - "act_file", # write result to a file - "act_kb", # save finding to the knowledge base - "act_escalate", # escalate to the Director for review - "act_chain", # run multiple actions in sequence + "act_record", # write observation to the observation ledger + "act_alert", # send an alert (inbox message or log) + "act_file", # write result to a file + "act_kb", # save finding to the knowledge base + "act_escalate", # escalate to the Director for review + "act_chain", # run multiple actions in sequence + "act_agent_session", # run a manifest of kernel instructions in order }, } diff --git a/project/bots/bot_runner_act.py b/project/bots/bot_runner_act.py index 8410a41..fdf988d 100644 --- a/project/bots/bot_runner_act.py +++ b/project/bots/bot_runner_act.py @@ -1,9 +1,6 @@ """ bot_runner_act.py - Act stage — take actions in response to signals. -Split from bot_runner.py during Phase 7 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. - Defines 19 action functions. Each _act_* function performs one type of side effect (record, alert, file, chain, ingest, partition, spawn_bot, kernel, displace, transform, emit, merge, tag, link, @@ -69,11 +66,132 @@ def _act(config, data, bot): return _act_llm(config, data, bot) elif atype == "act_paper_trade": return _act_paper_trade(config, data, bot) + elif atype == "act_agent_session": + return _act_agent_session(config, data, bot) else: print(f" [BotRunner] Unknown act type: {atype}") return False +def _act_agent_session(config, data, bot): + """Execute a manifest of kernel instructions in order. + + Pairs with ``perceive_manifest``: the perceive stage loads + ``data["steps"]`` (a list of ``{instruction, args}`` dicts), and + this act handler iterates them, calling ``kernel.execute()`` for + each. Each step's outcome is logged to the observation ledger as + a progress note, and a final summary note is written to + ``output_scope`` if provided. + + The bot self-terminates by setting its own status to ``"stopped"`` + on exit so the scheduler does not re-fire it (one-shot scripted + bots are configured with ``interval_seconds=0`` and rely on + self-termination since the scheduler's ``max_cycles`` field is + informational only). + + Cancellation: between every step the handler re-fetches its own + bot record and exits early when status flips to ``"stopped"`` — + this lets ``BOT_STOP`` interrupt a long manifest in flight. + """ + steps = data.get("steps") or [] + output_scope = data.get("output_scope") or config.get("output_scope") or "" + script_id = data.get("script_id") or config.get("script_id") or "unnamed" + bot_id = bot.get("bot_id") or bot.get("id") or "" + + if not steps: + print(f" [BotRunner] act_agent_session: no steps in manifest " + f"(script_id={script_id!r}, bot={bot_id})") + # Still self-terminate so the bot doesn't loop forever on an + # empty manifest. + try: + import bot_factory as _bf + _bf.update_bot_status(bot_id, "stopped") + except Exception: + pass + return False + + try: + import kernel as _kern + import bot_factory as _bf + except ImportError as e: + print(f" [BotRunner] act_agent_session: missing dependency: {e}") + return False + + completed = 0 + errors = [] + + for i, step in enumerate(steps): + # Cancellation check + try: + current = _bf.get_bot(bot_id) or {} + if current.get("status") == "stopped": + print(f" [BotRunner] act_agent_session: cancelled at " + f"step {i + 1}/{len(steps)} (bot={bot_id})") + break + except Exception: + pass + + instruction = step.get("instruction", "") if isinstance(step, dict) else "" + args = step.get("args", {}) if isinstance(step, dict) else {} + if not instruction: + errors.append(f"step {i + 1}: missing instruction") + continue + + ok = False + note_detail = "" + try: + result = _kern.execute(instruction, args) + ok = bool(result.get("ok", True)) if isinstance(result, dict) else True + note_detail = ( + "ok" if ok + else (result.get("error", "?") if isinstance(result, dict) else "?") + ) + if ok: + completed += 1 + except Exception as e: + note_detail = f"exception: {e}" + errors.append(f"step {i + 1} {instruction}: {note_detail}") + + # Progress note to the observation ledger + try: + import observation_ledger as _ol + _ol.write( + text=f"step {i + 1}/{len(steps)} {instruction}: {note_detail}", + source=f"bot:{bot_id}", + service_id=bot.get("service_id", "global"), + obs_type="progress", + confidence=1.0 if ok else 0.5, + ) + except Exception: + pass + + # Final summary + summary = ( + f"Script {script_id} done: {completed}/{len(steps)} steps ok, " + f"{len(errors)} errors" + ) + if output_scope: + try: + _kern.execute("SCOPE_NOTE", { + "scope_id": output_scope, + "note": summary, + "author": f"bot:{bot_id}", + }) + except Exception: + pass + + # Self-terminate so the scheduler does not re-fire this one-shot. + try: + _bf.update_bot_status(bot_id, "stopped") + except Exception: + pass + + # Surface counts for caller introspection (e.g. tests). + data["_script_completed"] = completed + data["_script_errors"] = errors + return completed > 0 + + def _act_record(config, data, bot): """Record the observation to the observation log.""" msg_template = config.get("message_template", "{observation}") diff --git a/project/bots/bot_runner_perceive.py b/project/bots/bot_runner_perceive.py index dc8cffe..4c853e1 100644 --- a/project/bots/bot_runner_perceive.py +++ b/project/bots/bot_runner_perceive.py @@ -1,12 +1,9 @@ """ bot_runner_perceive.py - Perceive stage — data gathering from sources. -Split from bot_runner.py during Phase 7 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. - Defines 16 perceive functions plus an HTTP retry helper. -Each _perceive_* function knows how to gather data from one source -type (web, api, rss, file, dir, site, kb, partition, observation, +Each _perceive_* function gathers data from one source type +(web, api, rss, file, dir, site, kb, partition, observation, kernel, diff, multi, output, port). Public dispatcher: _perceive(config) -- reads config["perceive_type"] @@ -29,10 +26,8 @@ from bot_runner_chart import _perceive_chart_analysis from bot_runner_helpers import _extract_xml_field, _get_last_observation -# Tunables re-imported from the shim. Keeping these in the shim -# preserves MAX_RESPONSE_SIZE as the single source of truth and -# keeps external callers like app_executor.py able to reach them -# at bot_runner.HTTP_TIMEOUT (same module attribute as before the split). +# Tunables re-imported from the shim so they remain accessible at +# bot_runner.HTTP_TIMEOUT / MAX_RESPONSE_SIZE for external callers. from bot_runner import HTTP_TIMEOUT, MAX_RESPONSE_SIZE @@ -74,11 +69,50 @@ def _perceive(config): return _perceive_port(config) elif ptype == "perceive_chart_analysis": return _perceive_chart_analysis(config) + elif ptype == "perceive_manifest": + return _perceive_manifest(config) else: print(f" [BotRunner] Unknown perceive type: {ptype}") return None +def _perceive_manifest(config): + """Load a script manifest — a sequence of {instruction, args} pairs. + + Two source modes: + * Inline ``steps`` in config (preferred for one-shot scripted bots). + * ``scope_id`` reference — load the manifest from a SCOPE note, + useful when the script is generated dynamically and parked in a + scope before the bot is spawned. + + Returns: + dict with ``steps``, ``script_id``, ``output_scope``, ``text``. + Empty ``steps`` is allowed; the act handler treats it as a no-op + and self-terminates the bot. + """ + steps = config.get("steps") or config.get("manifest") or [] + if not steps and config.get("scope_id"): + try: + import kernel as _kern + r = _kern.execute("SCOPE_GET", {"scope_id": config["scope_id"]}) + import json as _json + content = (r or {}).get("content") or "[]" + parsed = _json.loads(content) + if isinstance(parsed, list): + steps = parsed + except Exception: + steps = [] + return { + "steps": list(steps or []), + "script_id": config.get("script_id", "unnamed"), + "output_scope": config.get("output_scope", ""), + "text": ( + f"manifest:{config.get('script_id', 'unnamed')} " + f"({len(steps)} steps)" + ), + } + + def _http_fetch_with_retry(url, headers, max_retries=3, timeout=HTTP_TIMEOUT): """Fetch a URL with exponential backoff retry. @@ -161,7 +195,7 @@ def _perceive_web(config): return None headers = { - "User-Agent": "MooseBot/1.0 (research automation)", + "User-Agent": "leOS-Bot/1.0 (research automation)", } # Allow per-bot custom headers (e.g. Authorization: Bearer ) extra_headers = config.get("headers", {}) @@ -205,7 +239,7 @@ def _perceive_api(config): return None headers = { - "User-Agent": "MooseBot/1.0", + "User-Agent": "leOS-Bot/1.0", "Accept": "application/json", } extra_headers = config.get("headers", {}) @@ -246,7 +280,7 @@ def _perceive_rss(config): try: req = urllib.request.Request(url, headers={ - "User-Agent": "MooseBot/1.0", + "User-Agent": "leOS-Bot/1.0", }) with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as resp: xml_text = resp.read(MAX_RESPONSE_SIZE).decode("utf-8", errors="replace") diff --git a/project/bots/companion_brain.py b/project/bots/companion_brain.py index cb11654..cc87f02 100644 --- a/project/bots/companion_brain.py +++ b/project/bots/companion_brain.py @@ -304,7 +304,7 @@ def handle(self, text, status=None, recent_turns=None): # surface threshold), short-circuit and return without invoking # any LLM-based handler. # - # This is the fix for the Xavier 344s regression: queries that + # This is the fix for the token-report 344s regression: queries that # leOS could answer from its KB were instead going to the # research router → handoff_open → main_agent loop, which # spent multiple minutes calling tools that returned the same @@ -379,7 +379,7 @@ def handle(self, text, status=None, recent_turns=None): # intern auto-escalation -- overriding # substrate's render with the intern's slow # tool-loop output. See the 2026-04-25 - # Xavier regression where substrate + # token-triage regression where substrate # surfaced format_combined conf=0.80 in # 654ms and the chat API immediately # auto-escalated to a 41s intern session diff --git a/project/bots/companion_brain_seeds.py b/project/bots/companion_brain_seeds.py index cc31dcf..b225629 100644 --- a/project/bots/companion_brain_seeds.py +++ b/project/bots/companion_brain_seeds.py @@ -273,14 +273,14 @@ # # History: # - Originally 0.70 ("solid" band). Worked for the simple cases -# but failed the Xavier "generate a report" case: substrate +# but failed the token-report case: substrate # found 10 strong KB hits at 0.82 similarity, the data was # stale-by-policy (24h crypto), the math landed at 0.45 (later # 0.60 after the 4-case stale tuning), and the brain fell # through to a 376-second intern loop that produced no answer. # The KB content was right there and the user never saw it. # - Now 0.55 ("low-hedged" band). At this threshold, the -# Xavier case (conf=0.60 with 10 KB hits, stale crypto, non-TS) +# token-triage case (conf=0.60 with 10 KB hits, stale crypto, non-TS) # surfaces with appropriate caveats. Substrate's renderer # already includes a "*Note: this comes from stored knowledge # dated ; it may be out of date.*" line for stale data, so diff --git a/project/bots/kb_reflection_listener.py b/project/bots/kb_reflection_listener.py new file mode 100644 index 0000000..0268e0b --- /dev/null +++ b/project/bots/kb_reflection_listener.py @@ -0,0 +1,340 @@ +"""kb_reflection_listener.py — Autonomous reflection on new KB articles. + +Subscribes to ``kb.article_saved`` via the signal bus. When a fresh KB +article is saved (e.g. by the staged review pipeline), spawns one +``REFLECT`` call so the agent narrates its thoughts about the new +content. That call produces: + + - a `monologue_*.jsonl` event log under voice/data/monologues/ + - rendered narration audio via ChatterboxTTS (deferred to idle if + monologue_renderer's auto_defer_on_active_use is true) + - a thinking_log post-mortem (written by REFLECT internally) + - splat / drift state on the /thought canvas + +Mirrors `bots/ambient_reactor.py` for the subscription pattern: +serial=True, bounded max_queue, dedup by article_id, blackboard error +notes on failure. No new threading primitives — everything runs on +the signal bus's own serial worker. +""" + +from __future__ import annotations + +import logging +import threading +import time +from typing import Any, Dict, Optional, Set + +logger = logging.getLogger("kb_reflection_listener") + +_DEFAULT_CFG: Dict[str, Any] = { + "enabled": True, + "max_queue": 10, + "debounce_s": 2.0, + "render_audio": True, + "min_importance": 0.0, + "source_agent_filter": ["review"], +} + + +class KBReflectionListener: + """One reflection per fresh article. Single-process, single-worker.""" + + def __init__(self, kernel, bus=None, config: Optional[Dict[str, Any]] = None): + self.kernel = kernel + self._cfg = {**_DEFAULT_CFG, **(config or {})} + self._running = False + self._subscribed = False + self._reflected_ids: Set[str] = set() + self._dedup_lock = threading.Lock() + + if bus is None: + try: + import signal_bus + bus = signal_bus.get_bus() + except Exception as e: + logger.warning("signal_bus unavailable: %s", e) + bus = None + self._bus = bus + + # -- lifecycle ------------------------------------------------------ + def start(self) -> None: + if self._running: + return + if not self._cfg.get("enabled", True): + logger.info("kb_reflection_listener: disabled by config") + return + if self._bus is None: + logger.warning("kb_reflection_listener: no signal bus, cannot start") + return + max_queue = int(self._cfg.get("max_queue", 10)) + try: + self._bus.subscribe( + "kb.article_saved", + self._on_kb_saved, + subscriber_id="kb_reflection_listener", + serial=True, + max_queue=max_queue, + ) + self._subscribed = True + self._running = True + logger.info( + "kb_reflection_listener: started (max_queue=%d, render_audio=%s, " + "source_agent_filter=%s)", + max_queue, self._cfg.get("render_audio"), + self._cfg.get("source_agent_filter"), + ) + except Exception as e: + logger.exception("kb_reflection_listener subscribe failed: %s", e) + + def stop(self) -> None: + if not self._running: + return + self._running = False + if self._subscribed and self._bus is not None: + try: + self._bus.unsubscribe("kb.article_saved", "kb_reflection_listener") + except Exception as e: + logger.debug("unsubscribe failed: %s", e) + self._subscribed = False + logger.info("kb_reflection_listener: stopped") + + # -- handler -------------------------------------------------------- + def _on_kb_saved(self, **payload: Any) -> None: + if not self._running: + return + try: + self._handle(payload) + except Exception as e: + logger.exception("kb_reflection_listener: unhandled error: %s", e) + + def _handle(self, payload: Dict[str, Any]) -> None: + article_id = payload.get("article_id") or payload.get("id") or "" + if not article_id: + logger.debug("no article_id in payload, skipping") + return + + importance = float(payload.get("importance", 0.5)) + if importance < float(self._cfg.get("min_importance", 0.0)): + return + + agent_filter = self._cfg.get("source_agent_filter") or [] + source_agent = payload.get("source_agent", "") + if agent_filter and source_agent not in agent_filter: + logger.debug( + "skipping %s: source_agent=%r not in filter %s", + article_id, source_agent, agent_filter, + ) + return + + with self._dedup_lock: + if article_id in self._reflected_ids: + logger.debug("already reflected on %s", article_id) + return + self._reflected_ids.add(article_id) + + debounce = float(self._cfg.get("debounce_s", 2.0)) + if debounce > 0: + time.sleep(debounce) + + title = payload.get("title", "") + summary = payload.get("summary", "") + topics = payload.get("topics", []) + logger.info("reflecting on %r (%s)", title, article_id) + + # Fetch the full article so the prompt contains facts, claims, + # and themes. A prompt with only title + summary produces + # reflection about the storage event rather than the ideas. + article = None + try: + from knowledge import get_kb + article = get_kb().get_article(article_id) or {} + except Exception as e: + logger.debug("KB get_article failed: %s", e) + article = {} + + full_summary = ( + article.get("summary") or article.get("content") + or summary or "" + ) + facts = article.get("facts") or [] + claims = article.get("claims") or [] + themes = article.get("themes") or topics or [] + entities = article.get("entities") or [] + + # Pull a few related KB articles for connect-to-prior-knowledge + # — quality of the reflection improves significantly when the + # model can compare the new article against existing context. + related_titles = [] + try: + sr = self.kernel.execute("KB_SEARCH", { + "query": f"{title} {full_summary}".strip(), + "top_k": 3, + }) + for r in (sr or {}).get("results") or []: + rid = r.get("id") or r.get("article_id") or "" + t = r.get("title") or "" + if rid and rid != article_id and t: + related_titles.append(t) + except Exception as e: + logger.debug("KB_SEARCH for related failed: %s", e) + + # Build a rich content payload — what the article actually says. + # Order: short summary, then top facts, top claims (with speakers + # if available), themes. Capped to keep prompt under ESCALATE + # context budget after the renderer truncates to 3000 chars. + parts = [f"Title: {title}"] + if full_summary: + parts.append(f"\nSummary:\n{full_summary[:1200]}") + if facts: + parts.append("\nKey facts:") + for f in facts[:10]: + if isinstance(f, dict): + txt = f.get("text") or f.get("statement") or str(f) + else: + txt = str(f) + parts.append(f" - {txt[:200]}") + if claims: + parts.append("\nClaims made (with speakers):") + for c in claims[:10]: + if isinstance(c, dict): + spk = c.get("speaker") or "unknown" + txt = c.get("text") or c.get("statement") or str(c) + parts.append(f" - [{spk}] {txt[:200]}") + else: + parts.append(f" - {str(c)[:200]}") + if themes: + theme_strs = [] + for t in themes[:8]: + if isinstance(t, dict): + theme_strs.append(t.get("name") or t.get("text") or str(t)) + else: + theme_strs.append(str(t)) + parts.append(f"\nThemes: {', '.join(theme_strs)}") + if entities: + ent_strs = [] + for e in entities[:10]: + if isinstance(e, dict): + ent_strs.append(e.get("name") or str(e)) + else: + ent_strs.append(str(e)) + parts.append(f"\nEntities: {', '.join(ent_strs)}") + + experience = "\n".join(parts) + if related_titles: + experience += ( + "\n\nRelated articles already in your knowledge base: " + + "; ".join(related_titles) + ) + + try: + if getattr(self.kernel, "thought_canvas", None) is None: + self.kernel.execute( + "THOUGHT_CANVAS_OPEN", {"width": 256, "height": 224} + ) + except Exception as e: + logger.warning("THOUGHT_CANVAS_OPEN failed: %s", e) + self._post_blackboard( + "warning", + f"Reflection skipped for {article_id}: canvas unavailable ({e})", + ) + return + + render = bool(self._cfg.get("render_audio", True)) + try: + result = self.kernel.execute("REFLECT", { + "experience": experience, + # 'insight' switches the prompt to content-engagement + # framing inside ReflectionGenerator — react to the + # ideas, don't introspect about the storage event. + "experience_type": "insight", + "task": title, + "reflection_context": { + "kb_article_id": article_id, + "title": title, + "summary": full_summary, + "topics": themes, + "importance": importance, + "related_articles": related_titles, + "trigger": "kb.article_saved", + }, + "render": render, + }) + except Exception as e: + logger.exception("REFLECT raised: %s", e) + self._post_blackboard("warning", f"Reflection error for {article_id}: {e}") + return + + if not (result or {}).get("ok"): + err = (result or {}).get("error", "unknown error") + # If audio rendering failed, retry text-only — preserves the + # post-mortem and KB-graph-update side effects. + if render and "voice" in str(err).lower(): + logger.info("retrying %s without audio render", article_id) + try: + result = self.kernel.execute("REFLECT", { + "experience": experience, + "experience_type": "insight", + "task": title, + "reflection_context": { + "kb_article_id": article_id, + "title": title, + "trigger": "kb.article_saved", + }, + "render": False, + }) + except Exception as e2: + logger.exception("text-only fallback failed: %s", e2) + self._post_blackboard( + "warning", + f"Reflection failed (both modes) for {article_id}: {e2}", + ) + return + else: + self._post_blackboard( + "warning", f"Reflection not-ok for {article_id}: {err}" + ) + return + + duration = (result or {}).get("duration_s", 0) + logger.info("reflection complete for %s (%.1fs)", article_id, duration) + + # -- helpers -------------------------------------------------------- + def _post_blackboard(self, priority: str, text: str) -> None: + try: + from status_blackboard import get_status_blackboard + get_status_blackboard().post( + from_id="kb_reflection_listener", + text=text, + audience="*", + priority=priority, + ttl_s=600, + ) + except Exception as e: + logger.debug("blackboard post failed: %s", e) + + def get_stats(self) -> Dict[str, Any]: + with self._dedup_lock: + return { + "running": self._running, + "enabled": self._cfg.get("enabled", True), + "reflected_count": len(self._reflected_ids), + "config": dict(self._cfg), + } + + +# Module-level singleton ------------------------------------------------ +_instance: Optional[KBReflectionListener] = None +_instance_lock = threading.Lock() + + +def get_kb_reflection_listener(kernel=None, config=None) -> Optional[KBReflectionListener]: + """Return the singleton listener, creating it on first call.""" + global _instance + if _instance is not None: + return _instance + if kernel is None: + return None + with _instance_lock: + if _instance is None: + _instance = KBReflectionListener(kernel, config=config) + return _instance diff --git a/project/bots/substrate_gather/__init__.py b/project/bots/substrate_gather/__init__.py index e9e08fe..0caa795 100644 --- a/project/bots/substrate_gather/__init__.py +++ b/project/bots/substrate_gather/__init__.py @@ -408,10 +408,10 @@ def _format_dict_brief(d, max_keys=8, depth=0, max_depth=1): runaway nesting on deeply-structured API payloads. For dexscreener-style data this means {baseToken: {symbol: - XAVIER, name: Xavier}} renders as: + ALPHA, name: Alpha}} renders as: - baseToken: - - symbol: XAVIER - - name: Xavier + - symbol: ALPHA + - name: Alpha instead of "baseToken: (dict, 2 items)". """ if not d: diff --git a/project/bots/substrate_gather/common.py b/project/bots/substrate_gather/common.py index e93e86a..956e934 100644 --- a/project/bots/substrate_gather/common.py +++ b/project/bots/substrate_gather/common.py @@ -131,7 +131,7 @@ # the same as "what's the latest price of X" queries. The user # experience was: substrate found 10 strong KB hits, refused to # surface them because crypto was technically stale, and dumped -# the user into a slow intern loop. See the 2026-04-25 Xavier +# the user into a slow intern loop. See the token-triage 2026-04-25 # regression where conf landed at 0.45 (kb=0.70 - stale_with_policy=0.25) # for a query that didn't actually want fresh data. CONF_STALE_PENALTY_POLICY_TIME_SENSITIVE = 0.25 # case 1 diff --git a/project/config.json b/project/config.json index d6f4e1e..bbf163a 100644 --- a/project/config.json +++ b/project/config.json @@ -18,10 +18,10 @@ "model": "nomic-ai/nomic-embed-text-v1.5", "vision_enabled": true, "vision_model": "nomic-ai/nomic-embed-vision-v1.5", - "vision_device": "cpu", + "vision_device": "cuda", "vision_max_image_size": 1024, "imagebind_enabled": true, - "imagebind_device": "cpu", + "imagebind_device": "cuda", "imagebind_max_image_size": 1024, "imagebind_audio_sample_rate": 16000, "imagebind_audio_clip_duration": 2, @@ -34,6 +34,7 @@ }, "knowledgebase_path": "./knowledgebase", "max_kb_entries": 500, + "orchestrator_hard_cap_minutes": 30, "perspective": { "enabled": true, "mask_floor": 0.3, @@ -84,15 +85,15 @@ "utility": { "enabled": true, "model_server": "http://localhost:11434", - "model": "qwen3.5:0.8b", + "model": "qwen3:0.6b", "num_gpu": 99, "max_tokens": 1024, "timeout": 120, - "hf_model": "Qwen/Qwen3.5-0.8B", + "hf_model": "Qwen/Qwen3-0.6B", "use_direct_loading": true, "hidden_dim": 1024, "prefer_ollama": true, - "ollama_model": "qwen3.5:0.8b" + "ollama_model": "qwen3:0.6b" }, "comfyui": { "enabled": true, @@ -102,5 +103,34 @@ "monologue": { "auto_defer_on_active_use": true, "active_use_threshold_s": 60 + }, + "kb_reflection": { + "enabled": true, + "max_queue": 10, + "debounce_s": 2.0, + "render_audio": true, + "min_importance": 0.0, + "source_agent_filter": ["review"] + }, + "media": { + "_comment": "Media ingest tuning. whisper_backend: auto|faster|openai (auto prefers faster-whisper, falls back to openai-whisper). whisper_model_size: tiny|base|small|medium|large|large-v2|large-v3|large-v3-turbo. whisper_device: auto|cpu|cuda|cuda:0|mps — auto resolves via torch. whisper_compute_type: float16|int8_float16|int8|float32 (faster-whisper only — float16 on GPU, int8 on CPU). whisper_beam_size: 1=greedy/fastest, 5=default/balanced. Tested empirically: large-v3 via faster-whisper is the best WER (~4.4% vs medium's ~5.5%) AND ~2× faster than openai-whisper at the same size; the openai-whisper repetition hallucination on quiet sections does not reproduce in CTranslate2.", + "whisper_backend": "whisperx", + "whisper_model_size": "large-v3", + "whisper_device": "auto", + "whisper_compute_type": "float16", + "whisper_beam_size": 5, + "whisper_language": null, + "whisperx_batch_size": 16, + "hf_token": "", + "_serial_comment": "serial_review_mode blocks the media_ingest queue worker after each transcription until the scheduled idle review task reaches a terminal state (completed/failed/cancelled). Trades throughput for steady KB growth — disk and review backlog don't decouple from ingest progress. Timeout caps how long any single review can hold the queue; review runs that exceed it release the gate so backlog isn't permanently stuck.", + "serial_review_mode": true, + "serial_review_timeout_s": 1800, + "serial_review_poll_s": 5, + "_review_tuning_comment": "Knobs that bound the per-episode review cost. Only relevant when review_pipeline = director_run. review_max_rounds caps how many tool-call cycles the director_run agent gets per subtask. review_no_think prepends /no_think to the review goal so Qwen3 skips thinking-mode prefill.", + "review_max_rounds": 5, + "review_no_think": true, + "_review_pipeline_comment": "review_pipeline selects the post-transcribe insight extraction strategy. 'staged' (default) uses media_ingest_review_v2 — a 7-stage pipeline (semantic segmentation, JSON-mode atomic extraction, episode reduce, RAPTOR tree, deterministic auto-tag, cross-episode linking, KG emission, structured KB save) that runs synchronously in the audio worker. 'director_run' is the legacy multi-round agent_session approach. See project/knowledge/media_ingest_review_v2.py for stage details.", + "review_pipeline": "staged", + "review_model": null } } \ No newline at end of file diff --git a/project/infra/decision_trees/trees.json b/project/infra/decision_trees/trees.json new file mode 100644 index 0000000..e741da8 --- /dev/null +++ b/project/infra/decision_trees/trees.json @@ -0,0 +1,2415 @@ +{ + "trees": [ + { + "id": "post-step-validation", + "name": "Post-Step Validation", + "description": "Evaluates an agent's output after completing a plan step. Checks for empty responses, file errors, tool error rates, and task completion. Routes to continue, retry, escalate, or replan.", + "created": "2026-04-27T15:57:50.106408+00:00", + "updated": "2026-04-27T15:57:50.106408+00:00", + "root": { + "type": "choice", + "question": "response empty?", + "options": [ + { + "label": "empty", + "next": { + "type": "yes_no", + "question": "already retried?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Agent returned empty response after retry. May need a different agent or approach." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "Agent returned empty or near-empty response." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + } + }, + { + "label": "has_content", + "next": { + "type": "choice", + "question": "drift check", + "options": [ + { + "label": "redshift", + "next": { + "type": "yes_no", + "question": "already retried (redshift)?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Response drifted far from task intent (redshift detected by displacement analysis) even after retry. Needs a different approach or a more constrained task description." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "DRIFT WARNING: Your response drifted far from the assigned task (redshift detected). Re-read the task description and focus strictly on what was asked. Do not go off on tangents." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + } + }, + { + "label": "blueshift", + "next": { + "type": "yes_no", + "question": "already retried (blueshift)?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Response is too shallow or echoes the task (blueshift detected) even after retry." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "DRIFT WARNING: Your response appears to echo the task or is too shallow (blueshift detected). You must produce substantive work output, not just acknowledge the task." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + } + }, + { + "label": "converging", + "next": { + "type": "leaf", + "action": "replan", + "reason": "LOOP DETECTED: Recent responses are converging to the same semantic point (convergence detected by displacement analysis). The current approach is stuck. A fundamentally different strategy is needed." + } + }, + { + "label": "normal", + "next": { + "type": "choice", + "question": "response quality check", + "options": [ + { + "label": "non_answer", + "next": { + "type": "yes_no", + "question": "already retried (non-answer)?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Agent acknowledged the task but produced no work output (no tool calls, no deliverables) even after retry. Needs a different agent or a more explicit task description." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "Agent acknowledged the task but did not actually perform any work -- no tools were called and no deliverables were produced. You MUST use your tools to complete the task, not just describe your intentions. Start working immediately." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + } + }, + { + "label": "plan_only", + "next": { + "type": "yes_no", + "question": "already retried (plan-only)?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Agent wrote a plan but did not execute it, even after retry. Needs a more action-oriented agent or explicit instructions to execute." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "Agent wrote a detailed plan or outlined steps but did NOT execute any of them. Do NOT plan. You MUST execute the work using your tools now. Call file_write, code_run, web_search, or whatever tools the task requires. Start working immediately." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + } + }, + { + "label": "process_only", + "next": { + "type": "yes_no", + "question": "already retried (process-only)?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Agent described its process without producing deliverables, even after retry." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "Agent described what it would do or how it would approach the task, but produced no actual output. Stop describing your process. You MUST produce concrete deliverables using your tools. Start working immediately." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + } + }, + { + "label": "placeholder_content", + "next": { + "type": "yes_no", + "question": "already retried (placeholder)?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Agent produced placeholder content instead of real work, even after retry." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "Agent produced placeholder or template content with sections like 'TODO' or 'insert X here' instead of real, complete work. You MUST fill in ALL content with real data and complete text. No placeholders, no TODOs, no template markers." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + } + }, + { + "label": "off_topic", + "next": { + "type": "yes_no", + "question": "already retried (off-topic)?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Agent went off-topic and did not address the assigned task, even after retry." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "Agent's response does not address the assigned task. Re-read the task description carefully and focus specifically on what was requested. Do not work on anything else." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + } + }, + { + "label": "real_answer", + "next": { + "type": "choice", + "question": "file validation", + "options": [ + { + "label": "has_errors", + "next": { + "type": "yes_no", + "question": "already retried (file errors)?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Files still have errors after retry. Director should review and reassign." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "Output files have syntax or format errors. Retrying with error details." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + } + }, + { + "label": "clean", + "next": { + "type": "choice", + "question": "deliverable check", + "options": [ + { + "label": "undelivered", + "next": { + "type": "leaf", + "action": "retry", + "reason": "All task plan steps complete but deliverable not delivered -- retry and call task_plan deliver." + } + }, + { + "label": "ok", + "next": { + "type": "choice", + "question": "tool error rate", + "options": [ + { + "label": "high_errors", + "next": { + "type": "choice", + "question": "duplicate errors?", + "options": [ + { + "label": "has_duplicates", + "next": { + "type": "leaf", + "action": "replan", + "reason": "Repeated identical tool failures. The current approach isn't working and retrying won't help." + } + }, + { + "label": "no_duplicates", + "next": { + "type": "yes_no", + "question": "already retried (tool errors)?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "High tool error rate persists after retry." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "Most tool calls failed. Retrying may resolve transient issues." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + } + } + ], + "node_class": "check", + "check_fn": "_check_has_duplicate_errors" + } + }, + { + "label": "some_errors", + "next": { + "type": "yes_no", + "question": "Did the agent accomplish its assigned task despite the tool errors? Answer 'yes' if the agent produced useful output. Answer 'no' if the task is incomplete.", + "yes": { + "type": "leaf", + "action": "continue", + "reason": "Task completed despite some tool errors." + }, + "no": { + "type": "yes_no", + "question": "already retried (some errors)?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Task incomplete with tool errors after retry." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "Task incomplete due to tool errors. Retrying." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + }, + "node_class": "llm", + "context_key": "agent_response" + } + }, + { + "label": "low_errors", + "next": { + "type": "yes_no", + "question": "Did the agent complete its assigned task? Answer 'yes' if it produced useful output. Answer 'no' only if the task is clearly unfinished.", + "yes": { + "type": "leaf", + "action": "continue", + "reason": "Task completed successfully." + }, + "no": { + "type": "yes_no", + "question": "Is this agent blocked by a missing resource, permission, or external dependency? Answer 'yes' if the agent cannot proceed without something it doesn't have. Answer 'no' if it just needs to try harder or use a different approach.", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Agent is blocked by an external dependency. Retrying won't help." + }, + "no": { + "type": "yes_no", + "question": "already retried (incomplete)?", + "yes": { + "type": "leaf", + "action": "replan", + "reason": "Task still incomplete after retry. A different approach is needed." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "Task incomplete but not blocked. Retrying with clearer instructions." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + }, + "node_class": "llm", + "context_key": "agent_response" + }, + "node_class": "llm", + "context_key": "agent_response" + } + }, + { + "label": "no_tools", + "next": { + "type": "yes_no", + "question": "The agent didn't use any tools. Did it complete its task using only its own knowledge? Answer 'yes' if the response adequately addresses the task. Answer 'no' if tools should have been used.", + "yes": { + "type": "leaf", + "action": "continue", + "reason": "Task completed without tool use." + }, + "no": { + "type": "yes_no", + "question": "already retried (no tools)?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Agent didn't use tools after retry. May need different agent." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "Agent should have used tools but didn't. Retrying with explicit tool instructions." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + }, + "node_class": "llm", + "context_key": "agent_response" + } + } + ], + "node_class": "check", + "check_fn": "_check_tool_error_rate" + } + } + ], + "node_class": "check", + "check_fn": "_check_deliverable_status" + } + }, + { + "label": "no_files_dir", + "next": { + "type": "yes_no", + "question": "Did the agent complete its assigned task? Answer 'yes' if it produced useful output. Answer 'no' if the task is incomplete.", + "yes": { + "type": "leaf", + "action": "continue", + "reason": "Task completed successfully." + }, + "no": { + "type": "yes_no", + "question": "already retried (no files dir)?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Task incomplete after retry." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "Task incomplete. Retrying." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + }, + "node_class": "llm", + "context_key": "agent_response" + } + } + ], + "node_class": "check", + "check_fn": "_check_has_file_errors" + } + } + ], + "node_class": "check", + "check_fn": "_check_response_quality" + } + }, + { + "label": "unavailable", + "next": { + "type": "yes_no", + "question": "Did the agent complete its assigned task? Answer 'yes' if it produced useful output. Answer 'no' if the task is clearly unfinished.", + "yes": { + "type": "leaf", + "action": "continue", + "reason": "Task completed (drift detection unavailable, LLM check passed)." + }, + "no": { + "type": "yes_no", + "question": "already retried (no drift)?", + "yes": { + "type": "leaf", + "action": "escalate", + "reason": "Task incomplete after retry (drift detection unavailable)." + }, + "no": { + "type": "leaf", + "action": "retry", + "reason": "Task appears incomplete. Retrying." + }, + "node_class": "check", + "check_fn": "_check_already_retried" + }, + "node_class": "llm", + "context_key": "agent_response" + } + } + ], + "node_class": "check", + "check_fn": "_check_drift_status" + } + } + ], + "node_class": "check", + "check_fn": "_check_response_empty" + }, + "category": "orchestration", + "system": true, + "triggers": [ + { + "event": "post_step", + "description": "After an agent completes a plan step", + "context_keys": [ + "agent_response", + "task_description", + "tool_stats", + "files_path", + "agent_name", + "step_num", + "is_last_step", + "already_retried" + ] + } + ], + "seed_version": 2 + }, + { + "id": "error-recovery", + "name": "Error Recovery", + "description": "Classifies an error or exception and picks a recovery strategy: retry, escalate, replan, or skip.", + "created": "2026-04-27T15:57:50.111425+00:00", + "updated": "2026-04-27T15:57:50.111425+00:00", + "root": { + "type": "choice", + "question": "What type of problem occurred? Pick the best category.", + "options": [ + { + "label": "timeout", + "next": { + "type": "leaf", + "action": "retry", + "reason": "Operation timed out. Retrying with simpler instructions.", + "details": { + "simplify": true + } + } + }, + { + "label": "blocked", + "next": { + "type": "leaf", + "action": "escalate", + "reason": "Agent is blocked by external dependency.", + "details": { + "notify_director": true + } + } + }, + { + "label": "wrong_approach", + "next": { + "type": "leaf", + "action": "replan", + "reason": "The approach isn't working. Need a different strategy." + } + }, + { + "label": "minor_error", + "next": { + "type": "leaf", + "action": "retry", + "reason": "Minor error that may resolve on retry." + } + }, + { + "label": "capability_gap", + "next": { + "type": "leaf", + "action": "escalate", + "reason": "Agent lacks the tools or skills for this task. Need a different agent.", + "details": { + "suggest_agent_change": true + } + } + } + ], + "node_class": "llm", + "context_key": "error_text" + }, + "category": "orchestration", + "system": true, + "triggers": [ + { + "event": "error_recovery", + "description": "When the orchestrator catches an exception", + "context_keys": [ + "error_context" + ] + } + ], + "seed_version": 1 + }, + { + "id": "web-fetch-recovery", + "name": "Web Fetch Recovery", + "description": "Evaluates a web_fetch failure (blocked, timeout, 404, rate limit, etc.) and picks the right recovery strategy.", + "created": "2026-04-27T15:57:50.113931+00:00", + "updated": "2026-04-27T15:57:50.113931+00:00", + "root": { + "type": "choice", + "question": "fetch error type", + "options": [ + { + "label": "bot_blocked", + "next": { + "type": "leaf", + "action": "use_alternative_source", + "reason": "Site has bot protection. Do NOT retry this URL. Search for the same information on a different site, check the reference library for official docs, or look for a public API endpoint instead.", + "details": { + "blacklist_url": true + } + } + }, + { + "label": "timeout", + "next": { + "type": "leaf", + "action": "retry_simpler", + "reason": "Request timed out. Try the same site with a simpler URL (no query params) or try a different page on the same domain. If it times out again, switch sites." + } + }, + { + "label": "not_found", + "next": { + "type": "leaf", + "action": "search_again", + "reason": "Page not found (404). The URL may be outdated. Do a fresh web_search for the same topic to find current URLs." + } + }, + { + "label": "forbidden", + "next": { + "type": "leaf", + "action": "use_alternative_source", + "reason": "Access forbidden (403). This site requires authentication or blocks automated access. Find the same information elsewhere.", + "details": { + "blacklist_url": true + } + } + }, + { + "label": "rate_limited", + "next": { + "type": "leaf", + "action": "wait_and_retry", + "reason": "Rate limited (429). Wait before making more requests. In the meantime, process any data you already have.", + "details": { + "wait_seconds": 10 + } + } + }, + { + "label": "connection_error", + "next": { + "type": "leaf", + "action": "check_url", + "reason": "Connection failed. Verify the URL is correct. If it is, the site may be down -- try alternatives." + } + }, + { + "label": "other_error", + "next": { + "type": "choice", + "question": "Can this web fetch error be resolved by retrying with different parameters, or should the agent try a completely different source?", + "options": [ + { + "label": "retry_different_params", + "next": { + "type": "leaf", + "action": "retry_modified", + "reason": "Try the same site with different parameters (different page, simplified URL, etc.)" + } + }, + { + "label": "use_different_source", + "next": { + "type": "leaf", + "action": "use_alternative_source", + "reason": "This error suggests the site won't work. Find the information on a different site." + } + } + ], + "node_class": "llm", + "context_key": "error_text" + } + } + ], + "node_class": "check", + "check_fn": "_check_fetch_error_type" + }, + "category": "tool-recovery", + "system": true, + "triggers": [ + { + "event": "tool_error", + "tool": "web_fetch", + "description": "After web_fetch returns an error", + "context_keys": [ + "tool_result" + ] + } + ], + "seed_version": 1 + }, + { + "id": "code-run-recovery", + "name": "Code Run Recovery", + "description": "Classifies a code_run failure (syntax error, missing module, type error, etc.) and routes to the right debugging approach.", + "created": "2026-04-27T15:57:50.119534+00:00", + "updated": "2026-04-27T15:57:50.119534+00:00", + "root": { + "type": "choice", + "question": "code error type", + "options": [ + { + "label": "syntax_error", + "next": { + "type": "leaf", + "action": "patch_syntax", + "reason": "Syntax error detected. Use file_read to see the exact line, then file_patch to fix just that line. Do NOT rewrite the entire file.", + "details": { + "use_file_patch": true, + "read_error_line": true + } + } + }, + { + "label": "missing_module", + "next": { + "type": "leaf", + "action": "install_module", + "reason": "Missing Python module. Use code_run with 'pip install ' first, then retry. Check requirements or project docs for the right package name (it may differ from the import name).", + "details": { + "install_first": true + } + } + }, + { + "label": "file_not_found", + "next": { + "type": "leaf", + "action": "check_files", + "reason": "File not found. Use file_list to see what files exist. The filename might be misspelled, or the file might not have been created by a previous step.", + "details": { + "use_file_list": true + } + } + }, + { + "label": "data_access_error", + "next": { + "type": "leaf", + "action": "inspect_data", + "reason": "Data access error (KeyError/IndexError). The data structure doesn't match what the code expects. Add a print statement to see the actual data shape before the failing line, then fix the access pattern.", + "details": { + "add_debug_print": true + } + } + }, + { + "label": "type_error", + "next": { + "type": "leaf", + "action": "inspect_types", + "reason": "Type error. A function received the wrong type of argument. Add a print(type(var)) before the failing line to see what's actually being passed, then fix the type conversion or function call.", + "details": { + "add_debug_print": true + } + } + }, + { + "label": "network_error", + "next": { + "type": "leaf", + "action": "add_retry_logic", + "reason": "Network error. The script should have retry logic with timeouts. Add try/except around network calls with a retry loop (max 3 attempts, 2-second delay).", + "details": { + "add_try_except": true + } + } + }, + { + "label": "permission_error", + "next": { + "type": "leaf", + "action": "check_paths", + "reason": "Permission denied. The script is trying to access a file or directory outside the project files folder. All file operations must stay within the project." + } + }, + { + "label": "resource_error", + "next": { + "type": "leaf", + "action": "reduce_scope", + "reason": "Out of memory or process killed. The script is processing too much data at once. Reduce batch size, process in chunks, or simplify the computation." + } + }, + { + "label": "command_error", + "next": { + "type": "leaf", + "action": "fix_command", + "reason": "Command not found or bad arguments. Check the command spelling and argument format." + } + }, + { + "label": "timeout", + "next": { + "type": "leaf", + "action": "optimize_or_split", + "reason": "Script timed out. Either optimize the slow operation or split it into smaller steps with higher timeout. Consider processing data in smaller batches.", + "details": { + "increase_timeout": true + } + } + }, + { + "label": "runtime_error", + "next": { + "type": "choice", + "question": "Is this a logic error that needs the code approach changed, or a fixable bug at a specific line?", + "options": [ + { + "label": "logic_error", + "next": { + "type": "leaf", + "action": "rethink_approach", + "reason": "The code's approach has a fundamental problem. Step back and reconsider the algorithm before patching individual lines.", + "details": { + "read_full_file": true + } + } + }, + { + "label": "fixable_bug", + "next": { + "type": "leaf", + "action": "patch_bug", + "reason": "Fix the specific bug at the error location. Read the traceback, identify the line, and use file_patch for a targeted fix.", + "details": { + "use_file_patch": true, + "read_error_line": true + } + } + } + ], + "node_class": "llm", + "context_key": "error_text" + } + } + ], + "node_class": "check", + "check_fn": "_check_code_error_type" + }, + "category": "tool-recovery", + "system": true, + "triggers": [ + { + "event": "tool_error", + "tool": "code_run", + "description": "After code_run returns a non-zero exit code", + "context_keys": [ + "stderr", + "stdout", + "exit_code" + ] + } + ], + "seed_version": 1 + }, + { + "id": "comfyui-template-selection", + "name": "ComfyUI Template Selection", + "description": "Routes a media generation request to the right ComfyUI workflow template based on whether the task wants images or video, and whether input images are provided.", + "created": "2026-04-27T15:57:50.123595+00:00", + "updated": "2026-04-27T15:57:50.123595+00:00", + "root": { + "type": "choice", + "question": "wants video?", + "options": [ + { + "label": "wants_video", + "next": { + "type": "choice", + "question": "has input images?", + "options": [ + { + "label": "has_images", + "next": { + "type": "leaf", + "action": "use_template", + "reason": "Use 'itv_video' (image-to-video). The prompt should describe MOTION, not the scene. Example: 'The camera slowly pans right as wind blows through the trees.'", + "details": { + "template": "itv_video" + } + } + }, + { + "label": "no_images", + "next": { + "type": "leaf", + "action": "use_template", + "reason": "Use 'ttv_video' (text-to-video). The prompt should describe BOTH the scene AND the motion. Example: 'A sunset over mountains, clouds drift slowly, camera pushes forward.'", + "details": { + "template": "ttv_video" + } + } + } + ], + "node_class": "check", + "check_fn": "_check_has_input_images" + } + }, + { + "label": "wants_image", + "next": { + "type": "choice", + "question": "has input images?", + "options": [ + { + "label": "has_images", + "next": { + "type": "choice", + "question": "How many input images does this task involve? Count the distinct images referenced.", + "options": [ + { + "label": "one", + "next": { + "type": "leaf", + "action": "use_template", + "reason": "Use 'image_edit_1' (single image edit). The prompt describes what to CHANGE.", + "details": { + "template": "image_edit_1" + } + } + }, + { + "label": "two", + "next": { + "type": "leaf", + "action": "use_template", + "reason": "Use 'image_edit_2' (two reference images). The prompt explains how to combine/use both.", + "details": { + "template": "image_edit_2" + } + } + }, + { + "label": "three_or_more", + "next": { + "type": "leaf", + "action": "use_template", + "reason": "Use 'image_edit_3' (three reference images). The prompt explains how all three are used.", + "details": { + "template": "image_edit_3" + } + } + } + ], + "node_class": "llm", + "context_key": "task_description" + } + }, + { + "label": "no_images", + "next": { + "type": "leaf", + "action": "use_template", + "reason": "Use 'image' (text-to-image). Write a detailed prompt: Subject + Details + Style + Quality modifiers.", + "details": { + "template": "image" + } + } + } + ], + "node_class": "check", + "check_fn": "_check_has_input_images" + } + } + ], + "node_class": "check", + "check_fn": "_check_wants_video" + }, + "category": "tool-routing", + "system": true, + "triggers": [ + { + "event": "pre_tool", + "tool": "comfyui_generate", + "description": "Before calling comfyui_generate", + "context_keys": [ + "task_description" + ] + } + ], + "seed_version": 1 + }, + { + "id": "comfyui-error-recovery", + "name": "ComfyUI Error Recovery", + "description": "Classifies a ComfyUI generation error (VRAM, missing input, ComfyUI down, template error) and picks recovery.", + "created": "2026-04-27T15:57:50.128800+00:00", + "updated": "2026-04-27T15:57:50.128800+00:00", + "root": { + "type": "choice", + "question": "comfyui error type", + "options": [ + { + "label": "vram_error", + "next": { + "type": "leaf", + "action": "reduce_resolution", + "reason": "Out of VRAM. Reduce the image/video dimensions. For images, try 512x512 instead of 1024x1024. For video, reduce frame count or resolution.", + "details": { + "reduce_dimensions": true + } + } + }, + { + "label": "missing_input", + "next": { + "type": "leaf", + "action": "check_files", + "reason": "Required input file is missing. Use file_list to verify the image exists in the project files folder. The file must be uploaded to ComfyUI's input folder.", + "details": { + "use_file_list": true + } + } + }, + { + "label": "comfyui_down", + "next": { + "type": "leaf", + "action": "notify_user", + "reason": "ComfyUI is not running. Send a message to the user via inbox_send explaining that ComfyUI needs to be started before media generation can work.", + "details": { + "use_inbox": true + } + } + }, + { + "label": "template_error", + "next": { + "type": "leaf", + "action": "list_templates", + "reason": "Template is invalid or incompatible. Run comfyui_generate with action='list_templates' to see available workflows and pick a different one." + } + }, + { + "label": "unknown_error", + "next": { + "type": "choice", + "question": "Is this ComfyUI error likely fixable by changing inputs (prompt, dimensions, template), or does it need user intervention?", + "options": [ + { + "label": "fixable_by_agent", + "next": { + "type": "leaf", + "action": "retry_modified", + "reason": "Try again with modified parameters. Check the prompt quality, dimensions, and template choice." + } + }, + { + "label": "needs_user_help", + "next": { + "type": "leaf", + "action": "notify_user", + "reason": "This error may require the user's help. Send them a message with the error details.", + "details": { + "use_inbox": true + } + } + } + ], + "node_class": "llm", + "context_key": "error_text" + } + } + ], + "node_class": "check", + "check_fn": "_check_comfyui_error_type" + }, + "category": "tool-recovery", + "system": true, + "triggers": [ + { + "event": "tool_error", + "tool": "comfyui_generate", + "description": "After comfyui_generate returns an error", + "context_keys": [ + "error_text" + ] + } + ], + "seed_version": 1 + }, + { + "id": "staffing-decision", + "name": "Staffing Decision", + "description": "Evaluates whether to approve, deny, modify, or reactivate in response to an agent creation request.", + "created": "2026-04-27T15:57:50.133878+00:00", + "updated": "2026-04-27T15:57:50.133878+00:00", + "root": { + "type": "choice", + "question": "existing agent check", + "options": [ + { + "label": "active_exists", + "next": { + "type": "leaf", + "action": "deny", + "reason": "An active agent of this type already exists. Delegate to the existing agent instead of creating a duplicate.", + "details": { + "reason": "duplicate" + } + } + }, + { + "label": "inactive_exists", + "next": { + "type": "leaf", + "action": "reactivate", + "reason": "An inactive agent of this type exists. Reactivate it instead of creating a new one -- it retains its history and context from previous work.", + "details": { + "reason": "reuse_existing" + } + } + }, + { + "label": "none_available", + "next": { + "type": "choice", + "question": "Is this agent creation request justified? Consider: Is the task outside the requesting agent's capabilities? Is the scope large enough to warrant a new specialist? Answer 'justified' if the request makes sense, 'unjustified' if the requesting agent should handle it themselves.", + "options": [ + { + "label": "justified", + "next": { + "type": "choice", + "question": "Is the requested agent type the right one for this task, or would a different type be more appropriate?", + "options": [ + { + "label": "right_type", + "next": { + "type": "leaf", + "action": "approve", + "reason": "Request is justified and the right agent type was requested." + } + }, + { + "label": "wrong_type", + "next": { + "type": "leaf", + "action": "modify", + "reason": "Request is justified but a different agent type would be more appropriate. Modify the request before approving.", + "details": { + "suggest_alternative": true + } + } + } + ], + "node_class": "llm", + "context_key": "request_context" + } + }, + { + "label": "unjustified", + "next": { + "type": "leaf", + "action": "deny", + "reason": "The requesting agent should be able to handle this task with its existing tools and capabilities.", + "details": { + "reason": "can_self_handle" + } + } + } + ], + "node_class": "llm", + "context_key": "request_context" + } + } + ], + "node_class": "check", + "check_fn": "_check_existing_agent_available" + }, + "category": "orchestration", + "system": true, + "triggers": [ + { + "event": "staffing_request", + "description": "When an agent requests a new agent", + "context_keys": [ + "requested_type", + "active_agents", + "inactive_agents", + "request_context" + ] + } + ], + "seed_version": 1 + }, + { + "id": "data-analysis-tool-routing", + "name": "Data Analysis Tool Routing", + "description": "Routes a data analysis task to the right tool based on file type (CSV, JSON, Excel, text), file size, and the kind of analysis needed.", + "created": "2026-04-27T15:57:50.140134+00:00", + "updated": "2026-04-27T15:57:50.140134+00:00", + "root": { + "type": "choice", + "question": "data file type", + "options": [ + { + "label": "csv", + "next": { + "type": "choice", + "question": "csv file size", + "options": [ + { + "label": "small", + "next": { + "type": "leaf", + "action": "use_csv_query", + "reason": "Small CSV file. Use csv_query for exploration (info, stats, filter actions). Only write a custom script if csv_query can't answer the specific question.", + "details": { + "tool": "csv_query" + } + } + }, + { + "label": "medium", + "next": { + "type": "leaf", + "action": "use_csv_query_first", + "reason": "Medium CSV file. Start with csv_query for structure and basic stats. Use code_run with pandas for complex analysis. Do NOT file_read the whole file -- it will flood your context.", + "details": { + "tool": "csv_query", + "fallback": "code_run" + } + } + }, + { + "label": "large", + "next": { + "type": "leaf", + "action": "use_code_run", + "reason": "Large CSV file. Use csv_query for info/stats only. For any real analysis, write a pandas script with code_run. Process in chunks if the file is very large. NEVER file_read this.", + "details": { + "tool": "code_run", + "chunked": true + } + } + }, + { + "label": "unknown_size", + "next": { + "type": "leaf", + "action": "check_size_first", + "reason": "Check the file size first with file_read_lines info_only=true. Then pick the right approach based on size.", + "details": { + "tool": "file_read_lines" + } + } + } + ], + "node_class": "check", + "check_fn": "_check_file_size" + } + }, + { + "label": "json", + "next": { + "type": "leaf", + "action": "use_code_run", + "reason": "JSON/JSONL file. Write a Python script to load and analyze it. For JSONL, process line by line. For nested JSON, use code_run to explore the structure.", + "details": { + "tool": "code_run" + } + } + }, + { + "label": "text", + "next": { + "type": "leaf", + "action": "use_file_search", + "reason": "Text/log file. Start with file_search to find relevant patterns. Use file_read_lines for specific sections. Write a script only for complex parsing.", + "details": { + "tool": "file_search" + } + } + }, + { + "label": "excel", + "next": { + "type": "leaf", + "action": "use_code_run", + "reason": "Excel file. Write a Python script with openpyxl or pandas to read it. csv_query won't work on .xlsx.", + "details": { + "tool": "code_run" + } + } + }, + { + "label": "unknown", + "next": { + "type": "choice", + "question": "Based on the task description, what kind of data is this and what's the best tool to start with?", + "options": [ + { + "label": "structured_data", + "next": { + "type": "leaf", + "action": "use_code_run", + "reason": "Structured data -- use code_run with pandas.", + "details": { + "tool": "code_run" + } + } + }, + { + "label": "text_data", + "next": { + "type": "leaf", + "action": "use_file_search", + "reason": "Text data -- use file_search and file_read_lines.", + "details": { + "tool": "file_search" + } + } + }, + { + "label": "binary_data", + "next": { + "type": "leaf", + "action": "notify_limitation", + "reason": "Binary data format. May need a specialized Python library. Check what's available with 'pip list' via code_run.", + "details": { + "tool": "code_run" + } + } + } + ], + "node_class": "llm", + "context_key": "task_description" + } + } + ], + "node_class": "check", + "check_fn": "_check_data_file_type" + }, + "category": "tool-routing", + "system": true, + "triggers": [ + { + "event": "pre_tool", + "tool": "csv_query", + "description": "Before starting a data analysis task", + "context_keys": [ + "filename", + "file_size_bytes", + "task_description" + ] + }, + { + "event": "pre_tool", + "tool": "code_run", + "description": "Before running code on data files", + "context_keys": [ + "filename", + "file_size_bytes", + "task_description" + ] + } + ], + "seed_version": 1 + }, + { + "id": "handoff-quality-check", + "name": "Handoff Quality Check", + "description": "Verifies that an agent properly saved its output before the next agent takes over. Catches cases where output only exists in the chat response but wasn't persisted.", + "created": "2026-04-27T15:57:50.147279+00:00", + "updated": "2026-04-27T15:57:50.147279+00:00", + "root": { + "type": "choice", + "question": "task needs output?", + "options": [ + { + "label": "no_output_needed", + "next": { + "type": "leaf", + "action": "handoff_ok", + "reason": "Task type doesn't require persistent output. The response itself is sufficient." + } + }, + { + "label": "output_required", + "next": { + "type": "yes_no", + "question": "Did this agent save its work to a file using file_write or similar? Answer 'yes' if files were created or modified. Answer 'no' if the work only exists in the chat response.", + "yes": { + "type": "leaf", + "action": "handoff_ok", + "reason": "Agent saved output files. Handoff is complete." + }, + "no": { + "type": "leaf", + "action": "handoff_incomplete", + "reason": "Agent produced output but didn't save it to a file. The next agent won't be able to access it. Re-delegate with instructions to save the output using file_write.", + "details": { + "re_delegate": true, + "add_save_instruction": true + } + }, + "node_class": "llm", + "context_key": "agent_response" + } + }, + { + "label": "findings_required", + "next": { + "type": "yes_no", + "question": "Did this agent save its findings to the knowledgebase (kb_save) or to a file (file_write)? Answer 'yes' if findings were persisted anywhere. Answer 'no' if they only exist in the response.", + "yes": { + "type": "leaf", + "action": "handoff_ok", + "reason": "Agent saved findings. Handoff is complete." + }, + "no": { + "type": "leaf", + "action": "handoff_incomplete", + "reason": "Agent did research but didn't save findings to the KB or a file. Re-delegate with instructions to save key findings using kb_save.", + "details": { + "re_delegate": true, + "add_kb_instruction": true + } + }, + "node_class": "llm", + "context_key": "agent_response" + } + }, + { + "label": "unclear", + "next": { + "type": "yes_no", + "question": "Does the next step in the plan depend on output from this step? Answer 'yes' if the next agent needs files or data from this agent. Answer 'no' if the steps are independent.", + "yes": { + "type": "yes_no", + "question": "Did this agent save its work to a file or the knowledgebase? Answer 'yes' if work was persisted. Answer 'no' if it only exists in the chat response.", + "yes": { + "type": "leaf", + "action": "handoff_ok", + "reason": "Agent saved output. Handoff is complete." + }, + "no": { + "type": "leaf", + "action": "handoff_incomplete", + "reason": "Agent didn't persist output that the next step needs. Re-delegate with save instructions.", + "details": { + "re_delegate": true + } + }, + "node_class": "llm", + "context_key": "agent_response" + }, + "no": { + "type": "leaf", + "action": "handoff_ok", + "reason": "Steps are independent. No handoff needed." + }, + "node_class": "llm", + "context_key": "task_description" + } + } + ], + "node_class": "check", + "check_fn": "_check_task_needs_output" + }, + "category": "orchestration", + "system": true, + "triggers": [ + { + "event": "task_handoff", + "description": "After a step completes, before the next step starts", + "context_keys": [ + "task_description", + "agent_response" + ] + } + ], + "seed_version": 1 + }, + { + "id": "loop-recovery", + "name": "Loop Recovery", + "description": "When the loop detector identifies a repetition, oscillation, or no-progress loop, this tree picks a specific recovery action: force different agent, replan, merge tasks, or terminate.", + "created": "2026-04-27T15:57:50.155674+00:00", + "updated": "2026-04-27T15:57:50.155674+00:00", + "root": { + "type": "choice", + "question": "loop type", + "options": [ + { + "label": "repetition", + "next": { + "type": "choice", + "question": "intervention count (repetition)", + "options": [ + { + "label": "first", + "next": { + "type": "leaf", + "action": "force_different_agent", + "reason": "Same agent + same task repeating. Force the Director to use a DIFFERENT agent for this task, or modify the task significantly.", + "details": { + "block_current_agent": true + } + } + }, + { + "label": "some", + "next": { + "type": "leaf", + "action": "force_replan", + "reason": "Repetition loop persists after intervention. Force a replan -- the current approach isn't working and needs to be redesigned.", + "details": { + "trigger_architect": true + } + } + }, + { + "label": "too_many", + "next": { + "type": "leaf", + "action": "force_terminate", + "reason": "Repeated interventions have not broken the loop. Terminate the session and save a failure report.", + "details": { + "save_failure_report": true + } + } + } + ], + "node_class": "check", + "check_fn": "_check_intervention_count" + } + }, + { + "label": "oscillation", + "next": { + "type": "leaf", + "action": "merge_tasks", + "reason": "Two agents ping-ponging. Merge their tasks into a single delegation to one of them, or create a new agent that combines both capabilities.", + "details": { + "combine_agents": true + } + } + }, + { + "label": "no_progress", + "next": { + "type": "choice", + "question": "intervention count (no progress)", + "options": [ + { + "label": "first", + "next": { + "type": "leaf", + "action": "inject_concrete_guidance", + "reason": "Agent producing similar output each time. Inject very specific, concrete instructions that tell the agent exactly what to do differently.", + "details": { + "add_explicit_instructions": true + } + } + }, + { + "label": "some", + "next": { + "type": "leaf", + "action": "force_replan", + "reason": "No-progress loop continues. The task needs to be broken down differently or reassigned.", + "details": { + "trigger_architect": true + } + } + }, + { + "label": "too_many", + "next": { + "type": "leaf", + "action": "force_terminate", + "reason": "Cannot break the no-progress loop.", + "details": { + "save_failure_report": true + } + } + } + ], + "node_class": "check", + "check_fn": "_check_intervention_count" + } + }, + { + "label": "cross_step", + "next": { + "type": "leaf", + "action": "skip_step", + "reason": "Same agent+task appearing across different plan steps. This step may be a duplicate of earlier work. Skip it and move to the next step, using the earlier results.", + "details": { + "use_previous_results": true + } + } + }, + { + "label": "embedding_drift", + "next": { + "type": "leaf", + "action": "inject_concrete_guidance", + "reason": "Agent's output is drifting away from the task semantically. Re-inject the original task description with emphasis on the specific deliverables required.", + "details": { + "add_explicit_instructions": true + } + } + }, + { + "label": "unknown", + "next": { + "type": "leaf", + "action": "force_replan", + "reason": "Unknown loop pattern detected. Trigger a replan to get a fresh approach.", + "details": { + "trigger_architect": true + } + } + } + ], + "node_class": "check", + "check_fn": "_check_loop_type" + }, + "category": "orchestration", + "system": true, + "triggers": [ + { + "event": "loop_detected", + "description": "When loop_detector.check() returns a warning", + "context_keys": [ + "loop_type", + "intervention_count" + ] + } + ], + "seed_version": 1 + }, + { + "id": "cross-modal-embedding-operations-routing", + "name": "Cross-Modal Embedding Operations Routing", + "description": "Intercepts media_search calls to detect when the agent is trying to do cross-modal embedding arithmetic (blend audio+image, solve analogies, search by spatial layout, extract motion, compare modality gaps). Redirects to imagebind_ops when embedding computation is needed.", + "created": "2026-04-27T15:57:50.162759+00:00", + "updated": "2026-04-27T15:57:50.162759+00:00", + "root": { + "type": "choice", + "question": "cross-modal intent", + "options": [ + { + "label": "motion", + "next": { + "type": "leaf", + "action": "redirect_tool", + "reason": "This task involves motion, temporal, or video action analysis. Use imagebind_ops action='motion' instead of media_search. For extracting motion: sub_action='extract'. For applying motion to images: sub_action='transfer'.", + "details": { + "redirect_to": "imagebind_ops", + "suggested_action": "motion" + } + } + }, + { + "label": "blend", + "next": { + "type": "leaf", + "action": "redirect_tool", + "reason": "This task wants to combine multiple modalities (e.g. audio mood + image scene). Use imagebind_ops action='blend' with the relevant modalities and search=true to find matches.", + "details": { + "redirect_to": "imagebind_ops", + "suggested_action": "blend" + } + } + }, + { + "label": "analogy", + "next": { + "type": "leaf", + "action": "redirect_tool", + "reason": "This task involves cross-modal analogy or translation. Use imagebind_ops action='analogy' with A:B::C:? pattern. For example, quiet_forest_image : forest_audio :: city_image : ???", + "details": { + "redirect_to": "imagebind_ops", + "suggested_action": "analogy" + } + } + }, + { + "label": "depth", + "next": { + "type": "leaf", + "action": "redirect_tool", + "reason": "This task needs spatial layout + text meaning combined. Use imagebind_ops action='depth_search' with a text query and a depth_id reference image for layout.", + "details": { + "redirect_to": "imagebind_ops", + "suggested_action": "depth_search" + } + } + }, + { + "label": "gap", + "next": { + "type": "leaf", + "action": "redirect_tool", + "reason": "This task is about comparing what different modalities reveal. Use imagebind_ops action='gap' to measure the semantic difference between two modalities on the same record.", + "details": { + "redirect_to": "imagebind_ops", + "suggested_action": "gap" + } + } + }, + { + "label": "basic_search", + "next": { + "type": "leaf", + "action": "continue", + "reason": "This is a standard media search — proceed with media_search." + } + } + ], + "node_class": "check", + "check_fn": "_check_cross_modal_intent" + }, + "category": "tool-routing", + "system": true, + "triggers": [ + { + "event": "pre_tool", + "tool": "media_search", + "description": "Before a media_search call, check if the query involves cross-modal computation", + "context_keys": [ + "task_description", + "query" + ] + } + ], + "seed_version": 1 + }, + { + "id": "context-overflow", + "name": "Context Overflow", + "description": "Decides how to handle context that exceeds the token budget: compact, flush-and-compact, or full reset.", + "created": "2026-04-27T15:57:50.170379+00:00", + "updated": "2026-04-27T15:57:50.170379+00:00", + "root": { + "type": "choice", + "question": "overflow severity", + "options": [ + { + "label": "mild", + "next": { + "type": "leaf", + "action": "compact_and_continue", + "reason": "Context is slightly over budget. Compact older messages and continue. No data loss expected." + } + }, + { + "label": "moderate", + "next": { + "type": "leaf", + "action": "flush_and_compact", + "reason": "Context significantly over budget. Flush all critical data to files first, then compact aggressively. Some detail will be lost from older rounds.", + "details": { + "flush_first": true, + "aggressive": true + } + } + }, + { + "label": "severe", + "next": { + "type": "leaf", + "action": "reset_with_briefing", + "reason": "Context is far over budget even after compaction. Reset the agent's history and rebuild context from briefings + blackboard + current step only.", + "details": { + "full_reset": true + } + } + } + ], + "node_class": "check", + "check_fn": "_check_overflow_severity" + }, + "category": "orchestration", + "system": true, + "triggers": [ + { + "event": "context_overflow", + "description": "When building agent context exceeds token budget", + "context_keys": [ + "tokens_needed", + "tokens_available" + ] + } + ], + "seed_version": 1 + }, + { + "id": "retry-strategy", + "name": "Retry Strategy", + "description": "When an agent step fails, this tree picks a specific recovery strategy based on the failure type and how many retries have already been attempted.", + "created": "2026-04-27T15:57:50.179014+00:00", + "updated": "2026-04-27T15:57:50.179014+00:00", + "root": { + "type": "choice", + "question": "retry count", + "options": [ + { + "label": "first_retry", + "next": { + "type": "choice", + "question": "failure type (first retry)", + "options": [ + { + "label": "empty_response", + "next": { + "type": "leaf", + "action": "retry_with_simpler_prompt", + "reason": "Agent produced empty output. Retry with a simplified task description and lower temperature.", + "details": { + "simplify_prompt": true, + "reduce_temp": true + } + } + }, + { + "label": "repetition_loop", + "next": { + "type": "leaf", + "action": "retry_with_context_reset", + "reason": "Agent is in a repetition loop. Clear its recent history and retry with explicit 'do not repeat' instruction.", + "details": { + "clear_history": true, + "add_anti_repeat": true + } + } + }, + { + "label": "tool_failure", + "next": { + "type": "leaf", + "action": "retry_with_different_tools", + "reason": "Agent's tools failed. Retry with alternative tools or a different approach to the same task.", + "details": { + "swap_tools": true + } + } + }, + { + "label": "other", + "next": { + "type": "leaf", + "action": "retry_with_guidance", + "reason": "Retry with more specific instructions about what went wrong and how to fix it.", + "details": { + "add_error_context": true + } + } + } + ], + "node_class": "check", + "check_fn": "_check_failure_type" + } + }, + { + "label": "second_retry", + "next": { + "type": "leaf", + "action": "retry_with_different_agent", + "reason": "Second retry failed. Try a different agent type for this task.", + "details": { + "force_different_agent": true, + "reduce_temp": true + } + } + }, + { + "label": "third_or_more", + "next": { + "type": "leaf", + "action": "escalate_or_skip", + "reason": "Multiple retries failed. Skip this step and note the failure, or escalate to the user if the step is critical.", + "details": { + "skip_if_non_critical": true, + "escalate_if_critical": true + } + } + } + ], + "node_class": "check", + "check_fn": "_check_retry_count" + }, + "category": "orchestration", + "system": true, + "triggers": [ + { + "event": "error_recovery", + "description": "When post-step validation indicates failure", + "context_keys": [ + "retry_count", + "output_issues" + ] + } + ], + "seed_version": 1 + }, + { + "id": "routing-validation", + "name": "Routing Validation", + "description": "Validates the Director's agent choice before delegation. Checks if the chosen agent has the required tools and whether a better-equipped agent is available.", + "created": "2026-04-27T15:57:50.187536+00:00", + "updated": "2026-04-27T15:57:50.187536+00:00", + "root": { + "type": "choice", + "question": "agent tool check", + "options": [ + { + "label": "has_tools", + "next": { + "type": "leaf", + "action": "accept_routing", + "reason": "Director's agent choice has the tools needed for this task." + } + }, + { + "label": "missing_tools", + "next": { + "type": "choice", + "question": "better agent available?", + "options": [ + { + "label": "better_exists", + "next": { + "type": "leaf", + "action": "override_routing", + "reason": "A better-equipped agent is available for this task. Override the Director's choice.", + "details": { + "use_suggested_agent": true + } + } + }, + { + "label": "no_better", + "next": { + "type": "leaf", + "action": "accept_with_warning", + "reason": "No better agent available. Accept the Director's choice but warn about missing tools." + } + } + ], + "node_class": "check", + "check_fn": "_check_better_agent_available" + } + }, + { + "label": "breaker_open", + "next": { + "type": "leaf", + "action": "force_different_agent", + "reason": "The chosen agent's circuit breaker is open. Must use a different agent.", + "details": { + "skip_agent": true + } + } + } + ], + "node_class": "check", + "check_fn": "_check_agent_has_required_tools" + }, + "category": "orchestration", + "system": true, + "triggers": [ + { + "event": "pre_delegation", + "description": "After Director picks an agent, before delegation", + "context_keys": [ + "chosen_agent", + "agent_tools", + "task_type", + "available_agents" + ] + } + ], + "seed_version": 1 + }, + { + "id": "director-review-decision", + "name": "Director Review Decision", + "description": "After the Director reviews a completed step, this tree decides whether to continue with the plan or replan. Users can add more outcomes like 'escalate_to_user' or 'partial_replan'.", + "created": "2026-04-27T15:57:50.195058+00:00", + "updated": "2026-04-27T15:57:50.195058+00:00", + "root": { + "type": "yes_no", + "question": "Based on the Director's step review, should we continue with the current plan or replan with new steps?", + "node_class": "check", + "check_fn": "_check_review_has_replan", + "context_key": "review_text", + "yes": { + "type": "leaf", + "action": "replan", + "reason": "Director's review indicates replanning is needed." + }, + "no": { + "type": "choice", + "question": "The review doesn't explicitly say REPLAN. Should we continue, or does the tone suggest problems?", + "node_class": "llm", + "context_key": "review_text", + "options": [ + { + "label": "continue", + "next": { + "type": "leaf", + "action": "continue", + "reason": "Step review is positive, continuing." + } + }, + { + "label": "replan", + "next": { + "type": "leaf", + "action": "replan", + "reason": "Director's review suggests issues warrant replanning." + } + } + ] + } + }, + "category": "orchestration", + "system": true, + "triggers": [ + { + "event": "director_review", + "description": "After Director reviews a step, decide continue or replan", + "context_keys": [ + "review_text" + ] + } + ], + "seed_version": 1 + }, + { + "id": "task-completion-check", + "name": "Task Completion Check", + "description": "Verifies whether an agent completed its assigned task. Uses fast deterministic checks first (response length, tool usage, keyword detection) to short-circuit the LLM call in obvious cases.", + "created": "2026-04-27T15:57:50.205120+00:00", + "updated": "2026-04-27T15:57:50.205120+00:00", + "root": { + "type": "yes_no", + "question": "Is the agent's response extremely short (under 50 chars)?", + "node_class": "check", + "check_fn": "_check_response_too_short", + "context_key": "agent_response", + "yes": { + "type": "leaf", + "action": "incomplete", + "reason": "Response is too short to contain meaningful work.", + "details": { + "confidence": "0.9" + } + }, + "no": { + "type": "yes_no", + "question": "Does the response contain error indicators?", + "node_class": "check", + "check_fn": "_check_has_error_indicators", + "context_key": "agent_response", + "yes": { + "type": "leaf", + "action": "incomplete", + "reason": "Response contains error indicators suggesting failure.", + "details": { + "confidence": "0.7" + } + }, + "no": { + "type": "yes_no", + "question": "Did the agent accomplish its assigned task? Answer 'yes' if it produced useful output.", + "node_class": "llm", + "context_key": "task_context", + "yes": { + "type": "leaf", + "action": "complete", + "reason": "Task appears complete.", + "details": { + "confidence": "0.8" + } + }, + "no": { + "type": "leaf", + "action": "incomplete", + "reason": "LLM determined task is not complete.", + "details": { + "confidence": "0.7" + } + } + } + } + }, + "category": "validation", + "system": true, + "triggers": [ + { + "event": "task_completion", + "description": "After agent responds, check if task is done", + "context_keys": [ + "task_description", + "agent_response", + "agent_plan_steps", + "completed_steps" + ] + } + ], + "seed_version": 1 + }, + { + "id": "guardrail-retry-strategy", + "name": "Guardrail Retry Strategy", + "description": "When guardrails detect quality issues, this tree decides the retry strategy. Different failure types (placeholder text, stub code, empty file, minor issues) get different treatment instead of a one-size-fits-all retry.", + "created": "2026-04-27T15:57:50.215191+00:00", + "updated": "2026-04-27T15:57:50.215191+00:00", + "root": { + "type": "yes_no", + "question": "Have we already exhausted retry attempts?", + "node_class": "check", + "check_fn": "_check_retries_exhausted", + "context_key": "retry_count", + "yes": { + "type": "leaf", + "action": "accept_with_warnings", + "reason": "Max retries reached, accepting output with warnings." + }, + "no": { + "type": "choice", + "question": "What type of guardrail failure is most severe?", + "node_class": "check", + "check_fn": "_check_guardrail_failure_type", + "context_key": "failure_types", + "options": [ + { + "label": "placeholder_text", + "next": { + "type": "leaf", + "action": "retry_gentle", + "reason": "Placeholder text detected -- retry with reminder." + } + }, + { + "label": "stub_code", + "next": { + "type": "leaf", + "action": "retry_strict", + "reason": "Stub code detected -- retry with stronger instructions." + } + }, + { + "label": "empty_output", + "next": { + "type": "leaf", + "action": "escalate", + "reason": "Empty output -- escalate, retry unlikely to help." + } + }, + { + "label": "minor_issues", + "next": { + "type": "leaf", + "action": "retry_gentle", + "reason": "Minor issues found -- retry with specific feedback." + } + } + ] + } + }, + "category": "validation", + "system": true, + "triggers": [ + { + "event": "guardrail_failure", + "description": "When guardrails detect issues in agent output", + "context_keys": [ + "failure_count", + "failure_types", + "retry_count", + "max_retries" + ] + } + ], + "seed_version": 1 + }, + { + "id": "goal-completion-verification", + "name": "Goal Completion Verification", + "description": "Verifies whether the original user goal has been fully addressed before moving to post-mortem. Users can customize what 'fully addressed' means for different project types (e.g. coding projects need file checks, research needs citation counts).", + "created": "2026-04-27T15:57:50.225332+00:00", + "updated": "2026-04-27T15:57:50.225332+00:00", + "root": { + "type": "yes_no", + "question": "Does the Director's response explicitly say INCOMPLETE?", + "node_class": "check", + "check_fn": "_check_response_says_incomplete", + "context_key": "director_response", + "yes": { + "type": "leaf", + "action": "incomplete", + "reason": "Director explicitly marked the goal as incomplete." + }, + "no": { + "type": "yes_no", + "question": "Does the Director's response confirm COMPLETE?", + "node_class": "check", + "check_fn": "_check_response_says_complete", + "context_key": "director_response", + "yes": { + "type": "leaf", + "action": "verified", + "reason": "Director confirmed the goal is fully addressed." + }, + "no": { + "type": "yes_no", + "question": "Has the original goal been FULLY addressed? Answer 'yes' only if ALL requirements are met.", + "node_class": "llm", + "context_key": "verification_context", + "yes": { + "type": "leaf", + "action": "verified", + "reason": "LLM assessment confirms goal is met." + }, + "no": { + "type": "leaf", + "action": "incomplete", + "reason": "LLM assessment indicates goal is not fully met." + } + } + } + }, + "category": "orchestration", + "system": true, + "triggers": [ + { + "event": "goal_verification", + "description": "After Director believes work is complete, verify goal", + "context_keys": [ + "user_message", + "director_response", + "project_type" + ] + } + ], + "seed_version": 1 + }, + { + "id": "benchmark-analysis-routing", + "name": "Benchmark Analysis Routing", + "description": "Decides how to handle performance analysis requests. Routes to HR for post-mortem analysis, to Analyst for deep data investigation, or provides a quick summary when the question is simple.", + "created": "2026-04-27T15:57:50.235359+00:00", + "updated": "2026-04-27T15:57:50.235359+00:00", + "root": { + "type": "yes_no", + "question": "Is this a post-mortem analysis request (triggered automatically after a Director session)?", + "node_class": "check", + "check_fn": "_check_context_equals", + "context_key": "request_source", + "check_value": "post_mortem", + "yes": { + "type": "yes_no", + "question": "Is an HR agent available in this project?", + "node_class": "check", + "check_fn": "_check_context_truthy", + "context_key": "has_hr_agent", + "yes": { + "type": "leaf", + "action": "route_to_hr", + "reason": "HR agent handles post-mortem benchmark analysis. It will use benchmark_query to pull timing data and correlate with the post-mortem text." + }, + "no": { + "type": "leaf", + "action": "log_to_kb", + "reason": "No HR agent available. Save benchmark summary to the knowledgebase under topic 'system-health' so it can be reviewed later." + } + }, + "no": { + "type": "yes_no", + "question": "Does the request involve comparing multiple sessions or deep statistical analysis?", + "node_class": "llm", + "context_key": "question", + "yes": { + "type": "yes_no", + "question": "Is an Analyst agent available in this project?", + "node_class": "check", + "check_fn": "_check_context_truthy", + "context_key": "has_analyst_agent", + "yes": { + "type": "leaf", + "action": "route_to_analyst", + "reason": "Complex analysis request routed to Analyst agent, which has chart_create, code_run, and benchmark_query for deep investigation." + }, + "no": { + "type": "leaf", + "action": "route_to_hr", + "reason": "No Analyst available. HR can handle benchmark analysis with benchmark_query, though without chart generation capabilities." + } + }, + "no": { + "type": "leaf", + "action": "quick_summary", + "reason": "Simple performance question. Use benchmark_query action=summary session_id=latest to get a quick overview without full agent involvement." + } + } + }, + "category": "orchestration", + "system": true, + "triggers": [ + { + "event": "benchmark_analysis", + "description": "When an agent or the system needs performance data analyzed -- after sessions, during reviews, or on user request.", + "context_keys": [ + "request_source", + "question", + "has_hr_agent", + "has_analyst_agent", + "session_count" + ] + } + ], + "seed_version": 1 + }, + { + "id": "synesthesia-routing", + "name": "Synesthesia Routing", + "description": "Route data conversion and modality transformation requests to leos_synesthesia.", + "created": "2026-04-27T15:57:50.245387+00:00", + "updated": "2026-04-27T15:57:50.245387+00:00", + "root": { + "type": "check", + "label": "needs_modality_conversion", + "question": "Does the user want to convert data between modalities (image, audio, thermal, QR, steganography)?", + "yes": { + "type": "check", + "label": "is_steganography", + "question": "Is this about hiding data in images (steganography)?", + "yes": { + "type": "leaf", + "action": "leos_synesthesia", + "reason": "Use leos_synesthesia action=stego_embed or stego_extract for steganographic operations." + }, + "no": { + "type": "leaf", + "action": "leos_synesthesia", + "reason": "Use leos_synesthesia with to_image, to_audio, to_thermal, to_qr, encode_all, etc." + } + }, + "no": { + "type": "leaf", + "action": "pass", + "reason": "Not a modality conversion request — check other tools." + } + }, + "category": "tool_selection", + "system": true, + "triggers": [ + { + "event": "synesthesia", + "description": "Plan 19 composability surface — modality routing" + } + ], + "seed_version": 1 + }, + { + "id": "cross-modal-routing", + "name": "Cross-Modal Routing", + "description": "Route cross-modal blend/analogy/gap requests to leos_crossmodal.", + "created": "2026-04-27T15:57:50.255431+00:00", + "updated": "2026-04-27T15:57:50.255431+00:00", + "root": { + "type": "check", + "label": "needs_crossmodal", + "question": "Does the user want to blend, compare, or do analogies across text/image/audio modalities?", + "yes": { + "type": "leaf", + "action": "leos_crossmodal", + "reason": "Use leos_crossmodal for blend, analogy, gap, depth_search, motion, resonance in ImageBind space." + }, + "no": { + "type": "leaf", + "action": "pass", + "reason": "Not a cross-modal operation — check other tools." + } + }, + "category": "tool-routing", + "system": true, + "triggers": [ + { + "event": "pre_tool", + "tool": "media_search", + "description": "Before a media_search call, check if the query involves cross-modal computation", + "context_keys": [ + "task_description", + "query" + ] + } + ], + "seed_version": 1 + }, + { + "id": "embedding-science-routing", + "name": "Embedding Science Routing", + "description": "Route embedding topology, clustering, and analysis requests to leos_science.", + "created": "2026-04-27T15:57:50.265612+00:00", + "updated": "2026-04-27T15:57:50.265612+00:00", + "root": { + "type": "check", + "label": "needs_embedding_analysis", + "question": "Does the user want to analyze embedding topology, cluster vectors, or use advanced embedding operations?", + "yes": { + "type": "check", + "label": "is_predictive", + "question": "Is this about deciding whether to skip the LLM for a familiar task?", + "yes": { + "type": "leaf", + "action": "leos_science", + "reason": "Use leos_science action=predict_check to test if System 1 (embedding-only) can handle this." + }, + "no": { + "type": "leaf", + "action": "leos_science", + "reason": "Use leos_science for cluster_kmeans, dim_pca, sim_matrix, dark_matter_scan, etc." + } + }, + "no": { + "type": "leaf", + "action": "pass", + "reason": "Not an embedding analysis request — check other tools." + } + }, + "category": "tool_selection", + "system": true, + "triggers": [ + { + "event": "embedding_analysis", + "description": "Plan 19 composability surface — embedding analysis routing" + } + ], + "seed_version": 1 + } + ] +} \ No newline at end of file diff --git a/project/infra/dev_tools.py b/project/infra/dev_tools.py index 717d3be..8d982e5 100644 --- a/project/infra/dev_tools.py +++ b/project/infra/dev_tools.py @@ -44,7 +44,6 @@ logger = logging.getLogger("dev_tools") -# Import shell_guard for safe execution try: from shell_guard import execute_safe, check_command _HAS_GUARD = True diff --git a/project/infra/gpu_coordinator.py b/project/infra/gpu_coordinator.py new file mode 100644 index 0000000..8ac3c37 --- /dev/null +++ b/project/infra/gpu_coordinator.py @@ -0,0 +1,417 @@ +""" +gpu_coordinator.py – Cross-framework VRAM coordinator for leOS. + +The single-host AI workstation problem in 2025-2026: Ollama, faster- +whisper, ImageBind, and the embedding processors all want GPU memory +and none of them know about each other. Ollama's KEEP_ALIVE policy +keeps an LLM resident for 5 minutes after use; meanwhile a podcast +ingest fires up faster-whisper which silently thrashes when free VRAM +is too tight. Result: a transcription that should take 70 seconds +hangs for half an hour, or OOMs. + +This module is the cooperator that ties them together. It does NOT +own model loading itself — each framework keeps its own loaders. +What it owns is the *priority decision*: when a leOS subsystem is +about to ask the GPU for N MB, the coordinator first checks whether +that fits in current free VRAM, and if not, evicts lower-priority +framework state until it does. + +Stays pure stdlib (subprocess + json) so it can run anywhere torch +runs and isn't load-bearing on its own. + +Public API: + + query_free_vram_mb() → int | None on no-GPU host + query_loaded_ollama_models() → list[dict] (name, size_mb) + evict_ollama_for(needed_mb) → bool (True if evicted any) + gpu_priority(name, needed_mb) → context manager + self_test() → 0 on success, 1 on failure + +Author: leOS Tier 2 GPU coordination — Ralph loop iteration 1 +""" + +import contextlib +import json +import logging +import os +import re +import shutil +import subprocess +import sys +import time + +logger = logging.getLogger("gpu_coordinator") + +# --------------------------------------------------------------------------- +# Tunables (overridable via config.json "gpu" section or env) +# --------------------------------------------------------------------------- + +# How aggressive the coordinator is about evicting Ollama models when a +# leOS subsystem requests VRAM. "always" evicts before any heavy work; +# "on_pressure" evicts only when free < needed_mb; "never" disables +# eviction entirely (cooperative-only mode, relies on KEEP_ALIVE). +DEFAULT_POLICY = "on_pressure" + +# Safety margin added to every needed_mb request. Accounts for CUDA +# context overhead, allocator fragmentation, framework working set. +SAFETY_MARGIN_MB = 1024 + +# Where Ollama's CLI lives. Resolved lazily so non-Windows hosts that +# install via brew/apt also work. +def _ollama_bin(): + if shutil.which("ollama"): + return "ollama" + candidates = [ + os.path.expanduser( + r"~\AppData\Local\Programs\Ollama\ollama.exe" + ), + r"C:\Program Files\Ollama\ollama.exe", + "/usr/local/bin/ollama", + "/opt/homebrew/bin/ollama", + ] + for c in candidates: + if os.path.exists(c): + return c + return None + + +def _run(cmd, timeout=10): + """Run a command and return (stdout, returncode). Returns ('', -1) on failure.""" + try: + r = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=timeout, + check=False, + ) + return r.stdout.decode("utf-8", errors="replace"), r.returncode + except Exception as e: + logger.debug("subprocess failed %s: %s", cmd, e) + return "", -1 + + +# --------------------------------------------------------------------------- +# VRAM probe — nvidia-smi for now; gpu_detect.py covers the wider matrix +# --------------------------------------------------------------------------- + +def query_free_vram_mb(): + """Return free VRAM in MB on the primary GPU, or None on a non-NVIDIA host. + + Doesn't import torch — works pre-install and avoids holding a CUDA + context just to ask a question. + """ + if not shutil.which("nvidia-smi"): + return None + out, rc = _run([ + "nvidia-smi", + "--query-gpu=memory.free", + "--format=csv,noheader,nounits", + ]) + if rc != 0 or not out.strip(): + return None + try: + # Multi-GPU: pick the largest free slice + return max(int(x.strip()) for x in out.strip().splitlines() if x.strip()) + except Exception: + return None + + +def query_total_vram_mb(): + """Total VRAM on the primary GPU, MB.""" + if not shutil.which("nvidia-smi"): + return None + out, rc = _run([ + "nvidia-smi", + "--query-gpu=memory.total", + "--format=csv,noheader,nounits", + ]) + if rc != 0 or not out.strip(): + return None + try: + return max(int(x.strip()) for x in out.strip().splitlines() if x.strip()) + except Exception: + return None + + +# --------------------------------------------------------------------------- +# Ollama probe + eviction +# --------------------------------------------------------------------------- + +_OLLAMA_PS_HEADER = re.compile(r"^\s*NAME\s+ID\s+SIZE", re.IGNORECASE) +_OLLAMA_PS_ROW = re.compile( + r"^(\S+)\s+\S+\s+([\d.]+)\s*([KMG]?B)\s+", + re.IGNORECASE, +) + + +def query_loaded_ollama_models(): + """Return [{"name": str, "size_mb": int, "raw": line}, ...]. + + Parses ``ollama ps`` text output. Returns [] if Ollama isn't + installed, isn't running, or has nothing loaded. + """ + bin_ = _ollama_bin() + if not bin_: + return [] + out, rc = _run([bin_, "ps"], timeout=5) + if rc != 0: + return [] + models = [] + for line in out.splitlines(): + if _OLLAMA_PS_HEADER.match(line): + continue + m = _OLLAMA_PS_ROW.match(line) + if not m: + continue + name, val, unit = m.group(1), float(m.group(2)), m.group(3).upper() + unit_mb = {"KB": 1 / 1024.0, "MB": 1.0, "GB": 1024.0, "B": 1 / (1024 * 1024.0)}.get(unit, 1.0) + size_mb = int(val * unit_mb) + models.append({"name": name, "size_mb": size_mb, "raw": line.strip()}) + return models + + +def evict_ollama_model(name): + """Stop one Ollama model. Returns True on success.""" + bin_ = _ollama_bin() + if not bin_: + return False + _, rc = _run([bin_, "stop", name], timeout=15) + if rc == 0: + logger.info("Evicted Ollama model: %s", name) + return True + return False + + +def evict_ollama_for(needed_mb, policy=None): + """Evict Ollama models until free VRAM >= needed_mb (with safety margin). + + Returns the list of evicted model names. No-op on hosts without + nvidia-smi or Ollama. + """ + policy = policy or DEFAULT_POLICY + if policy == "never": + return [] + + target_mb = needed_mb + SAFETY_MARGIN_MB + free = query_free_vram_mb() + if free is None: + return [] + + if policy == "on_pressure" and free >= target_mb: + return [] # already enough room + + loaded = query_loaded_ollama_models() + # Largest first — biggest reclaim per eviction + loaded.sort(key=lambda m: m["size_mb"], reverse=True) + evicted = [] + for m in loaded: + if evict_ollama_model(m["name"]): + evicted.append(m["name"]) + # Give Ollama ~1s to actually free the memory before re-checking + time.sleep(1.0) + new_free = query_free_vram_mb() or 0 + if new_free >= target_mb: + break + return evicted + + +# --------------------------------------------------------------------------- +# Public context manager — what callers actually use +# --------------------------------------------------------------------------- + +@contextlib.contextmanager +def gpu_priority(name, needed_mb=2048, policy=None, sustain_ollama=True): + """Reserve VRAM for a leOS subsystem during a model load. + + Usage:: + + from gpu_coordinator import gpu_priority + + with gpu_priority("whisper", needed_mb=4096): + model = WhisperModel("large-v3", device="cuda") + # ... transcribe ... + + Two-phase coordination: + + 1. **Eviction (one-shot, on enter)**: if free VRAM is below + ``needed_mb`` (plus a safety margin), the largest Ollama model + is stopped to make room. + + 2. **Sustained lease (held until exit)**: when ``sustain_ollama`` + is true, every Ollama generate/chat request from this process + gets ``keep_alive=0`` injected via the OllamaDriver's module- + global override. That means any agent_session call that fires + *while* Whisper is transcribing still gets its answer, but + Ollama unloads the LLM immediately after responding instead of + holding it resident for the default 5 minutes. Without this, + a single ambient_reactor / goal_auto_populator call mid- + transcribe pulls qwen3:8b right back in and OOMs the GPU. + + ``policy``: "always" | "on_pressure" | "never". Default comes + from the global ``DEFAULT_POLICY`` (settable via env or config). + """ + t0 = time.time() + free_before = query_free_vram_mb() + evicted = evict_ollama_for(needed_mb, policy=policy) + if evicted: + free_after = query_free_vram_mb() + logger.info( + "[%s] reserved ~%dMB; evicted %s; free %s→%s MB", + name, needed_mb, ", ".join(evicted), + free_before, free_after, + ) + elif free_before is not None: + logger.info( + "[%s] reserved ~%dMB; %d MB free, no eviction needed", + name, needed_mb, free_before, + ) + + # Sustained lease: force every concurrent Ollama call from this + # process to release VRAM immediately after responding. Imported + # lazily so a CPU-only host without ollama_driver still works. + _prev_override = _MISSING = object() + if sustain_ollama: + try: + import ollama_driver # type: ignore + _prev_override = ollama_driver.set_keep_alive_override(0) + logger.info( + "[%s] sustained Ollama keep_alive=0 lease active", name, + ) + except Exception as _e: + logger.debug( + "[%s] sustain_ollama unavailable (%s)", name, _e, + ) + + try: + yield + finally: + if _prev_override is not _MISSING: + try: + import ollama_driver # type: ignore + ollama_driver.set_keep_alive_override(_prev_override) + logger.info( + "[%s] released sustained Ollama lease", name, + ) + except Exception: + pass + logger.debug( + "[%s] gpu_priority block ended after %.1fs", + name, time.time() - t0, + ) + + +# --------------------------------------------------------------------------- +# CLI / self-test +# --------------------------------------------------------------------------- + +def status(): + """Snapshot dict for diagnostics / leos_status.""" + return { + "free_vram_mb": query_free_vram_mb(), + "total_vram_mb": query_total_vram_mb(), + "loaded_ollama_models": query_loaded_ollama_models(), + "ollama_available": _ollama_bin() is not None, + "policy": os.environ.get("LEOS_GPU_POLICY", DEFAULT_POLICY), + } + + +def self_test(): + """Exercise the public API. Exit 0 on success, 1 on failure. + + Runs as ``python -m infra.gpu_coordinator self_test`` from the + Ralph completion check, and as a stand-alone smoke test from CI. + """ + failures = [] + + # 1. VRAM probes don't crash and return either None or int + free = query_free_vram_mb() + total = query_total_vram_mb() + if free is not None and not isinstance(free, int): + failures.append(f"query_free_vram_mb returned {type(free)}") + if total is not None and not isinstance(total, int): + failures.append(f"query_total_vram_mb returned {type(total)}") + if free is not None and total is not None and free > total: + failures.append(f"free ({free}) > total ({total}) — impossible") + + # 2. Ollama probes don't crash and return list[dict] + loaded = query_loaded_ollama_models() + if not isinstance(loaded, list): + failures.append(f"query_loaded_ollama_models returned {type(loaded)}") + for m in loaded: + if not isinstance(m, dict) or "name" not in m or "size_mb" not in m: + failures.append(f"malformed ollama model entry: {m}") + + # 3. evict_ollama_for is callable with policy=never (no-op) and returns list + e = evict_ollama_for(1024, policy="never") + if not isinstance(e, list): + failures.append(f"evict_ollama_for returned {type(e)}") + if e: + failures.append("policy=never should not evict but returned " + str(e)) + + # 4. context manager doesn't blow up + try: + with gpu_priority("self_test", needed_mb=128, policy="never"): + pass + except Exception as ex: + failures.append(f"gpu_priority raised: {ex}") + + # 5. status() returns the documented shape + s = status() + for key in ("free_vram_mb", "total_vram_mb", "loaded_ollama_models", + "ollama_available", "policy"): + if key not in s: + failures.append(f"status() missing key: {key}") + + # 6. Sustained Ollama lease engages and disengages cleanly. We + # don't require ollama_driver to be importable in this stdlib + # context — the test passes whether or not the import works. + try: + import ollama_driver # type: ignore + before = ollama_driver.get_keep_alive_override() + with gpu_priority("self_test_lease", needed_mb=128, policy="never"): + during = ollama_driver.get_keep_alive_override() + if during != 0: + failures.append( + f"sustain_ollama lease should set override=0, got {during}" + ) + after = ollama_driver.get_keep_alive_override() + if after != before: + failures.append( + f"override not restored: before={before} after={after}" + ) + except ImportError: + pass # ollama_driver not on path during stdlib-only invocation + + if failures: + print("self_test FAILED:") + for f in failures: + print(f" - {f}") + return 1 + print("self_test OK") + print(json.dumps(status(), indent=2)) + return 0 + + +def _main(): + args = sys.argv[1:] + cmd = args[0] if args else "status" + if cmd == "self_test": + sys.exit(self_test()) + if cmd == "status": + print(json.dumps(status(), indent=2)) + return + if cmd == "evict": + needed = int(args[1]) if len(args) > 1 else 4096 + evicted = evict_ollama_for(needed) + print(json.dumps({"evicted": evicted, "free_after": query_free_vram_mb()}, indent=2)) + return + if cmd == "free": + print(query_free_vram_mb()) + return + print(f"unknown subcommand: {cmd}", file=sys.stderr) + sys.exit(2) + + +if __name__ == "__main__": + _main() diff --git a/project/infra/gpu_detect.py b/project/infra/gpu_detect.py new file mode 100644 index 0000000..e56ff94 --- /dev/null +++ b/project/infra/gpu_detect.py @@ -0,0 +1,372 @@ +""" +gpu_detect.py – Pre-install GPU/accelerator detection for leOS. + +The bootstrap script needs to know which torch wheel to fetch *before* +any torch is installed, so this module touches no Python ML library. +It uses OS-level signals only: + + - ``nvidia-smi`` on Windows + Linux → CUDA driver version + GPU name + - ``rocm-smi`` on Linux → AMD ROCm version + - ``uname``/sys.platform on Darwin → MPS (Apple Silicon) + - falls through to CPU when nothing else lights up + +The result is shaped so the same dict can drive both the install-time +wheel selection and the runtime ``leos_status`` panel — no second +detection pass needed. + +Result schema: + + { + "backend": "cuda" | "rocm" | "mps" | "cpu", + "available": bool, + "gpu_name": str | None, + "driver_cuda_version": str | None, # max CUDA the driver allows (nvidia-smi) + "rocm_version": str | None, + "vram_gb": float | None, # NVIDIA only without extra deps + "platform": "windows" | "linux" | "darwin", + "torch_index_url": str, # the URL to hand pip --index-url + "torch_pip_extras": list[str], # extra args, e.g. ["--pre"] for rocm + "detected_at": ISO timestamp, + "detection_method": str, # short trace of how we decided + } + +The picked ``torch_index_url`` follows PyTorch's published index map. +We default to a known-good CUDA tier rather than chasing the literal +driver version: a driver that supports CUDA 12.4 will happily run a +12.1 wheel, and that's the build with the broadest compatibility for +common torch versions in the leOS pinset. + +Author: leOS GPU detection — Phase 1 +""" + +import json +import os +import platform +import re +import shutil +import subprocess +import sys +import time + +# --------------------------------------------------------------------------- +# Wheel index map +# --------------------------------------------------------------------------- +# +# Keep this in one place so both the bootstrap and the runtime status +# panel agree on what got picked. When PyTorch publishes new tiers +# (cu126, cu130, …) just add an entry; the picker prefers the highest +# tier the driver supports. +# +_CUDA_INDEX = { + # cuda toolkit version → published wheel index + "12.4": "https://download.pytorch.org/whl/cu124", + "12.1": "https://download.pytorch.org/whl/cu121", + "11.8": "https://download.pytorch.org/whl/cu118", +} +_ROCM_INDEX = { + "6.2": "https://download.pytorch.org/whl/rocm6.2", + "6.1": "https://download.pytorch.org/whl/rocm6.1", + "5.7": "https://download.pytorch.org/whl/rocm5.7", +} +_CPU_INDEX = "https://download.pytorch.org/whl/cpu" + +# Order matters: newest first. We pick the highest tier that the +# driver supports. +_CUDA_PREF_ORDER = ["12.4", "12.1", "11.8"] +_ROCM_PREF_ORDER = ["6.2", "6.1", "5.7"] + + +# --------------------------------------------------------------------------- +# Caching +# --------------------------------------------------------------------------- + +def _cache_path(): + base = os.environ.get("LEOS_DATA_DIR") + if not base: + # data dir lives next to project root + base = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "data", + ) + return os.path.join(base, "gpu_capabilities.json") + + +def _load_cached(max_age_seconds=86400): + path = _cache_path() + if not os.path.exists(path): + return None + try: + age = time.time() - os.path.getmtime(path) + if age > max_age_seconds: + return None + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except Exception: + return None + + +def _save_cached(result): + path = _cache_path() + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(result, f, indent=2) + except Exception: + pass # cache is a convenience, never load-bearing + + +# --------------------------------------------------------------------------- +# Detectors +# --------------------------------------------------------------------------- + +def _run(cmd, timeout=5): + """Run a command and return stdout. Returns "" on any failure.""" + try: + out = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=timeout, + check=False, + ).stdout + return out.decode("utf-8", errors="replace") + except Exception: + return "" + + +def _detect_nvidia(): + """Parse ``nvidia-smi`` for driver CUDA version, GPU name, VRAM.""" + if not shutil.which("nvidia-smi"): + return None + raw = _run([ + "nvidia-smi", + "--query-gpu=name,memory.total,driver_version", + "--format=csv,noheader", + ]) + if not raw.strip(): + return None + # First line of CSV: "NVIDIA GeForce RTX 4090, 24564 MiB, 565.57.01" + first = raw.strip().splitlines()[0] + parts = [p.strip() for p in first.split(",")] + name = parts[0] if parts else None + vram_gb = None + if len(parts) > 1: + m = re.match(r"(\d+)\s*MiB", parts[1]) + if m: + vram_gb = round(int(m.group(1)) / 1024.0, 1) + + # Driver's max CUDA — separate query + cuda_raw = _run(["nvidia-smi"]) + driver_cuda = None + m = re.search(r"CUDA Version:\s*([0-9]+\.[0-9]+)", cuda_raw) + if m: + driver_cuda = m.group(1) + + return { + "gpu_name": name, + "vram_gb": vram_gb, + "driver_cuda_version": driver_cuda, + } + + +def _detect_rocm(): + """Parse ``rocm-smi`` for ROCm presence.""" + if not shutil.which("rocm-smi"): + return None + raw = _run(["rocm-smi", "--showproductname"]) + if not raw.strip(): + return None + name = None + for line in raw.splitlines(): + m = re.search(r"Card series:\s*(.+)$", line) + if m: + name = m.group(1).strip() + break + # ROCm version — best-effort from rocminfo + ver = None + info = _run(["rocminfo"]) + m = re.search(r"ROCm Version:\s*([0-9]+\.[0-9]+)", info) + if m: + ver = m.group(1) + return {"gpu_name": name, "rocm_version": ver} + + +def _detect_mps(): + """macOS Apple Silicon (M1/M2/M3) → MPS backend.""" + if sys.platform != "darwin": + return None + machine = platform.machine().lower() + if machine not in ("arm64", "aarch64"): + return None + return {"gpu_name": f"Apple Silicon ({machine})"} + + +# --------------------------------------------------------------------------- +# Index-URL picker +# --------------------------------------------------------------------------- + +def _pick_cuda_index(driver_cuda_version): + """Choose the highest CUDA wheel tier the driver allows. + + Drivers are forward-compatible inside a major/minor pair: a driver + that reports 12.4 will run 12.1 and 11.8 wheels. We pick the + highest tier ≤ driver, falling back to the lowest known tier so + the install doesn't refuse to proceed. + """ + if not driver_cuda_version: + return _CUDA_INDEX[_CUDA_PREF_ORDER[-1]] + try: + d_major, d_minor = (int(x) for x in driver_cuda_version.split(".")[:2]) + except Exception: + return _CUDA_INDEX[_CUDA_PREF_ORDER[-1]] + for tier in _CUDA_PREF_ORDER: + t_major, t_minor = (int(x) for x in tier.split(".")) + if (t_major, t_minor) <= (d_major, d_minor): + return _CUDA_INDEX[tier] + return _CUDA_INDEX[_CUDA_PREF_ORDER[-1]] + + +def _pick_rocm_index(rocm_version): + if not rocm_version: + return _ROCM_INDEX[_ROCM_PREF_ORDER[-1]] + try: + r_major, r_minor = (int(x) for x in rocm_version.split(".")[:2]) + except Exception: + return _ROCM_INDEX[_ROCM_PREF_ORDER[-1]] + for tier in _ROCM_PREF_ORDER: + t_major, t_minor = (int(x) for x in tier.split(".")) + if (t_major, t_minor) <= (r_major, r_minor): + return _ROCM_INDEX[tier] + return _ROCM_INDEX[_ROCM_PREF_ORDER[-1]] + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def detect(force=False): + """Run detection (or return cached result) and pick the wheel index. + + ``force=True`` skips the cache and re-probes. The cache lives at + ``data/gpu_capabilities.json`` and is treated as advisory — never + load-bearing on its own. + """ + if not force: + cached = _load_cached() + if cached: + return cached + + result = { + "backend": "cpu", + "available": False, + "gpu_name": None, + "driver_cuda_version": None, + "rocm_version": None, + "vram_gb": None, + "platform": "windows" if sys.platform.startswith("win") + else ("darwin" if sys.platform == "darwin" else "linux"), + "torch_index_url": _CPU_INDEX, + "torch_pip_extras": [], + "detected_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "detection_method": "cpu_fallback", + } + + # CUDA first — most common on leOS hosts. + nv = _detect_nvidia() + if nv: + result.update({ + "backend": "cuda", + "available": True, + "gpu_name": nv["gpu_name"], + "driver_cuda_version": nv["driver_cuda_version"], + "vram_gb": nv["vram_gb"], + "torch_index_url": _pick_cuda_index(nv["driver_cuda_version"]), + "detection_method": "nvidia-smi", + }) + _save_cached(result) + return result + + # ROCm next (Linux only in practice). + amd = _detect_rocm() + if amd: + result.update({ + "backend": "rocm", + "available": True, + "gpu_name": amd["gpu_name"], + "rocm_version": amd["rocm_version"], + "torch_index_url": _pick_rocm_index(amd["rocm_version"]), + "detection_method": "rocm-smi", + }) + _save_cached(result) + return result + + # MPS — Apple Silicon. PyTorch ships MPS support in the standard + # CPU wheel since 1.12; no special index URL needed. + mps = _detect_mps() + if mps: + result.update({ + "backend": "mps", + "available": True, + "gpu_name": mps["gpu_name"], + "torch_index_url": _CPU_INDEX, # MPS is enabled by default in stable wheels + "detection_method": "darwin+arm64", + }) + _save_cached(result) + return result + + _save_cached(result) + return result + + +def summary(): + """One-line human-readable summary, suitable for boot logs.""" + r = detect() + if r["backend"] == "cuda": + bits = [r["gpu_name"] or "NVIDIA GPU"] + if r.get("vram_gb"): + bits.append(f"{r['vram_gb']} GB VRAM") + if r.get("driver_cuda_version"): + bits.append(f"driver CUDA {r['driver_cuda_version']}") + return "CUDA – " + ", ".join(bits) + if r["backend"] == "rocm": + return f"ROCm – {r.get('gpu_name') or 'AMD GPU'}, ROCm {r.get('rocm_version') or 'unknown'}" + if r["backend"] == "mps": + return f"MPS – {r.get('gpu_name') or 'Apple Silicon'}" + return "CPU – no supported accelerator detected" + + +def index_url(): + """Convenience: just the torch wheel index URL.""" + return detect()["torch_index_url"] + + +# --------------------------------------------------------------------------- +# CLI entry — used by run.bat / run.sh during bootstrap +# --------------------------------------------------------------------------- + +def _main(): + """Print machine-readable detection output. + + Usage: + python -m infra.gpu_detect # full JSON + python -m infra.gpu_detect index # just the index URL (one line) + python -m infra.gpu_detect summary # human-readable line + python -m infra.gpu_detect --force # bypass cache + """ + args = sys.argv[1:] + force = "--force" in args + args = [a for a in args if a != "--force"] + mode = args[0] if args else "json" + + if mode == "index": + print(detect(force=force)["torch_index_url"]) + elif mode == "summary": + if force: + detect(force=True) # warm cache + print(summary()) + else: + print(json.dumps(detect(force=force), indent=2)) + + +if __name__ == "__main__": + _main() diff --git a/project/infra/leos_secrets.py b/project/infra/leos_secrets.py new file mode 100644 index 0000000..1ce0277 --- /dev/null +++ b/project/infra/leos_secrets.py @@ -0,0 +1,142 @@ +"""secrets.py — Encrypted on-disk secret store. + +Used to persist API tokens (HF_TOKEN today, OpenAI / Anthropic / etc. +later) so users enter them once via the settings UI and they survive +restarts without ending up in plain text on disk or in shell history. + +Storage shape: + data/secrets.enc — single Fernet-encrypted JSON blob: {key: value} + +Key derivation: + SHA-256(hostname + home_dir + "leOS-secrets-v1") → 32 bytes → + urlsafe_b64encode → Fernet key. Tied to the host so a stolen + secrets.enc on its own can't be decrypted on a different machine. + Not bulletproof against a determined local attacker (they'd run + the same derivation) — defends against accidental disclosure + (git, backups, sharing the data dir), not against an adversary + with full filesystem access. + +Public API: + set_token(name, value) — encrypt + persist + get_token(name) — decrypt + return, or None + clear_token(name) — delete one + list_tokens() — names only, never values + inject_into_environ() — load all, set os.environ for any not + already present in the process env + +inject_into_environ() runs at server boot so subprocesses (whisperx +loader, monologue_renderer) pick up tokens via os.environ.get(). +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import logging +import os +import socket +import threading +from pathlib import Path +from typing import Dict, List, Optional + +logger = logging.getLogger("secrets") + +_LOCK = threading.Lock() +_PATH: Optional[Path] = None +_FERNET = None + + +def _key_material() -> bytes: + host = socket.gethostname() or "unknown-host" + home = str(Path.home()) + seed = (host + "::" + home + "::leOS-secrets-v1").encode("utf-8") + digest = hashlib.sha256(seed).digest() + return base64.urlsafe_b64encode(digest) + + +def _fernet(): + global _FERNET + if _FERNET is None: + from cryptography.fernet import Fernet + _FERNET = Fernet(_key_material()) + return _FERNET + + +def init(base_dir: Optional[str] = None) -> None: + """Set the secrets file path. Call once at boot before set/get.""" + global _PATH + base = Path(base_dir or os.getcwd()) + _PATH = base / "data" / "secrets.enc" + _PATH.parent.mkdir(parents=True, exist_ok=True) + + +def _read_all() -> Dict[str, str]: + if _PATH is None or not _PATH.exists(): + return {} + try: + blob = _PATH.read_bytes() + if not blob: + return {} + plain = _fernet().decrypt(blob) + return json.loads(plain.decode("utf-8")) + except Exception as e: + logger.warning("secrets read failed: %s — treating as empty", e) + return {} + + +def _write_all(d: Dict[str, str]) -> None: + if _PATH is None: + raise RuntimeError("secrets.init() not called") + blob = _fernet().encrypt(json.dumps(d).encode("utf-8")) + tmp = _PATH.with_suffix(_PATH.suffix + ".tmp") + tmp.write_bytes(blob) + os.replace(tmp, _PATH) + try: + os.chmod(_PATH, 0o600) + except OSError: + pass # best-effort; Windows may not honor + + +def set_token(name: str, value: str) -> None: + if not name: + raise ValueError("token name required") + with _LOCK: + d = _read_all() + if value: + d[name] = value + else: + d.pop(name, None) + _write_all(d) + + +def get_token(name: str) -> Optional[str]: + with _LOCK: + return _read_all().get(name) + + +def clear_token(name: str) -> None: + set_token(name, "") + + +def list_tokens() -> List[str]: + """Return the list of stored token names (never their values).""" + with _LOCK: + return sorted(_read_all().keys()) + + +def inject_into_environ(overwrite: bool = False) -> List[str]: + """Set os.environ[name] for every stored token whose name isn't + already present (or always, if overwrite=True). Returns the list + of names that were injected. + """ + injected = [] + with _LOCK: + d = _read_all() + for name, value in d.items(): + if not value: + continue + if overwrite or not os.environ.get(name): + os.environ[name] = value + injected.append(name) + return injected diff --git a/project/infra/plugin_loader.py b/project/infra/plugin_loader.py index 1af955c..5edab33 100644 --- a/project/infra/plugin_loader.py +++ b/project/infra/plugin_loader.py @@ -1,5 +1,3 @@ -# plugin_loader.py — Adapted - """ plugin_loader.py - Plugin Discovery, Loading, and Hook Registry @@ -206,7 +204,7 @@ def is_loaded(plugin_id): } # --------------------------------------------------------------------------- -# Plan 20: Mapping from legacy hook names to signal bus signal names. +# Mapping from hook names (manifest "hooks" key) to signal bus names, for cross-dispatch. # # This allows plugins that use the old "hooks" manifest key to # automatically receive events from the new signal bus, and vice @@ -279,7 +277,7 @@ def emit_hook(event_name, **kwargs): Handlers are called synchronously in the current thread. Exceptions in handlers are caught and logged, never propagated. - Plan 20: Also emits through the signal bus so non-plugin + Also emits through the signal bus so non-plugin subscribers (internal modules) can react too. Args: @@ -297,7 +295,7 @@ def emit_hook(event_name, **kwargs): # Log but never crash -- a broken plugin must not break core print(f" [Plugin:{plugin_id}] Hook {event_name} error: {e}") - # ── Plan 20: Also emit through the signal bus ── + # Also emit through the signal bus so bus-native subscribers receive the event. # This lets internal modules that subscribed to the mapped signal # name (e.g. "agent.tool_result") receive events that were emitted # via the legacy emit_hook("on_tool_result") path. @@ -484,7 +482,7 @@ def _load_single_plugin(pid, manifest, app): else: print(f" [Plugin:{pid}] WARNING: Could not resolve handler: {handler_path}") - # --- Register signal bus handlers (Plan 20 "signals" manifest key) --- + # --- Register signal bus handlers ("signals" manifest key) --- # The "signals" key maps signal bus names directly (e.g. # "agent.step_complete") to handler paths. These go straight onto # the signal bus — no legacy _hook_handlers involvement. @@ -723,7 +721,7 @@ def set_plugin_enabled(plugin_id, enabled): if pid != plugin_id ] - # Plan 20: Also remove from the signal bus + # Also remove from signal bus to stop bus-native subscribers receiving events. try: from signal_bus import get_bus get_bus().unsubscribe_all(f"plugin:{plugin_id}") diff --git a/project/infra/service_manager.py b/project/infra/service_manager.py index b7a4b02..e158660 100644 --- a/project/infra/service_manager.py +++ b/project/infra/service_manager.py @@ -1,5 +1,3 @@ -# service_manager.py — Adapted - """ service_manager.py - Lifecycle Management for Living Services @@ -344,8 +342,6 @@ def get_service_summary(service_id): return "\n".join(lines) -# =================================================================== -# Internal helpers # =================================================================== def _generate_id(name): diff --git a/project/infra/state.py b/project/infra/state.py index 076b3ef..0380506 100644 --- a/project/infra/state.py +++ b/project/infra/state.py @@ -1,5 +1,3 @@ -# state.py — Adapted - """ state.py - Shared State & Helper Functions @@ -56,8 +54,7 @@ # Re-export these from tool_utils so existing code that does # "from state import parse_tool_metadata, _get_tools_dir" still works. -# They used to live in this file but were moved to tool_utils.py to -# break a circular import with agent_factory.py. +# Re-exported from tool_utils.py (lives there to avoid circular import with agent_factory.py). from tool_utils import parse_tool_metadata, _get_tools_dir @@ -134,7 +131,7 @@ def get_loaded_tools(): # Ollama server URL helper # # leOS stores the Ollama URL under coprocessors.ollama.model_server, -# while the old agent-swarm codebase used llm.model_server. This +# Checks coprocessors.ollama.model_server then llm.model_server as fallback. This # helper checks both paths so everything works regardless of which # config layout is active. # --------------------------------------------------------------------------- @@ -1653,14 +1650,6 @@ def clear_comms_log(project_id): _comms_seq.pop(project_id, None) -# --------------------------------------------------------------------------- -# Tool metadata parser -# parse_tool_metadata() and _get_tools_dir() have been moved to -# tool_utils.py to break a circular import with agent_factory.py. -# They are re-exported at the top of this file so existing code -# that does "from state import parse_tool_metadata" still works. - - # --------------------------------------------------------------------------- # Agent role lookup helpers # diff --git a/project/kernel.py b/project/kernel.py index f63ec18..9fb08ac 100644 --- a/project/kernel.py +++ b/project/kernel.py @@ -7,8 +7,7 @@ Every operation that can be done with pure vector math happens here. The kernel NEVER calls an LLM. If something requires language -generation, the kernel would issue an ESCALATE instruction to the -coprocessor bay (not implemented until Phase 6). +generation, the kernel issues ESCALATE to the coprocessor bay. All vector arithmetic uses spherical geometry (exp/log maps, SLERP, parallel transport) unless explicitly noted otherwise. @@ -90,13 +89,6 @@ class KernelError(Exception): # The Kernel # ================================================================ -# ============================================================================ -# Phase 12 mixin composition -# ============================================================================ -# Kernel composes from 17 subsystem mixins, one per instruction family. -# Each mixin lives in a sibling file and contains only the methods for -# its subsystem; the shim holds __init__, execute, list_instructions, -# and KernelError. from kernel_core_instructions import _CoreInstructionsMixin # noqa: F401 from kernel_signal_bus import _SignalBusMixin # noqa: F401 from kernel_storage import _StorageMixin # noqa: F401 @@ -172,8 +164,6 @@ def __init__(self, processors, state, config): - handler aliases + tool bridge come after the handler table - signal-bus subscribers come last (they reference self._on_X reactor methods on other mixins) - - Decomposed during Phase 12 Part 2 of the monolith-breakdown plan. """ self.processors = processors or {} self.state = state @@ -222,9 +212,6 @@ def __init__(self, processors, state, config): def _init_signal_bus(self, state, config): """Signal Bus (Plan 20) — central event infrastructure.""" - # ── Signal Bus (Plan 20) ── - # Central event infrastructure. Modules subscribe to named - # signals instead of being called directly. See signal_bus.py. try: from signal_bus import get_bus self.signal_bus = get_bus() @@ -406,7 +393,6 @@ def _init_phase_14_research(self, state, config): ) # Web directory — curated whitelist of known-good sites. - # Replaces unreliable third-party search APIs (DDG, Brave, etc.) # with a local, embedding-searchable directory of useful URLs. # The agent searches this first, then reads matching URLs directly. try: @@ -1108,6 +1094,7 @@ def _build_handler_table(self, state, config): "ASSIST_LIST_STEERING": self._assist_list_steering, "ASSIST_TEST_STEERING": self._assist_test_steering, "COPROCESSOR_STATUS": self._coprocessor_status, + "COPROCESSOR_PROBE": self._coprocessor_probe, # Phase 7: Reflex arc production "REFLEX_STATS": self._reflex_stats, "REFLEX_VOLATILE": self._reflex_volatile, @@ -1327,6 +1314,9 @@ def _build_handler_table(self, state, config): "MEDIA_INGEST_VIDEO": self._media_ingest_video, "MEDIA_INGEST_FILE": self._media_ingest_file, "MEDIA_INGEST_URL": self._media_ingest_url, + "MEDIA_REVIEW": self._media_review, + "RETRANSCRIBE_RECORD": self._retranscribe_record, + "RESOLVE_SPEAKERS": self._resolve_speakers, "MEDIA_EDIT": self._media_edit, "MEDIA_QUERY": self._media_query, "MEDIA_LIST": self._media_list, @@ -1460,8 +1450,13 @@ def _init_handler_aliases(self, state, config): # handlers live in tool_bridge (TOOL_KB_*) or have slightly # different registered names. Adding aliases here means both # the short and the long names work. - self.handlers["KB_SAVE"] = lambda args: self.execute("TOOL_KB_SAVE", args) - self.handlers["KB_SEARCH"] = lambda args: self.execute("TOOL_KB_SEARCH", args) + self.handlers["KB_SAVE"] = lambda args: self.execute("TOOL_KB_SAVE", args) + self.handlers["KB_SEARCH"] = lambda args: self.execute("TOOL_KB_SEARCH", args) + self.handlers["KB_NEAREST"] = lambda args: self.execute("TOOL_KB_NEAREST", args) + # HYBRID_SEARCH: concurrent BM25 + nomic + qwen RRF retrieval, + # returns a structured dict suitable for programmatic consumption + # (unlike KB_SEARCH which returns a formatted string for LLM output). + self.handlers["HYBRID_SEARCH"] = self._hybrid_search # DISPLACEMENT_RECORD: record a task→response displacement self.handlers["DISPLACEMENT_RECORD"] = lambda args: self.execute("DISPLACE", args) # OBSERVE: embed + store an observation @@ -1567,6 +1562,106 @@ def _init_dev_handlers(self, state, config): print(f"[kernel] Dev tools: {dev_count} handlers installed") + def _hybrid_search(self, args: dict) -> dict: + """HYBRID_SEARCH instruction — concurrent BM25 + nomic + qwen RRF. + + Returns a structured dict (not a formatted string) for programmatic + callers such as other kernel instructions and agent tools that want + scores, not prose. KBSearchTool / KB_SEARCH remain the LLM-facing + path; this is the machine-facing entry point. + + Args (all optional except query): + query str — required; the search string. + k int — max results (default 10). + partition str — ignored for now; KB is single-partition. + mmr_diversity float — 0..1 diversity for MMR re-rank (default 0.3). + min_threshold float — drop results with fused score below this. + + Returns: + { + "results": [{"id", "score", "title", "snippet"}, ...], + "weight_label": str, # e.g. "conceptual" + "retrievers_used": list, # e.g. ["bm25", "nomic", "qwen"] + "query": str, + "total_found": int, + } + """ + query = (args.get("query") or "").strip() + if not query: + return {"error": "query is required", "results": []} + + k = int(args.get("k", 10)) + mmr_diversity = float(args.get("mmr_diversity", 0.3)) + min_threshold = args.get("min_threshold") + if min_threshold is not None: + min_threshold = float(min_threshold) + + # Note: the KB singleton lives in ``knowledge.py``, not in a + # ``kb_store`` module — there is no such module. An older + # version of this handler had ``from kb_store import get_kb`` + # which silently failed at import-time and produced a "KB + # unavailable" error on every call. Fixed to the correct path. + try: + from knowledge import get_kb + kb = get_kb() + except Exception as exc: + return {"ok": False, + "error": f"KB unavailable: {exc}", "results": []} + + try: + from rrf_search import hybrid_rrf_search, is_enabled as _rrf_on, _select_weights + except ImportError as exc: + return {"ok": False, + "error": f"rrf_search not importable: {exc}", + "results": []} + + if not _rrf_on(): + return {"error": "LEOS_RRF_SEARCH=0; hybrid search disabled", "results": []} + + # Reuse _search_rrf to build the callables and MMR vectors — this + # keeps the retriever construction logic in one place. + fused_entries = kb._search_rrf( + query, + project_id=None, + limit=k, + hybrid_fn=hybrid_rrf_search, + ) + + # Determine weight_label from the query directly (cheap, no extra call). + weight_label, _ = _select_weights(query) + + # Determine which retrievers were actually active. + retrievers_used = ["bm25", "nomic"] + try: + import embeddings as _emb + if _emb.is_qwen_enabled(): + retrievers_used.append("qwen") + except Exception: + pass + + results = [] + for entry in fused_entries: + score = entry.get("_rrf_score", 0.0) + if min_threshold is not None and score < min_threshold: + continue + content = entry.get("content") or "" + summary = entry.get("summary") or "" + snippet = (summary or content)[:300] + results.append({ + "id": entry.get("id", ""), + "score": round(score, 6), + "title": entry.get("title") or entry.get("subject") or "", + "snippet": snippet, + }) + + return { + "results": results, + "weight_label": weight_label, + "retrievers_used": retrievers_used, + "query": query, + "total_found": len(results), + } + def _init_kb_seed_articles(self, state, config): """Seed KB articles for pre-loaded APIs.""" # ── Seed KB articles for pre-loaded APIs ── @@ -1743,9 +1838,8 @@ def execute(self, instruction, args=None): dict with the result. Always includes 'ok' (bool) and 'elapsed_ms' (float). On error, includes 'error' (string). - Phase 5 change: no longer raises on error. All errors are - returned as {ok: False, error: "..."} so callers don't need - try/except. This matters for bone chains and the REST API. + All errors are returned as {ok: False, error: '...'} rather than + raised, so callers do not need try/except. """ args = args or {} start = time.perf_counter() @@ -1787,7 +1881,7 @@ def execute(self, instruction, args=None): ) except Exception: logger.warning("Suppressed error in %s", "execute", exc_info=True) - # Plan 20: emit kernel.error signal + # emit kernel.error signal if self.signal_bus is not None: try: self.signal_bus.async_emit( @@ -1807,17 +1901,19 @@ def execute(self, instruction, args=None): elapsed = (time.perf_counter() - start) * 1000.0 - # Wrap result in a standard envelope + # Wrap result in a standard envelope. + # ``setdefault`` preserves ``{"ok": False, ...}`` from error paths — + # unconditional ``= True`` would silently mask handler failures. if isinstance(result, dict): - result["ok"] = True + result.setdefault("ok", True) result["elapsed_ms"] = round(elapsed, 2) else: result = {"ok": True, "result": result, "elapsed_ms": round(elapsed, 2)} - # Phase C: record telemetry and update activity timestamp + # record telemetry and update activity timestamp self._record_telemetry(instruction, elapsed, error=False) - # Plan 20: emit kernel.instruction signal (async to avoid latency) + # emit kernel.instruction signal async (avoids adding latency to the handler path) if self.signal_bus is not None: try: self.signal_bus.async_emit( diff --git a/project/kernel_adapter_cert.py b/project/kernel_adapter_cert.py index 52b943d..7bf4dbf 100644 --- a/project/kernel_adapter_cert.py +++ b/project/kernel_adapter_cert.py @@ -1,8 +1,6 @@ """ kernel_adapter_cert.py - API adapter + model certification + Phase C integration -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _AdapterCertIntegrationMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via diff --git a/project/kernel_advanced.py b/project/kernel_advanced.py index 16257c8..6cac1f9 100644 --- a/project/kernel_advanced.py +++ b/project/kernel_advanced.py @@ -1,8 +1,6 @@ """ kernel_advanced.py - Phase H: Advanced capabilities -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _AdvancedMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via @@ -24,7 +22,7 @@ logger = logging.getLogger("kernel") -# Additional imports needed by this mixin (Phase 12 post-split fix) +# numpy required by speculative-execution and media-ingest handlers import numpy as np @@ -393,6 +391,434 @@ def _media_ingest_url(self, args): except Exception as e: return {"ok": False, "error": str(e)} + def _resolve_speakers(self, args): + """RESOLVE_SPEAKERS — map acoustic SPEAKER_NN labels to real names. + + Layered resolution, strongest signal first: + + 1. **Transcript-intro mining** — the first ~60s of a podcast + almost always contains self-introductions ("I'm Alie") + and host-introducing-guest cues ("joined by", "my guest"). + Each match is attributed to the speaker label that uttered + it via diarization, giving us name→speaker bindings with + the strongest possible evidence (the speaker said their + own name out loud). + 2. **iTunes show lookup** — `artistName` from the iTunes + Search API is split on `" and "`, `"&"`, `","`, `"/"` so + co-hosted shows yield a list of host candidates instead of + a single combined string. + 3. **Episode title regex** — `with X`, `feat. X`, + `featuring X`, `| X` patterns extract the guest. + 4. **Acoustic anchoring fallback** — for any speaker label + still nameless, fall back to talk-time ranking: top talker + gets the next unclaimed host name, runner-up gets the + guest name, etc. + + Roles: `host` for any speaker bound to an iTunes-listed name + or top talker; `guest` for the title/intro-cued guest; + `additional` for residual speakers without a name. + + Writes ``analysis.speakers_resolved``: + + [ + {"id": "SPEAKER_02", "role": "host", "name": "Shane Mauss", + "talk_time_s": 2468.8, "confidence": 0.9, "evidence": "intro"}, + {"id": "SPEAKER_03", "role": "host", "name": "Ramin Nazer", + "talk_time_s": 1358.6, "confidence": 0.7, "evidence": "itunes"}, + ] + + Args: + record_id: media record to resolve + + Returns: + ok, record_id, speakers_resolved (the list above) + """ + import json as _json + import re as _re + import urllib.parse as _up + import urllib.request as _ur + record_id = args.get("record_id", "") + if not record_id: + return {"ok": False, "error": "record_id required"} + try: + import media_library + if not media_library.is_initialized(): + from state import get_base_dir + media_library.init(get_base_dir()) + rec = media_library.get_record(record_id) + if not rec: + return {"ok": False, "error": f"record {record_id} not found"} + + title = (rec.get("title") or "").strip() + analysis = rec.get("analysis") or {} + segments = analysis.get("transcript_segments") or [] + + # ---- show name (everything before em-dash / colon) ---- + show = _re.split(r"\s*[—:|]\s*|\s+-\s+", title, maxsplit=1)[0].strip() + + # ---- 1. iTunes lookup → list of host candidates ------ + # + # Co-hosted shows return artistName like "Shane Mauss and + # Ramin Nazer" — split it so we can match each name to a + # distinct acoustic speaker rather than concatenating both + # into one label. + host_candidates = [] + if show: + try: + url = ( + "https://itunes.apple.com/search?" + + _up.urlencode({ + "term": show, "entity": "podcast", "limit": 1, + }) + ) + with _ur.urlopen(url, timeout=8) as r: + data = _json.loads(r.read().decode("utf-8")) + if data.get("results"): + raw = (data["results"][0].get("artistName") or "").strip() + for part in _re.split(r"\s+and\s+|\s*&\s*|\s*,\s*|\s*/\s*", raw): + p = part.strip() + if p and p.lower() not in (h.lower() for h in host_candidates): + host_candidates.append(p) + except Exception: + pass # offline / API blip — fall through + + # ---- 2. Episode-title regex → guest candidate -------- + # + # Stop-words for the FIRST captured token: words that + # superficially look like names (capitalised) but are + # really role/episode descriptors. These reject "Solo", + # "Encore", "Special" being misread as a guest's first + # name. Distinct from the transcript stop-words below. + _TITLE_STOP = { + "solo", "encore", "special", "live", "rerun", "rebroadcast", + "bonus", "preview", "pilot", "trailer", "episode", "part", + "patreon", "minisode", + } + guest_name = None + for pat in ( + # Honorific-led: "with Dr. Foo Bar", "with Paleontologist Foo" + r"\bwith\s+(?:Paleontologist|Biologist|Doctor|Dr\.?|Professor|Prof\.?|Comedian|Author)\s+([A-Z][\w'.\-]+(?:\s+[A-Z][\w'.\-]+){0,3})", + # Plain "with X" + r"\bwith\s+([A-Z][\w'.\-]+(?:\s+[A-Z][\w'.\-]+){0,3})", + # Slash form: "w/Emily J Willingham", "w/ Pete Holmes" + r"\bw/\s*([A-Z][\w'.\-]+(?:\s+[A-Z][\w'.\-]+){0,3})", + # feat. / featuring + r"\b(?:feat\.?|featuring|ft\.?)\s+([A-Z][\w'.\-]+(?:\s+[A-Z][\w'.\-]+){0,3})", + # Pipe-separator at end: "Comedy Sex God | Pete Holmes" + r"\|\s+([A-Z][\w'.\-]+(?:\s+[A-Z][\w'.\-]+){0,3})\s*$", + ): + m = _re.search(pat, title) + if m: + candidate = m.group(1).strip() + # Strip trailing role/episode suffixes + candidate = _re.sub( + r"\s+(?:Solo|Encore|Special|Live|Bonus|Preview|Patreon)?\s*" + r"(?:Solo Episode|Encore Presentation|Special|Episode|Part \d+|Pt\.? \d+|Pt \d+)$", + "", candidate, + ).strip() + if not candidate: + continue + # Reject if first captured token is a title-stop word + first = candidate.split()[0].lower() + if first in _TITLE_STOP: + continue + if candidate.lower() in (h.lower() for h in host_candidates): + continue + guest_name = candidate + break + + # ---- 3. Transcript-intro mining ---------------------- + # + # Strongest signal: the speaker said their own name on + # the air. Scan the first 90s of segments for self-intro + # ("I'm Alie", "my name is Alie") and host-introducing- + # guest ("joined by Mike", "my guest Mike", "today we + # have Mike"). Each match is attributed to the speaker + # label that uttered it. + # + # Captured names override talk-time guesses below. + speaker_to_name = {} # SPEAKER_NN -> resolved name + speaker_evidence = {} # SPEAKER_NN -> "intro" | "itunes" | "talk_time" + speaker_role = {} # SPEAKER_NN -> "host" | "guest" + + self_intro_pats = [ + _re.compile(r"\bI'?m\s+([A-Z][a-z][\w'.\-]+(?:\s+[A-Z][\w'.\-]+){0,2})\b"), + _re.compile(r"\bmy\s+name(?:'s|\s+is)\s+([A-Z][\w'.\-]+(?:\s+[A-Z][\w'.\-]+){0,2})\b"), + _re.compile(r"\bthis\s+is\s+([A-Z][\w'.\-]+(?:\s+[A-Z][\w'.\-]+){0,2})\b"), + ] + guest_intro_pats = [ + _re.compile(r"\b(?:joined\s+by|my\s+guest|today\s+(?:we\s+have|I'?m\s+(?:talking\s+(?:to|with)|joined\s+by))|welcome(?:ing)?)\s+([A-Z][\w'.\-]+(?:\s+[A-Z][\w'.\-]+){0,3})\b"), + _re.compile(r"\bI'?m\s+(?:talking|chatting|here)\s+with\s+([A-Z][\w'.\-]+(?:\s+[A-Z][\w'.\-]+){0,3})\b"), + _re.compile(r"\bplease\s+welcome\s+([A-Z][\w'.\-]+(?:\s+[A-Z][\w'.\-]+){0,3})\b"), + ] + # Common false-positive words after "I'm" / "this is" — drop + # them rather than claim them as someone's name. Includes + # speech-marker words ("Solo"), state words ("alone", "tired"), + # and common throwaways the model might capitalise. + _STOP = { + "the", "a", "an", "going", "gonna", "so", "just", "really", + "very", "your", "my", "our", "his", "her", "their", "that", + "this", "today", "here", "back", "still", "actually", + "solo", "alone", "tired", "excited", "happy", "sorry", + "live", "ready", "back", "up", "down", "out", "in", "on", + "good", "great", "fine", "okay", "ok", "yeah", "yes", "no", + "thinking", "talking", "saying", "trying", "telling", + "sure", "right", "wrong", "old", "young", + "joined", "introducing", "welcoming", "presenting", + } + + def _is_namey(s): + if not s: + return False + # Reject single tokens that are common english words. + first = s.split()[0].lower() + if first in _STOP: + return False + # Two-token capitalised names beat single tokens. + return True + + # First-90s window: enough for nearly every podcast intro, + # short enough to avoid mid-show name drops on guests. + for s in segments: + if float(s.get("start") or 0.0) > 90.0: + break + spk = s.get("speaker") + text = s.get("text") or "" + if not spk or not text: + continue + # Self-intro → bind name to THIS speaker + for pat in self_intro_pats: + m = pat.search(text) + if m and _is_namey(m.group(1)) and spk not in speaker_to_name: + speaker_to_name[spk] = m.group(1).strip() + speaker_evidence[spk] = "intro" + speaker_role[spk] = "host" # self-intro almost always = host + break + # Guest intro: speaker who SAID it is a host; the + # name they uttered should be bound to a DIFFERENT + # speaker — we'll resolve that below by talk-time. + for pat in guest_intro_pats: + m = pat.search(text) + if m and _is_namey(m.group(1)) and not guest_name: + guest_name = m.group(1).strip() + # The introducer is a host + if spk not in speaker_role: + speaker_role[spk] = "host" + break + + # ---- 4. Acoustic anchoring: talk-time per speaker ---- + talk_time = {} + for s in segments: + spk = s.get("speaker") + if not spk: + continue + dur = float(s.get("end") or 0.0) - float(s.get("start") or 0.0) + if dur > 0: + talk_time[spk] = talk_time.get(spk, 0.0) + dur + ranked = sorted(talk_time.items(), key=lambda kv: -kv[1]) + + # ---- 5. Bind remaining unclaimed names --------------- + # + # Names that intro-mining didn't already nail down: + # - host_candidates from iTunes (could be 1 or N) + # - guest_name from title regex / guest-intro pat + # Speakers without names: top-by-talk-time first, in + # rank order, get the unclaimed host names; next gets + # guest; then "additional" with no name. + claimed_names = {n.lower() for n in speaker_to_name.values()} + unclaimed_hosts = [ + h for h in host_candidates if h.lower() not in claimed_names + ] + unclaimed_guest = ( + guest_name + if guest_name and guest_name.lower() not in claimed_names + else None + ) + + for spk, dur in ranked: + if spk in speaker_to_name: + continue # already bound by intro + if unclaimed_hosts: + name = unclaimed_hosts.pop(0) + speaker_to_name[spk] = name + speaker_evidence[spk] = "itunes" + speaker_role.setdefault(spk, "host") + elif unclaimed_guest: + speaker_to_name[spk] = unclaimed_guest + speaker_evidence[spk] = "title_regex" + speaker_role.setdefault(spk, "guest") + unclaimed_guest = None + else: + # No name candidates left for this speaker — + # role decided by ranking position. + speaker_role.setdefault( + spk, + "guest" if not any(r == "guest" for r in speaker_role.values()) else "additional", + ) + + # ---- 6. Build resolved list -------------------------- + resolved = [] + total_talk = sum(t for _, t in ranked) or 1.0 + for spk, dur in ranked: + name = speaker_to_name.get(spk) + role = speaker_role.get(spk) or ("additional" if name is None else "host") + evidence = speaker_evidence.get(spk, "talk_time" if name else "none") + share = dur / total_talk + # Confidence: intro evidence is gold (0.9), iTunes is + # solid (0.7), title-regex is decent (0.6), pure + # talk-time is weak (0.4 × share). No name → 0. + conf = { + "intro": 0.9, + "itunes": 0.7, + "title_regex": 0.6, + "talk_time": round(min(1.0, share * 1.5), 2), + "none": 0.0, + }.get(evidence, 0.0) + if not name: + conf = 0.0 + resolved.append({ + "id": spk, + "role": role, + "name": name, + "talk_time_s": round(dur, 1), + "confidence": conf, + "evidence": evidence, + }) + + updates = {"analysis": {**analysis, "speakers_resolved": resolved}} + media_library.update_record(record_id, updates) + + return { + "ok": True, + "record_id": record_id, + "show": show, + "host_candidates": host_candidates, + "guest_name": guest_name, + "speakers_resolved": resolved, + } + except Exception as e: + return {"ok": False, "error": str(e)} + + def _retranscribe_record(self, args): + """RETRANSCRIBE_RECORD — re-run Whisper on the local audio of + an existing record and overwrite ``analysis.transcript`` and + ``analysis.transcript_segments``. Use after switching backends + (e.g. faster-whisper → whisperx) so existing records pick up + the new capability (speaker labels) without re-ingesting. + + Args: + record_id: media record to re-transcribe + wipe_review: if true, also clear ``analysis.review`` so the + next ``MEDIA_REVIEW`` writes a fresh structured block. + Default false. + + Returns: + ok, record_id, transcript_chars, segments_count, speakers_detected, language + """ + import os + record_id = args.get("record_id", "") + if not record_id: + return {"ok": False, "error": "record_id required"} + wipe_review = bool(args.get("wipe_review", False)) + try: + import media_library + if not media_library.is_initialized(): + from state import get_base_dir + media_library.init(get_base_dir()) + rec = media_library.get_record(record_id) + if not rec: + return {"ok": False, "error": f"record {record_id} not found"} + files_dir = os.path.join( + media_library._BASE_DIR, "media_library", "files", + ) + audio_path = None + for ext in (".mp3", ".m4a", ".wav", ".ogg", ".flac"): + p = os.path.join(files_dir, f"{record_id}{ext}") + if os.path.exists(p): + audio_path = p + break + if audio_path is None: + return {"ok": False, "error": f"no audio file on disk for {record_id}"} + + from media_ingest_models import _get_whisper_model + model = _get_whisper_model() + if not model: + return {"ok": False, "error": "no Whisper backend available"} + + result = model.transcribe(audio_path) or {} + text = (result.get("text") or "").strip() + segments = result.get("segments") or [] + language = result.get("language") + + updates = {"analysis": {**(rec.get("analysis") or {})}} + updates["analysis"]["transcript"] = text + updates["analysis"]["transcript_segments"] = segments + if language: + updates["analysis"]["transcript_language"] = language + if wipe_review: + updates["analysis"].pop("review", None) + updates["analysis"].pop("review_v2", None) + media_library.update_record(record_id, updates) + + n_speakers = len({ + s.get("speaker") for s in segments if s.get("speaker") + }) + + # Chain: resolve SPEAKER_NN labels to real names via iTunes + # lookup + episode-title regex + talk-time anchoring. Lets + # downstream review prompts use "Alie Ward (host)" instead + # of "SPEAKER_00". + speakers_resolved = [] + if n_speakers > 0: + try: + rs = self._resolve_speakers({"record_id": record_id}) + if rs.get("ok"): + speakers_resolved = rs.get("speakers_resolved") or [] + except Exception as e: + logger.debug("resolve_speakers chain failed: %s", e) + + return { + "ok": True, + "record_id": record_id, + "audio_path": audio_path, + "transcript_chars": len(text), + "segments_count": len(segments), + "speakers_detected": n_speakers, + "speakers_resolved": speakers_resolved, + "language": language, + "wiped_review": wipe_review, + } + except Exception as e: + return {"ok": False, "error": str(e)} + + def _media_review(self, args): + """MEDIA_REVIEW — run the staged review pipeline synchronously. + + Blocks for the duration of the pipeline (typically 50-300s + depending on transcript length and Ollama warmth). Writes + the structured result back to ``record["analysis"]["review"]`` + and returns it. This is the single canonical media reviewer. + + Args: + record_id: media record to review + + Returns: + ok, record_id, review (the structured dict) + """ + record_id = args.get("record_id", "") + if not record_id: + return {"ok": False, "error": "record_id required"} + try: + import media_library + from media_ingest_review import run_review + if not media_library.is_initialized(): + from state import get_base_dir + media_library.init(get_base_dir()) + result = run_review(record_id) + return {"ok": True, "record_id": record_id, "review": result} + except Exception as e: + return {"ok": False, "error": str(e)} + def _media_edit(self, args): """MEDIA_EDIT — Semantic edit via vector arithmetic. @@ -690,7 +1116,7 @@ def _bot_create(self, args): ) result["ok"] = True - # Plan 20: emit bot.created signal + # emit bot.created signal if self.signal_bus is not None: try: self.signal_bus.emit( @@ -751,7 +1177,7 @@ def _bot_stop(self, args): bf.stop_bot(bot_id) - # Plan 20: emit bot.stopped signal + # emit bot.stopped signal if self.signal_bus is not None: try: self.signal_bus.emit( @@ -847,13 +1273,12 @@ def _bot_list(self, args): def _bot_run_cycle(self, args): """BOT_RUN_CYCLE — Execute one perceive→evaluate→act cycle. - Phase B2: Runs the cycle as a bone chain through app_executor, - which means each stage gets displacement recording, results - feed the nutrient field, performance is tracked in perf_store, - and the reflex arc can learn to shortcut known patterns. + Runs the cycle as a bone chain through app_executor, which means + each stage gets displacement recording, results feed the nutrient + field, performance is tracked in perf_store, and the reflex arc + can learn to shortcut known patterns. - Falls back to direct bot_runner.run_cycle() if app_executor - is not available (backward compatibility). + Falls back to bot_runner.run_cycle() if app_executor is unavailable. Args: bot_id: the bot instance to cycle @@ -892,7 +1317,7 @@ def _bot_run_cycle(self, args): input_data["bot_name"] = bot.get("name", "") input_data["service_id"] = bot.get("service_id", "") - # ---- Phase B7: Reflex shortcut for bot cycles ---- + # ---- Reflex shortcut for bot cycles (System 1 fast path) ---- # Before running the full perceive→evaluate→act chain, check # the reflex arc. If we have a cached prediction for this bot's # pattern (high confidence, System 1 fast path), we can skip @@ -954,7 +1379,7 @@ def _bot_run_cycle(self, args): "steps": chain_result.get("steps", []), } else: - # ---- Fallback: direct bot_runner call (pre-B2 compat) ---- + # ---- Fallback: direct bot_runner.run_cycle() ---- import bot_runner bot_runner.init(self.state.base_dir) result = bot_runner.run_cycle(bot_id) @@ -975,13 +1400,9 @@ def _bot_run_cycle(self, args): except Exception: pass # Don't let bookkeeping errors break the bot cycle - # ---- Phase B5: Feed nutrient field on successful bone chain ---- - # NOW HANDLED VIA SIGNAL BUS (Plan 20). - # bot_runner.run_cycle() emits "bot.cycle_complete" which - # constraint_hooks receives and calls record_bot_observation(). - # This works for both bone_chain and direct paths. + # Nutrient field update handled via "bot.cycle_complete" signal (both bone-chain and direct paths). - # ---- Update bot stats (backward compat) ---- + # ---- Update bot stats ---- try: bf.update_bot_stats(bot_id, result) except Exception: diff --git a/project/kernel_apps.py b/project/kernel_apps.py index 05fa722..b9e376b 100644 --- a/project/kernel_apps.py +++ b/project/kernel_apps.py @@ -1,8 +1,6 @@ """ kernel_apps.py - Phase 3: App framework instructions -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _AppsMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via diff --git a/project/kernel_blackboard.py b/project/kernel_blackboard.py index e7a59fe..353ca46 100644 --- a/project/kernel_blackboard.py +++ b/project/kernel_blackboard.py @@ -1,8 +1,6 @@ """ kernel_blackboard.py - Status companion ephemeral blackboard (Plan 10) -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _BlackboardMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via diff --git a/project/kernel_core_instructions.py b/project/kernel_core_instructions.py index 2751d71..dda2a0c 100644 --- a/project/kernel_core_instructions.py +++ b/project/kernel_core_instructions.py @@ -1,8 +1,6 @@ """ kernel_core_instructions.py - Core instruction handlers (Phase 1 geometry, state mutation) -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _CoreInstructionsMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via @@ -15,8 +13,7 @@ even though many of the names it references are not defined here. Core instruction handlers covering Phase 1 spherical geometry -instructions and state-mutation instructions. These are the original -handlers from before subsystem mixins existed — the foundational math +instructions and state-mutation instructions. The foundational math and state ops that everything else builds on. """ @@ -24,7 +21,7 @@ logger = logging.getLogger("kernel") -# Additional imports needed by this mixin (Phase 12 post-split fix) +# Spherical geometry primitives used directly by this mixin's handlers import numpy as np from spherical_geometry import batch_cosine_similarity, cosine_similarity, exp_map, geodesic_distance, log_map, mean_on_sphere, mrl_truncate, normalize, parallel_transport, slerp @@ -775,5 +772,30 @@ def _status(self, args): except ImportError: result["render_learner"] = {"running": False, "reason": "not installed"} + # Accelerator (GPU/CUDA/ROCm/MPS/CPU) — driver-level facts plus + # the runtime confirmation from torch. The pre-install + # detector runs from OS signals; the runtime check confirms + # torch was actually built against the matching backend. + try: + from infra.gpu_detect import detect as _gpu_detect + gpu = dict(_gpu_detect()) + try: + import torch as _torch + gpu["torch_version"] = _torch.__version__ + gpu["torch_cuda_built"] = getattr(_torch.version, "cuda", None) + gpu["torch_cuda_available"] = bool(_torch.cuda.is_available()) + gpu["torch_mps_available"] = bool( + getattr(_torch.backends, "mps", None) + and _torch.backends.mps.is_available() + ) + if gpu["torch_cuda_available"]: + gpu["torch_device_name"] = _torch.cuda.get_device_name(0) + except Exception: + gpu["torch_cuda_available"] = False + gpu["torch_mps_available"] = False + result["gpu"] = gpu + except Exception as _gpu_err: + result["gpu"] = {"available": False, "error": str(_gpu_err)} + return result diff --git a/project/kernel_crawl_drift.py b/project/kernel_crawl_drift.py index a3423df..dc6c605 100644 --- a/project/kernel_crawl_drift.py +++ b/project/kernel_crawl_drift.py @@ -1,8 +1,6 @@ """ kernel_crawl_drift.py - Site crawl + drift detection + baseline calibration + source reader + code change -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _CrawlDriftBaselineSourceMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via @@ -357,10 +355,6 @@ def _drift_relevance(self, args): Batch-scores candidate embeddings against a reference embedding. Returns three-tier classification (high/medium/low). - Replaces the old context_compactor's semantic scoring and the - old briefing's semantic filtering — one instruction for all - relevance checks. - Args: reference: text to embed as the reference (or reference_vec) candidates: list of texts to score (or candidate_vecs) diff --git a/project/kernel_display_coproc.py b/project/kernel_display_coproc.py index 14e7603..08cda76 100644 --- a/project/kernel_display_coproc.py +++ b/project/kernel_display_coproc.py @@ -1,8 +1,6 @@ """ kernel_display_coproc.py - Phase 4: Display + Phase 5: Coprocessor instructions -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _DisplayCoprocessorMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via @@ -133,12 +131,21 @@ def _escalate(self, args): if user_id: self.user_manager.record_activity(user_id, "escalation") + # Auto-RAG: same as ASSIST — prepend KB snippets so the + # primary agent grounds answers in the corpus. Tool-using + # paths still work; this only changes the prompt body. + # Disabled for reflection-style calls via context['no_tools']. + prompt, context = self._rag_augment(prompt, context or {}) + # Fall through to coprocessor result = self.coprocessor.escalate( task_vec=task_vec, context=context, coprocessor=coprocessor, prompt=prompt, messages=messages, status_update=status_update, ) + if isinstance(result, dict) and context.get("_rag_applied"): + result["rag_applied"] = True + result["rag_hits"] = context.get("_rag_hits", 0) # Phase 12: auto-compile successful escalations for one-shot learning if (result.get("ok") and task_vec is not None @@ -164,6 +171,120 @@ def _coprocessor_status(self, args): """COPROCESSOR_STATUS: check which backends are available.""" return self.coprocessor.status() + def _coprocessor_probe(self, args): + """COPROCESSOR_PROBE: re-probe backends and return refreshed status. + + Use after restoring a downed Ollama/ComfyUI daemon — the cached + ``_available`` flags are not refreshed by ESCALATE/ASSIST gates, + only by ``coprocessor_bay.probe()``. Without this instruction + the only way to recover from "Ollama died and came back" is a + full leOS restart. + """ + return self.coprocessor.probe() + + # ---------------------------------------------------------------- + # Auto-RAG: inject KB context before the LLM ever sees the prompt. + # ---------------------------------------------------------------- + # + # Without this, ASSIST/ESCALATE ground answers in model weights only + # and hallucinate corpus content. Pre-retrieving top-K snippets and + # prepending them as "Reference material" eliminates that blind spot. + # + # Retrieval path: HYBRID_SEARCH (BM25 + nomic + qwen, RRF-fused) + # rather than raw SEARCH (nomic-only cosine ANN). nomic encodes + # task directives ("summarize", "know about") into the query vector, + # drifting it toward generic system docs at ~0.65-0.70 cosine + # baseline. BM25 lexical matching for domain terms ("menopause", + # "evolution") surfaces the right article while system docs (zero + # lexical overlap) fall out of the fused top-K. min_threshold=0.50 + # drops generic system-doc hits that BM25 cannot discriminate. + # + # Default-on for prompty task types (explain, describe, summarize, + # answer, text, q&a). Opt out per-call via ``context['rag'] = False`` + # (format/translate tasks) or ``context['no_tools'] = True`` + # (disables agent tools — also disables RAG for parity). + + _RAG_TASK_TYPES = { + "", "text", "explain", "describe", "summarize", "answer", + "qa", "q&a", "research", "elaborate", "compare_text", + } + + def _rag_augment(self, prompt, context): + """Return (augmented_prompt, augmented_context) with KB + snippets prepended. Pure pass-through if RAG is off, the + prompt is empty, or retrieval fails — never breaks the call.""" + if context is None: + return prompt, context + if not prompt or not isinstance(prompt, str): + return prompt, context + # Explicit opt-out + if context.get("rag") is False: + return prompt, context + if context.get("no_tools") and context.get("rag") is not True: + # Reflection-style calls: same content already supplied + # by caller; double-retrieving is noise. + return prompt, context + # Default-on for prompty task types + task_type = (context.get("type") or "").lower() + if context.get("rag") is not True and task_type not in self._RAG_TASK_TYPES: + return prompt, context + + try: + top_k = int(context.get("rag_top_k") or 3) + top_k = max(1, min(top_k, 8)) + # top_k + 2 gives RRF room to MMR-diversify before cutting to + # top_k. min_threshold=0.50 drops generic system-doc hits. + r = self.execute("HYBRID_SEARCH", { + "query": prompt[:512], + "k": top_k + 2, + "min_threshold": 0.50, + }) + except Exception: + return prompt, context + results = (r or {}).get("results") or [] + if not results: + return prompt, context + + # Build a compact reference block — title + first ~400 chars + # of content per hit, capped at top-K. Total reference budget + # is ~2 KB; the LLM still has room for the actual question. + refs = [] + for i, hit in enumerate(results[:top_k], 1): + # HYBRID_SEARCH returns top-level 'title'/'snippet'; raw SEARCH + # nests title inside 'metadata'. Handle both shapes. + title = ( + hit.get("title") + or (hit.get("metadata") or {}).get("title") + or hit.get("id") + or "?" + ).strip() + content = (hit.get("snippet") or hit.get("content") or hit.get("text") + or hit.get("summary") or "").strip() + sim = hit.get("score") or hit.get("similarity") or 0.0 + snippet = content[:500].replace("\n", " ") + refs.append(f"[{i}] {title} (sim={sim:.2f})\n {snippet}") + ref_block = ( + "Reference material from your knowledge base — use these " + "to ground your answer; cite the bracketed numbers:\n\n" + + "\n\n".join(refs) + ) + new_prompt = f"{ref_block}\n\n---\n\n{prompt}" + + # Strengthen system prompt to actually use the refs + new_ctx = dict(context) + existing_sys = new_ctx.get("system_prompt") or "" + rag_addendum = ( + "\n\nYou have been given reference material from the user's " + "knowledge base. Ground your answer in those references; " + "cite the bracketed numbers ([1], [2]) when you use them. " + "If the references don't cover what was asked, say so plainly " + "instead of inventing facts." + ) + new_ctx["system_prompt"] = (existing_sys + rag_addendum).strip() + new_ctx["_rag_applied"] = True + new_ctx["_rag_hits"] = len(results[:top_k]) + return new_prompt, new_ctx + def _assist(self, args): """ASSIST: delegate a lightweight task to the CPU-only utility model. @@ -225,11 +346,18 @@ def _assist(self, args): if user_id: self.user_manager.record_activity(user_id, "assist") + # Auto-RAG: pre-retrieve KB context for prompty tasks so the + # intern grounds answers in the corpus instead of hallucinating. + prompt, context = self._rag_augment(prompt, context or {}) + # Route to the utility model (CPU-only) result = self.coprocessor.assist( prompt=prompt, messages=messages, context=context, task_vec=task_vec, ) + if context.get("_rag_applied"): + result["rag_applied"] = True + result["rag_hits"] = context.get("_rag_hits", 0) return result diff --git a/project/kernel_dreaming_extensions.py b/project/kernel_dreaming_extensions.py index 744f312..a57a97e 100644 --- a/project/kernel_dreaming_extensions.py +++ b/project/kernel_dreaming_extensions.py @@ -1,8 +1,6 @@ """ kernel_dreaming_extensions.py - Phase 8-13 extensions (dreaming, purple, gravity, holography, self-extend, multi-user) -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _DreamingExtensionsMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via @@ -30,7 +28,7 @@ logger = logging.getLogger("kernel") -# Additional imports needed by this mixin (Phase 12 post-split fix) +# numpy used by speculative-execution helpers (_subspace_build, _topology) import numpy as np @@ -50,7 +48,7 @@ def _dream_consolidate(self, args): """ seconds = float(args.get("idle_seconds", 30)) - # Plan 20: emit dream.consolidate_start signal + # emit dream.consolidate_start signal if self.signal_bus is not None: try: self.signal_bus.emit( @@ -62,7 +60,7 @@ def _dream_consolidate(self, args): result = self.dreaming.on_idle(seconds) - # Plan 20: emit dream.consolidate_end signal + # emit dream.consolidate_end signal if self.signal_bus is not None: try: self.signal_bus.emit( @@ -83,7 +81,7 @@ def _dream_cycle(self, args): Also clears expired ephemeral blackboard notes (Plan 10). """ - # Plan 20: emit dream.cycle_start signal + # emit dream.cycle_start signal if self.signal_bus is not None: try: self.signal_bus.emit("dream.cycle_start") @@ -100,7 +98,7 @@ def _dream_cycle(self, args): result = self.dreaming.on_sleep() - # Plan 20: emit dream.cycle_end signal + # emit dream.cycle_end signal if self.signal_bus is not None: try: self.signal_bus.emit("dream.cycle_end") diff --git a/project/kernel_launcher_fs.py b/project/kernel_launcher_fs.py index 78bd342..1f73f5e 100644 --- a/project/kernel_launcher_fs.py +++ b/project/kernel_launcher_fs.py @@ -1,8 +1,6 @@ """ kernel_launcher_fs.py - Launcher + App repair + Filesystem instructions -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _LauncherRepairFsMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via diff --git a/project/kernel_network_web.py b/project/kernel_network_web.py index 169ca72..8a4ae6d 100644 --- a/project/kernel_network_web.py +++ b/project/kernel_network_web.py @@ -1,8 +1,6 @@ """ kernel_network_web.py - Network + web directory + site knowledge + Playwright/Scrapy overhaul -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _NetworkWebMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via @@ -26,8 +24,6 @@ logger = logging.getLogger("kernel") -# Additional imports needed by this mixin (Phase 12 post-split fix) - class _NetworkWebMixin: """Network + web directory + site knowledge + Playwright/Scrapy overhaul. Intended to be mixed into Kernel. @@ -139,7 +135,7 @@ def _net_search(self, args): max_results = args.get("max_results", 10) - # ── Plan 16: Site Knowledge first-pass check ── + # ── Site Knowledge first-pass check ── # Four-step flow: # 1. Search site knowledge for the query # 2. Fresh matches (sim > 0.6, not stale) → return directly diff --git a/project/kernel_reflex.py b/project/kernel_reflex.py index d876849..a977a48 100644 --- a/project/kernel_reflex.py +++ b/project/kernel_reflex.py @@ -1,8 +1,6 @@ """ kernel_reflex.py - Phase 7: Reflex arc production instructions -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _ReflexMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via diff --git a/project/kernel_research_watchdog.py b/project/kernel_research_watchdog.py index f31f033..37ad331 100644 --- a/project/kernel_research_watchdog.py +++ b/project/kernel_research_watchdog.py @@ -1,8 +1,6 @@ """ kernel_research_watchdog.py - Phase 14 research + watchdog self-healing -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _ResearchWatchdogMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via diff --git a/project/kernel_scoped_work.py b/project/kernel_scoped_work.py index e83730c..c6454c9 100644 --- a/project/kernel_scoped_work.py +++ b/project/kernel_scoped_work.py @@ -1,8 +1,6 @@ """ kernel_scoped_work.py - Plan 14 scoped work + Plan 14b thought externalization + Plan 9 I/O membrane -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _ScopedWorkThoughtMembraneMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via @@ -91,7 +89,7 @@ def _scope_create(self, args): scope_copy = dict(scope) scope_copy.pop("vector", None) - # Plan 20: emit scope.created signal + # emit scope.created signal if self.signal_bus is not None: try: self.signal_bus.emit( @@ -134,7 +132,7 @@ def _scope_update(self, args): if "status" in updates: self._evaluate_referencing_wires(scope_id) - # Plan 20: emit scope.status_changed signal + # emit scope.status_changed signal if self.signal_bus is not None: try: self.signal_bus.emit( @@ -492,7 +490,7 @@ def _scope_archive(self, args): if result is None: return {"ok": False, "error": f"Scope '{scope_id}' not found"} - # Plan 20: emit scope.archived signal + # emit scope.archived signal if self.signal_bus is not None: try: self.signal_bus.emit("scope.archived", scope_id=scope_id) @@ -533,7 +531,7 @@ def _scope_complete(self, args): # dependency or trigger wires on other scopes should fire. self._evaluate_referencing_wires(scope_id) - # Plan 20: emit scope.completed signal + # emit scope.completed signal if self.signal_bus is not None: try: self.signal_bus.emit( diff --git a/project/kernel_signal_bus.py b/project/kernel_signal_bus.py index 3e10724..ec0cb2f 100644 --- a/project/kernel_signal_bus.py +++ b/project/kernel_signal_bus.py @@ -1,8 +1,6 @@ """ kernel_signal_bus.py - Signal bus subscriber handlers (Plan 20, Phase 4) -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _SignalBusMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via diff --git a/project/kernel_state.py b/project/kernel_state.py index 49ea089..117ae8a 100644 --- a/project/kernel_state.py +++ b/project/kernel_state.py @@ -12,9 +12,8 @@ displacement (via log_map), the outcome, and metadata. Used by the REFLEX instruction to fire learned patterns. - - Confidence Map: Per-region density estimate derived from the displacement - log. Phase 1: simple neighbor counting. - Phase 7: conformal prediction with geodesic balls. + - Confidence Map: Density estimate using geodesic balls; backed by + conformal prediction for calibrated p-values. - Nutrient Field: (Stub) Success density map for gravitational lensing. Phase 9 builds this out. @@ -60,8 +59,8 @@ class SDFRegistry: sdf(point) = mahalanobis_distance(point, centroid, inv_cov) - threshold Negative = inside the region. Positive = outside. - Phase 1: Store regions as simple dicts. The covariance is identity - (spherical regions only). Phase 7 adds learned ellipsoidal covariance. + Currently uses spherical regions (identity covariance). Ellipsoidal + covariance is not yet implemented; inv_cov is always None. """ def __init__(self, data_dir): @@ -156,7 +155,7 @@ def _save(self): for name, region in self.regions.items(): serializable[name] = { "centroid": _vector_to_b64(region["centroid"]), - "inv_cov": None, # Phase 1: always None (spherical) + "inv_cov": None, # spherical region — ellipsoidal covariance not yet implemented "threshold": region["threshold"], "metadata": region["metadata"], "created_at": region["created_at"], @@ -205,16 +204,21 @@ class DisplacementLog: - timestamp: when this happened - metadata: any extra info (instruction type, model used, etc.) - Phase 1: Simple in-memory list with JSON persistence. - Phase 2: H.264-style compression (I/P/B frames, codebook). - Phase 7: STDP-style temporal weighting. + In-memory list with JSON persistence. H.264-style compression and + STDP temporal weighting are planned but not yet active. """ def __init__(self, data_dir, max_entries=10000): self.data_dir = data_dir self.max_entries = max_entries self.entries = [] - self._vectors_cache = None # numpy array of task_vecs for fast search + self._vectors_cache = None # (N, 768) float32 matrix of task_vecs + self._response_cache = None # (N, 768) float32 matrix of response_vecs + # Invariant: both caches are always None together or both valid. + # len(entries) == cache.shape[0] whenever cache is not None. + # Both are invalidated together on prune (single compound assignment). + # Not thread-safe across concurrent append() calls -- callers must + # serialise if multi-threaded ingestion is ever added. self._load() def append(self, task_vec, response_vec, outcome=1.0, metadata=None): @@ -242,16 +246,27 @@ def append(self, task_vec, response_vec, outcome=1.0, metadata=None): } self.entries.append(entry) - # Append to cache instead of invalidating -- avoids rebuilding - # the entire numpy matrix on every append. + # Append to both caches instead of invalidating -- avoids rebuilding + # the entire numpy matrices on every append. if self._vectors_cache is not None: - new_row = task_vec.reshape(1, -1) - self._vectors_cache = np.vstack([self._vectors_cache, new_row]) + try: + self._vectors_cache = np.vstack( + [self._vectors_cache, task_vec.reshape(1, -1)] + ) + self._response_cache = np.vstack( + [self._response_cache, response_vec.reshape(1, -1)] + ) + except Exception: + # Shape mismatch or allocation failure -- invalidate both + # caches atomically so entries and cache stay in sync. + self._vectors_cache = self._response_cache = None # Trim if over max (drop oldest) if len(self.entries) > self.max_entries: self.entries = self.entries[-self.max_entries:] - self._vectors_cache = None # must rebuild after prune + # Invalidate both caches together -- a single assignment ensures + # no caller can see one valid and one None. + self._vectors_cache = self._response_cache = None # Save periodically (every 50 entries) if len(self.entries) % 50 == 0: @@ -267,10 +282,11 @@ def find_similar(self, task_vec, k=15): task_vec = np.array(task_vec, dtype=np.float32) - # Build vector cache if needed + # Build vector cache if needed (dtype=float32 explicit to match + # stored entry dtype and avoid float64 inference on legacy loads). if self._vectors_cache is None: self._vectors_cache = np.array( - [e["task_vec"] for e in self.entries] + [e["task_vec"] for e in self.entries], dtype=np.float32 ) # Batch cosine similarity -- unit vectors, skip re-normalization @@ -300,13 +316,25 @@ def find_nearest_response(self, predicted_vec): """Find the stored response vector closest to a predicted vector. Used by the reflex engine to snap predictions to known responses. + Semantics: pure cosine similarity, no recency or displacement weighting. + + Uses _response_cache (N, 768) float32 to avoid rebuilding the matrix + on every call -- same pattern as find_similar / density_at use for + _vectors_cache. """ if len(self.entries) == 0: return None predicted_vec = np.array(predicted_vec, dtype=np.float32) - response_vecs = np.array([e["response_vec"] for e in self.entries]) - sims = batch_cosine_similarity(predicted_vec, response_vecs, + + # Build response cache if needed (dtype explicit to guard against + # float64 inference on legacy-loaded entries). + if self._response_cache is None: + self._response_cache = np.array( + [e["response_vec"] for e in self.entries], dtype=np.float32 + ) + + sims = batch_cosine_similarity(predicted_vec, self._response_cache, normalized=True) best_idx = int(np.argmax(sims)) @@ -339,7 +367,7 @@ def density_at(self, point, radius=0.3): point = np.array(point, dtype=np.float32) if self._vectors_cache is None: self._vectors_cache = np.array( - [e["task_vec"] for e in self.entries] + [e["task_vec"] for e in self.entries], dtype=np.float32 ) # Use cosine similarity (faster than geodesic for thresholding) @@ -405,13 +433,9 @@ def _load(self): class ConfidenceMap: """Per-region confidence derived from displacement log density. - Phase 1: Simple density counting -- how many displacement log entries - fall near a given point? More entries = more confidence = more likely - to fire a reflex instead of escalating to deliberation. - - Phase 7: Conformal prediction with geodesic balls. The confidence - score becomes a p-value that tells you "how unusual is this input - compared to what I've successfully handled before?" + Computes confidence as a neighbor-count density estimate within a + geodesic ball, with thresholds for void/low/medium/high. Backed by + conformal prediction when calibration data is available. """ def __init__(self, displacement_log, config=None): diff --git a/project/kernel_storage.py b/project/kernel_storage.py index 26de04e..c67ffb3 100644 --- a/project/kernel_storage.py +++ b/project/kernel_storage.py @@ -1,8 +1,6 @@ """ kernel_storage.py - Phase 2: Storage and compression instructions -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _StorageMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via @@ -24,7 +22,7 @@ logger = logging.getLogger("kernel") -# Additional imports needed by this mixin (Phase 12 post-split fix) +# exp_map and numpy are used directly by codec and compression handlers from spherical_geometry import exp_map import numpy as np @@ -170,6 +168,20 @@ def _search(self, args): k = int(args.get("k", args.get("top_k", 10))) skip_lens = args.get("skip_lens", False) + # Optional metadata filter — restrict results to entries whose + # ``metadata.source_agent`` matches. Used to suppress + # source-domain bleed (e.g. system docs ranking in podcast + # topical queries because their embeddings sit near the corpus + # centroid). Accepts a single string or a list; None means no + # filter (default — preserves the previous behaviour). + source_agent = args.get("source_agent") + if isinstance(source_agent, str): + source_agent = [source_agent] + filter_fn = None + if source_agent: + _sa = set(source_agent) + filter_fn = lambda e: (e.get("metadata") or {}).get("source_agent") in _sa + # Step 1: Gravitational lensing search_vec = vector lens_applied = False @@ -179,8 +191,10 @@ def _search(self, args): search_vec = np.array(lens_result["lensed"], dtype=np.float32) lens_applied = True - # Step 2: Holographic cache check - if partition in self.holo_cache.caches: + # Step 2: Holographic cache check. Skip the cache when a + # source filter is requested — the cache returns raw (idx, sim) + # pairs without metadata, so we can't honour filter_fn there. + if filter_fn is None and partition in self.holo_cache.caches: holo_result = self.holo_cache.reconstruct(partition, search_vec, k) if holo_result.get("cache_hit") and holo_result.get("results"): return { @@ -197,7 +211,7 @@ def _search(self, args): # Step 3: Full partition search results = self.state.partition_manager.nearest( - partition, search_vec, k + partition, search_vec, k, filter_fn=filter_fn, ) return { diff --git a/project/kernel_subsystems.py b/project/kernel_subsystems.py index 8699901..917851f 100644 --- a/project/kernel_subsystems.py +++ b/project/kernel_subsystems.py @@ -152,11 +152,6 @@ def _plan_create(kernel, args): None-value trap (where get(key, default) returns None instead of default when the value is explicitly None). - Plan 22 (retry escalation) added two optional args that flow - through to plan_manager.create_plan as kwargs. Both default to - "" — callers that don't know about plan 22 see no behaviour - change at all. - Args (all optional except `goal`): goal: what the plan aims to achieve (REQUIRED) steps: list of step dicts @@ -172,8 +167,7 @@ def _plan_create(kernel, args): steps = args.get("steps") scope_id = args.get("scope_id") tags = args.get("tags") - # Plan 22 fields — additive, default to empty strings so old - # callers who don't pass these get the same behaviour as before + # Retry-tracking fields — additive; callers that omit these see no change in behaviour tier_used = args.get("tier_used", "") or "" retry_of = args.get("retry_of", "") or "" try: @@ -755,26 +749,68 @@ def _get_kb(kernel): def _kb_graph_expand(kernel, args): - """KB_GRAPH_EXPAND: traverse the knowledge graph from a starting entry.""" + """KB_GRAPH_EXPAND: traverse the knowledge graph from a starting entry. + + Accepted args (all callers should use these names): + article_id / entry_id — KB article to expand from (article_id preferred) + max_hops / depth — BFS depth (default 1) + max_neighbors — max total nodes (default 20) + query — optional free-text for relevance filtering + + Response shape: + { + "neighbors": [{"id": str, "title": str, "summary": str, + "score": float, "via": str}, ...], + "count": int + } + """ from knowledge_graph import expand, GraphNode + kb = _get_kb(kernel) - entry_id = args.get("entry_id", "") - depth = args.get("depth", 1) + + # Accept both "article_id" (canonical) and "entry_id" (legacy alias). + entry_id = args.get("article_id") or args.get("entry_id", "") + + # Accept both "max_hops" (canonical) and "depth" (legacy alias). + depth = int(args.get("max_hops") or args.get("depth") or 1) + + max_nodes = int(args.get("max_neighbors") or 20) + query = args.get("query", "") - # Build seed nodes from the entry_id + + # --- Build the seed node --- seed_nodes = [] if kb and entry_id: try: - entry = kb.get_entry(entry_id) if hasattr(kb, 'get_entry') else None + entry = kb.get_entry(entry_id) if hasattr(kb, "get_entry") else None if entry: - seed_nodes = [GraphNode(node_id=entry_id, node_type="kb_entry", data=entry)] + # node_type must be "kb" (not "kb_entry") — GraphNode and + # the embedding helper both branch on type == "kb". + seed_nodes = [GraphNode(node_id=entry_id, node_type="kb", data=entry)] except Exception: pass - # expand(seed_nodes, kb=None, max_depth=1, max_nodes=20, query_text=None, ...) - result = expand(seed_nodes=seed_nodes, kb=kb, max_depth=depth, query_text=query or None) - if hasattr(result, 'format_for_context'): - return {"nodes": result.format_for_context(), "count": len(result.all_nodes())} - return {"nodes": [], "count": 0} + + result = expand( + seed_nodes=seed_nodes, + kb=kb, + max_depth=depth, + max_nodes=max_nodes, + query_text=query or None, + ) + + # Serialise related nodes into the shape the probe (and callers) expect. + # Exclude the seed itself — callers already have it. + neighbors = [] + for node in result.related_nodes(): + neighbors.append({ + "id": node.id, + "title": node.get_title(), + "summary": node.get_summary(), + "score": round(float(node.query_relevance), 4), + "via": node.via, # "link" or "semantic" + }) + + return {"neighbors": neighbors, "count": len(neighbors)} def _kb_graph_autolink(kernel, args): @@ -1599,7 +1635,7 @@ def _ta_library_stats(kernel, args): # ==================================================================== -# Plan 20: Signal Bus instructions +# Signal Bus instructions # ==================================================================== def _signal_emit(kernel, args): @@ -1779,7 +1815,7 @@ def _ambient_drain(kernel, args): SUBSYSTEM_HANDLERS_AMBIENT = { - # Plan 20: Ambient Reactor read-only surface + # Ambient Reactor read-only surface "AMBIENT_STATS": _ambient_stats, "AMBIENT_CONTEXT": _ambient_context, "AMBIENT_DRAIN": _ambient_drain, @@ -1787,7 +1823,7 @@ def _ambient_drain(kernel, args): SUBSYSTEM_HANDLERS_SIGNAL = { - # Plan 20: Signal Bus + # Signal Bus "SIGNAL_EMIT": _signal_emit, "SIGNAL_LIST": _signal_list, "SIGNAL_STATS": _signal_stats, diff --git a/project/kernel_ui_sdol.py b/project/kernel_ui_sdol.py index fac45ce..c50e88a 100644 --- a/project/kernel_ui_sdol.py +++ b/project/kernel_ui_sdol.py @@ -1,8 +1,6 @@ """ kernel_ui_sdol.py - Phases D-G: Semantic UI + session + SDOL + semantic rendering -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _UiSdolRenderingMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via diff --git a/project/kernel_vsa.py b/project/kernel_vsa.py index 8e44372..8f7f5d8 100644 --- a/project/kernel_vsa.py +++ b/project/kernel_vsa.py @@ -1,8 +1,6 @@ """ kernel_vsa.py - Phase A: VSA (Vector Symbolic Architecture) + cleanup memory seeding -Split from kernel.py during Phase 12 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. Defines _VsaMixin, one of seventeen mixins composed into Kernel in kernel.py. The mixin is not useful on its own — it is joined via diff --git a/project/knowledge/atomic_extractor.py b/project/knowledge/atomic_extractor.py new file mode 100644 index 0000000..46d37db --- /dev/null +++ b/project/knowledge/atomic_extractor.py @@ -0,0 +1,465 @@ +""" +atomic_extractor.py — JSON-schema-constrained extraction via Ollama. + +Wraps ``ollama_driver.generate(format="json", ...)`` with a tight +prompt and a schema-shaped expected return. Each call: + + • sees ONLY the segment text + the prompt template (narrow context + → no cross-record bleed) + • runs with ``format=json`` so qwen3 emits valid JSON or fails (vs + free-form prose with hallucinated URLs) + • is a single Ollama generation, not an agent_session round — + minimal hallucination surface + +Schema returned per segment: + + { + "topic": str, // ≤ 80 chars, the segment's theme + "summary": str, // ≤ 280 chars, what was discussed + "facts": [ + {"subject": str, "predicate": str, "object": str, "confidence": 0..1} + ], + "entities": [ + {"name": str, "type": "person|place|organization|concept|work"} + ], + "questions_raised": [str], + "claims_made": [ + {"claim": str, "speaker": str|null} + ] + } + +References: + - AFEV (arxiv 2506.07446) — atomic-fact decomposition + - LLM×MapReduce (ACL 2025) — map-stage structured extraction + - n1n.ai 2026 — JSON-schema design for hallucination mitigation +""" + +import json +import logging +import re + +logger = logging.getLogger("atomic_extractor") + + +# NOTE: Prompts are str.Template-style — we use $segment_text / +# $segment_json / $cluster_text rather than .format() so the literal +# curly braces in the schema documentation don't get interpreted as +# format placeholders (KeyError: '\n "topic"' was the bite). +from string import Template + + +SEGMENT_PROMPT = Template("""\ +You are extracting structured information from a podcast transcript segment. +Read ONLY the text between the BEGIN/END markers. Return STRICT JSON that +EXACTLY matches the example shape below. Each fact/entity/claim must be +an OBJECT — strings will be REJECTED. Do NOT invent URLs, citations, +references, or content beyond what the text supports. + +Example output (mirror this structure exactly): +{ + "topic": "Volcanology and lava chemistry", + "summary": "Host A asks volcanologist B about magma composition, eruption types, and a recent Hawaiian event.", + "facts": [ + {"subject": "basaltic lava", "predicate": "has", "object": "low silica content", "confidence": 0.9}, + {"subject": "guest B", "predicate": "studies", "object": "active volcanoes", "confidence": 0.95} + ], + "entities": [ + {"name": "guest B", "type": "person"}, + {"name": "Hawaii", "type": "place"}, + {"name": "magma", "type": "concept"} + ], + "questions_raised": [ + "What triggers explosive vs effusive eruptions?", + "Can lava chemistry predict an eruption's behaviour?" + ], + "claims_made": [ + {"claim": "Pahoehoe and a'a are two basaltic lava textures.", "speaker": "guest B"} + ] +} + +Rules: + - Use ONLY information in the text below. No outside knowledge. + - Each facts/entities/claims item MUST be an object — never a string. + - Confidence in 0.0-1.0. + - Entity types: person | place | organization | concept | work. + - Speaker null when unattributable. + - Output ONLY the JSON object. No prose before or after. + +BEGIN TRANSCRIPT SEGMENT +$segment_text +END TRANSCRIPT SEGMENT +""") + + +REDUCE_PROMPT = Template("""\ +You are aggregating per-segment extractions of a single podcast episode. +Each segment has already been individually extracted; your job is to +deduplicate, resolve conflicts, and produce one episode-level structure. + +Below is a JSON list of per-segment extractions. Return a single JSON +object that EXACTLY matches the example shape — every facts entry must +be an OBJECT with subject/predicate/object keys, every entities entry +must be an OBJECT with name/type keys, every claims entry must be an +OBJECT with claim/speaker keys. String entries will be REJECTED. + +Example output (mirror this structure exactly): +{ + "summary": "Host A and guest B explore X. They cover topic 1, topic 2, and topic 3, including specific findings and questions about Y.", + "themes": ["Topic Area 1", "Topic Area 2", "Topic Area 3"], + "facts": [ + {"subject": "guest B", "predicate": "is", "object": "an expert in X", "confidence": 0.9}, + {"subject": "fossil X", "predicate": "was discovered in", "object": "year 2020 in Region Z","confidence": 0.85} + ], + "entities": [ + {"name": "guest B", "type": "person"}, + {"name": "Region Z", "type": "place"}, + {"name": "topic 1", "type": "concept"} + ], + "questions": ["How does mechanism M work?", "Why does pattern P happen?"], + "claims": [ + {"claim": "X is the best example of Y", "speaker": "guest B"} + ] +} + +Rules: + - Use ONLY information present in the per-segment input below. + - Every facts/entities/claims item MUST be an object — never a string. + - Confidence stays in 0.0-1.0. + - Entity types: person | place | organization | concept | work + - When two segments disagree on a fact, prefer the higher-confidence one. + - Output ONLY the JSON object. No prose before or after. + +PER-SEGMENT INPUT: +$segment_json +""") + + +CLUSTER_PROMPT = Template("""\ +You are summarising a cluster of related segment summaries from a single +podcast episode. Produce a JSON object with this schema: +{ + "label": string up to 60 chars, naming the unifying theme, + "summary": string up to 400 chars, the synthesis of the cluster +} + +Use only what is present in the cluster summaries below. Output ONLY +the JSON object. + +CLUSTER SUMMARIES: +$cluster_text +""") + + +def _strip_to_json(text): + """Trim leading/trailing prose and Markdown fences from an LLM + response to leave just the JSON object.""" + if text is None: + return "" + s = text.strip() + # Strip ```json ... ``` fences + if s.startswith("```"): + s = re.sub(r"^```(?:json)?\s*", "", s) + s = re.sub(r"\s*```$", "", s) + # Take from first { to last } + a = s.find("{") + b = s.rfind("}") + if a >= 0 and b > a: + s = s[a:b + 1] + return s.strip() + + +def _clean_extraction(obj): + """Coerce/sanitise an extraction dict into the schema's shape.""" + if not isinstance(obj, dict): + obj = {} + out = {} + out["topic"] = (obj.get("topic") or "")[:200] + out["summary"] = (obj.get("summary") or "")[:600] + facts = obj.get("facts") or [] + out["facts"] = [] + for f in facts if isinstance(facts, list) else []: + if isinstance(f, dict): + s, p, o = f.get("subject"), f.get("predicate"), f.get("object") + if not (s and p and o): + continue + try: + conf = float(f.get("confidence", 0.7)) + except Exception: + conf = 0.7 + elif isinstance(f, str) and f.strip(): + s, p, o = "episode", "states", f.strip() + conf = 0.5 + else: + continue + out["facts"].append({ + "subject": str(s)[:200], + "predicate": str(p)[:120], + "object": str(o)[:400], + "confidence": max(0.0, min(1.0, conf)), + }) + ents = obj.get("entities") or [] + out["entities"] = [] + for e in ents if isinstance(ents, list) else []: + if isinstance(e, dict): + n, t = e.get("name"), e.get("type") + if not n: + continue + if t not in ("person", "place", "organization", "concept", "work"): + t = "concept" + elif isinstance(e, str) and e.strip(): + n, t = e.strip(), "concept" + else: + continue + out["entities"].append({"name": str(n)[:160], "type": t}) + qs = obj.get("questions_raised") or [] + out["questions_raised"] = [ + str(q)[:300] for q in (qs if isinstance(qs, list) else []) + if q + ][:20] + cs = obj.get("claims_made") or [] + out["claims_made"] = [] + for c in cs if isinstance(cs, list) else []: + if not isinstance(c, dict): + continue + cl = c.get("claim") + if not cl: + continue + sp = c.get("speaker") + out["claims_made"].append({ + "claim": str(cl)[:400], + "speaker": str(sp)[:80] if sp else None, + }) + return out + + +def _ollama_generate_json(prompt, driver=None, model=None, num_ctx=None, timeout_s=120): + """One-shot Ollama generate with format=json, return parsed dict. + + The ``driver`` arg is the dependency-injected OllamaDriver — pass a + long-lived instance from ``run_review()`` so a single ``warmup()`` + call covers all 12-17 LLM calls in a review run. When ``driver`` + is None, falls back to creating a fresh driver per call (used by + direct callers and tests; a 5-15s cold-load tax applies per call). + """ + if driver is None: + try: + import ollama_driver + except Exception as e: + raise RuntimeError(f"ollama_driver unavailable: {e}") + driver = ollama_driver.OllamaDriver() + if num_ctx is None: + # Mid-range context — segments are 6k chars max ≈ 1500 tokens, + # plus prompt overhead. 8192 is plenty and cheaper than 16k. + num_ctx = 8192 + # Inject /no_think for speed unless caller already did so — qwen3 + # honours it anywhere in the prompt. + if "/no_think" not in prompt: + prompt = "/no_think\n" + prompt + r = driver.generate( + prompt=prompt, + model=model, + temperature=0.1, + max_tokens=2048, + num_ctx=num_ctx, + json_mode=True, + ) + if not r.get("ok"): + raise RuntimeError(f"ollama generate failed: {r.get('error')}") + text = r.get("text") or "" + text = _strip_to_json(text) + if not text: + return {} + try: + return json.loads(text) + except json.JSONDecodeError as e: + logger.warning( + "atomic_extractor: JSON parse failed (%s); raw head: %s", + e, text[:200], + ) + return {} + + +def extract_segment(segment_text, model=None, driver=None): + """Extract structured info from one transcript segment.""" + prompt = SEGMENT_PROMPT.substitute(segment_text=segment_text) + raw = _ollama_generate_json(prompt, driver=driver, model=model) + return _clean_extraction(raw) + + +def _deterministic_reduce(per_segment): + """Fallback reducer used when the LLM reducer returns nothing. + + Deduplicates facts by lowered (subject, predicate, object). + Deduplicates entities by lowered name. Concatenates segment + summaries into a coarse episode summary (truncated). Themes are + the unique segment topics. No LLM call, no synthesis — just a + union — but it preserves the per-segment work rather than + discarding it. + """ + summary_pieces = [] + themes_seen = [] + facts = [] + fact_keys = set() + entities = [] + ent_keys = set() + questions = [] + claims = [] + for s in per_segment: + if not isinstance(s, dict): + continue + topic = (s.get("topic") or "").strip() + if topic and topic not in themes_seen: + themes_seen.append(topic) + sm = (s.get("summary") or "").strip() + if sm: + summary_pieces.append(sm) + for f in s.get("facts") or []: + if not isinstance(f, dict): + continue + sb, pr, ob = f.get("subject"), f.get("predicate"), f.get("object") + if not (sb and pr and ob): + continue + key = (str(sb).lower(), str(pr).lower(), str(ob).lower()) + if key in fact_keys: + continue + fact_keys.add(key) + facts.append(f) + for e in s.get("entities") or []: + if not isinstance(e, dict) or not e.get("name"): + continue + key = str(e["name"]).lower() + if key in ent_keys: + continue + ent_keys.add(key) + entities.append(e) + for q in s.get("questions_raised") or []: + if q and q not in questions: + questions.append(q) + for c in s.get("claims_made") or []: + if isinstance(c, dict) and c.get("claim"): + claims.append(c) + summary = " ".join(summary_pieces)[:1500] + return { + "summary": summary, + "themes": themes_seen[:8], + "facts": facts, + "entities": entities, + "questions": questions[:30], + "claims": claims, + } + + +def reduce_episode(per_segment_extractions, model=None, driver=None): + """Aggregate the per-segment extractions into one episode-level dict. + + Two robustness mechanics: + 1. Strip pipeline-internal fields (index/start_char/end_char/elapsed_s) + before serializing — they have no value to the reducer and cost + 30%+ of the payload size on long episodes. + 2. If the reducer returns nothing on the first try (empty raw dict + on non-empty input — usually a context-window overrun on long + episodes), fall back to a deterministic union of per-segment + outputs. Degraded vs the LLM-reduced shape (no dedup, no + theme synthesis, no episode-level summary) but non-empty. + """ + INTERNAL = {"index", "start_char", "end_char", "elapsed_s"} + trimmed = [] + for s in per_segment_extractions: + if not isinstance(s, dict): + continue + trimmed.append({k: v for k, v in s.items() if k not in INTERNAL}) + payload = json.dumps(trimmed, ensure_ascii=False) + prompt = REDUCE_PROMPT.substitute(segment_json=payload) + # 32768 num_ctx gives headroom for ~12-segment episodes with + # dense per-segment extractions (~6-10 facts/segment). Tested + # against MUM episodes that previously OOM'd the reducer at 16k. + raw = _ollama_generate_json(prompt, driver=driver, model=model, num_ctx=32768) + if not raw and per_segment_extractions: + logger.warning( + "reduce_episode: LLM returned empty on %d segments — " + "falling back to deterministic union", + len(per_segment_extractions), + ) + raw = _deterministic_reduce(per_segment_extractions) + if not isinstance(raw, dict): + raw = {} + # Sanitise the reduced shape + out = { + "summary": (raw.get("summary") or "")[:1500], + "themes": [ + str(t)[:120] for t in (raw.get("themes") or [])[:8] if t + ], + "facts": [], + "entities": [], + "questions": [ + str(q)[:300] for q in (raw.get("questions") or [])[:30] if q + ], + "claims": [], + } + for f in raw.get("facts") or []: + if isinstance(f, dict): + s, p, o = f.get("subject"), f.get("predicate"), f.get("object") + if not (s and p and o): + continue + try: + conf = float(f.get("confidence", 0.7)) + except Exception: + conf = 0.7 + elif isinstance(f, str) and f.strip(): + # Permissive fallback — promote a bare claim string into a + # generic SVO triple so the fact survives even when the + # model ignored the structure. Predicate "states" tags it + # as low-structure but preserves the content. + s, p, o = "episode", "states", f.strip() + conf = 0.5 + else: + continue + out["facts"].append({ + "subject": str(s)[:200], + "predicate": str(p)[:120], + "object": str(o)[:400], + "confidence": max(0.0, min(1.0, conf)), + }) + for e in raw.get("entities") or []: + if isinstance(e, dict): + n, t = e.get("name"), e.get("type") + if not n: + continue + if t not in ("person", "place", "organization", "concept", "work"): + t = "concept" + elif isinstance(e, str) and e.strip(): + # Permissive fallback — bare-string entities get type=concept. + n, t = e.strip(), "concept" + else: + continue + out["entities"].append({"name": str(n)[:160], "type": t}) + for c in raw.get("claims") or []: + if isinstance(c, dict): + cl = c.get("claim") + if not cl: + continue + sp = c.get("speaker") + elif isinstance(c, str) and c.strip(): + cl, sp = c.strip(), None + else: + continue + out["claims"].append({ + "claim": str(cl)[:400], + "speaker": str(sp)[:80] if sp else None, + }) + return out + + +def cluster_summary(cluster_segment_summaries, model=None, driver=None): + """Summarise a cluster of related segment summaries (RAPTOR step).""" + text = "\n\n".join( + f"- {s}" for s in cluster_segment_summaries if s + ) + prompt = CLUSTER_PROMPT.substitute(cluster_text=text) + raw = _ollama_generate_json(prompt, driver=driver, model=model) + if not isinstance(raw, dict): + raw = {} + return { + "label": (raw.get("label") or "")[:120], + "summary": (raw.get("summary") or "")[:800], + } diff --git a/project/knowledge/blackboard.py b/project/knowledge/blackboard.py index 7effc09..53c240d 100644 --- a/project/knowledge/blackboard.py +++ b/project/knowledge/blackboard.py @@ -1,5 +1,3 @@ -# blackboard.py — Already routes through kernel ASSIST when available - """ blackboard.py - Shared Project Blackboard @@ -464,7 +462,7 @@ def render(self, max_chars=2500): lines.append(line) sections.append("\n".join(lines)) - # --- Phase 6: Deliverables --- + # --- Deliverables: read taskplan_*.json to surface status for Director --- # Read taskplan_*.json files to show the Director what deliverables # exist and whether they've been delivered/verified. This gives # the Director visibility into deliverable status without needing diff --git a/project/knowledge/goal_auto_populate.py b/project/knowledge/goal_auto_populate.py index 407f6ae..23391c1 100644 --- a/project/knowledge/goal_auto_populate.py +++ b/project/knowledge/goal_auto_populate.py @@ -72,21 +72,16 @@ class ExtractedGoals(BaseModel): # ----------------------------------------------------------------------- # Helper: read Ollama URL and model from config.json # -# leOS stores LLM config under coprocessors.ollama, while the old -# agent-swarm codebase used an llm section. We check both paths -# so this works in either context. +# Check both coprocessors.ollama (leOS) and llm (legacy) config paths. # ----------------------------------------------------------------------- def _find_config_file(): """Locate config.json by walking up from this module's directory. - This module can be installed in any subdirectory (knowledge/ or - elsewhere), so the old `os.path.join(os.path.dirname(__file__), - "config.json")` check failed because config.json actually lives - in the project root, not the subdirectory. + Walk up to 5 levels looking for config.json; this module may be + installed in any subdirectory. - Walk up to 5 levels looking for config.json. Returns the path - if found, or None. + Returns the path if found, or None. """ current = os.path.dirname(os.path.abspath(__file__)) for _ in range(5): @@ -115,8 +110,16 @@ def _get_config(): cfg = json.load(f) # --- leOS path: coprocessors.ollama --- - ollama_cfg = cfg.get("coprocessors", {}).get("ollama", {}) - if ollama_cfg.get("enabled", False): + # If the leOS section is present at all, treat it as authoritative. + # An explicit `enabled: false` means the user has decided not to + # run the LLM, so the auto-populator should bow out — falling + # through to the legacy `llm.*` path produced + # `Invalid URL '/api/generate'` errors at the caller because the + # legacy section is rarely populated under leOS configs. + ollama_cfg = cfg.get("coprocessors", {}).get("ollama") + if ollama_cfg is not None: + if not ollama_cfg.get("enabled", False): + return None, None, 65536 url = ollama_cfg.get("model_server", "") if url.endswith("/v1"): url = url[:-3] @@ -124,8 +127,12 @@ def _get_config(): num_ctx = ollama_cfg.get("num_ctx", 8192) if url and model: return url, model, num_ctx + # Section enabled but missing required fields — refuse rather + # than fall through to a probably-stale legacy section. + return None, None, 65536 # --- Old agent-swarm path: llm --- + # Only reached when no `coprocessors.ollama` section exists at all. llm = cfg.get("llm", {}) url = llm.get("model_server", "") if url.endswith("/v1"): diff --git a/project/knowledge/hashtag_extraction.py b/project/knowledge/hashtag_extraction.py new file mode 100644 index 0000000..2a6c928 --- /dev/null +++ b/project/knowledge/hashtag_extraction.py @@ -0,0 +1,104 @@ +"""hashtag_extraction.py — Markdown-aware inline #hashtag extraction. + +Pulls user-authored ``#tag`` markers out of an article's body without +mistaking markdown structures for tags. Pattern derived from a Rust +reference implementation of the same rule set. + +Rules (in order): + 1. Hashtags must start with ``#`` followed by a letter. + 2. Body chars are letters, digits, hyphens, underscores. + 3. Numeric-only tags excluded (``#123``). + 4. Markdown headings excluded (``# H1``, ``## H2`` — ``#`` followed + by whitespace or end-of-line). + 5. Code blocks (fenced ``` ``` ``` ``` ```) excluded. + 6. Inline code (`` `code` ``) excluded. + 7. URL fragments excluded (``http://x.com/#anchor``). + 8. Markdown link anchors excluded (``[text](#anchor)``). + 9. Tags normalised to lowercase. + 10. Duplicates removed. + +Output is a sorted list of unique tag strings. +""" + +from __future__ import annotations + +import re +from typing import List, Set + + +_FENCED_CODE = re.compile(r"```[a-zA-Z0-9_+-]*\n.*?```", re.DOTALL) +_INLINE_CODE = re.compile(r"`[^`]+`") +_MARKDOWN_LINK = re.compile(r"\[([^\]]+)\]\(([^)]+)\)") +_URL = re.compile( + r"https?://[^\s<>\[\]()]+" + r"|www\.[^\s<>\[\]()]+" + r"|[a-zA-Z0-9.\-]+\.[a-z]{2,}[^\s<>\[\]()]*", + re.IGNORECASE, +) +_HASHTAG = re.compile(r"(?:^|[^a-zA-Z0-9_-])#([a-zA-Z][a-zA-Z0-9_-]*)") + + +def _strip_fenced_code(text: str) -> str: + return _FENCED_CODE.sub("", text) + + +def _strip_inline_code(text: str) -> str: + return _INLINE_CODE.sub("", text) + + +def _strip_headings(text: str) -> str: + """Drop any line that begins with ``#`` followed by whitespace or + end-of-line — those are markdown headings, not hashtags.""" + out_lines = [] + for line in text.splitlines(): + stripped = line.lstrip() + if stripped.startswith("#"): + n_hash = 0 + for ch in stripped: + if ch == "#": + n_hash += 1 + else: + break + after = stripped[n_hash : n_hash + 1] + # Heading if ``#``s followed by space or nothing (line end) + if after == "" or after.isspace(): + continue + out_lines.append(line) + return "\n".join(out_lines) + + +def _strip_markdown_links(text: str) -> str: + """Replace ``[label](url)`` with just ``label`` — strips the URL + portion (which may contain ``#anchor``).""" + return _MARKDOWN_LINK.sub(r"\1", text) + + +def _strip_urls(text: str) -> str: + return _URL.sub("", text) + + +def extract_inline_hashtags(content: str) -> List[str]: + """Return sorted, lowercased, deduped list of ``#tag`` mentions. + + Empty input returns ``[]``; never raises. + """ + if not content or not isinstance(content, str): + return [] + text = _strip_fenced_code(content) + text = _strip_inline_code(text) + text = _strip_headings(text) + text = _strip_markdown_links(text) + text = _strip_urls(text) + + found: Set[str] = set() + for m in _HASHTAG.finditer(text): + tag = m.group(1) + if not tag: + continue + # Numeric-only check — regex already requires leading letter, + # so this is a defensive belt-and-braces check for future + # regex modifications. + if tag.isdigit(): + continue + found.add(tag.lower()) + return sorted(found) diff --git a/project/knowledge/knowledge.py b/project/knowledge/knowledge.py index a46149b..1ec0ef2 100644 --- a/project/knowledge/knowledge.py +++ b/project/knowledge/knowledge.py @@ -51,9 +51,7 @@ from datetime import datetime, timezone from rapidfuzz import fuzz -# Phase 4: BM25 for much better relevance ranking on natural language queries. -# bm25s is a fast pure-Python BM25 implementation -- near Elasticsearch speed -# for small-to-medium corpora, no heavy dependencies. +# BM25 ranking via bm25s — near-Elasticsearch speed for small-to-medium corpora, pure-Python. try: import bm25s HAS_BM25 = True @@ -67,8 +65,6 @@ # --------------------------------------------------------------------------- -# Phase 2: Embedding helpers -# # These functions bridge the KB and the embeddings module. They handle # composing the embeddable text from entry fields, calling embed_text(), # and storing results in the embeddings cache. The KB entry JSON files @@ -266,6 +262,16 @@ def __init__(self, kb_dir_or_kernel=None, max_entries=500, on_add=None): self._bm25_index = None self._bm25_dirty = True # Forces rebuild on first search + # Embedding matrix cache — built lazily by ``nearest()``, + # invalidated whenever ``self.entries`` changes. Pattern mirrors + # ``DisplacementLog._vectors_cache`` in kernel_state.py. + # At N=1000 the BLAS GEMV (~150 µs) replaces the Python loop + # in ``auto_link_entry`` (~15 ms per call). + import threading as _threading + self._emb_matrix = None # (N, 768) float32 unit-normed | None + self._emb_ids = None # list[str] aligned to matrix rows | None + self._emb_lock = _threading.Lock() + # Track whether default articles have been populated self._defaults_populated = False @@ -501,6 +507,29 @@ def add_entry(self, content, topics, project_id=None, source_agent="system", # This is non-blocking if embeddings are disabled (returns None). _embed_entry(entry) + # Phase 6: Append the new vector to the cached embedding matrix + # (instead of triggering a full rebuild on next nearest() call). + # Best-effort — on any failure we drop the cache and the next + # caller rebuilds from scratch. + _vec_for_matrix = _get_entry_embedding(entry) + if _vec_for_matrix is not None: + try: + import numpy as _np + _row = _np.asarray(_vec_for_matrix, dtype=_np.float32) + _n = _np.linalg.norm(_row) + if _n > 0: + _row = _row / _n + with self._emb_lock: + if self._emb_matrix is not None: + self._emb_matrix = _np.vstack( + [self._emb_matrix, _row.reshape(1, -1)] + ) + if self._emb_ids is not None: + self._emb_ids.append(entry["id"]) + # else: leave None — lazy build on next nearest() + except Exception: + self._invalidate_emb_cache() + # Auto-discover related KB entries and media records by # embedding similarity. Uses a hard threshold (0.55) so # unrelated entries in a small KB don't get falsely linked. @@ -531,6 +560,24 @@ def add_entry(self, content, topics, project_id=None, source_agent="system", except Exception: pass # Don't let a callback error break the KB + # Emit kb.article_saved on the signal bus so subscribers (e.g. + # the kb_reflection_listener) can spawn autonomous follow-ups + # without coupling KB writes to any specific consumer. Routed + # via plugin_loader.emit_hook so legacy plugin hooks fire too. + try: + from infra.plugin_loader import emit_hook + emit_hook( + "on_kb_entry_saved", + article_id=entry_id, + title=entry.get("title", ""), + summary=entry.get("summary", ""), + topics=topics, + importance=entry.get("importance", 0.5), + source_agent=source_agent, + ) + except Exception: + pass # signal emit must never break KB writes + # Sync bidirectional links: update referenced media records # so they know this KB entry points to them. if media_refs: @@ -593,6 +640,8 @@ def update_entry(self, entry_id, content=None, topics=None, importance=None, entry["updated"] = datetime.now(timezone.utc).isoformat() self._save_entry(entry) self._bm25_dirty = True + # Embedding may have changed — drop the matrix cache. + self._invalidate_emb_cache() # Phase 2: Re-embed the updated entry. # Content-hash cache means no wasted work if content didn't change. @@ -634,9 +683,120 @@ def delete_entry(self, entry_id): self._delete_entry_file(entry_id) self._remove_from_partition(entry_id) self._bm25_dirty = True + self._invalidate_emb_cache() return True return False + # ------------------------------------------------------------------- + # Embedding matrix cache — fast nearest-K + # ------------------------------------------------------------------- + # + # (N, 768) float32 matrix of unit-normed embeddings, parallel to + # ``_emb_ids``. Built on first ``nearest()`` call, appended on + # ``add_entry``, invalidated on update / delete / prune. + # Shared by all similarity scans (``auto_link_entry``, ``KB_NEAREST``). + + def _invalidate_emb_cache(self): + """Atomically drop the embedding matrix cache. Call after any + operation that changes ``self.entries`` or an entry's embedding.""" + with self._emb_lock: + self._emb_matrix = None + self._emb_ids = None + + def _build_emb_matrix(self): + """Lazy-build the (N, 768) float32 embedding matrix. + + Skips entries whose embedding is not yet available — they'll + rejoin the matrix on the next invalidate-then-rebuild cycle. + Caller must NOT hold ``self._emb_lock``. + """ + try: + import numpy as np + except ImportError: + return + with self._emb_lock: + if self._emb_matrix is not None: + return # built by a concurrent caller while we waited + vecs = [] + ids = [] + for e in self.entries: + v = _get_entry_embedding(e) + if v is None: + continue + arr = np.asarray(v, dtype=np.float32) + # Defensive unit-norm — nomic-text already normalises but + # cached entries may predate that contract. + n = np.linalg.norm(arr) + if n > 0: + arr = arr / n + vecs.append(arr) + ids.append(e["id"]) + if vecs: + self._emb_matrix = np.vstack(vecs).astype(np.float32, copy=False) + self._emb_ids = ids + else: + self._emb_matrix = np.empty((0, 768), dtype=np.float32) + self._emb_ids = [] + + def nearest(self, query_vec, k=10, threshold=None, exclude_ids=None): + """Return the ``k`` nearest KB entries to ``query_vec`` by + cosine similarity, using the cached embedding matrix. + + Args: + query_vec: list[float] or numpy array, 768-dim. + k: maximum results (default 10). + threshold: minimum cosine similarity to return; ``None`` + means no floor. + exclude_ids: iterable of entry ids to drop from the result + (typically the source entry id). + + Returns: + list of ``{"id", "score", "title", "summary"}`` dicts + sorted by descending similarity. + """ + try: + import numpy as np + except ImportError: + return [] + if self._emb_matrix is None: + self._build_emb_matrix() + # Snapshot under the lock so a concurrent invalidate doesn't + # leave us holding a stale reference mid-matmul. + with self._emb_lock: + mat = self._emb_matrix + ids = self._emb_ids + if mat is None or mat.shape[0] == 0 or not ids: + return [] + + q = np.asarray(query_vec, dtype=np.float32) + n = float(np.linalg.norm(q)) + if n > 0: + q = q / n + sims = mat @ q # (N,) BLAS GEMV — single call + + exclude = set(exclude_ids or []) + scored = [] + for i, eid in enumerate(ids): + if eid in exclude: + continue + s = float(sims[i]) + if threshold is not None and s < threshold: + continue + scored.append((s, eid)) + scored.sort(key=lambda kv: -kv[0]) + + by_id = {e["id"]: e for e in self.entries} + out = [] + for s, eid in scored[:k]: + entry = by_id.get(eid) or {} + out.append({ + "id": eid, + "score": s, + "title": entry.get("title", ""), + "summary": (entry.get("summary") or "")[:200], + }) + return out + # ------------------------------------------------------------------- # Article format validation # ------------------------------------------------------------------- @@ -778,6 +938,9 @@ def import_entry(self, foreign_entry): self._save_entry(entry) self.entries.append(entry) self._bm25_dirty = True + # Bulk-import path — invalidate the embedding matrix; the + # next nearest() call rebuilds with the new entry included. + self._invalidate_emb_cache() # Phase 2: Embed the imported entry _embed_entry(entry) @@ -934,7 +1097,12 @@ def search_semantic(self, query, project_id=None, limit=5): scored.sort(key=lambda x: x[0], reverse=True) return scored[:limit] - def search(self, query, project_id=None, limit=5): + def search(self, query, project_id=None, limit=5, **_kwargs): + # ``**_kwargs`` swallows forward-compat fields like ``agent_name`` + # (from the Agent Lens G2 path in tools/kb_search.py) without + # crashing. When the perspective wiring is fully implemented + # those fields can be promoted to real keyword args. + """ Combined search: hybrid BM25 + semantic (Phase 2), with fuzzy fallback. @@ -967,6 +1135,24 @@ def search(self, query, project_id=None, limit=5): embeddings_available = False if embeddings_available and HAS_BM25: + # Preferred path: RRF fusion of BM25 + nomic (+ qwen when + # available) with adaptive weights and MMR diversification. + # Falls back to the linear-blend hybrid if anything errors. + try: + from rrf_search import hybrid_rrf_search, is_enabled as _rrf_on + except Exception: + _rrf_on = lambda: False + hybrid_rrf_search = None # noqa: F841 + if _rrf_on() and hybrid_rrf_search is not None: + try: + results = self._search_rrf( + query, project_id=project_id, limit=limit, + hybrid_fn=hybrid_rrf_search, + ) + if results: + return results + except Exception as e: + logger.debug("RRF search failed (%s); falling to linear blend", e) results = self._search_hybrid(query, project_id=project_id, limit=limit) if results: return results @@ -1063,6 +1249,136 @@ def _search_hybrid(self, query, project_id=None, limit=5, blended.sort(key=lambda x: x[0], reverse=True) return [entry for _, entry in blended[:limit]] + def _search_rrf(self, query, project_id=None, limit=5, hybrid_fn=None): + """RRF + adaptive-weights + MMR hybrid search (Fortemi pattern). + + Builds three retriever callables — BM25, nomic, and (when + available) qwen3-embedding — and hands them to + ``rrf_search.hybrid_rrf_search``. The fused list is then + MMR-diversified before being mapped back to entry dicts. + + Falls through to the existing linear-blend ``_search_hybrid`` + when this path returns nothing or errors out. + """ + if hybrid_fn is None: + return [] + # Pull a deeper candidate pool from each retriever — RRF + # rewards consensus across lists, so each list should be + # generous (3-5x the final limit). + pool = max(15, limit * 4) + + # --- Build per-retriever callables --- + # BM25 + def _bm25(q, k): + return [(e["id"], s) for s, e in + self._search_bm25_scored(q, project_id=project_id, limit=k)] + + # Nomic (default text embedder) + def _nomic(q, k): + return [(e["id"], s) for s, e in + self.search_semantic(q, project_id=project_id, limit=k)] + + # Qwen3-embedding retriever (third RRF channel). + # + # Default OFF — gated behind ``LEOS_RRF_QWEN=1`` env flag. + # Reason: this path embeds every KB entry on first call + # (cached thereafter), but during startup KB-seed-init the + # cold-cache pass blocked the kernel for 60+ seconds and + # sometimes deadlocked on GPU contention with Chatterbox + # and ImageBind which load just before kernel init. Until + # we move qwen embedding generation to index-time + # (computed once per article on add_entry), the live-search + # path skips qwen and RRF runs with BM25 + nomic only. + qwen_fn = None + _qwen_on = ( + os.environ.get("LEOS_RRF_QWEN", "0").strip() + not in ("0", "false", "no", "off", "") + ) + try: + import embeddings as _emb + if _qwen_on and _emb.is_qwen_enabled(): + def _qwen(q, k): + qv = _emb.embed_text_qwen( + q, + task_instruction=( + "Given a knowledge base query, retrieve " + "relevant articles" + ), + ) + if not qv: + return [] + scored = [] + for entry in self.entries: + if project_id is not None: + ep = entry.get("project_id") + if ep is not None and ep != project_id: + continue + ev = entry.get("_qwen_emb") + if ev is None: + # Build compact representation; cap at 8000 + # chars to stay within nomic-text budget. + et = "\n".join([ + entry.get("title", ""), + entry.get("summary", ""), + entry.get("content", "") or "", + ])[:8000] + try: + ev = _emb.embed_text_qwen( + et, + task_instruction=( + "Given a knowledge base article, " + "represent it for retrieval" + ), + ) + except Exception: + ev = None + # Cache (or sentinel) on the entry so we don't + # retry every call when embedding fails. + entry["_qwen_emb"] = ev or [] + if not ev: + continue + s = _emb.cosine_similarity(qv, ev) + if s > 0: + scored.append((s, entry)) + scored.sort(key=lambda x: -x[0]) + return [(e["id"], s) for s, e in scored[:k]] + qwen_fn = _qwen + except Exception: + qwen_fn = None # qwen unavailable; RRF runs on bm25+nomic + + # MMR vectors: use each entry's cached embedding (nomic) so + # the post-fusion diversification is cheap. + mmr_vectors = {} + for e in self.entries: + v = _get_entry_embedding(e) + if v is not None: + mmr_vectors[e["id"]] = v + + fused = hybrid_fn( + query, + bm25_fn=_bm25, + nomic_fn=_nomic, + qwen_fn=qwen_fn, + limit=limit, + candidate_pool=pool, + mmr_vectors=mmr_vectors, + ) + if not fused: + return [] + + # Map ids back to full entries. Build an index once. + by_id = {e["id"]: e for e in self.entries} + out = [] + for iid, score, _label in fused: + entry = by_id.get(iid) + if entry: + # Annotate the score onto a shallow copy so we don't + # mutate the canonical KB entry. + ann = dict(entry) + ann["_rrf_score"] = score + out.append(ann) + return out + def _search_bm25_scored(self, query, project_id=None, limit=15): """ BM25 search that returns (raw_score, entry) tuples instead of @@ -1463,6 +1779,8 @@ def _prune(self): if e["id"] not in ids_to_remove ] self._bm25_dirty = True + # Pruned rows must drop from the matrix cache too. + self._invalidate_emb_cache() print(f"[KB] Pruned {excess} low-importance entries. {len(self.entries)} remaining.") diff --git a/project/knowledge/knowledge_base.py b/project/knowledge/knowledge_base.py index a1833a2..3ffdf82 100644 --- a/project/knowledge/knowledge_base.py +++ b/project/knowledge/knowledge_base.py @@ -8,7 +8,7 @@ from knowledge_base import KnowledgeBase # works from knowledge_base import get_kb # works from knowledge import KnowledgeBase # also works - from knowledge import Knowledgebase # also works (old name) + from knowledge import Knowledgebase # lowercase alias retained for compatibility from knowledge import get_kb # also works The KnowledgeBase class accepts either a directory path or a kernel diff --git a/project/knowledge/knowledge_graph.py b/project/knowledge/knowledge_graph.py index beecad0..0782188 100644 --- a/project/knowledge/knowledge_graph.py +++ b/project/knowledge/knowledge_graph.py @@ -70,7 +70,22 @@ # Auto-link: minimum similarity to create a 'related' or 'media_refs' # link automatically when a new KB entry is saved. Conservative to # avoid false connections. -LINK_THRESHOLD = 0.55 +LINK_THRESHOLD = 0.62 +# Hard cap on how many auto_related entries get attached to a single +# article. Without this, dense corpora (e.g. ~30 podcast reviews +# whose summaries all share narrative-podcast vocabulary) end up +# linking to every other article in their show, producing +# article.related lists with 30-40 weak matches that drown out the +# semantically strong ones. We keep only the top-N by similarity. +MAX_AUTO_RELATED = 8 + +# MMR diversity weight for related-link selection. See knowledge/mmr.py. +# 0.0 = pure relevance (top-K by cosine) +# 0.4 = moderate diversity (default — balances relevance & spread) +# 1.0 = pure diversity (rarely useful) +# 0.4 is effective for podcast corpora where same-show episodes +# cluster densely — produces 1-2 same-show matches plus cross-show hits. +MMR_DIVERSITY = 0.4 # Semantic neighbors: minimum similarity to add an implicit connection # during graph expansion. Slightly lower than LINK_THRESHOLD because @@ -736,26 +751,112 @@ def auto_link_entry(entry, kb): if entry_vec is None: return [], [] - auto_related = [] + # Score every other article; we'll cap, MMR-diversify and sort after. + scored_related = [] # (other_id, sim) + candidate_vecs = {} # other_id -> embedding (for MMR diversity calc) auto_media = [] # --- Compare against existing KB entries --- - for other in kb.entries: - other_id = other.get("id", "") - if other_id == entry_id: - continue # Don't link to self + # + # Fast path: single BLAS GEMV against the cached (N, 768) matrix + # — ~150 µs at N=1000 vs ~15ms for the Python loop. Falls back + # to the pairwise loop if the matrix is unavailable (embeddings + # disabled, NumPy missing, lock contention). + matrix_path_used = False + try: + import numpy as _np + if getattr(kb, "_emb_matrix", None) is None: + kb._build_emb_matrix() + with kb._emb_lock: + mat = kb._emb_matrix + ids = kb._emb_ids + if mat is not None and mat.shape[0] > 0 and ids: + ev = _np.asarray(entry_vec, dtype=_np.float32) + ev_n = float(_np.linalg.norm(ev)) + if ev_n > 0: + ev = ev / ev_n + sims = mat @ ev # (N,) — single BLAS call replaces the loop + for i, other_id in enumerate(ids): + if other_id == entry_id: + continue + sim = float(sims[i]) + if sim < LINK_THRESHOLD: + continue + # MMR still needs the actual vector for diversity + # scoring — pull from the matrix row directly. + other_row = mat[i] + scored_related.append((other_id, sim)) + candidate_vecs[other_id] = other_row.tolist() + logger.debug("Auto-link: %s <-> %s (sim=%.3f)", + entry_id, other_id, sim) + matrix_path_used = True + except Exception as e: + logger.debug("Auto-link matrix path failed (%s); using loop", e) - other_vec = _get_embedding_for_entry(other) - if other_vec is None: - continue + if not matrix_path_used: + for other in kb.entries: + other_id = other.get("id", "") + if other_id == entry_id: + continue # Don't link to self - sim = _cosine_sim(entry_vec, other_vec) - if sim >= LINK_THRESHOLD: - auto_related.append(other_id) - logger.debug( - "Auto-link: %s <-> %s (sim=%.3f)", entry_id, other_id, sim) + other_vec = _get_embedding_for_entry(other) + if other_vec is None: + continue + + sim = _cosine_sim(entry_vec, other_vec) + if sim >= LINK_THRESHOLD: + scored_related.append((other_id, sim)) + candidate_vecs[other_id] = other_vec + logger.debug( + "Auto-link: %s <-> %s (sim=%.3f)", entry_id, other_id, sim) + + # MMR re-ranking: top-K-by-cosine is myopic in dense corpora — + # same-show episodes cluster tightly and sweep out the cap with + # near-duplicates. MMR penalises already-selected neighbours so + # the cap-of-8 list spans more semantic ground. + # diversity=0.4 (lambda=0.6): ~60% relevance, ~40% diversity. + # Tunable via MMR_DIVERSITY. + if scored_related: + try: + from mmr import mmr_rerank + mmr_input = [ + {"id": oid, "score": sim} for oid, sim in scored_related + ] + mmr_out = mmr_rerank( + mmr_input, + vectors=candidate_vecs, + diversity=MMR_DIVERSITY, + limit=MAX_AUTO_RELATED, + ) + auto_related = [c["id"] for c in mmr_out] + # Keep score-by-target mapping for the typed-edge writer. + mmr_score_map = {c["id"]: float(c.get("_mmr_score", 0.0)) for c in mmr_out} + except Exception as e: + # Fallback to plain top-K if MMR module fails — never + # block save on a re-ranking import error. + logger.debug("MMR re-rank failed (%s); using raw top-K", e) + scored_related.sort(key=lambda kv: -kv[1]) + auto_related = [oid for oid, _ in scored_related[:MAX_AUTO_RELATED]] + mmr_score_map = dict(scored_related[:MAX_AUTO_RELATED]) + + # Mirror into LinkStore as kind=related so callers can query + # by edge kind/score/metadata instead of parsing the flat + # article.related[] array. Best-effort — never blocks save. + try: + from links import get_link_store, KIND_RELATED + link_store = get_link_store() + link_store.replace_outgoing( + entry_id, KIND_RELATED, + [(rid, mmr_score_map.get(rid, 0.0)) for rid in auto_related], + metadata={"source": "auto_link_entry"}, + ) + except Exception as e: + logger.debug("Typed-link write failed (%s); legacy related[] still set", e) + else: + auto_related = [] # --- Compare against media records --- + media_scores = [] # (media_id, sim) pairs for typed-edge writer below try: import media_library all_records, _ = media_library.list_records(limit=500) @@ -773,6 +874,7 @@ def auto_link_entry(entry, kb): sim = _cosine_sim(entry_vec, rec_vec) if sim >= LINK_THRESHOLD: auto_media.append(mid) + media_scores.append((mid, sim)) logger.debug( "Auto-link: %s <-> media %s (sim=%.3f)", entry_id, mid, sim) @@ -782,6 +884,18 @@ def auto_link_entry(entry, kb): except Exception as e: logger.debug("Auto-link media scan failed: %s", e) + # Mirror kb_article → media_record links as kind=derived_from: + # the KB article was generated from review of the media file. + if media_scores: + try: + from links import get_link_store, KIND_DERIVED_FROM + link_store = get_link_store() + for mid, score in media_scores: + link_store.add(entry_id, mid, KIND_DERIVED_FROM, float(score), + metadata={"source": "auto_link_entry"}) + except Exception as e: + logger.debug("Typed media-link write failed: %s", e) + # --- Apply the discovered links --- changed = False diff --git a/project/knowledge/links.py b/project/knowledge/links.py new file mode 100644 index 0000000..1df5094 --- /dev/null +++ b/project/knowledge/links.py @@ -0,0 +1,378 @@ +"""links.py — Typed link edges for the leOS KB / media graph. + +JSONL-backed edge store with two in-memory indexes (forward + +reverse) built once at load. No relational dependency — all state +lives in ``data/links.jsonl`` and the in-process maps below. + +Schema per edge (one JSON object per line): + + { + "id": "lnk_", + "from": "kb_3a6bbe13", # any node id (kb / media / etc) + "to": "kb_5853007d", + "kind": "related", # see KIND_* constants below + "score": 0.72, # [0, 1] + "created_at": "2026-04-29T12:34:56+00:00", + "metadata": { ... } # optional; source, evidence, etc. + } + +Kinds (mirror SKOS where applicable): + + related - associative; symmetric. + broader - 'from' is more specific; 'to' is more general. + narrower - inverse of broader. + exact_match - same concept (duplicate ingestions, e.g. two RSS pulls of one episode). + close_match - near-equivalent. + derived_from - 'from' was generated from 'to' (review article <- media record). + authored_by - human / agent attribution (rarely used). + +Coexists with the legacy ``article.related[]`` field on KB articles +— that's now treated as the canonical *kind=related* outgoing view. +The two stay in sync via ``LinkStore.set_related_for(article_id, +[ids], score=...)`` which writes both the legacy field AND typed +edges; new code should call typed APIs directly. +""" + +from __future__ import annotations + +import json +import logging +import os +import threading +import time +import uuid +from datetime import datetime, timezone +from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple + +logger = logging.getLogger("links") + +KIND_RELATED = "related" +KIND_BROADER = "broader" +KIND_NARROWER = "narrower" +KIND_EXACT_MATCH = "exact_match" +KIND_CLOSE_MATCH = "close_match" +KIND_DERIVED_FROM = "derived_from" +KIND_AUTHORED_BY = "authored_by" + +VALID_KINDS = { + KIND_RELATED, KIND_BROADER, KIND_NARROWER, KIND_EXACT_MATCH, + KIND_CLOSE_MATCH, KIND_DERIVED_FROM, KIND_AUTHORED_BY, +} + +# Inverse pairs for reciprocal-link generation. +_INVERSE = { + KIND_BROADER: KIND_NARROWER, + KIND_NARROWER: KIND_BROADER, + KIND_RELATED: KIND_RELATED, + KIND_EXACT_MATCH: KIND_EXACT_MATCH, + KIND_CLOSE_MATCH: KIND_CLOSE_MATCH, + KIND_DERIVED_FROM: None, # not symmetric + KIND_AUTHORED_BY: None, +} + + +def _data_dir() -> str: + base = os.environ.get("LEOS_DATA_DIR") + if not base: + base = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "data", + ) + return base + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _new_link_id() -> str: + return "lnk_" + uuid.uuid4().hex[:12] + + +class LinkStore: + """Append-only JSONL edge store with in-memory indexes.""" + + def __init__(self, path: Optional[str] = None) -> None: + self._path = path or os.path.join(_data_dir(), "links.jsonl") + self._lock = threading.RLock() + # Edges by id + self._edges: Dict[str, dict] = {} + # Forward index: from_id -> list of edge_ids + self._out: Dict[str, List[str]] = {} + # Reverse index: to_id -> list of edge_ids + self._in: Dict[str, List[str]] = {} + # Dedup key: (from, to, kind) -> edge_id + self._dedup: Dict[Tuple[str, str, str], str] = {} + self._load() + + # ----- persistence ---------------------------------------------- + + def _load(self) -> None: + if not os.path.exists(self._path): + return + try: + with open(self._path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + edge = json.loads(line) + except Exception: + continue + self._index(edge) + except Exception as e: + logger.warning("LinkStore load failed: %s", e) + + def _index(self, edge: dict) -> None: + eid = edge.get("id") + if not eid: + return + self._edges[eid] = edge + self._out.setdefault(edge["from"], []).append(eid) + self._in.setdefault(edge["to"], []).append(eid) + self._dedup[(edge["from"], edge["to"], edge["kind"])] = eid + + def _append(self, edge: dict) -> None: + try: + os.makedirs(os.path.dirname(self._path), exist_ok=True) + with open(self._path, "a", encoding="utf-8") as f: + f.write(json.dumps(edge, ensure_ascii=False) + "\n") + except Exception as e: + logger.warning("LinkStore append failed: %s", e) + + def _rewrite(self) -> None: + """Rewrite the file from in-memory state — used after deletes + (which can't be expressed as appends in JSONL).""" + tmp = self._path + ".tmp" + try: + os.makedirs(os.path.dirname(self._path), exist_ok=True) + with open(tmp, "w", encoding="utf-8") as f: + for edge in self._edges.values(): + f.write(json.dumps(edge, ensure_ascii=False) + "\n") + os.replace(tmp, self._path) + except Exception as e: + logger.warning("LinkStore rewrite failed: %s", e) + + # ----- mutators ------------------------------------------------- + + def add( + self, + from_id: str, + to_id: str, + kind: str = KIND_RELATED, + score: float = 0.0, + metadata: Optional[dict] = None, + ) -> str: + """Add an edge if it doesn't already exist; return edge id.""" + if not from_id or not to_id or from_id == to_id: + return "" + if kind not in VALID_KINDS: + kind = KIND_RELATED + with self._lock: + existing = self._dedup.get((from_id, to_id, kind)) + if existing: + # Update score on existing if higher; preserve created_at. + old = self._edges[existing] + if score > float(old.get("score", 0.0)): + old["score"] = float(score) + if metadata: + merged = dict(old.get("metadata") or {}) + merged.update(metadata) + old["metadata"] = merged + self._rewrite() + return existing + edge = { + "id": _new_link_id(), + "from": from_id, + "to": to_id, + "kind": kind, + "score": float(max(0.0, min(1.0, score))), + "created_at": _now_iso(), + "metadata": metadata or {}, + } + self._index(edge) + self._append(edge) + return edge["id"] + + def add_reciprocal( + self, + a: str, + b: str, + kind: str = KIND_RELATED, + score: float = 0.0, + metadata: Optional[dict] = None, + ) -> Tuple[str, str]: + """Create A→B and the appropriate inverse B→A in one shot.""" + forward_id = self.add(a, b, kind, score, metadata) + inverse = _INVERSE.get(kind) + if not inverse: + return forward_id, "" + backward_id = self.add(b, a, inverse, score, metadata) + return forward_id, backward_id + + def delete(self, edge_id: str) -> bool: + with self._lock: + edge = self._edges.pop(edge_id, None) + if not edge: + return False + try: + self._out.get(edge["from"], []).remove(edge_id) + except ValueError: + pass + try: + self._in.get(edge["to"], []).remove(edge_id) + except ValueError: + pass + self._dedup.pop((edge["from"], edge["to"], edge["kind"]), None) + self._rewrite() + return True + + def delete_for_node(self, node_id: str) -> int: + """Remove every edge touching ``node_id`` (in either + direction). Returns count removed.""" + with self._lock: + to_remove = set(self._out.get(node_id, [])) | set(self._in.get(node_id, [])) + for eid in to_remove: + edge = self._edges.pop(eid, None) + if not edge: + continue + self._dedup.pop((edge["from"], edge["to"], edge["kind"]), None) + self._out.pop(node_id, None) + self._in.pop(node_id, None) + # Clean up dangling references in other nodes' out/in lists + for lst in self._out.values(): + lst[:] = [e for e in lst if e in self._edges] + for lst in self._in.values(): + lst[:] = [e for e in lst if e in self._edges] + if to_remove: + self._rewrite() + return len(to_remove) + + def replace_outgoing( + self, + from_id: str, + kind: str, + targets: Sequence[Tuple[str, float]], + metadata: Optional[dict] = None, + ) -> int: + """Replace ALL outgoing edges of ``kind`` from ``from_id`` + with the new ``targets`` list of (to_id, score). Used by + the auto-link pass to refresh related links cleanly without + leftover stale entries. Returns count of new edges added.""" + with self._lock: + for eid in list(self._out.get(from_id, [])): + edge = self._edges.get(eid) + if edge and edge["kind"] == kind: + self.delete(eid) + added = 0 + for to_id, score in targets: + if self.add(from_id, to_id, kind, score, metadata): + added += 1 + return added + + # ----- queries -------------------------------------------------- + + def outgoing( + self, + node_id: str, + kind: Optional[str] = None, + *, + min_score: float = 0.0, + limit: Optional[int] = None, + ) -> List[dict]: + """Edges where ``from == node_id``. Sorted by score DESC.""" + eids = self._out.get(node_id, []) + out = [] + for eid in eids: + e = self._edges.get(eid) + if not e: + continue + if kind and e["kind"] != kind: + continue + if e["score"] < min_score: + continue + out.append(e) + out.sort(key=lambda e: -float(e.get("score", 0.0))) + return out[:limit] if limit else out + + def incoming( + self, + node_id: str, + kind: Optional[str] = None, + *, + min_score: float = 0.0, + limit: Optional[int] = None, + ) -> List[dict]: + eids = self._in.get(node_id, []) + out = [] + for eid in eids: + e = self._edges.get(eid) + if not e: + continue + if kind and e["kind"] != kind: + continue + if e["score"] < min_score: + continue + out.append(e) + out.sort(key=lambda e: -float(e.get("score", 0.0))) + return out[:limit] if limit else out + + def neighbors( + self, + node_id: str, + kind: Optional[str] = None, + *, + min_score: float = 0.0, + ) -> Set[str]: + """Union of out-target ids and in-source ids.""" + out_ids = {e["to"] for e in self.outgoing(node_id, kind, min_score=min_score)} + in_ids = {e["from"] for e in self.incoming(node_id, kind, min_score=min_score)} + return out_ids | in_ids + + def stats(self) -> Dict[str, Any]: + kinds: Dict[str, int] = {} + for e in self._edges.values(): + kinds[e["kind"]] = kinds.get(e["kind"], 0) + 1 + return { + "edges": len(self._edges), + "nodes": len(set(self._out) | set(self._in)), + "by_kind": kinds, + } + + # ----- migration ------------------------------------------------ + + def import_legacy_related( + self, + article_id: str, + related_ids: Iterable[str], + scores: Optional[Dict[str, float]] = None, + ) -> int: + """Convert a legacy ``article.related[]`` list to typed + ``related`` edges. Returns count added.""" + scores = scores or {} + added = 0 + for rid in related_ids: + if not rid or rid == article_id: + continue + score = float(scores.get(rid, 0.5)) + if self.add(article_id, rid, KIND_RELATED, score, + metadata={"source": "legacy_related"}): + added += 1 + return added + + +# ---- module-level singleton ------------------------------------------ + +_INSTANCE: Optional[LinkStore] = None +_INSTANCE_LOCK = threading.Lock() + + +def get_link_store() -> LinkStore: + global _INSTANCE + if _INSTANCE is not None: + return _INSTANCE + with _INSTANCE_LOCK: + if _INSTANCE is None: + _INSTANCE = LinkStore() + return _INSTANCE diff --git a/project/knowledge/media_ingest.py b/project/knowledge/media_ingest.py index 0a382c3..af4badc 100644 --- a/project/knowledge/media_ingest.py +++ b/project/knowledge/media_ingest.py @@ -96,7 +96,6 @@ logger = logging.getLogger("media_ingest") -# Additional imports needed by this module (Phase 5 post-split fix) import media_library import embeddings from media_ingest_helpers import _HAS_BS4, _HAS_CV2, _HAS_DOCX, _HAS_PIL, _HAS_PYPDF, _HAS_TESSERACT, _HAS_WHISPER, _HAS_YOLO, _stage_timers, _HAS_YTDLP @@ -308,11 +307,8 @@ def _run_url_pipeline(record_id, url, url_type): _stage_timers.pop(record_id, None) # ============================================================================ -# Phase 5 re-exports +# Re-exports: public surface preserved for callers that import directly from this module. # ============================================================================ -# Every function that used to live at module level in media_ingest.py is -# re-exported here so that existing callers (`import media_ingest; -# media_ingest.some_fn(...)`) continue to work without any call-site changes. from media_ingest_helpers import ( # noqa: F401 -- re-export _update_stage, _format_duration, _HAS_FFMPEG, _HAS_FFPROBE, @@ -339,14 +335,9 @@ def _run_url_pipeline(record_id, url, url_type): _pipeline_url_text, _ingest_web_page, _ingest_youtube, _ingest_direct_download, _classify_url, ) -from media_ingest_review import ( # noqa: F401 -- re-export - ensure_media_project, -) - __all__ = [ # Public API "ingest", "ingest_url", "get_capabilities", "queue_status", - "ensure_media_project", # Internal dispatchers + extracted helpers (kept importable for # existing code that may reach for them) "_run_pipeline", "_run_url_pipeline", diff --git a/project/knowledge/media_ingest_audio_pipeline.py b/project/knowledge/media_ingest_audio_pipeline.py index 78eb806..4e8dd9a 100644 --- a/project/knowledge/media_ingest_audio_pipeline.py +++ b/project/knowledge/media_ingest_audio_pipeline.py @@ -40,7 +40,7 @@ # Cross-file imports from sibling media_ingest_* modules. from media_ingest_helpers import _format_duration, _update_stage from media_ingest_models import _get_whisper_model -from media_ingest_review import _schedule_media_review +from media_ingest_review import run_review def _pipeline_audio(record_id, file_path): """Full analysis pipeline for an audio file.""" @@ -54,11 +54,23 @@ def _pipeline_audio(record_id, file_path): if metadata: results["metadata"] = metadata - # Step 2: Transcribe speech + # Step 2: Transcribe speech. Result carries both the flat text + # (used everywhere downstream that just wants the transcript) and + # the timestamped segment list (carries optional speaker labels + # when WhisperX + pyannote diarization is wired). Both land on + # the record so atomic_extractor can use the flat text and any + # future caller can hop to a specific moment by timestamp. _update_stage(record_id, "Transcribing speech...") - transcript = _transcribe_media(file_path) - if transcript: - results["transcript"] = transcript + tx = _transcribe_media(file_path) + if isinstance(tx, dict): + if tx.get("text"): + results["transcript"] = tx["text"] + if tx.get("segments"): + results["transcript_segments"] = tx["segments"] + if tx.get("language"): + results["transcript_language"] = tx["language"] + elif tx: + results["transcript"] = tx # Step 3: Generate spectrogram _update_stage(record_id, "Generating spectrogram...") @@ -203,9 +215,15 @@ def _pipeline_audio(record_id, file_path): if embeddings.is_enabled(): embeddings.save_cache_to_disk() - # Schedule an agent to review the audio when idle. - # Transcribed speech often contains extractable knowledge. - _schedule_media_review(record_id) + # Run the staged review pipeline synchronously inside this worker + # thread. The pipeline is the single canonical reviewer — no + # legacy director_run path, no config branch, no fallback. See + # media_ingest_review.run_review for stage breakdown. + try: + run_review(record_id) + except Exception as e: + logger.exception("media_ingest_review failed for %s: %s", record_id, e) + def _get_audio_metadata(file_path): """Get audio metadata via ffprobe.""" @@ -239,17 +257,27 @@ def _get_audio_metadata(file_path): return {} def _transcribe_media(file_path): - """Transcribe speech in audio/video using Whisper.""" + """Transcribe speech in audio/video using the configured Whisper backend. + + Returns the full result dict — ``{"text": ..., "segments": [...], + "language": ...}`` — so callers can persist both the flat transcript + and the timestamped segments. Empty dict on failure rather than + raising, so ingest doesn't abort over transcription errors. + """ model = _get_whisper_model() if not model: - return "" + return {} try: - result = model.transcribe(file_path) - text = result.get("text", "").strip() - return text + result = model.transcribe(file_path) or {} + text = (result.get("text") or "").strip() + return { + "text": text, + "segments": result.get("segments") or [], + "language": result.get("language"), + } except Exception as e: logger.warning("Transcription failed: %s", e) - return "" + return {} def _generate_spectrogram(record_id, file_path): """Generate a spectrogram image from audio/video. Returns relative path.""" diff --git a/project/knowledge/media_ingest_document_pipeline.py b/project/knowledge/media_ingest_document_pipeline.py index 8a9c22f..59bbd4a 100644 --- a/project/knowledge/media_ingest_document_pipeline.py +++ b/project/knowledge/media_ingest_document_pipeline.py @@ -39,7 +39,7 @@ # Cross-file imports from sibling media_ingest_* modules. from media_ingest_helpers import _update_stage -from media_ingest_review import _schedule_media_review +from media_ingest_review import run_review def _pipeline_document(record_id, file_path): """Full analysis pipeline for a document file. @@ -147,8 +147,10 @@ def _pipeline_document(record_id, file_path): if embeddings.is_enabled(): embeddings.save_cache_to_disk() - # Schedule an agent to review the document contents when idle - _schedule_media_review(record_id) + try: + run_review(record_id) + except Exception as e: + logger.exception("media_ingest_review failed for %s: %s", record_id, e) def _generate_pdf_thumbnail(record_id, file_path, dpi=150, max_size=512): """ diff --git a/project/knowledge/media_ingest_helpers.py b/project/knowledge/media_ingest_helpers.py index a0aab89..bc0cfa5 100644 --- a/project/knowledge/media_ingest_helpers.py +++ b/project/knowledge/media_ingest_helpers.py @@ -87,13 +87,7 @@ # --------------------------------------------------------------------------- -# yt-dlp is a required dependency (see requirements.txt). Historically -# this file had a try/except that fell back to the CLI binary when the -# Python package was missing — now that the package is required, the -# fallback is dead code and the bare import will surface ImportError -# loudly if the install is broken. _HAS_YTDLP is kept as an always-True -# constant because downstream modules (media_ingest, media_ingest_url_ -# pipeline, media_fetch) guard their code on it. +# yt-dlp required. _HAS_YTDLP kept as always-True constant — downstream modules gate on it. import yt_dlp # noqa: F401 _HAS_YTDLP = True diff --git a/project/knowledge/media_ingest_image_pipeline.py b/project/knowledge/media_ingest_image_pipeline.py index 76f1d26..8a39702 100644 --- a/project/knowledge/media_ingest_image_pipeline.py +++ b/project/knowledge/media_ingest_image_pipeline.py @@ -40,7 +40,7 @@ # Cross-file imports from sibling media_ingest_* modules. from media_ingest_helpers import _update_stage from media_ingest_models import _get_yolo_model -from media_ingest_review import _schedule_media_review +from media_ingest_review import run_review def _upscale_for_analysis(file_path, target_min_dimension=1280): """ @@ -489,7 +489,10 @@ def _pipeline_image(record_id, file_path): if embeddings.is_enabled(): embeddings.save_cache_to_disk() - _schedule_media_review(record_id) + try: + run_review(record_id) + except Exception as e: + logger.exception("media_ingest_review failed for %s: %s", record_id, e) def _generate_thumbnail(record_id, image_path, max_size=256): """Generate a thumbnail image. Returns relative path or None.""" diff --git a/project/knowledge/media_ingest_models.py b/project/knowledge/media_ingest_models.py index a28d392..2f69dd4 100644 --- a/project/knowledge/media_ingest_models.py +++ b/project/knowledge/media_ingest_models.py @@ -1,10 +1,7 @@ """ media_ingest_models.py - Lazy model loaders for YOLO and Whisper. -Split from media_ingest.py during Phase 5 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. - -Two loader functions backed by module-level caches so a model is loaded +Two loader functions backed by module-level caches so each model is loaded at most once per process: _get_yolo_model(model_type) -- object detection / segmentation @@ -17,9 +14,18 @@ logger = logging.getLogger("media_ingest") -# Additional imports needed by this module (Phase 5 post-split fix) from media_ingest_helpers import YOLO, _HAS_WHISPER, _HAS_YOLO, _whisper_module +# Optional config — used to pick the Whisper model size and device when +# the caller doesn't pass one explicitly. Imported lazily to avoid a +# circular dependency through leos_orchestrator at module load time. +def _media_config(): + try: + from leos_orchestrator import load_global_config + return (load_global_config() or {}).get("media", {}) + except Exception: + return {} + # --------------------------------------------------------------------------- # Job Queue @@ -36,8 +42,12 @@ _YOLO_MODELS = {} # model_type -> YOLO instance _YOLO_LOCK = threading.Lock() -# Whisper model cache -_WHISPER_MODEL = None +# Whisper model cache. Keyed by (model_size, device) so callers can +# request different sizes during a session without colliding — if size +# or device changes between calls, we keep both loaded rather than +# replacing the singleton mid-flight. In practice almost everyone +# uses one combination per process. +_WHISPER_MODELS = {} # (size, device) -> model _WHISPER_LOCK = threading.Lock() def _get_yolo_model(model_type): @@ -70,19 +80,486 @@ def _get_yolo_model(model_type): logger.warning("Failed to load YOLO %s: %s", model_file, e) return None -def _get_whisper_model(model_size="base"): - """Load or return cached Whisper model.""" - global _WHISPER_MODEL - if not _HAS_WHISPER: - return None +# Adapter: normalises faster-whisper's output to the same .transcribe() +# shape as openai-whisper so the audio pipeline is backend-agnostic. +# faster-whisper is preferred — equal accuracy at small/medium, better +# at large-v3 (openai-whisper's repetition-hallucination bug is absent +# in CTranslate2), and ~2× faster on a CUDA host. +class _WhisperFasterAdapter: + def __init__(self, model, beam_size=5, language=None): + self._model = model + self._beam_size = beam_size + self._language = language + + def transcribe(self, file_path, **_kwargs): + kwargs = {"beam_size": self._beam_size} + if self._language: + kwargs["language"] = self._language + seg_iter, info = self._model.transcribe(file_path, **kwargs) + seg_list = list(seg_iter) + text = " ".join(s.text.strip() for s in seg_list).strip() + return { + "text": text, + "segments": [ + {"start": s.start, "end": s.end, "text": s.text} + for s in seg_list + ], + "language": getattr(info, "language", None), + } + + +# Adapter: faster-whisper for transcription + pyannote.audio for +# speaker diarization, composed directly without the whisperx wrapper +# package (whisperx 3.8.x pins torch 2.8, incompatible with the +# project's torch 2.5.1 stack). Output segments carry an optional +# ``speaker`` key when diarization succeeded; absent on failure (no +# HF_TOKEN, download failure, OOM). In every failure case, ``text`` +# and start/end timestamps are always populated. +class _DiarizingWhisperAdapter: + def __init__(self, fw_model, diarize_pipeline, beam_size=5, language=None): + self._fw = fw_model # faster_whisper.WhisperModel + self._diarize = diarize_pipeline # pyannote.audio Pipeline (or None) + self._beam_size = beam_size + self._language = language + + def transcribe(self, file_path, **_kwargs): + kwargs = {"beam_size": self._beam_size} + if self._language: + kwargs["language"] = self._language + seg_iter, info = self._fw.transcribe(file_path, **kwargs) + seg_list = list(seg_iter) + segments = [ + { + "start": float(s.start), + "end": float(s.end), + "text": (s.text or "").strip(), + } + for s in seg_list + ] + + # Speaker diarization — only runs when the pyannote pipeline + # loaded successfully. On OOM or any failure, return segments + # without speaker labels rather than aborting the transcript. + if self._diarize is not None and segments: + try: + # pyannote 3.3.2 crashes when the final audio chunk is + # shorter than its 10s window (160000 samples at 16 kHz). + # Load at 16 kHz mono and pad to the next 10s boundary; + # padding is appended after real audio so timestamps are + # unaffected. + import torch as _torch + import torchaudio as _ta + _wav, _sr = _ta.load(file_path) + if _wav.dim() == 2 and _wav.shape[0] > 1: + _wav = _wav.mean(dim=0, keepdim=True) + target_sr = 16000 + if _sr != target_sr: + _wav = _ta.functional.resample(_wav, _sr, target_sr) + _sr = target_sr + _window = 10 * _sr + _len = _wav.shape[-1] + _target = ((_len // _window) + 1) * _window + if _target > _len: + _pad = _torch.zeros( + _wav.shape[0], _target - _len, dtype=_wav.dtype, + ) + _wav = _torch.cat([_wav, _pad], dim=-1) + diarize_result = self._diarize( + {"waveform": _wav, "sample_rate": _sr} + ) + # diarize_result.itertracks(yield_label=True) → + # (Segment(start, end), track_id, speaker_label) + import numpy as _np + _spans = [ + (float(turn.start), float(turn.end), str(speaker)) + for turn, _, speaker in diarize_result.itertracks(yield_label=True) + ] + # Vectorised speaker-to-segment mapping. Build (T,2) + # boundary arrays, compute the full (S,T) overlap matrix + # in one broadcast, argmax per segment. At S=400, T=200 + # the matrix is ~320 KB float32 — fits in L3 cache. + if _spans: + _s_arr = _np.array([[s, e] for s, e, _ in _spans], dtype=_np.float32) + _labels = [lbl for _, _, lbl in _spans] + _s_starts = _s_arr[:, 0] # (T,) + _s_ends = _s_arr[:, 1] # (T,) + + _seg_starts = _np.array([s["start"] for s in segments], dtype=_np.float32) # (S,) + _seg_ends = _np.array([s["end"] for s in segments], dtype=_np.float32) # (S,) + _seg_mids = (_seg_starts + _seg_ends) * 0.5 # (S,) + + # (S,T) overlap matrix — clip negatives to 0 + _overlaps = _np.maximum( + 0.0, + _np.minimum(_seg_ends[:, None], _s_ends[None, :]) + - _np.maximum(_seg_starts[:, None], _s_starts[None, :]), + ) # float32, (S, T) + + _best_idx = _overlaps.argmax(axis=1) # (S,) — argmax keeps first on tie + _best_val = _overlaps[_np.arange(len(segments)), _best_idx] # (S,) + + for i, seg in enumerate(segments): + if _best_val[i] > 0.0: + seg["speaker"] = _labels[_best_idx[i]] + else: + # Fallback: use the speaker whose turn contains + # this segment's mid-point (handles silence gaps). + _mid = float(_seg_mids[i]) + for s_start, s_end, s_label in _spans: + if s_start <= _mid <= s_end: + seg["speaker"] = s_label + break + # If still no match, seg["speaker"] is not set + # (segment falls in silence — correct behaviour). + except Exception as e: + try: + import torch + if hasattr(torch, "cuda") and torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + logger.warning("pyannote diarize failed: %s — transcript without speaker labels", e) + + text = " ".join(s["text"] for s in segments if s["text"]).strip() + return { + "text": text, + "segments": segments, + "language": getattr(info, "language", None) or self._language, + } + + +# Rough VRAM estimates per faster-whisper size at fp16. int8 halves +# the model weights but the runtime working set still wants the +# bigger figure. These are starting estimates with a generous safety +# margin; the coordinator adds another 1 GB on top. +_FW_VRAM_FP16_MB = { + "tiny": 400, + "base": 600, + "small": 1200, + "medium": 2500, + "large": 4500, + "large-v2": 4500, + "large-v3": 4500, + "large-v3-turbo": 3000, +} + + +def _whisper_vram_mb(size, compute_type): + base = _FW_VRAM_FP16_MB.get(size, 4500) + if (compute_type or "").lower().startswith("int8"): + base = max(int(base * 0.6), 600) + return base + + +def _resolve_device(dev): + """Resolve "auto" to a concrete backend by querying torch.""" + dev = (dev or "auto").strip().lower() + if dev != "auto": + return dev + try: + import torch + if torch.cuda.is_available(): + return "cuda" + if (getattr(torch.backends, "mps", None) + and torch.backends.mps.is_available()): + return "mps" + except Exception: + pass + return "cpu" + + +def _get_whisper_model(model_size=None, device=None, backend=None): + """Load or return a cached Whisper model. + + When a caller doesn't pass arguments explicitly, we read from the + ``media`` section of the global config: + + whisper_model_size — tiny | base | small | medium | large | + large-v2 | large-v3 | large-v3-turbo + whisper_device — auto | cpu | cuda | cuda:N | mps + whisper_backend — auto | faster | openai + whisper_compute_type — float16 | int8_float16 | int8 | float32 + (faster-whisper only) + whisper_beam_size — int (default 5) + whisper_language — ISO code or null for auto-detect + + Backend selection: + ``auto`` — prefer faster-whisper, fall back to openai-whisper + ``faster`` — require faster-whisper (return None if unavailable) + ``openai`` — require openai-whisper + + Why the default leans on faster-whisper: same accuracy at small/ + medium, BETTER accuracy at large-v3 (it doesn't reproduce the + openai-whisper repetition-hallucination bug on quiet sections), + and ~2× faster on CUDA. CTranslate2 also runs on int8 cleanly, + which is the path for hosts with limited VRAM or no GPU at all. + """ + cfg = _media_config() + size = (model_size or cfg.get("whisper_model_size") or "base").strip() + dev = _resolve_device(device or cfg.get("whisper_device")) + backend = ( + (backend or cfg.get("whisper_backend") or "auto").strip().lower() + ) + cache_key = (backend, size, dev) with _WHISPER_LOCK: - if _WHISPER_MODEL is not None: - return _WHISPER_MODEL - try: - _WHISPER_MODEL = _whisper_module.load_model(model_size) - logger.info("Loaded Whisper model: %s", model_size) - return _WHISPER_MODEL - except Exception as e: - logger.warning("Failed to load Whisper: %s", e) - return None + cached = _WHISPER_MODELS.get(cache_key) + if cached is not None: + return cached + + # ---- diarizing whisper path (faster-whisper + pyannote) ---- + if backend == "whisperx": + wrapper = _load_diarizing_whisper(size, dev, cfg) + if wrapper is not None: + _WHISPER_MODELS[cache_key] = wrapper + return wrapper + # Fall through to plain faster-whisper if diarization + # couldn't initialize. The transcript still gets produced; + # speaker labels just won't be present. + logger.warning( + "diarizing whisper unavailable — falling back to " + "faster-whisper (transcripts will not carry speaker labels)" + ) + + # ---- faster-whisper path ----------------------------------- + if backend in ("auto", "faster"): + logger.info( + "[whisper] entering faster-whisper path: size=%s device=%s", + size, dev, + ) + try: + from faster_whisper import WhisperModel as _FasterWhisper + logger.info("[whisper] faster_whisper module imported") + ct_default = "float16" if dev == "cuda" else "int8" + ct = (cfg.get("whisper_compute_type") or ct_default).strip() + fw_device = "cuda" if dev.startswith("cuda") else dev + if fw_device == "mps": + # CTranslate2 doesn't support MPS yet — fall back + # to CPU int8 rather than refuse. + fw_device, ct = "cpu", "int8" + # Coordinate VRAM with other GPU consumers (Ollama, + # ImageBind, embeddings). Without this, a long-lived + # Ollama LLM with KEEP_ALIVE high enough crowds Whisper + # off the device and the model load silently thrashes. + # See infra/gpu_coordinator.py for the policy. + _vram_estimate = _whisper_vram_mb(size, ct) + try: + # infra/ is added to sys.path as a directory (not a + # package) by server.py, so import as a top-level + # module rather than `infra.gpu_coordinator`. + from gpu_coordinator import gpu_priority + _ctx = gpu_priority( + f"whisper:{size}", needed_mb=_vram_estimate, + ) + logger.info( + "[whisper] gpu_coordinator engaged " + "(needed=%dMB)", _vram_estimate, + ) + except Exception as _ce: + logger.warning( + "[whisper] gpu_coordinator unavailable: %s", _ce, + ) + _ctx = None + logger.info( + "[whisper] about to load %s on %s (%s)", + size, fw_device, ct, + ) + if _ctx is not None: + with _ctx: + m = _FasterWhisper( + size, device=fw_device, compute_type=ct, + ) + else: + m = _FasterWhisper( + size, device=fw_device, compute_type=ct, + ) + logger.info("[whisper] WhisperModel constructed") + wrapper = _WhisperFasterAdapter( + m, + beam_size=int(cfg.get("whisper_beam_size") or 5), + language=cfg.get("whisper_language") or None, + ) + _WHISPER_MODELS[cache_key] = wrapper + logger.info( + "Loaded faster-whisper: %s on %s (%s) — reserved ~%dMB VRAM", + size, fw_device, ct, _vram_estimate, + ) + return wrapper + except ImportError: + if backend == "faster": + logger.warning( + "whisper_backend=faster but faster-whisper is not " + "installed; install with `pip install faster-whisper`" + ) + return None + # auto: fall through to openai-whisper + except Exception as e: + logger.warning( + "faster-whisper load failed (size=%s, device=%s): %s", + size, dev, e, + ) + if backend == "faster": + return None + # auto: fall through + + # ---- openai-whisper path ----------------------------------- + if backend in ("auto", "openai"): + if not _HAS_WHISPER: + return None + try: + kwargs = {} + if dev != "auto": + kwargs["device"] = dev + model = _whisper_module.load_model(size, **kwargs) + _WHISPER_MODELS[cache_key] = model + logger.info( + "Loaded openai-whisper: %s (device=%s)", + size, dev, + ) + return model + except Exception as e: + logger.warning( + "openai-whisper load failed (size=%s, device=%s): %s", + size, dev, e, + ) + return None + + return None + + +# --------------------------------------------------------------------------- +# Diarizing-Whisper loader: faster-whisper for transcription + +# pyannote.audio Pipeline for speaker diarization, both running on the +# pinned torch 2.5.1 stack. The whisperx wrapper package is NOT used +# (its 3.8.x line pins torch 2.8 — incompatible). Gated by HF_TOKEN +# for pyannote's gated models. Without HF_TOKEN, returns a wrapper +# with diarization disabled (transcripts still produced, no speakers). +# --------------------------------------------------------------------------- +def _load_diarizing_whisper(size, dev, cfg): + import os + try: + from faster_whisper import WhisperModel as _FasterWhisper + except ImportError: + logger.warning("faster-whisper not installed") + return None + + hf_token = os.environ.get("HF_TOKEN") or cfg.get("hf_token") or None + + fw_device = "cuda" if dev.startswith("cuda") else dev + if fw_device == "mps": + fw_device = "cpu" + compute_type = (cfg.get("whisper_compute_type") + or ("float16" if fw_device == "cuda" else "int8")).strip() + if fw_device == "cpu" and compute_type.lower().startswith("float"): + compute_type = "int8" + beam_size = int(cfg.get("whisper_beam_size") or 5) + language = cfg.get("whisper_language") or None + + vram_estimate = _whisper_vram_mb(size, compute_type) + 1200 # +diarize overhead + try: + from gpu_coordinator import gpu_priority + ctx = gpu_priority(f"whisper-diar:{size}", needed_mb=vram_estimate) + except Exception: + ctx = None + + def _do_load(): + fw = _FasterWhisper(size, device=fw_device, compute_type=compute_type) + + diarize_pipe = None + if hf_token: + try: + # Compat shim 1: chatterbox-tts pins transformers 5.2.0, + # which requires huggingface_hub >=1.0 (provides + # `is_offline_mode`). pyannote.audio 3.3.2 was written + # against the pre-1.0 API and calls + # `hf_hub_download(use_auth_token=...)` — that kwarg was + # removed in 1.0 (replaced by `token=`). Both packages + # are pinned by upstream needs, so we translate the + # kwarg in-place rather than fight the version graph. + # Idempotent: only wraps once. + import huggingface_hub as _hfh + if not getattr(_hfh.hf_hub_download, "_leos_kw_shim", False): + _orig_hfd = _hfh.hf_hub_download + def _hfd_shim(*args, **kwargs): + if "use_auth_token" in kwargs and "token" not in kwargs: + kwargs["token"] = kwargs.pop("use_auth_token") + return _orig_hfd(*args, **kwargs) + _hfd_shim._leos_kw_shim = True + _hfh.hf_hub_download = _hfd_shim + + # Compat shim 2: speechbrain (pyannote.audio dep) + # registers many optional integrations (k2_fsa, nlp, + # etc.) as LazyModules. pytorch_lightning's + # `inspect.stack()` walks all loaded modules and + # accidentally triggers those lazy loads, which raise + # ImportError when their optional native deps (k2, + # nltk extras, etc.) are missing. Diarization doesn't + # use any of them; turn the failure into a no-op. + import sys as _sys, types as _types + _sys.modules.setdefault("k2", _types.ModuleType("k2")) + try: + from speechbrain.utils import importutils as _sbi + if not getattr(_sbi.LazyModule, "_leos_safe", False): + _orig_ensure = _sbi.LazyModule.ensure_module + def _safe_ensure(self, stacklevel=1): + try: + return _orig_ensure(self, stacklevel + 1) + except ImportError: + # Return an empty stub instead of raising + # — pyannote.audio 3.3.2 never touches + # these optional integrations directly. + stub = _types.ModuleType(self.target) + self.lazy_module = stub + return stub + _sbi.LazyModule.ensure_module = _safe_ensure + _sbi.LazyModule._leos_safe = True + except Exception: + pass # speechbrain not present yet — fine + + from pyannote.audio import Pipeline as _PyannotePipeline + import torch as _torch + diarize_pipe = _PyannotePipeline.from_pretrained( + "pyannote/speaker-diarization-3.1", + use_auth_token=hf_token, + ) + if fw_device.startswith("cuda") and _torch.cuda.is_available(): + diarize_pipe.to(_torch.device("cuda")) + except Exception as e: + logger.warning( + "pyannote diarization pipeline load failed: %s — " + "transcripts will not carry speaker labels. Verify " + "HF_TOKEN is valid and pyannote model terms are " + "accepted on huggingface.co (segmentation-3.0 + " + "speaker-diarization-3.1).", e, + ) + else: + logger.warning( + "HF_TOKEN not set — speaker diarization disabled. " + "Add it via http://localhost:5000/credentials to enable." + ) + + return _DiarizingWhisperAdapter( + fw_model=fw, + diarize_pipeline=diarize_pipe, + beam_size=beam_size, + language=language, + ) + + try: + if ctx is not None: + with ctx: + wrapper = _do_load() + else: + wrapper = _do_load() + logger.info( + "Loaded diarizing whisper: %s on %s (%s) — diarize=%s, ~%dMB VRAM", + size, fw_device, compute_type, + wrapper._diarize is not None, vram_estimate, + ) + return wrapper + except Exception as e: + logger.warning( + "diarizing whisper load failed (size=%s, device=%s): %s", + size, dev, e, + ) + return None diff --git a/project/knowledge/media_ingest_queue.py b/project/knowledge/media_ingest_queue.py index 71e27ab..4024c29 100644 --- a/project/knowledge/media_ingest_queue.py +++ b/project/knowledge/media_ingest_queue.py @@ -23,7 +23,6 @@ logger = logging.getLogger("media_ingest") -# Additional imports needed by this module (Phase 5 post-split fix) import media_library diff --git a/project/knowledge/media_ingest_review.py b/project/knowledge/media_ingest_review.py index d2cd426..4d5d14e 100644 --- a/project/knowledge/media_ingest_review.py +++ b/project/knowledge/media_ingest_review.py @@ -1,861 +1,623 @@ """ -media_ingest_review.py - Post-ingestion review scheduling. - -Split from media_ingest.py during Phase 5 of the monolith-breakdown plan. -See 24__monolith_breakdown_plan.md for context. - -After a record is ingested, this module decides whether it deserves -active agent review (looks interesting, is memetic, came from a monitored -source) and builds a review task with a goal + tool plan. - -Public functions: - - _schedule_media_review(record_id, ...) - ensure_media_project(...) -- ensure the media-processing project - scope exists +media_ingest_review.py — Staged insight extraction pipeline. + +Replaces the multi-round director_run agent with a deterministic 7-stage +pipeline that produces structured, hallucination-resistant analysis of +podcast transcripts: + + Stage 0 semantic topic segmentation (no LLM) + Stage 1 per-segment atomic extraction (1 LLM call/segment, JSON-mode) + Stage 2 episode-level reduce (1 LLM call, JSON-mode) + Stage 3 RAPTOR tree (~3-5 LLM calls) + Stage 4 auto-tag (no LLM, cosine sim vs vocab) + Stage 5 cross-episode linking (no LLM, leos_search) + Stage 6 knowledge graph emission (no LLM, deterministic) + Stage 7 structured KB save (no LLM, structured fields) + +All stages write into ``record["analysis"]["review"]``. No agent +session is spawned, no director_run is scheduled. The serial-mode +gate doesn't apply because there's no idle review task to wait on — +this runs synchronously inside the audio pipeline worker thread. + +References: + - RAPTOR (arxiv 2401.18059) + - LLM×MapReduce (ACL 2025.acl-long.1341) + - AFEV (arxiv 2506.07446) + - PODTILE / multi-level transcript segmentation (Interspeech 2025) + - Constrained JSON-mode for hallucination mitigation (n1n.ai 2026) """ - -import json import logging -import os +import time -logger = logging.getLogger("media_ingest") - -# Additional imports needed by this module (Phase 5 post-split fix) import media_library +logger = logging.getLogger("media_ingest") -# --------------------------------------------------------------------------- -# Job Queue -# -# All media processing runs through a single-worker queue to prevent -# concurrent jobs from fighting over YOLO models, Whisper, VRAM, CPU, -# and disk I/O. When you submit a new file or URL while something is -# already processing, it simply gets added to the queue and waits its -# turn. The UI polls status and will show "queued" for waiting items. -# --------------------------------------------------------------------------- - - -# Review idle-gate tunables -_REVIEW_IDLE_GATE = 120 - -# Max chars of text content to include in the goal prompt. -_REVIEW_PREVIEW_CHARS = 3000 - -# MEDIA_PROJECT_ID constant -MEDIA_PROJECT_ID = "_media-processing" - -def ensure_media_project(workspace_root=None): - """ - Public entry point: create the media processing project if it - doesn't exist yet. Called from server.py at startup and also - lazily from _schedule_media_review on first use. - - Args: - workspace_root: Path to the projects directory. If None, - we try to find it from state.py. - """ - if workspace_root is None: - try: - from state import WORKSPACE - workspace_root = WORKSPACE - except (ImportError, AttributeError): - logger.debug("Cannot ensure media project: no workspace_root") - return False - - return _ensure_media_project(workspace_root) - -def _ensure_media_project(workspace_root): - """ - Make sure the _media-processing project exists on disk. - Creates it with a purpose-built team if missing. - - Team: - Director -- coordinates the review, delegates to specialists - Analyst -- deep visual/content analysis using vision + media tools - Researcher -- web research on topics discovered in the media - Librarian -- documents findings into KB, cross-links references - """ - from datetime import datetime, timezone - - project_path = os.path.join(workspace_root, MEDIA_PROJECT_ID) - project_file = os.path.join(project_path, "project.json") - - if os.path.isdir(project_path) and os.path.isfile(project_file): - return True # Already exists - - logger.info("Creating media processing project at %s", project_path) - - agents = [ - # ---- Director: coordinates the review cycle ---- - { - "name": "Director", - "role": "director", - "description": ( - "Coordinates media review sessions. Examines what the " - "pipeline found, then delegates deep analysis, research, " - "and documentation to the specialist agents." - ), - "system_message": ( - "You are the Director of the media processing project. " - "When a new piece of media is ingested, you coordinate a " - "team to extract maximum value from it.\n\n" - "YOUR TEAM:\n" - "- Analyst: Deep visual/content analysis, data extraction, " - "chart reading, code interpretation\n" - "- Researcher: Web research on topics, people, places, or " - "technologies found in the media\n" - "- Librarian: Documents findings in the KB, creates " - "references, cross-links with existing knowledge\n\n" - "WORKFLOW:\n" - "1. Read the media record with media_search action='get'\n" - "2. Assess what kind of content it is and what's valuable\n" - "3. Delegate to Analyst for deep content examination\n" - "4. If topics need external context, delegate to Researcher\n" - "5. Delegate to Librarian to document everything found\n" - "6. Say DONE with a summary of what was extracted\n\n" - "DELEGATION FORMAT:\n" - "ACTION: DELEGATE\nAGENT: \n" - "TASK: \nREASON: \n\n" - "COMPLETION FORMAT:\n" - "ACTION: DONE\nSUMMARY: \n\n" - "RULES:\n" - "- Quality over speed. Extract real knowledge.\n" - "- Not every media item needs all agents. A simple photo " - "might only need Analyst + Librarian.\n" - "- Videos with transcripts are goldmines -- extract key " - "insights, techniques, and facts.\n" - "- Charts and diagrams contain structured data -- have " - "Analyst extract it.\n" - "- Max 12 delegations per review session.\n" - "- Always end with Librarian documenting findings." - ), - "tools": [ - "media_search", "kb_topics", "kb_search", - "inbox_send", "issue_query", - ], - "max_history": 30, - "temperature": 0.3, - }, - - # ---- Analyst: deep content examination ---- - { - "name": "Analyst", - "role": "analyst", - "description": ( - "Deep media analysis specialist. Uses vision models, OCR, " - "and content tools to thoroughly examine media content and " - "extract structured information." - ), - "system_message": ( - "You are the Analyst in the media processing project. " - "Your job is to deeply examine media content and extract " - "every piece of useful information.\n\n" - "CAPABILITIES:\n" - "- media_search: Load full records, search by content\n" - "- image_analyze: Classify, detect objects, segment, " - "describe images using YOLO + OpenCV\n" - "- image_ocr: Extract text from images with preprocessing\n" - "- video_info: Get video metadata, extract frames\n" - "- audio_process: Get audio metadata, transcribe\n\n" - "WHAT TO EXTRACT:\n" - "- For images: What's shown, any text, data from charts/" - "graphs, notable details, style/quality assessment\n" - "- For videos: Key topics from transcript, important " - "visual moments, on-screen text/code, people/speakers\n" - "- For audio: Speaker topics, key quotes, technical " - "terminology, actionable information\n" - "- For documents: Structure, key findings, data tables, " - "conclusions, references\n\n" - "OUTPUT: Report your findings clearly so the Librarian " - "can create proper KB entries. Include specific facts, " - "data points, and quotes (with timestamps for video/audio).\n\n" - "RULES:\n" - "- Be thorough but focused on genuinely useful content\n" - "- Extract structured data from charts/tables when possible\n" - "- Note timestamps for important moments in video/audio\n" - "- Distinguish facts from opinions in the content" - ), - "tools": [ - "media_search", - "image_analyze", "image_ocr", - "image_upscale", "image_edit", - "video_info", "audio_process", - "file_read", "file_write", - "kb_search", - ], - "max_history": 40, - "temperature": 0.3, - }, - - # ---- Researcher: web research on topics found ---- - { - "name": "Researcher", - "role": "researcher", - "description": ( - "Researches topics, people, technologies, and references " - "discovered during media analysis. Adds context and " - "background that enriches the extracted knowledge." - ), - "system_message": ( - "You are the Researcher in the media processing project. " - "After the Analyst examines media content, you research " - "the topics found to add context and depth.\n\n" - "WHAT TO RESEARCH:\n" - "- Technologies or tools mentioned in tutorials/talks\n" - "- People, companies, or organizations referenced\n" - "- Technical concepts that need explanation\n" - "- URLs, projects, or resources mentioned\n" - "- Historical context for events or developments\n\n" - "TOOLS:\n" - "- web_search + web_fetch for external research\n" - "- kb_search to check what we already know\n" - "- ref_lookup to find existing references\n" - "- ref_add to save useful URLs and resources\n\n" - "OUTPUT: Concise research summaries with sources. " - "Focus on facts that add value to what the Analyst found. " - "Don't repeat what's already in the KB.\n\n" - "RULES:\n" - "- Only research topics that genuinely need context\n" - "- Check KB first to avoid duplicate research\n" - "- Save useful URLs to the reference library\n" - "- Keep summaries focused and factual" - ), - "tools": [ - "web_search", "web_fetch", - "media_search", "kb_search", "kb_save", - "ref_lookup", "ref_add", - ], - "max_history": 30, - "temperature": 0.3, - }, - - # ---- Librarian: documents everything into KB ---- - { - "name": "Librarian", - "role": "librarian", - "description": ( - "Documents media analysis findings into the knowledgebase. " - "Creates well-tagged, cross-linked entries that make the " - "extracted knowledge findable and useful for other agents." - ), - "system_message": ( - "You are the Librarian in the media processing project. " - "Your job is to take the Analyst's findings and the " - "Researcher's context and create proper KB entries.\n\n" - "WHAT TO CREATE:\n" - "- One main entry per media item summarizing what it is\n" - "- Separate entries for distinct facts, techniques, or " - "insights extracted from the content\n" - "- Reference library entries for URLs and resources found\n\n" - "KB ENTRY QUALITY:\n" - "- Clear, specific titles (not 'Video Notes')\n" - "- Proper topic tags (lowercase-with-hyphens)\n" - "- Set importance: 3-4 for useful facts, 5+ for key insights\n" - "- Include media_refs to link back to the source record\n" - "- Cross-link with existing KB entries on the same topics\n\n" - "TOOLS:\n" - "- kb_save: Create new entries (include media_refs!)\n" - "- kb_search: Find related existing entries to cross-link\n" - "- kb_update: Add cross-references to existing entries\n" - "- ref_add: Save useful URLs to reference library\n" - "- ref_lookup: Check for existing references\n\n" - "MEDIA_REFS FORMAT:\n" - "When saving KB entries, include media_refs=[record_id] " - "to create a bidirectional link between the KB entry and " - "the media record. This is critical for traceability.\n\n" - "RULES:\n" - "- Every KB entry from media MUST have media_refs set\n" - "- Don't create entries for trivial observations\n" - "- Merge with existing entries when appropriate\n" - "- Use specific topic tags, not 'media' or 'misc'\n" - "- Quality over quantity -- fewer good entries beat many bad ones" - ), - "tools": [ - "kb_topics", "kb_search", "kb_save", "kb_update", "kb_delete", - "kb_maintenance", - "ref_lookup", "ref_add", "ref_update", - "media_search", - ], - "max_history": 40, - "temperature": 0.3, - }, - ] +def _config(): try: - os.makedirs(project_path, exist_ok=True) - os.makedirs(os.path.join(project_path, "history"), exist_ok=True) - os.makedirs(os.path.join(project_path, "files"), exist_ok=True) - os.makedirs(os.path.join(project_path, "skills"), exist_ok=True) - os.makedirs(os.path.join(project_path, "postmortems"), exist_ok=True) - os.makedirs(os.path.join(project_path, "postmortems", "processed"), - exist_ok=True) - - project_data = { - "project_id": MEDIA_PROJECT_ID, - "name": "Media Processing", - "description": ( - "Automated media analysis project. When media is added to " - "the library, an agent team reviews it: the Analyst examines " - "content in depth, the Researcher adds context from the web, " - "and the Librarian documents findings in the knowledgebase. " - "Runs during idle time. Managed by the system -- do not delete." - ), - "created": datetime.now(timezone.utc).isoformat(), - "status": "active", - "system_project": True, - "agents": agents, - "default_agent": "Director", - "notes": "Auto-created by the media processing system.", - } - - with open(project_file, "w", encoding="utf-8") as f: - json.dump(project_data, f, indent=4, ensure_ascii=False) - - # Copy skill templates if available - try: - import shutil as _shutil - skill_templates = os.path.join( - os.path.dirname(workspace_root), "skill-templates" - ) - skills_dir = os.path.join(project_path, "skills") - if os.path.isdir(skill_templates): - for name in os.listdir(skill_templates): - src = os.path.join(skill_templates, name) - dst = os.path.join(skills_dir, name) - if os.path.isdir(src): - if os.path.exists(dst): - _shutil.rmtree(dst) - _shutil.copytree(src, dst) - except Exception: - pass # Skills are nice-to-have, not critical - - logger.info("Created media processing project at %s", project_path) - return True + from leos_orchestrator import load_global_config + return (load_global_config() or {}).get("media", {}) + except Exception: + return {} - except Exception as e: - logger.error("Failed to create media processing project: %s", e) - return False -def _schedule_media_review(record_id): - """ - Notify the appropriate Director about new media that needs review. - - Two modes depending on context: - - 1. PROJECT CONTEXT (record has a project_id): - The media was ingested during an active project. Drop an - agent message in that project's files/ directory so the - Director picks it up on the next round. The Director then - delegates to the right agent on the team (Analyst, Coder, - Researcher -- whoever fits). If the session has already - ended, the message waits as a pending message for next time. - - 2. MANUAL DROP (no project_id): - No project context -- route to the dedicated _media-processing - project with an idle-gated director_run task so it doesn't - interrupt anything. - """ - record = media_library.get_record(record_id) - if not record: - return - - title = record.get("title") or record.get("filename") or record_id - media_type = record.get("type", "unknown") - analysis = record.get("analysis", {}) - tags = record.get("tags", []) - source_project = record.get("project_id") - - if source_project: - # ----------------------------------------------------------- - # PROJECT CONTEXT: send an agent message to the Director - # ----------------------------------------------------------- - _send_media_message_to_director( - record_id, source_project, title, media_type, analysis, tags - ) - else: - # ----------------------------------------------------------- - # MANUAL DROP: schedule a task in _media-processing - # ----------------------------------------------------------- - _schedule_media_review_task( - record_id, title, media_type, analysis, tags - ) - -def _send_media_message_to_director(record_id, project_id, - title, media_type, analysis, tags): - """ - Drop an agent message into the project's files/ directory so the - Director picks it up on the next round and delegates the review - to the appropriate agent on the team. +def _kb_save(title, summary, content, tags, refs, project_id=None): + """Save a single KB article via leOS's KnowledgeBase singleton. - This keeps the work inside the project's existing team and context, - rather than spinning up a separate session. + Hallucinated URLs are impossible because URLs come from ``refs`` + which we built from the record's own metadata. """ try: - from state import WORKSPACE - except ImportError: - logger.debug("Cannot send media message: state module not available") - return - - files_path = os.path.join(WORKSPACE, project_id, "files") - os.makedirs(files_path, exist_ok=True) - - # Build a concise but informative message for the Director. - lines = [ - f"New {media_type} has been ingested and analyzed by the pipeline.", - f"", - f"TITLE: {title}", - f"RECORD ID: {record_id}", - f"TYPE: {media_type}", - ] - - if tags: - lines.append(f"TAGS: {', '.join(tags)}") - - # Add key metadata hints so the Director knows what's worth examining - if media_type == "image": - cls = analysis.get("classification", []) - if cls and isinstance(cls, list) and isinstance(cls[0], dict): - lines.append(f"Classification: {cls[0].get('class', '?')}") - ocr = analysis.get("ocr_text", "") - if ocr: - lines.append(f"OCR text: {len(ocr)} chars found") - segs = analysis.get("segmentation", []) - if segs: - dominant = [f"{s['class']} ({s['area_pct']}%)" for s in segs[:3] - if s.get("area_pct", 0) > 5] - if dominant: - lines.append(f"Scene: {', '.join(dominant)}") - - elif media_type == "video": - meta = analysis.get("metadata", {}) - if meta.get("duration_str"): - lines.append(f"Duration: {meta['duration_str']}") - transcript = analysis.get("transcript", "") - if transcript: - lines.append(f"Transcript: {len(transcript)} chars " - f"(~{len(transcript.split())} words)") - ocr = analysis.get("ocr_text", "") - if ocr: - lines.append(f"On-screen text: {len(ocr)} chars") - - elif media_type == "audio": - transcript = analysis.get("transcript", "") - if transcript: - lines.append(f"Transcript: {len(transcript)} chars") - - elif media_type == "document": - content = analysis.get("content_text", "") - if content: - lines.append(f"Content: {len(content)} chars, " - f"{len(content.split())} words") - - lines.extend([ - "", - "ACTION NEEDED:", - f"Use media_search action='get' id='{record_id}' to load the full", - "record, then delegate to the appropriate agent to examine the", - "content and extract useful knowledge into the KB.", - "", - "Consider which agent is best suited:", - "- Analyst/Researcher for deep content examination", - "- Coder for code-related content", - "- Writer for documentation or article content", - "- Or create a specialist if none of the team fits", - ]) - - content = "\n".join(lines) - - # Write the message file in the same format as agent_msg.py - import time as _time - timestamp = _time.strftime("%Y%m%d_%H%M%S") - filename = f"msg_media_pipeline_to_director_{timestamp}_{record_id[:8]}.json" - filepath = os.path.join(files_path, filename) - - msg_data = { - "type": "agent_message", - "from": "Media Pipeline", - "to": "Director", - "subject": f"New {media_type} ready for review: {title[:60]}", + from knowledge_base import get_kb + except Exception as e: + logger.warning("kb save: knowledge_base unavailable (%s)", e) + return None + media_refs = [r["id"] for r in (refs or []) if r.get("kind") == "media_record" and r.get("id")] + references = [r["url"] for r in (refs or []) if r.get("url")] + article = { + "title": title, + "summary": summary, "content": content, - "timestamp": _time.strftime("%Y-%m-%d %H:%M:%S"), - "media_record_id": record_id, + "tags": list(tags or []), + "media_refs": media_refs, + "references": references, + "subject": "media review", } - try: - import json as _json - with open(filepath, "w", encoding="utf-8") as f: - _json.dump(msg_data, f, indent=2) - - # Tag the record so we know a message was sent - media_library.update_record(record_id, { - "pending_review": True, - "review_project": project_id, - "review_method": "agent_message", - }) - - logger.info("Sent media review message to Director in project %s " - "for %s '%s'", project_id, media_type, title[:60]) - - # Also broadcast to the comms feed so the UI shows it in real time - try: - from sse import broadcast_comms - broadcast_comms(project_id, { - "type": "agent_direct_message", - "from_agent": "Media Pipeline", - "to_agent": "Director", - "subject": f"New {media_type} ready for review: {title[:60]}", - "content": content[:300], - "timestamp": msg_data["timestamp"], - }) - except Exception: - pass # Don't let comms failure break the flow - + kb = get_kb() + result = kb.save_article(article, author="review") except Exception as e: - logger.warning("Failed to send media message for %s: %s", - record_id, e) - # Fall back to scheduling a task - _schedule_media_review_task( - record_id, title, media_type, - analysis, tags - ) + logger.warning("kb save raised: %s", e) + return None + if not isinstance(result, dict) or not result.get("ok"): + err = result.get("error") if isinstance(result, dict) else str(result) + logger.warning("kb save returned not-ok: %s", err) + return None + return result.get("article_id") -def _schedule_media_review_task(record_id, title, media_type, analysis, tags): - """ - Schedule a director_run task in the _media-processing project. - Used for media dropped in without a project context. Waits for - idle time before running so it doesn't interrupt active work. +def _vector_or_none(text): + try: + import embeddings + v = embeddings.embed_text(text) + return list(map(float, v)) if v is not None else None + except Exception: + return None + + +def _suggest_tags(text_vec, source_text: str = ""): + """Auto-tag via tiered cosine similarity against the topic vocabulary. + + Uses ``suggest_tags_tiered`` (three deterministic cosine passes, no LLM) + so that narrative articles that fall below the strict 0.50 floor still + receive at least one retrieval anchor when a centroid scores ≥ 0.40. + + Tier summary (see topic_vocabulary.suggest_tags_tiered for full detail): + Tier 1 — threshold 0.50, relative_margin 0.85 (strict, multi-tag) + Tier 2 — threshold 0.45, relative_margin 0.92 (relaxed, tighter margin) + Tier 3 — threshold 0.40, single best tag, no margin filter + + Returns [] only when the best centroid score is < 0.40. In that case the + optional LLM fallback below can be enabled via config key + ``media.topic_llm_fallback: true`` to do a single constrained ASSIST call + asking the model to pick 1–2 tags from the vocabulary list. The LLM path + is disabled by default because the deterministic tiers handle ≥ 95 % of + articles and the LLM adds ~1–3 s of latency per affected article. """ + if text_vec is None: + return [] try: - from state import get_scheduler - except ImportError: - logger.debug("Cannot schedule review: state module not available") - return + from topic_vocabulary import suggest_tags_tiered + return suggest_tags_tiered(text_vec, source_text=source_text, + max_tags=6) + except Exception as e: + logger.debug("suggest_tags unavailable: %s", e) + return [] - scheduler = get_scheduler() - if scheduler is None: - logger.debug("Cannot schedule review: scheduler not initialized") - return - project_id = MEDIA_PROJECT_ID - ensure_media_project() +def _suggest_tags_llm_fallback(summary: str, record_id: str) -> list: + """LLM safety-net for articles where all cosine tiers return []. - goal = _build_review_goal(record_id, title, media_type, analysis, tags) + Sends a single constrained ASSIST call with the full topic vocabulary as + a numbered list and asks the model to return 1–2 tag indices. The + response is validated against the vocabulary — no free-form tags, so + hallucination is structurally impossible. + Enable with ``media.topic_llm_fallback: true`` in config.json. + Disabled by default; ~1–3 s and ~$0.001–0.01 per affected article. + """ try: - task_id = scheduler.create_task( - task_type="pipeline", - project_id=project_id, - name=f"Review: {title[:60]}", - description=( - f"Agent review of {media_type} '{title}' from the media library. " - f"Will examine the content and extract useful information " - f"into the KB, reference library, or project plans." - ), - schedule={}, # Run as soon as conditions are met - steps=[ - { - "type": "director_run", - "project_id": project_id, - "goal": goal, - "max_rounds": 15, - "idle_gate_seconds": _REVIEW_IDLE_GATE, - "condition": { - "type": "no_active_session", - }, - }, - { - "type": "inbox", - "subject": f"Media reviewed: {title[:60]}", - "content": ( - f"An agent has finished reviewing {media_type} " - f"'{title}' from the media library (record " - f"{record_id}). Check the KB and reference library " - f"for any new entries." - ), - "category": "media_review", - "priority": "low", - }, - ], - created_by="media_ingest", + from topic_vocabulary import SEED_TOPICS + from models.ollama_driver import get_driver + labels = [ + (entry[0] if isinstance(entry, tuple) else entry) + for entry in SEED_TOPICS + ] + numbered = "\n".join(f"{i}. {lbl}" for i, lbl in enumerate(labels)) + prompt = ( + "You are a topic tagger. Given the article summary below, " + "return ONLY a JSON array of 1–2 integers that are the indices " + "of the best-matching topic labels from the numbered list. " + "Return [] if none fit.\n\n" + f"TOPIC LIST:\n{numbered}\n\n" + f"ARTICLE SUMMARY:\n{summary[:1200]}" ) + driver = get_driver() + raw = driver.generate(prompt, max_tokens=32, temperature=0.0) + import json as _json + indices = _json.loads(raw.strip()) + if not isinstance(indices, list): + return [] + result = [] + for idx in indices: + if isinstance(idx, int) and 0 <= idx < len(labels): + result.append((labels[idx], 0.0)) # score 0.0 = LLM-assigned + logger.info( + "[%s] stage4 LLM fallback: %d tag(s) (%s)", + record_id, len(result), + ", ".join(t[0] for t in result) if result else "(none)", + ) + return result + except Exception as e: + logger.debug("[%s] stage4 LLM fallback failed: %s", record_id, e) + return [] - media_library.update_record(record_id, { - "review_task_id": task_id, - "pending_review": True, - "review_method": "scheduled_task", + +def _link_similar_episodes(record_id, text_vec, top_k=5): + """Use leOS's media library search to find related ingested + episodes by transcript similarity. Returns list of {record_id, + title, score}.""" + if text_vec is None: + return [] + try: + results = media_library.search_by_embedding( + text_vec, media_type="audio", top_k=top_k + 1, + ) + except Exception as e: + logger.debug("link search failed: %s", e) + return [] + out = [] + for item in results or []: + # search_by_embedding returns list of (record, score) tuples, + # not dicts — unpack accordingly, but stay defensive in case + # the upstream signature changes back to dicts. + if isinstance(item, tuple) and len(item) == 2: + rec, score = item + elif isinstance(item, dict): + rec, score = item, item.get("score", 0.0) + else: + continue + rid = rec.get("id") if isinstance(rec, dict) else None + if not rid or rid == record_id: + continue + out.append({ + "record_id": rid, + "title": (rec.get("title") or "")[:160], + "score": float(score or 0.0), }) + if len(out) >= top_k: + break + return out - logger.info("Scheduled idle review for %s -> task %s (project: %s)", - record_id, task_id, project_id) - except Exception as e: - logger.warning("Failed to schedule media review for %s: %s", - record_id, e) +def run_review(record_id): + """Execute the full 7-stage v2 review on the given media record. -def _build_review_goal(record_id, title, media_type, analysis, tags): + Returns the ``review`` dict that was attached to the record's + analysis, or None on failure. """ - Build a type-appropriate goal prompt for the Director session - that will review this media record. + t0 = time.time() + record = media_library.get_record(record_id) + if not record: + logger.warning("review: record %s not found", record_id) + return None + transcript = (record.get("analysis") or {}).get("transcript") or "" + if not transcript: + logger.info("review: %s has no transcript, skipping", record_id) + return None - Content (transcripts, OCR text, document text) is NOT embedded - in the goal -- it's available on disk via media_search action='get'. - The goal only includes cheap metadata so the agent knows what kind - of thing it's looking at and can pull the full data itself. + cfg = _config() + model = cfg.get("review_model") # None = use default driver model - All KB entries created during review should use media_refs to - link back to this record, which auto-syncs a back-link on the - media record. This creates a bidirectional knowledge graph. - """ - # --------------------------------------------------------------- - # Header -- common to all types - # --------------------------------------------------------------- - lines = [ - f"A {media_type} was added to the media library and needs review.", - f"", - f"TITLE: {title}", - f"RECORD ID: {record_id}", - f"TYPE: {media_type}", - f"TAGS: {', '.join(tags)}", - f"", - f"STEP 1: Use media_search action='get' id='{record_id}' to load", - f" the full record -- transcripts, OCR text, descriptions,", - f" keyframe info, child records, and all analysis results.", - f"", - ] - - # --------------------------------------------------------------- - # Quick metadata hints -- just enough to orient, not content - # --------------------------------------------------------------- - if media_type == "image": - lines.append("--- QUICK SUMMARY ---") - - cls = analysis.get("classification", []) - if cls: - top = cls[0] if isinstance(cls, list) else cls - if isinstance(top, dict): - lines.append(f"Classification: {top.get('class', '?')} " - f"({top.get('confidence', 0):.0%})") - - objs = analysis.get("objects_detected", []) - if objs: - obj_counts = {} - for o in objs: - c = o.get("class", "?") - obj_counts[c] = obj_counts.get(c, 0) + 1 - obj_str = ", ".join( - f"{n}x {c}" if n > 1 else c for c, n in obj_counts.items() + # Stage 0 — semantic topic segmentation + try: + from transcript_segmenter import segment_transcript + segments = segment_transcript(transcript) + except Exception as e: + logger.warning("review stage0 failed: %s", e) + segments = [] + if not segments: + logger.warning("review: %s produced no segments", record_id) + return None + logger.info( + "[%s] review stage0: %d segments (%s)", record_id, + len(segments), segments[0].get("method"), + ) + + # Single OllamaDriver instance reused across all 12-17 LLM calls + # (Stage 1 per-segment, Stage 2 reduce, Stage 3 RAPTOR). + # + # warmup+pin only when get_keep_alive_override() == 0 (GPU-priority + # lease active — Whisper is transcribing and would evict the LLM + # between segments). When the override is None, Ollama's default + # 5-minute keep_alive already holds the model warm for the review's + # full duration; a warmup call adds ~5-10s for nothing. Released + # in the finally block only when pinned. + import ollama_driver + driver = ollama_driver.OllamaDriver() + pin_model = model or driver.default_model + pinned = False + if ollama_driver.get_keep_alive_override() == 0: + warm = driver.warmup(model=pin_model, keep_alive=-1) + if warm.get("ok"): + pinned = True + logger.info( + "[%s] review: warmed %s in %.1fs (pinned, GPU-priority lease active)", + record_id, pin_model, warm.get("elapsed_s") or 0, + ) + else: + logger.warning( + "[%s] review: warmup failed (%s) — proceeding without pin", + record_id, warm.get("error"), ) - lines.append(f"Objects: {obj_str}") - segs = analysis.get("segmentation", []) - if segs: - seg_str = ", ".join( - f"{s['class']} ({s['area_pct']}%)" for s in segs[:5] - if s.get("area_pct", 0) > 3 + try: + # Stage 1 — per-segment atomic extraction. Parallelised via + # ThreadPoolExecutor (OLLAMA_NUM_PARALLEL=3); order is preserved + # by index slot-fill on the result list. Failures are logged + # but never raise — partial results survive. + # + # When the GPU-priority lease is active (keep_alive=0), parallel + # calls each pay a cold-load tax that cancels the speedup; degrade + # to serial in that case. + from atomic_extractor import extract_segment, reduce_episode + from concurrent.futures import ThreadPoolExecutor, as_completed + + # Build a char-index over diarization segments so each Stage 0 + # text segment can declare which real speakers are present. + # Diarization segments come with time stamps but not char + # positions — we reconstruct char positions by walking the + # same `" ".join(text)` join that produced the flat transcript. + analysis = record.get("analysis") or {} + ts_segments = analysis.get("transcript_segments") or [] + speakers_resolved = analysis.get("speakers_resolved") or [] + spk_to_name = {} + for sr in speakers_resolved: + label = sr.get("name") or sr.get("id") + role = sr.get("role") + if role and sr.get("name"): + label = f"{sr['name']} ({role})" + if sr.get("id"): + spk_to_name[sr["id"]] = label + + ts_with_chars = [] + _pos = 0 + for t in ts_segments: + txt = (t.get("text") or "").strip() + ts_with_chars.append({ + "start_char": _pos, + "end_char": _pos + len(txt), + "speaker": t.get("speaker"), + }) + _pos += len(txt) + 1 # +1 for the space joiner + + def _speakers_in(start_c, end_c): + found = [] + for t in ts_with_chars: + if t["end_char"] < start_c or t["start_char"] > end_c: + continue + spk = t.get("speaker") + if spk and spk not in found: + found.append(spk) + return [spk_to_name.get(s, s) for s in found] + + def _extract_one(packed): + i, seg = packed + s_t0 = time.time() + # Build a speaker hint header so the LLM uses real names in + # claims_made. When diarization didn't run or didn't find + # multiple speakers, we send the raw text and let the LLM + # infer from textual cues as before. + speakers = _speakers_in( + seg.get("start_char", 0), seg.get("end_char", 0), + ) if ts_with_chars else [] + if speakers: + hint = ( + "Speakers present in this segment: " + + ", ".join(speakers) + + ".\nUse these exact names for the `speaker` field " + "of claims_made. When the speaker is unclear from " + "the text, omit `speaker` rather than guess.\n\n" + ) + seg_text = hint + (seg.get("text") or "") + else: + seg_text = seg.get("text") or "" + try: + ext = extract_segment(seg_text, model=model, driver=driver) + except Exception as e: + logger.warning( + "review segment %d extract failed: %s", + seg.get("index"), e, + ) + ext = {} + ext["index"] = seg.get("index") + ext["start_char"] = seg.get("start_char") + ext["end_char"] = seg.get("end_char") + ext["elapsed_s"] = round(time.time() - s_t0, 2) + return i, ext + + if ollama_driver.get_keep_alive_override() == 0: + stage1_workers = 1 + logger.info( + "[%s] review stage1: GPU-priority lease active — serial to avoid cold-load tax", + record_id, ) - if seg_str: - lines.append(f"Segmentation (coverage): {seg_str}") + else: + stage1_workers = int(cfg.get("stage1_workers") or 3) + + per_segment = [None] * len(segments) + with ThreadPoolExecutor(max_workers=stage1_workers) as pool: + fmap = { + pool.submit(_extract_one, (i, seg)): i + for i, seg in enumerate(segments) + } + for fut in as_completed(fmap): + pos, ext = fut.result() + per_segment[pos] = ext + # Defensive: any None (shouldn't happen since _extract_one always + # returns) gets a placeholder so reduce_episode doesn't choke. + for i, ps in enumerate(per_segment): + if ps is None: + per_segment[i] = { + "index": segments[i].get("index"), + "start_char": segments[i].get("start_char"), + "end_char": segments[i].get("end_char"), + "elapsed_s": 0.0, + } + logger.info( + "[%s] review stage1: %d segment extractions " + "(workers=%d, avg %.1fs each)", + record_id, len(per_segment), stage1_workers, + sum(s.get("elapsed_s", 0) for s in per_segment) / max(len(per_segment), 1), + ) - faces = analysis.get("faces_detected", 0) - if faces: - lines.append(f"Faces: {faces}") + # Stage 2 — episode-level reduce + try: + episode = reduce_episode(per_segment, model=model, driver=driver) + except Exception as e: + logger.warning("review stage2 failed: %s", e) + episode = { + "summary": "", + "themes": [], + "facts": [], + "entities": [], + "questions": [], + "claims": [], + } + logger.info( + "[%s] review stage2: episode summary=%d chars, %d facts, " + "%d entities, %d themes", + record_id, len(episode.get("summary") or ""), + len(episode.get("facts") or []), + len(episode.get("entities") or []), + len(episode.get("themes") or []), + ) - ocr = analysis.get("ocr_text", "") - if ocr: - lines.append(f"OCR text found: {len(ocr)} chars (read via media_search)") - else: - lines.append("No text detected in image.") - - desc = analysis.get("description", "") - if desc: - lines.append(f"Auto-description: {desc}") - - lines.append("---") - lines.append("") - - lines.extend([ - "REVIEW ACTIONS for images:", - "1. LOOK AT IT: Use vision_describe to examine the actual image", - " with the VL model. The ML analysis above gives labels and", - " bounding boxes, but vision_describe gives you real understanding", - " (chart interpretation, UI context, code reading, etc.).", - " Example: vision_describe input='' context='ML found: ...'", - "2. CATALOGUE: KB entry describing what this image shows and", - " what context it's useful in.", - "3. EXTRACT DATA: If chart/graph/diagram, extract the data into", - " structured KB entries.", - "4. EXTRACT TEXT: If OCR found substantial text, create KB entries", - " from that content.", - "5. VISUAL STYLE: Note aesthetics, quality, style (professional,", - " casual, generated, hand-drawn).", - "6. CONTEXT: Recognizable locations, logos, UI patterns, cultural", - " references -- document them.", - ]) - - elif media_type == "video": - lines.append("--- QUICK SUMMARY ---") - - meta = analysis.get("metadata", {}) - if meta: - dur = meta.get("duration_str", "") - res = meta.get("resolution", "") - if dur: - lines.append(f"Duration: {dur}") - if res: - lines.append(f"Resolution: {res}") - - # Note that the video file has been deleted - if analysis.get("video_cleaned"): - freed = analysis.get("bytes_freed", 0) - freed_mb = freed / (1024 * 1024) if freed else 0 - lines.append(f"NOTE: Original video deleted ({freed_mb:.0f} MB freed).") - lines.append("Keyframes, audio track, and transcript are preserved.") - - # Per-keyframe analysis summaries - kf_analysis = analysis.get("keyframes", []) - if kf_analysis: - lines.append(f"Keyframes: {len(kf_analysis)} frames with ML analysis") - # Show first few frame descriptions as preview - for kf in kf_analysis[:3]: - desc = kf.get("description", "") - if desc: - fn = kf.get("frame_number", "?") - lines.append(f" Frame {fn}: {desc[:120]}") - if len(kf_analysis) > 3: - lines.append(f" ... and {len(kf_analysis) - 3} more frames") - - transcript = analysis.get("transcript", "") - if transcript: - lines.append(f"Transcript: {len(transcript)} chars available " - f"(~{len(transcript.split())} words, read via media_search)") - else: - lines.append("No transcript (no speech detected).") - - ocr = analysis.get("ocr_text", "") - if ocr: - lines.append(f"On-screen text found: {len(ocr)} chars (slides/code/overlays)") - - seg_summary = analysis.get("segmentation_summary", []) - if seg_summary: - lines.append(f"Scene composition: {'; '.join(seg_summary[:3])}") - - faces = analysis.get("faces_detected", 0) - if faces: - lines.append(f"Faces detected: {faces}") - - desc = analysis.get("description", "") - if desc: - lines.append(f"Auto-description: {desc}") - - lines.append("---") - lines.append("") - - lines.extend([ - "REVIEW ACTIONS for video:", - "1. READ THE FULL TRANSCRIPT via media_search before anything else.", - "2. LOOK AT KEYFRAMES: Use vision_describe on 3-5 keyframes to", - " actually see what the video shows. The original video has been", - " deleted -- these keyframes are the ONLY visual record.", - "", - " Each keyframe has ML analysis stored (use media_search action='get'", - " to see analysis.keyframes[].description). Pass this as context", - " to vision_describe so the VL model builds on what ML already found.", - "", - " Example: vision_describe input='derived/abc_kf_003.jpg'", - " context='Video: Kubernetes Tutorial. Frame 3 of 8.", - " ML found: laptop, monitor, person. OCR: kubectl apply'", - "", - " Look at frames in order to understand the video's narrative flow.", - "3. EXTRACT KNOWLEDGE: Break distinct topics, strategies, data", - " points, and insights into individual KB entries.", - "4. SUMMARIZE: KB entry covering who speaks, main points,", - " and timestamps for key moments.", - "5. SOURCE: Assess credibility -- authoritative source, influencer", - " opinion, news clip, entertainment.", - "6. REFERENCE: If tutorial/how-to, add to reference library.", - ]) - - elif media_type == "audio": - lines.append("--- QUICK SUMMARY ---") - - meta = analysis.get("metadata", {}) - if meta: - dur = meta.get("duration_str", "") - if dur: - lines.append(f"Duration: {dur}") - - transcript = analysis.get("transcript", "") - if transcript: - lines.append(f"Transcript: {len(transcript)} chars available " - f"(~{len(transcript.split())} words, read via media_search)") - else: - lines.append("No transcript available.") - - lines.append("---") - lines.append("") - - lines.extend([ - "REVIEW ACTIONS for audio:", - "1. READ THE FULL TRANSCRIPT via media_search before anything else.", - "2. EXTRACT KNOWLEDGE: KB entries for key takeaways.", - "3. SUMMARIZE: KB entry describing content, speakers, main topics.", - "4. REFERENCE: If instructional, add to reference library.", - ]) - - elif media_type in ("document", "url"): - lines.append("--- QUICK SUMMARY ---") - - content = analysis.get("content_text", "") - word_count = len(content.split()) if content else 0 - lines.append(f"Content: {word_count} words available (read via media_search)") - - if media_type == "url": - source_url = analysis.get("source_url", "") - if source_url: - lines.append(f"Source: {source_url}") - - lines.append("---") - lines.append("") - - lines.extend([ - "REVIEW ACTIONS for documents/pages:", - "1. READ THE FULL CONTENT via media_search before anything else.", - "2. KNOWLEDGE: Break factual info, strategies, domain knowledge", - " into structured KB entries. One per distinct topic.", - "3. REFERENCE: If API docs, spec, or how-to, add to ref library.", - "4. PLAN: If it's a strategy/roadmap, create a project plan.", - "5. RESEARCH: Note sources to verify or questions to investigate.", - ]) - - else: - lines.extend([ - "Review this record using media_search and determine if any", - "useful information should be extracted into the KB or", - "reference library.", - ]) - - # --------------------------------------------------------------- - # Footer -- common to all types, now with media_refs instruction - # --------------------------------------------------------------- - lines.extend([ - "", - "LINKING (CRITICAL):", - f"- When creating KB entries from this media, ALWAYS include", - f" media_refs='{record_id}' in your kb_save call. This creates", - f" a bidirectional link: the KB article points to the media record", - f" and the media record points back to the KB article. This lets", - f" future agents discover related content by following links in", - f" either direction.", - "", - "GENERAL:", - f"- Use media_search action='get' id='{record_id}' for full details", - "- Every piece of media gets at least a brief KB entry so the", - " system knows what it has. A catalogued library beats a big one.", - "- Prioritize quality over speed. Extract all useful information", - " rather than rushing through.", - ]) - - return "\n".join(lines) + # Stage 3 — RAPTOR tree (over per-segment summaries) + try: + from raptor_tree import build_tree, root_themes + leaves = [ + { + "index": s.get("index"), + "topic": s.get("topic"), + "summary": s.get("summary") or s.get("topic") or "", + } + for s in per_segment if (s.get("summary") or s.get("topic")) + ] + raptor_nodes = build_tree(leaves, model=model, driver=driver) + raptor_roots = root_themes(raptor_nodes) + except Exception as e: + logger.warning("review stage3 RAPTOR failed: %s", e) + raptor_nodes = [] + raptor_roots = [] + logger.info( + "[%s] review stage3: RAPTOR tree %d nodes, %d roots", + record_id, len(raptor_nodes), len(raptor_roots), + ) + finally: + # Unload only when explicitly pinned. Without a pin the model + # remains on Ollama's normal 5-min keep_alive for the next call. + if pinned: + try: + driver.unload_model(pin_model) + logger.info("[%s] review: unloaded %s", record_id, pin_model) + except Exception as e: + logger.debug("[%s] review: unload failed (%s)", record_id, e) + + # Stage 4 — auto-tag. + # + # Embed the stage-2 summary rather than the ingest-time text_vector: + # the summary is dense in domain terms (titanosaur, foraging, fossils) + # and lands closer to the right centroids than the title + transcript + # head captured at ingest time. Falls back to the transcript prefix + # when the summary is empty. + # + # _suggest_tags uses suggest_tags_tiered (three cosine tiers: + # 0.50 / 0.45 / 0.40) — no LLM in the normal path. LLM fallback + # is opt-in via ``media.topic_llm_fallback: true``; ~1–3 s per + # affected article. + summary_for_embed = (episode.get("summary") or "").strip() + text_vec = _vector_or_none(summary_for_embed or transcript[:2000]) + # Pass the summary + transcript prefix as source_text so keyword + # guards in topic_vocabulary (e.g. the astronomy guard) can confirm + # domain vocabulary is actually present before accepting the tag. + tag_source_text = summary_for_embed + "\n" + transcript[:4000] + auto_tags = _suggest_tags(text_vec, source_text=tag_source_text) + + # Optional LLM safety net for articles that cosine tiers cannot tag. + if not auto_tags and _config().get("topic_llm_fallback", False): + auto_tags = _suggest_tags_llm_fallback(summary_for_embed, record_id) + + # Mine inline ``#hashtag`` mentions from the summary + content. + # Markdown headings, code, and URLs are stripped first to prevent + # ``# Heading`` being mistaken for ``#tag``. + try: + from hashtag_extraction import extract_inline_hashtags + inline_tags = extract_inline_hashtags( + (summary_for_embed + "\n" + transcript[:8000]) + ) + except Exception: + inline_tags = [] + if inline_tags: + logger.info( + "[%s] review stage4: +%d inline hashtags (%s)", + record_id, len(inline_tags), + ", ".join(inline_tags[:5]), + ) + # User-authored hashtags take precedence over similarity-derived + # tags: inserted at score 1.0 for sort stability, then any + # auto_tags whose label isn't already covered are appended. + existing = {t[0].lower() for t in auto_tags} + for ht in inline_tags: + if ht.lower() not in existing: + auto_tags.insert(0, (ht, 1.0)) + existing.add(ht.lower()) + + logger.info( + "[%s] review stage4: %d auto-tags (top: %s)", + record_id, len(auto_tags), + ", ".join(t[0] for t in auto_tags[:3]) if auto_tags else "(none)", + ) + + # Stage 5 — cross-episode linking + linked = _link_similar_episodes(record_id, text_vec, top_k=5) + logger.info( + "[%s] review stage5: %d linked episodes", + record_id, len(linked), + ) + + # Stage 6 — knowledge graph emission (deterministic from facts) + # Dedup by (subject, predicate) — same subject+predicate+different + # object means a multi-valued relation, keep both objects. Same + # triple twice is collapsed. + seen = set() + triples = [] + for f in episode.get("facts") or []: + key = ( + (f.get("subject") or "").lower(), + (f.get("predicate") or "").lower(), + (f.get("object") or "").lower(), + ) + if key in seen: + continue + seen.add(key) + triples.append({ + "subject": f["subject"], + "predicate": f["predicate"], + "object": f["object"], + "confidence": f.get("confidence", 0.7), + "record_id": record_id, + }) + logger.info( + "[%s] review stage6: %d unique KG triples", record_id, len(triples), + ) + + # Stage 7 — structured KB write. No LLM here. References are + # built from leOS's own record metadata so they cannot be fake. + refs = [] + if record.get("source_url"): + refs.append({"kind": "audio_source", "url": record["source_url"]}) + refs.append({"kind": "media_record", "id": record_id}) + title = (record.get("title") or "").strip() or f"Media {record_id}" + summary = (episode.get("summary") or "").strip() + bullet_facts = "\n".join( + f"- ({f['subject']}) {f['predicate']} ({f['object']}) " + f"[conf {f.get('confidence', 0.7):.2f}]" + for f in (episode.get("facts") or [])[:50] + ) + bullet_entities = "\n".join( + f"- {e['name']} ({e['type']})" + for e in (episode.get("entities") or [])[:50] + ) + bullet_questions = "\n".join( + f"- {q}" for q in (episode.get("questions") or [])[:30] + ) + bullet_claims = "\n".join( + f"- {c['claim']}" + + (f" — {c['speaker']}" if c.get("speaker") else "") + for c in (episode.get("claims") or [])[:30] + ) + bullet_themes = "\n".join( + f"- {t}" for t in (episode.get("themes") or [])[:8] + ) + content = ( + f"# {title}\n\n" + f"## Summary\n{summary}\n\n" + f"## Themes\n{bullet_themes or '(none extracted)'}\n\n" + f"## Atomic Facts\n{bullet_facts or '(none extracted)'}\n\n" + f"## Entities\n{bullet_entities or '(none extracted)'}\n\n" + f"## Questions Raised\n{bullet_questions or '(none extracted)'}\n\n" + f"## Claims (speaker-attributed)\n{bullet_claims or '(none extracted)'}\n\n" + f"## Source\n- Media record: {record_id}\n" + f"- Source URL: {record.get('source_url') or '(none)'}\n" + ) + kb_id = _kb_save( + title=title, + summary=summary, + content=content, + tags=[t[0] for t in auto_tags], + refs=refs, + project_id=record.get("project_id"), + ) + logger.info( + "[%s] review stage7: KB article saved (id=%s)", record_id, kb_id, + ) + + elapsed = round(time.time() - t0, 2) + review = { + "elapsed_s": elapsed, + "segments": [ + { + "index": s.get("index"), + "start_char": s.get("start_char"), + "end_char": s.get("end_char"), + "method": s.get("method"), + "topic": ps.get("topic"), + "summary": ps.get("summary"), + } + for s, ps in zip(segments, per_segment) + ], + "summary": episode.get("summary") or "", + "themes": episode.get("themes") or [], + "facts": episode.get("facts") or [], + "entities": episode.get("entities") or [], + "questions": episode.get("questions") or [], + "claims": episode.get("claims") or [], + "tags": [{"name": t[0], "score": t[1]} for t in auto_tags], + "linked_episodes": linked, + "knowledge_graph": triples, + "raptor": raptor_nodes, + "raptor_roots": raptor_roots, + "kb_article_id": kb_id, + } + + media_library.update_record(record_id, { + "analysis": {**(record.get("analysis") or {}), "review": review}, + "pending_review": False, + "review_method": "staged", + "status": "complete", + "processing_stage": "Review complete", + }) + + logger.info( + "[%s] review DONE in %.1fs " + "(%d segs, %d facts, %d entities, %d themes, %d tags, %d links)", + record_id, elapsed, len(segments), + len(episode.get("facts") or []), + len(episode.get("entities") or []), + len(episode.get("themes") or []), + len(auto_tags), len(linked), + ) + return review diff --git a/project/knowledge/media_ingest_url_pipeline.py b/project/knowledge/media_ingest_url_pipeline.py index f2b700d..6d7fce4 100644 --- a/project/knowledge/media_ingest_url_pipeline.py +++ b/project/knowledge/media_ingest_url_pipeline.py @@ -51,7 +51,7 @@ from media_ingest_document_pipeline import _pipeline_document from media_ingest_helpers import _update_stage from media_ingest_image_pipeline import _extract_ocr_text, _pipeline_image -from media_ingest_review import _schedule_media_review +from media_ingest_review import run_review from media_ingest_video_pipeline import _pipeline_video # _YTDLP_PATTERNS + _DIRECT_MEDIA_EXTENSIONS @@ -88,8 +88,28 @@ r"(?:https?://)?[^.]+\.bandcamp\.com/track/", ] -# File extensions that indicate a direct download -_DIRECT_MEDIA_EXTENSIONS = set() +# File extensions that indicate a direct download — checked after the +# yt-dlp pattern list, so anything matching one of these jumps straight +# into the "direct_media" branch and runs the format-specific pipeline. +# Without these populated every direct-link audio/video URL falls +# through to the generic web_page handler and gets HTML-extracted +# instead of transcribed/keyframed. +_DIRECT_MEDIA_EXTENSIONS = { + # Audio + ".mp3", ".m4a", ".aac", ".wav", ".flac", ".ogg", ".oga", ".opus", + ".wma", ".aiff", ".aif", + # Video + ".mp4", ".m4v", ".mkv", ".webm", ".mov", ".avi", ".wmv", ".flv", + ".mpg", ".mpeg", ".3gp", ".ts", + # Images + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tiff", ".tif", + ".svg", ".heic", ".avif", + # Documents (handled by the document pipeline, not the web reader) + ".pdf", ".docx", ".doc", ".rtf", ".odt", ".epub", + # Tabular / data + ".csv", ".tsv", ".xlsx", ".xls", ".json", ".jsonl", ".ndjson", + ".parquet", +} def _pipeline_url_text(record_id): """Embed whatever text content a URL record already has.""" @@ -391,8 +411,10 @@ def _ingest_web_page(record_id, url): media_library.update_record(record_id, updates) - # Schedule an agent to review the page contents when idle - _schedule_media_review(record_id) + try: + run_review(record_id) + except Exception as e: + logger.exception("media_ingest_review failed for %s: %s", record_id, e) def _extract_embedded_media(html_content, page_url): """ diff --git a/project/knowledge/media_ingest_video_pipeline.py b/project/knowledge/media_ingest_video_pipeline.py index 7f61237..b5224d1 100644 --- a/project/knowledge/media_ingest_video_pipeline.py +++ b/project/knowledge/media_ingest_video_pipeline.py @@ -43,7 +43,7 @@ from media_ingest_audio_pipeline import _generate_spectrogram, _transcribe_media from media_ingest_helpers import _format_duration, _update_stage from media_ingest_image_pipeline import _build_image_description, _classify_image, _crop_and_embed_faces, _crop_and_embed_objects, _deduplicate_keyframes, _detect_faces, _detect_faces_with_bboxes, _detect_objects, _ocr_with_upscale, _segment_image, _upscale_for_analysis -from media_ingest_review import _schedule_media_review +from media_ingest_review import run_review def _analyze_keyframes(record_id, keyframe_paths, results): @@ -600,7 +600,10 @@ def _pipeline_video(record_id, file_path): if embeddings.is_enabled(): embeddings.save_cache_to_disk() - _schedule_media_review(record_id) + try: + run_review(record_id) + except Exception as e: + logger.exception("media_ingest_review failed for %s: %s", record_id, e) def _get_video_metadata(file_path): """Get video metadata via ffprobe.""" diff --git a/project/knowledge/media_library.py b/project/knowledge/media_library.py index 5a4c16e..bedeb75 100644 --- a/project/knowledge/media_library.py +++ b/project/knowledge/media_library.py @@ -191,14 +191,13 @@ def create_record(media_type, filename=None, title=None, source_url=None, "depth_vector": None, # 1024-dim ImageBind via Depth Anything V2 "keyframe_vectors": [], - # Phase 3D: New embedding fields for the full multimodal fingerprint "qwen_text_vector": None, # 1024-dim Qwen3 instruction-aware text embedding "imagebind_text_vector": None, # 1024-dim ImageBind CLIP text embedding "imagebind_depth_native": None, # 1024-dim ImageBind native DEPTH encoder "video_clip_vectors": [], # list of {timestamp, vector} for video temporal "audio_timeline_vectors": [], # list of {start, end, vector} for audio windows - # Phase 5A: Per-object and per-face crop embeddings + # Per-object and per-face crop embeddings — each crop embedded individually for semantic search. # Each detected object/face is cropped, optionally upscaled, # and embedded individually with ImageBind. This lets us search # for "all images containing a laptop" by vector similarity diff --git a/project/knowledge/mmr.py b/project/knowledge/mmr.py new file mode 100644 index 0000000..fc32d55 --- /dev/null +++ b/project/knowledge/mmr.py @@ -0,0 +1,240 @@ +"""mmr.py — Maximal Marginal Relevance re-ranking. + +Implementation of the MMR algorithm (Carbonell & Goldstein, SIGIR 1998) +for balancing relevance and diversity in ranked lists. + + MMR = argmax_{d in R\\S} [ lambda * Sim(d, q) - (1-lambda) * max_{d' in S} Sim(d, d') ] + +where: + R = candidate set + S = already-selected set + q = query (or anchor article for related-link generation) + lambda = 1 - diversity (1.0 = pure relevance, 0.0 = pure diversity) + +Used for two purposes in leOS: + * `mmr_rerank` — cap-and-diversify a ranked candidate list. Eliminates + show-clustering effects in `auto_link_entry` (without MMR, all 10 + near-duplicate MUM episodes win against one diverse-topic match). + * `mmr_select_top_k` — same algorithm, friendlier wrapper for the + KB graph expansion path. + +Reference: Carbonell & Goldstein (SIGIR 1998). Python port of an +internal Rust implementation, adapted to leOS's dict-based candidate +shape and Python list vectors. + +Performance: + * All candidate vectors are stacked into a single float32 (N, 768) matrix. + nomic-embed-text outputs are unit-normed, so cosine reduces to dot product. + * The full (N, N) pairwise similarity matrix is computed once via a single + BLAS dgemm call and sliced during greedy selection — no Python-level + cosine calls inside the selection loop. + * Relevance scores are vectorised before the loop; the first pick is a + plain argmax with no matmul. + * Pre-allocated index lists avoid list.append / np.vstack in the hot loop. +""" + +from __future__ import annotations + +import math +from typing import Any, Iterable, List, Optional, Sequence, Tuple + +import numpy as np + + +def cosine_similarity(a: Sequence[float], b: Sequence[float]) -> float: + """Cosine similarity between two equal-length numeric sequences. + Returns 0.0 for empty inputs, mismatched lengths, or zero-norm + vectors — never raises. + + Kept for external callers and ``mmr_select_top_k`` score seeding. + The hot path inside ``mmr_rerank`` no longer calls this function. + """ + if not a or not b or len(a) != len(b): + return 0.0 + dot = norm_a = norm_b = 0.0 + for x, y in zip(a, b): + dot += x * y + norm_a += x * x + norm_b += y * y + denom = math.sqrt(norm_a) * math.sqrt(norm_b) + return dot / denom if denom else 0.0 + + +def mmr_rerank( + candidates: Sequence[dict], + vectors: dict, + diversity: float = 0.5, + limit: int = 10, + *, + id_key: str = "id", + score_key: str = "score", + assume_normed: bool = False, +) -> List[dict]: + """Re-rank ``candidates`` using MMR. + + Args: + candidates: list of dicts with at least ``id_key`` and + ``score_key``. Pre-sorted by score is fine but + not required (MMR will pick its own order). + vectors: dict mapping ``candidate[id_key]`` -> list[float] + embedding vector. Candidates with no entry are + kept but appended at the end (no diversity + calculation possible without a vector). + diversity: 0.0 = pure relevance (preserves original ranking), + 1.0 = max diversity. 0.3-0.5 is a typical + balance for related-article selection. + limit: maximum number of results to return. + assume_normed: set True when vectors are already unit-normed + (e.g. direct nomic-embed-text output) to skip + the defensive L2 normalisation step at entry. + + Returns: + Newly-ordered list of up to ``limit`` candidate dicts, with + the original metadata preserved. Includes a ``_mmr_score`` + key on each returned dict for diagnostic visibility. + """ + if not candidates or limit <= 0: + return [] + diversity = max(0.0, min(1.0, float(diversity))) + lam = 1.0 - diversity + + # Diversity disabled — return top-K by raw score. + if diversity == 0.0: + ordered = sorted(candidates, key=lambda c: c.get(score_key, 0.0), reverse=True) + return [dict(c, _mmr_score=c.get(score_key, 0.0)) for c in ordered[:limit]] + + # Partition candidates by whether they have an embedding. + with_vec: List[Tuple[dict, list]] = [] + without_vec: List[dict] = [] + for c in candidates: + v = vectors.get(c.get(id_key)) + if v: + with_vec.append((c, v)) + else: + without_vec.append(c) + + if not with_vec: + # Nothing to MMR — return raw-score ranking. + ordered = sorted(without_vec, key=lambda c: c.get(score_key, 0.0), reverse=True) + return [dict(c, _mmr_score=c.get(score_key, 0.0)) for c in ordered[:limit]] + + # ------------------------------------------------------------------ + # Vectorised MMR core + # ------------------------------------------------------------------ + N = len(with_vec) + cand_dicts = [c for c, _ in with_vec] + + # Stack all vectors into a single (N, dim) float32 matrix. + mat = np.asarray([v for _, v in with_vec], dtype=np.float32) # (N, dim) + + # Defensive L2 normalisation (one pass, O(N*dim)). + # Skip only when the caller guarantees unit norms. + if not assume_normed: + norms = np.linalg.norm(mat, axis=1, keepdims=True) + norms = np.where(norms == 0.0, 1.0, norms) # avoid /0 for zero vecs + mat /= norms + + # Compute full (N, N) pairwise similarity matrix in one BLAS call. + # At N=1000 this is 4 MB float32 — well within budget. + # sim_mat[i, j] = cosine(candidate_i, candidate_j) (dot product on + # unit-normed vectors). + sim_mat = mat @ mat.T # (N, N) + + # Normalise relevance scores into [0, 1]. + raw_scores = np.array( + [c.get(score_key, 0.0) for c in cand_dicts], dtype=np.float64 + ) + s_min = raw_scores.min() + s_max = raw_scores.max() + s_rng = s_max - s_min + if s_rng > 0.0: + rel_scores = (raw_scores - s_min) / s_rng + else: + rel_scores = np.ones(N, dtype=np.float64) + + # Track which indices are still available. + remaining_mask = np.ones(N, dtype=bool) + + # Pre-allocate the output buffer for selected indices. + selected_indices: List[int] = [] + selected_mmr_scores: List[float] = [] + + # First pick: pure relevance, no diversity penalty. + # Use remaining_mask for consistency (all True at this point). + remaining_idx = np.where(remaining_mask)[0] + first = int(remaining_idx[np.argmax(rel_scores[remaining_idx])]) + selected_indices.append(first) + selected_mmr_scores.append(float(rel_scores[first])) + remaining_mask[first] = False + + # Greedy selection for picks 2 … limit. + while len(selected_indices) < limit and remaining_mask.any(): + remaining_idx = np.where(remaining_mask)[0] # indices still available + + # Diversity penalty: max sim to any already-selected candidate. + # sim_mat[selected_indices][:, remaining_idx] is (sel, rem); + # .max(axis=0) gives the per-remaining worst-case overlap. + sel_arr = np.array(selected_indices, dtype=np.intp) + max_sim_to_selected = sim_mat[np.ix_(sel_arr, remaining_idx)].max(axis=0) + # shape: (len(remaining_idx),) + + mmr_scores = ( + lam * rel_scores[remaining_idx] + - (1.0 - lam) * max_sim_to_selected + ) + best_local = int(np.argmax(mmr_scores)) + best_global = int(remaining_idx[best_local]) + + selected_indices.append(best_global) + selected_mmr_scores.append(float(mmr_scores[best_local])) + remaining_mask[best_global] = False + + # Build output list preserving original dict metadata. + selected: List[dict] = [] + for idx, mmr_score in zip(selected_indices, selected_mmr_scores): + cand_out = dict(cand_dicts[idx]) + cand_out["_mmr_score"] = mmr_score + selected.append(cand_out) + + # Vectorless candidates fill remaining capacity in original order. + if len(selected) < limit and without_vec: + for c in without_vec[: limit - len(selected)]: + selected.append(dict(c, _mmr_score=c.get(score_key, 0.0))) + + return selected + + +def mmr_select_top_k( + candidates: Iterable[dict], + anchor_vec: Optional[Sequence[float]], + vectors: dict, + k: int = 8, + diversity: float = 0.4, + *, + id_key: str = "id", + score_key: str = "score", + assume_normed: bool = False, +) -> List[dict]: + """Convenience wrapper: same as ``mmr_rerank`` but takes an + ``anchor_vec`` (the article whose related-list we're building) + so initial relevance can be computed from cosine if scores are + missing. When candidates already carry meaningful ``score_key`` + values, ``anchor_vec`` can be None. + + Pass ``assume_normed=True`` when vectors come directly from + ``leos_embed`` (nomic-embed-text returns unit-normed float32) to + skip the defensive normalisation step inside ``mmr_rerank``. + """ + cands = list(candidates) + if not cands: + return [] + # If candidates lack scores, compute them from anchor_vec. + if anchor_vec and any(score_key not in c for c in cands): + for c in cands: + if score_key not in c: + v = vectors.get(c.get(id_key)) + c[score_key] = cosine_similarity(anchor_vec, v) if v else 0.0 + return mmr_rerank( + cands, vectors=vectors, diversity=diversity, limit=k, + id_key=id_key, score_key=score_key, assume_normed=assume_normed, + ) diff --git a/project/knowledge/plan_coherence.py b/project/knowledge/plan_coherence.py index 5f7e58f..22e1aaf 100644 --- a/project/knowledge/plan_coherence.py +++ b/project/knowledge/plan_coherence.py @@ -29,8 +29,6 @@ logger = logging.getLogger("plan_coherence") -# --------------------------------------------------------------------------- -# Phase 12: Coherence scoring # --------------------------------------------------------------------------- def score_plan_coherence(phases, goal_text=""): @@ -247,8 +245,6 @@ def _empty_result(phases): } -# --------------------------------------------------------------------------- -# Phase 14: Goal gradient / progression analysis # --------------------------------------------------------------------------- def get_step_progression(phases, goal_text): diff --git a/project/knowledge/plan_manager.py b/project/knowledge/plan_manager.py index e7e9c48..1e212d7 100644 --- a/project/knowledge/plan_manager.py +++ b/project/knowledge/plan_manager.py @@ -254,12 +254,7 @@ def create_plan(self, goal, steps=None, description="", overview="", "warnings": [], "notes": [], "scope_id": scope_id, - # Plan 22 (retry escalation) additive fields. Old plan - # records on disk that lack these keys still load fine — - # readers should default-handle their absence. Empty - # string for tier_used and retry_of, empty list for - # retry_history, lets every existing caller remain - # unaware of the new fields. + # Additive retry fields; absent in older records — callers should default-handle missing keys. "tier_used": tier_used or "", "retry_of": retry_of or "", "retry_history": [], diff --git a/project/knowledge/raptor_tree.py b/project/knowledge/raptor_tree.py new file mode 100644 index 0000000..460da1f --- /dev/null +++ b/project/knowledge/raptor_tree.py @@ -0,0 +1,224 @@ +""" +raptor_tree.py — Recursive embed-cluster-summarize for episode segments. + +Per RAPTOR (arxiv 2401.18059) and the 2025 enhancements (Frontiers +"semantic chunking + adaptive graph clustering"): build a tree of +summaries at multiple levels of abstraction so retrieval can hit fine +detail OR top-level themes. + +This module operates over already-extracted segment summaries (from +atomic_extractor) so the LLM cost is bounded — only ``cluster_summary`` +calls during the recursion, no fresh per-segment work. + +Algorithm (simplified Leiden / GMM via numpy-only k-means): + 1. Embed each segment's ``summary`` field. + 2. K-means cluster — k chosen as ceil(sqrt(N/2)), capped 2..6. + 3. For each cluster, call ``atomic_extractor.cluster_summary`` to + produce {label, summary}. + 4. Recurse on the cluster summaries until ≤ 3 nodes remain (the + RAPTOR root layer). + 5. Return tree as a flat list of {id, level, parent_id, label, + summary, child_ids[], embedding[]}. + +Trees are stored on the record's ``analysis.review.raptor`` field +and are queryable by leOS's existing ``leos_search`` over text_store +when the tree nodes are written there. +""" + +import logging +import math +import random + +logger = logging.getLogger("raptor_tree") + + +def _embed(text): + try: + import embeddings + v = embeddings.embed_text(text) + if v is None: + return None + return list(map(float, v)) + except Exception as e: + logger.debug("embed failed: %s", e) + return None + + +def _kmeans(vectors, k, max_iter=20, seed=42): + """Tiny pure-Python k-means. Vectors are unit-normalised so cosine + similarity is just a dot product; we use squared Euclidean (which + is monotonic with cosine on unit vectors) for assignment.""" + if not vectors or k <= 1: + return [0] * len(vectors) + rng = random.Random(seed) + n, d = len(vectors), len(vectors[0]) + k = min(k, n) + # Init: pick k random vectors as centroids + idxs = rng.sample(range(n), k) + centroids = [list(vectors[i]) for i in idxs] + assignments = [0] * n + for _ in range(max_iter): + # Assign + changed = False + for i, v in enumerate(vectors): + best = 0 + best_d = float("inf") + for c_idx, c in enumerate(centroids): + dist = 0.0 + for x, y in zip(v, c): + dist += (x - y) * (x - y) + if dist < best_d: + best_d = dist + best = c_idx + if assignments[i] != best: + assignments[i] = best + changed = True + # Update centroids + new_cents = [[0.0] * d for _ in range(k)] + counts = [0] * k + for i, v in enumerate(vectors): + cl = assignments[i] + for j in range(d): + new_cents[cl][j] += v[j] + counts[cl] += 1 + for cl in range(k): + if counts[cl]: + for j in range(d): + new_cents[cl][j] /= counts[cl] + else: + # empty cluster — re-seed from a random vector + new_cents[cl] = list(vectors[rng.randrange(n)]) + centroids = new_cents + if not changed: + break + return assignments + + +def build_tree(segment_summaries, *, max_root_nodes=3, max_levels=4, + model=None, driver=None): + """Build a RAPTOR tree from a list of dicts that each have a + ``summary`` and an ``index`` (the segment index in the original + transcript). + + Returns a flat list of nodes: + { + "id": "n_0_3", # level_index + "level": int, # 0 = leaves (segments) + "parent_id": str|None, + "label": str, + "summary": str, + "embedding": [float..]|None, + "child_ids": [str..], # empty for leaves + "segment_indices": [int..] # for leaves: [own idx]; else flat + } + """ + # Lazy import — needs ollama for cluster_summary. + from atomic_extractor import cluster_summary + + if not segment_summaries: + return [] + + # Build leaf nodes + nodes = [] + leaf_ids = [] + for i, seg in enumerate(segment_summaries): + nid = f"n_0_{i}" + leaf_ids.append(nid) + text = seg.get("summary") or seg.get("topic") or "" + nodes.append({ + "id": nid, + "level": 0, + "parent_id": None, + "label": (seg.get("topic") or f"segment {seg.get('index', i)}")[:120], + "summary": text, + "embedding": _embed(text) if text else None, + "child_ids": [], + "segment_indices": [seg.get("index", i)], + }) + + # Recurse upward. + current_level_ids = leaf_ids + level = 1 + while len(current_level_ids) > max_root_nodes and level <= max_levels: + # Cluster the current frontier + frontier_nodes = [n for n in nodes if n["id"] in current_level_ids] + # Filter to nodes with embeddings; without embeddings we can't + # cluster — fall back to grouping in chronological order. + embedded = [n for n in frontier_nodes if n.get("embedding")] + if len(embedded) < 2: + # Not enough to cluster meaningfully — stop here. + break + n_count = len(embedded) + k = max(2, min(max_root_nodes, math.ceil(math.sqrt(n_count / 2)))) + vectors = [n["embedding"] for n in embedded] + assignments = _kmeans(vectors, k) + + # Build parent nodes for each cluster + clusters = {} + for node, cl in zip(embedded, assignments): + clusters.setdefault(cl, []).append(node) + # Nodes without embeddings are appended to the cluster of their + # nearest already-clustered neighbour by index proximity. + unembedded = [n for n in frontier_nodes if not n.get("embedding")] + for ne in unembedded: + ne_idx = ne["segment_indices"][0] + best_cl = 0 + best_diff = float("inf") + for cl, members in clusters.items(): + for m in members: + diff = abs(m["segment_indices"][0] - ne_idx) + if diff < best_diff: + best_diff = diff + best_cl = cl + clusters.setdefault(best_cl, []).append(ne) + + next_ids = [] + for cl, members in clusters.items(): + child_summaries = [m["summary"] for m in members if m.get("summary")] + if not child_summaries: + continue + try: + cs = cluster_summary(child_summaries, model=model, driver=driver) + except Exception as e: + logger.warning("cluster_summary failed at level %d: %s", level, e) + cs = { + "label": f"cluster {cl}", + "summary": " | ".join(s[:140] for s in child_summaries), + } + parent_id = f"n_{level}_{cl}" + child_ids = [m["id"] for m in members] + seg_indices = [] + for m in members: + seg_indices.extend(m.get("segment_indices") or []) + m["parent_id"] = parent_id + nodes.append({ + "id": parent_id, + "level": level, + "parent_id": None, + "label": cs.get("label") or f"cluster {cl}", + "summary": cs.get("summary") or "", + "embedding": _embed(cs.get("summary") or ""), + "child_ids": child_ids, + "segment_indices": sorted(set(seg_indices)), + }) + next_ids.append(parent_id) + + if not next_ids or len(next_ids) >= len(current_level_ids): + # Failed to reduce — stop to avoid infinite loop. + break + current_level_ids = next_ids + level += 1 + + return nodes + + +def root_themes(nodes): + """Return the top-level (highest level number) node summaries — + the RAPTOR roots, useful for episode-level theme display.""" + if not nodes: + return [] + max_lvl = max(n["level"] for n in nodes) + return [ + {"label": n["label"], "summary": n["summary"]} + for n in nodes if n["level"] == max_lvl + ] diff --git a/project/knowledge/research_router.py b/project/knowledge/research_router.py index d32bf73..a169fdf 100644 --- a/project/knowledge/research_router.py +++ b/project/knowledge/research_router.py @@ -188,13 +188,7 @@ def _refine_entity_kinds(entities): # constant or the sentinel "" which binds to the entity's # raw text at dispatch time. # -# Stage 3 covers the crypto_market_data group only -- the group -# where the 61-retry regression lives. Extending to other groups -# (scientific_papers, encyclopedia, etc.) is the same table edit: -# add (source_id, kind) -> recipe rows. Stage 7's tagging handshake -# can eventually produce these entries automatically from each -# learned API spec's agent_hints, but hand-curation is the right -# starting point while the vocabulary is small. +# DISPATCH_RECIPES covers crypto_market_data; extend by adding (source_id, kind) rows. Hand-curation is appropriate while the vocabulary is small. # ================================================================ DISPATCH_RECIPES = { diff --git a/project/knowledge/rrf_search.py b/project/knowledge/rrf_search.py new file mode 100644 index 0000000..7a49ee6 --- /dev/null +++ b/project/knowledge/rrf_search.py @@ -0,0 +1,231 @@ +"""rrf_search.py — Reciprocal Rank Fusion over BM25 + nomic + qwen. + +Three retrieval upgrades over the previous linear-blend hybrid: + + 1. **RRF fusion** instead of linear-blend score combination. Linear + blends require both signals to share a score scale (or aggressive + normalization); RRF operates on rank positions, dodging the + normalization headache. Score = Σ_i w_i / (k + rank_i + 1). + We use ``RRF_K = 20`` (Elasticsearch BEIR 2024 grid-search + finding) instead of the original Cormack 60. + + 2. **Adaptive weights** — short keyword queries weight BM25 higher + (lexical precision), long natural-language queries weight + semantic higher (intent). Quoted phrases pin BM25. See the + weight table below. + + 3. **MMR re-rank** of the fused top-K — kills the redundant + near-duplicates that all three retrievers tend to surface + together (e.g. multiple ingestions of the same Phonology + episode). Tunable via ``mmr_diversity``. + +The qwen3-embedding signal is included as a third dense retriever +when ``embeddings.is_qwen_enabled()`` is true. qwen has a 32K +context (vs nomic's 8K) and instruction-aware embedding, so it +catches different match cases than nomic for long articles. + +Used directly by ``KnowledgeBase.search`` when ``LEOS_RRF_SEARCH`` +env var is set to "1" (default), with a graceful fallback to the +existing ``_search_hybrid`` linear-blend path if anything goes +wrong. + +Reference: + - Cormack, Clarke, Buettcher (2009) — Reciprocal Rank Fusion + - Elasticsearch BEIR analysis (2024) — k=20 optimum + - Carbonell, Goldstein (SIGIR 1998) — MMR +""" + +from __future__ import annotations + +import logging +import os +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +logger = logging.getLogger("rrf_search") + +# Cormack-Clarke-Buettcher (2009) used K=60. Elasticsearch's 2024 +# BEIR grid search found K=20 better for small-to-medium corpora — +# tighter top-rank emphasis at the cost of deep recall, which is +# exactly the tradeoff for a personal podcast KB. +RRF_K = 20.0 + +# MMR diversity for the fused top-K. 0.3 = moderate diversity. +MMR_DIVERSITY = 0.3 + +# Adaptive weight table. Each entry is (bm25, nomic, qwen). +# Weights sum to 1.0. Picked by query characteristics in +# ``_select_weights``. Three signals: BM25 + nomic + qwen. +_WEIGHT_TABLE: Dict[str, Tuple[float, float, float]] = { + # Quoted phrases — lexical precision matters. + "quoted": (0.70, 0.20, 0.10), + # Exact-match patterns (UUIDs, IDs, codes). + "exact": (0.80, 0.15, 0.05), + # Short keyword queries (1-2 tokens) — FTS handles keywords well. + "keyword": (0.60, 0.25, 0.15), + # Balanced (3-5 tokens) — equal split. + "balanced": (0.40, 0.35, 0.25), + # Long conceptual queries (6+ tokens) — semantic captures intent. + "conceptual": (0.25, 0.45, 0.30), +} + +_QUOTE_RE = re.compile(r'"[^"]+"|“[^”]+”|\'[^\']+\'') +_EXACT_RE = re.compile( + r"^[A-Z0-9_-]{6,}$" # ID-like all-caps/digits + r"|^[a-f0-9]{8,}(-[a-f0-9]{4,}){0,4}$" # uuid-ish + r"|^kb_[a-f0-9]+$|^media_[a-f0-9]+$" # leOS ids +) + + +def _select_weights(query: str) -> Tuple[str, Tuple[float, float, float]]: + """Pick (label, (bm25, nomic, qwen)) for this query.""" + q = query.strip() + if not q: + return "balanced", _WEIGHT_TABLE["balanced"] + if _EXACT_RE.match(q): + return "exact", _WEIGHT_TABLE["exact"] + if _QUOTE_RE.search(q): + return "quoted", _WEIGHT_TABLE["quoted"] + n = len(q.split()) + if n <= 2: + return "keyword", _WEIGHT_TABLE["keyword"] + if n <= 5: + return "balanced", _WEIGHT_TABLE["balanced"] + return "conceptual", _WEIGHT_TABLE["conceptual"] + + +def rrf_fuse( + ranked_lists: Sequence[Tuple[float, List[Tuple[Any, float]]]], + *, + k: float = RRF_K, +) -> List[Tuple[Any, float]]: + """Fuse multiple ranked lists into one using weighted RRF. + + Each input is ``(weight, [(item_id, score), ...])`` where the + inner list is **already sorted** best-first. The score values + inside each list are not used — only ranks. Per-list weights + let the caller emphasise some retrievers (e.g. boost BM25 for + short queries). + + Returns ``[(item_id, normalised_rrf_score), ...]`` sorted + best-first. Score is normalised to [0, 1] using the maximum + possible score (item appearing rank-0 in every list). + """ + if not ranked_lists: + return [] + fused: Dict[Any, float] = {} + total_weight = sum(w for w, _ in ranked_lists if w > 0) or 1.0 + for weight, hits in ranked_lists: + if weight <= 0 or not hits: + continue + for rank, (item_id, _score) in enumerate(hits): + contribution = weight / (k + rank + 1.0) + fused[item_id] = fused.get(item_id, 0.0) + contribution + + # Max possible score: item is rank-0 in every weighted list. + max_score = total_weight / (k + 1.0) + out = [ + (iid, min(1.0, s / max_score) if max_score > 0 else 0.0) + for iid, s in fused.items() + ] + out.sort(key=lambda kv: kv[1], reverse=True) + return out + + +def hybrid_rrf_search( + query: str, + *, + bm25_fn: Optional[Callable[[str, int], List[Tuple[Any, float]]]], + nomic_fn: Optional[Callable[[str, int], List[Tuple[Any, float]]]], + qwen_fn: Optional[Callable[[str, int], List[Tuple[Any, float]]]] = None, + limit: int = 5, + candidate_pool: int = 25, + mmr_diversity: float = MMR_DIVERSITY, + mmr_vectors: Optional[Dict[Any, List[float]]] = None, +) -> List[Tuple[Any, float, str]]: + """Run hybrid retrieval via RRF + MMR. + + Args: + query: the search string + bm25_fn: callable(query, k) -> [(id, score), ...] or None + nomic_fn: callable(query, k) -> [(id, score), ...] or None + qwen_fn: callable(query, k) -> [(id, score), ...] or None + limit: final result count + candidate_pool: per-retriever fetch size before fusion + mmr_diversity: 0..1 diversity weight for the post-fusion MMR + pass. 0 disables MMR (pure RRF order). Set + to 0 if ``mmr_vectors`` not supplied. + mmr_vectors: dict id -> embedding vector for MMR. When + absent or empty, MMR is skipped. + + Returns: + ``[(id, fused_score, weights_label), ...]`` sorted best-first. + """ + if not query or not isinstance(query, str): + return [] + label, (w_bm, w_no, w_qw) = _select_weights(query) + + _tasks: List[Tuple[str, float, Callable]] = [] + if bm25_fn and w_bm > 0: + _tasks.append(("bm25", w_bm, bm25_fn)) + if nomic_fn and w_no > 0: + _tasks.append(("nomic", w_no, nomic_fn)) + if qwen_fn and w_qw > 0: + _tasks.append(("qwen", w_qw, qwen_fn)) + + # Sequential retrieval: ThreadPoolExecutor deadlocks here because + # worker threads compete for embedding-model locks already held by + # the Flask request thread (shared singleton in ``embeddings.py``). + # At ~50 articles the sum-vs-max latency gap (80-150ms vs 50-80ms) + # is tolerable. Re-entrant embedding access is the prerequisite + # for returning to parallel fan-out. + ranked: List[Tuple[float, List[Tuple[Any, float]]]] = [] + for name, weight, fn in _tasks: + try: + hits = fn(query, candidate_pool) or [] + if hits: + ranked.append((weight, list(hits))) + except Exception as exc: + logger.debug("%s retriever failed: %s", name, exc) + + fused = rrf_fuse(ranked) + if not fused: + return [] + + # Optional MMR pass over fused top-N to diversify. + if mmr_diversity > 0 and mmr_vectors: + try: + from mmr import mmr_rerank + mmr_input = [{"id": iid, "score": s} for iid, s in fused[: limit * 3]] + mmr_out = mmr_rerank( + mmr_input, + vectors=mmr_vectors, + diversity=mmr_diversity, + limit=limit, + ) + return [(c["id"], c["score"], label) for c in mmr_out] + except Exception as e: + logger.debug("MMR re-rank failed (%s); using raw RRF", e) + + return [(iid, s, label) for iid, s in fused[:limit]] + + +def query_characteristics(query: str) -> Dict[str, Any]: + """Diagnostic: returns the same labels/weights ``hybrid_rrf_search`` + would pick. Useful for surfacing why retrieval ranked something + a particular way.""" + label, weights = _select_weights(query) + return { + "query": query, + "token_count": len(query.split()), + "has_quotes": bool(_QUOTE_RE.search(query or "")), + "is_exact_pattern": bool(_EXACT_RE.match(query or "")), + "weight_label": label, + "weights": {"bm25": weights[0], "nomic": weights[1], "qwen": weights[2]}, + } + + +def is_enabled() -> bool: + """Honor the ``LEOS_RRF_SEARCH`` env flag. Default ON.""" + return os.environ.get("LEOS_RRF_SEARCH", "1").strip() not in ("0", "false", "no", "off") diff --git a/project/knowledge/scope_context.py b/project/knowledge/scope_context.py index 0417e15..3307c23 100644 --- a/project/knowledge/scope_context.py +++ b/project/knowledge/scope_context.py @@ -342,8 +342,7 @@ def _layer_3_activity(self, scope, task, task_vector, include_raw): def _layer_4_knowledge(self, scope, task, task_vector, include_raw): """Layer 4: Relevant KB articles tagged with this scope.""" - # This is a placeholder that searches by scope name/description. - # Full integration would filter KB articles by scope_id metadata. + # FIXME: filter by scope_id metadata; currently falls back to name/description search. if task_vector is None: return ("", 0, 0) @@ -448,8 +447,7 @@ def _layer_5_parent(self, scope, task, task_vector, include_raw): def _layer_6_memory(self, scope, task, task_vector, include_raw): """Layer 6: Institutional memory — similar past tasks.""" - # This would query the displacement log for similar task outcomes. - # For now, check compacted memory for observation clusters. + # FIXME: query displacement log for similar task outcomes; currently uses compacted memory only. memory = scope.get("memory", {}) if not memory: return ("", 0, 0) diff --git a/project/knowledge/scope_store.py b/project/knowledge/scope_store.py index 122beeb..e86ea8a 100644 --- a/project/knowledge/scope_store.py +++ b/project/knowledge/scope_store.py @@ -36,14 +36,7 @@ # Valid note types for scope notes. # -# `user_correction` was added for the retry-escalation flow (plan 22). -# When a user clicks "Try again" on an unsatisfactory response and -# optionally provides feedback text, that feedback is written as a -# user_correction note on the scope so the next tier's planner can -# read it and adjust. Distinct from `decision` (which records what -# the system chose) and `observation` (passive context); a -# user_correction is an active, in-flight signal that the prior -# attempt didn't satisfy the user. +# user_correction: active signal that the prior attempt didn't satisfy the user — distinct from observation (passive) and decision (system choice). VALID_NOTE_TYPES = {"observation", "decision", "post_mortem", "general", "step_data", "user_correction"} diff --git a/project/knowledge/skos.py b/project/knowledge/skos.py new file mode 100644 index 0000000..a9cf4d8 --- /dev/null +++ b/project/knowledge/skos.py @@ -0,0 +1,346 @@ +"""skos.py — Minimal W3C SKOS-inspired tag hierarchy for leOS. + +A pragmatic subset of the SKOS data model focused on the operations +that actually move the needle for podcast-corpus retrieval and +related-link inference: + + - **prefLabel / altLabel** — canonical name + synonyms + - **broader / narrower** — hierarchical parent/child + - **related** — associative (computed from siblings + manual links) + - **PMEST facet** — Personality / Matter / Energy / Space / Time + classification (Ranganathan) + - **ancestor expansion** — given an article tagged ``[bears]``, the + inferred concept set is ``{bears, animals_pets, life_sciences}``, + so a query for "biology" still finds the bears article via the + inferred ancestor. + +A pragmatic subset only — concept schemes, audit logs, governance +stats, mapping relations, etc., are deliberately out of scope. The +hierarchy is a hand-built JSON file (``data/skos_hierarchy.json``) +served by a small read-mostly Python API. + +Typical use: + + from skos import get_hierarchy + h = get_hierarchy() + # All ancestor concepts for a tag (transitive) + h.ancestors("bears garbage dump") # -> {"animals and pets", "life sciences"} + # Concepts considered related (siblings + direct related links) + h.related_concepts("oncology and cancer", depth=1) + # Concept overlap score between two articles + h.concept_overlap_score(["oncology and cancer"], ["microbiome and gut health"]) +""" + +from __future__ import annotations + +import json +import logging +import os +import threading +from collections import deque +from typing import Dict, FrozenSet, Iterable, List, Optional, Set, Tuple + +logger = logging.getLogger("skos") + +_HIERARCHY_FILE = "skos_hierarchy.json" +_LOCK = threading.Lock() +_INSTANCE: Optional["SkosHierarchy"] = None + + +def _data_dir() -> str: + base = os.environ.get("LEOS_DATA_DIR") + if not base: + base = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "data", + ) + return base + + +class SkosHierarchy: + """Read-mostly view over the hand-built SKOS-style hierarchy. + + Loaded once from disk; thread-safe for read operations after + construction. Reload via ``reload()`` if the source file + changes. + """ + + def __init__(self, path: Optional[str] = None) -> None: + self._path = path or os.path.join(_data_dir(), _HIERARCHY_FILE) + self._concepts: Dict[str, dict] = {} + # Reverse lookups built once on load. + self._narrower_of: Dict[str, Set[str]] = {} + self._alt_to_pref: Dict[str, str] = {} + self.load() + + # ----- loading --------------------------------------------------- + + def load(self) -> None: + try: + with open(self._path, "r", encoding="utf-8") as f: + raw = json.load(f) + except FileNotFoundError: + logger.warning("SKOS hierarchy file not found: %s", self._path) + self._concepts = {} + return + except Exception as e: + logger.warning("Failed to load SKOS hierarchy: %s", e) + self._concepts = {} + return + self._concepts = dict(raw.get("concepts") or {}) + self._build_indexes() + + def reload(self) -> None: + with _LOCK: + self.load() + + def _build_indexes(self) -> None: + """Build reverse-index of broader -> narrower, alt -> pref, and + precomputed transitive ancestor/descendant closures.""" + self._narrower_of = {} + self._alt_to_pref = {} + for label, c in self._concepts.items(): + for parent in (c.get("broader") or []): + self._narrower_of.setdefault(parent, set()).add(label) + for alt in (c.get("alt_labels") or []): + self._alt_to_pref[alt.lower().strip()] = label + self._alt_to_pref[label.lower().strip()] = label + self._build_closures() + + def _build_closures(self) -> None: + """Precompute transitive ancestor and descendant frozensets for every + concept using a topological sort (Kahn's algorithm) so each closure is + computed exactly once via set-union over already-complete parent + closures. + + Raises ValueError if the broader/narrower graph contains a cycle, + which SKOS forbids but hand-edited JSON may introduce accidentally. + """ + concepts = self._concepts + + # --- Step 1: build children map and in-degree for Kahn's sort ------- + children: Dict[str, Set[str]] = {c: set() for c in concepts} + in_degree: Dict[str, int] = {c: 0 for c in concepts} + for node, data in concepts.items(): + for parent in (data.get("broader") or []): + if parent in children: + children[parent].add(node) + in_degree[node] += 1 + # Parents referenced but not defined are skipped gracefully; + # _build_indexes already validates all_broader vs concepts. + + # --- Step 2: Kahn's topological sort (roots first) ------------------- + queue: deque = deque(c for c in concepts if in_degree[c] == 0) + topo: List[str] = [] + while queue: + node = queue.popleft() + topo.append(node) + for child in children[node]: + in_degree[child] -= 1 + if in_degree[child] == 0: + queue.append(child) + + # --- Step 3: cycle detection ------------------------------------------ + if len(topo) != len(concepts): + in_cycle = set(concepts) - set(topo) + raise ValueError( + f"SKOS hierarchy contains a broader/narrower cycle — " + f"nodes involved: {sorted(in_cycle)}" + ) + + # --- Step 4: ancestor closures bottom-up (parents before children) --- + # anc_closure[node] = frozenset of ALL transitive parents (not self). + anc_closure: Dict[str, FrozenSet[str]] = {} + for node in topo: + direct_parents = frozenset( + p for p in (concepts[node].get("broader") or []) + if p in concepts + ) + ancestors: Set[str] = set(direct_parents) + for p in direct_parents: + ancestors |= anc_closure[p] # already complete, guaranteed by topo order + anc_closure[node] = frozenset(ancestors) + + # --- Step 5: descendant closures top-down (children before parents) -- + desc_sets: Dict[str, Set[str]] = {c: set() for c in concepts} + for node in reversed(topo): + for parent in (concepts[node].get("broader") or []): + if parent in desc_sets: + desc_sets[parent].add(node) + desc_sets[parent] |= desc_sets[node] + desc_closure: Dict[str, FrozenSet[str]] = { + c: frozenset(v) for c, v in desc_sets.items() + } + + self._anc_closure = anc_closure + self._desc_closure = desc_closure + + # ----- normalization -------------------------------------------- + + def normalize(self, tag: str) -> Optional[str]: + """Map any tag (preferred or alt label) to its canonical + prefLabel, or None if the concept is unknown.""" + if not tag: + return None + key = tag.lower().strip() + return self._alt_to_pref.get(key) + + def has(self, label: str) -> bool: + return self.normalize(label) is not None + + # ----- hierarchy queries ---------------------------------------- + + def broader(self, label: str) -> List[str]: + """Direct parents (skos:broader).""" + c = self._concepts.get(self.normalize(label) or "") + return list(c.get("broader") or []) if c else [] + + def narrower(self, label: str) -> List[str]: + """Direct children (skos:narrower).""" + pref = self.normalize(label) + return sorted(self._narrower_of.get(pref or "", set())) + + def ancestors(self, label: str, *, max_depth: int = 6) -> FrozenSet[str]: # noqa: ARG002 + """All transitive parents for a label (excludes the label itself). + + Returns a precomputed frozenset — O(1) dict lookup. ``max_depth`` + is accepted for API compatibility but ignored; the full transitive + closure is always returned (actual max depth in the current hierarchy + is 3). + """ + pref = self.normalize(label) + if not pref: + return frozenset() + return self._anc_closure.get(pref, frozenset()) + + def descendants(self, label: str, *, max_depth: int = 6) -> FrozenSet[str]: # noqa: ARG002 + """All transitive children (excludes the label itself). + + Returns a precomputed frozenset — O(1) dict lookup. ``max_depth`` + is accepted for API compatibility but ignored. + """ + pref = self.normalize(label) + if not pref: + return frozenset() + return self._desc_closure.get(pref, frozenset()) + + def siblings(self, label: str) -> Set[str]: + """Concepts sharing a direct parent.""" + pref = self.normalize(label) + if not pref: + return set() + out: Set[str] = set() + for parent in self.broader(pref): + for sib in self.narrower(parent): + if sib != pref: + out.add(sib) + return out + + def related_concepts(self, label: str, *, depth: int = 1) -> Set[str]: + """Loose 'related' set: parents + children + siblings. + ``depth`` controls how many ancestor levels to include.""" + pref = self.normalize(label) + if not pref: + return set() + out: Set[str] = set() + out |= self.ancestors(pref, max_depth=depth) + out |= self.descendants(pref, max_depth=depth) + out |= self.siblings(pref) + out.discard(pref) + return out + + def expand(self, labels: Iterable[str], *, with_ancestors: bool = True) -> Set[str]: + """Expand a list of tags to include their ancestors. Returns + the union of all canonical prefLabels (input tags + their + ancestors). Unknown tags are passed through verbatim so an + article can still surface a free-text label.""" + out: Set[str] = set() + for raw in labels: + pref = self.normalize(raw) + if pref: + out.add(pref) + if with_ancestors: + out |= self.ancestors(pref) + elif raw: + out.add(raw) # preserve unknown tags + return out + + def pmest(self, label: str) -> Optional[str]: + """Return the PMEST facet for a concept (or any of its + ancestors).""" + pref = self.normalize(label) + if not pref: + return None + chain = [pref] + list(self.ancestors(pref)) + for c in chain: + f = (self._concepts.get(c) or {}).get("pmest") + if f: + return f + return None + + # ----- scoring helpers ------------------------------------------ + + def concept_overlap_score( + self, + a: Iterable[str], + b: Iterable[str], + *, + ancestor_weight: float = 0.5, + ) -> float: + """Score how related two tag-sets are, in [0, 1]. + + Direct shared concepts count fully. Shared ANCESTORS count + with ``ancestor_weight`` (default 0.5) — so two articles + tagged ``bears`` and ``manatees`` share ``animals and pets`` + and ``life sciences`` ancestors but neither shares a direct + concept; they get a partial overlap score instead of zero. + + Score is normalised by the smaller of the two expanded sets, + so identical tag-sets always score 1.0. + """ + set_a = set(a or []) + set_b = set(b or []) + if not set_a or not set_b: + return 0.0 + + norm_a = {self.normalize(t) or t for t in set_a} + norm_b = {self.normalize(t) or t for t in set_b} + + anc_a = set() + for t in norm_a: + anc_a |= self.ancestors(t) + anc_b = set() + for t in norm_b: + anc_b |= self.ancestors(t) + + direct = norm_a & norm_b + ancestor = (anc_a & norm_b) | (anc_b & norm_a) | (anc_a & anc_b) + ancestor -= direct # don't double-count + + score = len(direct) + ancestor_weight * len(ancestor) + denom = min(len(norm_a | anc_a), len(norm_b | anc_b)) or 1 + return min(1.0, score / denom) + + # ----- introspection -------------------------------------------- + + def all_pref_labels(self) -> List[str]: + return sorted(self._concepts.keys()) + + def stats(self) -> Dict[str, int]: + roots = sum(1 for c in self._concepts.values() if not c.get("broader")) + return { + "concepts": len(self._concepts), + "roots": roots, + "alt_labels": len(self._alt_to_pref), + } + + +def get_hierarchy() -> SkosHierarchy: + """Singleton accessor.""" + global _INSTANCE + if _INSTANCE is not None: + return _INSTANCE + with _LOCK: + if _INSTANCE is None: + _INSTANCE = SkosHierarchy() + return _INSTANCE diff --git a/project/knowledge/topic_vocabulary.py b/project/knowledge/topic_vocabulary.py new file mode 100644 index 0000000..1e5b465 --- /dev/null +++ b/project/knowledge/topic_vocabulary.py @@ -0,0 +1,514 @@ +""" +topic_vocabulary.py — Seed topic centroids for deterministic auto-tagging. + +Auto-tagging via embedding similarity (Fortemi pattern, refined for +podcast content): instead of asking the LLM to invent tags (which +hallucinates), we maintain a fixed vocabulary of topic phrases, embed +each one with leOS's nomic-text encoder, and tag each podcast segment +by cosine similarity against those centroids. Threshold 0.45 by +default — narrative spoken content scores lower against tag prompts +than written technical text does, so the threshold is gentler than +the 0.7 typical for documentation. + +Adding a new tag is just appending to ``SEED_TOPICS``; the centroid +is computed on first use and cached in ``data/topic_centroids.json``. + +References: + - Fortemi auto-tag.ts (cosine sim vs tag-vocab centroids) + - Extend.ai 2026 chunking-thresholds guide (0.5–0.6 narrative) +""" + +import json +import logging +import os +import threading + +logger = logging.getLogger("topic_vocabulary") + +# Seed topics — each entry is (label, embedding_phrase). +# +# The label is the canonical tag stored on the article. The embedding +# phrase is the longer descriptive sentence whose nomic-text embedding +# becomes the centroid. Richer phrases produce more discriminating +# centroids: "computer science and AI" alone embeds near the corpus +# centre and matches everything; "software programming algorithms data +# structures machine learning" locates a distinct semantic neighborhood. +# +# Format-level descriptors that fit every podcast (e.g. "podcasting +# and broadcasting", "interviewing and storytelling") have been dropped +# — they were attractors that polluted every article's tag set. +SEED_TOPICS = [ + ("biology and life sciences", + "biology life sciences cell organism species genome physiology"), + ("neuroscience and the brain", + "neuroscience brain neurons synapses cortex cognition neural"), + ("psychology and mental health", + "psychology mental health depression anxiety therapy emotion behavior"), + ("evolution and natural selection", + "evolution natural selection adaptation speciation Darwin fitness ancestry"), + ("paleontology and fossils", + "paleontology fossils dinosaurs prehistoric extinct rocks ancient bones"), + ("astronomy and the cosmos", + "astronomy astrophysics telescope spectroscopy stellar photometry " + "exoplanet supernova nebula pulsar redshift parsec Hubble JWST " + "radio-telescope interferometry binary-star parallax magnitude", + # keyword guard: at least one astronomy-specific term must appear + # in the article summary before this tag is accepted. + # "cosmos", "cosmology", "universe", "orbit", "light-year", + # "constellation" are intentionally excluded from the guard list — + # all carry heavy metaphorical usage in motivational/podcast + # language and would generate false positives. + {"require_any_keyword": [ + "telescope", "exoplanet", "supernova", "nebula", "pulsar", + "redshift", "parsec", "hubble", "jwst", "spectroscopy", + "astrophysics", "photometry", "stellar", "asteroid", "comet", + "galaxy", "galaxies", "quasar", "neutron star", "binary star", + "radio telescope", "interferometry", "parallax", "magnitude", + ]}), + ("physics and quantum theory", + "physics quantum mechanics particles energy relativity field forces"), + ("chemistry and materials", + "chemistry molecules reactions atoms compounds materials elements"), + ("mathematics and logic", + "mathematics logic numbers proof equations theorems geometry algebra"), + ("ecology and environment", + "ecology environment ecosystems habitat biodiversity species conservation"), + ("climate change and sustainability", + "climate change global warming sustainability carbon emissions renewable"), + ("geology and earth science", + "geology earth science rocks tectonics minerals volcanoes mountains"), + ("engineering and technology", + "engineering technology mechanical electrical hardware design systems"), + ("computer science and AI", + "software programming algorithms data structures machine learning" + " artificial intelligence neural networks code"), + ("medicine and human health", + "medicine doctor disease diagnosis treatment hospital patient surgery"), + ("genetics and DNA", + "genetics DNA genes chromosomes heredity mutation genome inheritance"), + ("history and archaeology", + "history archaeology ancient civilizations empires artifacts ruins"), + ("anthropology and culture", + "anthropology culture rituals customs societies tribes traditions"), + ("philosophy and ethics", + "philosophy ethics morality meaning consciousness existence wisdom"), + ("religion and spirituality", + "religion spirituality god prayer faith church meditation soul"), + ("comedy and humor", + "comedy humor jokes stand up funny laughter sketches pranks"), + ("education and learning", + "education learning students teachers school classroom curriculum study"), + ("language and linguistics", + "language linguistics grammar syntax phonology speech accent dialect"), + ("music and sound", + "music sound song melody rhythm instruments composer performance"), + ("film and television", + "film television movies actors directors cinema episodes screenwriting"), + ("art and visual culture", + "art visual painting sculpture artists galleries aesthetic design"), + ("literature and books", + "literature books novels authors writing reading fiction poetry"), + ("food and nutrition", + "food nutrition cooking recipes diet ingredients vitamins meals"), + ("fitness and exercise", + "fitness exercise workout strength cardio gym training muscles"), + ("addiction and recovery", + "addiction recovery substance abuse alcohol drugs dependence relapse"), + ("psychedelics and altered states", + "psychedelics psilocybin LSD ayahuasca altered consciousness trip mushrooms"), + ("consciousness and meditation", + "consciousness meditation mindfulness awareness mental practice attention"), + ("relationships and dating", + "relationships dating love romance partner couple marriage intimacy"), + ("family and parenting", + "family parenting children kids parents siblings home upbringing"), + ("career and work", + "career work job profession workplace employment colleagues hiring"), + ("money and economics", + "money economics finance markets investment dollars wealth income"), + ("politics and government", + "politics government election democracy president congress policy"), + ("law and justice", + "law justice courts judges trial verdict crime legal attorneys"), + ("war and military", + "war military soldiers battles armies weapons combat strategy"), + ("urban planning and architecture", + "urban planning architecture cities buildings infrastructure design streets"), + ("transportation and travel", + "transportation travel cars trains planes journey trip tourism"), + ("sports and athletics", + "sports athletics game team players competition match league championship"), + ("gaming and play", + "gaming videogames play console controller multiplayer board games"), + ("agriculture and farming", + "agriculture farming crops livestock harvest soil rural farmer"), + ("animals and pets", + "animals pets dogs cats wildlife mammals creatures domestic"), + ("insects and arthropods", + "insects arthropods bugs spiders bees ants beetles invertebrates"), + ("plants and botany", + # Phrase includes foraging vocabulary ("edible wild fungi berries + # identification") because field-botany episodes use that register, + # not the indoor/academic terms ("photosynthesis", "garden") that + # would anchor a shorter phrase too close to the corpus centre. + "plants botany trees flowers leaves seeds fungi foraging edible wild " + "herbs berries mushrooms identification ecological botany vegetation"), + ("birds and ornithology", + "birds ornithology feathers wings flight nesting species bird watching"), + ("marine life and oceans", + "marine ocean fish whales dolphins coral reef sharks sea creatures"), + ("geography and places", + "geography places countries cities landscapes regions mountains rivers"), + ("social media and the internet", + "social media internet twitter facebook online platforms users posts"), + ("privacy and security", + "privacy security encryption surveillance data hackers passwords cyber"), + ("death and dying", + "death dying mortality grief funeral end of life loss"), + ("sleep and dreams", + "sleep dreams insomnia rem nightmares circadian rest unconscious"), + ("memory and learning", + "memory learning recall memorization neural plasticity cognition"), + ("emotion and affect", + "emotion affect feelings mood happiness fear anger sadness"), + ("creativity and inspiration", + "creativity inspiration imagination artistic ideas innovation original"), + ("science communication", + "science communication public outreach explanation popular science"), + # "interviewing and storytelling" and "podcasting and broadcasting" + # are intentionally absent — format descriptors that match every + # episode equally and pollute every article's tag set. + ("conservation and wildlife", + "conservation wildlife endangered species habitat protection biology"), + ("oncology and cancer", + "cancer oncology tumor chemotherapy radiation screening colonoscopy patient"), + ("anatomy and physiology", + "anatomy physiology organs body systems cells tissues function"), + ("microbiome and gut health", + "microbiome gut bacteria intestines fermentation digestion probiotics"), + ("aging and longevity", + "aging longevity lifespan senescence elderly healthspan retirement"), + ("identity and gender", + "identity gender sexuality queer transgender feminism representation"), + ("race and ethnicity", + "race ethnicity racism diversity culture immigration heritage"), + ("trauma and healing", + "trauma healing PTSD recovery therapy abuse resilience grief"), +] + + +_CENTROID_FILE = "topic_centroids.json" +_LOCK = threading.Lock() +_CENTROIDS = None # name -> list[float] (768d nomic-text) + +# Vectorised companion to _CENTROIDS, built lazily via ``_get_matrix``. +# Pre-normed so cosine reduces to a single BLAS GEMV (matrix @ query). +# At ~70 centroids on 768-dim float32 the GEMV is ~2-3 µs vs ~1.2 ms +# for the Python-loop cosine. +_MATRIX = None # np.ndarray (N, 768) float32, unit-normed rows +_NAMES = None # list[str] aligned to _MATRIX rows + +# Optional per-topic keyword guards — built once from SEED_TOPICS at import +# time. Each entry is a ``{"require_any_keyword": [...]}`` dict extracted +# from the optional third element of a SEED_TOPICS tuple. When a guard is +# present, at least one listed keyword must appear (case-insensitive) in the +# article text before the tag is accepted, regardless of cosine score. +# Guards block the "narrative cosmic awe" class of false positives where an +# embedding drifts toward a centroid because the podcast host uses the +# centroid's metaphor-prone vocabulary in an unrelated context. +_KEYWORD_GUARDS: dict = {} # label -> {"require_any_keyword": [...]} + + +def _build_keyword_guards() -> dict: + guards: dict = {} + for entry in SEED_TOPICS: + if not isinstance(entry, tuple) or len(entry) < 3: + continue + label = entry[0] + guard = entry[2] + if isinstance(guard, dict) and "require_any_keyword" in guard: + guards[label] = guard + return guards + + +_KEYWORD_GUARDS = _build_keyword_guards() + + +def _cache_path(): + base = os.environ.get("LEOS_DATA_DIR") + if not base: + base = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "data", + ) + return os.path.join(base, _CENTROID_FILE) + + +def _load_cached(): + path = _cache_path() + if not os.path.exists(path): + return None + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except Exception: + return None + + +def _save_cached(centroids): + path = _cache_path() + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(centroids, f) + except Exception as e: + logger.warning("failed to cache topic centroids: %s", e) + + +def _build_centroids(): + """Embed each SEED_TOPICS phrase with nomic-text. Each entry is + a (label, embedding_phrase) pair; the embedding phrase carries + extra anchor words so the centroid lands in a discriminating + semantic region instead of near the corpus center.""" + import embeddings + out = {} + for entry in SEED_TOPICS: + # Backwards compat: a bare string still works as both label + # and phrase; new entries use a tuple. + if isinstance(entry, str): + label, phrase = entry, entry + else: + label, phrase = entry[0], entry[1] + try: + vec = embeddings.embed_text(phrase) + if vec is not None: + out[label] = list(map(float, vec)) + except Exception as e: + logger.warning("topic embed failed for %r: %s", label, e) + return out + + +def invalidate(): + """Drop both the centroid dict and the matmul matrix. Call after + rebuilding the seed vocabulary on disk so the next ``suggest_tags`` + rebuilds fresh.""" + global _CENTROIDS, _MATRIX, _NAMES + with _LOCK: + _CENTROIDS = None + _MATRIX = None + _NAMES = None + + +def get_centroids(): + """Return the topic-name → embedding dict, building on first call.""" + global _CENTROIDS + with _LOCK: + if _CENTROIDS is not None: + return _CENTROIDS + cached = _load_cached() + if cached: + _CENTROIDS = cached + return _CENTROIDS + logger.info("building topic centroids (one-time, %d topics)", + len(SEED_TOPICS)) + _CENTROIDS = _build_centroids() + if _CENTROIDS: + _save_cached(_CENTROIDS) + return _CENTROIDS + + +def cosine(a, b): + """Plain Python cosine on raw lists. Retained for backward + compatibility with any external callers — the hot path + (``suggest_tags``) now uses a vectorised matmul instead. + Assumes inputs are already unit-normed (nomic-text outputs are).""" + s = 0.0 + for x, y in zip(a, b): + s += x * y + return s + + +def _get_matrix(): + """Return ``(names, matrix)`` — built once, cached module-level. + + The matrix is the centroid dict stacked as a contiguous (N, 768) + float32 array with unit-normed rows. Pre-normalisation reduces + cosine similarity to a single BLAS GEMV (matrix @ query) — at + N=70 this is ~2-3 µs on CPU, vs the prior Python-loop cosine + which cost ~1.2 ms per article. Memory layout is C-contiguous + so NumPy dispatches directly to BLAS sgemv without a copy. + """ + global _MATRIX, _NAMES + # Fast path — no lock once the matrix is built. Reference + # assignment is atomic under the GIL, so a reader either sees + # the fully-built matrix or the prior None — never a torn write. + if _MATRIX is not None: + return _NAMES, _MATRIX + with _LOCK: + # Re-check under lock in case two threads raced the fast path. + if _MATRIX is not None: + return _NAMES, _MATRIX + cents = get_centroids() + if not cents: + return [], None + try: + import numpy as np + except ImportError: + return list(cents.keys()), None + names = list(cents.keys()) + mat = np.array(list(cents.values()), dtype=np.float32) + # Safety: nomic-text already returns unit vectors, but we + # don't fully trust the cache file (older runs may have + # written un-normed values). One-time normalisation here + # is paid only at first call. + norms = np.linalg.norm(mat, axis=1, keepdims=True) + mat /= np.where(norms > 0, norms, 1.0) + _NAMES = names + _MATRIX = mat + return _NAMES, _MATRIX + + +def suggest_tags(text_vector, threshold=0.50, max_tags=5, + relative_margin=0.85, source_text: str = ""): + """Return [(tag, score), ...] for the top tags whose centroid sim + exceeds ``threshold`` AND is within ``relative_margin`` of the + top tag's score. + + Three filters working together: + * Absolute floor (``threshold``): rejects weak matches. + * Relative margin: keeps only tags within ``relative_margin`` + of the top score, so long tails of mediocre tags get cut. + ``0.85`` means tags scoring below 85% of the leader drop. + * Keyword guard (optional, per-centroid): if a SEED_TOPICS entry + carries a ``{"require_any_keyword": [...]}`` guard, the tag is + only kept when at least one listed keyword appears (case- + insensitive) in ``source_text``. This blocks the "narrative + cosmic awe" class of false positives where an embedding drifts + toward a centroid because podcast hosts use the centroid's + metaphor-prone vocabulary ("universe", "cosmos") in an entirely + different context, but no domain-specific term is present. + Pass the article summary or transcript excerpt as ``source_text`` + to activate guards; omitting it skips keyword checks (backward- + compatible behaviour — callers that don't pass source_text are + unaffected). + + Together, the three filters prevent tag-set bloat: a 0.45 absolute + floor alone admits too many marginal matches; relative margin + trims the long tail; keyword guards block embedding drift from + metaphor-prone vocabulary. + + ``text_vector`` should be a normalised 768d nomic-text embedding. + The function pre-normalises defensively in case the caller + passes a vector from a different source. + + Vectorised path: matmul against the cached centroid matrix (~5 µs + per call). Falls back to the Python-loop path if NumPy is unavailable. + """ + # Pre-compute lowered source text once for all keyword guard checks. + source_lower = source_text.lower() if source_text else "" + + def _keyword_guard_passes(name: str) -> bool: + """Return True if the tag passes its keyword guard (or has none).""" + guard = _KEYWORD_GUARDS.get(name) + if not guard or not source_lower: + return True + required = guard.get("require_any_keyword", []) + return any(kw.lower() in source_lower for kw in required) + + names, matrix = _get_matrix() + if not names: + return [] + if matrix is None: + # NumPy unavailable — fall back to pure-Python cosine loop. + cents = get_centroids() + if not cents: + return [] + scored = [] + for name, c in cents.items(): + score = cosine(text_vector, c) + if score >= threshold and _keyword_guard_passes(name): + scored.append((name, float(score))) + if not scored: + return [] + scored.sort(key=lambda x: x[1], reverse=True) + cutoff = scored[0][1] * float(relative_margin) + return [(n, s) for n, s in scored if s >= cutoff][:max_tags] + + import numpy as np + q = np.asarray(text_vector, dtype=np.float32) + qn = float(np.linalg.norm(q)) + if qn > 0: + q = q / qn + + scores = matrix @ q # (N,) BLAS GEMV + passing_idx = np.where(scores >= threshold)[0] + if passing_idx.size == 0: + return [] + # Apply keyword guards before relative-margin filter so guarded tags + # cannot inflate the top_score used to compute the cutoff. + if source_lower and _KEYWORD_GUARDS: + guarded_mask = np.array( + [_keyword_guard_passes(names[int(i)]) for i in passing_idx], + dtype=bool, + ) + passing_idx = passing_idx[guarded_mask] + if passing_idx.size == 0: + return [] + top_score = float(scores[passing_idx].max()) + cutoff = top_score * float(relative_margin) + keep_mask = scores[passing_idx] >= cutoff + keep_idx = passing_idx[keep_mask] + # Sort survivors descending by score. At N<=10 survivors, + # argsort is faster than argpartition (which has higher fixed + # overhead) and produces the same result. + order = np.argsort(-scores[keep_idx]) + selected = keep_idx[order[:max_tags]] + return [(names[int(i)], float(scores[int(i)])) for i in selected] + + +def suggest_tags_tiered(text_vector, source_text: str = "", max_tags=6): + """Two-tier deterministic fallback for sparse-content articles. + + Tier 1 (strict): threshold=0.50, relative_margin=0.85 — same as the + normal pipeline. Returns immediately if any tags pass. + + Tier 2 (relaxed): threshold=0.45, relative_margin=0.92 — lower absolute + floor, tighter relative margin so only the closest tag(s) survive when + the content is genuinely borderline. Returns immediately if any pass. + + Tier 3 (single-tag guarantee): no margin filter; returns the single + highest-scoring tag if it scores ≥ 0.40. Ensures every article has at + least one retrieval anchor — a tag whose centroid is the nearest in + embedding space is always meaningful, even if it isn't perfect. + + Returns [] only when the best centroid score is < 0.40, indicating the + article is genuinely outside the vocabulary (worth a vocabulary addition + rather than a forced wrong tag). + + This function is ~3 × the cost of a single suggest_tags call (three + matmuls on a pre-cached (N, 768) matrix) ≈ 15 µs on CPU — negligible + per article. No LLM is called; no external I/O. + + Callers that need an LLM safety-net on top can check for an empty + return and call an ASSIST pass themselves — this function intentionally + stays pure-cosine. + """ + # Tier 1 — standard strict pass. + result = suggest_tags(text_vector, threshold=0.50, max_tags=max_tags, + relative_margin=0.85, source_text=source_text) + if result: + return result + + # Tier 2 — relaxed floor, tighter relative margin. + result = suggest_tags(text_vector, threshold=0.45, max_tags=max_tags, + relative_margin=0.92, source_text=source_text) + if result: + return result + + # Tier 3 — single-tag guarantee: best tag above a soft floor, no margin + # filter. Using threshold=0.40 with relative_margin=1.0 (i.e. only the + # top scorer) keeps the result meaningful without admitting a long tail + # of mediocre matches. + result = suggest_tags(text_vector, threshold=0.40, max_tags=1, + relative_margin=1.0, source_text=source_text) + return result # may still be [] if best score < 0.40 diff --git a/project/knowledge/transcript_segmenter.py b/project/knowledge/transcript_segmenter.py new file mode 100644 index 0000000..ef54529 --- /dev/null +++ b/project/knowledge/transcript_segmenter.py @@ -0,0 +1,211 @@ +""" +transcript_segmenter.py — Semantic topic segmentation for transcripts. + +Refines the Fortemi fixed-token chunker into something podcast-aware. +Approach (per Interspeech 2025 PODTILE + Extend.ai 2026 chunking +guidance for narrative content): + + 1. Sentence-tokenise the transcript with a regex tuned for spoken + ASR output (Whisper rarely emits paragraph breaks; sentences are + usually punctuated though). + 2. Embed each sentence with leOS's nomic-text encoder. + 3. Compute cosine similarity between consecutive sentence embeddings + — drops in similarity mark topic boundaries (TextTiling). + 4. Smooth the depth-score curve with a small moving average. + 5. Cut at local minima below the cohesion threshold; merge tiny + segments back into their neighbours so each segment is at least + ``min_chars`` (default 1500) and at most ``max_chars`` (default + 6000) — the chunk-size sweet spot for narrative summarisation + on qwen3:8b with 16k context (Extend.ai 2026). + +The segmenter never calls an LLM; it's purely deterministic vector +math over sentence embeddings. + +Graceful degradation: if embeddings are unavailable, fall back to a +fixed-size chunker (max_chars / 2 with 200 char overlap). +""" + +import logging +import re + +logger = logging.getLogger("transcript_segmenter") + +# Sentence splitter for ASR-style text. Whisper produces text without +# paragraph breaks but mostly with terminal punctuation. We split on +# .?! followed by whitespace + capital, and on long pauses encoded as +# multiple spaces or newlines. Simple and robust for English speech. +_SENT_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z\"'])|\n{2,}") + + +def split_sentences(text): + """Tokenise transcript text into sentences.""" + text = text.strip() + if not text: + return [] + # Collapse multi-space inside sentences but keep paragraph breaks + text = re.sub(r"[ \t]+", " ", text) + parts = _SENT_RE.split(text) + sents = [s.strip() for s in parts if s and s.strip()] + return sents + + +def _moving_average(values, window): + if window <= 1 or len(values) < window: + return list(values) + out = [] + half = window // 2 + for i in range(len(values)): + lo = max(0, i - half) + hi = min(len(values), i + half + 1) + chunk = values[lo:hi] + out.append(sum(chunk) / len(chunk)) + return out + + +def _cosine(a, b): + s = 0.0 + for x, y in zip(a, b): + s += x * y + return s + + +def _fixed_chunks(text, max_chars, overlap_chars): + """Fallback when embeddings aren't available.""" + if len(text) <= max_chars: + return [{ + "index": 0, + "start_char": 0, + "end_char": len(text), + "text": text, + "method": "fixed_full", + }] + out = [] + start = 0 + idx = 0 + while start < len(text): + end = min(start + max_chars, len(text)) + out.append({ + "index": idx, + "start_char": start, + "end_char": end, + "text": text[start:end], + "method": "fixed", + }) + idx += 1 + start += max_chars - overlap_chars + return out + + +def segment_transcript( + transcript, + *, + cohesion_threshold=0.55, + smoothing_window=3, + min_chars=1500, + max_chars=6000, +): + """Split ``transcript`` into topic-coherent segments. + + Returns a list of dicts: + {index, start_char, end_char, text, method} + + ``method`` is "semantic" when sentence embeddings drove the cut, + "fixed" / "fixed_full" when we fell back to a chunked split. + """ + if not transcript or not transcript.strip(): + return [] + + sents = split_sentences(transcript) + if len(sents) <= 1: + return _fixed_chunks(transcript, max_chars, 200) + + # Try to embed sentences; if anything goes wrong, fall back. + try: + import embeddings # leOS subsystem + if not getattr(embeddings, "is_enabled", lambda: False)(): + raise RuntimeError("embeddings disabled") + vecs = [] + for s in sents: + v = embeddings.embed_text(s) + if v is None: + raise RuntimeError("embed_text returned None") + vecs.append(list(map(float, v))) + except Exception as e: + logger.info( + "transcript_segmenter: embedding unavailable (%s) — " + "falling back to fixed chunks", e, + ) + return _fixed_chunks(transcript, max_chars, 200) + + # Cohesion = cosine similarity between adjacent sentences + # (assuming embeddings are L2-normalised, which nomic-text outputs). + cohesion = [] + for i in range(len(vecs) - 1): + cohesion.append(_cosine(vecs[i], vecs[i + 1])) + smoothed = _moving_average(cohesion, smoothing_window) + + # Build a char-offset map so we can express segment bounds in + # original-text coordinates (not sentence coordinates). + offsets = [] + cursor = 0 + src = transcript + for s in sents: + idx = src.find(s, cursor) + if idx < 0: + idx = cursor + offsets.append((idx, idx + len(s))) + cursor = idx + len(s) + + # Cut at local minima below threshold. + boundaries = [0] + for i in range(1, len(smoothed) - 1): + if (smoothed[i] < cohesion_threshold + and smoothed[i] < smoothed[i - 1] + and smoothed[i] <= smoothed[i + 1]): + boundaries.append(i + 1) # cut AFTER sentence i + boundaries.append(len(sents)) + + # Merge to satisfy size constraints. + raw_segments = [] + for k in range(len(boundaries) - 1): + a, b = boundaries[k], boundaries[k + 1] + start = offsets[a][0] + end = offsets[b - 1][1] + raw_segments.append((start, end)) + + merged = [] + for start, end in raw_segments: + if merged and (end - start < min_chars + or (merged[-1][1] - merged[-1][0]) < min_chars): + # Merge with previous if either is too small (and combined + # stays under max_chars + 50% leeway) + prev_start, prev_end = merged[-1] + if (end - prev_start) <= int(max_chars * 1.5): + merged[-1] = (prev_start, end) + continue + merged.append((start, end)) + + # Hard-split any segment that exceeds max_chars + final = [] + for start, end in merged: + if end - start <= max_chars: + final.append((start, end)) + continue + # Slice into max_chars windows, no overlap (sentences already + # collapsed back into one segment). + cut = start + while cut < end: + nxt = min(cut + max_chars, end) + final.append((cut, nxt)) + cut = nxt + + out = [] + for i, (start, end) in enumerate(final): + out.append({ + "index": i, + "start_char": start, + "end_char": end, + "text": transcript[start:end], + "method": "semantic", + }) + return out diff --git a/project/lvm/embeddings.py b/project/lvm/embeddings.py index ac40be1..662d9cc 100644 --- a/project/lvm/embeddings.py +++ b/project/lvm/embeddings.py @@ -1230,14 +1230,101 @@ def get_validation_embeddings_dict(): # Public API: Cache management # --------------------------------------------------------------------------- -def save_cache_to_disk(): +def _safetensors_paths(): + """Return (vectors_path, meta_sidecar_path) derived from _CACHE_FILE.""" + if _CACHE_FILE is None: + return None, None + base, _ = os.path.splitext(_CACHE_FILE) + return base + ".safetensors", base + "_meta.json" + + +def _save_safetensors(content_cache, meta_dict): + """Write content_cache as fp16 safetensors, meta as a JSON sidecar. + + The content_cache (~9k+ SHA-256-keyed 768-dim vectors) lives in the + binary file: 14 MB on disk vs 154 MB JSON, ~0.01s save vs ~3s. Keys + are stored as a (N, 64) uint8 matrix — SHA-256 hex is exactly 64 + ASCII chars per key. fp16 cosine error vs fp32 measures at ~1e-7, + six orders of magnitude below the 0.45 similarity threshold. + + The meta sidecar JSON holds the small auxiliary caches (tool / + preset / validation / image / qwen / imagebind), which together + are <5% of total cache bytes and stay readable for inspection. """ - Persist the in-memory embedding cache to a JSON file on disk. + import numpy as np + from safetensors.numpy import save_file + + vecs_path, meta_path = _safetensors_paths() + if vecs_path is None: + return + + if content_cache: + keys = list(content_cache.keys()) + vecs = np.array( + [content_cache[k] for k in keys], dtype=np.float16 + ) + key_arr = np.zeros((len(keys), 64), dtype=np.uint8) + for i, k in enumerate(keys): + b = k.encode("ascii")[:64] + key_arr[i, : len(b)] = np.frombuffer(b, dtype=np.uint8) + else: + vecs = np.zeros((0, 768), dtype=np.float16) + key_arr = np.zeros((0, 64), dtype=np.uint8) + + save_file( + {"vectors": vecs, "keys": key_arr}, + vecs_path + ".tmp", + ) + os.replace(vecs_path + ".tmp", vecs_path) + + with open(meta_path + ".tmp", "w", encoding="utf-8") as f: + json.dump(meta_dict, f) + os.replace(meta_path + ".tmp", meta_path) + + +def _load_safetensors(): + """Return (content_cache, meta_dict) or (None, None) if the + safetensors cache is missing or unreadable.""" + import numpy as np + + vecs_path, meta_path = _safetensors_paths() + if vecs_path is None or not os.path.exists(vecs_path): + return None, None + + try: + from safetensors.numpy import load_file + loaded = load_file(vecs_path) + keys_arr = loaded.get("keys") + vecs = loaded.get("vectors") + if keys_arr is None or vecs is None: + return None, None + content_cache = {} + for i in range(len(keys_arr)): + kb = keys_arr[i].tobytes().rstrip(b"\x00") + k = kb.decode("ascii", errors="ignore") + if k: + content_cache[k] = vecs[i].astype(np.float32).tolist() + + meta = {} + if os.path.exists(meta_path): + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) + return content_cache, meta + except Exception as e: + logger.warning( + "safetensors cache load failed: %s — falling back to JSON", + e, + ) + return None, None + + +def save_cache_to_disk(): + """Persist the in-memory embedding cache to disk. - Called after bulk embedding operations (rebuild, KB embed-all) and - periodically by the system. The cache file stores content hashes - mapped to embedding vectors, so unchanged content doesn't need - re-embedding across restarts. + Format: ``_embedding_cache.safetensors`` for the large content + vectors (fp16 binary), ``_embedding_cache_meta.json`` for the + small auxiliary caches. Both written via tmp+rename so a crash + can't leave a half-written file visible. """ if _CACHE_FILE is None: return @@ -1245,36 +1332,31 @@ def save_cache_to_disk(): with _LOCK: cache_copy = dict(_CACHE) + meta = { + "version": 6, + "tool_embeddings": dict(_TOOL_EMBEDDINGS), + "preset_embeddings": dict(_PRESET_EMBEDDINGS), + "validation_embeddings": dict(_VALIDATION_EMBEDDINGS), + "image_cache": dict(_IMAGE_CACHE), + "qwen_cache": dict(_QWEN_EMB_CACHE), + } + if _HAS_IMAGEBIND_MODULE: + meta.update(_imagebind.get_cache()) + try: - # Also save tool and preset embeddings in the same file - data = { - "version": 5, - "content_cache": cache_copy, - "tool_embeddings": dict(_TOOL_EMBEDDINGS), - "preset_embeddings": dict(_PRESET_EMBEDDINGS), - "validation_embeddings": dict(_VALIDATION_EMBEDDINGS), - "image_cache": dict(_IMAGE_CACHE), - "qwen_cache": dict(_QWEN_EMB_CACHE), - } - - # Include ImageBind caches if the module is available - if _HAS_IMAGEBIND_MODULE: - data.update(_imagebind.get_cache()) - - # Write atomically: write to temp file, then rename - tmp_path = _CACHE_FILE + ".tmp" - with open(tmp_path, "w", encoding="utf-8") as f: - json.dump(data, f) - os.replace(tmp_path, _CACHE_FILE) - - logger.debug("Saved embedding cache: %d content, %d tools, " - "%d presets, %d validation categories, %d images", - len(cache_copy), len(_TOOL_EMBEDDINGS), - len(_PRESET_EMBEDDINGS), len(_VALIDATION_EMBEDDINGS), - len(_IMAGE_CACHE)) + _save_safetensors(cache_copy, meta) + logger.debug( + "Saved embedding cache: %d content (safetensors), %d tools, " + "%d presets, %d validation, %d images", + len(cache_copy), len(_TOOL_EMBEDDINGS), + len(_PRESET_EMBEDDINGS), len(_VALIDATION_EMBEDDINGS), + len(_IMAGE_CACHE), + ) except Exception as e: - logger.warning("Could not save embedding cache to %s: %s", - _CACHE_FILE, e) + logger.warning( + "Could not save embedding cache to %s: %s", + _CACHE_FILE, e, + ) # --------------------------------------------------------------------------- @@ -2214,57 +2296,40 @@ def _image_cache_set(img_hash, vec): # --------------------------------------------------------------------------- def _load_cache_from_disk(): - """ - Load the embedding cache from the on-disk JSON file. - - Populates _CACHE, _TOOL_EMBEDDINGS, _PRESET_EMBEDDINGS, and - _VALIDATION_EMBEDDINGS from the saved file. If the file doesn't - exist or is corrupt, starts with empty caches (no error). + """Load the embedding cache from ``_embedding_cache.safetensors`` + (fp16 vectors) and ``_embedding_cache_meta.json`` (small auxiliary + caches). No file → empty in-memory caches, no error. """ global _CACHE, _TOOL_EMBEDDINGS, _PRESET_EMBEDDINGS, _VALIDATION_EMBEDDINGS global _IMAGE_CACHE, _QWEN_EMB_CACHE - if _CACHE_FILE is None or not os.path.exists(_CACHE_FILE): + vecs_path, meta_path = _safetensors_paths() + for p in (vecs_path, meta_path): + if p is None: + continue + tmp = p + ".tmp" + if os.path.exists(tmp): + try: + os.remove(tmp) + except OSError: + pass + + content_cache, meta = _load_safetensors() + if content_cache is None: return - try: - with open(_CACHE_FILE, "r", encoding="utf-8") as f: - data = json.load(f) - - if isinstance(data, dict): - version = data.get("version", 0) - - if version >= 1: - # Version 1+ format: separate sections - _CACHE = data.get("content_cache", {}) - _TOOL_EMBEDDINGS = data.get("tool_embeddings", {}) - _PRESET_EMBEDDINGS = data.get("preset_embeddings", {}) - # Version 2 added validation embeddings - _VALIDATION_EMBEDDINGS = data.get("validation_embeddings", {}) - # Version 3 added image cache - _IMAGE_CACHE = data.get("image_cache", {}) - # Version 4 added ImageBind caches - if _HAS_IMAGEBIND_MODULE: - _imagebind.load_cache(data) - # Version 5 added qwen embedding cache - _QWEN_EMB_CACHE = data.get("qwen_cache", {}) - else: - # Legacy format: flat dict of hash -> vector - _CACHE = data - - logger.debug("Loaded embedding cache from disk: %d content, " - "%d tools, %d presets, %d validation categories, " - "%d images", - len(_CACHE), len(_TOOL_EMBEDDINGS), - len(_PRESET_EMBEDDINGS), len(_VALIDATION_EMBEDDINGS), - len(_IMAGE_CACHE)) - - except (json.JSONDecodeError, IOError) as e: - logger.warning("Could not load embedding cache from %s: %s " - "(starting fresh)", _CACHE_FILE, e) - _CACHE = {} - _TOOL_EMBEDDINGS = {} - _PRESET_EMBEDDINGS = {} - _VALIDATION_EMBEDDINGS = {} - _IMAGE_CACHE = {} - _QWEN_EMB_CACHE = {} + _CACHE = content_cache + _TOOL_EMBEDDINGS = meta.get("tool_embeddings", {}) + _PRESET_EMBEDDINGS = meta.get("preset_embeddings", {}) + _VALIDATION_EMBEDDINGS = meta.get("validation_embeddings", {}) + _IMAGE_CACHE = meta.get("image_cache", {}) + _QWEN_EMB_CACHE = meta.get("qwen_cache", {}) + if _HAS_IMAGEBIND_MODULE: + _imagebind.load_cache(meta) + logger.debug( + "Loaded embedding cache: %d content, %d tools, %d presets, " + "%d validation, %d images", + len(_CACHE), len(_TOOL_EMBEDDINGS), + len(_PRESET_EMBEDDINGS), len(_VALIDATION_EMBEDDINGS), + len(_IMAGE_CACHE), + ) diff --git a/project/lvm/reflex_engine.py b/project/lvm/reflex_engine.py index 0f5bf62..98a3e05 100644 --- a/project/lvm/reflex_engine.py +++ b/project/lvm/reflex_engine.py @@ -179,6 +179,23 @@ def __init__(self, displacement_log, config=None): "hit_rate": 0.0, # fires / attempts "total_latency_ms": 0.0, "mean_latency_ms": 0.0, + # Per-gate suppression counts — break down the aggregate + # suppressions value so the bottleneck gate is identifiable + # without code instrumentation. + "suppressions_by_gate": { + "volatility": 0, # _is_volatile pattern match + "min_neighbors": 0, # find_similar returned < min_neighbors + "staleness": 0, # neighbors all older than max_age + "volatile_meta": 0, # neighbors flagged volatile in metadata + "consistency": 0, # displacement consistency below threshold + "match_quality": 0, # find_nearest_response below match_quality + "conformal": 0, # conformal predictor rejected + "drift": 0, # post-fire drift validation rejected + }, + # Sliding window of the last few near-misses; useful for + # diagnosing why the arc is silent on a given workload. + # Each entry: {gate, task_text, best_sim, n_neighbors, at} + "last_5_misses": [], } def set_conformal(self, conformal_reflex): @@ -225,6 +242,7 @@ def try_fire(self, task_vec, task_text=None, metadata=None, if self._is_volatile(task_text, metadata, region_name): self._record_latency(start) self.stats["volatile_skips"] += 1 + self._record_miss("volatility", task_text, 0.0, 0) self._update_hit_rate() return None @@ -232,6 +250,12 @@ def try_fire(self, task_vec, task_text=None, metadata=None, # Step 1: Find similar past tasks neighbors = self.log.find_similar(task_vec, k=15) + # Capture top-1 similarity BEFORE filtering — useful for + # diagnosing whether the floor or the count is the bottleneck. + _best_sim = ( + float(max((n["similarity"] for n in neighbors), default=0.0)) + if neighbors else 0.0 + ) # Filter by similarity floor neighbors = [ @@ -242,6 +266,8 @@ def try_fire(self, task_vec, task_text=None, metadata=None, if len(neighbors) < self.min_neighbors: self._record_latency(start) self.stats["suppressions"] += 1 + self._record_miss("min_neighbors", task_text, _best_sim, + len(neighbors)) self._update_hit_rate() return None @@ -255,6 +281,8 @@ def try_fire(self, task_vec, task_text=None, metadata=None, if (now - freshest) > self.max_age_seconds: self._record_latency(start) self.stats["stale_skips"] += 1 + self._record_miss("staleness", task_text, _best_sim, + len(neighbors)) self._update_hit_rate() return None @@ -269,6 +297,8 @@ def try_fire(self, task_vec, task_text=None, metadata=None, if len(neighbors) < self.min_neighbors: self._record_latency(start) self.stats["volatile_skips"] += 1 + self._record_miss("volatile_meta", task_text, _best_sim, + len(neighbors)) self._update_hit_rate() return None @@ -279,6 +309,8 @@ def try_fire(self, task_vec, task_text=None, metadata=None, if consistency < self.consistency_threshold: self._record_latency(start) self.stats["suppressions"] += 1 + self._record_miss("consistency", task_text, _best_sim, + len(neighbors)) self._update_hit_rate() return None @@ -296,6 +328,8 @@ def try_fire(self, task_vec, task_text=None, metadata=None, if total_w < 1e-10: self._record_latency(start) self.stats["suppressions"] += 1 + self._record_miss("consistency", task_text, _best_sim, + len(neighbors)) self._update_hit_rate() return None @@ -312,6 +346,8 @@ def try_fire(self, task_vec, task_text=None, metadata=None, if nearest is None: self._record_latency(start) self.stats["suppressions"] += 1 + self._record_miss("match_quality", task_text, _best_sim, + len(neighbors)) self._update_hit_rate() return None @@ -319,6 +355,8 @@ def try_fire(self, task_vec, task_text=None, metadata=None, if match_sim < self.match_quality: self._record_latency(start) self.stats["suppressions"] += 1 + self._record_miss("match_quality", task_text, match_sim, + len(neighbors)) self._update_hit_rate() return None @@ -332,6 +370,8 @@ def try_fire(self, task_vec, task_text=None, metadata=None, self._record_latency(start) self.stats["suppressions"] += 1 self.stats["conformal_gates"] += 1 + self._record_miss("conformal", task_text, match_sim, + len(neighbors)) self._update_hit_rate() return None @@ -349,6 +389,8 @@ def try_fire(self, task_vec, task_text=None, metadata=None, self._record_latency(start) self.stats["drift_gates"] += 1 self.stats["suppressions"] += 1 + self._record_miss("drift", task_text, match_sim, + len(neighbors)) self._update_hit_rate() return None @@ -623,7 +665,45 @@ def _record_latency(self, start): ) def _update_hit_rate(self): - """Update the running hit rate.""" - total = self.stats["fires"] + self.stats["suppressions"] + """Update the running hit rate. + + Denominator includes ``volatile_skips`` and ``stale_skips`` so + the rate matches the operator's intuition — fires per total + ``try_fire`` attempts. The previous implementation used only + ``fires + suppressions``, which under-counted by excluding the + skips and produced a misleadingly-high hit rate when a workload + was dominated by volatile content. + """ + total = ( + self.stats["fires"] + + self.stats["suppressions"] + + self.stats["volatile_skips"] + + self.stats["stale_skips"] + ) if total > 0: self.stats["hit_rate"] = self.stats["fires"] / total + + def _record_miss(self, gate, task_text, best_sim, n_neighbors): + """Record a near-miss for diagnostic visibility. + + Increments the per-gate counter and pushes a compact entry + onto the last-5-misses sliding window so an operator can run + ``REFLEX_STATS`` and see WHICH gate is blocking fires (and + with what evidence). Without this the system reports + ``fires=0`` and the operator has no path forward. + + ``gate`` must match a key in ``stats['suppressions_by_gate']``. + """ + bucket = self.stats.setdefault("suppressions_by_gate", {}) + bucket[gate] = bucket.get(gate, 0) + 1 + misses = self.stats.setdefault("last_5_misses", []) + misses.append({ + "gate": gate, + "task_text": (task_text or "")[:80], + "best_sim": round(float(best_sim or 0.0), 3), + "n_neighbors": int(n_neighbors or 0), + "at": time.time(), + }) + # Keep only the most recent 5 entries. + if len(misses) > 5: + del misses[: len(misses) - 5] diff --git a/project/models/assist_engine.py b/project/models/assist_engine.py index 7610a98..8b174a6 100644 --- a/project/models/assist_engine.py +++ b/project/models/assist_engine.py @@ -1,20 +1,7 @@ """ assist_engine.py - Direct transformers loader for the ASSIST model. -Phase L2 of the LLM Direct Loading Plan. - -Loads a small instruction model (default: Qwen/Qwen3-0.6B-Instruct) -directly via HuggingFace transformers, giving us full access to things -Ollama's HTTP API doesn't expose: - - - Token logprobs: real confidence signal per output token - - Hidden layer activations: for probing (L3) and steering (L4) - - Activation injection: steering vectors applied via forward hooks - -The interface mirrors OllamaDriver so coprocessor_bay.py can swap -between them with minimal changes. The key difference: the 'confidence' -field in the return dict is a real number from token logprobs, not the -hardcoded 1.0 we were using before. +Loads a small instruction model directly via transformers, exposing logprobs, hidden states, and steering unavailable through Ollama's HTTP API. Why 0.6B is the right model for this: - Fits comfortably in RAM alongside the four embedding models @@ -1082,9 +1069,6 @@ def unload(self): """ import gc try: - # Note: torch used to be imported here for torch.cuda.empty_cache(). - # After the macOS refactor, device_select.empty_cache() handles - # the cross-platform cache clearing and imports torch internally. if self.model is not None: del self.model self.model = None @@ -1102,11 +1086,7 @@ def unload(self): self._vl_capable = False gc.collect() try: - # device_select.empty_cache() is a cross-platform wrapper: - # calls torch.cuda.empty_cache() on CUDA, torch.mps. - # empty_cache() on Apple Silicon MPS, no-op on CPU. The - # old `torch.cuda.empty_cache()` here was harmless on CPU - # but left MPS users without any cache cleanup. + # Cross-platform cache clear: CUDA / MPS / no-op on CPU. from device_select import empty_cache as _empty_device_cache _empty_device_cache() except Exception: diff --git a/project/models/claude_api.py b/project/models/claude_api.py index 0a5a6fa..51c6f63 100644 --- a/project/models/claude_api.py +++ b/project/models/claude_api.py @@ -855,11 +855,7 @@ def run_claude_agent(agent, messages, on_tool_call=None, on_output=None, "tools_used": set(), } - # --- Phase 17: Mid-execution course correction --- - # Pre-embed the task description so we can check relevance between - # tool rounds. If the agent drifts off-course, we inject a reminder - # message directly into the conversation -- Claude can then self-correct - # or explain why it's doing something that seems unrelated. + # Pre-embed task description to check mid-run relevance; low cosine score triggers a self-correction reminder. _course_correction_sent = False # Only send once _task_vec = None if task_description and len(task_description) > 20: @@ -1016,38 +1012,14 @@ def run_claude_agent(agent, messages, on_tool_call=None, on_output=None, # (this is Claude's expected format for tool results) claude_messages.append({"role": "user", "content": tool_results}) - # --- Phase 17: Mid-execution course correction --- - # - # After 3+ tool rounds with meaningful text output, check if the - # agent is still working on the assigned task. We embed the - # agent's current text and compare against the task embedding. - # - # If cosine similarity is below 0.20, the agent has drifted - # significantly off-course. We inject a reminder as a text - # block inside the tool results message (can't add a separate - # user message -- Claude requires alternating user/assistant). - # - # This is a CONVERSATION injection, not a hard interrupt. Claude - # gets to respond to the reminder naturally. The message: - # - Reminds the agent of the original task - # - Points out the apparent drift - # - Asks it to either refocus or explain its approach - # - Offers the option to request clarification - # - # We only do this once to avoid nagging. If the agent says "yes - # this is relevant because X", we trust it and let it continue. + # After 3+ tool rounds, embed the agent's text and compare against task_vec. Cosine < 0.20 injects a single correction message (only once) into the conversation. if (_task_vec is not None and not _course_correction_sent and round_num >= 2 and full_response and len(full_response) > 200): try: - # Phase 17.1: Vision-aware relevance check. - # If the agent used vision/media tools, their findings - # should be factored into the relevance check. Otherwise - # an agent doing visual analysis might look "off-course" - # because its text output is sparse (e.g. "Analyzing the - # image...") while the real work is in the tool results. + # Include vision-tool findings in the relevance check so image-heavy agents aren't falsely flagged as off-course. _check_text = full_response[:500] if tool_cache and files_path and agent_name: try: diff --git a/project/models/coprocessor_bay.py b/project/models/coprocessor_bay.py index 6942cc6..6ecc375 100644 --- a/project/models/coprocessor_bay.py +++ b/project/models/coprocessor_bay.py @@ -17,29 +17,6 @@ Used for complex reasoning, multi-step tool calling, agentic workflows. -Phase L1 improvements (Ollama Track A): - - Full sampling parameters forwarded (top_k, top_p, repeat_penalty, seed) - - json_mode support for forcing valid JSON from the ESCALATE model - - KV cache reuse: context token IDs threaded through ESCALATE rounds - - keep_alive management: ESCALATE model pinned during active sessions - -Phase L2 additions (Direct ASSIST loading): - - AssistEngine loaded directly via transformers (bypasses Ollama HTTP) - - Real confidence signal from token logprobs (replaces hardcoded 1.0) - - _record_displacement() uses result["confidence"] as outcome weight - - Graceful fallback to Ollama utility path if transformers load fails - -Phase L3 additions (Pre-generation probe): - - GenerationProbe reads ASSIST model hidden state before generating - - If probe says "reflex" with high confidence, short-circuits the call - - probe.record_outcome() called after every routing decision - -Phase L4 additions (Steering vectors): - - SteeringLibrary computes json_output, concise, classification vectors - - assist() picks the right vector based on context type - - Kernel instructions: ASSIST_COMPUTE_STEERING, ASSIST_LIST_STEERING, - ASSIST_TEST_STEERING - Usage: bay = CoprocessorBay(config, kernel) result = bay.escalate(task_vec, context, prompt="Analyze: ...") @@ -187,9 +164,7 @@ def __init__(self, config=None, kernel=None): use_direct = utility_cfg.get("use_direct_loading", True) # Explicit VL declaration from operator config. # true/false → honored over the architecture heuristic - # missing → AssistEngine falls back to heuristic - # Added 2026-04-14 to solve the Qwen3.5 unified-VL case - # where the arch-name heuristic misses the model. + # None if missing — AssistEngine falls back to arch-name heuristic, which misses unified-VL models like Qwen3.5. vl_capable_cfg = utility_cfg.get("vl_capable") # None if missing if use_direct: try: @@ -1132,9 +1107,14 @@ def _call_ollama(self, prompt, messages, context, status_update=None): seed=seed, ) - # Text tasks -- use agent session with tool calling + # Text tasks -- use agent session with tool calling, UNLESS the + # caller explicitly asked for a tool-less single-shot generation + # (e.g. reflection prompts that should monologue, not search). + # Set context['no_tools'] = True to skip the agent loop and go + # straight through to the raw Ollama driver. else: - agent = self._get_agent_session() + no_tools = bool(context.get("no_tools")) + agent = None if no_tools else self._get_agent_session() if agent is not None: if messages: @@ -1213,7 +1193,6 @@ def _call_ollama(self, prompt, messages, context, status_update=None): if result.get("ok"): result["response"] = result.get("text", "") result["coprocessor_used"] = "ollama" - # Ollama doesn't give real confidence -- use conservative default if "confidence" not in result: result["confidence"] = 0.8 else: @@ -1332,9 +1311,7 @@ def _record_displacement(self, task_vec, result, context, metadata): response_vec = embed_result["vector"] - # L2: Use real confidence signal instead of hardcoded 1.0. - # Default 0.8 is conservative: slightly less than "certain" - # since we can't verify quality without ground truth. + # Default 0.8: conservative confidence for Ollama calls that don't expose logprobs. outcome = float(result.get("confidence", 0.8)) # Clamp to valid range just in case outcome = max(0.0, min(1.0, outcome)) diff --git a/project/models/model_certification.py b/project/models/model_certification.py index 096eec9..f477ab4 100644 --- a/project/models/model_certification.py +++ b/project/models/model_certification.py @@ -56,24 +56,9 @@ def get_progress(): # ================================================================ -# Main-agent mutex (2026-04-14) -# -# The main agent model (e.g. qwen3.5:9b on GPU) is a singleton -# resource. Only one thing can run on it at a time: either a -# certification benchmark suite OR a live agent chat session. -# Running both concurrently would produce interleaved generations, -# tool-call confusion, and VRAM pressure. -# -# This lock enforces that contract. Certification acquires it for -# the full duration of a cert run (can be 2-5 minutes). Chat -# callers probe main_agent_in_use() before invoking the main agent -# and, if True, degrade to intern-only for that request. -# -# The lock is module-level so it's visible from agent_chat_api -# (for the probe) and from server.py's startup thread (to run cert -# safely). It intentionally does NOT cover intern/ASSIST calls — -# those run on a separate CPU model and have their own concurrency -# characteristics. +# Serializes access to the main agent model. Certification and live chat are +# mutually exclusive — concurrent use causes interleaved generations and VRAM +# pressure. Does NOT cover intern/ASSIST calls. # ================================================================ _main_agent_lock = threading.Lock() diff --git a/project/models/ollama_driver.py b/project/models/ollama_driver.py index bf6197b..42dcad3 100644 --- a/project/models/ollama_driver.py +++ b/project/models/ollama_driver.py @@ -16,13 +16,6 @@ the full response (simpler for displacement recording). - Timeouts are generous (120s) because local LLMs on CPU can be slow. -Phase L1 additions: - - Full sampling parameters: top_k, top_p, repeat_penalty, seed, stop - - json_mode: forces valid JSON output via Ollama's format constraint - - KV cache reuse: return context token IDs for follow-up calls - - keep_alive management: keep model resident during active sessions - - pin_model() / unload_model() for session lifecycle management - Usage: driver = OllamaDriver("http://localhost:11434") if driver.is_available(): @@ -44,14 +37,65 @@ import urllib.error -# Module logger. Used by probe_capabilities() and similar best-effort -# calls that should warn on failure rather than crash. Was missing on -# initial check-in -- the logger.debug() call inside probe_capabilities' -# exception handler would NameError, masking the real cause of the probe -# failure. logger = logging.getLogger(__name__) +# ============================================================================ +# Module-global keep_alive override — used by infra/gpu_coordinator.py to +# force Ollama to release VRAM after every call while another GPU consumer +# (Whisper transcription, ImageBind, etc.) holds a lease. +# +# When set, every *.generate / *.chat / *.chat_with_tools / *.structured_chat +# request from THIS process will inject `keep_alive=` into the +# Ollama request body, regardless of what the caller passed. Setting to +# 0 makes Ollama unload the model immediately after responding (per +# https://docs.ollama.com/api/generate). +# +# Read+write are atomic in CPython (single int slot), so no lock needed +# for the simple set/clear/read pattern. The override is stored as +# ``None`` when inactive (so callers fall back to their own keep_alive), +# and as the override value (typically 0) while a lease is active. +# ============================================================================ +_KEEP_ALIVE_OVERRIDE = None + + +def set_keep_alive_override(value): + """Force every subsequent Ollama request to use this keep_alive. + + Pass ``0`` to unload the model after each response. Pass ``None`` or + call :func:`clear_keep_alive_override` to restore caller-supplied + behaviour. + + Returns the previous override (so callers can restore it on exit + via try/finally). + """ + global _KEEP_ALIVE_OVERRIDE + prev = _KEEP_ALIVE_OVERRIDE + _KEEP_ALIVE_OVERRIDE = value + if value is None: + logger.info("Ollama keep_alive override cleared") + else: + logger.info("Ollama keep_alive override set to %r", value) + return prev + + +def clear_keep_alive_override(): + """Convenience: undo a previous :func:`set_keep_alive_override` call.""" + set_keep_alive_override(None) + + +def get_keep_alive_override(): + """Read the current override (or ``None`` if inactive).""" + return _KEEP_ALIVE_OVERRIDE + + +def _resolve_keep_alive(caller_value): + """Pick the effective keep_alive: override wins if set.""" + if _KEEP_ALIVE_OVERRIDE is not None: + return _KEEP_ALIVE_OVERRIDE + return caller_value + + class OllamaDriver: """Driver for Ollama text generation API. @@ -646,8 +690,13 @@ def generate(self, prompt, model=None, system=None, temperature=0.7, body["system"] = system if context: body["context"] = context # KV cache token IDs from previous call - if keep_alive is not None: - body["keep_alive"] = keep_alive + # Resolve keep_alive — module-level override (set by the GPU + # coordinator while a Whisper / ImageBind lease is active) wins + # over the caller's value, so a long-resident LLM auto-evicts + # after each call instead of crowding the leaseholder. + _eff_keep_alive = _resolve_keep_alive(keep_alive) + if _eff_keep_alive is not None: + body["keep_alive"] = _eff_keep_alive try: data = self._post("/api/generate", body) @@ -739,8 +788,13 @@ def chat(self, messages, model=None, system=None, temperature=0.7, } if json_mode: body["format"] = "json" - if keep_alive is not None: - body["keep_alive"] = keep_alive + # Resolve keep_alive — module-level override (set by the GPU + # coordinator while a Whisper / ImageBind lease is active) wins + # over the caller's value, so a long-resident LLM auto-evicts + # after each call instead of crowding the leaseholder. + _eff_keep_alive = _resolve_keep_alive(keep_alive) + if _eff_keep_alive is not None: + body["keep_alive"] = _eff_keep_alive try: data = self._post("/api/chat", body) @@ -891,8 +945,13 @@ def chat_with_tools(self, messages, tools=None, model=None, body["tools"] = tools if json_mode: body["format"] = "json" - if keep_alive is not None: - body["keep_alive"] = keep_alive + # Resolve keep_alive — module-level override (set by the GPU + # coordinator while a Whisper / ImageBind lease is active) wins + # over the caller's value, so a long-resident LLM auto-evicts + # after each call instead of crowding the leaseholder. + _eff_keep_alive = _resolve_keep_alive(keep_alive) + if _eff_keep_alive is not None: + body["keep_alive"] = _eff_keep_alive try: data = self._post("/api/chat", body) @@ -1143,24 +1202,10 @@ def _post_stream(self, path, body, on_chunk=None, With it, we can detect a hang within chunk_timeout seconds and return a meaningful error up the stack. - Runaway-generation ceiling (optional, 2026-04-14): - chunk_timeout + liveness_check only catches WEDGED models - (Ollama stopped sending chunks AND the model was unloaded). - It does NOT catch a different failure: a model that keeps - cheerfully generating tokens forever without ever using a - tool or producing a useful answer. The liveness check - sees a healthy stream, never fires; the per-chunk socket - timeout never trips because chunks keep arriving; the - request runs until the model hits num_predict or the - caller gives up. - - max_total_seconds provides a hard wall-clock ceiling on - the entire streaming read. If the total elapsed time - (measured from just before the loop begins) exceeds this - value, we raise a hang-style RuntimeError so the caller - can distinguish a runaway generation from a normal error. - None = no ceiling, preserving the pre-existing behavior - for callers that haven't opted in. + Runaway-generation ceiling (optional): + max_total_seconds: hard wall-clock ceiling on the entire streaming read. + Catches runaway generations that chunk_timeout+liveness don't catch + because chunks keep arriving. Args: path: API path (e.g. "/api/chat") diff --git a/project/models/vram_manager.py b/project/models/vram_manager.py index fe3376d..041fde4 100644 --- a/project/models/vram_manager.py +++ b/project/models/vram_manager.py @@ -1,5 +1,3 @@ -# vram_manager.py — Adapted - """ vram_manager.py - Centralized GPU/VRAM Resource Manager @@ -9,17 +7,6 @@ - orchestrator.py (needs VRAM for the active agent's LLM) - task_scheduler.py (can trigger either of the above) -It replaces the narrow _unload/_reload functions in comfyui_generate.py -which only knew about one model from config.json. - -Key improvements over the old approach: - - Discovers ALL loaded models via Ollama's /api/ps endpoint - - Unloads ALL of them before ComfyUI (not just the config.json one) - - Reloads the specific model that was active (not always the default) - - Uses empty-prompt loading instead of wasteful "hi" prompt - - Provides a threading lock to prevent concurrent GPU access - - Context manager for clean acquire/release patterns - USAGE: from vram_manager import VRAMManager @@ -116,7 +103,6 @@ def _read_ollama_url(self): server = cfg.get("coprocessors", {}).get("ollama", {}).get( "model_server", "" ) - # Old path fallback if not server: server = cfg.get("llm", {}).get( "model_server", "http://localhost:11434/v1" diff --git a/project/page_agent.html b/project/page_agent.html index 754d987..9a77774 100644 --- a/project/page_agent.html +++ b/project/page_agent.html @@ -1649,6 +1649,7 @@ Agent Desktop Knowledge + Podcasts Plans Scopes Thought diff --git a/project/page_diagnostics.html b/project/page_diagnostics.html index dbc1171..6965eda 100644 --- a/project/page_diagnostics.html +++ b/project/page_diagnostics.html @@ -265,6 +265,7 @@ Agent Desktop Knowledge + Podcasts Plans Scopes Thought diff --git a/project/page_home.html b/project/page_home.html index af212bf..f08147e 100644 --- a/project/page_home.html +++ b/project/page_home.html @@ -320,6 +320,7 @@ Agent Desktop Knowledge + Podcasts Plans Scopes Thought diff --git a/project/page_knowledge.html b/project/page_knowledge.html index 6216499..180e16a 100644 --- a/project/page_knowledge.html +++ b/project/page_knowledge.html @@ -385,6 +385,7 @@ Agent Desktop Knowledge + Podcasts Plans Scopes Thought diff --git a/project/page_plans.html b/project/page_plans.html index 5805bff..29a0eb4 100644 --- a/project/page_plans.html +++ b/project/page_plans.html @@ -530,6 +530,7 @@ Agent Desktop Knowledge + Podcasts Plans Scopes Thought diff --git a/project/page_podcasts.html b/project/page_podcasts.html new file mode 100644 index 0000000..37f398d --- /dev/null +++ b/project/page_podcasts.html @@ -0,0 +1,446 @@ + + + + + +leOS — Podcasts + + + + + + + + +
+

Podcasts

+

Search for a show by name (iTunes lookup) or paste an RSS URL. Pick episodes to ingest — leOS transcribes, diarizes, reviews, and saves to the KB.

+ +
+
+ + +
+ +
+
+ + +
+
Hits Apple's iTunes Search API — no auth, no key. Pick a show below to load its episodes.
+ +
+ +
+
+ + +
+
Direct RSS / Atom feed URL — useful if the show isn't on iTunes or you want a specific feed.
+
+ +
+
+ + +
+ + + + diff --git a/project/page_scopes.html b/project/page_scopes.html index 6897967..7db0a25 100644 --- a/project/page_scopes.html +++ b/project/page_scopes.html @@ -382,6 +382,7 @@ Agent Desktop Knowledge + Podcasts Plans Scopes Thought diff --git a/project/page_settings.html b/project/page_settings.html index ef5cab2..f8331fc 100644 --- a/project/page_settings.html +++ b/project/page_settings.html @@ -218,6 +218,7 @@ Agent Desktop Knowledge + Podcasts Plans Scopes Thought @@ -363,6 +364,16 @@

Settings

+ +
+
🔐 Credentials
+
+
+ Encrypted at rest in data/secrets.enc with a key derived from this host. + Saved tokens are injected into the process environment immediately and persist across restarts. +
+
+
⚙️ System
@@ -731,6 +742,130 @@

Settings

window._lastSavedModel = model; if (model) checkModelCert('ollama-model', 'primary'); }, 500); + +// ─────────────────────────────────────────────────────────────────── +// Credentials section — encrypted token store at data/secrets.enc. +// Backend: /credentials/state, /credentials/save, /credentials/clear, +// /credentials/test/hf in server_routes_credentials.py. +// ─────────────────────────────────────────────────────────────────── +const CRED_KNOWN = [ + { + name: 'HF_TOKEN', + label: 'HuggingFace read token', + help: 'Required for speaker diarization (pyannote.audio). ' + + 'Create at huggingface.co/settings/tokens ' + + '(read access is enough). Then accept terms on ' + + 'pyannote/segmentation-3.0 and ' + + 'speaker-diarization-3.1.', + test: '/credentials/test/hf', + }, +]; + +function credMaskInfo(s) { + if (!s || !s.set) return 'not set'; + return 'stored (' + s.length + ' chars, ending …' + s.tail + ')'; +} + +function renderCredentials(state) { + const root = document.getElementById('credentials-rows'); + if (!root) return; + root.innerHTML = CRED_KNOWN.map(t => { + const m = (state || []).find(s => s.name === t.name) || {}; + return '
' + + '
' + + '
' + t.help + '
' + + '
' + + '' + + '' + + (t.test ? '' : '') + + '' + + '
' + + '
' + credMaskInfo(m) + '
' + + '
'; + }).join(''); +} + +async function credRefresh() { + try { + const r = await fetch('/credentials/state'); + const d = await r.json(); + renderCredentials(d.tokens || []); + } catch (e) { + renderCredentials([]); + } +} + +async function credSave(name, btn) { + const card = btn.closest('[data-cred]'); + const value = card.querySelector('input').value; + const status = card.querySelector('.cred-status'); + status.innerHTML = 'saving…'; + try { + const r = await fetch('/credentials/save', { + method:'POST', headers:{'Content-Type':'application/json'}, + body: JSON.stringify({name, value}), + }); + const d = await r.json(); + if (d.ok) { + status.innerHTML = '✓ saved — ' + + (d.status.set ? 'token stored (' + d.status.length + ' chars, …' + d.status.tail + ')' : 'cleared') + + ''; + card.querySelector('input').value = ''; + setTimeout(credRefresh, 1500); + } else { + status.innerHTML = '✗ ' + (d.error || 'error') + ''; + } + } catch (e) { + status.innerHTML = '✗ ' + e.message + ''; + } +} + +async function credTest(name, endpoint, btn) { + const card = btn.closest('[data-cred]'); + const value = card.querySelector('input').value; + const status = card.querySelector('.cred-status'); + if (!value) { + status.innerHTML = 'paste a token first'; + return; + } + status.innerHTML = 'testing…'; + try { + const r = await fetch(endpoint, { + method:'POST', headers:{'Content-Type':'application/json'}, + body: JSON.stringify({value}), + }); + const d = await r.json(); + if (d.ok) { + status.innerHTML = '✓ valid — ' + + (d.username ? '@' + d.username : 'token accepted') + ''; + } else { + status.innerHTML = '✗ ' + (d.error || 'rejected') + ''; + } + } catch (e) { + status.innerHTML = '✗ ' + e.message + ''; + } +} + +async function credClear(name, btn) { + if (!confirm('Clear stored ' + name + '?')) return; + const card = btn.closest('[data-cred]'); + const status = card.querySelector('.cred-status'); + try { + const r = await fetch('/credentials/clear', { + method:'POST', headers:{'Content-Type':'application/json'}, + body: JSON.stringify({name}), + }); + const d = await r.json(); + if (d.ok) { + status.innerHTML = '✓ cleared'; + setTimeout(credRefresh, 800); + } + } catch (e) { + status.innerHTML = '✗ ' + e.message + ''; + } +} + +credRefresh(); diff --git a/project/page_thought.html b/project/page_thought.html index e8ee961..c27e5f8 100644 --- a/project/page_thought.html +++ b/project/page_thought.html @@ -487,6 +487,7 @@ Agent Desktop Knowledge + Podcasts Plans Scopes Thought diff --git a/project/page_voice.html b/project/page_voice.html index 6db7b5a..3e2317a 100644 --- a/project/page_voice.html +++ b/project/page_voice.html @@ -337,6 +337,7 @@ Agent Desktop Knowledge + Podcasts Plans Scopes Thought diff --git a/project/page_work.html b/project/page_work.html index cd97170..04b3b2c 100644 --- a/project/page_work.html +++ b/project/page_work.html @@ -319,6 +319,7 @@ Agent Desktop Knowledge + Podcasts Plans Scopes Thought diff --git a/project/references/references.json b/project/references/references.json new file mode 100644 index 0000000..2c9a588 --- /dev/null +++ b/project/references/references.json @@ -0,0 +1,57 @@ +{ + "references": [], + "blacklist": [ + { + "id": "bl_0001", + "domain": "dexscreener.com", + "reason": "Blocks bots on the web UI -- saves bandwidth on charts and media. Same data is freely available via their public API.", + "added_by": "seed", + "added_at": "2026-04-27T16:21:59.751107+00:00", + "api_alternative": { + "base_url": "https://api.dexscreener.com", + "docs_url": "https://docs.dexscreener.com/api/reference", + "auth": "none", + "examples": [ + "GET https://api.dexscreener.com/latest/dex/tokens/{address}", + "GET https://api.dexscreener.com/latest/dex/search?q={query}", + "GET https://api.dexscreener.com/latest/dex/pairs/{chain}/{pair}" + ] + } + }, + { + "id": "bl_0002", + "domain": "coingecko.com", + "reason": "Aggressive Cloudflare bot challenge on the web UI. Free public API tier covers most use cases with no auth required.", + "added_by": "seed", + "added_at": "2026-04-27T16:21:59.752104+00:00", + "api_alternative": { + "base_url": "https://api.coingecko.com/api/v3", + "docs_url": "https://www.coingecko.com/api/documentation", + "auth": "none (free tier; demo key optional for higher limits)", + "examples": [ + "GET https://api.coingecko.com/api/v3/simple/price?ids={id}&vs_currencies=usd", + "GET https://api.coingecko.com/api/v3/coins/{id}", + "GET https://api.coingecko.com/api/v3/search?query={query}" + ] + } + }, + { + "id": "bl_0003", + "domain": "reddit.com", + "reason": "Blocks generic User-Agents and serves heavy JS for bot detection. The same content is available as JSON by appending .json to any URL or by hitting old.reddit.com.", + "added_by": "seed", + "added_at": "2026-04-27T16:21:59.753104+00:00", + "api_alternative": { + "base_url": "https://www.reddit.com", + "docs_url": "https://www.reddit.com/dev/api/", + "auth": "none for read-only public JSON; OAuth for write", + "examples": [ + "GET https://www.reddit.com/r/{subreddit}/hot.json", + "GET https://www.reddit.com/r/{subreddit}/comments/{id}.json", + "GET https://www.reddit.com/search.json?q={query}" + ] + } + } + ], + "seeded": true +} \ No newline at end of file diff --git a/project/rendering/multi_user.py b/project/rendering/multi_user.py index b13dce2..b0f8bdf 100644 --- a/project/rendering/multi_user.py +++ b/project/rendering/multi_user.py @@ -39,7 +39,7 @@ Usage: mgr = UserManager() - mgr.create_user("human_01", "human", "Moose", role="owner") + mgr.create_user("human_01", "human", "alice", role="owner") mgr.create_user("agent_claude", "llm", "Claude", role="agent") session = mgr.get_session("agent_claude") conference = mgr.conference_view("human_01") diff --git a/project/requirements.txt b/project/requirements.txt index e33c3b8..d3c3807 100644 --- a/project/requirements.txt +++ b/project/requirements.txt @@ -48,6 +48,13 @@ python-docx==1.1.2 pytesseract==0.3.13 opencv-python-headless==4.11.0.86 +# --- Speech-to-text --- +# openai-whisper is installed separately by run.bat with +# --no-build-isolation; this is the CTranslate2-backed alternative +# leOS prefers when available (~2x faster, better WER on large-v3). +# Pulls in onnxruntime + ctranslate2 wheels. +faster-whisper==1.2.1 + # --- Data analysis (chart bones, CSV query) --- matplotlib==3.10.3 pandas==2.2.3 diff --git a/project/science/theory_experiments_codec.py b/project/science/theory_experiments_codec.py index 909d013..1cd1d40 100644 --- a/project/science/theory_experiments_codec.py +++ b/project/science/theory_experiments_codec.py @@ -1191,7 +1191,6 @@ def test_steganographic_roundtrip(self): try: import sys - sys.path.insert(0, '/home/claude') if '/home/claude' not in sys.path else None from synesthetic_encoder import pack_payload, unpack_payload, lsb_capacity from PIL import Image except ImportError as e: @@ -1334,7 +1333,6 @@ def test_multichannel_density(self): try: import sys - sys.path.insert(0, '/home/claude') if '/home/claude' not in sys.path else None from synesthetic_encoder import ( encode_as_image, encode_as_depth, encode_as_thermal, encode_as_imu, encode_as_audio, lsb_capacity, diff --git a/project/science/theory_experiments_stereo.py b/project/science/theory_experiments_stereo.py index a951f1d..d4389b9 100644 --- a/project/science/theory_experiments_stereo.py +++ b/project/science/theory_experiments_stereo.py @@ -56,7 +56,6 @@ def test_cross_modal_parallax(self): try: import sys - sys.path.insert(0, '/home/claude') if '/home/claude' not in sys.path else None from synesthetic_encoder import ( encode_as_image, encode_as_depth, displacement_encode, displacement_decode, @@ -256,7 +255,6 @@ def test_autostereogram_dual_read(self): try: import sys - sys.path.insert(0, '/home/claude') if '/home/claude' not in sys.path else None from synesthetic_encoder import ( encode_as_autostereogram, autostereogram_depth_profile ) diff --git a/project/server.py b/project/server.py index c90936d..6e23bd4 100644 --- a/project/server.py +++ b/project/server.py @@ -62,6 +62,23 @@ if os.path.isdir(_sd_path) and _sd_path not in sys.path: sys.path.insert(1, _sd_path) +# Decrypt-and-inject any tokens stored via the credentials UI before +# subsystems that consume os.environ (whisperx loader, monologue +# renderer, etc.) are imported. Tokens already in the parent +# environment win over stored ones — env var override always works. +try: + import leos_secrets + leos_secrets.init(_BASE_DIR) + _injected = leos_secrets.inject_into_environ(overwrite=False) + if _injected: + logging.getLogger("server").info( + "leos_secrets injected: %s", ", ".join(_injected) + ) +except Exception as _se: + logging.getLogger("server").warning( + "leos_secrets init failed (continuing without): %s", _se + ) + logger = logging.getLogger("server") # --------------------------------------------------------------------------- @@ -218,11 +235,12 @@ def _register_route_blueprints(): from server_routes_collab import bp as collab_bp from server_routes_media import bp as media_bp from server_routes_misc import bp as misc_bp + from server_routes_credentials import bp as credentials_bp for bp in (health_bp, pages_bp, scene_bp, kernel_bp, thought_bp, voice_bp, ta_bp, adapter_bp, certification_bp, apps_bp, render_bp, desktop_bp, ports_bp, datasets_bp, sdol_bp, - collab_bp, media_bp, misc_bp): + collab_bp, media_bp, misc_bp, credentials_bp): app.register_blueprint(bp) @@ -2714,6 +2732,24 @@ def _init_ambient_reactor(_kernel): f" Ambient reactor: failed ({are})") + def _init_kb_reflection(_kernel): + """Wire kb.article_saved → autonomous REFLECT session.""" + try: + from kb_reflection_listener import get_kb_reflection_listener + cfg = (_kernel.config or {}).get("kb_reflection", {}) + listener = get_kb_reflection_listener(kernel=_kernel, config=cfg) + if listener is None: + _add_startup_status(" KB reflection listener: skipped") + return + listener.start() + _kernel.kb_reflection_listener = listener + import atexit + atexit.register(listener.stop) + _add_startup_status(" KB reflection listener: OK (started)") + except Exception as e: + _add_startup_status(f" KB reflection listener: failed ({e})") + + def _init_activity_monitor(_kernel): """Subscribe the activity monitor to the signal bus.""" # Heartbeat tracker for idle-based timeout detection. @@ -2928,37 +2964,6 @@ def _skip_flag(): ) - def _init_media_processing_project(): - """Ensure the _media-processing project exists on disk.""" - # Persisted scheduler tasks from prior runs may target - # the _media-processing project. If it was never - # created on disk, those tasks fail with "Project not - # found" errors on every retry cycle. The docstring - # for ensure_media_project() says this is called from - # server.py at startup, but the actual call was - # missing. Calling it here makes the project exist - # before the scheduler loop picks up any stale tasks. - try: - from media_ingest import ensure_media_project - # Must use _BASE_DIR (same as the scheduler's - # workspace_root above on line 8789). The default - # ensure_media_project() falls back to - # state.WORKSPACE which resolves to - # _BASE_DIR/projects — a DIFFERENT directory than - # the scheduler looks in. That mismatch caused - # the "_media-processing not found" retry spam - # even though the project was being created (at - # the wrong location). - if ensure_media_project(workspace_root=_BASE_DIR): - _add_startup_status( - " Media processing project: ready" - ) - except Exception as mpe: - _add_startup_status( - f" Media processing project: init skipped ({mpe})" - ) - - def _init_render_bridge(_kernel): """Register the WI-18 multi-channel render-bridge routes.""" try: @@ -3326,12 +3331,12 @@ def _background_init(): _init_bot_scheduler(_kernel) _init_io_membrane(_kernel) _init_ambient_reactor(_kernel) + _init_kb_reflection(_kernel) _init_activity_monitor(_kernel) _init_observation_ledger(_kernel) _init_api_adapter_report(_kernel) _init_chart_analysis(_kernel) _init_model_certification(_kernel) - _init_media_processing_project() _init_render_bridge(_kernel) _init_agent_chat_api(_kernel) _init_work_tray_summary(_kernel) diff --git a/project/server_routes_credentials.py b/project/server_routes_credentials.py new file mode 100644 index 0000000..507c94d --- /dev/null +++ b/project/server_routes_credentials.py @@ -0,0 +1,157 @@ +"""server_routes_credentials.py — UI for entering API tokens that +persist encrypted across restarts. + +Routes: + GET /credentials — settings form (lists known tokens + by name + masked preview, lets + user enter/replace each) + POST /credentials/save — body: {name, value}; encrypts and + persists; injects into os.environ + so live subsystems pick it up + without restart; clears the + Whisper model cache so the next + transcribe rebuilds with the new + token (relevant for HF_TOKEN → + pyannote diarization). + POST /credentials/clear — body: {name}; deletes the token. + POST /credentials/test/hf — calls HF /whoami-v2 with the + supplied token and returns + {ok, username} or {ok: false, error}. + Does NOT save the token — used to + validate before saving. + +The token value is never echoed back in plaintext. GET /credentials +returns only names + a length + a "tail" (last 4 chars) so the user +can confirm they have the right one without exposing the full secret. +""" + +from __future__ import annotations + +import json +import logging +import urllib.error +import urllib.request +from typing import Any, Dict + +from flask import Blueprint, jsonify, request + +import leos_secrets + +logger = logging.getLogger("credentials") +bp = Blueprint("credentials", __name__) + + +_KNOWN_TOKENS = [ + { + "name": "HF_TOKEN", + "label": "HuggingFace read token", + "help": ( + "Required for speaker diarization (pyannote.audio). " + "Create at https://huggingface.co/settings/tokens — read " + "access is sufficient. Accept the model terms on " + "pyannote/segmentation-3.0 and pyannote/speaker-" + "diarization-3.1 first." + ), + "test_endpoint": "/credentials/test/hf", + }, +] + + +def _mask(value: str) -> Dict[str, Any]: + if not value: + return {"set": False} + tail = value[-4:] if len(value) >= 8 else "****" + return {"set": True, "length": len(value), "tail": tail} + + +def _on_token_changed(name: str) -> None: + """Hot-apply effects when a stored token changes. HF_TOKEN flushes + the Whisper cache so the next transcribe rebuilds whisperx with + pyannote enabled.""" + if name == "HF_TOKEN": + try: + from media_ingest_models import ( + _WHISPER_MODELS as _wm, _WHISPER_LOCK as _wl, + ) + with _wl: + _wm.clear() + logger.info("HF_TOKEN updated — Whisper cache cleared, " + "next transcribe will rebuild with diarization") + except Exception as e: + logger.warning("could not clear whisper cache: %s", e) + + +@bp.route("/credentials/state", methods=["GET"]) +def credentials_state(): + """JSON view of which tokens are set (without values).""" + return jsonify({ + "tokens": [ + {"name": s["name"], **_mask(leos_secrets.get_token(s["name"]) or "")} + for s in _KNOWN_TOKENS + ], + }) + + +@bp.route("/credentials/save", methods=["POST"]) +def credentials_save(): + body = request.get_json(silent=True) or {} + name = (body.get("name") or "").strip() + value = body.get("value") or "" + if not name: + return jsonify({"ok": False, "error": "name required"}), 400 + if name not in {s["name"] for s in _KNOWN_TOKENS}: + return jsonify({"ok": False, "error": f"unknown token: {name}"}), 400 + leos_secrets.set_token(name, value) + if value: + # Inject into env so live subsystems pick it up without restart. + import os + os.environ[name] = value + _on_token_changed(name) + return jsonify({"ok": True, "name": name, "status": _mask(value)}) + + +@bp.route("/credentials/clear", methods=["POST"]) +def credentials_clear(): + body = request.get_json(silent=True) or {} + name = (body.get("name") or "").strip() + if not name: + return jsonify({"ok": False, "error": "name required"}), 400 + leos_secrets.clear_token(name) + import os + os.environ.pop(name, None) + _on_token_changed(name) + return jsonify({"ok": True, "name": name, "status": _mask("")}) + + +@bp.route("/credentials/test/hf", methods=["POST"]) +def credentials_test_hf(): + """Call HF /whoami-v2 with the supplied token and return the + result. Does NOT save the token — pass the raw value in the body + to validate before pressing Save.""" + body = request.get_json(silent=True) or {} + value = body.get("value") or "" + if not value: + return jsonify({"ok": False, "error": "value required"}), 400 + try: + req = urllib.request.Request( + "https://huggingface.co/api/whoami-v2", + headers={"Authorization": f"Bearer {value}"}, + method="GET", + ) + with urllib.request.urlopen(req, timeout=10) as resp: + data = json.loads(resp.read().decode("utf-8")) + return jsonify({ + "ok": True, + "username": data.get("name", ""), + "fullname": data.get("fullname", ""), + "type": data.get("type", ""), + }) + except urllib.error.HTTPError as e: + return jsonify({ + "ok": False, + "error": f"HF rejected token: HTTP {e.code}", + }), 200 + except Exception as e: + return jsonify({"ok": False, "error": str(e)}), 200 + + diff --git a/project/server_routes_pages.py b/project/server_routes_pages.py index d2de039..fe34a17 100644 --- a/project/server_routes_pages.py +++ b/project/server_routes_pages.py @@ -56,6 +56,12 @@ def scopes_page(): return server._serve_page("page_scopes.html") +@bp.route("/podcasts") +def podcasts_page(): + """Serve the Podcasts UI: search shows, fetch RSS, ingest episodes.""" + return server._serve_page("page_podcasts.html") + + @bp.route("/work") def work_page(): """Serve the Work Tray UI (Plan 22). diff --git a/project/subsystems/dreaming_engine.py b/project/subsystems/dreaming_engine.py index cdcedb0..ddffe46 100644 --- a/project/subsystems/dreaming_engine.py +++ b/project/subsystems/dreaming_engine.py @@ -45,10 +45,6 @@ log_map, exp_map, batch_cosine_similarity, ) -# Module logger. Used by the dreaming/idle code paths to report -# deferrals and exceptions. Was missing on initial check-in -- every -# logger.* call in this file used to NameError when its branch fired, -# turning small failures into crashes that hid the real cause. logger = logging.getLogger(__name__) @@ -731,13 +727,13 @@ def on_idle(self, seconds_idle): results["prune"] = self._prune_low_confidence() if seconds_idle > 300: - # Plan 14: Compact stale scopes during idle time. + # Compact stale scopes during idle time (compresses accumulated notes to stay within context budget). # Scopes with lots of accumulated notes get clustered and # summarized so that context assembly stays within budget. results["scope_compact"] = self._compact_stale_scopes() if seconds_idle > 300: - # Plan 14b: Process pending deferred monologue logs. + # Process pending deferred monologue logs (renders narrated thought videos captured during active work). # Renders narrated thought videos from JSONL event logs # that were captured in deferred mode during agent work. results["deferred_monologues"] = self._process_deferred_monologues() @@ -750,7 +746,7 @@ def on_idle(self, seconds_idle): results["reflections"] = self._render_pending_reflections() if seconds_idle > 600: - # Plan 14b: Visual practice — idle-time splat fitting. + # Visual practice — idle-time splat fitting (embedding feedback loop for concept visualization). # The system teaches itself to draw by practicing concept # visualization with the embedding feedback loop. results["visual_practice"] = self._run_visual_practice() @@ -1360,7 +1356,7 @@ def _run_conservation_check(self): } def _compact_stale_scopes(self): - """Plan 14: Compact scopes whose note count exceeds the + """Compact scopes whose note count exceeds the compact_after threshold in their notes_policy. Called during idle cycles (300s+) so it doesn't interfere with @@ -1408,7 +1404,7 @@ def _compact_stale_scopes(self): return {"scopes_compacted": compacted} # ------------------------------------------------------------------ - # Plan 14b: Deferred monologue rendering (idle-time) + # Deferred monologue rendering (idle-time) # ------------------------------------------------------------------ def _process_deferred_monologues(self): @@ -1620,7 +1616,7 @@ def _render_pending_reflections(self): } # ------------------------------------------------------------------ - # Plan 14b: Visual practice (idle-time skill building) + # Visual practice (idle-time skill building) # ------------------------------------------------------------------ def _run_visual_practice(self): diff --git a/project/subsystems/estimate_plan.py b/project/subsystems/estimate_plan.py index 12b17cc..7c413b6 100644 --- a/project/subsystems/estimate_plan.py +++ b/project/subsystems/estimate_plan.py @@ -169,8 +169,6 @@ def call(params, context=None): return json.dumps(output, indent=2, default=str) -# --------------------------------------------------------------------------- -# Helpers # --------------------------------------------------------------------------- def _format_duration(seconds): diff --git a/project/subsystems/estimate_task.py b/project/subsystems/estimate_task.py index 3ea06b5..d959786 100644 --- a/project/subsystems/estimate_task.py +++ b/project/subsystems/estimate_task.py @@ -308,8 +308,6 @@ def _handle_timeout(params, context): }, indent=2) -# --------------------------------------------------------------------------- -# Helpers # --------------------------------------------------------------------------- def _format_duration(seconds): diff --git a/project/subsystems/estimation_engine.py b/project/subsystems/estimation_engine.py index 1d6cbba..8ab751e 100644 --- a/project/subsystems/estimation_engine.py +++ b/project/subsystems/estimation_engine.py @@ -649,8 +649,6 @@ def compute_timeout(agent=None, model=None, step_description=None, return default_timeout -# --------------------------------------------------------------------------- -# Helpers # --------------------------------------------------------------------------- def _compute_critical_path_simple(step_estimates, dependencies): diff --git a/project/subsystems/kb_void_scheduled.py b/project/subsystems/kb_void_scheduled.py new file mode 100644 index 0000000..394f8fb --- /dev/null +++ b/project/subsystems/kb_void_scheduled.py @@ -0,0 +1,42 @@ +"""kb_void_scheduled.py — Lightweight wrapper to run KB_VOID_DETECT +on a periodic schedule independent of the heavier self-improvement +cycle. + +Wired via a recurring scheduler task (data/scheduler/task_kb_void_scan.json) +that calls ``run`` every 30 minutes. Cheap operation — pure-geometric +void detection, no LLM — so frequent firing is fine. +""" + +import logging + +logger = logging.getLogger("kb_void_scheduled") + + +def run(base_dir=None, config=None, kernel=None): + """Trigger KB_VOID_DETECT and log a one-line summary.""" + if kernel is None: + try: + from state import get_kernel + kernel = get_kernel() + except Exception as e: + logger.warning("kb_void_scheduled: no kernel available (%s)", e) + return {"ok": False, "error": "no kernel"} + if kernel is None: + return {"ok": False, "error": "no kernel"} + + result = kernel.execute("KB_VOID_DETECT") + if not result.get("ok"): + logger.warning("KB_VOID_DETECT failed: %s", result.get("error")) + return result + + voids = result.get("voids") or [] + coverage = result.get("coverage_score") or 0.0 + logger.info( + "KB void scan: %d voids, %.0f%% coverage", + len(voids), coverage * 100, + ) + return { + "ok": True, + "voids_found": len(voids), + "coverage_score": coverage, + } diff --git a/project/subsystems/learning_evaluator.py b/project/subsystems/learning_evaluator.py index 98f6f7c..7d1f605 100644 --- a/project/subsystems/learning_evaluator.py +++ b/project/subsystems/learning_evaluator.py @@ -511,7 +511,7 @@ def eval_timing(self, exercise_path=None): return result def eval_ambient_reactor(self): - """9. Ambient Reactor Health (Plan 20). + """9. Ambient Reactor Health. Reads AmbientReactor.get_stats() via the AMBIENT_STATS kernel instruction. Evaluates: diff --git a/project/subsystems/perf_store.py b/project/subsystems/perf_store.py index afcd8e1..61db1b8 100644 --- a/project/subsystems/perf_store.py +++ b/project/subsystems/perf_store.py @@ -106,8 +106,6 @@ def _invalidate_cache(): _cache_dirty = True -# --------------------------------------------------------------------------- -# Helpers # --------------------------------------------------------------------------- def _now_iso(): diff --git a/project/subsystems/predictive_coding.py b/project/subsystems/predictive_coding.py index 29b7e9c..84ee397 100644 --- a/project/subsystems/predictive_coding.py +++ b/project/subsystems/predictive_coding.py @@ -266,7 +266,7 @@ def get_task_embedding(task_text, speed_tier="fast"): System 1 gate checks) - "balanced": 512-dim qwen MRL (good precision/speed tradeoff) - "full": 1024-dim qwen (maximum precision, for System 2) - - "nomic": 768-dim nomic only (legacy fallback) + - "nomic": 768-dim nomic only (fallback when qwen is unavailable) Args: task_text: The task description to embed. diff --git a/project/subsystems/reorder_for_efficiency.py b/project/subsystems/reorder_for_efficiency.py index 0efb8c2..ac5db24 100644 --- a/project/subsystems/reorder_for_efficiency.py +++ b/project/subsystems/reorder_for_efficiency.py @@ -213,8 +213,6 @@ def analyze_reorder_savings(steps, confidence_threshold=0.7): } -# --------------------------------------------------------------------------- -# Helpers # --------------------------------------------------------------------------- def _get_model(step): diff --git a/project/subsystems/task_scheduler.py b/project/subsystems/task_scheduler.py index 015c74f..00c8884 100644 --- a/project/subsystems/task_scheduler.py +++ b/project/subsystems/task_scheduler.py @@ -1,5 +1,3 @@ -# task_scheduler.py — Adapted - """ task_scheduler.py - Task Scheduler Engine @@ -19,7 +17,7 @@ - tool: Execute a registered tool with given parameters. - inbox: Send a message to the user's inbox. - agent_msg: Send a message to an agent's inbox file. - - message: Queue a pending message to a project (legacy, lightweight). + - message: Queue a pending message to a project (lightweight, no LLM session required). - director_run: Run a full Director session on a project. Acquires the session lock, builds agents, calls run_directed(), and releases the lock when done. Supports an optional @@ -2154,8 +2152,6 @@ def _process_pending_messages(self): traceback.print_exc() # ------------------------------------------------------------------ - # Helpers - # ------------------------------------------------------------------ def _build_tool_config(self, project_id): """ diff --git a/project/subsystems/watchdog.py b/project/subsystems/watchdog.py index 69b4a20..f3e50a0 100644 --- a/project/subsystems/watchdog.py +++ b/project/subsystems/watchdog.py @@ -205,9 +205,8 @@ def _register_all(self): is_enabled_fn=lambda: not getattr(k, '_drift_disabled', False), ) - # Plan 20: ambient reactor. Same hasattr guard pattern as - # drift_detector so the watchdog only registers when the - # reactor was actually wired onto the kernel at boot. + # Ambient reactor — same hasattr guard pattern as drift_detector: + # only registers when the reactor was actually wired onto the kernel at boot. if hasattr(k, 'ambient_reactor') and k.ambient_reactor is not None: self._register("ambient_reactor", check_fn=lambda: self._check_ambient_reactor(), @@ -384,7 +383,7 @@ def _check_ambient_reactor(self): """Verify the ambient reactor is alive and not catastrophically backed up. - Plan 20 check. Three things are verified: + Three things are verified: 1. The reactor's background subscription is registered (is_running() returns True). 2. get_stats() returns a well-formed dict. diff --git a/project/tests/core/test_substrate_gather.py b/project/tests/core/test_substrate_gather.py index 42f19fa..e629e22 100644 --- a/project/tests/core/test_substrate_gather.py +++ b/project/tests/core/test_substrate_gather.py @@ -6,9 +6,9 @@ Test scenarios: - A. Xavier query with strong fresh KB coverage + A. EXAMPLECOIN query with strong fresh KB coverage → action=format_from_kb (the headline scenario) - B. Xavier query with stale KB + successful fresh API dispatch + B. EXAMPLECOIN query with stale KB + successful fresh API dispatch → action=format_combined (stale branch) C. Fresh token query with no KB, recipe binds, API returns data → action=format_from_api @@ -148,13 +148,13 @@ def check(condition, label): sg._CACHED_TREE = None # ------------------------------------------------------------ - # Scenario A: Xavier-style, strong fresh KB → format_from_kb + # Scenario A: EXAMPLECOIN-style, strong fresh KB → format_from_kb # ------------------------------------------------------------ print("\n--- A: strong fresh KB ---") now = time.time() entries = [ - MockEntry("kb_1", "Xavier Token Report", - "Detailed report on the Xavier Solana token " + MockEntry("kb_1", "EXAMPLECOIN Token Report", + "Detailed report on the EXAMPLECOIN Solana token " "69G8... with price, holders, liquidity...", timestamp=now - 3600), # 1 hour old ] @@ -209,7 +209,7 @@ def check(condition, label): print("\n--- B: strong stale KB + crypto + time-sensitive ---") old_ts = now - (48 * 3600) # 48 hours old — past crypto freshness entries = [ - MockEntry("kb_1", "Xavier Token Report", "... stale data", + MockEntry("kb_1", "EXAMPLECOIN Token Report", "... stale data", timestamp=old_ts), ] scored = [(0.85, entries[0])] @@ -414,8 +414,8 @@ def check(condition, label): # ------------------------------------------------------------ print("\n--- H: partial KB + recipe (self-pruned) ---") entries = [ - MockEntry("kb_mid", "Partial Xavier info", - "Some background on Xavier", + MockEntry("kb_mid", "Partial EXAMPLECOIN info", + "Some background on EXAMPLECOIN", timestamp=now - 100), ] scored = [(0.60, entries[0])] # between 0.50 and 0.75 = partial @@ -547,7 +547,7 @@ def check(condition, label): f"K.4: stale with no entities uses no-policy penalty " f"(got {c_stale_none})") - # Xavier-like scenario: stale crypto but 2 APIs confirmed + # EXAMPLECOIN-like scenario: stale crypto but 2 APIs confirmed # (the format_combined case with strong stale KB + APIs) c_xavier_recovered = sg._compute_confidence( {"kb_top_score": 0.85, @@ -705,7 +705,7 @@ def check(condition, label): # Entity-first: solana_addr_pump → ["text_store"] (today) now = time.time() entries = [MockEntry( - "kb_1", "Xavier", "info", + "kb_1", "EXAMPLECOIN", "info", timestamp=now - 3600)] scored = [(0.85, entries[0])] patch_kb_module(ScriptedKB(entries=entries, semantic_scores=scored)) diff --git a/project/tools/kb_nearest.py b/project/tools/kb_nearest.py new file mode 100644 index 0000000..eee7f2d --- /dev/null +++ b/project/tools/kb_nearest.py @@ -0,0 +1,135 @@ +"""kb_nearest.py — Find the K nearest KB entries to a vector or entry. + +Lightweight wrapper around ``KnowledgeBase.nearest()``: a single BLAS +GEMV against the cached embedding matrix. Replaces ad-hoc Python +loops over ``kb.entries`` for callers that just want +"top-K similar to X" without invoking the full hybrid search stack +(BM25 + nomic + qwen + RRF + MMR). + +Args (JSON): + entry_id — kb article id (mutually exclusive with query_vec) + query_vec — list[float] 768-dim embedding (mutually exclusive with entry_id) + k — max results (default 10) + threshold — minimum cosine similarity (optional, default no floor) + exclude_ids — list[str] to drop from results (the source entry id is + always added automatically when entry_id is supplied) + +Returns: + {"entries": [{"id", "score", "title", "summary"}, ...]} +""" + +from __future__ import annotations + +import json +from typing import Any, Dict + + +try: + from base_tool import BaseTool +except ImportError: # pragma: no cover — older deploys + class BaseTool: # type: ignore[no-redef] + description = "" + parameters: list = [] + + def call(self, params, **kwargs): + raise NotImplementedError + + +TOOL_DESC = ( + "Find the K KB entries whose embeddings are nearest to a source " + "vector or another KB entry. Single matmul against the cached " + "embedding matrix — no scoring loop on the caller side." +) + + +TOOL_PARAMETERS = [ + {"name": "entry_id", "type": "string", + "description": "KB article id whose embedding becomes the query.", + "required": False}, + {"name": "query_vec", "type": "array", + "description": "Raw 768-dim float vector (alternative to entry_id).", + "required": False}, + {"name": "k", "type": "integer", + "description": "Maximum results (default 10).", "required": False}, + {"name": "threshold", "type": "number", + "description": "Minimum cosine similarity (default: no floor).", + "required": False}, + {"name": "exclude_ids", "type": "array", + "description": "Article ids to drop from results.", + "required": False}, +] + + +class KBNearestTool(BaseTool): + description = TOOL_DESC + parameters = TOOL_PARAMETERS + + def call(self, params, **kwargs): + # Accept dict OR JSON string for compatibility with both the + # agent-loop (string) and direct kernel POST (dict) call paths. + if isinstance(params, str): + try: + params = json.loads(params) + except Exception: + return json.dumps({"status": "error", + "error": "Could not parse parameters as JSON"}) + params = params or {} + + entry_id = params.get("entry_id") or None + query_vec = params.get("query_vec") or None + k = int(params.get("k") or 10) + threshold = params.get("threshold") + if threshold is not None: + try: + threshold = float(threshold) + except (TypeError, ValueError): + threshold = None + exclude_ids = list(params.get("exclude_ids") or []) + + if not entry_id and not query_vec: + return json.dumps({"status": "error", + "error": "Provide entry_id or query_vec"}) + + # Prefer injected _kb_instance (agent context); fall back to + # the module singleton for direct /kernel/execute calls. + kb = (self.cfg.get("_kb_instance") if hasattr(self, "cfg") else None) + if kb is None: + try: + from knowledge import get_kb + kb = get_kb() + except Exception: + pass + if kb is None: + return json.dumps({"status": "error", + "error": "Knowledgebase not available"}) + + # Resolve query vector + vec = None + if entry_id: + entry = kb.get_entry(entry_id) + if entry is None: + return json.dumps({"status": "error", + "error": f"Entry {entry_id} not found"}) + try: + from knowledge import _get_entry_embedding + except Exception: + _get_entry_embedding = None # type: ignore[assignment] + if _get_entry_embedding is None: + return json.dumps({"status": "error", + "error": "Embedding helper unavailable"}) + vec = _get_entry_embedding(entry) + if vec is None: + return json.dumps({"status": "error", + "error": "Source entry has no embedding"}) + exclude_ids.append(entry_id) + else: + vec = query_vec + + results = kb.nearest(vec, k=k, threshold=threshold, + exclude_ids=exclude_ids) + return json.dumps({ + "status": "ok", + "action": "nearest", + "count": len(results), + "entries": results, + }) diff --git a/project/tools/kb_search.py b/project/tools/kb_search.py index 55bef43..160f5b1 100644 --- a/project/tools/kb_search.py +++ b/project/tools/kb_search.py @@ -97,13 +97,18 @@ def call(self, params: str, **kwargs) -> str: kb = self.cfg.get("_kb_instance") project_id = self.cfg.get("_project_id") - # Get the agent name for perspective-shifted search (G2). - # agent_factory.py passes _agent_name into every tool's config - # at line 719. When provided, the KB search uses the Agent Lens - # to apply this agent's attention mask, so a Coder sees code-relevant - # results while a Researcher sees research-relevant results. + # _agent_name enables perspective-shifted search via the Agent Lens. + # When absent (direct /kernel/execute, UI buttons, system probes), + # the Agent Lens stays off but search still works. agent_name = self.cfg.get("_agent_name") + if kb is None: + # Outside agent_factory context: grab the singleton directly. + try: + from knowledge import get_kb + kb = get_kb() + except Exception: + kb = None if kb is None: return "Error: Knowledgebase not available. Check tool configuration." @@ -119,54 +124,58 @@ def call(self, params: str, **kwargs) -> str: f"available topics." ) - # Phase 1C: Qwen re-ranking pass. - # Qwen3-Embedding has a 32K context window (vs nomic's 8K) and - # instruction-aware embedding, so it can re-rank results more - # accurately — especially for long articles or multilingual content. - # This is a lightweight pass: embed the query once, then score - # against each result's content. Falls back silently if qwen - # isn't available. - try: - import embeddings - if embeddings.is_qwen_enabled(): - qwen_query_vec = embeddings.embed_text_qwen( - query, - task_instruction=( - "Given a knowledge base query, retrieve " - "relevant articles" - ), - ) - if qwen_query_vec: - for entry in results: - # Build a compact text representation of the entry - content = entry.get("content", "") - title = entry.get("title", "") - summary = entry.get("summary", "") - entry_text = f"{title}\n{summary}\n{content}"[:8000] - - entry_vec = embeddings.embed_text_qwen( - entry_text, - task_instruction=( - "Given a knowledge base article, " - "represent it for retrieval" - ), - ) - if entry_vec: - qwen_sim = embeddings.cosine_similarity( - qwen_query_vec, entry_vec - ) - # Store qwen score for potential display - entry["_qwen_similarity"] = round(qwen_sim, 4) - - # Re-sort by qwen similarity when available, - # falling back to original order for entries - # that don't have a qwen score - results.sort( - key=lambda e: e.get("_qwen_similarity", 0.0), - reverse=True, + # Optional Qwen re-ranking pass (LEOS_RRF_QWEN=1, default OFF). + # Cost: ~5-10s per result × 5 = 25-50s per search. Redundant + # with the qwen retriever already in the RRF path; useful only + # when testing qwen re-ranking in isolation. + # ``_qwen_emb`` cached per-entry by ``_search_rrf`` to avoid + # re-embedding when the flag is on. + import os as _os + _qwen_rerank_on = ( + _os.environ.get("LEOS_RRF_QWEN", "0").strip() + not in ("0", "false", "no", "off", "") + ) + if _qwen_rerank_on: + try: + import embeddings + if embeddings.is_qwen_enabled(): + qwen_query_vec = embeddings.embed_text_qwen( + query, + task_instruction=( + "Given a knowledge base query, retrieve " + "relevant articles" + ), ) - except (ImportError, Exception): - pass # Qwen not available — no harm, original order stands + if qwen_query_vec: + for entry in results: + entry_vec = entry.get("_qwen_emb") + if entry_vec is None: + content = entry.get("content", "") + title = entry.get("title", "") + summary = entry.get("summary", "") + entry_text = f"{title}\n{summary}\n{content}"[:8000] + try: + entry_vec = embeddings.embed_text_qwen( + entry_text, + task_instruction=( + "Given a knowledge base article, " + "represent it for retrieval" + ), + ) + except Exception: + entry_vec = None + entry["_qwen_emb"] = entry_vec or [] + if entry_vec: + qwen_sim = embeddings.cosine_similarity( + qwen_query_vec, entry_vec + ) + entry["_qwen_similarity"] = round(qwen_sim, 4) + results.sort( + key=lambda e: e.get("_qwen_similarity", 0.0), + reverse=True, + ) + except (ImportError, Exception): + pass # Qwen not available — no harm, original order stands # ------------------------------------------------------------------ # Depth > 0: use knowledge graph expansion diff --git a/project/tools/podcast.py b/project/tools/podcast.py new file mode 100644 index 0000000..c1a9027 --- /dev/null +++ b/project/tools/podcast.py @@ -0,0 +1,493 @@ +""" +podcast.py - Podcast Search, Feed Inspection, and Episode Ingest + +A leOS-native tool for working with podcasts end-to-end: + + 1. Search Apple's public podcast directory by name (no key required). + 2. Fetch and parse an RSS feed into a clean episode list. + 3. Ingest selected episodes into the leOS media library, where the + existing audio pipeline (Whisper transcription + embedding) runs + through the single-worker queue. + +The tool is deliberately thin: search and feed parsing happen here, but +the moment an episode is selected for ingest the work is handed to +media_ingest.ingest_url() so it goes through the same pipeline as any +other audio source. No parallel transcription path; one pipeline. + +EXAMPLES: + # Find shows + podcast action="search" query="lex fridman" + + # See what's in a feed + podcast action="feed" feed_url="https://lexfridman.com/feed/podcast" + + # Ingest the latest 3 episodes + podcast action="ingest" feed_url="..." selection='{"mode":"latest","n":3}' + + # Ingest specific episodes by title substring + podcast action="ingest" feed_url="..." \\ + selection='{"mode":"name","patterns":["Sam Altman","Demis Hassabis"]}' + + # Ingest a contiguous range + podcast action="ingest" feed_url="..." selection='{"mode":"range","from":0,"to":4}' + + # Ingest specific indices (0-based, feed order) + podcast action="ingest" feed_url="..." selection='{"mode":"index","indices":[0,3,7]}' + + # Ingest everything (capped by max_episodes) + podcast action="ingest" feed_url="..." selection='{"mode":"all"}' max_episodes=20 +""" + +import json +import logging +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +from email.utils import parsedate_to_datetime + +import json5 + +from leos_tool_base import BaseTool, register_tool + +logger = logging.getLogger("podcast") + +ITUNES_SEARCH_URL = "https://itunes.apple.com/search" +USER_AGENT = "leOS/1.0 (+podcast-tool)" +DEFAULT_TIMEOUT = 15 +DEFAULT_MAX_EPISODES = 25 +HARD_MAX_EPISODES = 200 # never ingest more than this in one call + +# Namespaces commonly seen in podcast RSS feeds. ElementTree uses +# Clark notation ({uri}localname) for namespaced elements, so resolving +# with these prefixes is enough. +NS = { + "itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd", + "atom": "http://www.w3.org/2005/Atom", + "content": "http://purl.org/rss/1.0/modules/content/", +} + + +TOOL_DESC = ( + "Podcast tool: search Apple's public podcast directory by name, fetch " + "and parse RSS feeds into clean episode lists, and ingest selected " + "episodes into the leOS media library (Whisper transcription + embedding " + "happen automatically through the existing audio pipeline). Selection " + "modes: 'all', 'latest' (most recent N), 'range' (by feed-order index), " + "'index' (specific indices), 'name' (substring match against episode title)." +) + +TOOL_PARAMETERS = [ + { + "name": "action", + "type": "string", + "description": "What to do: 'search', 'feed', or 'ingest'.", + "required": True, + }, + { + "name": "query", + "type": "string", + "description": "Search term for action='search' (podcast name, host, topic).", + "required": False, + }, + { + "name": "limit", + "type": "integer", + "description": "Search results to return (default: 10, max: 50).", + "required": False, + }, + { + "name": "feed_url", + "type": "string", + "description": "RSS feed URL for action='feed' or action='ingest'.", + "required": False, + }, + { + "name": "selection", + "type": "object", + "description": ( + "Episode selection for action='ingest'. JSON object with one of: " + "{'mode':'all'}, {'mode':'latest','n':5}, " + "{'mode':'range','from':0,'to':9}, " + "{'mode':'index','indices':[0,3,7]}, " + "{'mode':'name','patterns':['Sam Altman','Demis']}." + ), + "required": False, + }, + { + "name": "max_episodes", + "type": "integer", + "description": ( + "Safety cap on how many episodes a single ingest call will " + f"enqueue (default: {DEFAULT_MAX_EPISODES}, hard max: " + f"{HARD_MAX_EPISODES})." + ), + "required": False, + }, + { + "name": "project_id", + "type": "string", + "description": "Optional leOS project to associate ingested episodes with.", + "required": False, + }, +] + + +@register_tool("podcast") +class Podcast(BaseTool): + description = TOOL_DESC + parameters = TOOL_PARAMETERS + + def call(self, params, **kwargs): + if isinstance(params, str): + try: + params = json5.loads(params) + except Exception: + return self._error("Could not parse parameters as JSON.") + + action = (params.get("action") or "").strip().lower() + if not action: + return self._error("Missing required parameter: action") + + if action == "search": + return self._search(params) + if action == "feed": + return self._feed(params) + if action == "ingest": + return self._ingest(params) + + return self._error( + f"Unknown action: '{action}'. Use: search, feed, ingest." + ) + + # ------------------------------------------------------------------ + # action: search + # ------------------------------------------------------------------ + def _search(self, params): + query = (params.get("query") or "").strip() + if not query: + return self._error("Missing required parameter: query") + + limit = int(params.get("limit") or 10) + limit = max(1, min(limit, 50)) + + url = ITUNES_SEARCH_URL + "?" + urllib.parse.urlencode({ + "term": query, + "media": "podcast", + "entity": "podcast", + "limit": limit, + }) + + try: + data = self._http_get_json(url) + except Exception as e: + return self._error(f"iTunes search failed: {e}") + + results = [] + for item in data.get("results", []): + feed = item.get("feedUrl") + if not feed: + continue # skip shows without a public RSS feed + results.append({ + "name": item.get("collectionName") or item.get("trackName"), + "author": item.get("artistName"), + "feed_url": feed, + "artwork": ( + item.get("artworkUrl600") + or item.get("artworkUrl100") + ), + "episode_count": item.get("trackCount"), + "primary_genre": item.get("primaryGenreName"), + "country": item.get("country"), + "itunes_collection_id": item.get("collectionId"), + }) + + return json.dumps({ + "status": "ok", + "action": "search", + "query": query, + "count": len(results), + "results": results, + }) + + # ------------------------------------------------------------------ + # action: feed + # ------------------------------------------------------------------ + def _feed(self, params): + feed_url = (params.get("feed_url") or "").strip() + if not feed_url: + return self._error("Missing required parameter: feed_url") + + try: + episodes, show = self._fetch_feed(feed_url) + except Exception as e: + return self._error(f"Feed fetch/parse failed: {e}") + + return json.dumps({ + "status": "ok", + "action": "feed", + "feed_url": feed_url, + "show": show, + "episode_count": len(episodes), + "episodes": episodes, + }) + + # ------------------------------------------------------------------ + # action: ingest + # ------------------------------------------------------------------ + def _ingest(self, params): + feed_url = (params.get("feed_url") or "").strip() + if not feed_url: + return self._error("Missing required parameter: feed_url") + + # Parse selection (accept either dict or JSON string) + selection = params.get("selection") + if isinstance(selection, str): + try: + selection = json5.loads(selection) + except Exception: + return self._error("Could not parse 'selection' as JSON.") + if not isinstance(selection, dict): + selection = {"mode": "latest", "n": 1} + + cap = int(params.get("max_episodes") or DEFAULT_MAX_EPISODES) + cap = max(1, min(cap, HARD_MAX_EPISODES)) + + try: + episodes, show = self._fetch_feed(feed_url) + except Exception as e: + return self._error(f"Feed fetch/parse failed: {e}") + + try: + picked = self._select_episodes(episodes, selection) + except ValueError as e: + return self._error(str(e)) + + if len(picked) > cap: + picked = picked[:cap] + + if not picked: + return json.dumps({ + "status": "ok", + "action": "ingest", + "feed_url": feed_url, + "show": show.get("title"), + "selection": selection, + "enqueued_count": 0, + "enqueued": [], + "note": "No episodes matched the selection.", + }) + + # Defer imports — these pull in the whole knowledge subsystem, + # which is heavy. Only pay for them when actually ingesting. + # + # We do NOT call media_ingest.ingest_url() directly: its + # _classify_url helper checks file extensions against the + # _DIRECT_MEDIA_EXTENSIONS set, which is empty in the upstream + # tree (project/knowledge/media_ingest_url_pipeline.py:92). + # Every podcast audio URL therefore mis-classifies as "web_page" + # and runs the HTML extractor instead of Whisper. We bypass the + # classifier by creating an audio record explicitly and queuing + # _ingest_direct_download — the same handler url_type == + # "direct_media" would have used. + try: + from functools import partial + from media_ingest_queue import _enqueue + from media_ingest_url_pipeline import _ingest_direct_download + import media_library + except Exception as e: + return self._error( + f"media_ingest unavailable (is leOS server running?): {e}" + ) + + project_id = params.get("project_id") or None + enqueued = [] + for ep in picked: + audio_url = ep.get("audio_url") + if not audio_url: + continue + title = self._compose_title(show, ep) + try: + record = media_library.create_record( + media_type="audio", + filename=None, + title=title, + source_url=audio_url, + project_id=project_id, + ) + record_id = record["id"] + _enqueue( + partial(_ingest_direct_download, record_id, audio_url), + record_id, + ) + enqueued.append({ + "index": ep["index"], + "title": ep["title"], + "audio_url": audio_url, + "record_id": record_id, + }) + except Exception as e: + logger.warning( + "podcast ingest enqueue failed for '%s': %s", title, e + ) + enqueued.append({ + "index": ep["index"], + "title": ep["title"], + "audio_url": audio_url, + "error": str(e), + }) + + return json.dumps({ + "status": "ok", + "action": "ingest", + "feed_url": feed_url, + "show": show.get("title"), + "selection": selection, + "enqueued_count": sum(1 for e in enqueued if "record_id" in e), + "enqueued": enqueued, + "note": ( + "Episodes are queued for background processing " + "(Whisper transcription + embedding). Use leos_media_search " + "or check the media library to find them once ready." + ), + }) + + # ------------------------------------------------------------------ + # helpers + # ------------------------------------------------------------------ + + def _http_get_json(self, url): + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(req, timeout=DEFAULT_TIMEOUT) as r: + return json.loads(r.read().decode("utf-8", errors="replace")) + + def _http_get_bytes(self, url): + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(req, timeout=DEFAULT_TIMEOUT) as r: + return r.read() + + def _fetch_feed(self, feed_url): + """Download an RSS feed and return (episodes, show_metadata).""" + raw = self._http_get_bytes(feed_url) + # Some feeds declare encodings ElementTree doesn't honor; let + # fromstring auto-detect from the XML declaration. + root = ET.fromstring(raw) + channel = root.find("channel") + if channel is None: + raise ValueError("RSS feed has no element") + + show = { + "title": _text(channel.find("title")), + "author": _text(channel.find(_q("itunes", "author"))) or _text( + channel.find("managingEditor") + ), + "description": _text(channel.find("description")), + "language": _text(channel.find("language")), + "link": _text(channel.find("link")), + } + + episodes = [] + for idx, item in enumerate(channel.findall("item")): + audio_url = None + audio_length = None + audio_type = None + enc = item.find("enclosure") + if enc is not None and enc.get("url"): + audio_url = enc.get("url") + audio_length = enc.get("length") + audio_type = enc.get("type") + + pub_iso = None + pub_raw = _text(item.find("pubDate")) + if pub_raw: + try: + pub_iso = parsedate_to_datetime(pub_raw).isoformat() + except Exception: + pub_iso = None + + episodes.append({ + "index": idx, + "title": _text(item.find("title")) or f"Episode {idx}", + "published": pub_iso, + "duration": _text(item.find(_q("itunes", "duration"))), + "guid": _text(item.find("guid")), + "audio_url": audio_url, + "audio_length": audio_length, + "audio_type": audio_type, + "summary": ( + _text(item.find(_q("itunes", "summary"))) + or _text(item.find("description")) + ), + }) + return episodes, show + + def _select_episodes(self, episodes, selection): + mode = (selection.get("mode") or "").strip().lower() + if not mode: + raise ValueError( + "selection requires a 'mode': all, latest, range, index, or name" + ) + + if mode == "all": + return list(episodes) + + if mode == "latest": + n = int(selection.get("n", 1)) + if n <= 0: + raise ValueError("selection.n must be a positive integer") + return list(episodes[:n]) + + if mode == "range": + lo = int(selection.get("from", 0)) + hi = int(selection.get("to", lo)) + if hi < lo: + raise ValueError("selection.to must be >= selection.from") + return [e for e in episodes if lo <= e["index"] <= hi] + + if mode == "index": + wanted = set(int(i) for i in (selection.get("indices") or [])) + if not wanted: + raise ValueError( + "selection.indices must be a non-empty list of integers" + ) + return [e for e in episodes if e["index"] in wanted] + + if mode == "name": + patterns = selection.get("patterns") or [] + if isinstance(patterns, str): + patterns = [patterns] + patterns = [p.lower() for p in patterns if p] + if not patterns: + raise ValueError( + "selection.patterns must be a non-empty list of strings" + ) + picked = [] + for ep in episodes: + title = (ep.get("title") or "").lower() + if any(p in title for p in patterns): + picked.append(ep) + return picked + + raise ValueError( + f"Unknown selection.mode: '{mode}'. " + "Use: all, latest, range, index, name." + ) + + def _compose_title(self, show, episode): + show_title = show.get("title") or "Podcast" + ep_title = episode.get("title") or f"Episode {episode.get('index', '?')}" + return f"{show_title} — {ep_title}" + + +# ---------------------------------------------------------------------- +# Local helpers (module-level) +# ---------------------------------------------------------------------- + +def _text(node): + if node is None: + return None + return (node.text or "").strip() or None + + +def _q(prefix, local): + """Compose a Clark-notation tag for a namespaced element.""" + return "{" + NS[prefix] + "}" + local diff --git a/project/voice/monologue_renderer.py b/project/voice/monologue_renderer.py index 438cfd3..dd11680 100644 --- a/project/voice/monologue_renderer.py +++ b/project/voice/monologue_renderer.py @@ -782,10 +782,8 @@ def apply_watermark(self, audio, *a, **kw): return audio sys.modules["perth"] = perth_stub # Try Turbo first (faster, paralinguistic tags). - # device_select.best_device() picks cuda/mps/cpu based on what's - # actually available; before this helper existed, this site used - # `"cuda" if torch.cuda.is_available() else "cpu"` which missed - # MPS acceleration entirely on Apple Silicon. + # best_device() probes cuda/mps/cpu; plain cuda-or-cpu misses + # MPS acceleration on Apple Silicon. from device_select import best_device device = best_device() try: @@ -2781,45 +2779,89 @@ def generate_reflection(self, experience_text, experience_type="lesson", if reflection_ctx: formatted_context = self._format_context_for_prompt(reflection_ctx) - # Build the reflection prompt. - prompt = ( - "You are reflecting on a recent experience that flagged a gap " - "in your knowledge or capability. Speak in first person as if " - "thinking aloud. Be honest and introspective, not clinical.\n\n" - f"Experience ({experience_type}):\n{experience_text[:800]}\n\n" - ) - - if context.get("task"): - prompt += f"Original task: {context['task'][:300]}\n\n" - - # Inject the full working context if available - if formatted_context: + # Build the reflection prompt. Two distinct framings: + # + # - 'insight' / 'discovery': engage with substantive new content + # (e.g. a freshly ingested KB article). The model should + # react to the IDEAS — what's surprising, what connects to + # prior knowledge, what new questions it raises. No + # introspection about the act of saving it. + # + # - everything else: reflect on a flagged gap in the agent's + # own capability — the original post-task / post-failure + # framing. Used for 'lesson', 'failure', 'post_mortem', + # 'surprise'. + if experience_type in ("insight", "discovery"): + prompt = ( + "You just absorbed new material into your knowledge. Speak " + "in first person as if thinking aloud about what you read. " + "React to the ideas themselves — be curious, opinionated, " + "and specific. Don't talk about the act of saving or " + "storing the article; talk about what it actually says.\n\n" + "New material:\n" + f"{experience_text[:3000]}\n\n" + ) + if context.get("task"): + prompt += f"Source: {context['task'][:300]}\n\n" + if formatted_context: + prompt += ( + "What I already knew that this connects to:\n\n" + f"{formatted_context}\n\n" + ) prompt += ( - "Here is what was happening when this experience occurred — " - "the scope I was working in, the plan I was executing, what " - "each step produced, and what was on the blackboard:\n\n" - f"{formatted_context}\n\n" + "Now think out loud about three things, in any order, as a " + "stream of consciousness — not a numbered list:\n" + " - The single most surprising, counterintuitive, or vivid " + "idea in this material. Quote or paraphrase a specific " + "claim, fact, or moment, and say why it lands.\n" + " - How it changes or extends what I already thought I " + "understood. If it agrees with my prior knowledge, what " + "does it sharpen? If it disagrees, where's the tension?\n" + " - One concrete question this opens up — something I'd " + "want to chase down or test next.\n\n" + "Speak naturally and substantively. Reference specific " + "names, claims, numbers, or examples from the material. " + "Avoid meta-commentary about reflection, ingestion, " + "storage, retrieval, or your own learning process — stay " + "with the content." + ) + else: + prompt = ( + "You are reflecting on a recent experience that flagged a " + "gap in your knowledge or capability. Speak in first " + "person as if thinking aloud. Be honest and introspective, " + "not clinical.\n\n" + f"Experience ({experience_type}):\n{experience_text[:800]}\n\n" + ) + if context.get("task"): + prompt += f"Original task: {context['task'][:300]}\n\n" + if formatted_context: + prompt += ( + "Here is what was happening when this experience " + "occurred — the scope I was working in, the plan I was " + "executing, what each step produced, and what was on " + "the blackboard:\n\n" + f"{formatted_context}\n\n" + ) + prompt += ( + "Reflect on three things, drawing on the working context above:\n" + "1. What can I learn from this experience? What pattern or " + "principle should I internalize for future work? Look at what " + "the plan was trying to accomplish, what the steps actually " + "produced, and where the gap appeared.\n" + "2. Why is this worth learning? What will go better next time " + "if I understand this? Consider the deliverables that were " + "expected vs. what was actually recorded.\n" + "3. Why didn't I already know how to handle this? What was the " + "gap in my understanding that led to this outcome? Look at the " + "blackboard messages and observations for clues about what I " + "was being told but didn't act on.\n\n" + "Speak naturally, as inner monologue. 2-4 sentences per point. " + "Reference specific details from the context — step numbers, " + "tool calls, blackboard messages — to ground your reflection " + "in what actually happened. Do not use numbered lists — this " + "is a stream of thought." ) - - prompt += ( - "Reflect on three things, drawing on the working context above:\n" - "1. What can I learn from this experience? What pattern or " - "principle should I internalize for future work? Look at what " - "the plan was trying to accomplish, what the steps actually " - "produced, and where the gap appeared.\n" - "2. Why is this worth learning? What will go better next time " - "if I understand this? Consider the deliverables that were " - "expected vs. what was actually recorded.\n" - "3. Why didn't I already know how to handle this? What was the " - "gap in my understanding that led to this outcome? Look at the " - "blackboard messages and observations for clues about what I " - "was being told but didn't act on.\n\n" - "Speak naturally, as inner monologue. 2-4 sentences per point. " - "Reference specific details from the context — step numbers, " - "tool calls, blackboard messages — to ground your reflection " - "in what actually happened. Do not use numbered lists — this " - "is a stream of thought." - ) try: # ESCALATE reads these fields from args["context"], not @@ -2828,20 +2870,46 @@ def generate_reflection(self, experience_text, experience_type="lesson", # silently dropped — reflections ran with default system # prompt, max_tokens, and temperature instead of the ones # specified here. + if experience_type in ("insight", "discovery"): + system_prompt = ( + "You are an AI thinking out loud after reading something " + "new. This is inner monologue — natural, curious, " + "opinionated. Engage with the substance: name specific " + "claims, examples, numbers, people from the material. " + "Treat the ideas as ideas, not as artifacts to file away. " + "Do not narrate the act of saving, storing, or learning " + "from the article — stay inside the content.\n\n" + "DO NOT CALL ANY TOOLS. Do not search the web, do not " + "search the knowledge base, do not call any function or " + "tool. The material in the user message is everything " + "you need. Respond with the monologue text directly — " + "no tool calls, no tags, no JSON. Just the " + "spoken inner monologue, ready to be read aloud." + ) + else: + system_prompt = ( + "You are an AI reflecting on your own learning process. " + "Speak in first person, honestly and thoughtfully. " + "This is inner monologue — natural, contemplative, not " + "a report. Reference specific details from the working " + "context (step outcomes, scope notes, blackboard messages) " + "to make your reflection concrete, not abstract. Include " + "moments of genuine realization." + ) + result = self.kernel.execute("ESCALATE", { "prompt": prompt, "context": { - "system_prompt": ( - "You are an AI reflecting on your own learning process. " - "Speak in first person, honestly and thoughtfully. " - "This is inner monologue — natural, contemplative, not " - "a report. Reference specific details from the working " - "context (step outcomes, scope notes, blackboard messages) " - "to make your reflection concrete, not abstract. Include " - "moments of genuine realization." - ), + "system_prompt": system_prompt, "max_tokens": 600, "temperature": 0.6, + # Reflection is a monologue, not a research task — + # bypass the agent's tool-calling loop and go + # straight to the LLM. Without this, an "engage + # with the content" prompt invites the agent to + # leos_web_search the topic instead of just + # speaking from the material we already provided. + "no_tools": True, }, }) diff --git a/run.bat b/run.bat index a1316b1..ae82fe9 100644 --- a/run.bat +++ b/run.bat @@ -5,11 +5,59 @@ cd /d "%~dp0" set "PROJ=project" set "VENV=%PROJ%\venv" set "PYTHON=%VENV%\Scripts\python.exe" +set "GPU_DETECT=%PROJ%\infra\gpu_detect.py" -set "DEPS_VER=15" +REM ===== GPU coordination defaults ===== +REM Ollama coexists with faster-whisper / ImageBind / embedding models +REM on the same GPU. The defaults below tell Ollama to release VRAM +REM faster than its 5-minute default so other leOS subsystems aren't +REM crowded off the device. infra/gpu_coordinator.py is the active +REM evictor that runs when these passive defaults aren't enough. +REM +REM OLLAMA_KEEP_ALIVE — how long an idle model stays loaded +REM OLLAMA_MAX_LOADED_MODELS — concurrent models cap (1 on a 24GB GPU +REM when Whisper / ImageBind also want room) +REM LEOS_GPU_POLICY — coordinator policy: always | on_pressure | never +REM +REM Override any of these from a parent shell to keep your own values. +if "%OLLAMA_KEEP_ALIVE%"=="" set "OLLAMA_KEEP_ALIVE=60s" +if "%OLLAMA_MAX_LOADED_MODELS%"=="" set "OLLAMA_MAX_LOADED_MODELS=1" +if "%LEOS_GPU_POLICY%"=="" set "LEOS_GPU_POLICY=on_pressure" +REM OLLAMA_NUM_PARALLEL — concurrent requests to a loaded model. 3 is +REM safe on a 24GB 4090 with qwen3.5:9b at num_ctx=8192 (each KV +REM slot is ~470MB at fp16, so 3 slots cost ~1.4GB; Whisper at +REM 4.5GB still fits with 12+GB headroom). Default upstream is 1. +REM OLLAMA_FLASH_ATTENTION — online attention reduces KV memory ~10-20%. +if "%OLLAMA_NUM_PARALLEL%"=="" set "OLLAMA_NUM_PARALLEL=3" +if "%OLLAMA_FLASH_ATTENTION%"=="" set "OLLAMA_FLASH_ATTENTION=1" + +REM ===== HuggingFace token for pyannote-audio (speaker diarization) ===== +REM pyannote/speaker-diarization-3.1 and pyannote/segmentation-3.0 are +REM gated models — accept their terms once on huggingface.co, create a +REM read token at https://huggingface.co/settings/tokens, then either +REM paste it into the line below (uncomment first) or export HF_TOKEN +REM in your shell. Without it, transcripts are produced but speaker +REM labels are not. +REM set "HF_TOKEN=hf_replace_me_with_your_token" + +REM Bootstrap Ollama daemon if not already serving (see :bootstrap_ollama) +call :bootstrap_ollama + +REM Bootstrap ComfyUI: install from upstream into project\external\ComfyUI +REM the first time, then launch headless on port 8000 if not already +REM serving. Self-contained — no Pinokio dependency. +call :bootstrap_comfyui + +REM Bumping DEPS_VER triggers a re-install — bump when the wheel +REM index URL strategy or PyTorch pin changes so existing checkouts +REM rebuild against the right backend instead of staying on the old +REM wheel set. +set "DEPS_VER=17" set "DATASCIENCE_VER=1" -set "IMAGEBIND_VER=1" +set "IMAGEBIND_VER=2" set "CHATTERBOX_PATCH_VER=3" +set "WHISPERX_VER=3" +set "COMFYUI_VER=1" if not exist "%PROJ%\server.py" ( echo ERROR: project\server.py not found. @@ -37,33 +85,74 @@ if not exist "%PYTHON%" ( exit /b 1 ) echo Upgrading pip... - "%PYTHON%" -m pip install --upgrade pip setuptools wheel + REM Pin setuptools<80 so pkg_resources stays bundled. setuptools 80 +REM split pkg_resources out into a separate package, which breaks +REM ImageBind (and any other library that still does `import +REM pkg_resources`) on a fresh venv. Until those upstreams migrate, +REM pinning here is the simplest way to keep the install reproducible. +"%PYTHON%" -m pip install --upgrade pip "setuptools<80" wheel ) "%PYTHON%" -m pip --version >nul 2>&1 if errorlevel 1 ( echo Bootstrapping pip... "%PYTHON%" -m ensurepip --upgrade - "%PYTHON%" -m pip install --upgrade pip setuptools wheel + REM Pin setuptools<80 so pkg_resources stays bundled. setuptools 80 +REM split pkg_resources out into a separate package, which breaks +REM ImageBind (and any other library that still does `import +REM pkg_resources`) on a fresh venv. Until those upstreams migrate, +REM pinning here is the simplest way to keep the install reproducible. +"%PYTHON%" -m pip install --upgrade pip "setuptools<80" wheel ) call :cleanup_stale_stubs deps !DEPS_VER! call :cleanup_stale_stubs datascience !DATASCIENCE_VER! call :cleanup_stale_stubs imagebind !IMAGEBIND_VER! call :cleanup_stale_stubs chatterbox_patch !CHATTERBOX_PATCH_VER! +call :cleanup_stale_stubs whisperx !WHISPERX_VER! +call :cleanup_stale_stubs comfyui !COMFYUI_VER! + +REM ----- Accelerator detection (runs even when deps are already installed +REM so leos_status can report the chosen wheel index without re-probing). +REM gpu_detect.py is pure stdlib so it runs before any torch is in place. +REM We capture the chosen index URL via a temp file rather than a piped +REM `for /f` to keep cmd.exe's parser out of trouble inside parens. +set "TORCH_INDEX_FILE=%TEMP%\leos_torch_index.txt" +if exist "%PYTHON%" ( + "%PYTHON%" "%GPU_DETECT%" index > "%TORCH_INDEX_FILE%" 2>nul + set /p TORCH_INDEX=<"%TORCH_INDEX_FILE%" +) +if "%TORCH_INDEX%"=="" set "TORCH_INDEX=https://download.pytorch.org/whl/cpu" +echo Accelerator wheel index: %TORCH_INDEX% if not exist ".deps_v!DEPS_VER!" ( echo. echo Installing core dependencies... echo. echo Ensuring build tools are current... - "%PYTHON%" -m pip install --upgrade pip setuptools wheel + REM Pin setuptools<80 so pkg_resources stays bundled. setuptools 80 +REM split pkg_resources out into a separate package, which breaks +REM ImageBind (and any other library that still does `import +REM pkg_resources`) on a fresh venv. Until those upstreams migrate, +REM pinning here is the simplest way to keep the install reproducible. +"%PYTHON%" -m pip install --upgrade pip "setuptools<80" wheel echo. - echo Installing PyTorch CPU... - "%PYTHON%" -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + if exist "%PYTHON%" "%PYTHON%" "%GPU_DETECT%" summary 2>nul + echo Installing PyTorch from %TORCH_INDEX% ... + REM Pinned to the same versions as requirements.txt so the later + REM `pip install -r requirements.txt` step doesn't try to downgrade + REM these packages (which can lock DLLs and fail on Windows). + REM --force-reinstall lets us migrate between wheel tiers (cpu -> + REM cu124, cu118 -> cu124, etc.) since the version pin matches + REM across all indexes — without --force pip would silently keep + REM the existing build. + REM --no-deps avoids dragging the rest of the dependency graph in + REM through the CUDA index (which is incomplete for non-torch + REM packages); requirements.txt resolves everything else from PyPI. + "%PYTHON%" -m pip install --force-reinstall --no-deps "torch==2.5.1" "torchvision==0.20.1" "torchaudio==2.5.1" --index-url "%TORCH_INDEX%" if errorlevel 1 ( - echo Trying default PyTorch... - "%PYTHON%" -m pip install torch torchvision torchaudio + echo Wheel index failed, falling back to CPU build... + "%PYTHON%" -m pip install --force-reinstall --no-deps "torch==2.5.1" "torchvision==0.20.1" "torchaudio==2.5.1" --index-url https://download.pytorch.org/whl/cpu ) echo. echo Installing requirements.txt... @@ -125,18 +214,63 @@ if not exist ".imagebind_v!IMAGEBIND_VER!" ( echo Installing ImageBind... "%PYTHON%" -m pip install --no-deps git+https://github.com/facebookresearch/ImageBind.git@main ) + REM ImageBind is intentionally installed --no-deps. Its model + REM architecture (imagebind.models.imagebind_model) only needs + REM torch + timm + einops at runtime; the heavier deps it declares + REM (pytorchvideo, ftfy, iopath, fvcore, decord, cartopy, moviepy) + REM are only used by imagebind.data for preprocessing, which leOS + REM replaces with its own transforms in lvm/imagebind_embed.py. + REM + REM Those packages are also chronically broken on modern stacks + REM (pytorchvideo's last release predates torchvision 0.17 and + REM imports the removed `torchvision.transforms.functional_tensor`), + REM so installing them does more harm than skipping them — the + REM leOS shim system in imagebind_embed.py papers over their + REM imports cleanly when they're missing. + REM + REM ONLY install timm + einops here. Do NOT add pytorchvideo or + REM friends; that breaks the shim path and leaves ImageBind + REM unable to load. "%PYTHON%" -c "import timm" >nul 2>&1 if errorlevel 1 "%PYTHON%" -m pip install "timm>=0.6.7" "%PYTHON%" -c "import einops" >nul 2>&1 if errorlevel 1 "%PYTHON%" -m pip install "einops>=0.6.0" + REM pkg_resources used to come bundled with setuptools but split + REM out at v80. ImageBind's model module imports it for version + REM checks; if it's missing the shim system handles it, but + REM having it real (via setuptools<80, pinned at the top of this + REM script) is preferred since the version checks can then return + REM accurate values. "%PYTHON%" -c "import torchaudio" >nul 2>&1 if errorlevel 1 ( echo Reinstalling torch suite for alignment... - "%PYTHON%" -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + "%PYTHON%" -m pip install --force-reinstall --no-deps "torch==2.5.1" "torchvision==0.20.1" "torchaudio==2.5.1" --index-url "%TORCH_INDEX%" ) echo Checked on %date% %time% > ".imagebind_v!IMAGEBIND_VER!" ) +if not exist ".whisperx_v!WHISPERX_VER!" ( + echo Installing pyannote.audio 3.3.2 for speaker diarization... + REM We use faster-whisper (already installed) for transcription and + REM pyannote.audio directly for diarization. We do NOT install the + REM whisperx wrapper package — its current line pins torch 2.8.x + REM which conflicts with our torch==2.5.1+cu124 stack and would + REM cascade-break chatterbox-tts, ImageBind, and the embedding + REM ecosystem. If a stale whisperx is present from an earlier + REM install, remove it and roll back deps it cascade-upgraded. + "%PYTHON%" -m pip uninstall -y whisperx 2>nul + "%PYTHON%" -m pip install --force-reinstall "numpy<2" + "%PYTHON%" -m pip install "transformers==5.2.0" + "%PYTHON%" -m pip install "pyannote.audio==3.3.2" + REM huggingface_hub stays at whatever transformers 5.2.0 installs + REM (1.x). pyannote.audio 3.3.2 was written for pre-1.0 hf_hub + REM and calls hf_hub_download(use_auth_token=...), which was + REM removed in 1.0 — but media_ingest_models._load_diarizing_whisper + REM monkey-patches hf_hub_download to translate use_auth_token → + REM token at runtime. No version pin needed. + echo Installed on %date% %time% > ".whisperx_v!WHISPERX_VER!" +) + if not exist ".chatterbox_patch_v!CHATTERBOX_PATCH_VER!" ( echo Patching chatterbox dtype and watermarker issues... "%PYTHON%" "%PROJ%\voice\patch_chatterbox.py" @@ -195,3 +329,84 @@ for %%F in (".%_grp%_v*") do ( if /i NOT "%%~nxF"==".%_grp%_v%_ver%" del "%%F" >nul 2>&1 ) goto :eof + + +:bootstrap_comfyui +REM Install ComfyUI into project\external\ComfyUI on first run, share +REM the leOS venv (deps overlap mostly cleanly with our pinned torch +REM 2.5.1 stack), then launch headless on port 8000 if no ComfyUI is +REM already serving. Self-contained — no Pinokio dependency. +curl --silent --max-time 1 http://127.0.0.1:8000/system_stats >nul 2>&1 +if not errorlevel 1 ( + echo ComfyUI: already serving at http://127.0.0.1:8000 + goto :eof +) +set "COMFYUI_DIR=%~dp0%PROJ%\external\ComfyUI" +if not exist ".comfyui_v!COMFYUI_VER!" ( + echo Installing ComfyUI ^(first-time clone + dep install^)... + if not exist "%~dp0%PROJ%\external" mkdir "%~dp0%PROJ%\external" >nul 2>&1 + where git >nul 2>&1 + if errorlevel 1 ( + echo ERROR: git not found on PATH. Install Git for Windows or + echo place a ComfyUI checkout at %COMFYUI_DIR% and re-run. + goto :eof + ) + if not exist "%COMFYUI_DIR%\main.py" ( + git clone --depth 1 https://github.com/comfyanonymous/ComfyUI.git "%COMFYUI_DIR%" + if errorlevel 1 ( + echo ERROR: ComfyUI clone failed. + goto :eof + ) + ) + if exist "%COMFYUI_DIR%\requirements.txt" ( + echo Installing ComfyUI requirements into the leOS venv... + "%PYTHON%" -m pip install -r "%COMFYUI_DIR%\requirements.txt" + ) + echo Installed on %date% %time% > ".comfyui_v!COMFYUI_VER!" +) +if not exist "%COMFYUI_DIR%\main.py" ( + echo NOTE: ComfyUI install missing at %COMFYUI_DIR% — skipping launch. + goto :eof +) +echo Starting ComfyUI on http://127.0.0.1:8000 ... +start "comfyui" /B "%PYTHON%" "%COMFYUI_DIR%\main.py" --port 8000 --cuda-device 0 +set "_COMFY_TRIES=0" +:bootstrap_comfyui_wait +timeout /t 1 /nobreak >nul +curl --silent --max-time 1 http://127.0.0.1:8000/system_stats >nul 2>&1 +if not errorlevel 1 goto :eof +set /a _COMFY_TRIES+=1 +if %_COMFY_TRIES% lss 30 goto bootstrap_comfyui_wait +echo WARNING: ComfyUI did not respond within 30s. leOS will boot without +echo image generation; COPROCESSOR_PROBE picks it up once ready. +goto :eof + + +:bootstrap_ollama +REM Power-cut recovery: the Ollama tray app normally autostarts at +REM login, but reboots from a hard crash sometimes skip it. Probe +REM 11434 and launch ollama serve if nobody's home. Env vars set +REM at the top of this script (KEEP_ALIVE, MAX_LOADED_MODELS) are +REM inherited by the child. Subroutine (not parenthesized block) so +REM the wait-loop labels parse correctly under cmd.exe. +curl --silent --max-time 1 http://localhost:11434/api/version >nul 2>&1 +if not errorlevel 1 goto :eof +set "OLLAMA_EXE=" +for /f "delims=" %%I in ('where ollama 2^>nul') do if not defined OLLAMA_EXE set "OLLAMA_EXE=%%I" +if not defined OLLAMA_EXE if exist "%LOCALAPPDATA%\Programs\Ollama\ollama.exe" set "OLLAMA_EXE=%LOCALAPPDATA%\Programs\Ollama\ollama.exe" +if not defined OLLAMA_EXE ( + echo NOTE: ollama.exe not found. leOS will boot without LLM coprocessor. + echo Install from https://ollama.com if you want ESCALATE/ASSIST. + goto :eof +) +echo Starting Ollama daemon: %OLLAMA_EXE% +start "ollama" /B "%OLLAMA_EXE%" serve +set /a _OLLAMA_TRIES=0 +:bootstrap_ollama_wait +timeout /t 1 /nobreak >nul +curl --silent --max-time 1 http://localhost:11434/api/version >nul 2>&1 +if not errorlevel 1 goto :eof +set /a _OLLAMA_TRIES+=1 +if %_OLLAMA_TRIES% lss 15 goto bootstrap_ollama_wait +echo WARNING: Ollama daemon did not respond within 15s. leOS will boot without LLM coprocessor. +goto :eof diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..3ba1e0b --- /dev/null +++ b/run.sh @@ -0,0 +1,290 @@ +#!/usr/bin/env bash +# POSIX equivalent of run.bat — handles first-time setup (venv, deps, +# torch wheel for the detected accelerator, ImageBind, chatterbox-tts, +# Playwright) and normal launches. Re-runs are fast: dependency steps +# are stamp-file gated and skip when already done. +# +# ./run.sh launch +# ./run.sh shell activate the venv and drop into a subshell + +set -euo pipefail + +cd "$(dirname "$0")" + +PROJ="project" +VENV="$PROJ/venv" +PYTHON="$VENV/bin/python" +GPU_DETECT="$PROJ/infra/gpu_detect.py" + +# ===== GPU coordination defaults ===== +# Mirrors run.bat: see the comment block there for what each var does. +: "${OLLAMA_KEEP_ALIVE:=60s}" +: "${OLLAMA_MAX_LOADED_MODELS:=1}" +: "${LEOS_GPU_POLICY:=on_pressure}" +: "${OLLAMA_NUM_PARALLEL:=3}" +: "${OLLAMA_FLASH_ATTENTION:=1}" +export OLLAMA_KEEP_ALIVE OLLAMA_MAX_LOADED_MODELS LEOS_GPU_POLICY \ + OLLAMA_NUM_PARALLEL OLLAMA_FLASH_ATTENTION + +# ===== HuggingFace token for pyannote-audio (speaker diarization) ===== +# Accept terms once on huggingface.co for pyannote/segmentation-3.0 and +# pyannote/speaker-diarization-3.1, create a read token, then uncomment +# below or export HF_TOKEN in your shell. Without it, transcripts are +# produced but speaker labels are not. +# export HF_TOKEN="hf_replace_me_with_your_token" + +# Stamp versions — bumping any of these triggers a re-install of that +# group on the next run (matches run.bat's DEPS_VER, IMAGEBIND_VER, etc.). +DEPS_VER=17 +DATASCIENCE_VER=1 +IMAGEBIND_VER=2 +CHATTERBOX_PATCH_VER=3 +WHISPERX_VER=3 +COMFYUI_VER=1 + +# ----- Bootstrap ComfyUI ----- +COMFYUI_DIR="$PROJ/external/ComfyUI" + +bootstrap_comfyui() { + if curl --silent --max-time 1 http://127.0.0.1:8000/system_stats > /dev/null 2>&1; then + echo "ComfyUI: already serving at http://127.0.0.1:8000" + return 0 + fi + if [[ ! -f ".comfyui_v${COMFYUI_VER}" ]]; then + echo "Installing ComfyUI (first-time clone + dep install)..." + mkdir -p "$PROJ/external" + if ! command -v git > /dev/null 2>&1; then + echo "ERROR: git not on PATH — install git or drop a ComfyUI checkout at $COMFYUI_DIR." + return 0 + fi + if [[ ! -f "$COMFYUI_DIR/main.py" ]]; then + git clone --depth 1 https://github.com/comfyanonymous/ComfyUI.git "$COMFYUI_DIR" || { + echo "ERROR: ComfyUI clone failed." + return 0 + } + fi + if [[ -f "$COMFYUI_DIR/requirements.txt" ]]; then + echo "Installing ComfyUI requirements into the leOS venv..." + "$PYTHON" -m pip install -r "$COMFYUI_DIR/requirements.txt" + fi + date > ".comfyui_v${COMFYUI_VER}" + fi + if [[ ! -f "$COMFYUI_DIR/main.py" ]]; then + echo "NOTE: ComfyUI install missing at $COMFYUI_DIR — skipping launch." + return 0 + fi + echo "Starting ComfyUI on http://127.0.0.1:8000 ..." + nohup "$PYTHON" "$COMFYUI_DIR/main.py" --port 8000 --cuda-device 0 \ + > /tmp/leos_comfyui.log 2>&1 & + for _ in $(seq 1 30); do + sleep 1 + if curl --silent --max-time 1 http://127.0.0.1:8000/system_stats > /dev/null 2>&1; then + return 0 + fi + done + echo "WARNING: ComfyUI did not respond within 30s. leOS will boot without image generation." +} + + +# ----- Bootstrap Ollama daemon if not already serving ----- +bootstrap_ollama() { + if curl --silent --max-time 1 http://localhost:11434/api/version > /dev/null 2>&1; then + return 0 + fi + if ! command -v ollama > /dev/null 2>&1; then + echo "NOTE: ollama not found on PATH. leOS will boot without LLM coprocessor." + echo " Install from https://ollama.com if you want ESCALATE/ASSIST." + return 0 + fi + echo "Starting Ollama daemon: $(command -v ollama)" + nohup ollama serve > /tmp/leos_ollama.log 2>&1 & + for _ in $(seq 1 15); do + sleep 1 + if curl --silent --max-time 1 http://localhost:11434/api/version > /dev/null 2>&1; then + return 0 + fi + done + echo "WARNING: Ollama daemon did not respond within 15s. leOS will boot without LLM coprocessor." +} + +# ----- Find / create system Python ----- +if [[ ! -f "$PROJ/server.py" ]]; then + echo "ERROR: $PROJ/server.py not found." + exit 1 +fi +if ! command -v python3 > /dev/null 2>&1; then + echo "ERROR: Python 3 not found. Install Python 3.10+ from https://python.org or via your package manager." + exit 1 +fi + +# Shell mode: drop into the venv subshell after ensuring it exists below. +SHELL_MODE=0 +if [[ "${1:-}" == "shell" ]]; then + SHELL_MODE=1 +fi + +# ----- Stale stamp cleanup ----- +cleanup_stale_stubs() { + local grp="$1" ver="$2" + for f in ".${grp}_v"*; do + [[ -e "$f" ]] || continue + if [[ "$f" != ".${grp}_v${ver}" ]]; then + rm -f "$f" + fi + done +} +cleanup_stale_stubs deps "$DEPS_VER" +cleanup_stale_stubs datascience "$DATASCIENCE_VER" +cleanup_stale_stubs imagebind "$IMAGEBIND_VER" +cleanup_stale_stubs chatterbox_patch "$CHATTERBOX_PATCH_VER" +cleanup_stale_stubs whisperx "$WHISPERX_VER" +cleanup_stale_stubs comfyui "$COMFYUI_VER" + +# ----- venv ----- +if [[ ! -x "$PYTHON" ]]; then + echo "Creating virtual environment..." + python3 -m venv "$VENV" + "$PYTHON" -m pip install --upgrade pip 'setuptools<80' wheel +fi +if ! "$PYTHON" -m pip --version > /dev/null 2>&1; then + echo "Bootstrapping pip..." + "$PYTHON" -m ensurepip --upgrade + "$PYTHON" -m pip install --upgrade pip 'setuptools<80' wheel +fi + +# ----- Accelerator detection ----- +TORCH_INDEX="" +if [[ -f "$GPU_DETECT" ]]; then + # gpu_detect.py is pure stdlib, runs before torch is installed. + TORCH_INDEX="$("$PYTHON" "$GPU_DETECT" index 2>/dev/null || true)" +fi +TORCH_INDEX="${TORCH_INDEX:-https://download.pytorch.org/whl/cpu}" +echo "Accelerator wheel index: $TORCH_INDEX" + +# ----- Core deps ----- +if [[ ! -f ".deps_v${DEPS_VER}" ]]; then + echo + echo "Installing core dependencies..." + "$PYTHON" -m pip install --upgrade pip 'setuptools<80' wheel + if [[ -f "$GPU_DETECT" ]]; then + "$PYTHON" "$GPU_DETECT" summary 2>/dev/null || true + fi + echo "Installing PyTorch from $TORCH_INDEX ..." + if ! "$PYTHON" -m pip install --force-reinstall --no-deps \ + "torch==2.5.1" "torchvision==0.20.1" "torchaudio==2.5.1" \ + --index-url "$TORCH_INDEX"; then + echo "Wheel index failed, falling back to CPU build..." + "$PYTHON" -m pip install --force-reinstall --no-deps \ + "torch==2.5.1" "torchvision==0.20.1" "torchaudio==2.5.1" \ + --index-url https://download.pytorch.org/whl/cpu + fi + echo "Installing openai-whisper (build-isolation workaround)..." + "$PYTHON" -m pip install --no-build-isolation "openai-whisper>=20230918" + echo "Installing requirements.txt..." + "$PYTHON" -m pip install -r "$PROJ/requirements.txt" + echo "Installing chatterbox-tts (--no-deps)..." + "$PYTHON" -m pip install --no-deps chatterbox-tts + "$PYTHON" -m pip install resemble-perth + if [[ -f "$PROJ/voice/patch_chatterbox.py" ]]; then + echo "Patching chatterbox dtype issues..." + "$PYTHON" "$PROJ/voice/patch_chatterbox.py" + fi + date > ".deps_v${DEPS_VER}" +fi + +# ----- Data-science check ----- +if [[ ! -f ".datascience_v${DATASCIENCE_VER}" ]]; then + DS_MISSING=() + "$PYTHON" -c "import matplotlib" 2>/dev/null || DS_MISSING+=("matplotlib>=3.7.0") + "$PYTHON" -c "import pandas" 2>/dev/null || DS_MISSING+=("pandas>=2.0.0") + "$PYTHON" -c "import numpy" 2>/dev/null || DS_MISSING+=("numpy>=1.24.0") + "$PYTHON" -c "import sklearn" 2>/dev/null || DS_MISSING+=("scikit-learn>=1.3.0") + if (( ${#DS_MISSING[@]} > 0 )); then + echo "Installing: ${DS_MISSING[*]}" + "$PYTHON" -m pip install "${DS_MISSING[@]}" + fi + date > ".datascience_v${DATASCIENCE_VER}" +fi + +# ----- ImageBind ----- +if [[ ! -f ".imagebind_v${IMAGEBIND_VER}" ]]; then + if ! "$PYTHON" -c "from imagebind.models import imagebind_model" 2>/dev/null; then + echo "Installing ImageBind..." + "$PYTHON" -m pip install --no-deps git+https://github.com/facebookresearch/ImageBind.git@main + fi + "$PYTHON" -c "import timm" 2>/dev/null || "$PYTHON" -m pip install "timm>=0.6.7" + "$PYTHON" -c "import einops" 2>/dev/null || "$PYTHON" -m pip install "einops>=0.6.0" + if ! "$PYTHON" -c "import torchaudio" 2>/dev/null; then + echo "Reinstalling torch suite for alignment..." + "$PYTHON" -m pip install --force-reinstall --no-deps \ + "torch==2.5.1" "torchvision==0.20.1" "torchaudio==2.5.1" \ + --index-url "$TORCH_INDEX" + fi + date > ".imagebind_v${IMAGEBIND_VER}" +fi + +# ----- Chatterbox dtype patch (Windows-bug fix that also helps on POSIX) ----- +if [[ ! -f ".whisperx_v${WHISPERX_VER}" ]]; then + echo "Installing pyannote.audio 3.3.2 for speaker diarization..." + # We use faster-whisper for transcription and pyannote.audio + # directly for diarization. We do NOT install the whisperx wrapper + # — its current line pins torch 2.8.x which conflicts with our + # torch==2.5.1 stack and would cascade-break chatterbox-tts, + # ImageBind, and the embedding ecosystem. Remove any stale + # whisperx and roll back deps it cascade-upgraded. + "$PYTHON" -m pip uninstall -y whisperx >/dev/null 2>&1 || true + "$PYTHON" -m pip install --force-reinstall "numpy<2" + "$PYTHON" -m pip install "transformers==5.2.0" + "$PYTHON" -m pip install "pyannote.audio==3.3.2" + # huggingface_hub stays at whatever transformers 5.2.0 installs + # (1.x). pyannote 3.3.2's call to hf_hub_download(use_auth_token=) + # is translated to token= by a runtime monkey-patch in + # media_ingest_models._load_diarizing_whisper. No version pin. + date > ".whisperx_v${WHISPERX_VER}" +fi + +if [[ ! -f ".chatterbox_patch_v${CHATTERBOX_PATCH_VER}" ]]; then + if [[ -f "$PROJ/voice/patch_chatterbox.py" ]]; then + echo "Patching chatterbox dtype and watermarker issues..." + "$PYTHON" "$PROJ/voice/patch_chatterbox.py" + fi + date > ".chatterbox_patch_v${CHATTERBOX_PATCH_VER}" +fi + +# ----- Playwright browser ----- +if ! "$PYTHON" -c "from playwright.sync_api import sync_playwright; p=sync_playwright().start(); b=p.chromium.launch(headless=True); b.close(); p.stop()" > /dev/null 2>&1; then + if "$PYTHON" -c "import playwright" 2>/dev/null; then + echo "Installing Chromium for Playwright (one-time, ~150MB)..." + "$PYTHON" -m playwright install chromium + # Linux only: install system-level dependencies if root or sudo available. + if [[ "$(uname -s)" == "Linux" ]] && command -v sudo > /dev/null 2>&1; then + sudo -n true 2>/dev/null && "$PYTHON" -m playwright install-deps chromium || true + fi + fi +fi + +# ----- Shell mode short-circuit ----- +if (( SHELL_MODE )); then + if [[ ! -f "$VENV/bin/activate" ]]; then + echo "ERROR: venv activate script missing." + exit 1 + fi + exec "${SHELL:-/bin/bash}" --rcfile <(echo "source '$VENV/bin/activate'; PS1='(leos) \w \$ '") +fi + +# ----- Bootstrap Ollama and ComfyUI just before launch ----- +bootstrap_ollama +bootstrap_comfyui + +echo +echo "====================================" +echo " leOS" +echo "====================================" +echo +echo " http://localhost:5000" +echo + +# server.py opens the browser itself once the port is accepting; see the +# _open_browser_when_ready thread in its __main__ block. +cd "$PROJ" +exec "../$PYTHON" server.py