diff --git a/.env.example b/.env.example index b74c681..d8be53e 100644 --- a/.env.example +++ b/.env.example @@ -103,7 +103,7 @@ AGENT_VERBOSE=false INGEST_POLL_INTERVAL=7 # Wiki scheduler HTTP read-timeout (seconds) on /wiki/maintain and -# /wiki/write calls. Default 1200 (20 min). Local quantised models +# /wiki/write calls. Default 2400 (40 min). Local quantised models # (Qwen 27B AWQ-INT4 on vLLM) routinely take 6-15 min for a full wiki # body; setting this below ~600 caused the scheduler to give up while # the api kept working — queue drained slower than reality. Raise if @@ -111,7 +111,7 @@ INGEST_POLL_INTERVAL=7 # write actually committed (check `wikis_ext.revision`); lower only if # you specifically want quicker scheduler turnover. The api itself is # unbounded by this; this only controls the scheduler's patience. -# WIKI_AGENT_TIMEOUT=1200 +# WIKI_AGENT_TIMEOUT=2400 # Per-wiki cooldown on attach claims (seconds). Default 300 (5 min). # Once the OLDEST pending attach for a given wiki is this old, the diff --git a/.gitignore b/.gitignore index dee2fc6..4fd6851 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,33 @@ data_test/ # Hermes sandbox (integrations/hermes/sandbox): throwaway agent profile dir — # holds a .env with the LLM key + provider state; never commit it. integrations/hermes/sandbox/hermes-data/ + +# Custom profiles — only the contract docs + the example/ profile are public. +# Every real profile (its prompt fragments, ingestor.py, and .env secret) is +# local-only: domain-specific content + credentials that must not be committed. +custom-profiles/* +!custom-profiles/README.md +!custom-profiles/SKILL.md +!custom-profiles/example/ +!custom-profiles/example/** +!custom-profiles/hackernews/ +!custom-profiles/hackernews/** +# ...but never the hackernews ingestor's runtime files (created when it runs) +custom-profiles/hackernews/.env +custom-profiles/hackernews/.state/ +custom-profiles/hackernews/__pycache__/ +!custom-profiles/gdrive/ +!custom-profiles/gdrive/** +# ...but never the gdrive ingestor's runtime files OR its credentials — the +# service-account JSON key must NEVER be committed (public repo). +custom-profiles/gdrive/.env +custom-profiles/gdrive/.state/ +custom-profiles/gdrive/__pycache__/ +custom-profiles/gdrive/*.json +!custom-profiles/hermes/ +!custom-profiles/hermes/** +# ...but never the hermes ingestor's runtime files OR its secret (the Hermes bearer +# token lives in .env, which must never be committed). +custom-profiles/hermes/.env +custom-profiles/hermes/.state/ +custom-profiles/hermes/__pycache__/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bcc338..c0bf9de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,57 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.9.0] — 2026-06-26 + +Headline: **custom profiles** — opt-in, self-contained overlays that reshape what BrainDB ingests +and how its auto-maintained wikis read, with ZERO effect on defaults when inactive. Two ship ready +to run: a keyless **Hacker News** chronicle that grows company/project/tech pages from the firehose, +and a **Hermes situational tracker** that asks an external web-search agent about an evolving +real-world situation and keeps a dated, sourced, dashboard-shaped status page. Also in this release: +the wiki writer now rejects blank/garbage bodies before persist, and the scheduler write-timeout +default is raised for slower local models. + +### Added + +- **Custom profiles framework.** An opt-in composition layer (`braindb/custom_profile.py`, + `braindb/profile_runner.py`) that shapes ingestion + wiki output from a single self-contained + folder under `custom-profiles//`, activated by `CUSTOM_PROFILE` — defaults are byte-for-byte + unchanged when unset. A profile may carry `wiki_maintainer.add.md` / `wiki_writer.add.md` prompt + fragments (appended after the base prompt; a `*.replace.md` swaps it wholesale) and an optional + supervised `ingestor.py` (plus its own gitignored `.env`) that feeds a custom source into the + existing file-watcher. Profiles compose (comma-separated). Core API, schema, and base prompts are + untouched. Ships with an `example/` template and a `gdrive/` source ingestor alongside the two + profiles below. +- **`hackernews` profile — keyless tech chronicle.** Polls the public Hacker News API, drops each new + story into `data/sources/`, and shapes the resulting entity pages as tech-entity chronicles + (type / homepage / category, a dated `current-developments` feed that ages into `background`, every + claim cited). No API key; runs immediately. +- **`hermes` profile — situational tracker.** On a schedule, asks an external + [Hermes Agent](https://github.com/NousResearch/hermes-agent) (web-search, OpenAI-compatible gateway) + a dated, stateless question and folds each answer into a central situational wiki shaped as a + dashboard — status + confidence + most-credible-source, then `now` / `recent-days` / `recent-weeks` + / `forecast` / `history`, every line dated and `[[ref:UUID]]`-cited, conflicts resolved by + credibility or flagged low-confidence — while the other entities each report mentions still get + their own cross-linked wikis. Dormant by default (no key, so it idles and never calls); a + `HERMES_SAMPLE_FILE` dry-run exercises the whole pipeline offline. +- **Current date in wiki prompts.** The maintainer and writer prompts now carry today's UTC date, so + recency / "as of" / dated-bucket reasoning has an anchor. + +### Changed + +- **Wiki scheduler write-timeout default raised 1200 -> 2400s (20 -> 40 min).** `WIKI_AGENT_TIMEOUT`; + smaller/slower quantised local models (e.g. Gemma 12B) can exceed 20 min on a full-body write, so + the old default abandoned slow-but-completing writes and forced retries. Client-patience only — the + API is not bounded by it and a write past the deadline still commits. + +### Fixed + +- **Writer rejects blank/garbage bodies before persist.** A weak model sometimes emits serialization + junk (`""`, `body=""`, `{}`) instead of a page; the previous whitespace-only gate let it through to + be written as content and bump the revision, self-perpetuating into empty, high-revision pages. A + single blank-body check now routes such writes to the existing retry path instead of persisting — + good pages can't be overwritten by junk, and the entity stays a healable orphan for the next cycle. + ## [0.8.0] — 2026-06-16 Headline: the wiki maintainer now batches related orphan triage into ONE LLM call — diff --git a/README.md b/README.md index 434356e..0b37765 100644 --- a/README.md +++ b/README.md @@ -352,6 +352,24 @@ automatically. To run without it, bring the stack up excluding the service or scale it to 0 (`docker compose up -d --scale wiki_scheduler=0`), exactly as you would for the watcher; or point `LLM_PROFILE` at a local model. +### Custom profiles — shape ingestion and wiki structure + +A **custom profile** is an opt-in, self-contained folder under [`custom-profiles/`](custom-profiles/) +that reshapes what BrainDB ingests and how its auto-maintained wikis read — without touching core. A +profile can append (or replace) the maintainer/writer prompts via `*.add.md` fragments and run its own +supervised `ingestor.py` to feed a custom source into the watcher. Activate with `CUSTOM_PROFILE=` +(comma-separated to compose); when unset, defaults are byte-for-byte unchanged. + +Two ship ready to run: + +- **[`hackernews`](custom-profiles/hackernews/)** — keyless: polls Hacker News and grows + company/project/tech pages as dated chronicles. +- **[`hermes`](custom-profiles/hermes/)** — asks an external web-search agent about an evolving + situation and keeps a dated, sourced, dashboard-shaped status page; dormant until configured. + +See [`custom-profiles/README.md`](custom-profiles/README.md) for the contract and +[`SKILL.md`](custom-profiles/SKILL.md) to author your own. + ## Frontend (read-only browser UI) A small vanilla-JS frontend ships under `frontend/` — no build step — and is served by the stack itself: after `docker compose up -d`, open **`http://localhost:8642`** (change the port with `FRONTEND_PORT` in `.env`). Three views (Reader for browsing wikis, Graph for visual exploration, Ops for watching the maintainer/writer pipeline) plus an Ask drawer that talks to the agent endpoint. The UI calls the BrainDB API at `http://localhost:8000` from your browser. diff --git a/braindb/custom_profile.py b/braindb/custom_profile.py new file mode 100644 index 0000000..4cc83aa --- /dev/null +++ b/braindb/custom_profile.py @@ -0,0 +1,84 @@ +""" +Optional custom-profile prompt composition. ZERO effect unless CUSTOM_PROFILE is set. + + CUSTOM_PROFILE=cityfalcon_news # one profile + CUSTOM_PROFILE=company_base,cityfalcon_news # several, composed in order + CUSTOM_PROFILE_DIR=custom-profiles # optional; default shown + +A profile is a folder under CUSTOM_PROFILE_DIR holding prompt fragments named +`.add.md` (appended to the base prompt) and/or `.replace.md` +(swaps the base template entirely). `target` is the prompt key a call site asks +for, e.g. "wiki_maintainer" or "wiki_writer". + +Two composition primitives, used at each prompt-assembly site: + + replace_or_base(base_template, target) -> the active `.replace.md` + (last active profile wins) or `base_template` unchanged. Applied to the + TEMPLATE *before* placeholder substitution, so a replace author keeps the + base's `{seeds}` / `%%MODE%%` tokens and owns the machine contract. + + additions(target) -> "\n\n" + joined `.add.md` fragments across the + active profiles in list order, or "". Appended *after* substitution, so a + fragment may contain any characters (including `{}`) with no str.format + hazard. + +Both degrade to the no-op result (the base, or "") when CUSTOM_PROFILE is unset, +a folder/file is missing, or anything goes wrong — they never raise into a +prompt-assembly path. Read-at-call: a dropped-in fragment takes effect on the +next assembly, and tests can monkeypatch CUSTOM_PROFILE without reimporting. +""" +import os +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent # braindb/.. -> repo root + + +def _active_profiles() -> list[str]: + """Profile names from CUSTOM_PROFILE (comma-separated), in order. [] when unset.""" + raw = os.getenv("CUSTOM_PROFILE", "").strip() + return [p.strip() for p in raw.split(",") if p.strip()] + + +def _profiles_root() -> Path: + base = os.getenv("CUSTOM_PROFILE_DIR", "custom-profiles").strip() or "custom-profiles" + p = Path(base) + return p if p.is_absolute() else _REPO_ROOT / p + + +def _read(path: Path) -> str | None: + try: + return path.read_text(encoding="utf-8") if path.is_file() else None + except Exception: + return None + + +def replace_or_base(base_template: str, target: str) -> str: + """Return the active `.replace.md` (last profile wins), else + `base_template` unchanged. Never raises.""" + try: + root = _profiles_root() + chosen = base_template + for name in _active_profiles(): + text = _read(root / name / f"{target}.replace.md") + if text is not None: + chosen = text + return chosen + except Exception: + return base_template + + +def additions(target: str) -> str: + """Return '\\n\\n' + joined `.add.md` fragments across active profiles + (list order), or '' when none / on any error. Never raises.""" + try: + root = _profiles_root() + frags: list[str] = [] + for name in _active_profiles(): + text = _read(root / name / f"{target}.add.md") + if text is not None: + stripped = text.strip() + if stripped: + frags.append(stripped) + return ("\n\n" + "\n\n".join(frags)) if frags else "" + except Exception: + return "" diff --git a/braindb/main.py b/braindb/main.py index bb092c4..21b4289 100644 --- a/braindb/main.py +++ b/braindb/main.py @@ -11,7 +11,7 @@ app = FastAPI( title="BrainDB", description="Memory database and REST API for LLM agents", - version="0.8.0", + version="0.9.0", ) app.add_middleware( diff --git a/braindb/profile_runner.py b/braindb/profile_runner.py new file mode 100644 index 0000000..baabd2b --- /dev/null +++ b/braindb/profile_runner.py @@ -0,0 +1,109 @@ +""" +Generic custom-profile ingestor supervisor — ONE loop, same posture as +ingest_watcher.py / wiki_scheduler.py. DORMANT unless CUSTOM_PROFILE is set. + +For each active profile (CUSTOM_PROFILE, comma-separated) that ships an +`ingestor.py`, this launches `python /ingestor.py` as an ISOLATED +subprocess and keeps it alive (restart with backoff on exit). It holds no +profile-specific knowledge and never imports profile code: a broken ingestor +can only crash its own subprocess, never the api / watcher / wiki. + +No active profile, or no active profile ships an ingestor.py -> the runner +sleeps and makes zero calls (exactly like WIKI_ENABLED=false). This is what +lets ONE switch (CUSTOM_PROFILE) bring up prompt-shaping AND a profile's +ingestor together, while the base docker-compose.yml carries only a neutral, +dormant-by-default service. See custom-profiles/README.md. +""" +import logging +import os +import subprocess +import sys +import time +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent # braindb/.. -> repo root + +# Liveness-check cadence and the minimum spacing between relaunch attempts for a +# crashing ingestor (prevents a tight crash-loop). Both are seconds. +POLL_INTERVAL = int(os.getenv("PROFILE_RUNNER_INTERVAL", "15")) +RESTART_BACKOFF = int(os.getenv("PROFILE_RUNNER_BACKOFF", "30")) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [profile-runner] %(message)s", + datefmt="%H:%M:%S", + stream=sys.stdout, +) +log = logging.getLogger("profile-runner") + + +def _profiles_root() -> Path: + base = os.getenv("CUSTOM_PROFILE_DIR", "custom-profiles").strip() or "custom-profiles" + p = Path(base) + return p if p.is_absolute() else _REPO_ROOT / p + + +def _active_profiles() -> list[str]: + raw = os.getenv("CUSTOM_PROFILE", "").strip() + return [p.strip() for p in raw.split(",") if p.strip()] + + +def _ingestors() -> dict[str, Path]: + """{profile_name: ingestor.py path} for active profiles that ship one.""" + root = _profiles_root() + out: dict[str, Path] = {} + for name in _active_profiles(): + script = root / name / "ingestor.py" + if script.is_file(): + out[name] = script + return out + + +def main() -> None: + ingestors = _ingestors() + if not ingestors: + log.info("no active profile ingestor (CUSTOM_PROFILE=%r). Idle.", + os.getenv("CUSTOM_PROFILE", "")) + # Sleep forever — container stays Up, zero calls. Toggle via env + restart. + while True: + time.sleep(3600) + + log.info("supervising %d profile ingestor(s): %s", + len(ingestors), ", ".join(ingestors)) + + procs: dict[str, subprocess.Popen] = {} + launched_at: dict[str, float] = {} + + def _launch(name: str, script: Path) -> None: + log.info("launching ingestor for profile '%s': %s", name, script) + # Isolated subprocess; cwd at repo root so the script's relative paths + # (its own folder, data/sources/) resolve exactly as in the containers. + procs[name] = subprocess.Popen([sys.executable, str(script)], cwd=str(_REPO_ROOT)) + launched_at[name] = time.time() + + for name, script in ingestors.items(): + _launch(name, script) + + while True: + try: + for name, script in ingestors.items(): + proc = procs.get(name) + if proc is None: + _launch(name, script) + continue + if proc.poll() is None: + continue # still running + # Exited — relaunch once the backoff window (since last launch) + # has elapsed, so a fast crash-loop is throttled while a + # long-lived process that dies restarts promptly. + if time.time() - launched_at[name] >= RESTART_BACKOFF: + log.warning("ingestor '%s' exited (code=%s); restarting", + name, proc.returncode) + _launch(name, script) + except Exception as e: + log.exception("supervisor loop error: %s", e) + time.sleep(POLL_INTERVAL) + + +if __name__ == "__main__": + main() diff --git a/braindb/routers/wiki.py b/braindb/routers/wiki.py index 2cc2e77..97e88eb 100644 --- a/braindb/routers/wiki.py +++ b/braindb/routers/wiki.py @@ -7,10 +7,13 @@ agent endpoint. """ import logging +import re +from datetime import datetime, timezone from pathlib import Path from fastapi import APIRouter, Query +from braindb import custom_profile from braindb.agent.agent import run_typed, get_maintainer_agent, get_writer_agent from braindb.agent.run_state import install_handoff_slot, release_handoff_slot from braindb.agent.schemas import MaintainerClusterDecision, WikiWriteResult @@ -28,6 +31,30 @@ _WRITER_PROMPT = (_PROMPTS / "wiki_writer_prompt.md").read_text(encoding="utf-8") +def _now_line() -> str: + """One neutral line giving the model the current date, so temporal reasoning + (recency, 'as of', dated/bucketed sections) has an anchor. Appended to every + maintainer/writer prompt; harmless when a subject has no time element.""" + return ( + "\n\n---\n**Current date (UTC):** " + f"{datetime.now(timezone.utc):%Y-%m-%d %H:%M} — treat this as \"now\" when " + "judging recency, dating claims, or bucketing events.\n" + ) + + +# A wiki body that carries no real page content: the empty string, whitespace, +# or a small-model serialization artifact echoed verbatim (`""`, `body=""`, +# `{}`). The regex matches ONLY an optional `body=`/`body:` prefix followed by +# quotes/braces/whitespace to the end — a genuine body (meta header + markdown) +# always has real characters, so it never matches. The single definition the +# router uses to decide a body is unpersistable. +_BLANK_BODY = re.compile(r'^\s*(?:body\s*[:=]\s*)?["\'{}\s]*$', re.IGNORECASE) + + +def _is_blank_body(body: str | None) -> bool: + return bool(_BLANK_BODY.match(body or "")) + + @router.post("/cron") def wiki_cron(): """Read-only orphan scan; enqueues one `triage` job per orphan. Idempotent.""" @@ -95,9 +122,14 @@ async def wiki_maintain(): or "(no existing wikis yet — attach/consolidate are impossible; " "use create/skip/ambiguous)" ) - prompt = _MAINTAINER_PROMPT.format( + # Optional custom-profile shaping: a `.replace.md` may swap the template + # (before `.format`), and `.add.md` fragments are appended AFTER `.format` + # (so a fragment's literal `{`/`}` can never reach str.format). No active + # profile -> both are no-ops and this prompt is byte-identical to the base. + _maintainer_tmpl = custom_profile.replace_or_base(_MAINTAINER_PROMPT, "wiki_maintainer") + prompt = _maintainer_tmpl.format( seeds=_seeds_block(seeds), wiki_catalog=catalog_txt - ) + ) + _now_line() + custom_profile.additions("wiki_maintainer") # `run_typed` returns an SDK-validated MaintainerClusterDecision, or raises # if the model never submitted — handled below like any agent failure. try: @@ -339,16 +371,20 @@ def _dupes_block(ds: list[dict]) -> str: for i, d in enumerate(ds, 1) ) - # 2. One focused agent call. + # 2. One focused agent call. Optional custom-profile shaping: a + # `.replace.md` may swap the template (it then owns the `%%...%%` tokens); + # `.add.md` fragments are appended after substitution. No active profile -> + # both are no-ops and this prompt is byte-identical to the base. + _writer_tmpl = custom_profile.replace_or_base(_WRITER_PROMPT, "wiki_writer") prompt = ( - _WRITER_PROMPT + _writer_tmpl .replace("%%MODE%%", mode) .replace("%%CANONICAL%%", canonical) .replace("%%WIKI_ID%%", bucket["target_wiki_id"] or "(assigned after write)") .replace("%%MEMBERS%%", _members_block(members)) .replace("%%CURRENT_BODY%%", _body_block_or_stub(mode, bucket.get("target_wiki_id"), old_body)) .replace("%%DUPLICATES%%", _dupes_block(dupes)) - ) + ) + _now_line() + custom_profile.additions("wiki_writer") # Capture pre-run revision on the target wiki for `attach` mode so we # can detect whether the writer used the section-edit tools (each # bumps `wikis_ext.revision` directly). The writer may then submit an @@ -445,7 +481,7 @@ def _dupes_block(ds: list[dict]) -> str: release_handoff_slot(handoff_token) used_section_edits = False - if not (res.body or "").strip(): + if _is_blank_body(res.body): # Empty body — only valid in attach mode if section edits bumped # the revision during the run. Otherwise the agent did nothing # persistable and we fail the jobs. @@ -519,8 +555,25 @@ def _dupes_block(ds: list[dict]) -> str: else: new_body = res.body - # 3. Persist (one transaction). No content gate — the LLM's body is - # authoritative; we only snapshot (reversible) and reconcile additively. + # Final blank-body backstop. `new_body` is now whatever we are about to + # persist — either the LLM's submission or the post-section-edit DB body. + # A genuine page always has real characters; only serialization garbage + # (`""`, `body=""`, `{}`) reaches here blank. Persisting it would write an + # empty page AND bump the revision, which self-perpetuates into a high-rev + # empty loop. Reject instead (same release/retry path as the gates above) — + # the entity stays a healable orphan and is re-tried next cycle. The legit + # no-op (all members already cited) already returned above, so this never + # touches it. + if _is_blank_body(new_body): + with get_conn() as conn: + disp = wiki_jobs.release_or_fail_jobs( + conn, job_ids, + f"blank/garbage body not persisted ({mode}): {new_body[:80]!r}") + return {"written": 0, "result": disp, "reason": "blank body"} + + # 3. Persist (one transaction). The LLM's body is authoritative (guarded + # above against blank); we only snapshot (reversible) and reconcile + # additively. with get_conn() as conn: summary, disambig = wiki_jobs.extract_summary_disambig(new_body) kw = wiki_jobs.keywords_from_meta(new_body) diff --git a/braindb/wiki_scheduler.py b/braindb/wiki_scheduler.py index c5d2ae1..afed2ce 100644 --- a/braindb/wiki_scheduler.py +++ b/braindb/wiki_scheduler.py @@ -42,11 +42,14 @@ # model, so a 600s deadline at the scheduler caused the client to give up # WHILE the api kept working in the background — the write still committed, # but the scheduler couldn't see the completion in time to drain the queue -# efficiently. With 1200s the client now waits long enough to see most -# writes finish, while still surfacing genuinely-stuck jobs as failures -# rather than blocking indefinitely. The api itself is not bounded by this -# value; this knob only controls how patient the scheduler's HTTP client is. -AGENT_TIMEOUT = int(os.getenv("WIKI_AGENT_TIMEOUT", "1200")) +# efficiently. Bumped again 1200 → 2400 (20 → 40 min) for smaller/slower +# quantised models (e.g. Gemma 12B QAT-W4A16): their full-body writes can +# exceed 20 min, so a 1200s deadline abandoned slow-but-completing writes +# and forced needless retries. With 2400s the client waits out most writes +# while still surfacing genuinely-stuck jobs as failures rather than blocking +# indefinitely. The api itself is not bounded by this value; this knob only +# controls how patient the scheduler's HTTP client is. +AGENT_TIMEOUT = int(os.getenv("WIKI_AGENT_TIMEOUT", "2400")) logging.basicConfig( level=logging.INFO, diff --git a/custom-profiles/README.md b/custom-profiles/README.md new file mode 100644 index 0000000..148cc98 --- /dev/null +++ b/custom-profiles/README.md @@ -0,0 +1,84 @@ +# Custom profiles + +Optional, self-contained overlays that shape BrainDB's wiki prompts and (optionally) +feed it a custom ingestion source — **with zero effect on default behaviour when none is +active**. One switch, one folder. + +A profile is a folder under this directory. You activate it with a single env var in the +repo-root `.env`: + +``` +CUSTOM_PROFILE=cityfalcon_news # one profile +CUSTOM_PROFILE=company_base,cityfalcon_news # several, composed left-to-right +``` + +When `CUSTOM_PROFILE` is empty (the default), every mechanism below is a strict no-op: +the prompts are the baked-in defaults, byte-for-byte, and the ingestor supervisor sleeps. + +Only this `README.md` and the `example/` profile are committed. **Every real profile is +gitignored** (see `.gitignore`) — its prompt fragments are domain-specific and its `.env` +holds credentials. + +## What a profile folder may contain + +``` +custom-profiles// +├── wiki_maintainer.add.md # appended to the maintainer prompt +├── wiki_maintainer.replace.md # OR: replaces the maintainer prompt entirely +├── wiki_writer.add.md # appended to the writer prompt +├── wiki_writer.replace.md # OR: replaces the writer prompt entirely +├── ingestor.py # optional: a standalone polling loop (see below) +└── .env # optional: the ingestor's own secrets/config +``` + +All files are optional. A profile that only ships `wiki_writer.add.md` touches nothing +else. + +## Prompt shaping — `add` vs `replace` + +Two filename conventions, two behaviours (`` ∈ `wiki_maintainer`, `wiki_writer`): + +- **`.add.md` — append.** Your text is glued onto the end of the base prompt, + *after* the base's placeholders are filled in. The base stays intact, so all the + machine-contract rules it carries (citation tokens, section markers, decision schema) + are preserved. This is the safe default and covers almost every need. +- **`.replace.md` — full swap.** Your file *becomes* the prompt (the base is + discarded). It is substituted for the template *before* placeholders are filled, so you + keep the dynamic tokens — but you also inherit responsibility for the machine contract: + the maintainer's `{seeds}` / `{wiki_catalog}`, the writer's `%%MODE%%` / `%%CANONICAL%%` + / `%%WIKI_ID%%` / `%%MEMBERS%%` / `%%CURRENT_BODY%%` / `%%DUPLICATES%%`, the + `[[ref:UUID]]` citation tokens, the `` markers, and the + `` header. Drop these and the pipeline silently breaks. + Prefer `add` unless you truly need to rewrite the base. + +With several active profiles, `add` fragments are concatenated in list order; the last +profile that supplies a `replace` for a target wins. + +## Custom ingestion — `ingestor.py` + +A profile may ship an `ingestor.py`: a standalone, long-running script (its own poll +loop) that feeds BrainDB. The committed `profile_runner` sidecar +(`braindb/profile_runner.py`) launches each active profile's `ingestor.py` as an isolated +subprocess and restarts it on exit. With no active profile it sleeps — so the base +`docker-compose.yml` carries only a neutral, dormant-by-default service, and the same +`CUSTOM_PROFILE` switch brings up both the prompt shaping and the ingestor. + +The simplest, least-coupled ingestor just **writes files into `data/sources/`** and lets +the existing watcher do ingestion + fact-extraction unchanged (the verbatim source is +stored in the DB at ingest, so the file is only a carrier and can be pruned afterwards). + +`ingestor.py` should load its own secrets from a sibling `.env`: + +```python +from pathlib import Path +from dotenv import load_dotenv +load_dotenv(Path(__file__).parent / ".env") +``` + +This keeps each profile self-contained and means no profile-specific variable is ever +named in the public `docker-compose.yml`. + +## The `example/` profile + +`example/` is a harmless, committed reference. Activate it with `CUSTOM_PROFILE=example` +to see the `add` mechanism in action against the writer prompt; it ships no ingestor. diff --git a/custom-profiles/SKILL.md b/custom-profiles/SKILL.md new file mode 100644 index 0000000..fffaf45 --- /dev/null +++ b/custom-profiles/SKILL.md @@ -0,0 +1,117 @@ +--- +name: braindb-custom-profile +description: How to author a BrainDB custom profile — prompt add/replace fragments and an optional keyless ingestor — that shapes wiki naming/structure and feeds a custom ingestion source, with zero effect on defaults when inactive. +allowed-tools: Read Write Edit Bash +--- + +# Authoring a BrainDB custom profile + +A **custom profile** lets you (a) shape how the wiki maintainer/writer **name and structure** +pages and (b) feed BrainDB a **custom ingestion source** — with **zero effect on default +behaviour when the profile is not active**. One env switch turns a profile on; one +self-contained folder holds everything it needs. + +The committed `custom-profiles/hackernews/` folder is a complete, **keyless** worked example +to copy from. `custom-profiles/README.md` is the short contract; this is the deep how-to. + +## The model + +- **One switch:** `CUSTOM_PROFILE` in the repo-root `.env` names the active profile(s), + comma-separated. **Unset ⇒ every prompt is byte-identical to the baked-in default** and the + ingestor supervisor sleeps. +- **One folder:** `custom-profiles//` holds the profile's prompt fragments, an optional + `ingestor.py`, and an optional `.env`. + +## Folder contract + +``` +custom-profiles// +├── wiki_maintainer.add.md # appended to the maintainer prompt +├── wiki_maintainer.replace.md # OR replaces it entirely (advanced) +├── wiki_writer.add.md # appended to the writer prompt +├── wiki_writer.replace.md # OR replaces it entirely (advanced) +├── ingestor.py # optional: a standalone source feeder +├── .env # optional: the ingestor's own config/secrets +└── README.md +``` + +Targets are `wiki_maintainer` and `wiki_writer`. `.add.md` is **appended** to the +base prompt; `.replace.md` **replaces** it. With several active profiles, `add` +fragments are concatenated in `CUSTOM_PROFILE` order. All files are optional. + +### What each prompt shapes +- **`wiki_maintainer`** decides, per orphan entity: skip / create (with a `proposed_name`) / + attach / consolidate. Shape **naming and dedup** here (e.g. "name companies by official + name; same ticker = same company"). +- **`wiki_writer`** authors the page body (free markdown). Shape **structure** here — the + sections, a labelled profile block, a dated chronicle. + +## Rules that keep it safe + +1. **Prefer `add` over `replace`.** `add` keeps the base prompt — and all the machine + contract it carries — intact and only layers your guidance on top. `replace` makes you + re-own that contract. +2. **Never break the machine contract** (the writer body is parsed for these): + - `[[ref:UUID]]` inline citations — how a wiki links to its evidence; + - `` markers — the section system (arbitrary NAMEs are allowed, so + you may add e.g. `profile`, `background`, `current-developments`); + - the `` header — the only source of a + page's keywords. +3. **Cite or `(unknown)` — never invent.** A structured field gets a value **and** its + `[[ref:UUID]]` only when a source supports it; otherwise write `(unknown)`. +4. **Make the clustering entity salient in the dropped file.** Facts cluster into a wiki by + the keyword they're tagged with, so put the entity (ticker, company, project) prominently + in the file (e.g. a `**Tickers:** MSFT` header line) so the extractor tags facts with it. + +## The ingestor (optional) + +A profile may ship `ingestor.py`: a standalone, long-running loop that feeds BrainDB. The +committed `profile_runner` sidecar launches each active profile's `ingestor.py` as an +**isolated subprocess** (restart-on-exit), and sleeps when no profile is active — so a broken +ingestor can never affect the api/watcher/wiki. + +The simplest, least-coupled ingestor just **writes files into `data/sources/`** and lets the +existing watcher do ingestion + fact-extraction — no DB or agent calls. Pattern: + +- load config from a sibling `.env` (a tiny parser into `os.environ`, or `python-dotenv`); +- poll your source; for each **new** item (track seen ids in a `.state/` file), write one + `-.md` into `data/sources/`, with the entity made salient; +- prune old files under `data/sources/ingested/` (the verbatim text is stored in the DB at + ingest, so the file is only a carrier); +- sleep and repeat. + +Resolve `data/sources/` from the script's own location so it works in the container and +standalone: `Path(__file__).resolve().parents[1] / "data" / "sources"`. Load secrets from the +sibling `.env` so **no profile-specific variable ever appears in the public +`docker-compose.yml`**. Keep files small (title + a few fields). + +## Activate and verify + +1. (if the profile has secrets) `cp custom-profiles//.env.example custom-profiles//.env` and fill it. +2. In the repo-root `.env`: `CUSTOM_PROFILE=` (or `a,b` for several). +3. `docker compose up -d` — the api picks up the prompt shaping; `profile_runner` launches the + ingestor. +4. Watch it flow: + ```bash + ls data/sources/ # files dropped by the ingestor + curl -s "http://localhost:8000/api/v1/entities?entity_type=wiki&limit=20" + curl -s -X POST http://localhost:8000/api/v1/memory/context \ + -H "Content-Type: application/json" -d '{"queries":[""]}' + ``` + +To deactivate: remove the `CUSTOM_PROFILE` line and `docker compose up -d` — defaults restored. + +## Worked example + +`custom-profiles/hackernews/` is a complete, **keyless** profile (Hacker News → tech-entity +wikis). Copy its `ingestor.py` + `wiki_*.add.md` as a starting point, then run it with just +`CUSTOM_PROFILE=hackernews` — no key required. + +`custom-profiles/gdrive/` is a second example: a **folder follower** that ingests a Google Drive +folder **incrementally** — only new files and, on edits, only the changed sections (a `.state` +manifest + per-doc snapshots, each delta self-describing *what part it is* and *where it belongs*). +Copy it when your source is a watched folder of documents that change over time rather than a feed. + +A note on privacy: keep real profiles that carry a domain prompt or an API key **gitignored** +(see `.gitignore`); only generic teaching profiles like `example/` and `hackernews/` are +committed. diff --git a/custom-profiles/example/README.md b/custom-profiles/example/README.md new file mode 100644 index 0000000..4f42ed3 --- /dev/null +++ b/custom-profiles/example/README.md @@ -0,0 +1,15 @@ +# example profile + +A harmless reference profile that documents the contract. Activate it with: + +``` +CUSTOM_PROFILE=example +``` + +It ships a single `wiki_writer.add.md` fragment, so activating it only *appends* a short +note to the wiki writer prompt — the base prompt, the maintainer prompt, and all default +behaviour are otherwise untouched. It has no `ingestor.py`, so the `profile_runner` +sidecar stays idle. + +Copy this folder to start a real profile (e.g. `custom-profiles/my_profile/`); real +profiles are gitignored. See `../README.md` for the full contract. diff --git a/custom-profiles/example/wiki_writer.add.md b/custom-profiles/example/wiki_writer.add.md new file mode 100644 index 0000000..9a0ccd0 --- /dev/null +++ b/custom-profiles/example/wiki_writer.add.md @@ -0,0 +1,11 @@ + + +## Custom-profile note (example) + +This paragraph was appended to the base wiki writer prompt by the `example` +custom profile. It does not change any rule above — it only demonstrates that a +profile can layer extra guidance onto the default prompt without editing it. + +If you can read this line inside the assembled writer prompt, the `add` +mechanism is working. Replace this file's content with real guidance in your own +profile. diff --git a/custom-profiles/gdrive/.env.example b/custom-profiles/gdrive/.env.example new file mode 100644 index 0000000..57de183 --- /dev/null +++ b/custom-profiles/gdrive/.env.example @@ -0,0 +1,40 @@ +# gdrive ingestor — copy this to `.env` and fill it in. ingestor.py loads THIS +# folder's `.env`. This file is committed; your real `.env` and the credential +# JSON are gitignored. +# +# AUTH: a Google Cloud *service account* with the Drive API enabled. A service +# account only sees what is SHARED with it — share the target folder with the +# service-account email (…iam.gserviceaccount.com) as Viewer. See README.md. + +# ── required ──────────────────────────────────────────────────────────────── +# The Drive folder to follow. It is the last path segment of the folder URL: +# https://drive.google.com/drive/folders/ +GDRIVE_FOLDER_ID= + +# Path to the service-account JSON key. Relative paths resolve against the repo +# root or this folder. Default: a service-account.json placed next to this file. +# GDRIVE_CREDENTIALS_FILE=custom-profiles/gdrive/service-account.json + +# ── behaviour (all optional) ──────────────────────────────────────────────── +# diff = ingest only changed/new sections of an edited doc (default, cheaper); +# full = re-ingest the whole doc on any change. +# GDRIVE_INGEST_MODE=diff + +# Walk subfolders recursively. +# GDRIVE_RECURSIVE=true + +# Poll cadence in seconds. +# GDRIVE_POLL_SECONDS=300 + +# Export mime types for native Google files. +# GDRIVE_EXPORT_DOC=text/markdown +# GDRIVE_EXPORT_SHEET=text/csv +# GDRIVE_EXPORT_SLIDES=text/plain + +# Delete ingested gdrive-*.* older than this many days (0 = keep forever). +# GDRIVE_PRUNE_DAYS=0 + +# On the very first run, seed state from the existing folder WITHOUT ingesting +# it (so only changes made from now on flow). Default false = backfill all docs +# once, then deltas. +# GDRIVE_SKIP_EXISTING_ON_FIRST_RUN=false diff --git a/custom-profiles/gdrive/README.md b/custom-profiles/gdrive/README.md new file mode 100644 index 0000000..64a9155 --- /dev/null +++ b/custom-profiles/gdrive/README.md @@ -0,0 +1,67 @@ +# gdrive profile (Google Drive folder follower) + +Follows a Google Drive folder and feeds BrainDB **only new files and only the changed parts of +edited docs** — generic and **env-driven**, so any user points it at their *own* folder and +credentials via `.env`, with no code change. See [`../SKILL.md`](../SKILL.md) for the profile +mechanism. + +## What it does + +- `ingestor.py` lists a Drive folder (recursively), exports native **Google Docs → markdown**, + **Sheets → CSV**, **Slides → text**, and drops the result into `data/sources/`, where the + existing watcher ingests + extracts it. (Binary `.docx`/`.pdf` are skipped — the pipeline is + text-only.) +- **Incremental.** A manifest (`.state/manifest.json`, `{file_id: change_key}`) skips unchanged + docs entirely. `change_key` = `md5Checksum` (uploads) or `modifiedTime` (native Docs). +- **Diff-aware (default `GDRIVE_INGEST_MODE=diff`).** First sight of a doc ingests the full + baseline; on later edits it emits **only the changed/new markdown sections**, each wrapped in a + self-describing breadcrumb (*what part it is* — heading path + position — and *where it belongs* + — doc title/id/URL, "full doc already in memory"). Set `GDRIVE_INGEST_MODE=full` to re-ingest the + whole doc on every change instead. + +This profile shapes ingestion only — it ships no wiki prompt fragments, so the wiki maintainer/ +writer keep their defaults. (Add `wiki_*.add.md` here later if you want shaped pages.) + +## One-time Google setup (any user, their own Drive) + +1. In Google Cloud, create (or reuse) a **service account** and **enable the Drive API** for its + project. Create a **JSON key** and save it as `service-account.json` in this folder (gitignored), + or point `GDRIVE_CREDENTIALS_FILE` at it. +2. **Share the folder with the service account.** A service account only sees what is shared with + it: open the Drive folder → Share → add the service-account email + (`…iam.gserviceaccount.com`) as **Viewer**. + *(For personal docs you don't want to share to a service account, an OAuth user flow is the + alternative — a future option, not wired here.)* + +## Run it + +1. `cp .env.example .env`, then set `GDRIVE_FOLDER_ID` (the last segment of the folder URL + `https://drive.google.com/drive/folders/`) and place the JSON key. +2. In the repo-root `.env`: `CUSTOM_PROFILE=gdrive` (or append, e.g. `gdrive,hackernews`). +3. `docker compose up -d`. +4. Watch it flow: + ```bash + docker compose logs -f profile_runner # "launching ingestor for profile 'gdrive'", + # then "found N / new M / changed K / skipped U" + ls data/sources/ # gdrive-*.md / gdrive-*.csv + curl -s -X POST http://localhost:8000/api/v1/memory/context \ + -H "Content-Type: application/json" -d '{"queries":[""]}' + ``` + +### Changing the tracked folder + +Edit `GDRIVE_FOLDER_ID` in `.env`, share the new folder with the service account, and +`docker compose up -d` (or restart `profile_runner`). To switch credentials, point +`GDRIVE_CREDENTIALS_FILE` at a different JSON key. No code change either way. + +## Honest notes / limits + +- A service account sees **only folders shared with it** (or that it owns). +- **Diff mode** anchors a delta's facts to the doc's existing in-memory cluster via the shared doc + title + the breadcrumb (the extractor can't live-fetch the source URL — it's preserved as + provenance). Editing a section may restate a prior fact (small near-dup surface the wiki + maintainer consolidates). +- **Deletions/trashing** are noted in the diff header but do **not** auto-retract already-extracted + facts. +- The Google client libs (`google-api-python-client`, `google-auth`) aren't in the BrainDB image; + the ingestor **self-installs** them on first run (the image stays untouched). diff --git a/custom-profiles/gdrive/ingestor.py b/custom-profiles/gdrive/ingestor.py new file mode 100644 index 0000000..331dd60 --- /dev/null +++ b/custom-profiles/gdrive/ingestor.py @@ -0,0 +1,487 @@ +""" +Google Drive folder-follower ingestor (gdrive profile). + +Launched by `braindb.profile_runner` when CUSTOM_PROFILE includes `gdrive`. It +does NOT touch BrainDB internals: it lists a Google Drive folder, exports native +Google Docs/Sheets/Slides to text, and writes files into `data/sources/`, where +the existing watcher ingests + extracts them. Generic + env-driven: every user +points it at their OWN folder and credentials via this folder's `.env` — no code +edit needed. + +Incremental by design — it ingests ONLY new files and ONLY the changed parts of +edited docs: + * a manifest (`.state/manifest.json`, `{file_id: change_key}`) decides which + docs are new/changed/unchanged. change_key = md5Checksum (uploads) or + modifiedTime (native Docs, which have no md5). Unchanged docs are skipped + with no download. + * a per-doc snapshot (`.state/snapshots/.md`) lets us, in `diff` mode + (default), emit only the changed/new markdown sections of an edited doc, each + wrapped in a self-describing breadcrumb (what part it is + where it belongs), + instead of re-ingesting the whole document (which would duplicate facts and + waste LLM time). First sight of a doc ingests the full baseline. + +Config — from this folder's own `.env` (copy `.env.example` -> `.env`): + GDRIVE_FOLDER_ID (required) the Drive folder to follow + GDRIVE_CREDENTIALS_FILE service-account JSON (default service-account.json here) + GDRIVE_INGEST_MODE diff (default) | full + GDRIVE_RECURSIVE walk subfolders (default true) + GDRIVE_POLL_SECONDS loop cadence (default 300) + GDRIVE_EXPORT_DOC/SHEET/SLIDES export mime types (defaults md / csv / plain) + GDRIVE_PRUNE_DAYS delete ingested gdrive-*.* older than this (default 0 = keep) + GDRIVE_SKIP_EXISTING_ON_FIRST_RUN seed state without ingesting existing docs (default false) + +IMPORTANT: a service account only sees what is SHARED with it. Share the target +folder with the service-account email (…iam.gserviceaccount.com) as Viewer. +""" +import difflib +import hashlib +import io +import json +import logging +import os +import re +import sys +import time +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +_REPO_ROOT = _HERE.parents[1] # custom-profiles/gdrive -> repo root +_SOURCES = _REPO_ROOT / "data" / "sources" +_INGESTED = _SOURCES / "ingested" +_STATE = _HERE / ".state" +_MANIFEST = _STATE / "manifest.json" +_SNAP_DIR = _STATE / "snapshots" + +_SCOPES = ["https://www.googleapis.com/auth/drive.readonly"] +_FOLDER_MIME = "application/vnd.google-apps.folder" +_FIELDS = ("nextPageToken, files(id,name,mimeType,modifiedTime,md5Checksum," + "webViewLink,trashed)") +# Extensions the BrainDB watcher accepts as plain text (binary docx/pdf are not +# ingestible, so we skip them rather than drop a file the watcher would reject). +_TEXT_EXTS = {"md", "txt", "json", "yaml", "yml", "csv", "log", "html", "xml"} +_MIME_EXT = {"text/markdown": "md", "text/plain": "txt", "text/csv": "csv"} +_MAX_DEPTH = 20 + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [gdrive-ingestor] %(message)s", + datefmt="%H:%M:%S", + stream=sys.stdout, +) +log = logging.getLogger("gdrive-ingestor") + + +# --------------------------------------------------------------------------- # +# env + small helpers (mirrors custom-profiles/hackernews/ingestor.py) +# --------------------------------------------------------------------------- # +def _load_env(path: Path) -> None: + """Minimal .env loader (no extra dependency). Container/host env wins.""" + if not path.is_file(): + return + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def _slug(s: str) -> str: + return re.sub(r"[^A-Za-z0-9._-]", "-", s)[:80] + + +def _resolve(p: str) -> str: + """Resolve a configured path against repo root / profile dir if relative.""" + pp = Path(p) + if pp.is_absolute(): + return str(pp) + for base in (_REPO_ROOT, _HERE): + cand = base / pp + if cand.is_file(): + return str(cand) + return str(_REPO_ROOT / pp) + + +def _ensure_deps() -> None: + """Self-bootstrap the Google client libs so the BrainDB image stays untouched.""" + try: + import googleapiclient # noqa: F401 + import google.oauth2.service_account # noqa: F401 + return + except ImportError: + pass + import subprocess + pkgs = ["google-api-python-client", "google-auth"] + log.info("installing Google Drive client libs: %s", ", ".join(pkgs)) + for extra in ([], ["--user"]): + try: + subprocess.check_call([sys.executable, "-m", "pip", "install", + "--quiet", *extra, *pkgs]) + return + except subprocess.CalledProcessError as e: + log.warning("pip install %sfailed: %s", "(--user) " if extra else "", e) + log.error("could not install Google Drive libs; exiting so the supervisor retries") + sys.exit(1) + + +# --------------------------------------------------------------------------- # +# Drive access (auth + list + export) — pattern from ir-evaluation +# --------------------------------------------------------------------------- # +def _build_service(): + from google.oauth2 import service_account + from googleapiclient.discovery import build + creds_path = _resolve(os.getenv("GDRIVE_CREDENTIALS_FILE", + str(_HERE / "service-account.json"))) + if not Path(creds_path).is_file(): + log.error("service-account JSON not found at %s " + "(set GDRIVE_CREDENTIALS_FILE). Exiting.", creds_path) + sys.exit(1) + creds = service_account.Credentials.from_service_account_file( + creds_path, scopes=_SCOPES) + return build("drive", "v3", credentials=creds, cache_discovery=False) + + +def _list_children(service, folder_id: str) -> list[dict]: + files, page = [], None + while True: + resp = service.files().list( + q=f"'{folder_id}' in parents and trashed=false", + fields=_FIELDS, + pageSize=1000, + pageToken=page, + includeItemsFromAllDrives=True, + supportsAllDrives=True, + ).execute() + files.extend(resp.get("files", [])) + page = resp.get("nextPageToken") + if not page: + break + return files + + +def _walk(service, folder_id: str, recursive: bool, depth: int = 0, + path: str = "") -> list[dict]: + """Return non-folder files under folder_id, each tagged with its `_drivePath`.""" + out = [] + for f in _list_children(service, folder_id): + if f.get("mimeType") == _FOLDER_MIME: + if recursive and depth < _MAX_DEPTH: + sub = f"{path}/{f['name']}" if path else f["name"] + out.extend(_walk(service, f["id"], recursive, depth + 1, sub)) + else: + f["_drivePath"] = path + out.append(f) + return out + + +def _content_for(service, f: dict) -> tuple[bytes | None, str | None]: + """Export/download a file's content as text bytes; (None, None) to skip.""" + from googleapiclient.http import MediaIoBaseDownload + mime = f.get("mimeType", "") + fid = f["id"] + exp_doc = os.getenv("GDRIVE_EXPORT_DOC", "text/markdown") + exp_sheet = os.getenv("GDRIVE_EXPORT_SHEET", "text/csv") + exp_slides = os.getenv("GDRIVE_EXPORT_SLIDES", "text/plain") + + def _export(targets: list[str]) -> tuple[bytes | None, str | None]: + for target in targets: + try: + data = service.files().export(fileId=fid, mimeType=target).execute() + return data, _MIME_EXT.get(target, "txt") + except Exception as e: # noqa: BLE001 — try the fallback target + log.warning("export %s as %s failed: %s", f.get("name"), target, e) + return None, None + + if mime == "application/vnd.google-apps.document": + return _export([exp_doc, "text/plain"]) + if mime == "application/vnd.google-apps.spreadsheet": + return _export([exp_sheet]) + if mime == "application/vnd.google-apps.presentation": + return _export([exp_slides]) + if mime.startswith("application/vnd.google-apps"): + log.info("skip native type %s (%s) - no text export", mime, f.get("name")) + return None, None + + # Uploaded file: only ingest text-ish ones (the watcher is text-only). + ext = f["name"].rsplit(".", 1)[-1].lower() if "." in f["name"] else "" + if ext in _TEXT_EXTS: + try: + buf, dl, done = io.BytesIO(), None, False + dl = MediaIoBaseDownload(buf, service.files().get_media(fileId=fid)) + while not done: + _, done = dl.next_chunk() + return buf.getvalue(), ext + except Exception as e: # noqa: BLE001 + log.warning("download %s failed: %s", f.get("name"), e) + return None, None + log.info("skip binary/unsupported %s (%s)", f.get("name"), mime or "?") + return None, None + + +# --------------------------------------------------------------------------- # +# section diff (stdlib only) +# --------------------------------------------------------------------------- # +_HEADING = re.compile(r"^(#{1,6})\s+(.*\S)\s*$") + + +def _split_sections(md: str) -> list[dict]: + """Split markdown into ordered sections, each with its heading breadcrumb + `path` (chain of enclosing headings) and `text` (heading line + body).""" + sections, stack = [], [] + cur = {"path": "(preamble)", "lines": []} + for line in md.splitlines(): + m = _HEADING.match(line) + if m: + sections.append(cur) + level, title = len(m.group(1)), m.group(2).strip() + while stack and stack[-1][0] >= level: + stack.pop() + stack.append((level, title)) + cur = {"path": " > ".join(t for _, t in stack), "lines": [line]} + else: + cur["lines"].append(line) + sections.append(cur) + out = [] + for s in sections: + text = "\n".join(l.rstrip() for l in s["lines"]).strip("\n") + if text: + out.append({"path": s["path"], "text": text}) + return out + + +def _keyed(sections: list[dict]) -> tuple[dict, list[tuple]]: + """Return ({key: text}, [(key, section)]) with duplicate paths disambiguated.""" + seen, mp, keyed = {}, {}, [] + for s in sections: + seen[s["path"]] = seen.get(s["path"], 0) + 1 + key = s["path"] if seen[s["path"]] == 1 else f"{s['path']} #{seen[s['path']]}" + mp[key] = s["text"] + keyed.append((key, s)) + return mp, keyed + + +def _diff_header(meta: dict, removed: list[str]) -> str: + lines = [ + f"", + "This file is an INCREMENTAL DIFF of a Google Doc: it contains only " + "changed/new parts, NOT the full document. Treat each part below as an " + "update to the named document; the full document is already in memory " + "under its title, and the Source URL is the original.", + f"**Document:** {meta['title']}", + f"**Doc id:** {meta['id']}", + ] + if meta.get("url"): + lines.append(f"**Source:** {meta['url']}") + if meta.get("modified"): + lines.append(f"**Modified:** {meta['modified']}") + if removed: + lines.append("_Parts removed since last sync (noted only; facts not " + f"auto-retracted): {'; '.join(removed)}_") + return "\n".join(lines) + + +def _breadcrumb(meta: dict, path: str, n: int, total: int) -> str: + return (f'> [DIFF | part: "{path}" | position {n}/{total} | belongs to: ' + f'"{meta["title"]}" ({meta["id"]}) | full doc in memory under ' + f'"{meta["title"]}" | source: {meta.get("url", "")}]') + + +def _build_diff(old_md: str, new_md: str, meta: dict) -> str | None: + """Self-describing delta of changed/new sections, or None if nothing changed.""" + new_secs = _split_sections(new_md) + if len(new_secs) < 2: # no usable headings -> line diff + return _build_line_diff(old_md, new_md, meta) + old_map, _ = _keyed(_split_sections(old_md)) + new_map, new_keyed = _keyed(new_secs) + total = len(new_keyed) + changed = [(i, s) for i, (key, s) in enumerate(new_keyed, 1) + if old_map.get(key) != s["text"]] + removed = [k for k in old_map if k not in new_map] + if not changed: + return None + parts = [_diff_header(meta, removed)] + for i, s in changed: + parts.append(_breadcrumb(meta, s["path"], i, total)) + parts.append(s["text"]) + return "\n\n".join(parts) + "\n" + + +def _build_line_diff(old_md: str, new_md: str, meta: dict) -> str | None: + """Fallback for heading-less docs (e.g. exported sheets): emit added/changed + lines as plain text under a breadcrumb.""" + diff = list(difflib.unified_diff(old_md.splitlines(), new_md.splitlines(), + lineterm="", n=0)) + added = [l[1:] for l in diff if l.startswith("+") and not l.startswith("+++")] + added_text = "\n".join(l for l in added if l.strip()) + if not added_text: + return None + crumb = (f'> [DIFF | line-level changes | belongs to: "{meta["title"]}" ' + f'({meta["id"]}) | full doc in memory under "{meta["title"]}" | ' + f'source: {meta.get("url", "")}]') + return f"{_diff_header(meta, [])}\n\n{crumb}\n\n{added_text}\n" + + +def _full_header(meta: dict) -> str: + lines = [ + f"", + f"**Document:** {meta['title']}", + f"**Doc id:** {meta['id']}", + ] + if meta.get("drive_path"): + lines.append(f"**Drive path:** {meta['drive_path']}") + if meta.get("url"): + lines.append(f"**Source:** {meta['url']}") + if meta.get("modified"): + lines.append(f"**Modified:** {meta['modified']}") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- # +# state +# --------------------------------------------------------------------------- # +def _load_manifest() -> dict: + try: + return json.loads(_MANIFEST.read_text(encoding="utf-8")) + except Exception: + return {} + + +def _save_manifest(m: dict) -> None: + _STATE.mkdir(parents=True, exist_ok=True) + _MANIFEST.write_text(json.dumps(m, indent=2, ensure_ascii=False), encoding="utf-8") + + +def _load_snapshot(fid: str) -> str: + try: + return (_SNAP_DIR / f"{_slug(fid)}.md").read_text(encoding="utf-8") + except Exception: + return "" + + +def _save_snapshot(fid: str, text: str) -> None: + _SNAP_DIR.mkdir(parents=True, exist_ok=True) + (_SNAP_DIR / f"{_slug(fid)}.md").write_text(text, encoding="utf-8") + + +def _change_key(f: dict) -> str: + return f.get("md5Checksum") or f.get("modifiedTime") or "" + + +def _write_source(name: str, content: str) -> bool: + dest = _SOURCES / name + if dest.exists() or (_INGESTED / name).exists(): + return False + dest.write_text(content, encoding="utf-8") + return True + + +def _prune(days: int) -> None: + if days <= 0 or not _INGESTED.is_dir(): + return + cutoff = time.time() - days * 86400 + for f in _INGESTED.glob("gdrive-*.*"): + try: + if f.stat().st_mtime < cutoff: + f.unlink() + except OSError: + pass + + +# --------------------------------------------------------------------------- # +# per-file processing +# --------------------------------------------------------------------------- # +def _process(service, f: dict, manifest: dict, mode: str, counts: dict, + seed_only: bool) -> None: + fid, cur = f["id"], _change_key(f) + prev = manifest.get(fid) + if prev is not None and prev == cur: + counts["skipped"] += 1 + return # unchanged: no download, no write + + data, ext = _content_for(service, f) + if data is None: # unsupported/native/error: record + move on + manifest[fid] = cur + return + text = data.decode("utf-8", errors="replace") if isinstance(data, bytes) else str(data) + + if seed_only: # first run, --skip-existing: snapshot only + _save_snapshot(fid, text) + manifest[fid] = cur + counts["seeded"] += 1 + return + + meta = {"id": fid, "title": f["name"], "url": f.get("webViewLink", ""), + "modified": f.get("modifiedTime", ""), "drive_path": f.get("_drivePath", "")} + is_new = prev is None + + if is_new or mode == "full": + out_text, out_ext, kind = _full_header(meta) + "\n\n" + text, ext, \ + ("new" if is_new else "changed") + else: + out_text = _build_diff(_load_snapshot(fid), text, meta) + out_ext, kind = "md", "changed" + if out_text is None: # no real content change (whitespace/meta) + _save_snapshot(fid, text) + manifest[fid] = cur + counts["skipped"] += 1 + return + + h8 = hashlib.sha256(text.encode("utf-8")).hexdigest()[:8] + name = f"gdrive-{_slug(f['name'])}-{h8}.{out_ext}" + if _write_source(name, out_text): + counts[kind] += 1 + _save_snapshot(fid, text) + manifest[fid] = cur + + +def main() -> None: + _load_env(_HERE / ".env") + _ensure_deps() + + folder_id = os.getenv("GDRIVE_FOLDER_ID", "").strip() + if not folder_id: + log.error("GDRIVE_FOLDER_ID not set (custom-profiles/gdrive/.env). Exiting.") + sys.exit(1) + mode = os.getenv("GDRIVE_INGEST_MODE", "diff").strip().lower() + recursive = os.getenv("GDRIVE_RECURSIVE", "true").strip().lower() == "true" + poll = int(os.getenv("GDRIVE_POLL_SECONDS", "300")) + prune_days = int(os.getenv("GDRIVE_PRUNE_DAYS", "0")) + skip_existing = os.getenv("GDRIVE_SKIP_EXISTING_ON_FIRST_RUN", + "false").strip().lower() == "true" + + _SOURCES.mkdir(parents=True, exist_ok=True) + service = _build_service() + log.info("gdrive-ingestor ready (folder=%s, mode=%s, recursive=%s, poll=%ss)", + folder_id, mode, recursive, poll) + + manifest = _load_manifest() + first_run = not manifest + while True: + try: + seed_only = first_run and skip_existing + counts = {"new": 0, "changed": 0, "skipped": 0, "seeded": 0} + try: + files = _walk(service, folder_id, recursive) + except Exception as e: # noqa: BLE001 — transient Drive error, retry next poll + log.warning("drive list error: %s", e) + files = [] + for f in files: + try: + _process(service, f, manifest, mode, counts, seed_only) + except Exception as e: # noqa: BLE001 — one bad doc must not kill the loop + log.warning("process %s error: %s", f.get("name"), e) + _save_manifest(manifest) + if seed_only: + log.info("first run: seeded %d doc(s) (changes-only from now)", + counts["seeded"]) + log.info("found %d / new %d / changed %d / skipped %d", + len(files), counts["new"], counts["changed"], counts["skipped"]) + _prune(prune_days) + first_run = False + except Exception as e: # noqa: BLE001 + log.exception("loop error: %s", e) + time.sleep(poll) + + +if __name__ == "__main__": + main() diff --git a/custom-profiles/hackernews/.env.example b/custom-profiles/hackernews/.env.example new file mode 100644 index 0000000..4fb16da --- /dev/null +++ b/custom-profiles/hackernews/.env.example @@ -0,0 +1,6 @@ +# hackernews ingestor — NO API KEY NEEDED. The defaults work as-is; copy this to +# `.env` only if you want to override them. ingestor.py loads THIS folder's `.env`. + +# HN_LIMIT=30 # top stories scanned per poll +# HN_POLL_SECONDS=600 # loop cadence +# HN_PRUNE_DAYS=14 # delete ingested hn-*.md older than this (0 = keep forever) diff --git a/custom-profiles/hackernews/README.md b/custom-profiles/hackernews/README.md new file mode 100644 index 0000000..bafbef7 --- /dev/null +++ b/custom-profiles/hackernews/README.md @@ -0,0 +1,32 @@ +# hackernews profile (keyless example) + +A complete, **keyless** custom profile — the worked example for [`../SKILL.md`](../SKILL.md). +It ingests Hacker News top stories and builds wiki pages about the companies / projects / +technologies they mention. + +## What it does + +- `ingestor.py` polls the public Hacker News API (no key) and drops one `hn-.md` per new + top story into `data/sources/`; the existing watcher ingests + extracts it. +- `wiki_maintainer.add.md` names pages by the entity's common name and dedups variants. +- `wiki_writer.add.md` shapes each page as a tech-entity profile (Type / Homepage / Category) + plus a dated `current-developments` chronicle. + +## Run it (no key needed) + +1. In the repo-root `.env`: `CUSTOM_PROFILE=hackernews` +2. `docker compose up -d` +3. Watch it flow: + ```bash + ls data/sources/ # hn-*.md files dropped by the ingestor + curl -s "http://localhost:8000/api/v1/entities?entity_type=wiki&limit=20" + ``` + +Optional tuning lives in `.env` (see `.env.example`): `HN_LIMIT`, `HN_POLL_SECONDS`, +`HN_PRUNE_DAYS`. No `.env` is required — the defaults work with no key. + +## Honest note + +HN gives a story **title + metadata** (no article body), so individual pages are sparser +than a per-entity feed; entity pages richen as the same company/project recurs across +stories. diff --git a/custom-profiles/hackernews/ingestor.py b/custom-profiles/hackernews/ingestor.py new file mode 100644 index 0000000..71397cb --- /dev/null +++ b/custom-profiles/hackernews/ingestor.py @@ -0,0 +1,185 @@ +""" +Hacker News ingestor (hackernews profile) — a KEYLESS worked example for the +custom-profiles SKILL. Launched by `braindb.profile_runner` when CUSTOM_PROFILE +includes `hackernews`. No API key, no auth. + +It polls the public Hacker News Firebase API and writes one Markdown file per new +top story into `data/sources/`, where the existing watcher ingests + extracts it. +The story title + source domain are made salient so the extraction step tags +facts with the company/project/technology named, which clusters them into entity +wikis. + +Config (all optional, NO key) — from this folder's own `.env`: + HN_LIMIT how many top stories to scan per poll (default 30) + HN_POLL_SECONDS loop cadence (default 600) + HN_PRUNE_DAYS delete ingested hn-*.md older than this (default 14; 0=keep) +""" +import logging +import os +import re +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlparse + +import requests + +_HERE = Path(__file__).resolve().parent +_REPO_ROOT = _HERE.parents[1] # custom-profiles/hackernews -> repo root +_SOURCES = _REPO_ROOT / "data" / "sources" +_INGESTED = _SOURCES / "ingested" +_STATE = _HERE / ".state" / "seen.txt" + +_HN = "https://hacker-news.firebaseio.com/v0" + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [hn-ingestor] %(message)s", + datefmt="%H:%M:%S", + stream=sys.stdout, +) +log = logging.getLogger("hn-ingestor") + + +def _load_env(path: Path) -> None: + """Minimal .env loader (no extra dependency). Container/host env wins.""" + if not path.is_file(): + return + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def _slug(s: str) -> str: + return re.sub(r"[^A-Za-z0-9._-]", "-", s)[:80] + + +def _domain(url: str) -> str: + try: + host = (urlparse(url).hostname or "").lower() + return host[4:] if host.startswith("www.") else host + except Exception: + return "" + + +def _fetch_json(path: str): + r = requests.get(f"{_HN}/{path}", timeout=30) + r.raise_for_status() + return r.json() + + +def _story(item_id: int) -> dict | None: + """Fetch one HN item; return normalised story fields, or None to skip + (non-story / dead / no title). Lets requests errors propagate so the caller + can retry next poll instead of marking the id seen.""" + it = _fetch_json(f"item/{item_id}.json") + if not isinstance(it, dict) or it.get("type") != "story": + return None + if it.get("dead") or it.get("deleted") or not it.get("title"): + return None + ts = it.get("time") + posted = (datetime.fromtimestamp(ts, tz=timezone.utc).isoformat() + if isinstance(ts, (int, float)) else "") + url = it.get("url") or "" + return { + "id": str(it["id"]), + "title": str(it["title"]).strip(), + "url": url, + "source": _domain(url) or "Hacker News (self post)", + "by": it.get("by", ""), + "posted": posted, + "score": it.get("score", 0), + "text": (it.get("text") or "").strip(), + } + + +def _write_story(s: dict) -> bool: + """Write one hn-*.md into data/sources/. Returns True if a file was written.""" + name = f"hn-{_slug(s['id'])}.md" + dest = _SOURCES / name + if dest.exists() or (_INGESTED / name).exists(): + return False + header = [ + f"**Source:** {s['source']}", + f"**By:** {s['by']}", + f"**Posted:** {s['posted']}", + f"**HN score:** {s['score']}", + ] + if s["url"]: + header.append(f"**URL:** {s['url']}") + body = f"# {s['title']}\n\n" + "\n".join(header) + f"\n\n{s['title']}\n" + if s["text"]: + body += f"\n{s['text']}\n" + dest.write_text(body, encoding="utf-8") + return True + + +def _load_seen() -> set[str]: + try: + return set(_STATE.read_text(encoding="utf-8").split()) + except Exception: + return set() + + +def _save_seen(seen: set[str], cap: int = 20000) -> None: + _STATE.parent.mkdir(parents=True, exist_ok=True) + _STATE.write_text("\n".join(list(seen)[-cap:]), encoding="utf-8") + + +def _prune(days: int) -> None: + if days <= 0 or not _INGESTED.is_dir(): + return + cutoff = time.time() - days * 86400 + for f in _INGESTED.glob("hn-*.md"): + try: + if f.stat().st_mtime < cutoff: + f.unlink() + except OSError: + pass + + +def main() -> None: + _load_env(_HERE / ".env") + limit = int(os.getenv("HN_LIMIT", "30")) + poll = int(os.getenv("HN_POLL_SECONDS", "600")) + prune_days = int(os.getenv("HN_PRUNE_DAYS", "14")) + + _SOURCES.mkdir(parents=True, exist_ok=True) + log.info("hn-ingestor ready (top %d, poll=%ss, sources=%s)", limit, poll, _SOURCES) + + seen = _load_seen() + while True: + try: + written = 0 + try: + ids = _fetch_json("topstories.json")[:limit] + except requests.RequestException as e: + log.warning("topstories fetch error: %s", e) + ids = [] + for sid in ids: + key = str(sid) + if key in seen: + continue + try: + s = _story(sid) + except requests.RequestException as e: + log.warning("item %s fetch error: %s", sid, e) + continue # transient — retry next poll, don't mark seen + seen.add(key) + if s and _write_story(s): + written += 1 + if written: + log.info("wrote %d new story file(s) into data/sources/", written) + _save_seen(seen) + _prune(prune_days) + except Exception as e: + log.exception("loop error: %s", e) + time.sleep(poll) + + +if __name__ == "__main__": + main() diff --git a/custom-profiles/hackernews/wiki_maintainer.add.md b/custom-profiles/hackernews/wiki_maintainer.add.md new file mode 100644 index 0000000..cf3eedf --- /dev/null +++ b/custom-profiles/hackernews/wiki_maintainer.add.md @@ -0,0 +1,15 @@ + + +## Tech-news subjects + +Seeds here come from Hacker News top stories — they concern **companies, projects, +technologies, and people** in tech. Apply these rules on top of the ones above: + +- **Name by the entity's common name.** When you **create**, set `proposed_name` to the + well-known name of the company / project / technology / person the story is about + (e.g. "OpenAI", "Rust", "PostgreSQL"), not a headline and not a URL. +- **One entity, one page.** Treat name variants and the entity's own domain as the same + subject — prefer **attach** / **consolidate** over minting duplicates. The source domain + (e.g. `openai.com`) is a strong hint to the owning company/project. +- **Skip non-subjects.** If a seed is a generic tech term, a verb, or a one-off with no + durable subject ("show hn", "ask hn", "a programmer"), **skip** it. diff --git a/custom-profiles/hackernews/wiki_writer.add.md b/custom-profiles/hackernews/wiki_writer.add.md new file mode 100644 index 0000000..f5e562b --- /dev/null +++ b/custom-profiles/hackernews/wiki_writer.add.md @@ -0,0 +1,34 @@ + + +## Tech-entity format (companies / projects / technologies / people) + +These pages are built from Hacker News stories. Keep EVERY base rule above — +`[[ref:UUID]]` citations, the `` markers, and the +`` header — and shape the page like this so every entity reads +consistently: + +- **Title** = the entity's common name, e.g. `# OpenAI`. +- **Meta keywords** = the entity name first, then key themes. + +Use these sections in addition to the base `contradictions` / `references`: + +``` + +**Type:** company / project / technology / person +**Homepage/Repo:** [[ref:UUID]] +**Category:** [[ref:UUID]] + durable facts about the entity + dated HN items, NEWEST FIRST, each with [[ref:UUID]] +``` + +**Profile rules** — one labeled field per line. Fill **Homepage/Repo** and **Category** +ONLY if a source supports it (and append its `[[ref:UUID]]`); otherwise write `(unknown)` +— **never invent**. **Type** is your best source-supported classification. + +### Chronicle aging + +`current-developments` is a rolling window of what's happening NOW. As items get old (relative to +the current date given above) or superseded, demote their lasting substance into `background` +(carrying the `[[ref:UUID]]` along) and drop the stale line from `current-developments`. Never +silently drop a cited fact — move its citation to `background`. Dated items use one consistent +format: `- 2026-06-20 — [[ref:UUID]]` (ISO date, em-dash, claim, citation). diff --git a/custom-profiles/hermes/.env.example b/custom-profiles/hermes/.env.example new file mode 100644 index 0000000..2999998 --- /dev/null +++ b/custom-profiles/hermes/.env.example @@ -0,0 +1,24 @@ +# hermes ingestor — copy this to `.env` and fill it in. ingestor.py loads THIS folder's `.env`. +# The Hormuz QUESTION is NOT here — it lives in `query.md` (the whole profile is Hormuz). This file +# is connection/operational only. Your real `.env` is gitignored. +# +# DORMANT BY DEFAULT: with no HERMES_API_KEY (and no HERMES_SAMPLE_FILE) the ingestor idles and +# NEVER calls Hermes. Hermes runs headless via `hermes gateway` (an OpenAI-compatible server) with a +# web-search backend configured — see README.md. This profile only CALLS Hermes; it does not run it. + +# ── connection (required to run live) ─────────────────────────────────────── +# HERMES_ASK_URL=http://127.0.0.1:8642 # Hermes gateway base URL +# HERMES_API_KEY= # bearer token (Hermes API_SERVER_KEY). UNSET => dormant. +# HERMES_MODEL=hermes-agent # model name sent in the request + +# ── behaviour (optional) ──────────────────────────────────────────────────── +# HERMES_POLL_SECONDS=1800 # cadence: 1800 = every 30 min; 3600 = hourly +# HERMES_ASK_TIMEOUT=600 # per-ask timeout seconds (the agent loop can be slow) +# HERMES_DEDUP=true # skip re-ingesting an unchanged answer; false = record every tick +# HERMES_PRUNE_DAYS=0 # delete ingested hermes-*.md older than this (0 = keep) + +# ── dry-run: test the pipeline WITHOUT Hermes ─────────────────────────────── +# Point this at a file with a canned Hormuz answer; the ingestor uses it as the "answer" instead of +# calling Hermes, so you can validate ingest + wiki shaping before Hermes is available. A ready-made +# fixture ships as sample_answer.md: +# HERMES_SAMPLE_FILE=custom-profiles/hermes/sample_answer.md diff --git a/custom-profiles/hermes/README.md b/custom-profiles/hermes/README.md new file mode 100644 index 0000000..4c9d199 --- /dev/null +++ b/custom-profiles/hermes/README.md @@ -0,0 +1,53 @@ +# hermes profile (situational monitor — "Hormuz status") + +On a schedule, asks an external **Hermes Agent** (which web-searches) for the latest **Strait of +Hormuz** status and feeds each dated answer into BrainDB as a datasource. The accumulating facts form +a central **"Hormuz status"** wiki structured for a future dashboard (short / mid / long-term + +forecast metrics), while BrainDB's default rules still build wikis for the other people / orgs / +countries mentioned — a situation-aware memory. See [`../SKILL.md`](../SKILL.md) for the mechanism. + +## What it does +- `ingestor.py` asks Hermes the question in `query.md` every `HERMES_POLL_SECONDS` (default 30 min), + as a **fresh, stateless discussion** each time, and drops the dated answer (with its web sources) + as one `hermes-hormuz-.md` into `data/sources/`; the existing watcher ingests + extracts it. +- `query.md` (committed) tells Hermes to web-search, **bring back context**, **search multiple times + until confident**, date everything precisely, give short/mid/long-term + the most credible source, + flag conflicts or admit uncertainty, and **include source URLs**. +- `wiki_maintainer.add.md` routes every report to the single **"Hormuz status"** page and keeps the + other entities on the default rules. +- `wiki_writer.add.md` shapes that page: a dashboard-parseable `metrics` block + `short-term` / + `mid-term` / `long-term` sections, leaning on the base `timeline` + `contradictions`. Timing is + treated as critical; conflicts are resolved by credibility or explicitly marked uncertain. + +## Hermes (external dependency) +Hermes runs headless via `hermes gateway` — an **OpenAI-compatible** server (default +`http://127.0.0.1:8642`, `POST /v1/chat/completions`, bearer auth). It needs a **web-search backend** +configured (Tavily / Firecrawl / Brave / SearXNG / etc.) so it can research the live status. See the +Hermes docs. **This profile does not run Hermes; it only calls it.** + +## Run it (live) +1. `cp .env.example .env`; set `HERMES_ASK_URL` + `HERMES_API_KEY` (and `HERMES_MODEL` if needed). +2. Repo-root `.env`: `CUSTOM_PROFILE=hermes` (or append, e.g. `gdrive,hermes`). +3. `docker compose up -d`; watch: + ```bash + docker compose logs -f profile_runner # "asking hermes (fresh discussion)" / "wrote hermes report" + curl -s -X POST localhost:8000/api/v1/memory/context \ + -H "Content-Type: application/json" -d '{"queries":["Hormuz status"]}' + ``` + +## Test WITHOUT Hermes (dry-run) +Set `HERMES_SAMPLE_FILE=custom-profiles/hermes/sample_answer.md` in `.env` (the shipped fixture); the +ingestor uses it as the "answer" instead of calling Hermes, so you can validate ingest + the wiki +shaping (metrics, short/mid/long, timeline, contradictions) before Hermes is live. + +## Dormant by default +With **no `HERMES_API_KEY`** (and no sample) the ingestor logs `idle (not configured)` and **never +calls Hermes** — safe to ship inactive. It only acts once you configure it and add `hermes` to +`CUSTOM_PROFILE`. + +## Notes / limits +- Answer parsing assumes OpenAI-compatible `choices[0].message.content` (adjust if your Hermes differs). +- Timing is carried as explicit dates in content + the `timeline` section — **no schema change**. +- `HERMES_DEDUP=true` skips re-ingesting an unchanged answer (avoids ~48 near-dup facts/day); set + `false` to record every tick as a time-series point. +- Very long answers may truncate (a known Hermes issue) — `query.md` asks for a focused report. diff --git a/custom-profiles/hermes/ingestor.py b/custom-profiles/hermes/ingestor.py new file mode 100644 index 0000000..60d4900 --- /dev/null +++ b/custom-profiles/hermes/ingestor.py @@ -0,0 +1,242 @@ +""" +Hermes situational ingestor (hermes profile) — Strait of Hormuz monitor. + +Launched by `braindb.profile_runner` when CUSTOM_PROFILE includes `hermes`. On a +schedule it asks an external Hermes Agent (its headless OpenAI-compatible gateway) +the question in this folder's `query.md`, and drops the answer into `data/sources/` +as one datasource, where the existing watcher ingests + extracts it. The +accumulating facts form the central "Hormuz status" wiki (shaped by +`wiki_writer.add.md`), while default rules build wikis for the other entities +mentioned — a situation-aware memory. + +DORMANT BY DEFAULT: with no HERMES_API_KEY (and no HERMES_SAMPLE_FILE) it logs +"idle" and NEVER calls Hermes. Each ask is a FRESH, STATELESS discussion (no +session header), so no context bleeds between polls. + +Config — from this folder's own `.env` (copy `.env.example` -> `.env`): + HERMES_ASK_URL Hermes gateway base URL (default http://127.0.0.1:8642) + HERMES_API_KEY bearer token (REQUIRED to run live; unset => dormant) + HERMES_MODEL model name sent in the request (default hermes-agent) + HERMES_ASK_TIMEOUT per-ask timeout seconds (default 600; agent loop is slow) + HERMES_POLL_SECONDS loop cadence (default 1800 = 30 min; 3600 = hourly) + HERMES_DEDUP skip re-ingesting an unchanged answer (default true) + HERMES_SAMPLE_FILE DRY-RUN: use this file's text as the "answer" instead of + calling Hermes (validate the pipeline before Hermes is live) + HERMES_PRUNE_DAYS delete ingested hermes-*.md older than this (default 0=keep) + +The Hormuz prompt itself lives in `query.md` (committed, profile-owned), not here. +""" +import hashlib +import logging +import os +import re +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +import requests + +_HERE = Path(__file__).resolve().parent +_REPO_ROOT = _HERE.parents[1] # custom-profiles/hermes -> repo root +_SOURCES = _REPO_ROOT / "data" / "sources" +_INGESTED = _SOURCES / "ingested" +_STATE = _HERE / ".state" +_LAST = _STATE / "last.txt" # substance-hash of the last dropped answer +_QUERY_FILE = _HERE / "query.md" + +_SUBJECT = "Hormuz status" # baked in — the whole profile is Hormuz + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [hermes-ingestor] %(message)s", + datefmt="%H:%M:%S", + stream=sys.stdout, +) +log = logging.getLogger("hermes-ingestor") + + +def _load_env(path: Path) -> None: + """Minimal .env loader (no extra dependency). Container/host env wins.""" + if not path.is_file(): + return + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def _slug(s: str) -> str: + return re.sub(r"[^A-Za-z0-9._-]", "-", s)[:80] + + +def _resolve(p: str) -> Path: + """Resolve a configured path against repo root / profile dir if relative.""" + pp = Path(p) + if pp.is_absolute(): + return pp + for base in (_REPO_ROOT, _HERE): + cand = base / pp + if cand.is_file(): + return cand + return _REPO_ROOT / pp + + +def _load_question() -> str: + try: + return _QUERY_FILE.read_text(encoding="utf-8").strip() + except Exception: + return "" + + +def _ask_hermes(question: str, url: str, key: str, model: str, timeout: int) -> str: + """One FRESH, STATELESS ask to the Hermes OpenAI-compatible gateway. + + No `X-Hermes-Session-Id` header is sent, so each call is an independent + discussion with no carried-over context.""" + endpoint = url.rstrip("/") + "/v1/chat/completions" + headers = {"Authorization": f"Bearer {key}", "Content-Type": "application/json"} + body = { + "model": model, + "messages": [{"role": "user", "content": question}], + "stream": False, + } + r = requests.post(endpoint, json=body, headers=headers, timeout=timeout) + r.raise_for_status() + data = r.json() + # OpenAI-compatible; tolerate minor shape differences. + try: + return (data["choices"][0]["message"]["content"] or "").strip() + except (KeyError, IndexError, TypeError): + if isinstance(data, dict): + return (data.get("answer") or data.get("content") or "").strip() + return "" + + +def _substance(answer: str) -> str: + """Hash the answer's substance (whitespace-collapsed) for change-detection.""" + return hashlib.sha256(re.sub(r"\s+", " ", answer).strip().encode("utf-8")).hexdigest() + + +def _load_last() -> str: + try: + return _LAST.read_text(encoding="utf-8").strip() + except Exception: + return "" + + +def _save_last(h: str) -> None: + _STATE.mkdir(parents=True, exist_ok=True) + _LAST.write_text(h, encoding="utf-8") + + +def _write_report(answer: str, as_of: str) -> bool: + """Drop one dated situational report into data/sources/. The subject is made + salient so the extractor clusters facts into the central "Hormuz status" wiki; + the framing tells the extractor that dates are authoritative and conflicts / + uncertainty must be preserved (not flattened).""" + name = f"hermes-hormuz-{_slug(as_of)}.md" + dest = _SOURCES / name + if dest.exists() or (_INGESTED / name).exists(): + return False + header = [ + f'', + f"# {_SUBJECT} - situational report", + "", + f"**Subject:** {_SUBJECT}", + f"**Fetched (poll time):** {as_of}", + "**Source:** external Hermes agent (web-search). This is an incoming situational report; its " + "web sources are inline below. The DATES INSIDE the report are authoritative for when each " + "thing was observed - 'Fetched' above is only when we polled. Record dates precisely; where " + "reports conflict, prefer the more credible and note it; where certainty is not available, " + "say so rather than guessing.", + "", + answer.strip(), + "", + ] + dest.write_text("\n".join(header), encoding="utf-8") + return True + + +def _prune(days: int) -> None: + if days <= 0 or not _INGESTED.is_dir(): + return + cutoff = time.time() - days * 86400 + for f in _INGESTED.glob("hermes-*.md"): + try: + if f.stat().st_mtime < cutoff: + f.unlink() + except OSError: + pass + + +def main() -> None: + _load_env(_HERE / ".env") + url = os.getenv("HERMES_ASK_URL", "http://127.0.0.1:8642").strip() + key = os.getenv("HERMES_API_KEY", "").strip() + model = os.getenv("HERMES_MODEL", "hermes-agent").strip() + timeout = int(os.getenv("HERMES_ASK_TIMEOUT", "600")) + poll = int(os.getenv("HERMES_POLL_SECONDS", "1800")) + dedup = os.getenv("HERMES_DEDUP", "true").strip().lower() == "true" + sample = os.getenv("HERMES_SAMPLE_FILE", "").strip() + prune_days = int(os.getenv("HERMES_PRUNE_DAYS", "0")) + + question = _load_question() + if not question: + log.error("query.md is empty/missing - no question to ask. Exiting.") + sys.exit(1) + + _SOURCES.mkdir(parents=True, exist_ok=True) + + # DORMANT guard: never call Hermes unless a key is configured OR a dry-run + # sample is provided. Until then, idle quietly so the supervisor doesn't churn. + if not key and not sample: + log.info("idle (not configured): set HERMES_API_KEY to enable, or HERMES_SAMPLE_FILE to " + "dry-run. Sleeping; will NOT call Hermes.") + while True: + time.sleep(max(poll, 300)) + + mode = "DRY-RUN (sample)" if sample else "live" + log.info("hermes-ingestor ready (%s, subject=%s, poll=%ss, url=%s)", + mode, _SUBJECT, poll, "(sample)" if sample else url) + + last = _load_last() + while True: + try: + now = datetime.now(timezone.utc) + as_of = now.isoformat() + answer = "" + if sample: + try: + answer = _resolve(sample).read_text(encoding="utf-8").strip() + except Exception as e: + log.warning("sample read error: %s", e) + else: + try: + log.info("asking hermes (fresh discussion)...") + dated_question = f"Today is {now:%Y-%m-%d %H:%M} UTC.\n\n{question}" + answer = _ask_hermes(dated_question, url, key, model, timeout) + except requests.RequestException as e: + log.warning("hermes ask error: %s", e) + + if not answer: + log.info("empty answer; will retry next poll") + else: + h = _substance(answer) + if dedup and h == last: + log.info("answer unchanged since last poll; skipping " + "(set HERMES_DEDUP=false to record every tick)") + elif _write_report(answer, as_of): + last = h + _save_last(h) + log.info("wrote hermes report into data/sources/ (as of %s)", as_of) + _prune(prune_days) + except Exception as e: # noqa: BLE001 — one bad poll must not kill the loop + log.exception("loop error: %s", e) + time.sleep(poll) + + +if __name__ == "__main__": + main() diff --git a/custom-profiles/hermes/query.md b/custom-profiles/hermes/query.md new file mode 100644 index 0000000..a851374 --- /dev/null +++ b/custom-profiles/hermes/query.md @@ -0,0 +1,48 @@ +You are tracking an evolving real-world situation: the operational status of the **Strait of Hormuz** +(the shipping chokepoint between the Persian Gulf and the Gulf of Oman). Search the web NOW and report +the most current verifiable picture, how it has evolved, and where it is heading. +The current date is given at the very top of this message — treat it as "today" and judge how recent +every source and event is relative to it. + +How to research: +- Make **several web searches from different angles** (official maritime/naval notices, major news + wires, shipping & marine-insurance advisories, regional sources) and **keep going until you are + reasonably confident** — never rely on a single source or a single search. +- **Date everything, and check it against today.** Give every event, report, and prediction an explicit + absolute date (and time + timezone when known); for any forecast, give its time horizon. State how many + days/weeks ago each item is relative to today. +- **Do not pass off old information as current.** Check each source's publication date. If the most + recent credible reporting is already days or weeks old, say so plainly and put those findings under + *History of key events* — **Current status** must reflect the freshest credible reporting, not the + first or most detailed source you happen to find. +- **Include the source URL(s) inline** with each claim. +- Work with **whatever verifiable information exists** — be thorough when there is a lot, concise when + there is little, and say so when a period is unclear. Do **not** invent specifics to fill gaps. + +Report in these labelled parts (keep the labels): + +**Current status** — the situation right now in a few words (in its own terms, e.g. how open/usable the +strait is), as of exactly when (date), per which source(s) — and if the latest solid information is +already days/weeks old, say so here. + +**Confidence** — high / medium / low, with a one-line reason (source agreement, recency, gaps). + +**Context** — what is happening and why: the key actors (states, militaries, operators) and causes. + +**Now (newest):** the very latest perception — the most recent reporting and what it says. + +**Recent (last ~3-4 days):** what has changed or been reported over the past few days. + +**Trajectory (last ~1-2 weeks):** the broader arc over the past one to two weeks. + +**Forecast (ahead):** where it is likely heading — ONE short forward outlook, with its time horizon and +how confident it is. (Do not split it into multiple short/long-term forecasts.) + +**History of key events** — a dated, chronological list (newest first) of the notable events, each with +its date + source, so a running timeline can be built up. + +**Most credible source** — the single most reliable source or assessment, and **why** you trust it most. + +**Conflicting reports** — if sources disagree, say so explicitly and either resolve it by credibility +(explain why) or state plainly that **certainty is not available**. Never present a confident answer you +cannot support. diff --git a/custom-profiles/hermes/sample_answer.md b/custom-profiles/hermes/sample_answer.md new file mode 100644 index 0000000..598f74e --- /dev/null +++ b/custom-profiles/hermes/sample_answer.md @@ -0,0 +1,46 @@ + + +**Current status** — The Strait of Hormuz is **OPEN but restricted** to commercial shipping as of +2026-06-22 (yesterday): traffic is moving, but several operators have paused transits and naval escorts +have increased. Per the Joint Maritime Information Center advisory, 2026-06-22 (https://example.com/jmic). + +**Confidence** — medium: the two freshest sources agree on "open but restricted", but reporting on the +last 24h is thin. + +**Context** — Heightened naval activity and an incident near Bandar Abbas have raised war-risk premiums; +the key actors are regional navies, multinational maritime bodies, and major tanker operators. + +**Now (newest):** 2026-06-22 — JMIC reports the strait open-but-restricted; escorts increased +(https://example.com/jmic). No newer credible reporting found for 2026-06-23. + +**Recent (last ~3-4 days):** +- 2026-06-21 — Regional authorities stated the strait "remains open to lawful navigation" + (https://example.com/reg-0621). +- 2026-06-20 — Two major tanker operators announced temporary rerouting / pauses pending clarity + (https://example.com/ship-0620). +- 2026-06-19 — A naval incident near Bandar Abbas raised insurance war-risk premiums + (https://example.com/wire-0619). + +**Trajectory (last ~1-2 weeks):** Since ~2026-06-10 the picture has shifted from normal traffic to +restricted-but-open, driven by escalating incidents and rising insurance premiums +(https://example.com/analysis-0618). + +**Forecast (ahead):** Likely stays open-but-restricted in the coming weeks; brief closures possible if +incidents recur — horizon ~weeks, confidence medium (https://example.com/jmic). + +**History of key events** (newest first): +- 2026-06-22 — JMIC: open but restricted (https://example.com/jmic). +- 2026-06-21 — Regional authorities: "open to lawful navigation" (https://example.com/reg-0621). +- 2026-06-20 — Tanker operators pause / reroute (https://example.com/ship-0620). +- 2026-06-19 — Naval incident near Bandar Abbas; premiums rise (https://example.com/wire-0619). +- 2026-05-30 — BACKGROUND only (~3 weeks old): an older baseline noted suppressed crossings; recorded + here as context, NOT the current status (https://example.com/baseline-0530). + +**Most credible source:** The Joint Maritime Information Center advisory — an official multinational +maritime body issuing dated, navigation-specific guidance, weighted above news aggregation. + +**Conflicting reports:** One regional outlet (https://example.com/outlet-0621) reported a brief FULL +closure on 2026-06-21, which JMIC and two wires do NOT corroborate. Treated as uncertain / not confirmed +— certainty on that specific point is not available at this time. diff --git a/custom-profiles/hermes/wiki_maintainer.add.md b/custom-profiles/hermes/wiki_maintainer.add.md new file mode 100644 index 0000000..2be7b82 --- /dev/null +++ b/custom-profiles/hermes/wiki_maintainer.add.md @@ -0,0 +1,19 @@ + + +## Tracked situation: "Hormuz status" + +Some seeds here are dated situational reports from an external agent about the **Strait of Hormuz** +(open / restricted / closed to shipping). Apply these on top of the rules above: + +- **One central page.** Route every Strait-of-Hormuz status report to a single canonical wiki named + **"Hormuz status"** — **attach** / **consolidate** onto it; never mint a second page for the same + situation (variants like "Strait of Hormuz", "Hormuz strait status" are the SAME subject). +- **Do NOT skip the surrounding entities.** The reports also name countries, militaries, shipping + companies, officials, vessels and organisations. Treat those by the **default rules** — create or + attach their own wikis as usual so the situation is richly cross-linked. Only skip true + non-subjects (generic terms, one-off verbs). +- **No abstract-concept pages, no empties.** Generic themes the reports use to *describe* the situation + — e.g. "maritime security", "commercial shipping", "global trade", "geopolitical risk" — are NOT + subjects: **skip** them (or fold their substance into "Hormuz status"); never mint a page for a theme, + and never create an empty / near-empty wiki. (Concrete named entities — a specific country, navy, + company, official or vessel — still get their own page per the rule above.) diff --git a/custom-profiles/hermes/wiki_writer.add.md b/custom-profiles/hermes/wiki_writer.add.md new file mode 100644 index 0000000..3537e26 --- /dev/null +++ b/custom-profiles/hermes/wiki_writer.add.md @@ -0,0 +1,55 @@ + + +## Ongoing-situation format + +When the subject is an ongoing real-world SITUATION tracked over time from dated reports (a strait's +status, a conflict, an outbreak, a market regime — anything that evolves), keep EVERY base rule — +`[[ref:UUID]]` citations, the `` markers, the `` header — and +lay the page out in clean, consistent, separately-parseable sections. For a situational page, use +**exactly the sections below, in this order** — these **REPLACE** the base "Recommended structure" list +(no `overview` / `timeline` / `sources`); add the base `contradictions` only when sources disagree; do +not invent extra sections. + +All windows below are RETROSPECTIVE — they look BACK from **today** (the current date given above). Only +`forecast` looks forward, and it is ONE outlook (never split into short/long-term tiers). Bucket each +event by its OWN date relative to today. + +``` + +**Current status:** [[ref:UUID]] +**As of:** [[ref:UUID]] +**Last checked:** +**Confidence:** high | medium | low — +**Most credible source:** — [[ref:UUID]] +**Open conflict:** none | + + today / latest perception — current state + most recent reporting, dated + the last ~3-4 days before today — what changed, newest first, dated + the last ~1-2 weeks before today — broader trajectory, dated + the ONLY forward-looking section — ONE short outlook with its time + horizon + confidence (a single statement; do NOT split into multiple + short/long-term forecasts) + running dated log of key events, NEWEST FIRST; grows with every report +``` + +`status` is **one labelled field per line** — a dashboard parses it. Give a field a value **and** its +`[[ref:UUID]]` only when a source supports it; otherwise write `(unknown)` — **never invent**. + +**Dated lines** use ONE consistent format, newest first: `- 2026-06-20 — [[ref:UUID]]` +(ISO date, em-dash, claim, citation). + +**Rules (generic — adapt to any situation, do not over-fit):** +- Use the situation's **own vocabulary** for status; don't force fixed categories. +- **Date every claim** ("as of "); `Current status` = the most recent credible report. +- Show **confidence cleanly** — once in `status`, and on the single `forecast` line. +- **Conflicts:** adopt the more credible (say why, cite **both** in `contradictions`) **or** set + `Confidence: low` and state plainly that certainty is not available — never fabricate one answer. + Hedge unverified claims ("reportedly", "per [[ref:UUID]]"). +- **Degrade gracefully:** if a period has little info, one dated line like "(no significant change + reported)" is fine — don't pad with invented detail. + +### Re-bucket every rewrite (this is YOUR job, each time) +Place every event by its OWN date relative to **today**: today/just now → `now`; within ~3-4 days → +`recent-days`; within ~1-2 weeks → `recent-weeks`; older → `history`. `history` is the durable record — +append new events (newest first), NEVER delete a cited one (keep its `[[ref:UUID]]`). So as days pass an +event moves now → recent-days → recent-weeks → history. diff --git a/docker-compose.yml b/docker-compose.yml index 0067848..2393885 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,10 @@ services: # the scheduler): an entity is wiki-eligible only once created_at is # older than this many minutes, so still-ingesting subjects settle first. WIKI_FRESHNESS_MINUTES: ${WIKI_FRESHNESS_MINUTES:-30} + # Optional custom-profile prompt shaping (wiki maintainer/writer). Empty + # by default -> the composition helper is a strict no-op and prompts are + # the baked-in defaults. Set CUSTOM_PROFILE in .env to activate. + CUSTOM_PROFILE: ${CUSTOM_PROFILE:-} extra_hosts: # Lets self-hosted profiles (e.g. vllm_workstation) reach a server bound # to the Docker host's loopback. Docker Desktop sets this implicitly; @@ -119,6 +123,27 @@ services: - .:/app command: python -m braindb.wiki_scheduler + # Generic custom-profile ingestor supervisor — DORMANT unless CUSTOM_PROFILE + # is set in .env. For each active profile that ships an `ingestor.py` it runs + # that script as an isolated subprocess (restart-on-exit); with no active + # profile it sleeps and makes zero calls. This is what lets a single switch + # (CUSTOM_PROFILE) bring up prompt-shaping AND a profile's ingestor at once, + # while this base file stays profile-agnostic. See custom-profiles/README.md. + profile_runner: + build: . + container_name: braindb_profile_runner + restart: unless-stopped + depends_on: + - api + networks: + - local-network + environment: + BRAINDB_API_URL: http://api:${API_PORT:-8000} + CUSTOM_PROFILE: ${CUSTOM_PROFILE:-} + volumes: + - .:/app + command: python -m braindb.profile_runner + # Read-only browser UI (frontend/ — plain static files, no build step). # Loopback-only on purpose: the UI's API calls go to http://localhost:8000 # from the BROWSER, so publishing this port beyond loopback wouldn't make diff --git a/pyproject.toml b/pyproject.toml index 7381158..9ab7233 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "braindb" -version = "0.8.0" +version = "0.9.0" description = "Persistent memory for LLM agents — thoughts, facts, sources, and behavioral rules with fuzzy + semantic search, graph traversal, and an internal agent." readme = "README.md" license = "Apache-2.0" diff --git a/tests/test_custom_profile.py b/tests/test_custom_profile.py new file mode 100644 index 0000000..0c29796 --- /dev/null +++ b/tests/test_custom_profile.py @@ -0,0 +1,106 @@ +""" +Pure-unit coverage for the optional custom-profile prompt composition +(braindb.custom_profile). No live stack, no Docker: these monkeypatch +CUSTOM_PROFILE / CUSTOM_PROFILE_DIR and write tiny fragment files into tmp_path, +so they run with `pytest` even when the stack is down (no `api` fixture). + +The load-bearing guarantee is the byte-identical test: with no active profile +the composition is a strict no-op, so the wiki prompts are exactly today's. +""" +import pytest + +from braindb import custom_profile + + +def _write(root, profile, filename, text): + d = root / profile + d.mkdir(parents=True, exist_ok=True) + (d / filename).write_text(text, encoding="utf-8") + + +@pytest.fixture +def profiles_dir(tmp_path, monkeypatch): + monkeypatch.setenv("CUSTOM_PROFILE_DIR", str(tmp_path)) + return tmp_path + + +def test_unset_is_strict_noop(profiles_dir, monkeypatch): + monkeypatch.delenv("CUSTOM_PROFILE", raising=False) + base = "BASE PROMPT {seeds} {wiki_catalog}" + assert custom_profile.replace_or_base(base, "wiki_maintainer") == base + assert custom_profile.additions("wiki_maintainer") == "" + assert custom_profile.additions("wiki_writer") == "" + + +def test_whitespace_only_profile_is_noop(profiles_dir, monkeypatch): + monkeypatch.setenv("CUSTOM_PROFILE", " ") + base = "BASE" + assert custom_profile.replace_or_base(base, "wiki_writer") == base + assert custom_profile.additions("wiki_writer") == "" + + +def test_maintainer_prompt_byte_identical_when_unset(profiles_dir, monkeypatch): + """Zero-default-impact proof: the wired assembly equals the bare base.""" + monkeypatch.delenv("CUSTOM_PROFILE", raising=False) + base = "RULES...\nSEEDS:\n{seeds}\nWIKIS:\n{wiki_catalog}\n" + bare = base.format(seeds="S", wiki_catalog="C") + wired = ( + custom_profile.replace_or_base(base, "wiki_maintainer") + .format(seeds="S", wiki_catalog="C") + + custom_profile.additions("wiki_maintainer") + ) + assert wired == bare + + +def test_add_one_profile(profiles_dir, monkeypatch): + _write(profiles_dir, "p1", "wiki_writer.add.md", "EXTRA RULE") + monkeypatch.setenv("CUSTOM_PROFILE", "p1") + assert custom_profile.additions("wiki_writer") == "\n\nEXTRA RULE" + # an untouched target stays empty + assert custom_profile.additions("wiki_maintainer") == "" + + +def test_add_multiple_profiles_joined_in_order(profiles_dir, monkeypatch): + _write(profiles_dir, "a", "wiki_writer.add.md", "FROM A") + _write(profiles_dir, "b", "wiki_writer.add.md", "FROM B") + monkeypatch.setenv("CUSTOM_PROFILE", "a,b") + assert custom_profile.additions("wiki_writer") == "\n\nFROM A\n\nFROM B" + monkeypatch.setenv("CUSTOM_PROFILE", "b,a") + assert custom_profile.additions("wiki_writer") == "\n\nFROM B\n\nFROM A" + + +def test_replace_swaps_template_last_wins(profiles_dir, monkeypatch): + base = "BASE" + _write(profiles_dir, "a", "wiki_writer.replace.md", "REPLACED BY A") + _write(profiles_dir, "b", "wiki_writer.replace.md", "REPLACED BY B") + monkeypatch.setenv("CUSTOM_PROFILE", "a,b") + assert custom_profile.replace_or_base(base, "wiki_writer") == "REPLACED BY B" + # a profile without a replace leaves the earlier one in place + monkeypatch.setenv("CUSTOM_PROFILE", "a,no_replace_here") + assert custom_profile.replace_or_base(base, "wiki_writer") == "REPLACED BY A" + + +def test_add_fragment_with_braces_is_format_safe(profiles_dir, monkeypatch): + """An `.add.md` with literal `{x}` must not break the maintainer's + `.format(...)`, because additions are appended AFTER substitution.""" + base = "RULES {seeds} {wiki_catalog}" + _write(profiles_dir, "p", "wiki_maintainer.add.md", "company note: use {ticker} as id") + monkeypatch.setenv("CUSTOM_PROFILE", "p") + tmpl = custom_profile.replace_or_base(base, "wiki_maintainer") + assembled = tmpl.format(seeds="S", wiki_catalog="C") + custom_profile.additions("wiki_maintainer") + assert assembled == "RULES S C\n\ncompany note: use {ticker} as id" + + +def test_blank_add_fragment_yields_no_suffix(profiles_dir, monkeypatch): + _write(profiles_dir, "p", "wiki_writer.add.md", " \n ") + monkeypatch.setenv("CUSTOM_PROFILE", "p") + assert custom_profile.additions("wiki_writer") == "" + + +def test_missing_profile_and_dir_never_raise(profiles_dir, monkeypatch): + monkeypatch.setenv("CUSTOM_PROFILE", "ghost") # folder doesn't exist under tmp + assert custom_profile.additions("wiki_writer") == "" + assert custom_profile.replace_or_base("BASE", "wiki_writer") == "BASE" + monkeypatch.setenv("CUSTOM_PROFILE_DIR", str(profiles_dir / "does_not_exist")) + assert custom_profile.additions("wiki_writer") == "" + assert custom_profile.replace_or_base("BASE", "wiki_writer") == "BASE"