feat: sensitive-data redaction, smarter grouping, OpenAI-compatible LLM, config hygiene

.gitignore

@@ -10,6 +10,14 @@ ChatExport*/
 # Tracked project config (re-include despite the broad *.json rule above)
 !configs/dataset_info.json
 
+# Synthetic demo input — safe to commit and needed for the reproducible demo.
+# (Generated demo outputs like demo/sample_sharegpt.json stay ignored via *.json.)
+!demo/sample_export.json
+
+# Personal training/export overrides — copy a tracked config to *.local.yaml and
+# edit that for your own model/hardware; it stays out of git.
+configs/*.local.yaml
+
 # Python cache / bytecode
 __pycache__/
 *.py[cod]

configs/train_lora.yaml

@@ -33,6 +33,10 @@ plot_loss: true
 overwrite_output_dir: true
 
 ### train
+# Loss is computed only on your (assistant/"gpt") turns; human turns are masked.
+# This is the SFT default, set explicitly here so --multi-speaker speaker labels
+# on the human side are safe — they condition the model but are never generated.
+train_on_prompt: false
 per_device_train_batch_size: 2
 gradient_accumulation_steps: 4
 learning_rate: 5.0e-5

demo/README.md

@@ -0,0 +1,31 @@
+# demo/
+
+Assets and the (dev-only) scripts used to generate the README banner/GIF. None of
+this is needed to run Doppelganger — it's tooling for regenerating the visuals.
+
+| File | What it is |
+|------|------------|
+| `parrot-mirror.jpg` | Source image for the mascot |
+| `mascot.txt` | The parrot converted to braille ASCII (committed art) |
+| `sample_export.json` | **Synthetic** Telegram export used by the demo (safe, fake PII) |
+| `demo.gif` | The README demo (ingest + sensitive-data scan) |
+| `img2ascii.py` | Convert an image to ASCII (brightness ramp) |
+| `build_final.py` | Rebuild `ingest/banner.py` and `demo/demo.gif` |
+
+## Regenerating
+
+These scripts need extra dev dependencies that the app itself does **not** require:
+
+```bash
+pip install pillow pyfiglet          # img2ascii.py / build_final.py
+# plus the asciinema 'agg' renderer (https://github.com/asciinema/agg):
+#   cargo install --git https://github.com/asciinema/agg
+#   or download a release binary and set:  export AGG=/path/to/agg
+```
+
+Then:
+
+```bash
+python demo/img2ascii.py parrot-mirror.jpg 72   # preview the mascot conversion
+python demo/build_final.py                      # rewrite the banner + demo.gif
+```

demo/build_final.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Build the final banner (ingest/banner.py) and demo GIF (demo/demo.gif).
+
+Layout: parrot (left) + ansi_shadow "Doppel"/"ganger" (right), amber wordmark,
+tagline centered beneath. Renders the GIF with agg's dracula theme.
+"""
+import json
+import os
+import subprocess
+
+import pyfiglet
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PY = os.path.join(ROOT, "venv", "bin", "python")
+AGG = os.environ.get("AGG", "agg")  # asciinema agg on PATH; override with $AGG
+GAP = "   "
+TAG = "fine-tune an LLM to write like you"
+CMD = "python -m ingest --source telegram --input demo/sample_export.json"
+AMBER, RESET = "\x1b[1;38;2;242;176;76m", "\x1b[0m"
+
+with open(os.path.join(ROOT, "demo/mascot.txt"), encoding="utf-8") as _f:
+    parrot = _f.read().rstrip("\n").split("\n")
+PW = max(len(l) for l in parrot)
+
+
+def _fig(t):
+    ls = [l.rstrip() for l in pyfiglet.figlet_format(t, font="ansi_shadow", width=200).rstrip("\n").split("\n")]
+    while ls and not ls[-1].strip(): ls.pop()
+    while ls and not ls[0].strip(): ls.pop(0)
+    return ls
+
+
+word = _fig("Doppel") + _fig("ganger")
+TOP = (len(parrot) - len(word)) // 2
+TOTAL_W = PW + len(GAP) + max(len(l) for l in word)
+
+
+def rows(on, off):
+    r = []
+    for i, pl in enumerate(parrot):
+        wl = word[i - TOP] if 0 <= i - TOP < len(word) else ""
+        wl = f"{on}{wl}{off}" if wl else ""
+        r.append((pl.ljust(PW) + GAP + wl).rstrip())
+    r.append("")
+    r.append(TAG.center(TOTAL_W).rstrip())  # tagline centred under the whole logo
+    return r
+
+
+def write_banner_module():
+    body = "\n".join(rows("<C>", "<R>"))  # sentinels; colourised at runtime
+    mod = (
+        '"""ASCII startup banner: a parrot in a mirror (it mimics your voice; the\n'
+        'mirror is the doppelganger) beside the wordmark. The wordmark is amber via\n'
+        'truecolor ANSI. Regenerate via demo/build_final.py.\n'
+        'Set DOPPELGANGER_NO_BANNER=1 to silence it."""\n\n'
+        'import os\n\n'
+        '_AMBER = "\\x1b[1;38;2;242;176;76m"  # truecolor amber\n'
+        '_RESET = "\\x1b[0m"\n\n'
+        '_BANNER = r"""\n' + body + '\n"""\n\n\n'
+        'def print_banner() -> None:\n'
+        '    if os.environ.get("DOPPELGANGER_NO_BANNER"):\n'
+        '        return\n'
+        '    print(_BANNER.replace("<C>", _AMBER).replace("<R>", _RESET) + "\\n")\n'
+    )
+    open(os.path.join(ROOT, "ingest/banner.py"), "w", encoding="utf-8").write(mod)
+
+
+def render_gif():
+    env = dict(os.environ, LLM_VALIDATE="false", DOPPELGANGER_NO_BANNER="1")
+    out = subprocess.run([PY, *CMD.split()[1:]], cwd=ROOT, env=env,
+                         capture_output=True, text=True, check=True)
+    report = ((out.stdout or "") + (out.stderr or "")).split("\n")
+
+    events, t = [], 0.0
+    def emit(d, dt):
+        nonlocal t
+        t += dt
+        events.append([round(t, 3), "o", d])
+    emit("\x1b[32m$\x1b[0m ", 0.3)
+    for ch in CMD:
+        emit(ch, 0.026)
+    emit("\r\n", 0.5)
+    for line in rows(AMBER, RESET) + report:
+        emit(line + "\r\n", 0.05)
+    emit("\x1b[32m$\x1b[0m ", 1.6)
+
+    cast = os.path.join(ROOT, "demo/demo.cast")
+    with open(cast, "w", encoding="utf-8") as f:
+        f.write(json.dumps({"version": 2, "width": 94, "height": 34}) + "\n")
+        for ev in events:
+            f.write(json.dumps(ev, ensure_ascii=False) + "\n")
+    subprocess.run([AGG, "--font-size", "18", "--theme", "dracula", cast,
+                    os.path.join(ROOT, "demo/demo.gif")],
+                   stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True, check=True)
+
+
+if __name__ == "__main__":
+    write_banner_module()
+    render_gif()
+    print("=== layout preview ===")
+    print("\n".join(rows("", "")))

demo/img2ascii.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Convert an image to ASCII art via a brightness ramp.
+
+Usage: python demo/img2ascii.py PATH [cols] [--invert]
+Transparent images are composited onto white first, so a dark subject on a
+transparent background (e.g. an OpenMoji black glyph) renders as the dense end
+of the ramp.
+"""
+import sys
+from PIL import Image
+
+RAMP = " .:-=+*#%@"
+
+
+def to_ascii(path: str, cols: int = 42, invert: bool = False) -> str:
+    img = Image.open(path)
+    if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
+        bg = Image.new("RGBA", img.size, (255, 255, 255, 255))
+        img = Image.alpha_composite(bg, img.convert("RGBA"))
+    g = img.convert("L")
+    rows = max(1, int(cols * (g.height / g.width) * 0.50))
+    g = g.resize((cols, rows))
+    px = g.load()
+    ramp = RAMP[::-1] if invert else RAMP
+    lines = []
+    for y in range(rows):
+        lines.append(
+            "".join(
+                ramp[int((255 - px[x, y]) / 255 * (len(ramp) - 1))] for x in range(cols)
+            ).rstrip()
+        )
+    return "\n".join(lines)
+
+
+if __name__ == "__main__":
+    args = [a for a in sys.argv[1:] if a != "--invert"]
+    if not args:
+        print("Usage: python demo/img2ascii.py PATH [cols] [--invert]", file=sys.stderr)
+        raise SystemExit(2)
+    print(to_ascii(args[0], int(args[1]) if len(args) > 1 else 42, "--invert" in sys.argv))

demo/mascot.txt

@@ -0,0 +1,17 @@
+⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⣀⡀
+⠀⠀⠀⠀⠀⠀⠀⠀⣀⣤⠶⠖⠛⠛⠋⠉⣿⣿⣿⣿⣷⣶⣤⣄⡀
+⠀⠀⠀⠀⠀⣠⡶⠛⠉⠀⠀⠀⠀⠀⠀⠀⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣄⡀
+⠀⠀⠀⣠⡾⠋⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣆
+⠀⠀⣴⢋⣤⣴⣶⣦⣤⡀⠀⠀⠀⠀⠀⠀⣿⣿⣿⣿⣿⣿⡿⠟⠋⠉⠉⠙⠻⢷⡀
+⠀⣸⣿⣿⣿⠿⠛⠻⣿⣿⣦⠀⠀⢀⣀⣤⣉⡙⠻⢿⣿⠏⠀⣠⠶⠚⠓⢶⣄⠀⠁
+⠀⣿⣿⣿⠁⠀⠸⠿⠊⣿⣿⠂⣴⠟⠁⠀⣿⣿⣷⣄⠙⠀⢸⠇⠀⠐⠿⠇⢹⡆
+⢸⣿⣿⣿⡀⠄⠀⠀⢀⣿⠇⣼⠃⠸⠁⠀⣿⣿⣄⣿⣇⠀⠸⣇⠠⠀⠀⠀⣼⠇
+⢸⡿⢿⣿⣿⣷⣶⣾⣿⣿⢰⡏⠀⠀⠀⠀⣿⣿⣿⣿⣿⡀⠀⠈⠛⠶⠶⠛⠁⠀⢠⡇
+⢸⡇⠈⠛⠿⣿⣿⣿⡿⠟⢸⣇⠀⠀⠀⠀⣿⣿⣿⣿⡿⠇⣀⡀⠀⠀⠀⢀⣠⣴⣿⡇
+⢸⡇⠀⠀⠀⠀⠀⠀⠀⠀⢰⡌⢳⣄⠀⠀⣿⣿⣿⠋⣠⠀⣿⣿⣿⣿⣿⣿⣿⣿⣿⡇
+⢸⡇⠀⠀⠀⠀⠀⠀⠀⠀⠈⢷⡀⢹⡆⠀⣿⣿⠃⣼⠏⣰⣿⣿⣿⣿⣿⣿⣿⣿⣿⡇
+⢸⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠳⣄⢻⡄⣿⡏⠐⢋⣴⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡇
+⢸⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⢷⡟⢠⣾⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡇
+⢸⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣴⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡇
+⢸⣇⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡇
+⠈⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠁

demo/sample_export.json

@@ -0,0 +1,22 @@
+{
+  "personal_information": { "first_name": "Alex", "last_name": "Tan" },
+  "chats": {
+    "list": [
+      {
+        "id": 4815162342,
+        "messages": [
+          { "type": "message", "id": 1, "from": "Jamie", "date_unixtime": "1700000000", "text": "yo did you book the airbnb for the trip?" },
+          { "type": "message", "id": 2, "from": "Alex Tan", "date_unixtime": "1700000035", "text": "yeah just sorted it! i'll forward you the invoice" },
+          { "type": "message", "id": 3, "from": "Jamie", "date_unixtime": "1700000070", "text": "nice send it to jamie.wong@gmail.com" },
+          { "type": "message", "id": 4, "from": "Alex Tan", "date_unixtime": "1700000110", "text": "done. they need your details for check-in too" },
+          { "type": "message", "id": 5, "from": "Jamie", "date_unixtime": "1700000150", "text": "ok my number is 8123 4567 and nric S1234567D" },
+          { "type": "message", "id": 6, "from": "Alex Tan", "date_unixtime": "1700000190", "text": "got it. i already paid the deposit with card 4111 1111 1111 1111" },
+          { "type": "message", "id": 7, "from": "Jamie", "date_unixtime": "1700000215", "text": "lmao did you just type your full card number" },
+          { "type": "message", "id": 8, "from": "Alex Tan", "date_unixtime": "1700000245", "text": "haha oops. anyway i'll send the confirmation to alex.tan@example.com" },
+          { "type": "message", "id": 9, "from": "Jamie", "date_unixtime": "1700000275", "text": "perfect, see you next week!" },
+          { "type": "message", "id": 10, "from": "Alex Tan", "date_unixtime": "1700000300", "text": "see ya, can't wait" }
+        ]
+      }
+    ]
+  }
+}

example.env

@@ -0,0 +1,22 @@
+# Copy to .env and fill in. .env is gitignored — never commit your keys.
+# Every value here is OPTIONAL; with none set, ingestion still runs (the LLM
+# features just stay off).
+
+# ── Optional LLM features (quality auditor + LLM redaction) ───────────────────
+# The CORE pipeline (parse -> dataset + regex sensitive-data scan) needs NONE of
+# this and runs with no setup. Uncomment below to ALSO enable the LLM auditor /
+# redaction.
+#
+# Run a LOCAL OpenAI-compatible server so your chat data never leaves your machine
+# (vLLM, LM Studio, llama.cpp). Serve an open model, then uncomment:
+#
+# LLM_VALIDATE=true
+# LLM_API_BASE_URL=http://localhost:8000/v1     # vLLM (LM Studio uses :1234/v1)
+# LLM_MODEL=Qwen/Qwen2.5-7B-Instruct            # the model your server serves
+# LLM_API_KEY=local                             # local servers accept any value
+
+# ── Optional: Hugging Face ────────────────────────────────────────────────────
+# Only needed to download GATED models during training (e.g. Gemma). The default
+# Qwen model in configs/train_lora.yaml is open and needs no token. Read by the
+# training stack (huggingface_hub), not by this repo's ingestion code.
+# HF_TOKEN=

ingest/adapters/telegram.py

@@ -64,14 +64,21 @@ def parse(
             for msg in chat.get("messages", []):
                 if not _is_valid(msg):
                     continue
-                sender = msg.get("from")
+                # "from" can be missing/None (e.g. anonymous channel posts); fall
+                # back to a label so sender_id stays a str (and multi-speaker
+                # mode doesn't emit a "None: " prefix).
+                sender = msg.get("from") or "Unknown"
+                reply_to = msg.get("reply_to_message_id")
+                msg_id = msg.get("id")
                 messages.append(
                     NormalizedMessage(
                         chat_id=chat_id,
                         timestamp=int(msg["date_unixtime"]),
                         sender_id=sender,
                         sender_is_self=(sender == self_name),
                         text=_get_text(msg),
+                        message_id=str(msg_id) if msg_id is not None else None,
+                        reply_to_id=str(reply_to) if reply_to is not None else None,
                     )
                 )
         return messages

ingest/banner.py

@@ -0,0 +1,37 @@
+"""ASCII startup banner: a parrot in a mirror (it mimics your voice; the
+mirror is the doppelganger) beside the wordmark. The wordmark is amber via
+truecolor ANSI. Regenerate via demo/build_final.py.
+Set DOPPELGANGER_NO_BANNER=1 to silence it."""
+
+import os
+
+_AMBER = "\x1b[1;38;2;242;176;76m"  # truecolor amber
+_RESET = "\x1b[0m"
+
+_BANNER = r"""
+⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⣀⡀
+⠀⠀⠀⠀⠀⠀⠀⠀⣀⣤⠶⠖⠛⠛⠋⠉⣿⣿⣿⣿⣷⣶⣤⣄⡀
+⠀⠀⠀⠀⠀⣠⡶⠛⠉⠀⠀⠀⠀⠀⠀⠀⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣄⡀       <C>██████╗  ██████╗ ██████╗ ██████╗ ███████╗██╗<R>
+⠀⠀⠀⣠⡾⠋⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣆      <C>██╔══██╗██╔═══██╗██╔══██╗██╔══██╗██╔════╝██║<R>
+⠀⠀⣴⢋⣤⣴⣶⣦⣤⡀⠀⠀⠀⠀⠀⠀⣿⣿⣿⣿⣿⣿⡿⠟⠋⠉⠉⠙⠻⢷⡀    <C>██║  ██║██║   ██║██████╔╝██████╔╝█████╗  ██║<R>
+⠀⣸⣿⣿⣿⠿⠛⠻⣿⣿⣦⠀⠀⢀⣀⣤⣉⡙⠻⢿⣿⠏⠀⣠⠶⠚⠓⢶⣄⠀⠁    <C>██║  ██║██║   ██║██╔═══╝ ██╔═══╝ ██╔══╝  ██║<R>
+⠀⣿⣿⣿⠁⠀⠸⠿⠊⣿⣿⠂⣴⠟⠁⠀⣿⣿⣷⣄⠙⠀⢸⠇⠀⠐⠿⠇⢹⡆     <C>██████╔╝╚██████╔╝██║     ██║     ███████╗███████╗<R>
+⢸⣿⣿⣿⡀⠄⠀⠀⢀⣿⠇⣼⠃⠸⠁⠀⣿⣿⣄⣿⣇⠀⠸⣇⠠⠀⠀⠀⣼⠇     <C>╚═════╝  ╚═════╝ ╚═╝     ╚═╝     ╚══════╝╚══════╝<R>
+⢸⡿⢿⣿⣿⣷⣶⣾⣿⣿⢰⡏⠀⠀⠀⠀⣿⣿⣿⣿⣿⡀⠀⠈⠛⠶⠶⠛⠁⠀⢠⡇   <C> ██████╗  █████╗ ███╗   ██╗ ██████╗ ███████╗██████╗<R>
+⢸⡇⠈⠛⠿⣿⣿⣿⡿⠟⢸⣇⠀⠀⠀⠀⣿⣿⣿⣿⡿⠇⣀⡀⠀⠀⠀⢀⣠⣴⣿⡇   <C>██╔════╝ ██╔══██╗████╗  ██║██╔════╝ ██╔════╝██╔══██╗<R>
+⢸⡇⠀⠀⠀⠀⠀⠀⠀⠀⢰⡌⢳⣄⠀⠀⣿⣿⣿⠋⣠⠀⣿⣿⣿⣿⣿⣿⣿⣿⣿⡇   <C>██║  ███╗███████║██╔██╗ ██║██║  ███╗█████╗  ██████╔╝<R>
+⢸⡇⠀⠀⠀⠀⠀⠀⠀⠀⠈⢷⡀⢹⡆⠀⣿⣿⠃⣼⠏⣰⣿⣿⣿⣿⣿⣿⣿⣿⣿⡇   <C>██║   ██║██╔══██║██║╚██╗██║██║   ██║██╔══╝  ██╔══██╗<R>
+⢸⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠳⣄⢻⡄⣿⡏⠐⢋⣴⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡇   <C>╚██████╔╝██║  ██║██║ ╚████║╚██████╔╝███████╗██║  ██║<R>
+⢸⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⢷⡟⢠⣾⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡇   <C> ╚═════╝ ╚═╝  ╚═╝╚═╝  ╚═══╝ ╚═════╝ ╚══════╝╚═╝  ╚═╝<R>
+⢸⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣴⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡇
+⢸⣇⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡇
+⠈⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠉⠁
+
+                           fine-tune an LLM to write like you
+"""
+
+
+def print_banner() -> None:
+    if os.environ.get("DOPPELGANGER_NO_BANNER"):
+        return
+    print(_BANNER.replace("<C>", _AMBER).replace("<R>", _RESET) + "\n")