Refactor RegexTokenizer to handle empty stats and update max function usage

.gitignore

@@ -173,7 +173,17 @@ cython_debug/
 # PyPI configuration file
 .pypirc
 
+checkpoints/
+AtlaSetCombined.txt
+# Prevent accidental commits of cleaned or redacted DM exports
+data/*_cleaned.txt
+data/*_redacted.txt
+# Ignore derived and private data exports
+data/*.jsonl
+data/dm_bpe.txt
 data/private/
+# Ignore generated notebook outputs and local workspace files
+notebooks/*_executed.ipynb
 output/
-checkpoints/
-AtlaSetCombined.txt
+.vscode/
+*.code-workspace

.patch_read_plain_chat.py

@@ -0,0 +1,83 @@
+import json
+from pathlib import Path
+path = Path(r'c:\Users\dglan\Documents\hackclub\llm\Train_Your_Language_Model_Course-clean\notebooks\1_DataCleaning.ipynb')
+nb = json.loads(path.read_text(encoding='utf-8'))
+changed = False
+for cell in nb['cells']:
+    if cell.get('cell_type') == 'code' and any('def read_plain_chat(file_path: str) -> pd.DataFrame:' in line for line in cell.get('source', [])):
+        cell['source'] = [
+            'def read_plain_chat(file_path: str) -> pd.DataFrame:\n',
+            '    import re\n',
+            '    # Generic cleaner for plain text chats (Discord or other exports)\n',
+            '    email_pattern = r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}"\n',
+            '    url_pattern = r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"\n',
+            '    media_pattern = "<Media omitted>"\n',
+            '    edited_message = "<This message was edited>"\n',
+            '    deleted_message = "You deleted this message"\n',
+            '    null_message = "null"\n',
+            '    created_group_message = "created group"\n',
+            '    added_you_to_group_message = "added you"\n',
+            '    tagging_pattern = r"@[\\w]+"\n',
+            '    export_marker = r"Exported .*message\\(s\\)"\n',
+            '    with open(file_path, "r", encoding="utf-8") as f:\n',
+            '        lines = f.readlines()\n',
+            '    filtered = []\n',
+            '    for line in lines:\n',
+            '        line = line.strip()\n',
+            '        if not line:\n',
+            '            continue\n',
+            '        if re.match(r"^[=-]{4,}$", line):\n',
+            '            continue\n',
+            '        if (\n',
+            '            deleted_message in line or\n',
+            '            null_message in line or\n',
+            '            media_pattern in line or\n',
+            '            created_group_message in line or\n',
+            '            added_you_to_group_message in line or\n',
+            '            re.search(email_pattern, line) or\n',
+            '            re.search(url_pattern, line) or\n',
+            '            re.search(export_marker, line)\n',
+            '        ):\n',
+            '            continue\n',
+            '        line = line.replace(edited_message, "").strip()\n',
+            '        line = re.sub(tagging_pattern, "", line).strip()\n',
+            '        if line:\n',
+            '            filtered.append(line)\n',
+            '    # Normalize unicode and join\n',
+            '    content = "\\n".join(filtered)\n',
+            '    content = content.replace("\\u202f", " ")\n',
+            '    content = content.replace("\\u200E", "").replace("\\u200F", "")\n',
+            '    # Parse lines: prefer dd/mm/yyyy style timestamps first, then ISO-like timestamps, then fall back to \'sender: message\'\n',
+            '    messages = []\n',
+            '    dm_pattern = r"^(\\d{1,2}/\\d{1,2}/\\d{2,4}, \\d{1,2}:\\d{2}(?::\\d{2})?)\\s*-\\s*(.*?):\\s*(.*)$"\n',
+            '    iso_pattern = r"^\\[?(\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2})\\]?\\s*(.*?):\\s*(.*)$"\n',
+            '    for line in content.split("\\n"):\n',
+            '        if not line.strip():\n',
+            '            continue\n',
+            '        m = re.match(dm_pattern, line)\n',
+            '        if m:\n',
+            '            ts, sender, msg = m.groups()\n',
+            '        else:\n',
+            '            m = re.match(iso_pattern, line)\n',
+            '            if m:\n',
+            '                ts, sender, msg = m.groups()\n',
+            '            else:\n',
+            '                m2 = re.match(r"^(.*?):\\s*(.*)$", line)\n',
+            '                if m2:\n',
+            '                    sender, msg = m2.groups()\n',
+            '                    ts = ""\n',
+            '                else:\n',
+            '                    sender = ""\n',
+            '                    msg = line\n',
+            '                    ts = ""\n',
+            '        messages.append((ts, sender.strip(), msg.strip()))\n',
+            '    df = pd.DataFrame(messages, columns=["timestamp","sender","message"])\n',
+            '    df["timestamp"] = pd.to_datetime(df["timestamp"], dayfirst=True, errors="coerce")\n',
+            '    return df\n',
+        ]
+        changed = True
+        break
+if not changed:
+    raise SystemExit('No matching read_plain_chat cell found')
+path.write_text(json.dumps(nb, indent=1, ensure_ascii=False), encoding='utf-8')
+print('patched')

data/anonymize_dm.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Anonymize DM export into a privacy-preserving JSONL file.
+
+Usage: python data/anonymize_dm.py path/to/raw.txt
+
+Output: creates path/to/raw_anonymized.jsonl with records:
+  {"date":"YYYY-MM-DD","user_id":"u1","message_hash":"abcd...","attachment_count":N}
+
+Notes: removes full timestamps (keeps date), replaces usernames with stable ids, hashes message text,
+and counts attachments instead of storing URLs.
+"""
+import re
+import sys
+import json
+import hashlib
+from pathlib import Path
+
+TS_USER_RE = re.compile(r"^\[(?P<ts>[^\]]+)\]\s*(?P<user>\S.*)$")
+URL_RE = re.compile(r"https?://\S+")
+
+
+def date_only(ts: str) -> str:
+    # Try to extract YYYY-MM-DD-like or MM/DD/YYYY and convert; fallback to raw date substring
+    # Example input: '8/31/2025 3:56 PM'
+    m = re.search(r"(\d{1,2})/(\d{1,2})/(\d{4})", ts)
+    if m:
+        mm, dd, yyyy = m.group(1), m.group(2), m.group(3)
+        return f"{int(yyyy):04d}-{int(mm):02d}-{int(dd):02d}"
+    # fallback: try YYYY-MM-DD
+    m2 = re.search(r"(\d{4})-(\d{2})-(\d{2})", ts)
+    if m2:
+        return f"{m2.group(1)}-{m2.group(2)}-{m2.group(3)}"
+    return ts.split()[0]
+
+
+def hash_text(s: str) -> str:
+    h = hashlib.sha256(s.encode('utf-8', errors='ignore')).hexdigest()
+    return h[:16]
+
+
+def anonymize(path: Path):
+    text = path.read_text(encoding='utf-8', errors='replace')
+    lines = text.splitlines()
+    user_map = {}
+    next_id = 1
+    out_lines = []
+    i = 0
+    while i < len(lines):
+        line = lines[i].rstrip()
+        m = TS_USER_RE.match(line)
+        if m:
+            ts = m.group('ts')
+            user = m.group('user').strip()
+            i += 1
+            msg_lines = []
+            att_count = 0
+            while i < len(lines):
+                nxt = lines[i]
+                if TS_USER_RE.match(nxt):
+                    break
+                if nxt.strip() == '{Attachments}':
+                    i += 1
+                    # count following URL lines
+                    while i < len(lines) and URL_RE.search(lines[i]):
+                        att_count += 1
+                        i += 1
+                    continue
+                # count inline urls as attachments and remove them from message
+                if URL_RE.search(nxt):
+                    att_count += len(URL_RE.findall(nxt))
+                    # strip urls
+                    nxt = URL_RE.sub('', nxt)
+                msg_lines.append(nxt.strip())
+                i += 1
+
+            msg = ' '.join(l for l in msg_lines if l)
+            # map user to stable id
+            if user not in user_map:
+                user_map[user] = f'u{next_id}'
+                next_id += 1
+            rec = {
+                'date': date_only(ts),
+                'user_id': user_map[user],
+                'message_hash': hash_text(msg) if msg else None,
+                'attachment_count': att_count,
+            }
+            out_lines.append(json.dumps(rec, ensure_ascii=False))
+        else:
+            i += 1
+
+    out = path.with_name(path.stem + '_anonymized.jsonl')
+    out.write_text('\n'.join(out_lines) + '\n', encoding='utf-8')
+    print(f'Wrote anonymized file: {out} (records: {len(out_lines)})')
+
+
+def main():
+    if len(sys.argv) < 2:
+        print('Usage: python data/anonymize_dm.py path/to/raw.txt')
+        sys.exit(1)
+    p = Path(sys.argv[1])
+    if not p.exists():
+        print('File not found:', p)
+        sys.exit(2)
+    anonymize(p)
+
+
+if __name__ == '__main__':
+    main()

data/clean_dm.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""Simple DM cleaning script.
+
+Usage: python data/clean_dm.py "path/to/file.txt"
+
+Outputs a cleaned file next to the input with suffix _cleaned.txt and prints a small summary.
+"""
+import re
+import sys
+from pathlib import Path
+
+
+OFFENSIVE = {"retards", "retard"}
+
+
+def redact(text: str) -> str:
+    def repl(m):
+        return "[redacted]"
+
+    # word-boundary replace for offensive words (case-insensitive)
+    pattern = re.compile(r"\\b(" + "|".join(re.escape(w) for w in OFFENSIVE) + r")\\b", flags=re.IGNORECASE)
+    text = pattern.sub(repl, text)
+    # normalize whitespace
+    text = re.sub(r"[\t\u00A0]+", " ", text)
+    text = re.sub(r" +", " ", text)
+    text = text.strip()
+    return text
+
+
+TS_USER_RE = re.compile(r"^\[(?P<ts>[^\]]+)\]\s*(?P<user>\S.*)$")
+URL_RE = re.compile(r"https?://\S+")
+
+
+def simplify_url(url: str) -> str:
+    # keep only base path (no query params)
+    return url.split("?")[0]
+
+
+def clean_messages(text: str):
+    lines = text.splitlines()
+    out_lines = []
+    i = 0
+    total_msgs = 0
+    removed = 0
+    while i < len(lines):
+        line = lines[i].rstrip()
+        m = TS_USER_RE.match(line)
+        if m:
+            ts = m.group("ts")
+            user = m.group("user").strip()
+            # gather following message lines until next timestamp or blank separator
+            i += 1
+            msg_lines = []
+            while i < len(lines):
+                nxt = lines[i]
+                if TS_USER_RE.match(nxt):
+                    break
+                # treat attachment marker and raw urls
+                if nxt.strip() == "{Attachments}":
+                    i += 1
+                    # collect URLs following
+                    att_urls = []
+                    while i < len(lines) and URL_RE.search(lines[i]):
+                        u = URL_RE.search(lines[i]).group(0)
+                        att_urls.append(simplify_url(u))
+                        i += 1
+                    if att_urls:
+                        msg_lines.append("[Attachment] " + ", ".join(att_urls))
+                    continue
+                msg_lines.append(nxt)
+                i += 1
+
+            msg = " ".join(l.strip() for l in msg_lines if l.strip())
+            msg = redact(msg)
+            if not msg:
+                removed += 1
+                continue
+            # turn into single-line record: timestamp \t user: message
+            out_lines.append(f"[{ts}]\t{user}: {msg}")
+            total_msgs += 1
+        else:
+            # non-timestamp lines: keep if they contain URLs or text
+            if URL_RE.search(line):
+                out_lines.append("[orphaned_url] " + simplify_url(URL_RE.search(line).group(0)))
+                total_msgs += 1
+            elif line.strip():
+                s = redact(line)
+                if s:
+                    out_lines.append(s)
+                    total_msgs += 1
+                else:
+                    removed += 1
+            i += 1
+
+    return "\n".join(out_lines) + "\n", total_msgs, removed
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python data/clean_dm.py path/to/file.txt")
+        sys.exit(1)
+    p = Path(sys.argv[1])
+    if not p.exists():
+        print("File not found:", p)
+        sys.exit(2)
+    raw = p.read_text(encoding="utf-8", errors="replace")
+    cleaned, kept, removed = clean_messages(raw)
+    out = p.with_name(p.stem + "_cleaned" + p.suffix)
+    out.write_text(cleaned, encoding="utf-8")
+    print(f"Cleaned: wrote {out}\nMessages kept: {kept}, removed/empty: {removed}")
+
+
+if __name__ == "__main__":
+    main()

data/make_combined.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+
+infile = Path('data') / 'dm_bpe.txt'
+outdir = Path('output')
+outdir.mkdir(parents=True, exist_ok=True)
+outfile = outdir / 'combined_text.txt'
+
+text = infile.read_text(encoding='utf-8', errors='replace')
+# join lines with a space to create a single long sequence
+text_sequence = ' '.join(line.strip() for line in text.splitlines() if line.strip())
+outfile.write_text(text_sequence, encoding='utf-8')
+print(f'Wrote {outfile} (chars: {len(text_sequence)})')