diff --git a/.gitignore b/.gitignore index e07275e..b77c66a 100644 --- a/.gitignore +++ b/.gitignore @@ -173,7 +173,17 @@ cython_debug/ # PyPI configuration file .pypirc +checkpoints/ +AtlaSetCombined.txt +# Prevent accidental commits of cleaned or redacted DM exports +data/*_cleaned.txt +data/*_redacted.txt +# Ignore derived and private data exports +data/*.jsonl +data/dm_bpe.txt data/private/ +# Ignore generated notebook outputs and local workspace files +notebooks/*_executed.ipynb output/ -checkpoints/ -AtlaSetCombined.txt \ No newline at end of file +.vscode/ +*.code-workspace \ No newline at end of file diff --git a/.patch_read_plain_chat.py b/.patch_read_plain_chat.py new file mode 100644 index 0000000..2221c82 --- /dev/null +++ b/.patch_read_plain_chat.py @@ -0,0 +1,83 @@ +import json +from pathlib import Path +path = Path(r'c:\Users\dglan\Documents\hackclub\llm\Train_Your_Language_Model_Course-clean\notebooks\1_DataCleaning.ipynb') +nb = json.loads(path.read_text(encoding='utf-8')) +changed = False +for cell in nb['cells']: + if cell.get('cell_type') == 'code' and any('def read_plain_chat(file_path: str) -> pd.DataFrame:' in line for line in cell.get('source', [])): + cell['source'] = [ + 'def read_plain_chat(file_path: str) -> pd.DataFrame:\n', + ' import re\n', + ' # Generic cleaner for plain text chats (Discord or other exports)\n', + ' email_pattern = r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}"\n', + ' url_pattern = r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"\n', + ' media_pattern = ""\n', + ' edited_message = ""\n', + ' deleted_message = "You deleted this message"\n', + ' null_message = "null"\n', + ' created_group_message = "created group"\n', + ' added_you_to_group_message = "added you"\n', + ' tagging_pattern = r"@[\\w]+"\n', + ' export_marker = r"Exported .*message\\(s\\)"\n', + ' with open(file_path, "r", encoding="utf-8") as f:\n', + ' lines = f.readlines()\n', + ' filtered = []\n', + ' for line in lines:\n', + ' line = line.strip()\n', + ' if not line:\n', + ' continue\n', + ' if re.match(r"^[=-]{4,}$", line):\n', + ' continue\n', + ' if (\n', + ' deleted_message in line or\n', + ' null_message in line or\n', + ' media_pattern in line or\n', + ' created_group_message in line or\n', + ' added_you_to_group_message in line or\n', + ' re.search(email_pattern, line) or\n', + ' re.search(url_pattern, line) or\n', + ' re.search(export_marker, line)\n', + ' ):\n', + ' continue\n', + ' line = line.replace(edited_message, "").strip()\n', + ' line = re.sub(tagging_pattern, "", line).strip()\n', + ' if line:\n', + ' filtered.append(line)\n', + ' # Normalize unicode and join\n', + ' content = "\\n".join(filtered)\n', + ' content = content.replace("\\u202f", " ")\n', + ' content = content.replace("\\u200E", "").replace("\\u200F", "")\n', + ' # Parse lines: prefer dd/mm/yyyy style timestamps first, then ISO-like timestamps, then fall back to \'sender: message\'\n', + ' messages = []\n', + ' dm_pattern = r"^(\\d{1,2}/\\d{1,2}/\\d{2,4}, \\d{1,2}:\\d{2}(?::\\d{2})?)\\s*-\\s*(.*?):\\s*(.*)$"\n', + ' iso_pattern = r"^\\[?(\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2})\\]?\\s*(.*?):\\s*(.*)$"\n', + ' for line in content.split("\\n"):\n', + ' if not line.strip():\n', + ' continue\n', + ' m = re.match(dm_pattern, line)\n', + ' if m:\n', + ' ts, sender, msg = m.groups()\n', + ' else:\n', + ' m = re.match(iso_pattern, line)\n', + ' if m:\n', + ' ts, sender, msg = m.groups()\n', + ' else:\n', + ' m2 = re.match(r"^(.*?):\\s*(.*)$", line)\n', + ' if m2:\n', + ' sender, msg = m2.groups()\n', + ' ts = ""\n', + ' else:\n', + ' sender = ""\n', + ' msg = line\n', + ' ts = ""\n', + ' messages.append((ts, sender.strip(), msg.strip()))\n', + ' df = pd.DataFrame(messages, columns=["timestamp","sender","message"])\n', + ' df["timestamp"] = pd.to_datetime(df["timestamp"], dayfirst=True, errors="coerce")\n', + ' return df\n', + ] + changed = True + break +if not changed: + raise SystemExit('No matching read_plain_chat cell found') +path.write_text(json.dumps(nb, indent=1, ensure_ascii=False), encoding='utf-8') +print('patched') diff --git a/data/anonymize_dm.py b/data/anonymize_dm.py new file mode 100644 index 0000000..c2f2138 --- /dev/null +++ b/data/anonymize_dm.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Anonymize DM export into a privacy-preserving JSONL file. + +Usage: python data/anonymize_dm.py path/to/raw.txt + +Output: creates path/to/raw_anonymized.jsonl with records: + {"date":"YYYY-MM-DD","user_id":"u1","message_hash":"abcd...","attachment_count":N} + +Notes: removes full timestamps (keeps date), replaces usernames with stable ids, hashes message text, +and counts attachments instead of storing URLs. +""" +import re +import sys +import json +import hashlib +from pathlib import Path + +TS_USER_RE = re.compile(r"^\[(?P[^\]]+)\]\s*(?P\S.*)$") +URL_RE = re.compile(r"https?://\S+") + + +def date_only(ts: str) -> str: + # Try to extract YYYY-MM-DD-like or MM/DD/YYYY and convert; fallback to raw date substring + # Example input: '8/31/2025 3:56 PM' + m = re.search(r"(\d{1,2})/(\d{1,2})/(\d{4})", ts) + if m: + mm, dd, yyyy = m.group(1), m.group(2), m.group(3) + return f"{int(yyyy):04d}-{int(mm):02d}-{int(dd):02d}" + # fallback: try YYYY-MM-DD + m2 = re.search(r"(\d{4})-(\d{2})-(\d{2})", ts) + if m2: + return f"{m2.group(1)}-{m2.group(2)}-{m2.group(3)}" + return ts.split()[0] + + +def hash_text(s: str) -> str: + h = hashlib.sha256(s.encode('utf-8', errors='ignore')).hexdigest() + return h[:16] + + +def anonymize(path: Path): + text = path.read_text(encoding='utf-8', errors='replace') + lines = text.splitlines() + user_map = {} + next_id = 1 + out_lines = [] + i = 0 + while i < len(lines): + line = lines[i].rstrip() + m = TS_USER_RE.match(line) + if m: + ts = m.group('ts') + user = m.group('user').strip() + i += 1 + msg_lines = [] + att_count = 0 + while i < len(lines): + nxt = lines[i] + if TS_USER_RE.match(nxt): + break + if nxt.strip() == '{Attachments}': + i += 1 + # count following URL lines + while i < len(lines) and URL_RE.search(lines[i]): + att_count += 1 + i += 1 + continue + # count inline urls as attachments and remove them from message + if URL_RE.search(nxt): + att_count += len(URL_RE.findall(nxt)) + # strip urls + nxt = URL_RE.sub('', nxt) + msg_lines.append(nxt.strip()) + i += 1 + + msg = ' '.join(l for l in msg_lines if l) + # map user to stable id + if user not in user_map: + user_map[user] = f'u{next_id}' + next_id += 1 + rec = { + 'date': date_only(ts), + 'user_id': user_map[user], + 'message_hash': hash_text(msg) if msg else None, + 'attachment_count': att_count, + } + out_lines.append(json.dumps(rec, ensure_ascii=False)) + else: + i += 1 + + out = path.with_name(path.stem + '_anonymized.jsonl') + out.write_text('\n'.join(out_lines) + '\n', encoding='utf-8') + print(f'Wrote anonymized file: {out} (records: {len(out_lines)})') + + +def main(): + if len(sys.argv) < 2: + print('Usage: python data/anonymize_dm.py path/to/raw.txt') + sys.exit(1) + p = Path(sys.argv[1]) + if not p.exists(): + print('File not found:', p) + sys.exit(2) + anonymize(p) + + +if __name__ == '__main__': + main() diff --git a/data/clean_dm.py b/data/clean_dm.py new file mode 100644 index 0000000..b47f252 --- /dev/null +++ b/data/clean_dm.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Simple DM cleaning script. + +Usage: python data/clean_dm.py "path/to/file.txt" + +Outputs a cleaned file next to the input with suffix _cleaned.txt and prints a small summary. +""" +import re +import sys +from pathlib import Path + + +OFFENSIVE = {"retards", "retard"} + + +def redact(text: str) -> str: + def repl(m): + return "[redacted]" + + # word-boundary replace for offensive words (case-insensitive) + pattern = re.compile(r"\\b(" + "|".join(re.escape(w) for w in OFFENSIVE) + r")\\b", flags=re.IGNORECASE) + text = pattern.sub(repl, text) + # normalize whitespace + text = re.sub(r"[\t\u00A0]+", " ", text) + text = re.sub(r" +", " ", text) + text = text.strip() + return text + + +TS_USER_RE = re.compile(r"^\[(?P[^\]]+)\]\s*(?P\S.*)$") +URL_RE = re.compile(r"https?://\S+") + + +def simplify_url(url: str) -> str: + # keep only base path (no query params) + return url.split("?")[0] + + +def clean_messages(text: str): + lines = text.splitlines() + out_lines = [] + i = 0 + total_msgs = 0 + removed = 0 + while i < len(lines): + line = lines[i].rstrip() + m = TS_USER_RE.match(line) + if m: + ts = m.group("ts") + user = m.group("user").strip() + # gather following message lines until next timestamp or blank separator + i += 1 + msg_lines = [] + while i < len(lines): + nxt = lines[i] + if TS_USER_RE.match(nxt): + break + # treat attachment marker and raw urls + if nxt.strip() == "{Attachments}": + i += 1 + # collect URLs following + att_urls = [] + while i < len(lines) and URL_RE.search(lines[i]): + u = URL_RE.search(lines[i]).group(0) + att_urls.append(simplify_url(u)) + i += 1 + if att_urls: + msg_lines.append("[Attachment] " + ", ".join(att_urls)) + continue + msg_lines.append(nxt) + i += 1 + + msg = " ".join(l.strip() for l in msg_lines if l.strip()) + msg = redact(msg) + if not msg: + removed += 1 + continue + # turn into single-line record: timestamp \t user: message + out_lines.append(f"[{ts}]\t{user}: {msg}") + total_msgs += 1 + else: + # non-timestamp lines: keep if they contain URLs or text + if URL_RE.search(line): + out_lines.append("[orphaned_url] " + simplify_url(URL_RE.search(line).group(0))) + total_msgs += 1 + elif line.strip(): + s = redact(line) + if s: + out_lines.append(s) + total_msgs += 1 + else: + removed += 1 + i += 1 + + return "\n".join(out_lines) + "\n", total_msgs, removed + + +def main(): + if len(sys.argv) < 2: + print("Usage: python data/clean_dm.py path/to/file.txt") + sys.exit(1) + p = Path(sys.argv[1]) + if not p.exists(): + print("File not found:", p) + sys.exit(2) + raw = p.read_text(encoding="utf-8", errors="replace") + cleaned, kept, removed = clean_messages(raw) + out = p.with_name(p.stem + "_cleaned" + p.suffix) + out.write_text(cleaned, encoding="utf-8") + print(f"Cleaned: wrote {out}\nMessages kept: {kept}, removed/empty: {removed}") + + +if __name__ == "__main__": + main() diff --git a/data/make_combined.py b/data/make_combined.py new file mode 100644 index 0000000..e7509fc --- /dev/null +++ b/data/make_combined.py @@ -0,0 +1,12 @@ +from pathlib import Path + +infile = Path('data') / 'dm_bpe.txt' +outdir = Path('output') +outdir.mkdir(parents=True, exist_ok=True) +outfile = outdir / 'combined_text.txt' + +text = infile.read_text(encoding='utf-8', errors='replace') +# join lines with a space to create a single long sequence +text_sequence = ' '.join(line.strip() for line in text.splitlines() if line.strip()) +outfile.write_text(text_sequence, encoding='utf-8') +print(f'Wrote {outfile} (chars: {len(text_sequence)})') diff --git a/data/make_private_dm.py b/data/make_private_dm.py new file mode 100644 index 0000000..dbb46c5 --- /dev/null +++ b/data/make_private_dm.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Convert a raw DM export into a privacy-safe chat log in the same plain-text format as DummyData.txt.""" +import re +import sys +from pathlib import Path +from datetime import datetime + +TS_USER_RE = re.compile(r"^\[(?P[^\]]+)\]\s*(?P\S.*)$") +URL_RE = re.compile(r"https?://\S+") +OFFENSIVE = {"retards", "retard"} + + +def redact(text: str) -> str: + pattern = re.compile(r"\b(" + "|".join(re.escape(w) for w in OFFENSIVE) + r")\b", flags=re.IGNORECASE) + text = pattern.sub("[redacted]", text) + text = re.sub(r"[\t\u00A0]+", " ", text) + text = re.sub(r" +", " ", text) + return text.strip() + + +def fmt_ts(raw: str) -> str: + # convert '8/31/2025 3:56 PM' -> '31/08/2025, 15:56' + try: + dt = datetime.strptime(raw, "%m/%d/%Y %I:%M %p") + except ValueError: + try: + dt = datetime.strptime(raw, "%m/%d/%Y %I:%M%p") + except ValueError: + return raw + return dt.strftime("%d/%m/%Y, %H:%M") + + +def parse_messages(text: str): + lines = text.splitlines() + msgs = [] + i = 0 + while i < len(lines): + line = lines[i].rstrip() + m = TS_USER_RE.match(line) + if m: + ts = m.group("ts") + user = m.group("user").strip() + i += 1 + msg_lines = [] + while i < len(lines): + nxt = lines[i] + if TS_USER_RE.match(nxt): + break + if nxt.strip() == "{Attachments}": + i += 1 + while i < len(lines) and URL_RE.search(lines[i]): + i += 1 + continue + msg_lines.append(nxt) + i += 1 + message = " ".join(l.strip() for l in msg_lines if l.strip()) + message = URL_RE.sub("", message) + message = redact(message) + if message: + msgs.append((fmt_ts(ts), user, message)) + continue + i += 1 + return msgs + + +def write_private_file(input_path: Path, output_path: Path): + msgs = parse_messages(input_path.read_text(encoding="utf-8", errors="replace")) + name_map = {} + name_counter = 1 + lines = [] + for ts, user, msg in msgs: + if user not in name_map: + name_map[user] = f"Person {name_counter}" + name_counter += 1 + lines.append(f"{ts} - {name_map[user]}: {msg}") + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"Wrote {output_path} ({len(lines)} messages)") + + +def main(): + if len(sys.argv) < 2: + print("Usage: python data/make_private_dm.py path/to/raw_dm.txt") + sys.exit(1) + input_path = Path(sys.argv[1]) + if not input_path.exists(): + print("File not found:", input_path) + sys.exit(2) + output_path = Path("data/private") / input_path.name + write_private_file(input_path, output_path) + + +if __name__ == "__main__": + main() diff --git a/data/prepare_bpe.py b/data/prepare_bpe.py new file mode 100644 index 0000000..c6c0cd6 --- /dev/null +++ b/data/prepare_bpe.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Prepare a BPE-ready corpus from a raw DM export. + +Output: data/dm_bpe.txt with one message per line. + +Behavior: +- Collapse multi-line messages into single lines +- Strip timestamps and usernames +- Remove URLs and attachment markers +- Lowercase and normalize whitespace +- Filter out very short messages (<3 chars) +""" +import re +import sys +from pathlib import Path + +TS_USER_RE = re.compile(r"^\[(?P[^\]]+)\]\s*(?P\S.*)$") +URL_RE = re.compile(r"https?://\S+") + + +def clean_msg_lines(msg_lines): + # join, remove urls, lowercase, normalize spaces + s = ' '.join(l.strip() for l in msg_lines if l and l.strip()) + s = URL_RE.sub('', s) + s = re.sub(r"[\t\u00A0]+", ' ', s) + s = re.sub(r" +", ' ', s) + s = s.strip().lower() + return s + + +def prepare(path: Path, outpath: Path): + text = path.read_text(encoding='utf-8', errors='replace') + lines = text.splitlines() + i = 0 + count_in = 0 + count_out = 0 + with outpath.open('w', encoding='utf-8') as f: + while i < len(lines): + line = lines[i].rstrip() + m = TS_USER_RE.match(line) + if m: + count_in += 1 + i += 1 + msg_lines = [] + while i < len(lines): + nxt = lines[i] + if TS_USER_RE.match(nxt): + break + if nxt.strip() == '{Attachments}': + # skip attachments and following url lines + i += 1 + while i < len(lines) and URL_RE.search(lines[i]): + i += 1 + continue + # treat inline url lines by removing urls later + msg_lines.append(nxt) + i += 1 + + cleaned = clean_msg_lines(msg_lines) + if len(cleaned) >= 3: + f.write(cleaned + '\n') + count_out += 1 + continue + else: + # ignore non-timestamp top-level lines + i += 1 + + print(f'Prepared BPE corpus: wrote {outpath} (messages in: {count_in}, kept: {count_out})') + + +def main(): + if len(sys.argv) < 2: + print('Usage: python data/prepare_bpe.py path/to/raw.txt') + sys.exit(1) + p = Path(sys.argv[1]) + if not p.exists(): + print('File not found:', p) + sys.exit(2) + out = Path('data') / 'dm_bpe.txt' + prepare(p, out) + + +if __name__ == '__main__': + main() diff --git a/data/redact_all.py b/data/redact_all.py new file mode 100644 index 0000000..a3bb9ea --- /dev/null +++ b/data/redact_all.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Create a fully redacted version of a cleaned DM file. + +Usage: python data/redact_all.py path/to/xxx_cleaned.txt [--delete] + +This writes a file with suffix _redacted.txt where every non-empty line is replaced +with the single token [REDACTED]. +""" +import sys +from pathlib import Path + + +def redact_file(p: Path, delete_original: bool = False): + text = p.read_text(encoding='utf-8', errors='replace') + lines = text.splitlines() + redacted_lines = [] + for ln in lines: + if ln.strip(): + redacted_lines.append('[REDACTED]') + else: + redacted_lines.append('') + out = p.with_name(p.stem.replace('_cleaned','') + '_redacted' + p.suffix) + out.write_text('\n'.join(redacted_lines) + '\n', encoding='utf-8') + print(f'Wrote redacted file: {out}') + if delete_original: + p.unlink() + print(f'Deleted original cleaned file: {p}') + + +def main(): + if len(sys.argv) < 2: + print('Usage: python data/redact_all.py path/to/xxx_cleaned.txt [--delete]') + sys.exit(1) + p = Path(sys.argv[1]) + if not p.exists(): + print('File not found:', p) + sys.exit(2) + delete = '--delete' in sys.argv[2:] + redact_file(p, delete_original=delete) + + +if __name__ == '__main__': + main() diff --git a/minbpe/regex.py b/minbpe/regex.py index 3d9fb28..92c4d08 100644 --- a/minbpe/regex.py +++ b/minbpe/regex.py @@ -53,8 +53,10 @@ def train(self, text, vocab_size, verbose=False): for chunk_ids in ids: # passing in stats will update it in place, adding up counts get_stats(chunk_ids, stats) + if not stats: + break # find the pair with the highest count - pair = max(stats, key=stats.get) + pair = max(stats.items(), key=lambda item: item[1])[0] # mint a new token: assign it the next available id idx = 256 + i # replace all occurrences of pair in ids with idx diff --git a/notebooks/1_DataCleaning.ipynb b/notebooks/1_DataCleaning.ipynb index 25bb1bf..59d95f8 100644 --- a/notebooks/1_DataCleaning.ipynb +++ b/notebooks/1_DataCleaning.ipynb @@ -65,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -139,10 +139,79 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 5, "metadata": {}, + "outputs": [], "source": [ - "The `all_chats` dictionary holds the content of each file as a dataframe with three columns: `timestamp`, `sender`, and `message`. " + "def read_plain_chat(file_path: str) -> pd.DataFrame:\n", + " import re\n", + " # Generic cleaner for plain text chats (Discord or other exports)\n", + " email_pattern = r\"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\"\n", + " url_pattern = r\"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+\"\n", + " media_pattern = \"\"\n", + " edited_message = \"\"\n", + " deleted_message = \"You deleted this message\"\n", + " null_message = \"null\"\n", + " created_group_message = \"created group\"\n", + " added_you_to_group_message = \"added you\"\n", + " tagging_pattern = r\"@[\\w]+\"\n", + " export_marker = r\"Exported .*message\\(s\\)\"\n", + " with open(file_path, \"r\", encoding=\"utf-8\") as f:\n", + " lines = f.readlines()\n", + " filtered = []\n", + " for line in lines:\n", + " line = line.strip()\n", + " if not line:\n", + " continue\n", + " if re.match(r\"^[=-]{4,}$\", line):\n", + " continue\n", + " if (\n", + " deleted_message in line or\n", + " null_message in line or\n", + " media_pattern in line or\n", + " created_group_message in line or\n", + " added_you_to_group_message in line or\n", + " re.search(email_pattern, line) or\n", + " re.search(url_pattern, line) or\n", + " re.search(export_marker, line)\n", + " ):\n", + " continue\n", + " line = line.replace(edited_message, \"\").strip()\n", + " line = re.sub(tagging_pattern, \"\", line).strip()\n", + " if line:\n", + " filtered.append(line)\n", + " # Normalize unicode and join\n", + " content = \"\\n\".join(filtered)\n", + " content = content.replace(\"\\u202f\", \" \")\n", + " content = content.replace(\"\\u200E\", \"\").replace(\"\\u200F\", \"\")\n", + " # Parse lines: prefer dd/mm/yyyy style timestamps first, then ISO-like timestamps, then fall back to 'sender: message'\n", + " messages = []\n", + " dm_pattern = r\"^(\\d{1,2}/\\d{1,2}/\\d{2,4}, \\d{1,2}:\\d{2}(?::\\d{2})?)\\s*-\\s*(.*?):\\s*(.*)$\"\n", + " iso_pattern = r\"^\\[?(\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2})\\]?\\s*(.*?):\\s*(.*)$\"\n", + " for line in content.split(\"\\n\"):\n", + " if not line.strip():\n", + " continue\n", + " m = re.match(dm_pattern, line)\n", + " if m:\n", + " ts, sender, msg = m.groups()\n", + " else:\n", + " m = re.match(iso_pattern, line)\n", + " if m:\n", + " ts, sender, msg = m.groups()\n", + " else:\n", + " m2 = re.match(r\"^(.*?):\\s*(.*)$\", line)\n", + " if m2:\n", + " sender, msg = m2.groups()\n", + " ts = \"\"\n", + " else:\n", + " sender = \"\"\n", + " msg = line\n", + " ts = \"\"\n", + " messages.append((ts, sender.strip(), msg.strip()))\n", + " df = pd.DataFrame(messages, columns=[\"timestamp\",\"sender\",\"message\"])\n", + " df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"], dayfirst=True, errors=\"coerce\")\n", + " return df\n" ] }, { @@ -150,14 +219,86 @@ "execution_count": null, "metadata": {}, "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `all_chats` dictionary holds the content of each file as a dataframe with three columns: `timestamp`, `sender`, and `message`. " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Direct Messages - Buncha retards [1411801796439183370]': timestamp sender \\\n", + " 0 2025-08-31 15:56:00 Person 1 \n", + " 1 2025-08-31 15:56:00 Person 1 \n", + " 2 2025-08-31 15:56:00 Person 2 \n", + " 3 2025-08-31 15:56:00 Person 3 \n", + " 4 2025-08-31 15:56:00 Person 1:\\n31/08/2025, 15:56 - Person 3 \n", + " ... ... ... \n", + " 27247 2026-06-30 12:35:00 Person 3 \n", + " 27248 2026-06-30 12:35:00 Person 3 \n", + " 27249 2026-06-30 12:36:00 Person 9 \n", + " 27250 2026-06-30 12:36:00 Person 1 \n", + " 27251 2026-06-30 12:36:00 Person 9 \n", + " \n", + " message \n", + " 0 Yo \n", + " 1 This confusing me texting both \n", + " 2 Lose \n", + " 3 wtf \n", + " 4 what \n", + " ... ... \n", + " 27247 youtube maybe \n", + " 27248 post rednote fr \n", + " 27249 wait y did bro get kicked \n", + " 27250 i aint gonna post that \n", + " 27251 what did bro do? =============================... \n", + " \n", + " [27252 rows x 3 columns],\n", + " 'discord': timestamp sender message\n", + " 0 2025-08-31 15:56:00 Person 1 Yo\n", + " 1 2025-08-31 15:56:00 Person 1 This confusing me texting both\n", + " 2 2025-08-31 15:56:00 Person 2 Lose\n", + " 3 2025-08-31 15:56:00 Person 3 wtf\n", + " 4 2025-08-31 15:56:00 Person 1 \n", + " ... ... ... ...\n", + " 28005 2026-06-30 12:35:00 Person 3 tik might\n", + " 28006 2026-06-30 12:35:00 Person 3 youtube maybe\n", + " 28007 2026-06-30 12:35:00 Person 3 post rednote fr\n", + " 28008 2026-06-30 12:36:00 Person 9 wait y did bro get kicked\n", + " 28009 2026-06-30 12:36:00 Person 1 i aint gonna post that\n", + " \n", + " [28010 rows x 3 columns]}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from pathlib import Path\n", "\n", "all_chats = {}\n", "data_directory = Path(\"../data/private\")\n", - "for file in data_directory.glob('*.txt'):\n", - " file_name = file.stem\n", - " all_chats[file_name] = read_whatsapp_chat(file)" + "if data_directory.exists():\n", + " for file in data_directory.glob(\"*.txt\"):\n", + " file_name = file.stem\n", + " all_chats[file_name] = read_whatsapp_chat(str(file))\n", + "\n", + "discord_file = Path(\"../data/private/Direct Messages - Buncha retards [1411801796439183370].txt\")\n", + "if discord_file.exists():\n", + " all_chats['discord'] = read_plain_chat(str(discord_file))\n", + "\n", + "all_chats\n" ] }, { @@ -176,9 +317,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "1036745" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "text_sequence = \"\"\n", "for file_name in all_chats.keys():\n", @@ -189,18 +341,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ - "with open(\"../output/combined_text.txt\", \"w\", encoding=\"utf-8\") as f:\n", + "output_file = Path(\"../output/combined_text.txt\")\n", + "output_file.parent.mkdir(parents=True, exist_ok=True)\n", + "\n", + "with open(output_file, \"w\", encoding=\"utf-8\") as f:\n", " f.write(text_sequence)" ] } ], "metadata": { "kernelspec": { - "display_name": "vincent", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -214,7 +369,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.10" + "version": "3.12.10" } }, "nbformat": 4, diff --git a/notebooks/2_BytePairEncoding.ipynb b/notebooks/2_BytePairEncoding.ipynb index 90ba6c9..0a85402 100644 --- a/notebooks/2_BytePairEncoding.ipynb +++ b/notebooks/2_BytePairEncoding.ipynb @@ -9,9 +9,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "1036745" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "with open(\"../output/combined_text.txt\", \"r\", encoding=\"utf-8\") as f:\n", " text_sequence = f.read()\n", @@ -35,7 +46,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -52,9 +63,17 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 768/768 [02:44<00:00, 4.68it/s]\n" + ] + } + ], "source": [ "from minbpe import BasicTokenizer\n", "\n", @@ -71,9 +90,1020 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{0: b'\\x00',\n", + " 1: b'\\x01',\n", + " 2: b'\\x02',\n", + " 3: b'\\x03',\n", + " 4: b'\\x04',\n", + " 5: b'\\x05',\n", + " 6: b'\\x06',\n", + " 7: b'\\x07',\n", + " 8: b'\\x08',\n", + " 9: b'\\t',\n", + " 10: b'\\n',\n", + " 11: b'\\x0b',\n", + " 12: b'\\x0c',\n", + " 13: b'\\r',\n", + " 14: b'\\x0e',\n", + " 15: b'\\x0f',\n", + " 16: b'\\x10',\n", + " 17: b'\\x11',\n", + " 18: b'\\x12',\n", + " 19: b'\\x13',\n", + " 20: b'\\x14',\n", + " 21: b'\\x15',\n", + " 22: b'\\x16',\n", + " 23: b'\\x17',\n", + " 24: b'\\x18',\n", + " 25: b'\\x19',\n", + " 26: b'\\x1a',\n", + " 27: b'\\x1b',\n", + " 28: b'\\x1c',\n", + " 29: b'\\x1d',\n", + " 30: b'\\x1e',\n", + " 31: b'\\x1f',\n", + " 32: b' ',\n", + " 33: b'!',\n", + " 34: b'\"',\n", + " 35: b'#',\n", + " 36: b'$',\n", + " 37: b'%',\n", + " 38: b'&',\n", + " 39: b\"'\",\n", + " 40: b'(',\n", + " 41: b')',\n", + " 42: b'*',\n", + " 43: b'+',\n", + " 44: b',',\n", + " 45: b'-',\n", + " 46: b'.',\n", + " 47: b'/',\n", + " 48: b'0',\n", + " 49: b'1',\n", + " 50: b'2',\n", + " 51: b'3',\n", + " 52: b'4',\n", + " 53: b'5',\n", + " 54: b'6',\n", + " 55: b'7',\n", + " 56: b'8',\n", + " 57: b'9',\n", + " 58: b':',\n", + " 59: b';',\n", + " 60: b'<',\n", + " 61: b'=',\n", + " 62: b'>',\n", + " 63: b'?',\n", + " 64: b'@',\n", + " 65: b'A',\n", + " 66: b'B',\n", + " 67: b'C',\n", + " 68: b'D',\n", + " 69: b'E',\n", + " 70: b'F',\n", + " 71: b'G',\n", + " 72: b'H',\n", + " 73: b'I',\n", + " 74: b'J',\n", + " 75: b'K',\n", + " 76: b'L',\n", + " 77: b'M',\n", + " 78: b'N',\n", + " 79: b'O',\n", + " 80: b'P',\n", + " 81: b'Q',\n", + " 82: b'R',\n", + " 83: b'S',\n", + " 84: b'T',\n", + " 85: b'U',\n", + " 86: b'V',\n", + " 87: b'W',\n", + " 88: b'X',\n", + " 89: b'Y',\n", + " 90: b'Z',\n", + " 91: b'[',\n", + " 92: b'\\\\',\n", + " 93: b']',\n", + " 94: b'^',\n", + " 95: b'_',\n", + " 96: b'`',\n", + " 97: b'a',\n", + " 98: b'b',\n", + " 99: b'c',\n", + " 100: b'd',\n", + " 101: b'e',\n", + " 102: b'f',\n", + " 103: b'g',\n", + " 104: b'h',\n", + " 105: b'i',\n", + " 106: b'j',\n", + " 107: b'k',\n", + " 108: b'l',\n", + " 109: b'm',\n", + " 110: b'n',\n", + " 111: b'o',\n", + " 112: b'p',\n", + " 113: b'q',\n", + " 114: b'r',\n", + " 115: b's',\n", + " 116: b't',\n", + " 117: b'u',\n", + " 118: b'v',\n", + " 119: b'w',\n", + " 120: b'x',\n", + " 121: b'y',\n", + " 122: b'z',\n", + " 123: b'{',\n", + " 124: b'|',\n", + " 125: b'}',\n", + " 126: b'~',\n", + " 127: b'\\x7f',\n", + " 128: b'\\x80',\n", + " 129: b'\\x81',\n", + " 130: b'\\x82',\n", + " 131: b'\\x83',\n", + " 132: b'\\x84',\n", + " 133: b'\\x85',\n", + " 134: b'\\x86',\n", + " 135: b'\\x87',\n", + " 136: b'\\x88',\n", + " 137: b'\\x89',\n", + " 138: b'\\x8a',\n", + " 139: b'\\x8b',\n", + " 140: b'\\x8c',\n", + " 141: b'\\x8d',\n", + " 142: b'\\x8e',\n", + " 143: b'\\x8f',\n", + " 144: b'\\x90',\n", + " 145: b'\\x91',\n", + " 146: b'\\x92',\n", + " 147: b'\\x93',\n", + " 148: b'\\x94',\n", + " 149: b'\\x95',\n", + " 150: b'\\x96',\n", + " 151: b'\\x97',\n", + " 152: b'\\x98',\n", + " 153: b'\\x99',\n", + " 154: b'\\x9a',\n", + " 155: b'\\x9b',\n", + " 156: b'\\x9c',\n", + " 157: b'\\x9d',\n", + " 158: b'\\x9e',\n", + " 159: b'\\x9f',\n", + " 160: b'\\xa0',\n", + " 161: b'\\xa1',\n", + " 162: b'\\xa2',\n", + " 163: b'\\xa3',\n", + " 164: b'\\xa4',\n", + " 165: b'\\xa5',\n", + " 166: b'\\xa6',\n", + " 167: b'\\xa7',\n", + " 168: b'\\xa8',\n", + " 169: b'\\xa9',\n", + " 170: b'\\xaa',\n", + " 171: b'\\xab',\n", + " 172: b'\\xac',\n", + " 173: b'\\xad',\n", + " 174: b'\\xae',\n", + " 175: b'\\xaf',\n", + " 176: b'\\xb0',\n", + " 177: b'\\xb1',\n", + " 178: b'\\xb2',\n", + " 179: b'\\xb3',\n", + " 180: b'\\xb4',\n", + " 181: b'\\xb5',\n", + " 182: b'\\xb6',\n", + " 183: b'\\xb7',\n", + " 184: b'\\xb8',\n", + " 185: b'\\xb9',\n", + " 186: b'\\xba',\n", + " 187: b'\\xbb',\n", + " 188: b'\\xbc',\n", + " 189: b'\\xbd',\n", + " 190: b'\\xbe',\n", + " 191: b'\\xbf',\n", + " 192: b'\\xc0',\n", + " 193: b'\\xc1',\n", + " 194: b'\\xc2',\n", + " 195: b'\\xc3',\n", + " 196: b'\\xc4',\n", + " 197: b'\\xc5',\n", + " 198: b'\\xc6',\n", + " 199: b'\\xc7',\n", + " 200: b'\\xc8',\n", + " 201: b'\\xc9',\n", + " 202: b'\\xca',\n", + " 203: b'\\xcb',\n", + " 204: b'\\xcc',\n", + " 205: b'\\xcd',\n", + " 206: b'\\xce',\n", + " 207: b'\\xcf',\n", + " 208: b'\\xd0',\n", + " 209: b'\\xd1',\n", + " 210: b'\\xd2',\n", + " 211: b'\\xd3',\n", + " 212: b'\\xd4',\n", + " 213: b'\\xd5',\n", + " 214: b'\\xd6',\n", + " 215: b'\\xd7',\n", + " 216: b'\\xd8',\n", + " 217: b'\\xd9',\n", + " 218: b'\\xda',\n", + " 219: b'\\xdb',\n", + " 220: b'\\xdc',\n", + " 221: b'\\xdd',\n", + " 222: b'\\xde',\n", + " 223: b'\\xdf',\n", + " 224: b'\\xe0',\n", + " 225: b'\\xe1',\n", + " 226: b'\\xe2',\n", + " 227: b'\\xe3',\n", + " 228: b'\\xe4',\n", + " 229: b'\\xe5',\n", + " 230: b'\\xe6',\n", + " 231: b'\\xe7',\n", + " 232: b'\\xe8',\n", + " 233: b'\\xe9',\n", + " 234: b'\\xea',\n", + " 235: b'\\xeb',\n", + " 236: b'\\xec',\n", + " 237: b'\\xed',\n", + " 238: b'\\xee',\n", + " 239: b'\\xef',\n", + " 240: b'\\xf0',\n", + " 241: b'\\xf1',\n", + " 242: b'\\xf2',\n", + " 243: b'\\xf3',\n", + " 244: b'\\xf4',\n", + " 245: b'\\xf5',\n", + " 246: b'\\xf6',\n", + " 247: b'\\xf7',\n", + " 248: b'\\xf8',\n", + " 249: b'\\xf9',\n", + " 250: b'\\xfa',\n", + " 251: b'\\xfb',\n", + " 252: b'\\xfc',\n", + " 253: b'\\xfd',\n", + " 254: b'\\xfe',\n", + " 255: b'\\xff',\n", + " 256: b'e ',\n", + " 257: b't ',\n", + " 258: b's ',\n", + " 259: b'o ',\n", + " 260: b'n ',\n", + " 261: b'y ',\n", + " 262: b'th',\n", + " 263: b'd ',\n", + " 264: b'in',\n", + " 265: b'r ',\n", + " 266: b'ou',\n", + " 267: b'g ',\n", + " 268: b'an',\n", + " 269: b'm ',\n", + " 270: b'on',\n", + " 271: b'ing ',\n", + " 272: b'l ',\n", + " 273: b'h ',\n", + " 274: b'ha',\n", + " 275: b'ea',\n", + " 276: b'i ',\n", + " 277: b'er',\n", + " 278: b'is ',\n", + " 279: b'k ',\n", + " 280: b' ',\n", + " 281: b'a ',\n", + " 282: b'ar',\n", + " 283: b'st',\n", + " 284: b'al',\n", + " 285: b'f ',\n", + " 286: b'\\xf0\\x9f',\n", + " 287: b'or',\n", + " 288: b'you',\n", + " 289: b'er ',\n", + " 290: b'li',\n", + " 291: b'ow',\n", + " 292: b'en',\n", + " 293: b'on ',\n", + " 294: b'ed ',\n", + " 295: b'it ',\n", + " 296: b'ly ',\n", + " 297: b'be',\n", + " 298: b'you ',\n", + " 299: b'st ',\n", + " 300: b'p ',\n", + " 301: b'the ',\n", + " 302: b'? ',\n", + " 303: b'an ',\n", + " 304: b'ke ',\n", + " 305: b'to ',\n", + " 306: b'me ',\n", + " 307: b'ot ',\n", + " 308: b'ti',\n", + " 309: b'la',\n", + " 310: b'wa',\n", + " 311: b'ts ',\n", + " 312: b'es ',\n", + " 313: b'or ',\n", + " 314: b'ow ',\n", + " 315: b'at ',\n", + " 316: b'en ',\n", + " 317: b'I ',\n", + " 318: b'in ',\n", + " 319: b'} ',\n", + " 320: b'oo',\n", + " 321: b'ra',\n", + " 322: b're',\n", + " 323: b'hat ',\n", + " 324: b' ',\n", + " 325: b'wh',\n", + " 326: b'so ',\n", + " 327: b'ro',\n", + " 328: b'ic',\n", + " 329: b'and ',\n", + " 330: b've ',\n", + " 331: b'\\xe2\\xa0',\n", + " 332: b'bed',\n", + " 333: b'Em',\n", + " 334: b'ma',\n", + " 335: b'{Em',\n", + " 336: b'{Embed',\n", + " 337: b'et ',\n", + " 338: b'{Embed} ',\n", + " 339: b'like ',\n", + " 340: b'sh',\n", + " 341: b'se ',\n", + " 342: b'll ',\n", + " 343: b'br',\n", + " 344: b'not ',\n", + " 345: b'\\xf0\\x9f\\x98',\n", + " 346: b'id',\n", + " 347: b'bu',\n", + " 348: b'sa',\n", + " 349: b'le ',\n", + " 350: b'ch',\n", + " 351: b'da',\n", + " 352: b'ba',\n", + " 353: b'ig',\n", + " 354: b'tu',\n", + " 355: b'ust ',\n", + " 356: b'th ',\n", + " 357: b'that ',\n", + " 358: b'no ',\n", + " 359: b'im ',\n", + " 360: b'\\xe2\\xa0\\x80',\n", + " 361: b'ca',\n", + " 362: b'ne',\n", + " 363: b'what ',\n", + " 364: b'le',\n", + " 365: b'ld ',\n", + " 366: b'ch ',\n", + " 367: b'one ',\n", + " 368: b'\\xf0\\x9f\\x98\\xad',\n", + " 369: b'tt',\n", + " 370: b'wi',\n", + " 371: b'lo',\n", + " 372: b'be ',\n", + " 373: b'om',\n", + " 374: b'he ',\n", + " 375: b'0 ',\n", + " 376: b'my ',\n", + " 377: b'do ',\n", + " 378: b'for ',\n", + " 379: b'ac',\n", + " 380: b'di',\n", + " 381: b'\\xe2\\xa0\\x80\\xe2\\xa0\\x80',\n", + " 382: b'bro ',\n", + " 383: b'ho',\n", + " 384: b'of ',\n", + " 385: b'su',\n", + " 386: b'se',\n", + " 387: b'just ',\n", + " 388: b'are ',\n", + " 389: b'ga',\n", + " 390: b'we ',\n", + " 391: b'ir',\n", + " 392: b'ht ',\n", + " 393: b'pla',\n", + " 394: b'he',\n", + " 395: b'uh ',\n", + " 396: b'ta',\n", + " 397: b'ere ',\n", + " 398: b'\\xf0\\x9f\\x98\\xad ',\n", + " 399: b' ',\n", + " 400: b'ev',\n", + " 401: b'un',\n", + " 402: b'rea',\n", + " 403: b'all ',\n", + " 404: b'can ',\n", + " 405: b'te ',\n", + " 406: b'to',\n", + " 407: b'u ',\n", + " 408: b'ss ',\n", + " 409: b'tr',\n", + " 410: b'ould ',\n", + " 411: b'get ',\n", + " 412: b'ck ',\n", + " 413: b'have ',\n", + " 414: b'was ',\n", + " 415: b'now ',\n", + " 416: b'this ',\n", + " 417: b'. ',\n", + " 418: b'w ',\n", + " 419: b'do',\n", + " 420: b'te',\n", + " 421: b'mo',\n", + " 422: b'ally ',\n", + " 423: b'gu',\n", + " 424: b'ck',\n", + " 425: b'ff ',\n", + " 426: b'ent ',\n", + " 427: b'its ',\n", + " 428: b'thin',\n", + " 429: b'el ',\n", + " 430: b'out ',\n", + " 431: b'ey ',\n", + " 432: b'if ',\n", + " 433: b', ',\n", + " 434: b'op',\n", + " 435: b'ant ',\n", + " 436: b'si',\n", + " 437: b'ys ',\n", + " 438: b'fu',\n", + " 439: b'pi',\n", + " 440: b'me',\n", + " 441: b'min',\n", + " 442: b'go',\n", + " 443: b'ore ',\n", + " 444: b'ri',\n", + " 445: b' \\xf0\\x9f',\n", + " 446: b'ce ',\n", + " 447: b'ight ',\n", + " 448: b'go ',\n", + " 449: b'2 ',\n", + " 450: b'than ',\n", + " 451: b'don',\n", + " 452: b'wor',\n", + " 453: b'You',\n", + " 454: b'bo',\n", + " 455: b'ter',\n", + " 456: b'but ',\n", + " 457: b'ood ',\n", + " 458: b'with ',\n", + " 459: b'cu',\n", + " 460: b'sp',\n", + " 461: b'vi',\n", + " 462: b'mu',\n", + " 463: b'cti',\n", + " 464: b'\\xe2\\x80',\n", + " 465: b'du',\n", + " 466: b'ge ',\n", + " 467: b'how ',\n", + " 468: b'll',\n", + " 469: b'day ',\n", + " 470: b'yea',\n", + " 471: b'b ',\n", + " 472: b'pp',\n", + " 473: b'T ',\n", + " 474: b'1 ',\n", + " 475: b'E ',\n", + " 476: b're ',\n", + " 477: b'use ',\n", + " 478: b'think ',\n", + " 479: b'oun',\n", + " 480: b'lea',\n", + " 481: b'c ',\n", + " 482: b'we',\n", + " 483: b'ex',\n", + " 484: b\"'s \",\n", + " 485: b'Wh',\n", + " 486: b'al ',\n", + " 487: b'\\xe2\\xa3',\n", + " 488: b'dont ',\n", + " 489: b'ill ',\n", + " 490: b'ying ',\n", + " 491: b'ge',\n", + " 492: b'fa',\n", + " 493: b'hh ',\n", + " 494: b'oug',\n", + " 495: b'5 ',\n", + " 496: b'Er',\n", + " 497: b'why ',\n", + " 498: b'id ',\n", + " 499: b'You ',\n", + " 500: b'What ',\n", + " 501: b'na',\n", + " 502: b'con',\n", + " 503: b'po',\n", + " 504: b'thing ',\n", + " 505: b'pro',\n", + " 506: b'na ',\n", + " 507: b'got ',\n", + " 508: b'know ',\n", + " 509: b'ds ',\n", + " 510: b'good ',\n", + " 511: b'ter ',\n", + " 512: b'co',\n", + " 513: b'ab',\n", + " 514: b'\\xe2\\x80\\x99',\n", + " 515: b'gr',\n", + " 516: b'Erm ',\n", + " 517: b'ani',\n", + " 518: b'est',\n", + " 519: b'so',\n", + " 520: b'No ',\n", + " 521: b'ons',\n", + " 522: b'S ',\n", + " 523: b'king ',\n", + " 524: b' ',\n", + " 525: b'com',\n", + " 526: b'your ',\n", + " 527: b'it',\n", + " 528: b'So ',\n", + " 529: b'mm ',\n", + " 530: b'ut ',\n", + " 531: b'de ',\n", + " 532: b'\\xe2\\xa0\\x80\\xe2\\xa0\\x80\\xe2\\xa0\\x80\\xe2\\xa0\\x80',\n", + " 533: b'holy ',\n", + " 534: b'Rea',\n", + " 535: b'tf ',\n", + " 536: b'ver',\n", + " 537: b'sc',\n", + " 538: b'Ca',\n", + " 539: b'3 ',\n", + " 540: b'is',\n", + " 541: b'ec',\n", + " 542: b'ybe ',\n", + " 543: b'need ',\n", + " 544: b'ctions',\n", + " 545: b'es',\n", + " 546: b'wait ',\n", + " 547: b'{Rea',\n", + " 548: b'{Reactions',\n", + " 549: b'{Reactions} ',\n", + " 550: b'rn ',\n", + " 551: b'no',\n", + " 552: b'{Embed} {Embed} ',\n", + " 553: b'Th',\n", + " 554: b'then ',\n", + " 555: b'em ',\n", + " 556: b'when ',\n", + " 557: b'lu',\n", + " 558: b'ong ',\n", + " 559: b'qu',\n", + " 560: b\"I'\",\n", + " 561: b'ic ',\n", + " 562: b'son ',\n", + " 563: b'tuff ',\n", + " 564: b'nt ',\n", + " 565: b'ver ',\n", + " 566: b'mar',\n", + " 567: b'yes ',\n", + " 568: b'play ',\n", + " 569: b'ro ',\n", + " 570: b'end ',\n", + " 571: b'ther ',\n", + " 572: b'ough ',\n", + " 573: b'up ',\n", + " 574: b'stu',\n", + " 575: b'ru',\n", + " 576: b'ted ',\n", + " 577: b'x ',\n", + " 578: b'par',\n", + " 579: b'am ',\n", + " 580: b'Li',\n", + " 581: b'el',\n", + " 582: b': ',\n", + " 583: b'more ',\n", + " 584: b\"I'm \",\n", + " 585: b'even ',\n", + " 586: b'did ',\n", + " 587: b'dy ',\n", + " 588: b'ff',\n", + " 589: b'cause ',\n", + " 590: b'bi',\n", + " 591: b'tha',\n", + " 592: b'ep ',\n", + " 593: b'Y ',\n", + " 594: b'actu',\n", + " 595: b'low',\n", + " 596: b'gra',\n", + " 597: b'also ',\n", + " 598: b'Wa',\n", + " 599: b'oh ',\n", + " 600: b'tion ',\n", + " 601: b'should ',\n", + " 602: b'An',\n", + " 603: b'they ',\n", + " 604: b'say ',\n", + " 605: b'tch ',\n", + " 606: b'bad ',\n", + " 607: b'ya',\n", + " 608: b'any ',\n", + " 609: b'ano ',\n", + " 610: b'But ',\n", + " 611: b'ing',\n", + " 612: b'lly ',\n", + " 613: b' H',\n", + " 614: b'yeah ',\n", + " 615: b'har',\n", + " 616: b'for',\n", + " 617: b'by ',\n", + " 618: b'idk ',\n", + " 619: b'6 ',\n", + " 620: b'pa',\n", + " 621: b'ine ',\n", + " 622: b'rom ',\n", + " 623: b'game ',\n", + " 624: b'key ',\n", + " 625: b'ethan ',\n", + " 626: b'Cap ',\n", + " 627: b'time ',\n", + " 628: b'too ',\n", + " 629: b'4 ',\n", + " 630: b'about ',\n", + " 631: b'sk',\n", + " 632: b'de',\n", + " 633: b'ding ',\n", + " 634: b'gi',\n", + " 635: b'tter ',\n", + " 636: b'us',\n", + " 637: b'mb',\n", + " 638: b'D ',\n", + " 639: b'right ',\n", + " 640: b'ft ',\n", + " 641: b'who ',\n", + " 642: b'IN',\n", + " 643: b'aniel ',\n", + " 644: b'way ',\n", + " 645: b'mi',\n", + " 646: b'hou',\n", + " 647: b'TH',\n", + " 648: b'Ho',\n", + " 649: b'cap ',\n", + " 650: b'We ',\n", + " 651: b'shit ',\n", + " 652: b'Why ',\n", + " 653: b'ting ',\n", + " 654: b'there ',\n", + " 655: b'wan',\n", + " 656: b'from ',\n", + " 657: b'said ',\n", + " 658: b'ean ',\n", + " 659: b'Yes ',\n", + " 660: b'pre',\n", + " 661: b'7 ',\n", + " 662: b'pe',\n", + " 663: b'much ',\n", + " 664: b'cant ',\n", + " 665: b'da ',\n", + " 666: b'O ',\n", + " 667: b'How ',\n", + " 668: b'\\xf0\\x9f\\x9f',\n", + " 669: b'per',\n", + " 670: b'only ',\n", + " 671: b'\\xe2\\xac',\n", + " 672: b'maybe ',\n", + " 673: b'hat',\n", + " 674: b'ome',\n", + " 675: b'gon',\n", + " 676: b'actually ',\n", + " 677: b'ahh ',\n", + " 678: b'??',\n", + " 679: b'Ma',\n", + " 680: b'ks ',\n", + " 681: b'ear',\n", + " 682: b'sta',\n", + " 683: b'still ',\n", + " 684: b'\\xe2\\x96',\n", + " 685: b'va',\n", + " 686: b'see ',\n", + " 687: b'cus ',\n", + " 688: b'ke',\n", + " 689: b'as ',\n", + " 690: b'ound ',\n", + " 691: b'ran',\n", + " 692: b'And ',\n", + " 693: b'i think ',\n", + " 694: b'mon',\n", + " 695: b'mean ',\n", + " 696: b'10 ',\n", + " 697: b'{Reactions} \\xf0\\x9f',\n", + " 698: b'cra',\n", + " 699: b'\" ',\n", + " 700: b'own ',\n", + " 701: b'..',\n", + " 702: b'\\x92\\x80',\n", + " 703: b'gonna ',\n", + " 704: b\"'t \",\n", + " 705: b'ent',\n", + " 706: b'z ',\n", + " 707: b'fir',\n", + " 708: b'ss',\n", + " 709: b'zy ',\n", + " 710: b'ur',\n", + " 711: b'yo ',\n", + " 712: b'im',\n", + " 713: b'would ',\n", + " 714: b'guys ',\n", + " 715: b'movi',\n", + " 716: b'Huh ',\n", + " 717: b'Holy ',\n", + " 718: b'ope ',\n", + " 719: b'af',\n", + " 720: b'all',\n", + " 721: b'ded ',\n", + " 722: b'though ',\n", + " 723: b'- ',\n", + " 724: b'Ok ',\n", + " 725: b'jo',\n", + " 726: b'R ',\n", + " 727: b'8 ',\n", + " 728: b'\\xe2\\xac\\x9b',\n", + " 729: b'going ',\n", + " 730: b'K ',\n", + " 731: b'star',\n", + " 732: b'us ',\n", + " 733: b'guy ',\n", + " 734: b'ever',\n", + " 735: b'ble ',\n", + " 736: b'In',\n", + " 737: b'ur ',\n", + " 738: b'yaan ',\n", + " 739: b'make ',\n", + " 740: b'ool ',\n", + " 741: b'cking ',\n", + " 742: b'huh ',\n", + " 743: b'better ',\n", + " 744: b'loo',\n", + " 745: b'Id',\n", + " 746: b'il',\n", + " 747: b'Wait ',\n", + " 748: b'AN',\n", + " 749: b'any',\n", + " 750: b'! ',\n", + " 751: b'00 ',\n", + " 752: b'will ',\n", + " 753: b'the',\n", + " 754: b'\\xef\\xb8',\n", + " 755: b'\\xef\\xb8\\x8f',\n", + " 756: b'ard ',\n", + " 757: b' ',\n", + " 758: b'did',\n", + " 759: b'ers ',\n", + " 760: b'lowkey ',\n", + " 761: b'ok ',\n", + " 762: b'p on ',\n", + " 763: b'fr',\n", + " 764: b'could ',\n", + " 765: b'The ',\n", + " 766: b'him ',\n", + " 767: b'now',\n", + " 768: b'\\xf0\\x9f\\x9f\\xa9',\n", + " 769: b'movie ',\n", + " 770: b'want ',\n", + " 771: b'blo',\n", + " 772: b'pl',\n", + " 773: b'pe ',\n", + " 774: b'ide',\n", + " 775: b'Idk ',\n", + " 776: b'thats ',\n", + " 777: b'Al',\n", + " 778: b'Like ',\n", + " 779: b'Not ',\n", + " 780: b'after ',\n", + " 781: b'ever ',\n", + " 782: b'tting ',\n", + " 783: b'9 ',\n", + " 784: b'grou',\n", + " 785: b'ass ',\n", + " 786: b'pu',\n", + " 787: b'fe',\n", + " 788: b'Its ',\n", + " 789: b'sm',\n", + " 790: b'work ',\n", + " 791: b'tea',\n", + " 792: b'hmm ',\n", + " 793: b'piano ',\n", + " 794: b'really ',\n", + " 795: b'xt ',\n", + " 796: b'30 ',\n", + " 797: b'\\xe2\\x95',\n", + " 798: b'Is ',\n", + " 799: b'other ',\n", + " 800: b'first ',\n", + " 801: b'bro',\n", + " 802: b'ep',\n", + " 803: b'eat ',\n", + " 804: b'in the ',\n", + " 805: b'\\xe2\\x80\\x99m ',\n", + " 806: b'lf ',\n", + " 807: b'fin',\n", + " 808: b'ear ',\n", + " 809: b'man ',\n", + " 810: b'tal',\n", + " 811: b'G ',\n", + " 812: b'ha ',\n", + " 813: b'ed to ',\n", + " 814: b'ay ',\n", + " 815: b'has ',\n", + " 816: b'hop on ',\n", + " 817: b'tor',\n", + " 818: b'doing ',\n", + " 819: b'int ',\n", + " 820: b'are you ',\n", + " 821: b'Oh ',\n", + " 822: b'sch',\n", + " 823: b'as',\n", + " 824: b'... ',\n", + " 825: b'play',\n", + " 826: b'i can ',\n", + " 827: b'kin',\n", + " 828: b'some',\n", + " 829: b'minu',\n", + " 830: b'ms ',\n", + " 831: b'terally ',\n", + " 832: b'ment ',\n", + " 833: b'ed',\n", + " 834: b'chan',\n", + " 835: b'**',\n", + " 836: b'\\x92\\x80 ',\n", + " 837: b'where ',\n", + " 838: b'ig ',\n", + " 839: b'mp',\n", + " 840: b'hh',\n", + " 841: b'playing ',\n", + " 842: b'der ',\n", + " 843: b'He',\n", + " 844: b'can',\n", + " 845: b'ons ',\n", + " 846: b'sure ',\n", + " 847: b'ph',\n", + " 848: b'i have ',\n", + " 849: b'ad',\n", + " 850: b'whi',\n", + " 851: b'sy ',\n", + " 852: b'heem ',\n", + " 853: b'his ',\n", + " 854: b') ',\n", + " 855: b'sh ',\n", + " 856: b'rs ',\n", + " 857: b'terest',\n", + " 858: b'ople ',\n", + " 859: b'fucking ',\n", + " 860: b'ton ',\n", + " 861: b'W ',\n", + " 862: b'igh',\n", + " 863: b'20',\n", + " 864: b'M ',\n", + " 865: b'dam',\n", + " 866: b'Or ',\n", + " 867: b'ence ',\n", + " 868: b'ead ',\n", + " 869: b'same ',\n", + " 870: b'It ',\n", + " 871: b'wanna ',\n", + " 872: b'vro ',\n", + " 873: b'ON',\n", + " 874: b'wee',\n", + " 875: b'lun',\n", + " 876: b'fi',\n", + " 877: b'\\xe2\\xa2',\n", + " 878: b'tra',\n", + " 879: b'Lo',\n", + " 880: b'buy ',\n", + " 881: b'ice ',\n", + " 882: b'might ',\n", + " 883: b'Hmm ',\n", + " 884: b'ans ',\n", + " 885: b'erm ',\n", + " 886: b'eak ',\n", + " 887: b'\\xf0\\x9f\\x98\\xad\\xf0\\x9f\\x98\\xad',\n", + " 888: b'prob ',\n", + " 889: b'hehe ',\n", + " 890: b'people ',\n", + " 891: b'vide',\n", + " 892: b'cla',\n", + " 893: b'kinda ',\n", + " 894: b'Daniel ',\n", + " 895: b'ning ',\n", + " 896: b'fun',\n", + " 897: b'i dont ',\n", + " 898: b'ry ',\n", + " 899: b'mes ',\n", + " 900: b'rr ',\n", + " 901: b'uh',\n", + " 902: b'lar',\n", + " 903: b'fore ',\n", + " 904: b'ty ',\n", + " 905: b'ju',\n", + " 906: b'tes ',\n", + " 907: b'back ',\n", + " 908: b'mus',\n", + " 909: b'\\xe2\\xa0\\x80\\xe2\\xa0\\x80\\xe2\\xa0\\x80\\xe2\\xa0\\x80\\xe2\\xa0\\x80\\xe2\\xa0\\x80\\xe2\\xa0\\x80\\xe2\\xa0\\x80',\n", + " 910: b'wtf ',\n", + " 911: b'some ',\n", + " 912: b'crazy ',\n", + " 913: b'call ',\n", + " 914: b'happ',\n", + " 915: b'H ',\n", + " 916: b'ir ',\n", + " 917: b'ird ',\n", + " 918: b'does ',\n", + " 919: b'Im ',\n", + " 920: b'marcus ',\n", + " 921: b'Maybe ',\n", + " 922: b'literally ',\n", + " 923: b'didnt ',\n", + " 924: b'tch',\n", + " 925: b'LL',\n", + " 926: b'tm',\n", + " 927: b'Also ',\n", + " 928: b'coo',\n", + " 929: b'down ',\n", + " 930: b'teresting ',\n", + " 931: b'dum ',\n", + " 932: b'fo',\n", + " 933: b'ick ',\n", + " 934: b'ked ',\n", + " 935: b'before ',\n", + " 936: b'ways ',\n", + " 937: b'hard ',\n", + " 938: b'year ',\n", + " 939: b'ty',\n", + " 940: b'were ',\n", + " 941: b'hats ',\n", + " 942: b'ine',\n", + " 943: b'ball ',\n", + " 944: b'bad',\n", + " 945: b'ents ',\n", + " 946: b'very ',\n", + " 947: b'tar',\n", + " 948: b'dle ',\n", + " 949: b'The',\n", + " 950: b'My ',\n", + " 951: b'10',\n", + " 952: b'ort ',\n", + " 953: b'?? ',\n", + " 954: b'cor',\n", + " 955: b'v ',\n", + " 956: b'watch ',\n", + " 957: b'He ',\n", + " 958: b'der',\n", + " 959: b'Ethan ',\n", + " 960: b'ad ',\n", + " 961: b'ster ',\n", + " 962: b'inst',\n", + " 963: b'This ',\n", + " 964: b'come ',\n", + " 965: b'our ',\n", + " 966: b'vo',\n", + " 967: b'fuck ',\n", + " 968: b'car',\n", + " 969: b'cou',\n", + " 970: b'Pi',\n", + " 971: b'\\xe2\\xa3\\xbf',\n", + " 972: b'them ',\n", + " 973: b'had ',\n", + " 974: b'ps ',\n", + " 975: b'OU',\n", + " 976: b'han',\n", + " 977: b'been ',\n", + " 978: b'ki',\n", + " 979: b'lig',\n", + " 980: b'jj',\n", + " 981: b' S',\n", + " 982: b'sw',\n", + " 983: b'ayaan ',\n", + " 984: b'bab',\n", + " 985: b'tw ',\n", + " 986: b'Bro ',\n", + " 987: b'fri',\n", + " 988: b'ich ',\n", + " 989: b'ze ',\n", + " 990: b'\\xe2\\x80\\x99s ',\n", + " 991: b'every',\n", + " 992: b'ye ',\n", + " 993: b'gay ',\n", + " 994: b'never ',\n", + " 995: b'gs ',\n", + " 996: b'i was ',\n", + " 997: b'of the ',\n", + " 998: b'If ',\n", + " 999: b'ler ',\n", + " ...}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "vocab = tokenizer.vocab\n", "vocab" @@ -88,31 +1118,42 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[497, 820, 389, 121]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "tokenizer.encode(\"Salam labas\")" + "tokenizer.encode(\"why are you gay\")" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'Salam labas'" + "'why are you gay'" ] }, - "execution_count": 9, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "tokenizer.decode([702, 310, 346, 115])" + "tokenizer.decode([497, 820, 389, 121])" ] }, { @@ -124,7 +1165,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -147,9 +1188,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "411925" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "len(tokenizer.encode(text_sequence))" ] @@ -163,7 +1215,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -173,7 +1225,7 @@ ], "metadata": { "kernelspec": { - "display_name": "vincent", + "display_name": "torch-rocm", "language": "python", "name": "python3" }, @@ -187,7 +1239,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.10" + "version": "3.10.20" } }, "nbformat": 4, diff --git a/notebooks/3_TransformerModel.ipynb b/notebooks/3_TransformerModel.ipynb index b4131ae..7163df4 100644 --- a/notebooks/3_TransformerModel.ipynb +++ b/notebooks/3_TransformerModel.ipynb @@ -9,7 +9,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -19,7 +19,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -31,7 +31,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -138,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -163,7 +163,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -214,7 +214,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -233,6 +233,13 @@ " return out" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "metadata": {}, @@ -242,7 +249,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -289,7 +296,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -388,18 +395,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "11.532552 M parameters\n" + "11.53409 M parameters\n" ] } ], "source": [ + "\n", + "\n", "model = GPTLanguageModel()\n", "model = model.to(device)\n", "# print the number of parameters in the model\n", @@ -408,14 +417,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "torch.Size([1, 6, 1032]) None\n" + "torch.Size([1, 6, 1034]) None\n" ] } ], @@ -438,14 +447,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "├─ token_embedding_table: Embedding (396,288 parameters)\n", + "├─ token_embedding_table: Embedding (397,056 parameters)\n", "├─ position_embedding_table: Embedding (98,304 parameters)\n", "├─ blocks: Sequential (10,639,872 parameters)\n", "│ ├─ 0: Block (1,773,312 parameters)\n", @@ -707,7 +716,7 @@ "│ │ ├─ layer_norm_1: LayerNorm (768 parameters)\n", "│ │ ├─ layer_norm_2: LayerNorm (768 parameters)\n", "├─ final_layer_norm: LayerNorm (768 parameters)\n", - "├─ final_linear_layer: Linear (397,320 parameters)\n" + "├─ final_linear_layer: Linear (398,090 parameters)\n" ] } ], @@ -727,398 +736,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [ { "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "Layer Name", - "rawType": "object", - "type": "string" - }, - { - "name": "Type", - "rawType": "object", - "type": "string" - }, - { - "name": "Parameters", - "rawType": "int64", - "type": "integer" - }, - { - "name": "Trainable", - "rawType": "int64", - "type": "integer" - } - ], - "conversionMethod": "pd.DataFrame", - "ref": "41f6a192-649f-4276-8acd-5e81c92f3aa9", - "rows": [ - [ - "0", - "token_embedding_table", - "Embedding", - "396288", - "396288" - ], - [ - "1", - "position_embedding_table", - "Embedding", - "98304", - "98304" - ], - [ - "2", - "blocks.0.self_attention.heads.0.key", - "Linear", - "24576", - "24576" - ], - [ - "3", - "blocks.0.self_attention.heads.0.query", - "Linear", - "24576", - "24576" - ], - [ - "4", - "blocks.0.self_attention.heads.0.value", - "Linear", - "24576", - "24576" - ], - [ - "5", - "blocks.0.self_attention.heads.0.dropout", - "Dropout", - "0", - "0" - ], - [ - "6", - "blocks.0.self_attention.heads.1.key", - "Linear", - "24576", - "24576" - ], - [ - "7", - "blocks.0.self_attention.heads.1.query", - "Linear", - "24576", - "24576" - ], - [ - "8", - "blocks.0.self_attention.heads.1.value", - "Linear", - "24576", - "24576" - ], - [ - "9", - "blocks.0.self_attention.heads.1.dropout", - "Dropout", - "0", - "0" - ], - [ - "10", - "blocks.0.self_attention.heads.2.key", - "Linear", - "24576", - "24576" - ], - [ - "11", - "blocks.0.self_attention.heads.2.query", - "Linear", - "24576", - "24576" - ], - [ - "12", - "blocks.0.self_attention.heads.2.value", - "Linear", - "24576", - "24576" - ], - [ - "13", - "blocks.0.self_attention.heads.2.dropout", - "Dropout", - "0", - "0" - ], - [ - "14", - "blocks.0.self_attention.heads.3.key", - "Linear", - "24576", - "24576" - ], - [ - "15", - "blocks.0.self_attention.heads.3.query", - "Linear", - "24576", - "24576" - ], - [ - "16", - "blocks.0.self_attention.heads.3.value", - "Linear", - "24576", - "24576" - ], - [ - "17", - "blocks.0.self_attention.heads.3.dropout", - "Dropout", - "0", - "0" - ], - [ - "18", - "blocks.0.self_attention.heads.4.key", - "Linear", - "24576", - "24576" - ], - [ - "19", - "blocks.0.self_attention.heads.4.query", - "Linear", - "24576", - "24576" - ], - [ - "20", - "blocks.0.self_attention.heads.4.value", - "Linear", - "24576", - "24576" - ], - [ - "21", - "blocks.0.self_attention.heads.4.dropout", - "Dropout", - "0", - "0" - ], - [ - "22", - "blocks.0.self_attention.heads.5.key", - "Linear", - "24576", - "24576" - ], - [ - "23", - "blocks.0.self_attention.heads.5.query", - "Linear", - "24576", - "24576" - ], - [ - "24", - "blocks.0.self_attention.heads.5.value", - "Linear", - "24576", - "24576" - ], - [ - "25", - "blocks.0.self_attention.heads.5.dropout", - "Dropout", - "0", - "0" - ], - [ - "26", - "blocks.0.self_attention.projection", - "Linear", - "147840", - "147840" - ], - [ - "27", - "blocks.0.self_attention.dropout", - "Dropout", - "0", - "0" - ], - [ - "28", - "blocks.0.feed_forward.net.0", - "Linear", - "591360", - "591360" - ], - [ - "29", - "blocks.0.feed_forward.net.1", - "ReLU", - "0", - "0" - ], - [ - "30", - "blocks.0.feed_forward.net.2", - "Linear", - "590208", - "590208" - ], - [ - "31", - "blocks.0.feed_forward.net.3", - "Dropout", - "0", - "0" - ], - [ - "32", - "blocks.0.layer_norm_1", - "LayerNorm", - "768", - "768" - ], - [ - "33", - "blocks.0.layer_norm_2", - "LayerNorm", - "768", - "768" - ], - [ - "34", - "blocks.1.self_attention.heads.0.key", - "Linear", - "24576", - "24576" - ], - [ - "35", - "blocks.1.self_attention.heads.0.query", - "Linear", - "24576", - "24576" - ], - [ - "36", - "blocks.1.self_attention.heads.0.value", - "Linear", - "24576", - "24576" - ], - [ - "37", - "blocks.1.self_attention.heads.0.dropout", - "Dropout", - "0", - "0" - ], - [ - "38", - "blocks.1.self_attention.heads.1.key", - "Linear", - "24576", - "24576" - ], - [ - "39", - "blocks.1.self_attention.heads.1.query", - "Linear", - "24576", - "24576" - ], - [ - "40", - "blocks.1.self_attention.heads.1.value", - "Linear", - "24576", - "24576" - ], - [ - "41", - "blocks.1.self_attention.heads.1.dropout", - "Dropout", - "0", - "0" - ], - [ - "42", - "blocks.1.self_attention.heads.2.key", - "Linear", - "24576", - "24576" - ], - [ - "43", - "blocks.1.self_attention.heads.2.query", - "Linear", - "24576", - "24576" - ], - [ - "44", - "blocks.1.self_attention.heads.2.value", - "Linear", - "24576", - "24576" - ], - [ - "45", - "blocks.1.self_attention.heads.2.dropout", - "Dropout", - "0", - "0" - ], - [ - "46", - "blocks.1.self_attention.heads.3.key", - "Linear", - "24576", - "24576" - ], - [ - "47", - "blocks.1.self_attention.heads.3.query", - "Linear", - "24576", - "24576" - ], - [ - "48", - "blocks.1.self_attention.heads.3.value", - "Linear", - "24576", - "24576" - ], - [ - "49", - "blocks.1.self_attention.heads.3.dropout", - "Dropout", - "0", - "0" - ] - ], - "shape": { - "columns": 4, - "rows": 196 - } - }, "text/html": [ "
\n", "