From 88e090acd825da65c0fbeb650b0810ca4cc58c3e Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 26 Nov 2025 12:09:53 +0800 Subject: [PATCH 1/3] feat: add 54 agent special tokens for conversational AI and reasoning Extends cl100k_base and o200k_base tokenizers with specialized tokens for building chat models, reasoning systems, and autonomous agents. Token IDs are allocated in non-conflicting ranges (cl100k: 100277-100330, o200k: 200019-200072). Token categories: - Conversation structure (system, user, assistant, ChatML delimiters) - Chain-of-Thought reasoning (think blocks for DeepSeek/o1-style reasoning) - ReAct agent loops (plan, step, act, observe) - Tool/function calling (function, result, error with retry handling) - Code execution (Jupyter-style code/output/language blocks) - RAG citations (context, quote, cite, source attribution) - Memory/state (memory store and recall for stateful agents) - Control flow (pad, stop, sep) - Multimodal (image, audio, video placeholders) - Document structure (title, section, summary for semantic parsing) Implementation: - Rust constants with comprehensive documentation in src/core/tokenizer.rs - Python bindings expose AgentTokens classes with token ID mappings - Full module integration across core, Python bindings, and public API - Updated README with feature descriptions and supported model details - New docs/special_tokens.md with design rationale and usage examples --- README.md | 84 ++-- benchmarks/benchmark.py | 20 +- docs/special_tokens.md | 612 ++++++++++++++++++++++++++++ python/splintr/__init__.py | 32 +- src/core/mod.rs | 5 +- src/core/tokenizer.rs | 541 +++++++++++++++++++++++++ src/lib.rs | 5 +- src/python/bindings.rs | 801 ++++++++++++++++++++++++++++++++++++- src/python/mod.rs | 2 +- 9 files changed, 2059 insertions(+), 43 deletions(-) create mode 100644 docs/special_tokens.md diff --git a/README.md b/README.md index 4497a23..d963504 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# splintr +# Splintr [![Crates.io](https://img.shields.io/crates/v/splintr.svg)](https://crates.io/crates/splintr) [![PyPI](https://img.shields.io/pypi/v/splintr-rs.svg)](https://pypi.org/project/splintr-rs/) @@ -8,7 +8,7 @@ A high-performance BPE tokenizer implemented in Rust with Python bindings, desig ## Features -splintr implements several optimizations that make tokenization faster and more efficient: +Splintr implements several optimizations that make tokenization faster and more efficient: - **PCRE2 with JIT compilation**: Uses PCRE2's just-in-time compilation for regex matching, providing 2-4x speedup over fancy-regex on pattern matching operations - **Rayon parallelism**: Leverages multiple CPU cores for encoding batches of text and individual regex chunks within each text @@ -17,6 +17,7 @@ splintr implements several optimizations that make tokenization faster and more - **Aho-Corasick for special tokens**: Employs the Aho-Corasick algorithm for fast multi-pattern matching of special tokens, avoiding regex alternation overhead - **LRU cache**: Caches frequently encoded text chunks to avoid redundant BPE encoding operations - **UTF-8 streaming decoder**: Safely handles token-by-token decoding for LLM output, buffering incomplete UTF-8 sequences across token boundaries +- **Extended agent tokens**: 54 special tokens for chat models, Chain-of-Thought reasoning, ReAct agents, tool calling, RAG citations, and multimodal applications (see [Special Tokens](docs/special_tokens.md)) ## Installation @@ -192,6 +193,7 @@ print(decoder.flush()) ``` This approach ensures that: + 1. Users see text as soon as complete characters are available 2. Multi-byte Unicode characters display correctly 3. No corruption occurs at token boundaries @@ -204,27 +206,28 @@ Benchmarks performed on Linux (6.16.8-arch3-1) with 24 CPU cores, comparing spli Performance on various text types: -| Content Type | Size | splintr (ms) | tiktoken (ms) | Speedup | -|--------------|------|--------------|---------------|---------| -| Long English | 450,000 chars | 7.94 | 19.91 | **2.5x** | -| Python Code | 59,200 chars | 1.67 | 5.90 | **3.5x** | -| JSON | 29,000 chars | 1.20 | 2.76 | **2.3x** | -| Numbers | 55,000 chars | 2.27 | 6.09 | **2.7x** | -| Whitespace-heavy | 50,000 chars | 1.36 | 4.91 | **3.6x** | -| Chinese | 11,500 chars | 1.09 | 1.45 | **1.3x** | +| Content Type | Size | splintr (ms) | tiktoken (ms) | Speedup | +| ---------------- | ------------- | ------------ | ------------- | -------- | +| Long English | 450,000 chars | 7.94 | 19.91 | **2.5x** | +| Python Code | 59,200 chars | 1.67 | 5.90 | **3.5x** | +| JSON | 29,000 chars | 1.20 | 2.76 | **2.3x** | +| Numbers | 55,000 chars | 2.27 | 6.09 | **2.7x** | +| Whitespace-heavy | 50,000 chars | 1.36 | 4.91 | **3.6x** | +| Chinese | 11,500 chars | 1.09 | 1.45 | **1.3x** | ### Batch Encoding Batch operations show significant speedup through parallelism: -| Configuration | splintr parallel (ms) | tiktoken (ms) | Speedup vs tiktoken | -|---------------|----------------------|---------------|---------------------| -| 10 × 1,000 chars | 0.25 | 0.48 | **1.9x** | -| 100 × 1,000 chars | 1.11 | 4.66 | **4.2x** | -| 1,000 × 100 chars | 1.42 | 6.95 | **4.9x** | -| 100 × 10,000 chars | 8.24 | 45.72 | **5.5x** | +| Configuration | splintr parallel (ms) | tiktoken (ms) | Speedup vs tiktoken | +| ------------------ | --------------------- | ------------- | ------------------- | +| 10 × 1,000 chars | 0.25 | 0.48 | **1.9x** | +| 100 × 1,000 chars | 1.11 | 4.66 | **4.2x** | +| 1,000 × 100 chars | 1.42 | 6.95 | **4.9x** | +| 100 × 10,000 chars | 8.24 | 45.72 | **5.5x** | **Parallel speedup within splintr:** + - 100 × 1,000 chars: 8.6x faster (parallel vs sequential) - 1,000 × 100 chars: 16.8x faster (parallel vs sequential) @@ -250,6 +253,7 @@ cat results/my_benchmark.md ``` The benchmark suite tests: + - Single text encoding across various content types (English, code, multilingual, etc.) - Batch encoding with different batch sizes and text lengths - Streaming decoder performance @@ -259,22 +263,55 @@ You can customize the benchmark by modifying `benchmark.py` or adding your own t ## Supported Models -| Model | Use Case | Vocabulary Size | Special Tokens | Import Constant | -|-------|----------|----------------|----------------|-----------------| -| `cl100k_base` | GPT-4, GPT-3.5-turbo | ~100,000 | 5 | `CL100K_BASE_PATTERN` | -| `o200k_base` | GPT-4o | ~200,000 | 2 | `O200K_BASE_PATTERN` | +| Model | Use Case | Vocabulary Size | Special Tokens | Import Constant | +| ------------- | -------------------- | --------------- | -------------- | --------------------- | +| `cl100k_base` | GPT-4, GPT-3.5-turbo | ~100,000 | 5 + 54 agent | `CL100K_BASE_PATTERN` | +| `o200k_base` | GPT-4o | ~200,000 | 2 + 54 agent | `O200K_BASE_PATTERN` | -**Special tokens:** +**OpenAI standard tokens:** - **cl100k_base**: `<|endoftext|>`, `<|fim_prefix|>`, `<|fim_middle|>`, `<|fim_suffix|>`, `<|endofprompt|>` - **o200k_base**: `<|endoftext|>`, `<|endofprompt|>` +**Agent tokens (54 per model):** + +Splintr extends both vocabularies with tokens for building agent systems. See [docs/special_tokens.md](docs/special_tokens.md) for complete documentation. + +```python +from splintr import Tokenizer, CL100K_AGENT_TOKENS + +tokenizer = Tokenizer.from_pretrained("cl100k_base") + +# Encode with special tokens +text = "<|think|>Let me reason...<|/think|>The answer is 42." +tokens = tokenizer.encode_with_special(text) + +# Access token IDs programmatically +print(CL100K_AGENT_TOKENS.THINK) # 100282 +print(CL100K_AGENT_TOKENS.FUNCTION) # 100292 +``` + +| Category | Tokens | Purpose | +| ------------ | --------------------------------------------------- | -------------------------- | +| Conversation | `system`, `user`, `assistant`, `im_start`, `im_end` | ChatML format | +| Thinking | `think` | Chain-of-Thought reasoning | +| ReAct | `plan`, `step`, `act`, `observe` | Agent action loops | +| Tools | `function`, `result`, `error` | Function calling | +| Code | `code`, `output`, `lang` | Code execution | +| RAG | `context`, `quote`, `cite`, `source` | Citations | +| Memory | `memory`, `recall` | State persistence | +| Control | `pad`, `stop`, `sep` | Sequence control | +| Multimodal | `image`, `audio`, `video` | Non-text content | +| Document | `title`, `section`, `summary` | Structured docs | + ## Use Cases -splintr is designed for: +Splintr is designed for: - **LLM applications**: Tokenizing prompts and streaming decoder for real-time output display +- **Agent systems**: Building ReAct agents, tool-calling systems, and Chain-of-Thought reasoning - **Training pipelines**: Fast batch encoding of large datasets for model training +- **RAG applications**: Structured context injection with citation support - **Token counting**: Estimating API costs or enforcing token limits - **Text preprocessing**: Converting text to tokens for embedding models or other NLP tasks @@ -321,7 +358,8 @@ This project is licensed under the MIT License - see the LICENSE file for detail ## Acknowledgments -splintr builds upon concepts from: +Splintr builds upon concepts from: + - [tiktoken](https://github.com/openai/tiktoken) - OpenAI's reference BPE tokenizer - [tokenizers](https://github.com/huggingface/tokenizers) - Hugging Face's tokenization library diff --git a/benchmarks/benchmark.py b/benchmarks/benchmark.py index 2ae17e0..e5409d0 100755 --- a/benchmarks/benchmark.py +++ b/benchmarks/benchmark.py @@ -194,7 +194,7 @@ def run_single_text_benchmarks( results["single_text"][name] = {"chars": len(text), "bytes": data_size} # Splintr - print(" splintr:") + print(" Splintr:") result = benchmark( lambda t=text: splintr_enc.encode(t), iterations=iterations, @@ -221,7 +221,7 @@ def run_single_text_benchmarks( speedup = result.mean_ms / splintr_result.mean_ms results["single_text"][name]["speedup"] = speedup - print(f" >>> splintr is {speedup:.2f}x {'faster' if speedup > 1 else 'slower'}") + print(f" >>> Splintr is {speedup:.2f}x {'faster' if speedup > 1 else 'slower'}") return results @@ -279,7 +279,7 @@ def run_batch_benchmarks( splintr_batch = result # Splintr sequential (for comparison) - print(" splintr sequential:") + print(" Splintr sequential:") result = benchmark( lambda: [splintr_enc.encode(t) for t in texts], iterations=iterations, @@ -307,7 +307,7 @@ def run_batch_benchmarks( results["batch"][config_name]["tiktoken_sequential"] = asdict(result) speedup = result.mean_ms / splintr_batch.mean_ms results["batch"][config_name]["vs_tiktoken_speedup"] = speedup - print(f" >>> splintr batch is {speedup:.2f}x faster than tiktoken sequential") + print(f" >>> Splintr batch is {speedup:.2f}x faster than tiktoken sequential") return results @@ -466,7 +466,7 @@ def run_special_tokens_benchmark( results["special_tokens"]["chars"] = len(text_with_special) results["special_tokens"]["bytes"] = data_size - print(" splintr encode_with_special:") + print(" Splintr encode_with_special:") result = benchmark( lambda: splintr_enc.encode_with_special(text_with_special), iterations=iterations, @@ -490,13 +490,13 @@ def run_special_tokens_benchmark( speedup = result.mean_ms / splintr_result.mean_ms results["special_tokens"]["speedup"] = speedup - print(f" >>> splintr is {speedup:.2f}x {'faster' if speedup > 1 else 'slower'}") + print(f" >>> Splintr is {speedup:.2f}x {'faster' if speedup > 1 else 'slower'}") return results def run_correctness_check(splintr_enc, tiktoken_enc) -> dict: - """Verify splintr produces identical output to tiktoken.""" + """Verify Splintr produces identical output to tiktoken.""" results = {"correctness": {"tests": [], "all_passed": False}} print("\n" + "=" * 70) @@ -537,7 +537,7 @@ def run_correctness_check(splintr_enc, tiktoken_enc) -> dict: }) if not match: - print(f" splintr: {splintr_tokens}") + print(f" Splintr: {splintr_tokens}") print(f" tiktoken: {tiktoken_tokens}") print("-" * 50) @@ -606,7 +606,7 @@ def generate_summary(results: dict) -> str: # Batch results if "batch" in results: lines.append("## Batch Encoding\n") - lines.append("| Config | splintr parallel (ms) | splintr seq (ms) | tiktoken (ms) | Parallel Speedup | vs tiktoken |") + lines.append("| Config | Splintr parallel (ms) | Splintr seq (ms) | Tiktoken (ms) | Parallel Speedup | vs Tiktoken |") lines.append("|--------|----------------------|------------------|---------------|------------------|-------------|") for config, data in results["batch"].items(): @@ -691,7 +691,7 @@ def main(): # Load tokenizers print(f"\nLoading tokenizers (model: {args.model})...") splintr_enc = SplintrTokenizer.from_pretrained(args.model) - print(f" splintr: {splintr_enc}") + print(f" Splintr: {splintr_enc}") tiktoken_enc = None if args.compare or args.correctness_only: diff --git a/docs/special_tokens.md b/docs/special_tokens.md new file mode 100644 index 0000000..3c8871d --- /dev/null +++ b/docs/special_tokens.md @@ -0,0 +1,612 @@ +# Special Tokens Reference + +This document describes the special tokens available in Splintr's `cl100k_base` and `o200k_base` tokenizers, including the extended agent token vocabulary. + +## Table of Contents + +- [Overview](#overview) +- [Design Rationale](#design-rationale) +- [Token ID Allocation](#token-id-allocation) +- [OpenAI Standard Tokens](#openai-standard-tokens) +- [Agent Token Categories](#agent-token-categories) + - [1. Conversation Structure](#1-conversation-structure) + - [2. Reasoning / Chain-of-Thought](#2-reasoning--chain-of-thought) + - [3. ReAct Agent Loop](#3-react-agent-loop) + - [4. Tool / Function Calling](#4-tool--function-calling) + - [5. Code Execution](#5-code-execution) + - [6. RAG / Citations](#6-rag--citations) + - [7. Memory / State](#7-memory--state) + - [8. Control Tokens](#8-control-tokens) + - [9. Multimodal](#9-multimodal) + - [10. Document Structure](#10-document-structure) +- [Usage Examples](#usage-examples) +- [Python API Reference](#python-api-reference) +- [Rust API Reference](#rust-api-reference) + +--- + +## Overview + +Splintr extends the standard OpenAI tokenizer vocabularies with **54 additional special tokens** designed for building modern AI agent systems. These tokens provide semantic structure for: + +- Multi-turn chat conversations (ChatML format) +- Chain-of-Thought reasoning (System 2 thinking) +- ReAct-style agent loops (Reason + Act) +- Tool/function calling with error handling +- Code execution environments +- Retrieval-Augmented Generation (RAG) with citations +- Long-term memory and state persistence +- Multimodal content placeholders +- Structured document parsing + +--- + +## Design Rationale + +### Why Special Tokens? + +Special tokens serve as **semantic markers** that help models understand the structure and intent of different parts of the input. Unlike regular text that gets split into subword tokens, special tokens are: + +1. **Atomic**: Always encoded as a single token ID, never split +2. **Unambiguous**: Cannot be confused with regular text +3. **Efficient**: Single token vs multiple tokens for delimiters +4. **Trainable**: Models can learn specific behaviors associated with each token + +### Why Extend the Vocabulary? + +OpenAI's standard tokenizers include only basic special tokens (`<|endoftext|>`, `<|fim_*|>`, etc.). Modern agent architectures require richer semantic markers to: + +- **Separate concerns**: Distinguish thinking from output, actions from observations +- **Enable parsing**: Reliably extract structured data from model outputs +- **Support training**: Provide clear signals for fine-tuning agent behaviors +- **Maintain compatibility**: Work alongside existing tokenizer infrastructure + +### Token Naming Convention + +All tokens follow the `<|name|>` / `<|/name|>` pattern: + +- Opening tags: `<|name|>` - marks the start of a semantic block +- Closing tags: `<|/name|>` - marks the end of a semantic block +- Standalone tokens: `<|name|>` - single markers (e.g., `<|pad|>`, `<|stop|>`) + +This convention mirrors XML/HTML for familiarity while using `<|...|>` to avoid conflicts with actual markup in training data. + +--- + +## Token ID Allocation + +### Avoiding Conflicts + +Token IDs are carefully allocated to avoid conflicts with OpenAI's reserved ranges: + +| Model | Regular Tokens | OpenAI Reserved | Agent Tokens | Total | +| ------------- | -------------- | --------------- | --------------- | ------- | +| `cl100k_base` | 0-100,255 | 100,257-100,276 | 100,277-100,330 | 100,331 | +| `o200k_base` | 0-199,997 | 199,999-200,018 | 200,019-200,072 | 200,073 | + +### Why These Ranges? + +- **OpenAI compatibility**: Agent tokens start after OpenAI's last known special token +- **Future-proofing**: Gap between OpenAI tokens and agent tokens allows for OpenAI additions +- **Consistency**: Same token semantics map to different IDs per vocabulary, but maintain relative ordering + +--- + +## OpenAI Standard Tokens + +These tokens are part of the original OpenAI tokenizer specification: + +### cl100k_base (GPT-4, GPT-3.5-turbo) + +| Token | ID | Purpose | +| ------------------- | ------ | -------------------------- | +| `<\|endoftext\|>` | 100257 | End of document marker | +| `<\|fim_prefix\|>` | 100258 | Fill-in-the-middle: prefix | +| `<\|fim_middle\|>` | 100259 | Fill-in-the-middle: middle | +| `<\|fim_suffix\|>` | 100260 | Fill-in-the-middle: suffix | +| `<\|endofprompt\|>` | 100276 | End of prompt marker | + +### o200k_base (GPT-4o) + +| Token | ID | Purpose | +| ------------------- | ------ | ---------------------- | +| `<\|endoftext\|>` | 199999 | End of document marker | +| `<\|endofprompt\|>` | 200018 | End of prompt marker | + +--- + +## Agent Token Categories + +### 1. Conversation Structure + +**Purpose**: Standard ChatML-style tokens for multi-turn conversations. + +| Token | cl100k ID | o200k ID | Description | +| ----------------- | --------- | -------- | ----------------------------------------------- | +| `<\|system\|>` | 100277 | 200019 | System instructions defining assistant behavior | +| `<\|user\|>` | 100278 | 200020 | User input/queries | +| `<\|assistant\|>` | 100279 | 200021 | Assistant responses | +| `<\|im_start\|>` | 100280 | 200022 | Generic message start (ChatML) | +| `<\|im_end\|>` | 100281 | 200023 | Generic message end (ChatML) | + +**Rationale**: These tokens implement the [ChatML format](https://github.com/openai/openai-python/blob/main/chatml.md) used by OpenAI and adopted widely for chat model training. The `im_start`/`im_end` tokens provide a generic wrapper, while role-specific tokens (`system`, `user`, `assistant`) enable direct role marking. + +**Example**: + +``` +<|im_start|>system +You are a helpful assistant.<|im_end|> +<|im_start|>user +What is the capital of France?<|im_end|> +<|im_start|>assistant +The capital of France is Paris.<|im_end|> +``` + +--- + +### 2. Reasoning / Chain-of-Thought + +**Purpose**: Enable System 2 (slow, deliberate) reasoning similar to DeepSeek-R1 or OpenAI o1. + +| Token | cl100k ID | o200k ID | Description | +| -------------- | --------- | -------- | ------------------------ | +| `<\|think\|>` | 100282 | 200024 | Start of reasoning block | +| `<\|/think\|>` | 100283 | 200025 | End of reasoning block | + +**Rationale**: Chain-of-Thought (CoT) prompting significantly improves model performance on complex tasks. Dedicated thinking tokens allow: + +- **Training**: Models learn to "think before answering" +- **Inference**: Thinking can be hidden from users in production +- **Analysis**: Reasoning traces can be extracted for debugging/evaluation + +**Example**: + +``` +<|think|> +The user is asking about the capital of France. +I know that Paris is the capital and largest city of France. +It has been the capital since the 10th century. +<|/think|> +The capital of France is Paris. +``` + +--- + +### 3. ReAct Agent Loop + +**Purpose**: Implement the ReAct (Reason + Act) paradigm for autonomous agents. + +| Token | cl100k ID | o200k ID | Description | +| ---------------- | --------- | -------- | ------------------------------- | +| `<\|plan\|>` | 100284 | 200026 | High-level strategy formulation | +| `<\|/plan\|>` | 100285 | 200027 | End of plan | +| `<\|step\|>` | 100286 | 200028 | Individual step within plan | +| `<\|/step\|>` | 100287 | 200029 | End of step | +| `<\|act\|>` | 100288 | 200030 | Action intent declaration | +| `<\|/act\|>` | 100289 | 200031 | End of action | +| `<\|observe\|>` | 100290 | 200032 | Environment feedback | +| `<\|/observe\|>` | 100291 | 200033 | End of observation | + +**Rationale**: The [ReAct paper](https://arxiv.org/abs/2210.03629) demonstrated that interleaving reasoning and acting improves agent performance. These tokens create a structured loop: + +1. **Plan**: Agent decides overall strategy +2. **Step**: Break plan into discrete actions +3. **Act**: Declare intent to perform action +4. **Observe**: Receive and process environment feedback +5. Repeat until task complete + +**Example**: + +``` +<|plan|> +To answer this question, I need to: +1. Search for current weather data +2. Extract the temperature +3. Format the response +<|/plan|> +<|step|>Searching for weather data<|/step|> +<|act|>search("London weather today")<|/act|> +<|observe|>Temperature: 18°C, Condition: Partly cloudy<|/observe|> +<|step|>Formatting response<|/step|> +The current temperature in London is 18°C with partly cloudy skies. +``` + +--- + +### 4. Tool / Function Calling + +**Purpose**: Structured tool use with explicit success/error handling. + +| Token | cl100k ID | o200k ID | Description | +| ----------------- | --------- | -------- | --------------------------- | +| `<\|function\|>` | 100292 | 200034 | Function call specification | +| `<\|/function\|>` | 100293 | 200035 | End of function call | +| `<\|result\|>` | 100294 | 200036 | Successful return value | +| `<\|/result\|>` | 100295 | 200037 | End of result | +| `<\|error\|>` | 100296 | 200038 | Execution error | +| `<\|/error\|>` | 100297 | 200039 | End of error | + +**Rationale**: Function calling is fundamental to agent capabilities. Separating `<|act|>` (intent) from `<|function|>` (technical payload) allows: + +- **Intent**: "I want to check the weather" (`<|act|>`) +- **Implementation**: `{"name": "get_weather", "args": {...}}` (`<|function|>`) + +The `<|error|>` token is critical for robust agents—it signals that the previous action failed, enabling retry logic without confusing errors with valid outputs. + +**Example**: + +``` +<|function|>{"name": "get_weather", "args": {"city": "London", "units": "celsius"}}<|/function|> +<|result|>{"temperature": 18, "condition": "partly_cloudy", "humidity": 65}<|/result|> +``` + +**Error handling**: + +``` +<|function|>{"name": "get_stock_price", "args": {"symbol": "INVALID"}}<|/function|> +<|error|>{"code": "SYMBOL_NOT_FOUND", "message": "Stock symbol 'INVALID' not found"}<|/error|> +``` + +--- + +### 5. Code Execution + +**Purpose**: Jupyter notebook-style code interpreter flow. + +| Token | cl100k ID | o200k ID | Description | +| --------------- | --------- | -------- | --------------------- | +| `<\|code\|>` | 100298 | 200040 | Code block to execute | +| `<\|/code\|>` | 100299 | 200041 | End of code block | +| `<\|output\|>` | 100300 | 200042 | Execution output | +| `<\|/output\|>` | 100301 | 200043 | End of output | +| `<\|lang\|>` | 100302 | 200044 | Language identifier | +| `<\|/lang\|>` | 100303 | 200045 | End of language tag | + +**Rationale**: Code execution is a powerful agent capability. These tokens model the notebook paradigm: + +- Code cells with explicit language tags +- Captured stdout/return values +- Clear separation between code and output + +**Example**: + +``` +<|code|><|lang|>python<|/lang|> +import math + +def calculate_circle_area(radius): + return math.pi * radius ** 2 + +area = calculate_circle_area(5) +print(f"Area: {area:.2f}") +<|/code|> +<|output|>Area: 78.54<|/output|> +``` + +--- + +### 6. RAG / Citations + +**Purpose**: Retrieval-Augmented Generation with source attribution. + +| Token | cl100k ID | o200k ID | Description | +| ---------------- | --------- | -------- | ----------------------- | +| `<\|context\|>` | 100304 | 200046 | Retrieved context block | +| `<\|/context\|>` | 100305 | 200047 | End of context | +| `<\|quote\|>` | 100306 | 200048 | Direct quotation | +| `<\|/quote\|>` | 100307 | 200049 | End of quote | +| `<\|cite\|>` | 100308 | 200050 | Citation reference | +| `<\|/cite\|>` | 100309 | 200051 | End of citation | +| `<\|source\|>` | 100310 | 200052 | Source metadata | +| `<\|/source\|>` | 100311 | 200053 | End of source | + +**Rationale**: RAG systems retrieve relevant documents to ground model responses. These tokens enable: + +- **Grounded generation**: Model sees retrieved context explicitly +- **Citation training**: Model learns to cite sources +- **Verification**: Outputs can be traced back to sources +- **Hallucination reduction**: Clear separation of retrieved vs generated content + +**Example**: + +``` +<|context|> +<|source|>wikipedia:Paris<|/source|> +Paris is the capital and most populous city of France. With an official +estimated population of 2,102,650 residents in January 2023 in an area of +more than 105 km², Paris is the fourth-most populated city in the European Union. +<|/context|> + +Based on the retrieved information, Paris is the capital of France with a +population of approximately <|quote|>2,102,650 residents<|/quote|> +<|cite|>wikipedia:Paris<|/cite|>. +``` + +--- + +### 7. Memory / State + +**Purpose**: Long-term memory and state persistence across sessions. + +| Token | cl100k ID | o200k ID | Description | +| --------------- | --------- | -------- | ------------------- | +| `<\|memory\|>` | 100312 | 200054 | Store information | +| `<\|/memory\|>` | 100313 | 200055 | End of memory block | +| `<\|recall\|>` | 100314 | 200056 | Retrieved memory | +| `<\|/recall\|>` | 100315 | 200057 | End of recall | + +**Rationale**: Persistent memory enables agents to: + +- Remember user preferences across conversations +- Build up knowledge over time +- Maintain continuity in long-running tasks + +The separation of `memory` (write) and `recall` (read) mirrors database semantics. + +**Example**: + +``` +<|memory|>User prefers concise responses. User's name is Alice.<|/memory|> + +... later in conversation ... + +<|recall|>User prefers concise responses. User's name is Alice.<|/recall|> +Hello Alice! Here's a brief answer: The capital of France is Paris. +``` + +--- + +### 8. Control Tokens + +**Purpose**: Sequence control and formatting. + +| Token | cl100k ID | o200k ID | Description | +| ------------ | --------- | -------- | --------------------------- | +| `<\|pad\|>` | 100316 | 200058 | Padding for batch alignment | +| `<\|stop\|>` | 100317 | 200059 | Generation stop signal | +| `<\|sep\|>` | 100318 | 200060 | Segment separator | + +**Rationale**: These are utility tokens for training and inference: + +- **pad**: Aligns sequences in batches (has no semantic meaning) +- **stop**: Alternative to `<|endoftext|>` for stopping generation +- **sep**: Separates segments without implying document boundaries + +--- + +### 9. Multimodal + +**Purpose**: Placeholders for non-text content. + +| Token | cl100k ID | o200k ID | Description | +| -------------- | --------- | -------- | ------------- | +| `<\|image\|>` | 100319 | 200061 | Image content | +| `<\|/image\|>` | 100320 | 200062 | End of image | +| `<\|audio\|>` | 100321 | 200063 | Audio content | +| `<\|/audio\|>` | 100322 | 200064 | End of audio | +| `<\|video\|>` | 100323 | 200065 | Video content | +| `<\|/video\|>` | 100324 | 200066 | End of video | + +**Rationale**: Multimodal models need to mark where non-text embeddings are inserted. These tokens serve as: + +- **Placeholders**: Mark positions for embedding injection +- **Delimiters**: Wrap base64-encoded or referenced content +- **Training signals**: Help models learn cross-modal attention + +**Example**: + +``` +Describe what you see in this image: +<|image|>base64_encoded_image_data_here<|/image|> + +The image shows a sunset over the ocean with vibrant orange and purple colors. +``` + +--- + +### 10. Document Structure + +**Purpose**: Semantic layout for parsing structured documents. + +| Token | cl100k ID | o200k ID | Description | +| ---------------- | --------- | -------- | ---------------------- | +| `<\|title\|>` | 100325 | 200067 | Document/section title | +| `<\|/title\|>` | 100326 | 200068 | End of title | +| `<\|section\|>` | 100327 | 200069 | Semantic section | +| `<\|/section\|>` | 100328 | 200070 | End of section | +| `<\|summary\|>` | 100329 | 200071 | Content summary | +| `<\|/summary\|>` | 100330 | 200072 | End of summary | + +**Rationale**: When processing structured documents (papers, reports, documentation), these tokens help: + +- **Preserve structure**: Maintain document hierarchy in tokenized form +- **Enable extraction**: Reliably parse titles, sections, summaries +- **Support generation**: Train models to produce well-structured output + +**Example**: + +``` +<|title|>Climate Change Impact Assessment<|/title|> + +<|summary|> +This report examines the effects of climate change on coastal ecosystems, +finding significant impacts on biodiversity and recommending adaptive strategies. +<|/summary|> + +<|section|> +<|title|>Introduction<|/title|> +Climate change represents one of the most significant challenges... +<|/section|> + +<|section|> +<|title|>Methodology<|/title|> +We analyzed data from 50 coastal monitoring stations... +<|/section|> +``` + +--- + +## Usage Examples + +### Python + +```python +from Splintr import Tokenizer, CL100K_AGENT_TOKENS + +tokenizer = Tokenizer.from_pretrained("cl100k_base") + +# Encode text with special tokens +text = "<|think|>Let me reason about this...<|/think|>The answer is 42." +tokens = tokenizer.encode_with_special(text) + +# Check for specific tokens +if CL100K_AGENT_TOKENS.THINK in tokens: + print("Contains thinking block") + +# Decode back to text +decoded = tokenizer.decode(tokens) +assert decoded == text + +# Access token IDs programmatically +print(f"THINK token ID: {CL100K_AGENT_TOKENS.THINK}") # 100282 +print(f"FUNCTION token ID: {CL100K_AGENT_TOKENS.FUNCTION}") # 100292 +``` + +### Rust + +```rust +use Splintr::{Tokenizer, cl100k_agent_tokens, CL100K_BASE_PATTERN}; + +// Access token constants +let think_id = cl100k_agent_tokens::THINK; // 100282 +let function_id = cl100k_agent_tokens::FUNCTION; // 100292 + +// Use in your agent implementation +fn extract_thinking(tokens: &[u32]) -> Option<(usize, usize)> { + let start = tokens.iter().position(|&t| t == cl100k_agent_tokens::THINK)?; + let end = tokens.iter().position(|&t| t == cl100k_agent_tokens::THINK_END)?; + Some((start, end)) +} +``` + +--- + +## Python API Reference + +### CL100K_AGENT_TOKENS + +```python +from Splintr import CL100K_AGENT_TOKENS + +# Conversation +CL100K_AGENT_TOKENS.SYSTEM # 100277 +CL100K_AGENT_TOKENS.USER # 100278 +CL100K_AGENT_TOKENS.ASSISTANT # 100279 +CL100K_AGENT_TOKENS.IM_START # 100280 +CL100K_AGENT_TOKENS.IM_END # 100281 + +# Thinking +CL100K_AGENT_TOKENS.THINK # 100282 +CL100K_AGENT_TOKENS.THINK_END # 100283 + +# ReAct +CL100K_AGENT_TOKENS.PLAN # 100284 +CL100K_AGENT_TOKENS.PLAN_END # 100285 +CL100K_AGENT_TOKENS.STEP # 100286 +CL100K_AGENT_TOKENS.STEP_END # 100287 +CL100K_AGENT_TOKENS.ACT # 100288 +CL100K_AGENT_TOKENS.ACT_END # 100289 +CL100K_AGENT_TOKENS.OBSERVE # 100290 +CL100K_AGENT_TOKENS.OBSERVE_END # 100291 + +# Tool/Function +CL100K_AGENT_TOKENS.FUNCTION # 100292 +CL100K_AGENT_TOKENS.FUNCTION_END # 100293 +CL100K_AGENT_TOKENS.RESULT # 100294 +CL100K_AGENT_TOKENS.RESULT_END # 100295 +CL100K_AGENT_TOKENS.ERROR # 100296 +CL100K_AGENT_TOKENS.ERROR_END # 100297 + +# Code +CL100K_AGENT_TOKENS.CODE # 100298 +CL100K_AGENT_TOKENS.CODE_END # 100299 +CL100K_AGENT_TOKENS.OUTPUT # 100300 +CL100K_AGENT_TOKENS.OUTPUT_END # 100301 +CL100K_AGENT_TOKENS.LANG # 100302 +CL100K_AGENT_TOKENS.LANG_END # 100303 + +# RAG +CL100K_AGENT_TOKENS.CONTEXT # 100304 +CL100K_AGENT_TOKENS.CONTEXT_END # 100305 +CL100K_AGENT_TOKENS.QUOTE # 100306 +CL100K_AGENT_TOKENS.QUOTE_END # 100307 +CL100K_AGENT_TOKENS.CITE # 100308 +CL100K_AGENT_TOKENS.CITE_END # 100309 +CL100K_AGENT_TOKENS.SOURCE # 100310 +CL100K_AGENT_TOKENS.SOURCE_END # 100311 + +# Memory +CL100K_AGENT_TOKENS.MEMORY # 100312 +CL100K_AGENT_TOKENS.MEMORY_END # 100313 +CL100K_AGENT_TOKENS.RECALL # 100314 +CL100K_AGENT_TOKENS.RECALL_END # 100315 + +# Control +CL100K_AGENT_TOKENS.PAD # 100316 +CL100K_AGENT_TOKENS.STOP # 100317 +CL100K_AGENT_TOKENS.SEP # 100318 + +# Multimodal +CL100K_AGENT_TOKENS.IMAGE # 100319 +CL100K_AGENT_TOKENS.IMAGE_END # 100320 +CL100K_AGENT_TOKENS.AUDIO # 100321 +CL100K_AGENT_TOKENS.AUDIO_END # 100322 +CL100K_AGENT_TOKENS.VIDEO # 100323 +CL100K_AGENT_TOKENS.VIDEO_END # 100324 + +# Document +CL100K_AGENT_TOKENS.TITLE # 100325 +CL100K_AGENT_TOKENS.TITLE_END # 100326 +CL100K_AGENT_TOKENS.SECTION # 100327 +CL100K_AGENT_TOKENS.SECTION_END # 100328 +CL100K_AGENT_TOKENS.SUMMARY # 100329 +CL100K_AGENT_TOKENS.SUMMARY_END # 100330 +``` + +### O200K_AGENT_TOKENS + +Same structure as above, with IDs starting at 200019. + +--- + +## Rust API Reference + +### cl100k_agent_tokens module + +```rust +use Splintr::cl100k_agent_tokens; + +// All constants follow the same naming as Python +cl100k_agent_tokens::SYSTEM // 100277 +cl100k_agent_tokens::THINK // 100282 +cl100k_agent_tokens::FUNCTION // 100292 +// ... etc +``` + +### o200k_agent_tokens module + +```rust +use Splintr::o200k_agent_tokens; + +o200k_agent_tokens::SYSTEM // 200019 +o200k_agent_tokens::THINK // 200024 +// ... etc +``` + +--- + +## See Also + +- [README.md](../README.md) - Project overview and quick start +- [ReAct Paper](https://arxiv.org/abs/2210.03629) - ReAct: Synergizing Reasoning and Acting in Language Models +- [ChatML Specification](https://github.com/openai/openai-python/blob/main/chatml.md) - Chat Markup Language diff --git a/python/splintr/__init__.py b/python/splintr/__init__.py index 3237b06..6535964 100644 --- a/python/splintr/__init__.py +++ b/python/splintr/__init__.py @@ -1,5 +1,5 @@ """ -splintr - Fast Rust BPE tokenizer with Python bindings +Splintr - Fast Rust BPE tokenizer with Python bindings A high-performance tokenizer featuring: - PCRE2 with JIT compilation (2-4x faster than fancy-regex) @@ -9,6 +9,7 @@ - Aho-Corasick for fast special token matching - LRU cache for frequently encoded chunks - UTF-8 streaming decoder for LLM output +- Agent tokens for chat/reasoning/tool-use applications Usage: from splintr import Tokenizer @@ -33,6 +34,31 @@ if text := decoder.add_token(token_id): print(text, end="", flush=True) print(decoder.flush()) + +Agent Tokens: + from splintr import Tokenizer, CL100K_AGENT_TOKENS + + tokenizer = Tokenizer.from_pretrained("cl100k_base") + + # Access token IDs programmatically + print(CL100K_AGENT_TOKENS.THINK) # 100282 + print(CL100K_AGENT_TOKENS.FUNCTION) # 100292 + + # Encode with special tokens + tokens = tokenizer.encode_with_special("<|think|>reasoning<|/think|>") + assert CL100K_AGENT_TOKENS.THINK in tokens + + # Token categories: + # - Conversation: SYSTEM, USER, ASSISTANT, IM_START, IM_END + # - Thinking: THINK, THINK_END (Chain-of-Thought) + # - ReAct: PLAN, STEP, ACT, OBSERVE (+ _END variants) + # - Tools: FUNCTION, RESULT, ERROR (+ _END variants) + # - Code: CODE, OUTPUT, LANG (+ _END variants) + # - RAG: CONTEXT, QUOTE, CITE, SOURCE (+ _END variants) + # - Memory: MEMORY, RECALL (+ _END variants) + # - Control: PAD, STOP, SEP + # - Multimodal: IMAGE, AUDIO, VIDEO (+ _END variants) + # - Document: TITLE, SECTION, SUMMARY (+ _END variants) """ from ._core import ( @@ -40,6 +66,8 @@ StreamingDecoder, CL100K_BASE_PATTERN, O200K_BASE_PATTERN, + CL100K_AGENT_TOKENS, + O200K_AGENT_TOKENS, ) __all__ = [ @@ -47,5 +75,7 @@ "StreamingDecoder", "CL100K_BASE_PATTERN", "O200K_BASE_PATTERN", + "CL100K_AGENT_TOKENS", + "O200K_AGENT_TOKENS", ] __version__ = "0.1.0b1" diff --git a/src/core/mod.rs b/src/core/mod.rs index f50396f..aebdaa8 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -31,5 +31,8 @@ mod vocab; pub use bpe::byte_pair_encode; pub use streaming::StreamingDecoder; -pub use tokenizer::{Tokenizer, TokenizerError, CL100K_BASE_PATTERN, O200K_BASE_PATTERN}; +pub use tokenizer::{ + cl100k_agent_tokens, o200k_agent_tokens, Tokenizer, TokenizerError, CL100K_BASE_PATTERN, + O200K_BASE_PATTERN, +}; pub use vocab::{build_decoder, load_tiktoken_bpe, load_tiktoken_bpe_file, VocabError}; diff --git a/src/core/tokenizer.rs b/src/core/tokenizer.rs index f9be129..4fa81ab 100644 --- a/src/core/tokenizer.rs +++ b/src/core/tokenizer.rs @@ -30,6 +30,488 @@ pub const CL100K_BASE_PATTERN: &str = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L} /// Default regex pattern for o200k_base (GPT-4o) pub const O200K_BASE_PATTERN: &str = r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"; +// ============================================================================= +// Agent Token Constants (cl100k_base: 100277+, o200k_base: 200019+) +// ============================================================================= +// These tokens extend the vocabulary for agent/chat applications without +// conflicting with OpenAI's reserved special token ranges. + +/// Agent tokens for cl100k_base (GPT-4, GPT-3.5-turbo). +/// +/// These special tokens extend the cl100k_base vocabulary for building chat models, +/// reasoning systems, and autonomous agents. Token IDs start at 100277 to avoid +/// conflicts with OpenAI's reserved range (100257-100276). +/// +/// # Token Categories +/// +/// ## Conversation Structure (100277-100281) +/// Standard ChatML-style tokens for multi-turn conversations: +/// - `<|system|>`: Marks system instructions that define assistant behavior +/// - `<|user|>`: Marks user input/queries +/// - `<|assistant|>`: Marks assistant responses +/// - `<|im_start|>`: Generic message start delimiter (ChatML format) +/// - `<|im_end|>`: Generic message end delimiter (ChatML format) +/// +/// Example: +/// ```text +/// <|im_start|>system +/// You are a helpful assistant.<|im_end|> +/// <|im_start|>user +/// Hello!<|im_end|> +/// <|im_start|>assistant +/// Hi there!<|im_end|> +/// ``` +/// +/// ## Reasoning/Thinking (100282-100283) +/// Chain-of-Thought (CoT) tokens for System 2 reasoning, similar to DeepSeek-R1 +/// or OpenAI o1-style thinking: +/// - `<|think|>`: Start of internal reasoning (hidden from user in production) +/// - `<|/think|>`: End of internal reasoning +/// +/// Example: +/// ```text +/// <|think|> +/// Let me break this down step by step... +/// First, I need to consider X. +/// Then, Y follows from X. +/// <|/think|> +/// The answer is Y. +/// ``` +/// +/// ## ReAct Agent Loop (100284-100291) +/// Tokens for ReAct (Reason + Act) agent architectures: +/// - `<|plan|>`: High-level planning phase where agent decides strategy +/// - `<|step|>`: Individual step within a plan +/// - `<|act|>`: Action intent declaration (what the agent wants to do) +/// - `<|observe|>`: Observation/feedback from environment after action +/// +/// Example: +/// ```text +/// <|plan|> +/// I need to: 1) Search for info, 2) Summarize findings +/// <|/plan|> +/// <|step|>Searching for relevant information<|/step|> +/// <|act|>search("climate change effects")<|/act|> +/// <|observe|>Found 3 relevant articles...<|/observe|> +/// ``` +/// +/// ## Tool/Function Calling (100292-100297) +/// Structured tool use with explicit success/error handling: +/// - `<|function|>`: Function call specification (name + arguments) +/// - `<|result|>`: Successful function return value +/// - `<|error|>`: Function execution error (enables retry logic) +/// +/// Example: +/// ```text +/// <|function|>{"name": "get_weather", "args": {"city": "London"}}<|/function|> +/// <|result|>{"temp": 18, "condition": "cloudy"}<|/result|> +/// ``` +/// +/// ## Code Execution (100298-100303) +/// Jupyter notebook-style code interpreter flow: +/// - `<|code|>`: Code block to execute +/// - `<|output|>`: Execution output (stdout, return values) +/// - `<|lang|>`: Programming language identifier +/// +/// Example: +/// ```text +/// <|code|><|lang|>python<|/lang|> +/// import math +/// print(math.sqrt(16)) +/// <|/code|> +/// <|output|>4.0<|/output|> +/// ``` +/// +/// ## RAG/Citations (100304-100311) +/// Retrieval-Augmented Generation with source attribution: +/// - `<|context|>`: Injected context from retrieval system +/// - `<|quote|>`: Direct quotation from source material +/// - `<|cite|>`: Citation reference marker +/// - `<|source|>`: Source metadata (URL, document ID, etc.) +/// +/// Example: +/// ```text +/// <|context|> +/// <|source|>doc_123<|/source|> +/// The Earth orbits the Sun in 365.25 days. +/// <|/context|> +/// According to the source<|cite|>doc_123<|/cite|>, <|quote|>The Earth orbits +/// the Sun in 365.25 days.<|/quote|> +/// ``` +/// +/// ## Memory/State (100312-100315) +/// Long-term memory and state persistence: +/// - `<|memory|>`: Store information for future reference +/// - `<|recall|>`: Retrieve previously stored information +/// +/// Example: +/// ```text +/// <|memory|>User prefers concise responses<|/memory|> +/// ...later... +/// <|recall|>User prefers concise responses<|/recall|> +/// ``` +/// +/// ## Control Tokens (100316-100318) +/// Sequence control and formatting: +/// - `<|pad|>`: Padding token for batch alignment +/// - `<|stop|>`: Generation stop signal +/// - `<|sep|>`: Separator between segments +/// +/// ## Multimodal (100319-100324) +/// Placeholders for non-text content: +/// - `<|image|>`: Image embedding or base64 data +/// - `<|audio|>`: Audio embedding or encoded data +/// - `<|video|>`: Video embedding or encoded data +/// +/// Example: +/// ```text +/// Describe this image: <|image|>base64_data_here<|/image|> +/// ``` +/// +/// ## Document Structure (100325-100330) +/// Semantic layout tokens for parsing structured documents: +/// - `<|title|>`: Document or section title +/// - `<|section|>`: Semantic section boundary +/// - `<|summary|>`: Condensed content summary +/// +/// Example: +/// ```text +/// <|title|>Introduction<|/title|> +/// <|section|> +/// This section covers the basics... +/// <|/section|> +/// <|summary|>Key points: X, Y, Z<|/summary|> +/// ``` +pub mod cl100k_agent_tokens { + // ========================================================================= + // Conversation Structure (100277-100281) + // ========================================================================= + + /// System message marker - defines assistant behavior and constraints. + pub const SYSTEM: u32 = 100277; + /// User message marker - marks human input in conversation. + pub const USER: u32 = 100278; + /// Assistant message marker - marks AI responses. + pub const ASSISTANT: u32 = 100279; + /// ChatML message start - generic delimiter for any role. + pub const IM_START: u32 = 100280; + /// ChatML message end - closes any message block. + pub const IM_END: u32 = 100281; + + // ========================================================================= + // Reasoning/Thinking - Chain-of-Thought (100282-100283) + // ========================================================================= + + /// Start of thinking/reasoning block (System 2 cognition). + /// Content between THINK and THINK_END represents internal reasoning + /// that may be hidden from users in production. + pub const THINK: u32 = 100282; + /// End of thinking/reasoning block. + pub const THINK_END: u32 = 100283; + + // ========================================================================= + // ReAct Agent Loop (100284-100291) + // ========================================================================= + + /// Start of planning phase - high-level strategy formulation. + pub const PLAN: u32 = 100284; + /// End of planning phase. + pub const PLAN_END: u32 = 100285; + /// Start of individual step - discrete action within a plan. + pub const STEP: u32 = 100286; + /// End of step. + pub const STEP_END: u32 = 100287; + /// Start of action - the intent to perform an operation. + pub const ACT: u32 = 100288; + /// End of action. + pub const ACT_END: u32 = 100289; + /// Start of observation - environment feedback after action. + pub const OBSERVE: u32 = 100290; + /// End of observation. + pub const OBSERVE_END: u32 = 100291; + + // ========================================================================= + // Tool/Function Calling (100292-100297) + // ========================================================================= + + /// Start of function call - contains function name and arguments (usually JSON). + pub const FUNCTION: u32 = 100292; + /// End of function call. + pub const FUNCTION_END: u32 = 100293; + /// Start of function result - successful return value. + pub const RESULT: u32 = 100294; + /// End of function result. + pub const RESULT_END: u32 = 100295; + /// Start of error block - function execution failure, enables retry logic. + pub const ERROR: u32 = 100296; + /// End of error block. + pub const ERROR_END: u32 = 100297; + + // ========================================================================= + // Code Execution (100298-100303) + // ========================================================================= + + /// Start of code block - executable code content. + pub const CODE: u32 = 100298; + /// End of code block. + pub const CODE_END: u32 = 100299; + /// Start of execution output - stdout, return values, rendered output. + pub const OUTPUT: u32 = 100300; + /// End of execution output. + pub const OUTPUT_END: u32 = 100301; + /// Start of language identifier (e.g., "python", "javascript"). + pub const LANG: u32 = 100302; + /// End of language identifier. + pub const LANG_END: u32 = 100303; + + // ========================================================================= + // RAG/Citations (100304-100311) + // ========================================================================= + + /// Start of retrieved context block - injected by RAG pipeline. + pub const CONTEXT: u32 = 100304; + /// End of context block. + pub const CONTEXT_END: u32 = 100305; + /// Start of direct quotation from source material. + pub const QUOTE: u32 = 100306; + /// End of quotation. + pub const QUOTE_END: u32 = 100307; + /// Start of citation marker - references a source. + pub const CITE: u32 = 100308; + /// End of citation marker. + pub const CITE_END: u32 = 100309; + /// Start of source identifier - URL, document ID, or metadata. + pub const SOURCE: u32 = 100310; + /// End of source identifier. + pub const SOURCE_END: u32 = 100311; + + // ========================================================================= + // Memory/State Management (100312-100315) + // ========================================================================= + + /// Start of memory block - information to persist across sessions. + pub const MEMORY: u32 = 100312; + /// End of memory block. + pub const MEMORY_END: u32 = 100313; + /// Start of recall block - retrieved persistent memory. + pub const RECALL: u32 = 100314; + /// End of recall block. + pub const RECALL_END: u32 = 100315; + + // ========================================================================= + // Control Tokens (100316-100318) + // ========================================================================= + + /// Padding token - used for batch alignment, has no semantic meaning. + pub const PAD: u32 = 100316; + /// Stop token - signals end of generation. + pub const STOP: u32 = 100317; + /// Separator token - delimits segments within a sequence. + pub const SEP: u32 = 100318; + + // ========================================================================= + // Multimodal Placeholders (100319-100324) + // ========================================================================= + + /// Start of image content - embedding vector or encoded image data. + pub const IMAGE: u32 = 100319; + /// End of image content. + pub const IMAGE_END: u32 = 100320; + /// Start of audio content - embedding vector or encoded audio data. + pub const AUDIO: u32 = 100321; + /// End of audio content. + pub const AUDIO_END: u32 = 100322; + /// Start of video content - embedding vector or encoded video data. + pub const VIDEO: u32 = 100323; + /// End of video content. + pub const VIDEO_END: u32 = 100324; + + // ========================================================================= + // Document Structure (100325-100330) + // ========================================================================= + + /// Start of title - document or section title for semantic parsing. + pub const TITLE: u32 = 100325; + /// End of title. + pub const TITLE_END: u32 = 100326; + /// Start of section - semantic document section boundary. + pub const SECTION: u32 = 100327; + /// End of section. + pub const SECTION_END: u32 = 100328; + /// Start of summary - condensed content summary. + pub const SUMMARY: u32 = 100329; + /// End of summary. + pub const SUMMARY_END: u32 = 100330; +} + +/// Agent tokens for o200k_base (GPT-4o). +/// +/// These special tokens extend the o200k_base vocabulary for building chat models, +/// reasoning systems, and autonomous agents. Token IDs start at 200019 to avoid +/// conflicts with OpenAI's reserved range (199999-200018). +/// +/// See [`cl100k_agent_tokens`] for detailed documentation on each token category. +/// The token semantics are identical; only the IDs differ. +pub mod o200k_agent_tokens { + // ========================================================================= + // Conversation Structure (200019-200023) + // ========================================================================= + + /// System message marker - defines assistant behavior and constraints. + pub const SYSTEM: u32 = 200019; + /// User message marker - marks human input in conversation. + pub const USER: u32 = 200020; + /// Assistant message marker - marks AI responses. + pub const ASSISTANT: u32 = 200021; + /// ChatML message start - generic delimiter for any role. + pub const IM_START: u32 = 200022; + /// ChatML message end - closes any message block. + pub const IM_END: u32 = 200023; + + // ========================================================================= + // Reasoning/Thinking - Chain-of-Thought (200024-200025) + // ========================================================================= + + /// Start of thinking/reasoning block (System 2 cognition). + pub const THINK: u32 = 200024; + /// End of thinking/reasoning block. + pub const THINK_END: u32 = 200025; + + // ========================================================================= + // ReAct Agent Loop (200026-200033) + // ========================================================================= + + /// Start of planning phase - high-level strategy formulation. + pub const PLAN: u32 = 200026; + /// End of planning phase. + pub const PLAN_END: u32 = 200027; + /// Start of individual step - discrete action within a plan. + pub const STEP: u32 = 200028; + /// End of step. + pub const STEP_END: u32 = 200029; + /// Start of action - the intent to perform an operation. + pub const ACT: u32 = 200030; + /// End of action. + pub const ACT_END: u32 = 200031; + /// Start of observation - environment feedback after action. + pub const OBSERVE: u32 = 200032; + /// End of observation. + pub const OBSERVE_END: u32 = 200033; + + // ========================================================================= + // Tool/Function Calling (200034-200039) + // ========================================================================= + + /// Start of function call - contains function name and arguments (usually JSON). + pub const FUNCTION: u32 = 200034; + /// End of function call. + pub const FUNCTION_END: u32 = 200035; + /// Start of function result - successful return value. + pub const RESULT: u32 = 200036; + /// End of function result. + pub const RESULT_END: u32 = 200037; + /// Start of error block - function execution failure, enables retry logic. + pub const ERROR: u32 = 200038; + /// End of error block. + pub const ERROR_END: u32 = 200039; + + // ========================================================================= + // Code Execution (200040-200045) + // ========================================================================= + + /// Start of code block - executable code content. + pub const CODE: u32 = 200040; + /// End of code block. + pub const CODE_END: u32 = 200041; + /// Start of execution output - stdout, return values, rendered output. + pub const OUTPUT: u32 = 200042; + /// End of execution output. + pub const OUTPUT_END: u32 = 200043; + /// Start of language identifier (e.g., "python", "javascript"). + pub const LANG: u32 = 200044; + /// End of language identifier. + pub const LANG_END: u32 = 200045; + + // ========================================================================= + // RAG/Citations (200046-200053) + // ========================================================================= + + /// Start of retrieved context block - injected by RAG pipeline. + pub const CONTEXT: u32 = 200046; + /// End of context block. + pub const CONTEXT_END: u32 = 200047; + /// Start of direct quotation from source material. + pub const QUOTE: u32 = 200048; + /// End of quotation. + pub const QUOTE_END: u32 = 200049; + /// Start of citation marker - references a source. + pub const CITE: u32 = 200050; + /// End of citation marker. + pub const CITE_END: u32 = 200051; + /// Start of source identifier - URL, document ID, or metadata. + pub const SOURCE: u32 = 200052; + /// End of source identifier. + pub const SOURCE_END: u32 = 200053; + + // ========================================================================= + // Memory/State Management (200054-200057) + // ========================================================================= + + /// Start of memory block - information to persist across sessions. + pub const MEMORY: u32 = 200054; + /// End of memory block. + pub const MEMORY_END: u32 = 200055; + /// Start of recall block - retrieved persistent memory. + pub const RECALL: u32 = 200056; + /// End of recall block. + pub const RECALL_END: u32 = 200057; + + // ========================================================================= + // Control Tokens (200058-200060) + // ========================================================================= + + /// Padding token - used for batch alignment, has no semantic meaning. + pub const PAD: u32 = 200058; + /// Stop token - signals end of generation. + pub const STOP: u32 = 200059; + /// Separator token - delimits segments within a sequence. + pub const SEP: u32 = 200060; + + // ========================================================================= + // Multimodal Placeholders (200061-200066) + // ========================================================================= + + /// Start of image content - embedding vector or encoded image data. + pub const IMAGE: u32 = 200061; + /// End of image content. + pub const IMAGE_END: u32 = 200062; + /// Start of audio content - embedding vector or encoded audio data. + pub const AUDIO: u32 = 200063; + /// End of audio content. + pub const AUDIO_END: u32 = 200064; + /// Start of video content - embedding vector or encoded video data. + pub const VIDEO: u32 = 200065; + /// End of video content. + pub const VIDEO_END: u32 = 200066; + + // ========================================================================= + // Document Structure (200067-200072) + // ========================================================================= + + /// Start of title - document or section title for semantic parsing. + pub const TITLE: u32 = 200067; + /// End of title. + pub const TITLE_END: u32 = 200068; + /// Start of section - semantic document section boundary. + pub const SECTION: u32 = 200069; + /// End of section. + pub const SECTION_END: u32 = 200070; + /// Start of summary - condensed content summary. + pub const SUMMARY: u32 = 200071; + /// End of summary. + pub const SUMMARY_END: u32 = 200072; +} + /// Default cache size for encoded chunks const DEFAULT_CACHE_SIZE: usize = 4096; @@ -426,4 +908,63 @@ mod tests { tokenizer.clear_cache(); assert_eq!(tokenizer.cache_len(), 0); } + + #[test] + fn test_agent_tokens_constants() { + // Verify cl100k agent tokens don't conflict with OpenAI's reserved range + assert!(super::cl100k_agent_tokens::SYSTEM > 100276); // After endofprompt + assert!(super::cl100k_agent_tokens::SUMMARY_END == 100330); // Last token + + // Verify o200k agent tokens don't conflict with OpenAI's reserved range + assert!(super::o200k_agent_tokens::SYSTEM > 200018); // After endofprompt + assert!(super::o200k_agent_tokens::SUMMARY_END == 200072); // Last token + + // Verify token ordering is correct (no gaps or overlaps) + assert_eq!( + super::cl100k_agent_tokens::USER, + super::cl100k_agent_tokens::SYSTEM + 1 + ); + assert_eq!( + super::o200k_agent_tokens::USER, + super::o200k_agent_tokens::SYSTEM + 1 + ); + } + + #[test] + fn test_agent_tokens_encode_decode() { + // Create a tokenizer with agent tokens for testing + let mut encoder: FxHashMap, u32> = FxHashMap::default(); + encoder.insert(b"Hello".to_vec(), 0); + encoder.insert(b" ".to_vec(), 1); + encoder.insert(b"World".to_vec(), 2); + + let mut special: FxHashMap = FxHashMap::default(); + // Add some agent tokens + special.insert("<|system|>".to_string(), 100277); + special.insert("<|user|>".to_string(), 100278); + special.insert("<|assistant|>".to_string(), 100279); + special.insert("<|think|>".to_string(), 100282); + special.insert("<|/think|>".to_string(), 100283); + + let pattern = r"\S+|\s+"; + let tokenizer = Tokenizer::new(encoder, special, pattern).unwrap(); + + // Test encoding with agent tokens + let text = "<|system|>Hello<|user|>World"; + let tokens = tokenizer.encode_with_special(text); + + // Should contain the special tokens + assert!(tokens.contains(&100277)); // <|system|> + assert!(tokens.contains(&100278)); // <|user|> + + // Test decoding back + let decoded = tokenizer.decode(&tokens).unwrap(); + assert_eq!(decoded, text); + + // Test think tokens + let think_text = "<|think|>reasoning here<|/think|>"; + let think_tokens = tokenizer.encode_with_special(think_text); + assert!(think_tokens.contains(&100282)); // <|think|> + assert!(think_tokens.contains(&100283)); // <|/think|> + } } diff --git a/src/lib.rs b/src/lib.rs index a912d62..32ade70 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,7 +7,7 @@ pub use core::{ StreamingDecoder, Tokenizer, TokenizerError, CL100K_BASE_PATTERN, O200K_BASE_PATTERN, }; -/// splintr - Fast Rust BPE tokenizer with Python bindings +/// Splintr - Fast Rust BPE tokenizer with Python bindings /// /// A high-performance tokenizer featuring: /// - PCRE2 with JIT compilation (2-4x faster than fancy-regex) @@ -17,10 +17,13 @@ pub use core::{ /// - Aho-Corasick for fast special token matching /// - LRU cache for frequently encoded chunks /// - UTF-8 streaming decoder for LLM output +/// - Agent tokens for chat/reasoning/tool-use applications #[pymodule] fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add("CL100K_BASE_PATTERN", CL100K_BASE_PATTERN)?; m.add("O200K_BASE_PATTERN", O200K_BASE_PATTERN)?; Ok(()) diff --git a/src/python/bindings.rs b/src/python/bindings.rs index a74dbcd..4f22886 100644 --- a/src/python/bindings.rs +++ b/src/python/bindings.rs @@ -47,22 +47,257 @@ const CL100K_BASE_VOCAB: &[u8] = include_bytes!("../../python/splintr/vocabs/cl1 /// Bundled o200k_base vocabulary (GPT-4o) const O200K_BASE_VOCAB: &[u8] = include_bytes!("../../python/splintr/vocabs/o200k_base.tiktoken"); +// ============================================================================= +// Special Token Definitions +// ============================================================================= +// +// This section defines special tokens for cl100k_base and o200k_base tokenizers. +// These tokens are used for chat formatting, agent architectures, and multimodal +// applications. +// +// ## Token ID Allocation +// +// OpenAI Reserved: +// - cl100k_base: 100257-100276 +// - o200k_base: 199999-200018 +// +// Agent Extensions (added by splintr): +// - cl100k_base: 100277-100324 (48 tokens) +// - o200k_base: 200019-200066 (48 tokens) +// +// ## Python Usage +// +// ```python +// from splintr import Tokenizer +// +// tokenizer = Tokenizer.from_pretrained("cl100k_base") +// +// # Encode with special tokens recognized +// tokens = tokenizer.encode_with_special("<|system|>You are helpful.<|user|>Hi!") +// +// # Decode back to text +// text = tokenizer.decode(tokens) +// ``` +// +// ## Agent Token Categories +// +// ### 1. Conversation Structure +// Standard ChatML-style tokens for multi-turn conversations: +// - `<|system|>`: System instructions defining assistant behavior +// - `<|user|>`: User input/queries +// - `<|assistant|>`: Assistant responses +// - `<|im_start|>`: Generic message start (ChatML format) +// - `<|im_end|>`: Generic message end (ChatML format) +// +// Example: +// ``` +// <|im_start|>system +// You are a helpful assistant.<|im_end|> +// <|im_start|>user +// Hello!<|im_end|> +// <|im_start|>assistant +// Hi there!<|im_end|> +// ``` +// +// ### 2. Reasoning/Thinking (Chain-of-Thought) +// For System 2 reasoning similar to DeepSeek-R1 or OpenAI o1: +// - `<|think|>` / `<|/think|>`: Internal reasoning block +// +// Example: +// ``` +// <|think|> +// Let me analyze this step by step... +// First, I need to consider X. +// <|/think|> +// The answer is Y. +// ``` +// +// ### 3. ReAct Agent Loop +// For ReAct (Reason + Act) agent architectures: +// - `<|plan|>` / `<|/plan|>`: High-level strategy +// - `<|step|>` / `<|/step|>`: Individual action step +// - `<|act|>` / `<|/act|>`: Action intent +// - `<|observe|>` / `<|/observe|>`: Environment feedback +// +// Example: +// ``` +// <|plan|>Search for info, then summarize<|/plan|> +// <|act|>search("climate change")<|/act|> +// <|observe|>Found 3 articles...<|/observe|> +// ``` +// +// ### 4. Tool/Function Calling +// Structured tool use with error handling: +// - `<|function|>` / `<|/function|>`: Function call (name + args) +// - `<|result|>` / `<|/result|>`: Successful return value +// - `<|error|>` / `<|/error|>`: Execution error (enables retry) +// +// Example: +// ``` +// <|function|>{"name": "get_weather", "args": {"city": "London"}}<|/function|> +// <|result|>{"temp": 18, "condition": "cloudy"}<|/result|> +// ``` +// +// ### 5. Code Execution +// Jupyter notebook-style code interpreter: +// - `<|code|>` / `<|/code|>`: Code block +// - `<|output|>` / `<|/output|>`: Execution output +// - `<|lang|>` / `<|/lang|>`: Language identifier +// +// Example: +// ``` +// <|code|><|lang|>python<|/lang|> +// print(2 + 2) +// <|/code|> +// <|output|>4<|/output|> +// ``` +// +// ### 6. RAG/Citations +// Retrieval-Augmented Generation with source attribution: +// - `<|context|>` / `<|/context|>`: Retrieved context +// - `<|quote|>` / `<|/quote|>`: Direct quotation +// - `<|cite|>` / `<|/cite|>`: Citation reference +// - `<|source|>` / `<|/source|>`: Source metadata +// +// Example: +// ``` +// <|context|><|source|>doc_123<|/source|> +// The Earth orbits the Sun.<|/context|> +// According to <|cite|>doc_123<|/cite|>, <|quote|>The Earth orbits the Sun.<|/quote|> +// ``` +// +// ### 7. Memory/State +// Long-term memory persistence: +// - `<|memory|>` / `<|/memory|>`: Store information +// - `<|recall|>` / `<|/recall|>`: Retrieve stored info +// +// ### 8. Control Tokens +// Sequence control: +// - `<|pad|>`: Padding for batch alignment +// - `<|stop|>`: Generation stop signal +// - `<|sep|>`: Segment separator +// +// ### 9. Multimodal +// Non-text content placeholders: +// - `<|image|>` / `<|/image|>`: Image data +// - `<|audio|>` / `<|/audio|>`: Audio data +// - `<|video|>` / `<|/video|>`: Video data +// +// ### 10. Document Structure +// Semantic layout for parsing structured documents: +// - `<|title|>` / `<|/title|>`: Document/section title +// - `<|section|>` / `<|/section|>`: Semantic section boundary +// - `<|summary|>` / `<|/summary|>`: Condensed content summary +// +// Example: +// ``` +// <|title|>Introduction<|/title|> +// <|section|>Content here...<|/section|> +// <|summary|>Key points: X, Y, Z<|/summary|> +// ``` +// +// ============================================================================= + /// Get the standard special tokens for cl100k_base encoding. /// /// Returns a map of special token strings to their token IDs for GPT-4 -/// and GPT-3.5-turbo models. Includes: -/// - `<|endoftext|>`: End of text marker -/// - `<|fim_prefix|>`: Fill-in-the-middle prefix -/// - `<|fim_middle|>`: Fill-in-the-middle middle section -/// - `<|fim_suffix|>`: Fill-in-the-middle suffix -/// - `<|endofprompt|>`: End of prompt marker +/// and GPT-3.5-turbo models. +/// +/// ## OpenAI Standard Tokens (100257-100276) +/// - `<|endoftext|>`: End of text marker (100257) +/// - `<|fim_prefix|>`: Fill-in-the-middle prefix (100258) +/// - `<|fim_middle|>`: Fill-in-the-middle middle (100259) +/// - `<|fim_suffix|>`: Fill-in-the-middle suffix (100260) +/// - `<|endofprompt|>`: End of prompt marker (100276) +/// +/// ## Agent Tokens (100277-100324) +/// Extended vocabulary for chat and agent applications. See module docs above. fn cl100k_base_special_tokens() -> FxHashMap { let mut special = FxHashMap::default(); + // OpenAI standard special tokens (100257-100276) special.insert("<|endoftext|>".to_string(), 100257); special.insert("<|fim_prefix|>".to_string(), 100258); special.insert("<|fim_middle|>".to_string(), 100259); special.insert("<|fim_suffix|>".to_string(), 100260); special.insert("<|endofprompt|>".to_string(), 100276); + + // Agent tokens (100277+) - These extend the vocabulary without conflicting + // with OpenAI's reserved range + + // Core conversation structure + special.insert("<|system|>".to_string(), 100277); + special.insert("<|user|>".to_string(), 100278); + special.insert("<|assistant|>".to_string(), 100279); + special.insert("<|im_start|>".to_string(), 100280); + special.insert("<|im_end|>".to_string(), 100281); + + // Reasoning/thinking tokens (System 2 / Chain-of-Thought) + special.insert("<|think|>".to_string(), 100282); + special.insert("<|/think|>".to_string(), 100283); + + // ReAct agent loop tokens + special.insert("<|plan|>".to_string(), 100284); + special.insert("<|/plan|>".to_string(), 100285); + special.insert("<|step|>".to_string(), 100286); + special.insert("<|/step|>".to_string(), 100287); + special.insert("<|act|>".to_string(), 100288); + special.insert("<|/act|>".to_string(), 100289); + special.insert("<|observe|>".to_string(), 100290); + special.insert("<|/observe|>".to_string(), 100291); + + // Tool/function calling + special.insert("<|function|>".to_string(), 100292); + special.insert("<|/function|>".to_string(), 100293); + special.insert("<|result|>".to_string(), 100294); + special.insert("<|/result|>".to_string(), 100295); + special.insert("<|error|>".to_string(), 100296); + special.insert("<|/error|>".to_string(), 100297); + + // Code execution + special.insert("<|code|>".to_string(), 100298); + special.insert("<|/code|>".to_string(), 100299); + special.insert("<|output|>".to_string(), 100300); + special.insert("<|/output|>".to_string(), 100301); + special.insert("<|lang|>".to_string(), 100302); + special.insert("<|/lang|>".to_string(), 100303); + + // RAG/context injection + special.insert("<|context|>".to_string(), 100304); + special.insert("<|/context|>".to_string(), 100305); + special.insert("<|quote|>".to_string(), 100306); + special.insert("<|/quote|>".to_string(), 100307); + special.insert("<|cite|>".to_string(), 100308); + special.insert("<|/cite|>".to_string(), 100309); + special.insert("<|source|>".to_string(), 100310); + special.insert("<|/source|>".to_string(), 100311); + + // Memory/state management + special.insert("<|memory|>".to_string(), 100312); + special.insert("<|/memory|>".to_string(), 100313); + special.insert("<|recall|>".to_string(), 100314); + special.insert("<|/recall|>".to_string(), 100315); + + // Control tokens + special.insert("<|pad|>".to_string(), 100316); + special.insert("<|stop|>".to_string(), 100317); + special.insert("<|sep|>".to_string(), 100318); + + // Multimodal placeholders + special.insert("<|image|>".to_string(), 100319); + special.insert("<|/image|>".to_string(), 100320); + special.insert("<|audio|>".to_string(), 100321); + special.insert("<|/audio|>".to_string(), 100322); + special.insert("<|video|>".to_string(), 100323); + special.insert("<|/video|>".to_string(), 100324); + + // Document structure (semantic layout for parsing structured data) + special.insert("<|title|>".to_string(), 100325); + special.insert("<|/title|>".to_string(), 100326); + special.insert("<|section|>".to_string(), 100327); + special.insert("<|/section|>".to_string(), 100328); + special.insert("<|summary|>".to_string(), 100329); + special.insert("<|/summary|>".to_string(), 100330); + special } @@ -73,8 +308,87 @@ fn cl100k_base_special_tokens() -> FxHashMap { /// - `<|endofprompt|>`: End of prompt marker fn o200k_base_special_tokens() -> FxHashMap { let mut special = FxHashMap::default(); + // OpenAI standard special tokens (199999-200018) special.insert("<|endoftext|>".to_string(), 199999); special.insert("<|endofprompt|>".to_string(), 200018); + + // Agent tokens (200019+) - These extend the vocabulary without conflicting + // with OpenAI's reserved range + + // Core conversation structure + special.insert("<|system|>".to_string(), 200019); + special.insert("<|user|>".to_string(), 200020); + special.insert("<|assistant|>".to_string(), 200021); + special.insert("<|im_start|>".to_string(), 200022); + special.insert("<|im_end|>".to_string(), 200023); + + // Reasoning/thinking tokens (System 2 / Chain-of-Thought) + special.insert("<|think|>".to_string(), 200024); + special.insert("<|/think|>".to_string(), 200025); + + // ReAct agent loop tokens + special.insert("<|plan|>".to_string(), 200026); + special.insert("<|/plan|>".to_string(), 200027); + special.insert("<|step|>".to_string(), 200028); + special.insert("<|/step|>".to_string(), 200029); + special.insert("<|act|>".to_string(), 200030); + special.insert("<|/act|>".to_string(), 200031); + special.insert("<|observe|>".to_string(), 200032); + special.insert("<|/observe|>".to_string(), 200033); + + // Tool/function calling + special.insert("<|function|>".to_string(), 200034); + special.insert("<|/function|>".to_string(), 200035); + special.insert("<|result|>".to_string(), 200036); + special.insert("<|/result|>".to_string(), 200037); + special.insert("<|error|>".to_string(), 200038); + special.insert("<|/error|>".to_string(), 200039); + + // Code execution + special.insert("<|code|>".to_string(), 200040); + special.insert("<|/code|>".to_string(), 200041); + special.insert("<|output|>".to_string(), 200042); + special.insert("<|/output|>".to_string(), 200043); + special.insert("<|lang|>".to_string(), 200044); + special.insert("<|/lang|>".to_string(), 200045); + + // RAG/context injection + special.insert("<|context|>".to_string(), 200046); + special.insert("<|/context|>".to_string(), 200047); + special.insert("<|quote|>".to_string(), 200048); + special.insert("<|/quote|>".to_string(), 200049); + special.insert("<|cite|>".to_string(), 200050); + special.insert("<|/cite|>".to_string(), 200051); + special.insert("<|source|>".to_string(), 200052); + special.insert("<|/source|>".to_string(), 200053); + + // Memory/state management + special.insert("<|memory|>".to_string(), 200054); + special.insert("<|/memory|>".to_string(), 200055); + special.insert("<|recall|>".to_string(), 200056); + special.insert("<|/recall|>".to_string(), 200057); + + // Control tokens + special.insert("<|pad|>".to_string(), 200058); + special.insert("<|stop|>".to_string(), 200059); + special.insert("<|sep|>".to_string(), 200060); + + // Multimodal placeholders + special.insert("<|image|>".to_string(), 200061); + special.insert("<|/image|>".to_string(), 200062); + special.insert("<|audio|>".to_string(), 200063); + special.insert("<|/audio|>".to_string(), 200064); + special.insert("<|video|>".to_string(), 200065); + special.insert("<|/video|>".to_string(), 200066); + + // Document structure (semantic layout for parsing structured data) + special.insert("<|title|>".to_string(), 200067); + special.insert("<|/title|>".to_string(), 200068); + special.insert("<|section|>".to_string(), 200069); + special.insert("<|/section|>".to_string(), 200070); + special.insert("<|summary|>".to_string(), 200071); + special.insert("<|/summary|>".to_string(), 200072); + special } @@ -492,3 +806,478 @@ impl PyStreamingDecoder { } } } + +// ============================================================================= +// Agent Token Constants for Python +// ============================================================================= + +/// Agent token IDs for cl100k_base (GPT-4, GPT-3.5-turbo). +/// +/// Provides constant token IDs for building chat models, reasoning systems, +/// and autonomous agents. Token IDs start at 100277 to avoid conflicts with +/// OpenAI's reserved range (100257-100276). +/// +/// # Python Example +/// +/// ```python +/// from splintr import CL100K_AGENT_TOKENS +/// +/// # Get token IDs +/// system_id = CL100K_AGENT_TOKENS.SYSTEM # 100277 +/// think_id = CL100K_AGENT_TOKENS.THINK # 100282 +/// +/// # Use with tokenizer +/// tokenizer = Tokenizer.from_pretrained("cl100k_base") +/// tokens = tokenizer.encode_with_special("<|think|>reasoning<|/think|>") +/// assert CL100K_AGENT_TOKENS.THINK in tokens +/// ``` +#[pyclass(name = "CL100K_AGENT_TOKENS", frozen)] +pub struct PyCL100KAgentTokens; + +#[pymethods] +impl PyCL100KAgentTokens { + // ========================================================================= + // Conversation Structure (100277-100281) + // ========================================================================= + + /// System message marker - defines assistant behavior (100277) + #[classattr] + const SYSTEM: u32 = 100277; + /// User message marker - human input (100278) + #[classattr] + const USER: u32 = 100278; + /// Assistant message marker - AI responses (100279) + #[classattr] + const ASSISTANT: u32 = 100279; + /// ChatML message start delimiter (100280) + #[classattr] + const IM_START: u32 = 100280; + /// ChatML message end delimiter (100281) + #[classattr] + const IM_END: u32 = 100281; + + // ========================================================================= + // Reasoning/Thinking (100282-100283) + // ========================================================================= + + /// Start of thinking block - Chain-of-Thought reasoning (100282) + #[classattr] + const THINK: u32 = 100282; + /// End of thinking block (100283) + #[classattr] + const THINK_END: u32 = 100283; + + // ========================================================================= + // ReAct Agent Loop (100284-100291) + // ========================================================================= + + /// Start of planning phase (100284) + #[classattr] + const PLAN: u32 = 100284; + /// End of planning phase (100285) + #[classattr] + const PLAN_END: u32 = 100285; + /// Start of step (100286) + #[classattr] + const STEP: u32 = 100286; + /// End of step (100287) + #[classattr] + const STEP_END: u32 = 100287; + /// Start of action (100288) + #[classattr] + const ACT: u32 = 100288; + /// End of action (100289) + #[classattr] + const ACT_END: u32 = 100289; + /// Start of observation (100290) + #[classattr] + const OBSERVE: u32 = 100290; + /// End of observation (100291) + #[classattr] + const OBSERVE_END: u32 = 100291; + + // ========================================================================= + // Tool/Function Calling (100292-100297) + // ========================================================================= + + /// Start of function call (100292) + #[classattr] + const FUNCTION: u32 = 100292; + /// End of function call (100293) + #[classattr] + const FUNCTION_END: u32 = 100293; + /// Start of function result (100294) + #[classattr] + const RESULT: u32 = 100294; + /// End of function result (100295) + #[classattr] + const RESULT_END: u32 = 100295; + /// Start of error block (100296) + #[classattr] + const ERROR: u32 = 100296; + /// End of error block (100297) + #[classattr] + const ERROR_END: u32 = 100297; + + // ========================================================================= + // Code Execution (100298-100303) + // ========================================================================= + + /// Start of code block (100298) + #[classattr] + const CODE: u32 = 100298; + /// End of code block (100299) + #[classattr] + const CODE_END: u32 = 100299; + /// Start of output (100300) + #[classattr] + const OUTPUT: u32 = 100300; + /// End of output (100301) + #[classattr] + const OUTPUT_END: u32 = 100301; + /// Start of language tag (100302) + #[classattr] + const LANG: u32 = 100302; + /// End of language tag (100303) + #[classattr] + const LANG_END: u32 = 100303; + + // ========================================================================= + // RAG/Citations (100304-100311) + // ========================================================================= + + /// Start of context block (100304) + #[classattr] + const CONTEXT: u32 = 100304; + /// End of context block (100305) + #[classattr] + const CONTEXT_END: u32 = 100305; + /// Start of quote (100306) + #[classattr] + const QUOTE: u32 = 100306; + /// End of quote (100307) + #[classattr] + const QUOTE_END: u32 = 100307; + /// Start of citation (100308) + #[classattr] + const CITE: u32 = 100308; + /// End of citation (100309) + #[classattr] + const CITE_END: u32 = 100309; + /// Start of source (100310) + #[classattr] + const SOURCE: u32 = 100310; + /// End of source (100311) + #[classattr] + const SOURCE_END: u32 = 100311; + + // ========================================================================= + // Memory/State (100312-100315) + // ========================================================================= + + /// Start of memory block (100312) + #[classattr] + const MEMORY: u32 = 100312; + /// End of memory block (100313) + #[classattr] + const MEMORY_END: u32 = 100313; + /// Start of recall block (100314) + #[classattr] + const RECALL: u32 = 100314; + /// End of recall block (100315) + #[classattr] + const RECALL_END: u32 = 100315; + + // ========================================================================= + // Control Tokens (100316-100318) + // ========================================================================= + + /// Padding token (100316) + #[classattr] + const PAD: u32 = 100316; + /// Stop token (100317) + #[classattr] + const STOP: u32 = 100317; + /// Separator token (100318) + #[classattr] + const SEP: u32 = 100318; + + // ========================================================================= + // Multimodal (100319-100324) + // ========================================================================= + + /// Start of image (100319) + #[classattr] + const IMAGE: u32 = 100319; + /// End of image (100320) + #[classattr] + const IMAGE_END: u32 = 100320; + /// Start of audio (100321) + #[classattr] + const AUDIO: u32 = 100321; + /// End of audio (100322) + #[classattr] + const AUDIO_END: u32 = 100322; + /// Start of video (100323) + #[classattr] + const VIDEO: u32 = 100323; + /// End of video (100324) + #[classattr] + const VIDEO_END: u32 = 100324; + + // ========================================================================= + // Document Structure (100325-100330) + // ========================================================================= + + /// Start of title - document/section title (100325) + #[classattr] + const TITLE: u32 = 100325; + /// End of title (100326) + #[classattr] + const TITLE_END: u32 = 100326; + /// Start of section - semantic document section (100327) + #[classattr] + const SECTION: u32 = 100327; + /// End of section (100328) + #[classattr] + const SECTION_END: u32 = 100328; + /// Start of summary - condensed content summary (100329) + #[classattr] + const SUMMARY: u32 = 100329; + /// End of summary (100330) + #[classattr] + const SUMMARY_END: u32 = 100330; +} + +/// Agent token IDs for o200k_base (GPT-4o). +/// +/// Provides constant token IDs for building chat models, reasoning systems, +/// and autonomous agents. Token IDs start at 200019 to avoid conflicts with +/// OpenAI's reserved range (199999-200018). +/// +/// # Python Example +/// +/// ```python +/// from splintr import O200K_AGENT_TOKENS +/// +/// # Get token IDs +/// system_id = O200K_AGENT_TOKENS.SYSTEM # 200019 +/// think_id = O200K_AGENT_TOKENS.THINK # 200024 +/// ``` +#[pyclass(name = "O200K_AGENT_TOKENS", frozen)] +pub struct PyO200KAgentTokens; + +#[pymethods] +impl PyO200KAgentTokens { + // ========================================================================= + // Conversation Structure (200019-200023) + // ========================================================================= + + /// System message marker - defines assistant behavior (200019) + #[classattr] + const SYSTEM: u32 = 200019; + /// User message marker - human input (200020) + #[classattr] + const USER: u32 = 200020; + /// Assistant message marker - AI responses (200021) + #[classattr] + const ASSISTANT: u32 = 200021; + /// ChatML message start delimiter (200022) + #[classattr] + const IM_START: u32 = 200022; + /// ChatML message end delimiter (200023) + #[classattr] + const IM_END: u32 = 200023; + + // ========================================================================= + // Reasoning/Thinking (200024-200025) + // ========================================================================= + + /// Start of thinking block - Chain-of-Thought reasoning (200024) + #[classattr] + const THINK: u32 = 200024; + /// End of thinking block (200025) + #[classattr] + const THINK_END: u32 = 200025; + + // ========================================================================= + // ReAct Agent Loop (200026-200033) + // ========================================================================= + + /// Start of planning phase (200026) + #[classattr] + const PLAN: u32 = 200026; + /// End of planning phase (200027) + #[classattr] + const PLAN_END: u32 = 200027; + /// Start of step (200028) + #[classattr] + const STEP: u32 = 200028; + /// End of step (200029) + #[classattr] + const STEP_END: u32 = 200029; + /// Start of action (200030) + #[classattr] + const ACT: u32 = 200030; + /// End of action (200031) + #[classattr] + const ACT_END: u32 = 200031; + /// Start of observation (200032) + #[classattr] + const OBSERVE: u32 = 200032; + /// End of observation (200033) + #[classattr] + const OBSERVE_END: u32 = 200033; + + // ========================================================================= + // Tool/Function Calling (200034-200039) + // ========================================================================= + + /// Start of function call (200034) + #[classattr] + const FUNCTION: u32 = 200034; + /// End of function call (200035) + #[classattr] + const FUNCTION_END: u32 = 200035; + /// Start of function result (200036) + #[classattr] + const RESULT: u32 = 200036; + /// End of function result (200037) + #[classattr] + const RESULT_END: u32 = 200037; + /// Start of error block (200038) + #[classattr] + const ERROR: u32 = 200038; + /// End of error block (200039) + #[classattr] + const ERROR_END: u32 = 200039; + + // ========================================================================= + // Code Execution (200040-200045) + // ========================================================================= + + /// Start of code block (200040) + #[classattr] + const CODE: u32 = 200040; + /// End of code block (200041) + #[classattr] + const CODE_END: u32 = 200041; + /// Start of output (200042) + #[classattr] + const OUTPUT: u32 = 200042; + /// End of output (200043) + #[classattr] + const OUTPUT_END: u32 = 200043; + /// Start of language tag (200044) + #[classattr] + const LANG: u32 = 200044; + /// End of language tag (200045) + #[classattr] + const LANG_END: u32 = 200045; + + // ========================================================================= + // RAG/Citations (200046-200053) + // ========================================================================= + + /// Start of context block (200046) + #[classattr] + const CONTEXT: u32 = 200046; + /// End of context block (200047) + #[classattr] + const CONTEXT_END: u32 = 200047; + /// Start of quote (200048) + #[classattr] + const QUOTE: u32 = 200048; + /// End of quote (200049) + #[classattr] + const QUOTE_END: u32 = 200049; + /// Start of citation (200050) + #[classattr] + const CITE: u32 = 200050; + /// End of citation (200051) + #[classattr] + const CITE_END: u32 = 200051; + /// Start of source (200052) + #[classattr] + const SOURCE: u32 = 200052; + /// End of source (200053) + #[classattr] + const SOURCE_END: u32 = 200053; + + // ========================================================================= + // Memory/State (200054-200057) + // ========================================================================= + + /// Start of memory block (200054) + #[classattr] + const MEMORY: u32 = 200054; + /// End of memory block (200055) + #[classattr] + const MEMORY_END: u32 = 200055; + /// Start of recall block (200056) + #[classattr] + const RECALL: u32 = 200056; + /// End of recall block (200057) + #[classattr] + const RECALL_END: u32 = 200057; + + // ========================================================================= + // Control Tokens (200058-200060) + // ========================================================================= + + /// Padding token (200058) + #[classattr] + const PAD: u32 = 200058; + /// Stop token (200059) + #[classattr] + const STOP: u32 = 200059; + /// Separator token (200060) + #[classattr] + const SEP: u32 = 200060; + + // ========================================================================= + // Multimodal (200061-200066) + // ========================================================================= + + /// Start of image (200061) + #[classattr] + const IMAGE: u32 = 200061; + /// End of image (200062) + #[classattr] + const IMAGE_END: u32 = 200062; + /// Start of audio (200063) + #[classattr] + const AUDIO: u32 = 200063; + /// End of audio (200064) + #[classattr] + const AUDIO_END: u32 = 200064; + /// Start of video (200065) + #[classattr] + const VIDEO: u32 = 200065; + /// End of video (200066) + #[classattr] + const VIDEO_END: u32 = 200066; + + // ========================================================================= + // Document Structure (200067-200072) + // ========================================================================= + + /// Start of title - document/section title (200067) + #[classattr] + const TITLE: u32 = 200067; + /// End of title (200068) + #[classattr] + const TITLE_END: u32 = 200068; + /// Start of section - semantic document section (200069) + #[classattr] + const SECTION: u32 = 200069; + /// End of section (200070) + #[classattr] + const SECTION_END: u32 = 200070; + /// Start of summary - condensed content summary (200071) + #[classattr] + const SUMMARY: u32 = 200071; + /// End of summary (200072) + #[classattr] + const SUMMARY_END: u32 = 200072; +} diff --git a/src/python/mod.rs b/src/python/mod.rs index f4e63bf..d4e4678 100644 --- a/src/python/mod.rs +++ b/src/python/mod.rs @@ -1,3 +1,3 @@ mod bindings; -pub use bindings::{PyStreamingDecoder, PyTokenizer}; +pub use bindings::{PyCL100KAgentTokens, PyO200KAgentTokens, PyStreamingDecoder, PyTokenizer}; From 76f0be2531aad60b787cd0a830cf7713066b0899 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 26 Nov 2025 12:34:33 +0800 Subject: [PATCH 2/3] feat: implement tag-based versioning system with single source of truth Add unified versioning system that uses .version file as the single source of truth for package versions. The system validates git tags against the base version and automatically converts versions to platform-specific formats during CI builds. Key components: - .version file stores base semantic version (0.2.0) - scripts/update_version.sh validates tags and updates Cargo.toml/pyproject.toml - Release workflow validates tags before building packages - Supports prerelease tags: alpha, beta, rc (case-insensitive) - Converts versions for each platform (Cargo semver vs PyPI PEP 440) - Fails with clear errors on tag/version mismatches This ensures version consistency across Rust and Python packages while supporting both stable and prerelease versions through git tags. --- .github/workflows/release.yml | 104 ++++++++++++++++++++++- .gitignore | 3 +- .version | 1 + Cargo.toml | 2 +- pyproject.toml | 2 +- scripts/update_version.sh | 155 ++++++++++++++++++++++++++++++++++ 6 files changed, 263 insertions(+), 4 deletions(-) create mode 100644 .version create mode 100755 scripts/update_version.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1130d21..87d2ea1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,9 +9,73 @@ permissions: contents: read jobs: + # Validate tag matches .version file + validate-version: + name: Validate Version Tag + runs-on: ubuntu-latest + outputs: + cargo_version: ${{ steps.version.outputs.cargo_version }} + pypi_version: ${{ steps.version.outputs.pypi_version }} + base_version: ${{ steps.version.outputs.base_version }} + steps: + - uses: actions/checkout@v4 + + - name: Validate and extract version + id: version + run: | + TAG="${GITHUB_REF_NAME}" + BASE_VERSION=$(cat .version | tr -d '[:space:]') + TAG_VERSION="${TAG#v}" + + echo "Tag: $TAG" + echo "Base version from .version: $BASE_VERSION" + + # Validate tag format + if [[ ! "$TAG_VERSION" =~ ^([0-9]+\.[0-9]+\.[0-9]+)(-([a-zA-Z]+)\.([0-9]+))?$ ]]; then + echo "::error::Invalid tag format '$TAG'. Expected: vX.Y.Z or vX.Y.Z-{alpha|beta|rc}.N" + exit 1 + fi + + TAG_BASE="${BASH_REMATCH[1]}" + PRERELEASE_TYPE="${BASH_REMATCH[3]}" + PRERELEASE_NUM="${BASH_REMATCH[4]}" + + # Validate base version matches + if [[ "$TAG_BASE" != "$BASE_VERSION" ]]; then + echo "::error::Version mismatch! Tag base '$TAG_BASE' does not match .version file '$BASE_VERSION'" + echo "::error::Valid tags: v$BASE_VERSION, v$BASE_VERSION-alpha.N, v$BASE_VERSION-beta.N, v$BASE_VERSION-rc.N" + exit 1 + fi + + # Determine version strings (convert prerelease type to lowercase) + if [[ -n "$PRERELEASE_TYPE" ]]; then + PRERELEASE_TYPE_LOWER=$(echo "$PRERELEASE_TYPE" | tr '[:upper:]' '[:lower:]') + CARGO_VERSION="$BASE_VERSION-$PRERELEASE_TYPE_LOWER.$PRERELEASE_NUM" + case "$PRERELEASE_TYPE_LOWER" in + alpha) PYPI_VERSION="${BASE_VERSION}a${PRERELEASE_NUM}" ;; + beta) PYPI_VERSION="${BASE_VERSION}b${PRERELEASE_NUM}" ;; + rc) PYPI_VERSION="${BASE_VERSION}rc${PRERELEASE_NUM}" ;; + *) + echo "::error::Unknown prerelease type '$PRERELEASE_TYPE'. Use: alpha, beta, rc (case-insensitive)" + exit 1 + ;; + esac + else + CARGO_VERSION="$BASE_VERSION" + PYPI_VERSION="$BASE_VERSION" + fi + + echo "Cargo version: $CARGO_VERSION" + echo "PyPI version: $PYPI_VERSION" + + echo "cargo_version=$CARGO_VERSION" >> $GITHUB_OUTPUT + echo "pypi_version=$PYPI_VERSION" >> $GITHUB_OUTPUT + echo "base_version=$BASE_VERSION" >> $GITHUB_OUTPUT + # Build and publish to crates.io publish-crate: name: Publish to crates.io + needs: validate-version runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -22,12 +86,25 @@ jobs: - name: Install PCRE2 dependencies run: sudo apt-get update && sudo apt-get install -y libpcre2-dev + - name: Update version in Cargo.toml + run: | + # Use awk to update version only in [package] section + awk -v ver="${{ needs.validate-version.outputs.cargo_version }}" ' + /^\[package\]/ { in_package=1 } + /^\[/ && !/^\[package\]/ { in_package=0 } + in_package && /^version = "/ { print "version = \"" ver "\""; next } + { print } + ' Cargo.toml > Cargo.toml.tmp && mv Cargo.toml.tmp Cargo.toml + echo "Updated Cargo.toml to version ${{ needs.validate-version.outputs.cargo_version }}" + grep "^version" Cargo.toml + - name: Publish to crates.io run: cargo publish --token ${{ secrets.CARGO_REGISTRY_TOKEN }} # Build Python wheels for multiple platforms build-wheels: name: Build wheels on ${{ matrix.os }} + needs: validate-version runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -42,6 +119,19 @@ jobs: with: python-version: '3.12' + - name: Update version in pyproject.toml + shell: bash + run: | + # Use awk to update version only in [project] section + awk -v ver="${{ needs.validate-version.outputs.pypi_version }}" ' + /^\[project\]/ { in_project=1 } + /^\[/ && !/^\[project\]/ { in_project=0 } + in_project && /^version = "/ { print "version = \"" ver "\""; next } + { print } + ' pyproject.toml > pyproject.toml.tmp && mv pyproject.toml.tmp pyproject.toml + echo "Updated pyproject.toml to version ${{ needs.validate-version.outputs.pypi_version }}" + grep "^version" pyproject.toml + - name: Install PCRE2 (Ubuntu) if: matrix.os == 'ubuntu-latest' run: sudo apt-get update && sudo apt-get install -y libpcre2-dev @@ -72,10 +162,22 @@ jobs: # Build source distribution build-sdist: name: Build source distribution + needs: validate-version runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Update version in pyproject.toml + run: | + # Use awk to update version only in [project] section + awk -v ver="${{ needs.validate-version.outputs.pypi_version }}" ' + /^\[project\]/ { in_project=1 } + /^\[/ && !/^\[project\]/ { in_project=0 } + in_project && /^version = "/ { print "version = \"" ver "\""; next } + { print } + ' pyproject.toml > pyproject.toml.tmp && mv pyproject.toml.tmp pyproject.toml + echo "Updated pyproject.toml to version ${{ needs.validate-version.outputs.pypi_version }}" + - name: Build sdist uses: PyO3/maturin-action@v1 with: @@ -91,7 +193,7 @@ jobs: # Publish to PyPI publish-pypi: name: Publish to PyPI - needs: [build-wheels, build-sdist] + needs: [validate-version, build-wheels, build-sdist] runs-on: ubuntu-latest environment: name: pypi diff --git a/.gitignore b/.gitignore index 30b93ae..9661d04 100644 --- a/.gitignore +++ b/.gitignore @@ -59,4 +59,5 @@ htmlcov/ coverage.xml *.cover -/research \ No newline at end of file +/research +.python-version \ No newline at end of file diff --git a/.version b/.version new file mode 100644 index 0000000..0ea3a94 --- /dev/null +++ b/.version @@ -0,0 +1 @@ +0.2.0 diff --git a/Cargo.toml b/Cargo.toml index ba8fb88..851ad9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "splintr" -version = "0.1.0-beta.1" +version = "0.2.0" edition = "2021" description = "Fast Rust BPE tokenizer with Python bindings" license = "MIT" diff --git a/pyproject.toml b/pyproject.toml index 8ceeda3..d52ff08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "splintr-rs" -version = "0.1.0b1" +version = "0.2.0" description = "Fast Rust BPE tokenizer with Python bindings" readme = "README.md" license = {text = "MIT"} diff --git a/scripts/update_version.sh b/scripts/update_version.sh new file mode 100755 index 0000000..2c51ba1 --- /dev/null +++ b/scripts/update_version.sh @@ -0,0 +1,155 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" + +VERSION_FILE="$PROJECT_ROOT/.version" +CARGO_TOML="$PROJECT_ROOT/Cargo.toml" +PYPROJECT_TOML="$PROJECT_ROOT/pyproject.toml" + +# Read base version from .version file +if [[ ! -f "$VERSION_FILE" ]]; then + echo "Error: .version file not found at $VERSION_FILE" + exit 1 +fi + +BASE_VERSION=$(cat "$VERSION_FILE" | tr -d '[:space:]') + +# Validate base version format (semver: X.Y.Z) +if [[ ! "$BASE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Error: Invalid base version format '$BASE_VERSION'. Expected X.Y.Z" + exit 1 +fi + +# Get tag from argument or environment +TAG="${1:-$GITHUB_REF_NAME}" + +if [[ -z "$TAG" ]]; then + # No tag provided, use base version + CARGO_VERSION="$BASE_VERSION" + PYPI_VERSION="$BASE_VERSION" + echo "No tag provided, using base version: $BASE_VERSION" +else + # Remove 'v' prefix if present + TAG_VERSION="${TAG#v}" + + # Extract base version and prerelease from tag + # Supports: v0.1.0, v0.1.0-beta.1, v0.1.0-alpha.2, v0.1.0-rc.1 + if [[ "$TAG_VERSION" =~ ^([0-9]+\.[0-9]+\.[0-9]+)(-([a-zA-Z]+)\.([0-9]+))?$ ]]; then + TAG_BASE="${BASH_REMATCH[1]}" + PRERELEASE_TYPE="${BASH_REMATCH[3]}" + PRERELEASE_NUM="${BASH_REMATCH[4]}" + + # Validate tag base matches .version file + if [[ "$TAG_BASE" != "$BASE_VERSION" ]]; then + echo "Error: Tag version mismatch!" + echo " Tag base version: $TAG_BASE" + echo " .version file: $BASE_VERSION" + echo "" + echo "The tag must match the base version in .version file." + echo "Valid tags for version $BASE_VERSION:" + echo " - v$BASE_VERSION" + echo " - v$BASE_VERSION-alpha.N" + echo " - v$BASE_VERSION-beta.N" + echo " - v$BASE_VERSION-rc.N" + exit 1 + fi + + if [[ -n "$PRERELEASE_TYPE" ]]; then + # Convert to lowercase for case-insensitive matching + PRERELEASE_TYPE_LOWER=$(echo "$PRERELEASE_TYPE" | tr '[:upper:]' '[:lower:]') + + # Convert prerelease type for Cargo (uses hyphen) and PyPI (uses different format) + # Cargo: 0.1.0-beta.1 + # PyPI: 0.1.0b1 (alpha=a, beta=b, rc=rc) + CARGO_VERSION="$BASE_VERSION-$PRERELEASE_TYPE_LOWER.$PRERELEASE_NUM" + + case "$PRERELEASE_TYPE_LOWER" in + alpha) + PYPI_VERSION="${BASE_VERSION}a${PRERELEASE_NUM}" + ;; + beta) + PYPI_VERSION="${BASE_VERSION}b${PRERELEASE_NUM}" + ;; + rc) + PYPI_VERSION="${BASE_VERSION}rc${PRERELEASE_NUM}" + ;; + *) + echo "Error: Unknown prerelease type '$PRERELEASE_TYPE'" + echo "Supported types: alpha, beta, rc (case-insensitive)" + exit 1 + ;; + esac + + echo "Prerelease version detected:" + echo " Cargo version: $CARGO_VERSION" + echo " PyPI version: $PYPI_VERSION" + else + # Stable release + CARGO_VERSION="$BASE_VERSION" + PYPI_VERSION="$BASE_VERSION" + echo "Stable release: $BASE_VERSION" + fi + else + echo "Error: Invalid tag format '$TAG'" + echo "Expected format: vX.Y.Z or vX.Y.Z-{alpha|beta|rc}.N" + echo "Examples: v0.1.0, v0.1.0-beta.1, v0.1.0-rc.2" + exit 1 + fi +fi + +echo "" +echo "Updating version files..." + +# Update Cargo.toml - only update version in [package] section +if [[ -f "$CARGO_TOML" ]]; then + # Use awk to update version only in [package] section + awk -v ver="$CARGO_VERSION" ' + /^\[package\]/ { in_package=1 } + /^\[/ && !/^\[package\]/ { in_package=0 } + in_package && /^version = "/ { print "version = \"" ver "\""; next } + { print } + ' "$CARGO_TOML" > "$CARGO_TOML.tmp" && mv "$CARGO_TOML.tmp" "$CARGO_TOML" + + # Verify the update worked (grep first version line and extract quoted string) + UPDATED_VERSION=$(grep '^version = "' "$CARGO_TOML" | head -1 | sed 's/.*"\(.*\)".*/\1/') + if [[ "$UPDATED_VERSION" != "$CARGO_VERSION" ]]; then + echo "Error: Failed to update version in $CARGO_TOML" + echo " Expected: $CARGO_VERSION" + echo " Got: $UPDATED_VERSION" + exit 1 + fi + echo " Updated $CARGO_TOML -> $CARGO_VERSION" +else + echo " Warning: $CARGO_TOML not found" +fi + +# Update pyproject.toml - only update version in [project] section +if [[ -f "$PYPROJECT_TOML" ]]; then + # Use awk to update version only in [project] section + awk -v ver="$PYPI_VERSION" ' + /^\[project\]/ { in_project=1 } + /^\[/ && !/^\[project\]/ { in_project=0 } + in_project && /^version = "/ { print "version = \"" ver "\""; next } + { print } + ' "$PYPROJECT_TOML" > "$PYPROJECT_TOML.tmp" && mv "$PYPROJECT_TOML.tmp" "$PYPROJECT_TOML" + + # Verify the update worked (grep first version line and extract quoted string) + UPDATED_VERSION=$(grep '^version = "' "$PYPROJECT_TOML" | head -1 | sed 's/.*"\(.*\)".*/\1/') + if [[ "$UPDATED_VERSION" != "$PYPI_VERSION" ]]; then + echo "Error: Failed to update version in $PYPROJECT_TOML" + echo " Expected: $PYPI_VERSION" + echo " Got: $UPDATED_VERSION" + exit 1 + fi + echo " Updated $PYPROJECT_TOML -> $PYPI_VERSION" +else + echo " Warning: $PYPROJECT_TOML not found" +fi + +echo "" +echo "Version update complete!" +echo " Base version: $BASE_VERSION" +echo " Cargo.toml: $CARGO_VERSION" +echo " pyproject.toml: $PYPI_VERSION" From 04a6e85e613c040da132be8d18671732fe87b295 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Wed, 26 Nov 2025 12:55:10 +0800 Subject: [PATCH 3/3] refactor(tokenizer): convert constant assertions to compile-time checks Replace runtime test assertions on constants with compile-time const block assertions to satisfy Clippy lint (assertions_on_constants). The verification logic remains unchanged but now executes at compile time rather than test time, providing earlier detection of constant constraint violations. --- src/core/tokenizer.rs | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/core/tokenizer.rs b/src/core/tokenizer.rs index 4fa81ab..ad77d85 100644 --- a/src/core/tokenizer.rs +++ b/src/core/tokenizer.rs @@ -909,26 +909,16 @@ mod tests { assert_eq!(tokenizer.cache_len(), 0); } - #[test] - fn test_agent_tokens_constants() { - // Verify cl100k agent tokens don't conflict with OpenAI's reserved range + // Compile-time verification that agent tokens don't conflict with OpenAI's reserved range + const _: () = { assert!(super::cl100k_agent_tokens::SYSTEM > 100276); // After endofprompt assert!(super::cl100k_agent_tokens::SUMMARY_END == 100330); // Last token - - // Verify o200k agent tokens don't conflict with OpenAI's reserved range assert!(super::o200k_agent_tokens::SYSTEM > 200018); // After endofprompt assert!(super::o200k_agent_tokens::SUMMARY_END == 200072); // Last token - - // Verify token ordering is correct (no gaps or overlaps) - assert_eq!( - super::cl100k_agent_tokens::USER, - super::cl100k_agent_tokens::SYSTEM + 1 - ); - assert_eq!( - super::o200k_agent_tokens::USER, - super::o200k_agent_tokens::SYSTEM + 1 - ); - } + // Verify token ordering is correct (no gaps or overlaps) + assert!(super::cl100k_agent_tokens::USER == super::cl100k_agent_tokens::SYSTEM + 1); + assert!(super::o200k_agent_tokens::USER == super::o200k_agent_tokens::SYSTEM + 1); + }; #[test] fn test_agent_tokens_encode_decode() {