Feat/versioning system

.github/workflows/release.yml

@@ -9,9 +9,73 @@ permissions:
   contents: read
 
 jobs:
+  # Validate tag matches .version file
+  validate-version:
+    name: Validate Version Tag
+    runs-on: ubuntu-latest
+    outputs:
+      cargo_version: ${{ steps.version.outputs.cargo_version }}
+      pypi_version: ${{ steps.version.outputs.pypi_version }}
+      base_version: ${{ steps.version.outputs.base_version }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Validate and extract version
+        id: version
+        run: |
+          TAG="${GITHUB_REF_NAME}"
+          BASE_VERSION=$(cat .version | tr -d '[:space:]')
+          TAG_VERSION="${TAG#v}"
+
+          echo "Tag: $TAG"
+          echo "Base version from .version: $BASE_VERSION"
+
+          # Validate tag format
+          if [[ ! "$TAG_VERSION" =~ ^([0-9]+\.[0-9]+\.[0-9]+)(-([a-zA-Z]+)\.([0-9]+))?$ ]]; then
+            echo "::error::Invalid tag format '$TAG'. Expected: vX.Y.Z or vX.Y.Z-{alpha|beta|rc}.N"
+            exit 1
+          fi
+
+          TAG_BASE="${BASH_REMATCH[1]}"
+          PRERELEASE_TYPE="${BASH_REMATCH[3]}"
+          PRERELEASE_NUM="${BASH_REMATCH[4]}"
+
+          # Validate base version matches
+          if [[ "$TAG_BASE" != "$BASE_VERSION" ]]; then
+            echo "::error::Version mismatch! Tag base '$TAG_BASE' does not match .version file '$BASE_VERSION'"
+            echo "::error::Valid tags: v$BASE_VERSION, v$BASE_VERSION-alpha.N, v$BASE_VERSION-beta.N, v$BASE_VERSION-rc.N"
+            exit 1
+          fi
+
+          # Determine version strings (convert prerelease type to lowercase)
+          if [[ -n "$PRERELEASE_TYPE" ]]; then
+            PRERELEASE_TYPE_LOWER=$(echo "$PRERELEASE_TYPE" | tr '[:upper:]' '[:lower:]')
+            CARGO_VERSION="$BASE_VERSION-$PRERELEASE_TYPE_LOWER.$PRERELEASE_NUM"
+            case "$PRERELEASE_TYPE_LOWER" in
+              alpha) PYPI_VERSION="${BASE_VERSION}a${PRERELEASE_NUM}" ;;
+              beta)  PYPI_VERSION="${BASE_VERSION}b${PRERELEASE_NUM}" ;;
+              rc)    PYPI_VERSION="${BASE_VERSION}rc${PRERELEASE_NUM}" ;;
+              *)
+                echo "::error::Unknown prerelease type '$PRERELEASE_TYPE'. Use: alpha, beta, rc (case-insensitive)"
+                exit 1
+                ;;
+            esac
+          else
+            CARGO_VERSION="$BASE_VERSION"
+            PYPI_VERSION="$BASE_VERSION"
+          fi
+
+          echo "Cargo version: $CARGO_VERSION"
+          echo "PyPI version: $PYPI_VERSION"
+
+          echo "cargo_version=$CARGO_VERSION" >> $GITHUB_OUTPUT
+          echo "pypi_version=$PYPI_VERSION" >> $GITHUB_OUTPUT
+          echo "base_version=$BASE_VERSION" >> $GITHUB_OUTPUT
+
   # Build and publish to crates.io
   publish-crate:
     name: Publish to crates.io
+    needs: validate-version
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -22,12 +86,25 @@ jobs:
       - name: Install PCRE2 dependencies
         run: sudo apt-get update && sudo apt-get install -y libpcre2-dev
 
+      - name: Update version in Cargo.toml
+        run: |
+          # Use awk to update version only in [package] section
+          awk -v ver="${{ needs.validate-version.outputs.cargo_version }}" '
+            /^\[package\]/ { in_package=1 }
+            /^\[/ && !/^\[package\]/ { in_package=0 }
+            in_package && /^version = "/ { print "version = \"" ver "\""; next }
+            { print }
+          ' Cargo.toml > Cargo.toml.tmp && mv Cargo.toml.tmp Cargo.toml
+          echo "Updated Cargo.toml to version ${{ needs.validate-version.outputs.cargo_version }}"
+          grep "^version" Cargo.toml
+
       - name: Publish to crates.io
         run: cargo publish --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
 
   # Build Python wheels for multiple platforms
   build-wheels:
     name: Build wheels on ${{ matrix.os }}
+    needs: validate-version
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -42,6 +119,19 @@ jobs:
         with:
           python-version: '3.12'
 
+      - name: Update version in pyproject.toml
+        shell: bash
+        run: |
+          # Use awk to update version only in [project] section
+          awk -v ver="${{ needs.validate-version.outputs.pypi_version }}" '
+            /^\[project\]/ { in_project=1 }
+            /^\[/ && !/^\[project\]/ { in_project=0 }
+            in_project && /^version = "/ { print "version = \"" ver "\""; next }
+            { print }
+          ' pyproject.toml > pyproject.toml.tmp && mv pyproject.toml.tmp pyproject.toml
+          echo "Updated pyproject.toml to version ${{ needs.validate-version.outputs.pypi_version }}"
+          grep "^version" pyproject.toml
+
       - name: Install PCRE2 (Ubuntu)
         if: matrix.os == 'ubuntu-latest'
         run: sudo apt-get update && sudo apt-get install -y libpcre2-dev
@@ -72,10 +162,22 @@ jobs:
   # Build source distribution
   build-sdist:
     name: Build source distribution
+    needs: validate-version
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
+      - name: Update version in pyproject.toml
+        run: |
+          # Use awk to update version only in [project] section
+          awk -v ver="${{ needs.validate-version.outputs.pypi_version }}" '
+            /^\[project\]/ { in_project=1 }
+            /^\[/ && !/^\[project\]/ { in_project=0 }
+            in_project && /^version = "/ { print "version = \"" ver "\""; next }
+            { print }
+          ' pyproject.toml > pyproject.toml.tmp && mv pyproject.toml.tmp pyproject.toml
+          echo "Updated pyproject.toml to version ${{ needs.validate-version.outputs.pypi_version }}"
+
       - name: Build sdist
         uses: PyO3/maturin-action@v1
         with:
@@ -91,7 +193,7 @@ jobs:
   # Publish to PyPI
   publish-pypi:
     name: Publish to PyPI
-    needs: [build-wheels, build-sdist]
+    needs: [validate-version, build-wheels, build-sdist]
     runs-on: ubuntu-latest
     environment:
       name: pypi

.gitignore

@@ -59,4 +59,5 @@ htmlcov/
 coverage.xml
 *.cover
 
-/research
+/research
+.python-version

.version

@@ -0,0 +1 @@
+0.2.0

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "splintr"
-version = "0.1.0-beta.1"
+version = "0.2.0"
 edition = "2021"
 description = "Fast Rust BPE tokenizer with Python bindings"
 license = "MIT"

README.md

@@ -1,4 +1,4 @@
-# splintr
+# Splintr
 
 [![Crates.io](https://img.shields.io/crates/v/splintr.svg)](https://crates.io/crates/splintr)
 [![PyPI](https://img.shields.io/pypi/v/splintr-rs.svg)](https://pypi.org/project/splintr-rs/)
@@ -8,7 +8,7 @@ A high-performance BPE tokenizer implemented in Rust with Python bindings, desig
 
 ## Features
 
-splintr implements several optimizations that make tokenization faster and more efficient:
+Splintr implements several optimizations that make tokenization faster and more efficient:
 
 - **PCRE2 with JIT compilation**: Uses PCRE2's just-in-time compilation for regex matching, providing 2-4x speedup over fancy-regex on pattern matching operations
 - **Rayon parallelism**: Leverages multiple CPU cores for encoding batches of text and individual regex chunks within each text
@@ -17,6 +17,7 @@ splintr implements several optimizations that make tokenization faster and more
 - **Aho-Corasick for special tokens**: Employs the Aho-Corasick algorithm for fast multi-pattern matching of special tokens, avoiding regex alternation overhead
 - **LRU cache**: Caches frequently encoded text chunks to avoid redundant BPE encoding operations
 - **UTF-8 streaming decoder**: Safely handles token-by-token decoding for LLM output, buffering incomplete UTF-8 sequences across token boundaries
+- **Extended agent tokens**: 54 special tokens for chat models, Chain-of-Thought reasoning, ReAct agents, tool calling, RAG citations, and multimodal applications (see [Special Tokens](docs/special_tokens.md))
 
 ## Installation
 
@@ -192,6 +193,7 @@ print(decoder.flush())
 ```
 
 This approach ensures that:
+
 1. Users see text as soon as complete characters are available
 2. Multi-byte Unicode characters display correctly
 3. No corruption occurs at token boundaries
@@ -204,27 +206,28 @@ Benchmarks performed on Linux (6.16.8-arch3-1) with 24 CPU cores, comparing spli
 
 Performance on various text types:
 
-| Content Type | Size | splintr (ms) | tiktoken (ms) | Speedup |
-|--------------|------|--------------|---------------|---------|
-| Long English | 450,000 chars | 7.94 | 19.91 | **2.5x** |
-| Python Code | 59,200 chars | 1.67 | 5.90 | **3.5x** |
-| JSON | 29,000 chars | 1.20 | 2.76 | **2.3x** |
-| Numbers | 55,000 chars | 2.27 | 6.09 | **2.7x** |
-| Whitespace-heavy | 50,000 chars | 1.36 | 4.91 | **3.6x** |
-| Chinese | 11,500 chars | 1.09 | 1.45 | **1.3x** |
+| Content Type     | Size          | splintr (ms) | tiktoken (ms) | Speedup  |
+| ---------------- | ------------- | ------------ | ------------- | -------- |
+| Long English     | 450,000 chars | 7.94         | 19.91         | **2.5x** |
+| Python Code      | 59,200 chars  | 1.67         | 5.90          | **3.5x** |
+| JSON             | 29,000 chars  | 1.20         | 2.76          | **2.3x** |
+| Numbers          | 55,000 chars  | 2.27         | 6.09          | **2.7x** |
+| Whitespace-heavy | 50,000 chars  | 1.36         | 4.91          | **3.6x** |
+| Chinese          | 11,500 chars  | 1.09         | 1.45          | **1.3x** |
 
 ### Batch Encoding
 
 Batch operations show significant speedup through parallelism:
 
-| Configuration | splintr parallel (ms) | tiktoken (ms) | Speedup vs tiktoken |
-|---------------|----------------------|---------------|---------------------|
-| 10 × 1,000 chars | 0.25 | 0.48 | **1.9x** |
-| 100 × 1,000 chars | 1.11 | 4.66 | **4.2x** |
-| 1,000 × 100 chars | 1.42 | 6.95 | **4.9x** |
-| 100 × 10,000 chars | 8.24 | 45.72 | **5.5x** |
+| Configuration      | splintr parallel (ms) | tiktoken (ms) | Speedup vs tiktoken |
+| ------------------ | --------------------- | ------------- | ------------------- |
+| 10 × 1,000 chars   | 0.25                  | 0.48          | **1.9x**            |
+| 100 × 1,000 chars  | 1.11                  | 4.66          | **4.2x**            |
+| 1,000 × 100 chars  | 1.42                  | 6.95          | **4.9x**            |
+| 100 × 10,000 chars | 8.24                  | 45.72         | **5.5x**            |
 
 **Parallel speedup within splintr:**
+
 - 100 × 1,000 chars: 8.6x faster (parallel vs sequential)
 - 1,000 × 100 chars: 16.8x faster (parallel vs sequential)
 
@@ -250,6 +253,7 @@ cat results/my_benchmark.md
 ```
 
 The benchmark suite tests:
+
 - Single text encoding across various content types (English, code, multilingual, etc.)
 - Batch encoding with different batch sizes and text lengths
 - Streaming decoder performance
@@ -259,22 +263,55 @@ You can customize the benchmark by modifying `benchmark.py` or adding your own t
 
 ## Supported Models
 
-| Model | Use Case | Vocabulary Size | Special Tokens | Import Constant |
-|-------|----------|----------------|----------------|-----------------|
-| `cl100k_base` | GPT-4, GPT-3.5-turbo | ~100,000 | 5 | `CL100K_BASE_PATTERN` |
-| `o200k_base` | GPT-4o | ~200,000 | 2 | `O200K_BASE_PATTERN` |
+| Model         | Use Case             | Vocabulary Size | Special Tokens | Import Constant       |
+| ------------- | -------------------- | --------------- | -------------- | --------------------- |
+| `cl100k_base` | GPT-4, GPT-3.5-turbo | ~100,000        | 5 + 54 agent   | `CL100K_BASE_PATTERN` |
+| `o200k_base`  | GPT-4o               | ~200,000        | 2 + 54 agent   | `O200K_BASE_PATTERN`  |
 
-**Special tokens:**
+**OpenAI standard tokens:**
 
 - **cl100k_base**: `<|endoftext|>`, `<|fim_prefix|>`, `<|fim_middle|>`, `<|fim_suffix|>`, `<|endofprompt|>`
 - **o200k_base**: `<|endoftext|>`, `<|endofprompt|>`
 
+**Agent tokens (54 per model):**
+
+Splintr extends both vocabularies with tokens for building agent systems. See [docs/special_tokens.md](docs/special_tokens.md) for complete documentation.
+
+```python
+from splintr import Tokenizer, CL100K_AGENT_TOKENS
+
+tokenizer = Tokenizer.from_pretrained("cl100k_base")
+
+# Encode with special tokens
+text = "<|think|>Let me reason...<|/think|>The answer is 42."
+tokens = tokenizer.encode_with_special(text)
+
+# Access token IDs programmatically
+print(CL100K_AGENT_TOKENS.THINK)      # 100282
+print(CL100K_AGENT_TOKENS.FUNCTION)   # 100292
+```
+
+| Category     | Tokens                                              | Purpose                    |
+| ------------ | --------------------------------------------------- | -------------------------- |
+| Conversation | `system`, `user`, `assistant`, `im_start`, `im_end` | ChatML format              |
+| Thinking     | `think`                                             | Chain-of-Thought reasoning |
+| ReAct        | `plan`, `step`, `act`, `observe`                    | Agent action loops         |
+| Tools        | `function`, `result`, `error`                       | Function calling           |
+| Code         | `code`, `output`, `lang`                            | Code execution             |
+| RAG          | `context`, `quote`, `cite`, `source`                | Citations                  |
+| Memory       | `memory`, `recall`                                  | State persistence          |
+| Control      | `pad`, `stop`, `sep`                                | Sequence control           |
+| Multimodal   | `image`, `audio`, `video`                           | Non-text content           |
+| Document     | `title`, `section`, `summary`                       | Structured docs            |
+
 ## Use Cases
 
-splintr is designed for:
+Splintr is designed for:
 
 - **LLM applications**: Tokenizing prompts and streaming decoder for real-time output display
+- **Agent systems**: Building ReAct agents, tool-calling systems, and Chain-of-Thought reasoning
 - **Training pipelines**: Fast batch encoding of large datasets for model training
+- **RAG applications**: Structured context injection with citation support
 - **Token counting**: Estimating API costs or enforcing token limits
 - **Text preprocessing**: Converting text to tokens for embedding models or other NLP tasks
 
@@ -321,7 +358,8 @@ This project is licensed under the MIT License - see the LICENSE file for detail
 
 ## Acknowledgments
 
-splintr builds upon concepts from:
+Splintr builds upon concepts from:
+
 - [tiktoken](https://github.com/openai/tiktoken) - OpenAI's reference BPE tokenizer
 - [tokenizers](https://github.com/huggingface/tokenizers) - Hugging Face's tokenization library