fix(ci): use correct rust-toolchain action and venv for maturin

.github/workflows/ci.yml

@@ -1,17 +1,37 @@
 name: CI
 
 on:
-  push:
-    branches: [main]
   pull_request:
     branches: [main]
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
+  # Lint and format check (single job)
+  lint:
+    name: Lint & Format
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+
+      - name: Install PCRE2
+        run: sudo apt-get update && sudo apt-get install -y libpcre2-dev
+
+      - name: Check formatting
+        run: cargo fmt --all --check
+
+      - name: Run clippy
+        run: cargo clippy --all-targets -- -D warnings
+
+  # Rust tests on multiple platforms
   test:
-    name: Test on ${{ matrix.os }}
+    name: Test (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -21,44 +41,54 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
       - name: Install Rust
-        uses: dtolnay/rust-action@stable
+        uses: dtolnay/rust-toolchain@stable
 
       - name: Install PCRE2 (Ubuntu)
         if: matrix.os == 'ubuntu-latest'
         run: sudo apt-get update && sudo apt-get install -y libpcre2-dev
 
+      - name: Configure Python for PyO3 (Ubuntu)
+        if: matrix.os == 'ubuntu-latest'
+        run: echo "PYO3_PYTHON=$(which python3)" >> $GITHUB_ENV
+
       - name: Install PCRE2 (macOS)
         if: matrix.os == 'macos-latest'
         run: brew install pcre2
 
+      - name: Configure Python for PyO3 (macOS)
+        if: matrix.os == 'macos-latest'
+        run: |
+          echo "PYO3_PYTHON=$(which python3)" >> $GITHUB_ENV
+          # Get Python library directory and set for linker
+          PYTHON_PREFIX=$(python3 -c "import sys; print(sys.prefix)")
+          echo "LIBRARY_PATH=${PYTHON_PREFIX}/lib" >> $GITHUB_ENV
+          echo "DYLD_LIBRARY_PATH=${PYTHON_PREFIX}/lib" >> $GITHUB_ENV
+          # Tell Cargo to link the Python framework
+          echo "CARGO_BUILD_RUSTFLAGS=-C link-arg=-undefined -C link-arg=dynamic_lookup" >> $GITHUB_ENV
+
       - name: Install PCRE2 (Windows)
         if: matrix.os == 'windows-latest'
         run: |
           vcpkg install pcre2:x64-windows
           echo "PCRE2_SYS_STATIC=1" >> $env:GITHUB_ENV
 
-      - name: Run tests
-        run: cargo test --verbose
-
-      - name: Run clippy
-        run: cargo clippy -- -D warnings
-
-  fmt:
-    name: Format check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Rust
-        uses: dtolnay/rust-action@stable
-        with:
-          components: rustfmt
+      - name: Configure Python for PyO3 (Windows)
+        if: matrix.os == 'windows-latest'
+        run: |
+          $pythonPath = (Get-Command python).Source
+          echo "PYO3_PYTHON=$pythonPath" >> $env:GITHUB_ENV
 
-      - name: Check formatting
-        run: cargo fmt --check
+      - name: Run tests
+        run: cargo test
 
-  python-test:
+  # Python bindings tests
+  python:
     name: Python tests
     runs-on: ubuntu-latest
     steps:
@@ -72,17 +102,17 @@ jobs:
       - name: Install PCRE2
         run: sudo apt-get update && sudo apt-get install -y libpcre2-dev
 
-      - name: Install dependencies
+      - name: Install dependencies and build
         run: |
+          python -m venv .venv
+          . .venv/bin/activate
           python -m pip install --upgrade pip
           pip install maturin tiktoken
-
-      - name: Build and install
-        run: maturin develop --release
+          maturin develop --release
 
       - name: Test Python bindings
         run: |
-          python -c "
+          .venv/bin/python -c "
           import splintr
           import tiktoken
 

.github/workflows/release.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Rust
-        uses: dtolnay/rust-action@stable
+        uses: dtolnay/rust-toolchain@stable
 
       - name: Install PCRE2 dependencies
         run: sudo apt-get update && sudo apt-get install -y libpcre2-dev

README.md

@@ -296,6 +296,10 @@ Contributions are welcome! Here's how you can help:
 git clone https://github.com/farhan/splintr.git
 cd splintr
 
+# Install pre-commit hook (recommended)
+cp hooks/pre-commit .git/hooks/pre-commit
+chmod +x .git/hooks/pre-commit
+
 # Build the Rust library
 cargo build --release
 
@@ -305,9 +309,12 @@ maturin develop --release
 
 # Run tests
 cargo test                    # Rust tests
-pytest python/tests/          # Python tests (if available)
+cargo clippy --all-targets    # Linting
+cargo fmt --all --check       # Format check
 ```
 
+The pre-commit hook automatically runs formatting, clippy, and tests before each commit.
+
 ## License
 
 This project is licensed under the MIT License - see the LICENSE file for details.

hooks/pre-commit

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Pre-commit hook for splintr
+#
+# Install by copying to .git/hooks/:
+#   cp hooks/pre-commit .git/hooks/pre-commit
+#   chmod +x .git/hooks/pre-commit
+#
+
+set -e
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+echo "Running pre-commit checks..."
+echo ""
+
+FAILED=0
+
+# 1. Rust formatting
+echo -n "Checking formatting... "
+if ! cargo fmt --all --check > /dev/null 2>&1; then
+    echo -e "${RED}FAILED${NC}"
+    echo -e "${YELLOW}Fix with:${NC} cargo fmt --all"
+    FAILED=1
+else
+    echo -e "${GREEN}OK${NC}"
+fi
+
+# 2. Clippy
+echo -n "Running clippy... "
+if ! cargo clippy --all-targets -- -D warnings > /dev/null 2>&1; then
+    echo -e "${RED}FAILED${NC}"
+    echo -e "${YELLOW}Run:${NC} cargo clippy --all-targets"
+    FAILED=1
+else
+    echo -e "${GREEN}OK${NC}"
+fi
+
+# 3. Tests
+echo -n "Running tests... "
+if ! cargo test --quiet > /dev/null 2>&1; then
+    echo -e "${RED}FAILED${NC}"
+    echo -e "${YELLOW}Run:${NC} cargo test"
+    FAILED=1
+else
+    echo -e "${GREEN}OK${NC}"
+fi
+
+echo ""
+
+if [ $FAILED -ne 0 ]; then
+    echo -e "${RED}Pre-commit checks failed.${NC}"
+    echo "Bypass with: git commit --no-verify"
+    exit 1
+fi
+
+echo -e "${GREEN}All checks passed!${NC}"

src/core/tokenizer.rs

@@ -332,10 +332,7 @@ impl Tokenizer {
 
     /// Get cache statistics (hits would require additional tracking).
     pub fn cache_len(&self) -> usize {
-        self.chunk_cache
-            .lock()
-            .map(|c| c.len())
-            .unwrap_or(0)
+        self.chunk_cache.lock().map(|c| c.len()).unwrap_or(0)
     }
 }
 

src/lib.rs

@@ -3,7 +3,9 @@ mod python;
 
 use pyo3::prelude::*;
 
-pub use core::{Tokenizer, TokenizerError, CL100K_BASE_PATTERN, O200K_BASE_PATTERN, StreamingDecoder};
+pub use core::{
+    StreamingDecoder, Tokenizer, TokenizerError, CL100K_BASE_PATTERN, O200K_BASE_PATTERN,
+};
 
 /// splintr - Fast Rust BPE tokenizer with Python bindings
 ///

src/python/bindings.rs

@@ -456,10 +456,10 @@ impl PyStreamingDecoder {
                 continue;
             }
 
-            if std::str::from_utf8(&bytes[..check_len]).is_ok() {
-                if Self::could_be_incomplete_sequence(&bytes[check_len..]) {
-                    return check_len;
-                }
+            if std::str::from_utf8(&bytes[..check_len]).is_ok()
+                && Self::could_be_incomplete_sequence(&bytes[check_len..])
+            {
+                return check_len;
             }
         }
 

src/python/mod.rs

@@ -1,3 +1,3 @@
 mod bindings;
 
-pub use bindings::{PyTokenizer, PyStreamingDecoder};
+pub use bindings::{PyStreamingDecoder, PyTokenizer};