Add @stdlib/tokenizer module: BPE tokenizer for tiktoken-compatible token counting

include/builtins_registry.h

@@ -317,6 +317,15 @@ typedef struct {
     X("__regex_replace",      3,  HML_BUILTIN_INTERNAL, HML_RET_STRING) \
     X("__regex_replace_all",  3,  HML_BUILTIN_INTERNAL, HML_RET_STRING)
 
+#define HML_BUILTINS_TOKENIZER(X) \
+    X("__tokenizer_create",      1,  HML_BUILTIN_INTERNAL, HML_RET_PTR) \
+    X("__tokenizer_free",        1,  HML_BUILTIN_INTERNAL, HML_RET_NULL) \
+    X("__tokenizer_encode",      2,  HML_BUILTIN_INTERNAL, HML_RET_ARRAY) \
+    X("__tokenizer_decode",      2,  HML_BUILTIN_INTERNAL, HML_RET_STRING) \
+    X("__tokenizer_count",       2,  HML_BUILTIN_INTERNAL, HML_RET_I32) \
+    X("__tokenizer_vocab_size",  1,  HML_BUILTIN_INTERNAL | HML_BUILTIN_PURE, HML_RET_I32) \
+    X("__tokenizer_add_special", 3,  HML_BUILTIN_INTERNAL, HML_RET_NULL)
+
 #define HML_BUILTINS_FFI(X) \
     X("callback",       2,  HML_BUILTIN_PUBLIC, HML_RET_PTR) \
     X("callback_free",  1,  HML_BUILTIN_PUBLIC, HML_RET_NULL) \
@@ -349,6 +358,7 @@ typedef struct {
     HML_BUILTINS_CRYPTO(X) \
     HML_BUILTINS_COMPRESSION(X) \
     HML_BUILTINS_REGEX(X) \
+    HML_BUILTINS_TOKENIZER(X) \
     HML_BUILTINS_FFI(X) \
     HML_BUILTINS_INTERNAL(X)
 

include/hemlock_limits.h

@@ -185,6 +185,24 @@
 #define HML_ATOMIC_I32_ALIGNMENT 4   // _Alignof(_Atomic int32_t)
 #define HML_ATOMIC_I64_ALIGNMENT 8   // _Alignof(_Atomic int64_t)
 
+// ========== TOKENIZER (BPE) CONSTANTS ==========
+
+// Initial hash table capacity for BPE vocabulary
+// Should be a prime number for better distribution
+#define HML_BPE_INITIAL_HASH_CAPACITY 131071
+
+// Maximum token byte sequence length
+#define HML_BPE_MAX_TOKEN_LENGTH 128
+
+// Initial capacity for the BPE merge work list
+#define HML_BPE_INITIAL_WORK_CAPACITY 256
+
+// Base64 decode buffer size for vocabulary loading
+#define HML_BPE_BASE64_DECODE_BUFSIZE 256
+
+// Maximum line length when reading vocabulary files
+#define HML_BPE_MAX_VOCAB_LINE_LENGTH 1024
+
 // ========== INLINE CACHE CONSTANTS ==========
 
 // Inline caching is used to speed up property access and method dispatch

runtime/include/hemlock_runtime.h

@@ -939,6 +939,19 @@ HmlValue hml_builtin_regex_error(HmlClosureEnv *env, HmlValue errcode, HmlValue
 HmlValue hml_builtin_regex_replace(HmlClosureEnv *env, HmlValue preg, HmlValue text, HmlValue replacement);
 HmlValue hml_builtin_regex_replace_all(HmlClosureEnv *env, HmlValue preg, HmlValue text, HmlValue replacement);
 
+// ========== TOKENIZER (BPE) FUNCTIONS ==========
+
+// Tokenizer lifecycle
+HmlValue hml_tokenizer_create(HmlValue path);
+HmlValue hml_tokenizer_free(HmlValue handle);
+
+// Tokenizer operations
+HmlValue hml_tokenizer_encode(HmlValue handle, HmlValue text);
+HmlValue hml_tokenizer_decode(HmlValue handle, HmlValue tokens);
+HmlValue hml_tokenizer_count(HmlValue handle, HmlValue text);
+HmlValue hml_tokenizer_vocab_size(HmlValue handle);
+HmlValue hml_tokenizer_add_special(HmlValue handle, HmlValue token, HmlValue rank);
+
 // ========== CALL STACK TRACKING ==========
 
 // Default maximum call stack depth (matches interpreter's limit)