Feature/hnsw vector search

CLAUDE.md

@@ -1209,7 +1209,17 @@ flutter_gemma/
 └── CLAUDE.md              # This file
 ```
 
-## Recent Updates (2026-01-18)
+## Recent Updates (2026-02-25)
+
+### ✅ iOS Tokenizer.json Support (v0.12.5)
+- **Problem**: SentencePiece C++ and TFLite conflict on iOS due to protobuf symbol clash
+- **Solution**: `iosPath` parameter in `tokenizerFromNetwork()` — downloads tokenizer.json on iOS instead of sentencepiece.model
+- **NetworkSource**: New `iosPath` field for platform-aware URL selection
+- **NetworkSourceHandler**: Auto-selects iosPath on iOS, throws `UnsupportedError` for `.model` without iosPath
+- **EmbeddingModel.swift**: Simple `.json`/`.model` branching (removed Bundle.main fallback)
+- **CDN**: Pre-converted files hosted on GitHub Releases v0.12.5
+- **BPETokenizer.swift**: Pure Swift BPE tokenizer matching SentencePiece C++ output, no C++ dependencies
+- **Zero breaking changes** — iosPath is optional, Android/Web unaffected
 
 ### ✅ Android LiteRT-LM Engine (v0.12.x+)
 - **Dual Engine Support** - MediaPipe and LiteRT-LM on Android

README.md

@@ -1604,6 +1604,36 @@ await FlutterGemma.installEmbedder()
   .install();
 ```
 
+### iOS: Tokenizer Compatibility
+
+> **Important:** On iOS, SentencePiece `.model` tokenizers are not supported due to a protobuf
+> conflict between SentencePiece C++ and TensorFlow Lite. Use `.json` tokenizers instead.
+
+Pre-converted tokenizer files are available on GitHub CDN:
+- **EmbeddingGemma:** `https://github.com/DenisovAV/flutter_gemma/releases/download/v0.12.5/embeddinggemma_tokenizer.json`
+- **Gecko:** `https://github.com/DenisovAV/flutter_gemma/releases/download/v0.12.5/gecko_tokenizer.json`
+
+```dart
+await FlutterGemma.installEmbedder()
+  .modelFromNetwork(modelUrl, token: hfToken)
+  .tokenizerFromNetwork(
+    'https://huggingface.co/.../sentencepiece.model',
+    token: hfToken,
+    iosPath: 'https://github.com/DenisovAV/flutter_gemma/releases/download/v0.12.5/embeddinggemma_tokenizer.json',
+  )
+  .install();
+```
+
+On Android and Web, the original `sentencepiece.model` URL is used. On iOS, the `iosPath` is
+automatically selected. If `iosPath` is not provided and the tokenizer URL ends with `.model`,
+an error is thrown with instructions.
+
+To convert your own tokenizer, use the provided script:
+```bash
+pip install -r tools/requirements.txt
+python tools/convert_sentencepiece_to_json.py --input path/to/sentencepiece.model --output tokenizer.json
+```
+
 ### Generate Text Embeddings
 
 ```dart

android/src/main/AndroidManifest.xml

@@ -1,3 +1,2 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
-  package="dev.flutterberlin.flutter_gemma">
-</manifest>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+</manifest>

android/src/main/kotlin/dev/flutterberlin/flutter_gemma/FlutterGemmaPlugin.kt

@@ -496,4 +496,28 @@ private class PlatformServiceImpl(
       callback(Result.failure(e))
     }
   }
+
+  override fun getAllDocumentsWithEmbeddings(callback: (Result<List<DocumentWithEmbedding>>) -> Unit) {
+    scope.launch {
+      try {
+        val results = vectorStore?.getAllDocumentsWithEmbeddings()
+          ?: throw IllegalStateException("Vector store not initialized")
+        callback(Result.success(results))
+      } catch (e: Exception) {
+        callback(Result.failure(e))
+      }
+    }
+  }
+
+  override fun getDocumentsByIds(ids: List<String>, callback: (Result<List<RetrievalResult>>) -> Unit) {
+    scope.launch {
+      try {
+        val results = vectorStore?.getDocumentsByIds(ids)
+          ?: throw IllegalStateException("Vector store not initialized")
+        callback(Result.success(results))
+      } catch (e: Exception) {
+        callback(Result.failure(e))
+      }
+    }
+  }
 }

android/src/main/kotlin/dev/flutterberlin/flutter_gemma/PigeonInterface.g.kt

@@ -115,6 +115,40 @@ data class VectorStoreStats (
     )
   }
 }
+
+/**
+ * Document with embedding for HNSW rebuild
+ *
+ * Used by [getAllDocumentsWithEmbeddings] to return documents
+ * with their vectors for in-memory index reconstruction.
+ *
+ * Generated class from Pigeon that represents data sent in messages.
+ */
+data class DocumentWithEmbedding (
+  val id: String,
+  val content: String,
+  val embedding: List<Double>,
+  val metadata: String? = null
+)
+ {
+  companion object {
+    fun fromList(pigeonVar_list: List<Any?>): DocumentWithEmbedding {
+      val id = pigeonVar_list[0] as String
+      val content = pigeonVar_list[1] as String
+      val embedding = pigeonVar_list[2] as List<Double>
+      val metadata = pigeonVar_list[3] as String?
+      return DocumentWithEmbedding(id, content, embedding, metadata)
+    }
+  }
+  fun toList(): List<Any?> {
+    return listOf(
+      id,
+      content,
+      embedding,
+      metadata,
+    )
+  }
+}
 private open class PigeonInterfacePigeonCodec : StandardMessageCodec() {
   override fun readValueOfType(type: Byte, buffer: ByteBuffer): Any? {
     return when (type) {
@@ -133,6 +167,11 @@ private open class PigeonInterfacePigeonCodec : StandardMessageCodec() {
           VectorStoreStats.fromList(it)
         }
       }
+      132.toByte() -> {
+        return (readValue(buffer) as? List<Any?>)?.let {
+          DocumentWithEmbedding.fromList(it)
+        }
+      }
       else -> super.readValueOfType(type, buffer)
     }
   }
@@ -150,6 +189,10 @@ private open class PigeonInterfacePigeonCodec : StandardMessageCodec() {
         stream.write(131)
         writeValue(stream, value.toList())
       }
+      is DocumentWithEmbedding -> {
+        stream.write(132)
+        writeValue(stream, value.toList())
+      }
       else -> super.writeValue(stream, value)
     }
   }
@@ -180,6 +223,33 @@ interface PlatformService {
   fun getVectorStoreStats(callback: (Result<VectorStoreStats>) -> Unit)
   fun clearVectorStore(callback: (Result<Unit>) -> Unit)
   fun closeVectorStore(callback: (Result<Unit>) -> Unit)
+  /**
+   * Get all documents with embeddings for HNSW index rebuild
+   *
+   * **Use case:**
+   * Called during initialize() to rebuild in-memory HNSW index
+   * from SQLite persistence layer.
+   *
+   * **Performance:**
+   * - Returns all documents in single call
+   * - Embeddings as List<double> (decoded from BLOB)
+   *
+   * Returns empty list if no documents stored.
+   */
+  fun getAllDocumentsWithEmbeddings(callback: (Result<List<DocumentWithEmbedding>>) -> Unit)
+  /**
+   * Get documents by IDs with full content
+   *
+   * **Use case:**
+   * After HNSW returns candidate IDs, fetch full documents
+   * for final result construction.
+   *
+   * **Parameters:**
+   * - [ids]: List of document IDs to retrieve
+   *
+   * Returns only documents that exist (missing IDs are skipped).
+   */
+  fun getDocumentsByIds(ids: List<String>, callback: (Result<List<RetrievalResult>>) -> Unit)
 
   companion object {
     /** The codec used by PlatformService. */
@@ -613,6 +683,44 @@ interface PlatformService {
           channel.setMessageHandler(null)
         }
       }
+      run {
+        val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.flutter_gemma.PlatformService.getAllDocumentsWithEmbeddings$separatedMessageChannelSuffix", codec)
+        if (api != null) {
+          channel.setMessageHandler { _, reply ->
+            api.getAllDocumentsWithEmbeddings{ result: Result<List<DocumentWithEmbedding>> ->
+              val error = result.exceptionOrNull()
+              if (error != null) {
+                reply.reply(wrapError(error))
+              } else {
+                val data = result.getOrNull()
+                reply.reply(wrapResult(data))
+              }
+            }
+          }
+        } else {
+          channel.setMessageHandler(null)
+        }
+      }
+      run {
+        val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.flutter_gemma.PlatformService.getDocumentsByIds$separatedMessageChannelSuffix", codec)
+        if (api != null) {
+          channel.setMessageHandler { message, reply ->
+            val args = message as List<Any?>
+            val idsArg = args[0] as List<String>
+            api.getDocumentsByIds(idsArg) { result: Result<List<RetrievalResult>> ->
+              val error = result.exceptionOrNull()
+              if (error != null) {
+                reply.reply(wrapError(error))
+              } else {
+                val data = result.getOrNull()
+                reply.reply(wrapResult(data))
+              }
+            }
+          }
+        } else {
+          channel.setMessageHandler(null)
+        }
+      }
     }
   }
 }

android/src/main/kotlin/dev/flutterberlin/flutter_gemma/VectorStore.kt

@@ -177,6 +177,91 @@ class VectorStore(
         detectedDimension = null
     }
 
+    /**
+     * Get all documents with embeddings for HNSW index rebuild
+     *
+     * Used during initialize() to rebuild in-memory HNSW index
+     * from SQLite persistence layer.
+     */
+    fun getAllDocumentsWithEmbeddings(): List<DocumentWithEmbedding> {
+        val db = database ?: throw IllegalStateException("Database not initialized")
+
+        val cursor = db.query(TABLE_DOCUMENTS, null, null, null, null, null, null)
+        val results = mutableListOf<DocumentWithEmbedding>()
+
+        cursor.use {
+            val idIndex = cursor.getColumnIndexOrThrow(COLUMN_ID)
+            val contentIndex = cursor.getColumnIndexOrThrow(COLUMN_CONTENT)
+            val embeddingIndex = cursor.getColumnIndexOrThrow(COLUMN_EMBEDDING)
+            val metadataIndex = cursor.getColumnIndexOrThrow(COLUMN_METADATA)
+
+            while (cursor.moveToNext()) {
+                val id = cursor.getString(idIndex)
+                val content = cursor.getString(contentIndex)
+                val embeddingBlob = cursor.getBlob(embeddingIndex)
+                val metadata = cursor.getString(metadataIndex)
+
+                // Convert BLOB to embedding
+                val embedding = blobToEmbedding(embeddingBlob)
+
+                results.add(DocumentWithEmbedding(
+                    id = id,
+                    content = content,
+                    embedding = embedding,
+                    metadata = metadata
+                ))
+            }
+        }
+
+        return results
+    }
+
+    /**
+     * Get documents by IDs with full content
+     *
+     * After HNSW returns candidate IDs, fetch full documents
+     * for final result construction.
+     */
+    fun getDocumentsByIds(ids: List<String>): List<RetrievalResult> {
+        val db = database ?: throw IllegalStateException("Database not initialized")
+
+        if (ids.isEmpty()) return emptyList()
+
+        // Build placeholder for IN clause
+        val placeholders = ids.joinToString(",") { "?" }
+        val cursor = db.query(
+            TABLE_DOCUMENTS,
+            arrayOf(COLUMN_ID, COLUMN_CONTENT, COLUMN_METADATA),
+            "$COLUMN_ID IN ($placeholders)",
+            ids.toTypedArray(),
+            null, null, null
+        )
+
+        val results = mutableListOf<RetrievalResult>()
+
+        cursor.use {
+            val idIndex = cursor.getColumnIndexOrThrow(COLUMN_ID)
+            val contentIndex = cursor.getColumnIndexOrThrow(COLUMN_CONTENT)
+            val metadataIndex = cursor.getColumnIndexOrThrow(COLUMN_METADATA)
+
+            while (cursor.moveToNext()) {
+                val id = cursor.getString(idIndex)
+                val content = cursor.getString(contentIndex)
+                val metadata = cursor.getString(metadataIndex)
+
+                // Similarity will be recalculated by Dart HNSW layer
+                results.add(RetrievalResult(
+                    id = id,
+                    content = content,
+                    similarity = 0.0,  // Placeholder, recalculated in Dart
+                    metadata = metadata
+                ))
+            }
+        }
+
+        return results
+    }
+
     private fun cosineSimilarity(a: List<Double>, b: List<Double>): Double {
         if (a.size != b.size) return 0.0
 

example/android/app/build.gradle.kts

@@ -49,6 +49,12 @@ android {
         targetSdk = 34
         versionCode = flutterVersionCode.toInt()
         versionName = flutterVersionName
+        testInstrumentationRunner = "pl.leancode.patrol.PatrolJUnitRunner"
+        testInstrumentationRunnerArguments["clearPackageData"] = "true"
+    }
+
+    testOptions {
+        execution = "ANDROIDX_TEST_ORCHESTRATOR"
     }
 
     buildTypes {
@@ -68,4 +74,6 @@ flutter {
     source = "../.."
 }
 
-dependencies {}
+dependencies {
+    androidTestUtil("androidx.test:orchestrator:1.5.1")
+}

example/android/app/src/androidTest/java/dev/flutterberlin/flutter_gemma_example/MainActivityTest.java

@@ -0,0 +1,31 @@
+package dev.flutterberlin.flutter_gemma_example;
+
+import androidx.test.platform.app.InstrumentationRegistry;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import pl.leancode.patrol.PatrolJUnitRunner;
+
+@RunWith(Parameterized.class)
+public class MainActivityTest {
+    @Parameters(name = "{0}")
+    public static Object[] testCases() {
+        PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation();
+        instrumentation.setUp(MainActivity.class);
+        instrumentation.waitForPatrolAppService();
+        return instrumentation.listDartTests();
+    }
+
+    public MainActivityTest(String dartTestName) {
+        this.dartTestName = dartTestName;
+    }
+
+    private final String dartTestName;
+
+    @Test
+    public void runDartTest() {
+        PatrolJUnitRunner instrumentation = (PatrolJUnitRunner) InstrumentationRegistry.getInstrumentation();
+        instrumentation.runDartTest(dartTestName);
+    }
+}