diff --git a/README.md b/README.md index 45d364f..35111b7 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # Sunflower -On-device translation, transcription, and speech Q&A for ten Ugandan languages (Luganda, Acholi, Ateso, Lugbara, Runyankole, Lusoga, Lumasaba, Swahili, Kinyarwanda, English). Runs a fine-tuned Gemma 4 E2B (Sunbird AI) — and optionally a Whisper-based ASR cascade — under [Cactus](https://github.com/cactus-compute/cactus)'s ARM-optimized engine. Android-first; offline. +On-device translation, transcription, and speech Q&A for ten Ugandan languages (Luganda, Acholi, Ateso, Lugbara, Runyankole, Lusoga, Lumasaba, Swahili, Kinyarwanda, English). Runs a fine-tuned Gemma 4 E2B (Sunbird AI) — and optionally a Whisper-based ASR cascade — under [Cactus](https://github.com/cactus-compute/cactus)'s ARM-optimized engine. Android & iOS; offline. For the architecture, threading model, and Cactus FFI contract, see [design-spec.md](./design-spec.md). For card-stack UI behaviour, see [card_spec.md](./card_spec.md). ## Run -Requirements: flagship Android device (Pixel 7+ / S22+), Flutter 3.11+, Android NDK r27.x, USB debugging. +Requirements: flagship Android (Pixel 7+ / S22+) or iOS device (iPhone 13+), Flutter 3.11+, macOS/Xcode (for iOS), Android NDK r27.x (for Android). ```bash git clone git@github.com:SunbirdAI/sunflower-app.git @@ -16,6 +16,13 @@ flutter pub get flutter run ``` +For iOS, ensure you have CocoaPods installed: + +```bash +cd ios && pod install && cd .. +flutter run +``` + The model isn't bundled (~4.7 GB INT4). On first launch the app shows "Get the model" — open the gear icon, tap Download. The tarball streams from HuggingFace into the app's private documents directory. For dev pushes from Mac (faster than the in-app download): @@ -104,6 +111,7 @@ scripts/ └── export_stitched.sh Whisper + Gemma dual-convert for the stitched bundle test/ chips, conversation_state, classroom_prompt android/app/src/main/jniLibs/arm64-v8a/libcactus.so +ios/cactus-ios.xcframework iOS native engine framework ``` ## How to @@ -185,7 +193,9 @@ cp flutter/cactus.dart /path/to/sunflower-app/lib/cactus.dart # then re-convert the model with the same vX.Y and re-upload to HF ``` -### Build a release APK +### Build a release + +#### Android (APK) ```bash flutter build apk --release @@ -194,6 +204,15 @@ flutter build apk --release The APK is signed with the Flutter debug key by default. Set up release signing in `android/app/build.gradle.kts` before distributing. +#### iOS (IPA) + +```bash +flutter build ios --release +``` + +Open `ios/Runner.xcworkspace` in Xcode to configure signing and capabilities (Microphone access is already in `Info.plist`). + + ### Run tests ```bash diff --git a/ios/.gitignore b/ios/.gitignore new file mode 100644 index 0000000..7a7f987 --- /dev/null +++ b/ios/.gitignore @@ -0,0 +1,34 @@ +**/dgph +*.mode1v3 +*.mode2v3 +*.moved-aside +*.pbxuser +*.perspectivev3 +**/*sync/ +.sconsign.dblite +.tags* +**/.vagrant/ +**/DerivedData/ +Icon? +**/Pods/ +**/.symlinks/ +profile +xcuserdata +**/.generated/ +Flutter/App.framework +Flutter/Flutter.framework +Flutter/Flutter.podspec +Flutter/Generated.xcconfig +Flutter/ephemeral/ +Flutter/app.flx +Flutter/app.zip +Flutter/flutter_assets/ +Flutter/flutter_export_environment.sh +ServiceDefinitions.json +Runner/GeneratedPluginRegistrant.* + +# Exceptions to above rules. +!default.mode1v3 +!default.mode2v3 +!default.pbxuser +!default.perspectivev3 diff --git a/ios/Flutter/AppFrameworkInfo.plist b/ios/Flutter/AppFrameworkInfo.plist new file mode 100644 index 0000000..391a902 --- /dev/null +++ b/ios/Flutter/AppFrameworkInfo.plist @@ -0,0 +1,24 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + App + CFBundleIdentifier + io.flutter.flutter.app + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + App + CFBundlePackageType + FMWK + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + 1.0 + + diff --git a/ios/Flutter/Debug.xcconfig b/ios/Flutter/Debug.xcconfig new file mode 100644 index 0000000..ec97fc6 --- /dev/null +++ b/ios/Flutter/Debug.xcconfig @@ -0,0 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig" +#include "Generated.xcconfig" diff --git a/ios/Flutter/Release.xcconfig b/ios/Flutter/Release.xcconfig new file mode 100644 index 0000000..c4855bf --- /dev/null +++ b/ios/Flutter/Release.xcconfig @@ -0,0 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig" +#include "Generated.xcconfig" diff --git a/ios/Podfile b/ios/Podfile new file mode 100644 index 0000000..d615bd1 --- /dev/null +++ b/ios/Podfile @@ -0,0 +1,38 @@ +platform :ios, '16.0' +ENV['COCOAPODS_DISABLE_STATS'] = 'true' + +project 'Runner', { + 'Debug' => :debug, + 'Profile' => :release, + 'Release' => :release, +} + +def flutter_root + generated_xcode_build_settings_path = File.expand_path(File.join('..', 'Flutter', 'Generated.xcconfig'), __FILE__) + unless File.exist?(generated_xcode_build_settings_path) + raise "#{generated_xcode_build_settings_path} must exist. If you're running pod install manually, make sure flutter pub get is executed first" + end + File.foreach(generated_xcode_build_settings_path) do |line| + matches = line.match(/FLUTTER_ROOT\=(.*)/) + return matches[1].strip if matches + end + raise "FLUTTER_ROOT not found in #{generated_xcode_build_settings_path}. Try deleting Generated.xcconfig, then run flutter pub get" +end + +require File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelper'), flutter_root) + +flutter_ios_podfile_setup + +target 'Runner' do + use_frameworks! :linkage => :static + flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__)) + target 'RunnerTests' do + inherit! :search_paths + end +end + +post_install do |installer| + installer.pods_project.targets.each do |target| + flutter_additional_ios_build_settings(target) + end +end diff --git a/ios/Podfile.lock b/ios/Podfile.lock new file mode 100644 index 0000000..7be776b --- /dev/null +++ b/ios/Podfile.lock @@ -0,0 +1,61 @@ +PODS: + - audioplayers_darwin (0.0.1): + - Flutter + - FlutterMacOS + - Flutter (1.0.0) + - flutter_foreground_task (0.0.1): + - Flutter + - flutter_onnxruntime (0.0.1): + - Flutter + - onnxruntime-objc (= 1.22.0) + - onnxruntime-c (1.22.0) + - onnxruntime-objc (1.22.0): + - onnxruntime-objc/Core (= 1.22.0) + - onnxruntime-objc/Core (1.22.0): + - onnxruntime-c (= 1.22.0) + - record_ios (1.2.0): + - Flutter + - shared_preferences_foundation (0.0.1): + - Flutter + - FlutterMacOS + +DEPENDENCIES: + - audioplayers_darwin (from `.symlinks/plugins/audioplayers_darwin/darwin`) + - Flutter (from `Flutter`) + - flutter_foreground_task (from `.symlinks/plugins/flutter_foreground_task/ios`) + - flutter_onnxruntime (from `.symlinks/plugins/flutter_onnxruntime/ios`) + - record_ios (from `.symlinks/plugins/record_ios/ios`) + - shared_preferences_foundation (from `.symlinks/plugins/shared_preferences_foundation/darwin`) + +SPEC REPOS: + trunk: + - onnxruntime-c + - onnxruntime-objc + +EXTERNAL SOURCES: + audioplayers_darwin: + :path: ".symlinks/plugins/audioplayers_darwin/darwin" + Flutter: + :path: Flutter + flutter_foreground_task: + :path: ".symlinks/plugins/flutter_foreground_task/ios" + flutter_onnxruntime: + :path: ".symlinks/plugins/flutter_onnxruntime/ios" + record_ios: + :path: ".symlinks/plugins/record_ios/ios" + shared_preferences_foundation: + :path: ".symlinks/plugins/shared_preferences_foundation/darwin" + +SPEC CHECKSUMS: + audioplayers_darwin: 835ced6edd4c9fc8ebb0a7cc9e294a91d99917d5 + Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467 + flutter_foreground_task: a159d2c2173b33699ddb3e6c2a067045d7cebb89 + flutter_onnxruntime: 515c2ab17b67fb027964ebbf2d02329771a482de + onnxruntime-c: 7f778680e96145956c0a31945f260321eed2611a + onnxruntime-objc: 83d28b87525bd971259a66e153ea32b5d023de19 + record_ios: 412daca2350b228e698fffcd08f1f94ceb1e3844 + shared_preferences_foundation: 7036424c3d8ec98dfe75ff1667cb0cd531ec82bb + +PODFILE CHECKSUM: 93e78d9efba2e1946f2eddcc1dd4ffef6df21e1a + +COCOAPODS: 1.16.2 diff --git a/ios/Runner.xcodeproj/project.pbxproj b/ios/Runner.xcodeproj/project.pbxproj new file mode 100644 index 0000000..974ddac --- /dev/null +++ b/ios/Runner.xcodeproj/project.pbxproj @@ -0,0 +1,746 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 54; + objects = { + +/* Begin PBXBuildFile section */ + 0A3438F200D4E7653FDE94AA /* Pods_RunnerTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = E39A2DE24A32093D9EFA6245 /* Pods_RunnerTests.framework */; }; + 1498D2341E8E89220040F4C2 /* GeneratedPluginRegistrant.m in Sources */ = {isa = PBXBuildFile; fileRef = 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */; }; + 331C808B294A63AB00263BE5 /* RunnerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 331C807B294A618700263BE5 /* RunnerTests.swift */; }; + 3B3967161E833CAA004F5970 /* AppFrameworkInfo.plist in Resources */ = {isa = PBXBuildFile; fileRef = 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */; }; + 74858FAF1ED2DC5600515810 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 74858FAE1ED2DC5600515810 /* AppDelegate.swift */; }; + 7884E8682EC3CC0700C636F2 /* SceneDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7884E8672EC3CC0400C636F2 /* SceneDelegate.swift */; }; + 87ED1B5A2FB615FD00137A76 /* cactus-ios.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 87ED1B582FB615E100137A76 /* cactus-ios.xcframework */; }; + 87ED1B5B2FB615FD00137A76 /* cactus-ios.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 87ED1B582FB615E100137A76 /* cactus-ios.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + 97C146FC1CF9000F007C117D /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FA1CF9000F007C117D /* Main.storyboard */; }; + 97C146FE1CF9000F007C117D /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FD1CF9000F007C117D /* Assets.xcassets */; }; + 97C147011CF9000F007C117D /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */; }; + A9BAB38550FCA6E33321C511 /* Pods_Runner.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 0CD8C9D04440C82664808F55 /* Pods_Runner.framework */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + 331C8085294A63A400263BE5 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 97C146E61CF9000F007C117D /* Project object */; + proxyType = 1; + remoteGlobalIDString = 97C146ED1CF9000F007C117D; + remoteInfo = Runner; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXCopyFilesBuildPhase section */ + 9705A1C41CF9048500538489 /* Embed Frameworks */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = ""; + dstSubfolderSpec = 10; + files = ( + 87ED1B5B2FB615FD00137A76 /* cactus-ios.xcframework in Embed Frameworks */, + ); + name = "Embed Frameworks"; + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXCopyFilesBuildPhase section */ + +/* Begin PBXFileReference section */ + 0CD8C9D04440C82664808F55 /* Pods_Runner.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Runner.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + 1498D2321E8E86230040F4C2 /* GeneratedPluginRegistrant.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GeneratedPluginRegistrant.h; sourceTree = ""; }; + 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GeneratedPluginRegistrant.m; sourceTree = ""; }; + 331C807B294A618700263BE5 /* RunnerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RunnerTests.swift; sourceTree = ""; }; + 331C8081294A63A400263BE5 /* RunnerTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = RunnerTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = AppFrameworkInfo.plist; path = Flutter/AppFrameworkInfo.plist; sourceTree = ""; }; + 3F704E798108FE8E0BF9745C /* Pods-Runner.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.debug.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig"; sourceTree = ""; }; + 633B2DF041FE201ED9703959 /* Pods-RunnerTests.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.debug.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.debug.xcconfig"; sourceTree = ""; }; + 74858FAD1ED2DC5600515810 /* Runner-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "Runner-Bridging-Header.h"; sourceTree = ""; }; + 74858FAE1ED2DC5600515810 /* AppDelegate.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; + 7884E8672EC3CC0400C636F2 /* SceneDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SceneDelegate.swift; sourceTree = ""; }; + 7AFA3C8E1D35360C0083082E /* Release.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; name = Release.xcconfig; path = Flutter/Release.xcconfig; sourceTree = ""; }; + 7B1B1D6D479BF19842A3534C /* Pods-RunnerTests.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.profile.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.profile.xcconfig"; sourceTree = ""; }; + 81F9D792A678E88223EAE03E /* Pods-RunnerTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.release.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.release.xcconfig"; sourceTree = ""; }; + 87ED1B582FB615E100137A76 /* cactus-ios.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = "cactus-ios.xcframework"; sourceTree = ""; }; + 9740EEB21CF90195004384FC /* Debug.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = Debug.xcconfig; path = Flutter/Debug.xcconfig; sourceTree = ""; }; + 9740EEB31CF90195004384FC /* Generated.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = Generated.xcconfig; path = Flutter/Generated.xcconfig; sourceTree = ""; }; + 97C146EE1CF9000F007C117D /* Runner.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Runner.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 97C146FB1CF9000F007C117D /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; + 97C146FD1CF9000F007C117D /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; + 97C147001CF9000F007C117D /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; + 97C147021CF9000F007C117D /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + A726CCF9522062C550887F12 /* Pods-Runner.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.profile.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.profile.xcconfig"; sourceTree = ""; }; + E39A2DE24A32093D9EFA6245 /* Pods_RunnerTests.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_RunnerTests.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + FE3C24750620191807070F8A /* Pods-Runner.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.release.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig"; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 4F91C6DB5872B6929FA57E78 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 0A3438F200D4E7653FDE94AA /* Pods_RunnerTests.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 97C146EB1CF9000F007C117D /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 87ED1B5A2FB615FD00137A76 /* cactus-ios.xcframework in Frameworks */, + A9BAB38550FCA6E33321C511 /* Pods_Runner.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 0B36191599E63AE71E3121BF /* Pods */ = { + isa = PBXGroup; + children = ( + 3F704E798108FE8E0BF9745C /* Pods-Runner.debug.xcconfig */, + FE3C24750620191807070F8A /* Pods-Runner.release.xcconfig */, + A726CCF9522062C550887F12 /* Pods-Runner.profile.xcconfig */, + 633B2DF041FE201ED9703959 /* Pods-RunnerTests.debug.xcconfig */, + 81F9D792A678E88223EAE03E /* Pods-RunnerTests.release.xcconfig */, + 7B1B1D6D479BF19842A3534C /* Pods-RunnerTests.profile.xcconfig */, + ); + path = Pods; + sourceTree = ""; + }; + 331C8082294A63A400263BE5 /* RunnerTests */ = { + isa = PBXGroup; + children = ( + 331C807B294A618700263BE5 /* RunnerTests.swift */, + ); + path = RunnerTests; + sourceTree = ""; + }; + 9740EEB11CF90186004384FC /* Flutter */ = { + isa = PBXGroup; + children = ( + 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */, + 9740EEB21CF90195004384FC /* Debug.xcconfig */, + 7AFA3C8E1D35360C0083082E /* Release.xcconfig */, + 9740EEB31CF90195004384FC /* Generated.xcconfig */, + ); + name = Flutter; + sourceTree = ""; + }; + 97C146E51CF9000F007C117D = { + isa = PBXGroup; + children = ( + 9740EEB11CF90186004384FC /* Flutter */, + 97C146F01CF9000F007C117D /* Runner */, + 97C146EF1CF9000F007C117D /* Products */, + 331C8082294A63A400263BE5 /* RunnerTests */, + 0B36191599E63AE71E3121BF /* Pods */, + DDC65B1797B144D328B044E6 /* Frameworks */, + ); + sourceTree = ""; + }; + 97C146EF1CF9000F007C117D /* Products */ = { + isa = PBXGroup; + children = ( + 97C146EE1CF9000F007C117D /* Runner.app */, + 331C8081294A63A400263BE5 /* RunnerTests.xctest */, + ); + name = Products; + sourceTree = ""; + }; + 97C146F01CF9000F007C117D /* Runner */ = { + isa = PBXGroup; + children = ( + 87ED1B582FB615E100137A76 /* cactus-ios.xcframework */, + 97C146FA1CF9000F007C117D /* Main.storyboard */, + 97C146FD1CF9000F007C117D /* Assets.xcassets */, + 97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */, + 97C147021CF9000F007C117D /* Info.plist */, + 1498D2321E8E86230040F4C2 /* GeneratedPluginRegistrant.h */, + 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */, + 74858FAE1ED2DC5600515810 /* AppDelegate.swift */, + 7884E8672EC3CC0400C636F2 /* SceneDelegate.swift */, + 74858FAD1ED2DC5600515810 /* Runner-Bridging-Header.h */, + ); + path = Runner; + sourceTree = ""; + }; + DDC65B1797B144D328B044E6 /* Frameworks */ = { + isa = PBXGroup; + children = ( + 0CD8C9D04440C82664808F55 /* Pods_Runner.framework */, + E39A2DE24A32093D9EFA6245 /* Pods_RunnerTests.framework */, + ); + name = Frameworks; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 331C8080294A63A400263BE5 /* RunnerTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 331C8087294A63A400263BE5 /* Build configuration list for PBXNativeTarget "RunnerTests" */; + buildPhases = ( + 84C5CD3ACB761C906EFF73AF /* [CP] Check Pods Manifest.lock */, + 331C807D294A63A400263BE5 /* Sources */, + 331C807F294A63A400263BE5 /* Resources */, + 4F91C6DB5872B6929FA57E78 /* Frameworks */, + ); + buildRules = ( + ); + dependencies = ( + 331C8086294A63A400263BE5 /* PBXTargetDependency */, + ); + name = RunnerTests; + productName = RunnerTests; + productReference = 331C8081294A63A400263BE5 /* RunnerTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; + 97C146ED1CF9000F007C117D /* Runner */ = { + isa = PBXNativeTarget; + buildConfigurationList = 97C147051CF9000F007C117D /* Build configuration list for PBXNativeTarget "Runner" */; + buildPhases = ( + 4104B38A38ACE2716C65C0AC /* [CP] Check Pods Manifest.lock */, + 9740EEB61CF901F6004384FC /* Run Script */, + 97C146EA1CF9000F007C117D /* Sources */, + 97C146EB1CF9000F007C117D /* Frameworks */, + 97C146EC1CF9000F007C117D /* Resources */, + 9705A1C41CF9048500538489 /* Embed Frameworks */, + 3B06AD1E1E4923F5004D2608 /* Thin Binary */, + 8E381D203F58DD0104FC4721 /* [CP] Copy Pods Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = Runner; + productName = Runner; + productReference = 97C146EE1CF9000F007C117D /* Runner.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 97C146E61CF9000F007C117D /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = YES; + LastUpgradeCheck = 1510; + ORGANIZATIONNAME = ""; + TargetAttributes = { + 331C8080294A63A400263BE5 = { + CreatedOnToolsVersion = 14.0; + TestTargetID = 97C146ED1CF9000F007C117D; + }; + 97C146ED1CF9000F007C117D = { + CreatedOnToolsVersion = 7.3.1; + LastSwiftMigration = 1100; + }; + }; + }; + buildConfigurationList = 97C146E91CF9000F007C117D /* Build configuration list for PBXProject "Runner" */; + compatibilityVersion = "Xcode 9.3"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 97C146E51CF9000F007C117D; + productRefGroup = 97C146EF1CF9000F007C117D /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 97C146ED1CF9000F007C117D /* Runner */, + 331C8080294A63A400263BE5 /* RunnerTests */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 331C807F294A63A400263BE5 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 97C146EC1CF9000F007C117D /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 97C147011CF9000F007C117D /* LaunchScreen.storyboard in Resources */, + 3B3967161E833CAA004F5970 /* AppFrameworkInfo.plist in Resources */, + 97C146FE1CF9000F007C117D /* Assets.xcassets in Resources */, + 97C146FC1CF9000F007C117D /* Main.storyboard in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXShellScriptBuildPhase section */ + 3B06AD1E1E4923F5004D2608 /* Thin Binary */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputPaths = ( + "${TARGET_BUILD_DIR}/${INFOPLIST_PATH}", + ); + name = "Thin Binary"; + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "/bin/sh \"$FLUTTER_ROOT/packages/flutter_tools/bin/xcode_backend.sh\" embed_and_thin"; + }; + 4104B38A38ACE2716C65C0AC /* [CP] Check Pods Manifest.lock */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + "${PODS_PODFILE_DIR_PATH}/Podfile.lock", + "${PODS_ROOT}/Manifest.lock", + ); + name = "[CP] Check Pods Manifest.lock"; + outputFileListPaths = ( + ); + outputPaths = ( + "$(DERIVED_FILE_DIR)/Pods-Runner-checkManifestLockResult.txt", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; + showEnvVarsInLog = 0; + }; + 84C5CD3ACB761C906EFF73AF /* [CP] Check Pods Manifest.lock */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + "${PODS_PODFILE_DIR_PATH}/Podfile.lock", + "${PODS_ROOT}/Manifest.lock", + ); + name = "[CP] Check Pods Manifest.lock"; + outputFileListPaths = ( + ); + outputPaths = ( + "$(DERIVED_FILE_DIR)/Pods-RunnerTests-checkManifestLockResult.txt", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; + showEnvVarsInLog = 0; + }; + 8E381D203F58DD0104FC4721 /* [CP] Copy Pods Resources */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-resources-${CONFIGURATION}-input-files.xcfilelist", + ); + name = "[CP] Copy Pods Resources"; + outputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-resources-${CONFIGURATION}-output-files.xcfilelist", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-resources.sh\"\n"; + showEnvVarsInLog = 0; + }; + 9740EEB61CF901F6004384FC /* Run Script */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputPaths = ( + ); + name = "Run Script"; + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "/bin/sh \"$FLUTTER_ROOT/packages/flutter_tools/bin/xcode_backend.sh\" build"; + }; +/* End PBXShellScriptBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 331C807D294A63A400263BE5 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 331C808B294A63AB00263BE5 /* RunnerTests.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 97C146EA1CF9000F007C117D /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 74858FAF1ED2DC5600515810 /* AppDelegate.swift in Sources */, + 1498D2341E8E89220040F4C2 /* GeneratedPluginRegistrant.m in Sources */, + 7884E8682EC3CC0700C636F2 /* SceneDelegate.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 331C8086294A63A400263BE5 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 97C146ED1CF9000F007C117D /* Runner */; + targetProxy = 331C8085294A63A400263BE5 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin PBXVariantGroup section */ + 97C146FA1CF9000F007C117D /* Main.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 97C146FB1CF9000F007C117D /* Base */, + ); + name = Main.storyboard; + sourceTree = ""; + }; + 97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 97C147001CF9000F007C117D /* Base */, + ); + name = LaunchScreen.storyboard; + sourceTree = ""; + }; +/* End PBXVariantGroup section */ + +/* Begin XCBuildConfiguration section */ + 249021D3217E4FDB00AE95B9 /* Profile */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + MTL_ENABLE_DEBUG_INFO = NO; + SDKROOT = iphoneos; + STRING_CATALOG_GENERATE_SYMBOLS = YES; + SUPPORTED_PLATFORMS = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + VALIDATE_PRODUCT = YES; + }; + name = Profile; + }; + 249021D4217E4FDB00AE95B9 /* Profile */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 7AFA3C8E1D35360C0083082E /* Release.xcconfig */; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CLANG_ENABLE_MODULES = YES; + CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; + DEVELOPMENT_TEAM = BUYL45TA3C; + ENABLE_BITCODE = NO; + INFOPLIST_FILE = Runner/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = ai.sunbird.sunflowerApp; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h"; + SWIFT_VERSION = 5.0; + VERSIONING_SYSTEM = "apple-generic"; + }; + name = Profile; + }; + 331C8088294A63A400263BE5 /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 633B2DF041FE201ED9703959 /* Pods-RunnerTests.debug.xcconfig */; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = ai.sunbird.sunflowerApp.RunnerTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_VERSION = 5.0; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/Runner.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Runner"; + }; + name = Debug; + }; + 331C8089294A63A400263BE5 /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 81F9D792A678E88223EAE03E /* Pods-RunnerTests.release.xcconfig */; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = ai.sunbird.sunflowerApp.RunnerTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 5.0; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/Runner.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Runner"; + }; + name = Release; + }; + 331C808A294A63A400263BE5 /* Profile */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 7B1B1D6D479BF19842A3534C /* Pods-RunnerTests.profile.xcconfig */; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = ai.sunbird.sunflowerApp.RunnerTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 5.0; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/Runner.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Runner"; + }; + name = Profile; + }; + 97C147031CF9000F007C117D /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + MTL_ENABLE_DEBUG_INFO = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + STRING_CATALOG_GENERATE_SYMBOLS = YES; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 97C147041CF9000F007C117D /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; + MTL_ENABLE_DEBUG_INFO = NO; + SDKROOT = iphoneos; + STRING_CATALOG_GENERATE_SYMBOLS = YES; + SUPPORTED_PLATFORMS = iphoneos; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; + TARGETED_DEVICE_FAMILY = "1,2"; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + 97C147061CF9000F007C117D /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 9740EEB21CF90195004384FC /* Debug.xcconfig */; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CLANG_ENABLE_MODULES = YES; + CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; + DEVELOPMENT_TEAM = BUYL45TA3C; + ENABLE_BITCODE = NO; + INFOPLIST_FILE = Runner/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = ai.sunbird.sunflowerApp; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h"; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_VERSION = 5.0; + VERSIONING_SYSTEM = "apple-generic"; + }; + name = Debug; + }; + 97C147071CF9000F007C117D /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 7AFA3C8E1D35360C0083082E /* Release.xcconfig */; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CLANG_ENABLE_MODULES = YES; + CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; + DEVELOPMENT_TEAM = BUYL45TA3C; + ENABLE_BITCODE = NO; + INFOPLIST_FILE = Runner/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = ai.sunbird.sunflowerApp; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h"; + SWIFT_VERSION = 5.0; + VERSIONING_SYSTEM = "apple-generic"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 331C8087294A63A400263BE5 /* Build configuration list for PBXNativeTarget "RunnerTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 331C8088294A63A400263BE5 /* Debug */, + 331C8089294A63A400263BE5 /* Release */, + 331C808A294A63A400263BE5 /* Profile */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 97C146E91CF9000F007C117D /* Build configuration list for PBXProject "Runner" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 97C147031CF9000F007C117D /* Debug */, + 97C147041CF9000F007C117D /* Release */, + 249021D3217E4FDB00AE95B9 /* Profile */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 97C147051CF9000F007C117D /* Build configuration list for PBXNativeTarget "Runner" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 97C147061CF9000F007C117D /* Debug */, + 97C147071CF9000F007C117D /* Release */, + 249021D4217E4FDB00AE95B9 /* Profile */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 97C146E61CF9000F007C117D /* Project object */; +} diff --git a/ios/Runner.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/ios/Runner.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..919434a --- /dev/null +++ b/ios/Runner.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/ios/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/ios/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 0000000..18d9810 --- /dev/null +++ b/ios/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/ios/Runner.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings b/ios/Runner.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings new file mode 100644 index 0000000..f9b0d7c --- /dev/null +++ b/ios/Runner.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings @@ -0,0 +1,8 @@ + + + + + PreviewsEnabled + + + diff --git a/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme new file mode 100644 index 0000000..c72ba24 --- /dev/null +++ b/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -0,0 +1,102 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ios/Runner.xcworkspace/contents.xcworkspacedata b/ios/Runner.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..21a3cc1 --- /dev/null +++ b/ios/Runner.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,10 @@ + + + + + + + diff --git a/ios/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/ios/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 0000000..18d9810 --- /dev/null +++ b/ios/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/ios/Runner.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings b/ios/Runner.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings new file mode 100644 index 0000000..f9b0d7c --- /dev/null +++ b/ios/Runner.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings @@ -0,0 +1,8 @@ + + + + + PreviewsEnabled + + + diff --git a/ios/Runner/AppDelegate.swift b/ios/Runner/AppDelegate.swift new file mode 100644 index 0000000..c30b367 --- /dev/null +++ b/ios/Runner/AppDelegate.swift @@ -0,0 +1,16 @@ +import Flutter +import UIKit + +@main +@objc class AppDelegate: FlutterAppDelegate, FlutterImplicitEngineDelegate { + override func application( + _ application: UIApplication, + didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? + ) -> Bool { + return super.application(application, didFinishLaunchingWithOptions: launchOptions) + } + + func didInitializeImplicitFlutterEngine(_ engineBridge: FlutterImplicitEngineBridge) { + GeneratedPluginRegistrant.register(with: engineBridge.pluginRegistry) + } +} diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 0000000..d36b1fa --- /dev/null +++ b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,122 @@ +{ + "images" : [ + { + "size" : "20x20", + "idiom" : "iphone", + "filename" : "Icon-App-20x20@2x.png", + "scale" : "2x" + }, + { + "size" : "20x20", + "idiom" : "iphone", + "filename" : "Icon-App-20x20@3x.png", + "scale" : "3x" + }, + { + "size" : "29x29", + "idiom" : "iphone", + "filename" : "Icon-App-29x29@1x.png", + "scale" : "1x" + }, + { + "size" : "29x29", + "idiom" : "iphone", + "filename" : "Icon-App-29x29@2x.png", + "scale" : "2x" + }, + { + "size" : "29x29", + "idiom" : "iphone", + "filename" : "Icon-App-29x29@3x.png", + "scale" : "3x" + }, + { + "size" : "40x40", + "idiom" : "iphone", + "filename" : "Icon-App-40x40@2x.png", + "scale" : "2x" + }, + { + "size" : "40x40", + "idiom" : "iphone", + "filename" : "Icon-App-40x40@3x.png", + "scale" : "3x" + }, + { + "size" : "60x60", + "idiom" : "iphone", + "filename" : "Icon-App-60x60@2x.png", + "scale" : "2x" + }, + { + "size" : "60x60", + "idiom" : "iphone", + "filename" : "Icon-App-60x60@3x.png", + "scale" : "3x" + }, + { + "size" : "20x20", + "idiom" : "ipad", + "filename" : "Icon-App-20x20@1x.png", + "scale" : "1x" + }, + { + "size" : "20x20", + "idiom" : "ipad", + "filename" : "Icon-App-20x20@2x.png", + "scale" : "2x" + }, + { + "size" : "29x29", + "idiom" : "ipad", + "filename" : "Icon-App-29x29@1x.png", + "scale" : "1x" + }, + { + "size" : "29x29", + "idiom" : "ipad", + "filename" : "Icon-App-29x29@2x.png", + "scale" : "2x" + }, + { + "size" : "40x40", + "idiom" : "ipad", + "filename" : "Icon-App-40x40@1x.png", + "scale" : "1x" + }, + { + "size" : "40x40", + "idiom" : "ipad", + "filename" : "Icon-App-40x40@2x.png", + "scale" : "2x" + }, + { + "size" : "76x76", + "idiom" : "ipad", + "filename" : "Icon-App-76x76@1x.png", + "scale" : "1x" + }, + { + "size" : "76x76", + "idiom" : "ipad", + "filename" : "Icon-App-76x76@2x.png", + "scale" : "2x" + }, + { + "size" : "83.5x83.5", + "idiom" : "ipad", + "filename" : "Icon-App-83.5x83.5@2x.png", + "scale" : "2x" + }, + { + "size" : "1024x1024", + "idiom" : "ios-marketing", + "filename" : "Icon-App-1024x1024@1x.png", + "scale" : "1x" + } + ], + "info" : { + "version" : 1, + "author" : "xcode" + } +} diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-1024x1024@1x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-1024x1024@1x.png new file mode 100644 index 0000000..dc9ada4 Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-1024x1024@1x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@1x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@1x.png new file mode 100644 index 0000000..7353c41 Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@1x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@2x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@2x.png new file mode 100644 index 0000000..797d452 Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@2x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@3x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@3x.png new file mode 100644 index 0000000..6ed2d93 Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-20x20@3x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@1x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@1x.png new file mode 100644 index 0000000..4cd7b00 Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@1x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@2x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@2x.png new file mode 100644 index 0000000..fe73094 Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@2x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@3x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@3x.png new file mode 100644 index 0000000..321773c Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-29x29@3x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@1x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@1x.png new file mode 100644 index 0000000..797d452 Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@1x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@2x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@2x.png new file mode 100644 index 0000000..502f463 Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@2x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@3x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@3x.png new file mode 100644 index 0000000..0ec3034 Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-40x40@3x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-60x60@2x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-60x60@2x.png new file mode 100644 index 0000000..0ec3034 Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-60x60@2x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-60x60@3x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-60x60@3x.png new file mode 100644 index 0000000..e9f5fea Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-60x60@3x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-76x76@1x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-76x76@1x.png new file mode 100644 index 0000000..84ac32a Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-76x76@1x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-76x76@2x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-76x76@2x.png new file mode 100644 index 0000000..8953cba Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-76x76@2x.png differ diff --git a/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-83.5x83.5@2x.png b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-83.5x83.5@2x.png new file mode 100644 index 0000000..0467bf1 Binary files /dev/null and b/ios/Runner/Assets.xcassets/AppIcon.appiconset/Icon-App-83.5x83.5@2x.png differ diff --git a/ios/Runner/Assets.xcassets/LaunchImage.imageset/Contents.json b/ios/Runner/Assets.xcassets/LaunchImage.imageset/Contents.json new file mode 100644 index 0000000..0bedcf2 --- /dev/null +++ b/ios/Runner/Assets.xcassets/LaunchImage.imageset/Contents.json @@ -0,0 +1,23 @@ +{ + "images" : [ + { + "idiom" : "universal", + "filename" : "LaunchImage.png", + "scale" : "1x" + }, + { + "idiom" : "universal", + "filename" : "LaunchImage@2x.png", + "scale" : "2x" + }, + { + "idiom" : "universal", + "filename" : "LaunchImage@3x.png", + "scale" : "3x" + } + ], + "info" : { + "version" : 1, + "author" : "xcode" + } +} diff --git a/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage.png b/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage.png new file mode 100644 index 0000000..9da19ea Binary files /dev/null and b/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage.png differ diff --git a/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage@2x.png b/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage@2x.png new file mode 100644 index 0000000..9da19ea Binary files /dev/null and b/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage@2x.png differ diff --git a/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage@3x.png b/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage@3x.png new file mode 100644 index 0000000..9da19ea Binary files /dev/null and b/ios/Runner/Assets.xcassets/LaunchImage.imageset/LaunchImage@3x.png differ diff --git a/ios/Runner/Assets.xcassets/LaunchImage.imageset/README.md b/ios/Runner/Assets.xcassets/LaunchImage.imageset/README.md new file mode 100644 index 0000000..89c2725 --- /dev/null +++ b/ios/Runner/Assets.xcassets/LaunchImage.imageset/README.md @@ -0,0 +1,5 @@ +# Launch Screen Assets + +You can customize the launch screen with your own desired assets by replacing the image files in this directory. + +You can also do it by opening your Flutter project's Xcode project with `open ios/Runner.xcworkspace`, selecting `Runner/Assets.xcassets` in the Project Navigator and dropping in the desired images. \ No newline at end of file diff --git a/ios/Runner/Base.lproj/LaunchScreen.storyboard b/ios/Runner/Base.lproj/LaunchScreen.storyboard new file mode 100644 index 0000000..f2e259c --- /dev/null +++ b/ios/Runner/Base.lproj/LaunchScreen.storyboard @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ios/Runner/Base.lproj/Main.storyboard b/ios/Runner/Base.lproj/Main.storyboard new file mode 100644 index 0000000..3ce0803 --- /dev/null +++ b/ios/Runner/Base.lproj/Main.storyboard @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ios/Runner/Info.plist b/ios/Runner/Info.plist new file mode 100644 index 0000000..18cb3a4 --- /dev/null +++ b/ios/Runner/Info.plist @@ -0,0 +1,76 @@ + + + + + NSMicrophoneUsageDescription + We need microphone access to record and transcribe your voice. + BGTaskSchedulerPermittedIdentifiers + + com.pravera.flutter_foreground_task.refresh + + CADisableMinimumFrameDurationOnPhone + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleDisplayName + Sunflower 🌻 + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + sunflower_app + CFBundlePackageType + APPL + CFBundleShortVersionString + $(FLUTTER_BUILD_NAME) + CFBundleSignature + ???? + CFBundleVersion + $(FLUTTER_BUILD_NUMBER) + LSRequiresIPhoneOS + + UIApplicationSceneManifest + + UIApplicationSupportsMultipleScenes + + UISceneConfigurations + + UIWindowSceneSessionRoleApplication + + + UISceneClassName + UIWindowScene + UISceneConfigurationName + flutter + UISceneDelegateClassName + $(PRODUCT_MODULE_NAME).SceneDelegate + UISceneStoryboardFile + Main + + + + + UIApplicationSupportsIndirectInputEvents + + UILaunchStoryboardName + LaunchScreen + UIMainStoryboardFile + Main + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + + diff --git a/ios/Runner/Runner-Bridging-Header.h b/ios/Runner/Runner-Bridging-Header.h new file mode 100644 index 0000000..308a2a5 --- /dev/null +++ b/ios/Runner/Runner-Bridging-Header.h @@ -0,0 +1 @@ +#import "GeneratedPluginRegistrant.h" diff --git a/ios/Runner/SceneDelegate.swift b/ios/Runner/SceneDelegate.swift new file mode 100644 index 0000000..b9ce8ea --- /dev/null +++ b/ios/Runner/SceneDelegate.swift @@ -0,0 +1,6 @@ +import Flutter +import UIKit + +class SceneDelegate: FlutterSceneDelegate { + +} diff --git a/ios/Runner/cactus-ios.xcframework/Info.plist b/ios/Runner/cactus-ios.xcframework/Info.plist new file mode 100644 index 0000000..55424d6 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/Info.plist @@ -0,0 +1,39 @@ + + + + + AvailableLibraries + + + LibraryIdentifier + ios-arm64 + LibraryPath + cactus.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + ios + + + LibraryIdentifier + ios-arm64-simulator + LibraryPath + cactus.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + ios + SupportedPlatformVariant + simulator + + + CFBundlePackageType + XFWK + XCFrameworkFormatVersion + 1.0 + + diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus.h b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus.h new file mode 100644 index 0000000..b5b6d80 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus.h @@ -0,0 +1,12 @@ +#ifndef CACTUS_H +#define CACTUS_H + +#include "graph/graph.h" +#include "kernel/kernel.h" +#include "kernel/kernel_utils.h" +#include "engine/engine.h" +#include "models/model.h" +#include "ffi/cactus_ffi.h" +#include "npu/npu.h" + +#endif // CACTUS_H \ No newline at end of file diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_cloud.h b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_cloud.h new file mode 100644 index 0000000..6911d9a --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_cloud.h @@ -0,0 +1,50 @@ +#ifndef CACTUS_CLOUD_H +#define CACTUS_CLOUD_H + +#include "cactus_utils.h" +#include +#include + +namespace cactus { +namespace ffi { + +struct CloudResponse { + std::string transcript; + std::string api_key_hash; + bool used_cloud = false; + std::string error; +}; + +struct CloudCompletionRequest { + std::vector messages; + std::vector tools; + std::string local_output; + std::vector local_function_calls; + bool has_images = false; + bool has_audio = false; + std::vector audio_pcm; + std::string cloud_key; +}; + +struct CloudCompletionResult { + bool ok = false; + bool used_cloud = false; + std::string response; + std::vector function_calls; + std::string error; +}; + +std::string cloud_base64_encode(const uint8_t* data, size_t len); +std::vector cloud_build_wav(const uint8_t* pcm, size_t pcm_bytes); +std::string resolve_cloud_api_key(const char* cloud_key_param); +CloudResponse cloud_transcribe_request(const std::string& audio_b64, + const std::string& fallback_text, + long timeout_seconds = 15L, + const char* cloud_key = nullptr); +CloudCompletionResult cloud_complete_request(const CloudCompletionRequest& request, + long timeout_ms); + +} // namespace ffi +} // namespace cactus + +#endif // CACTUS_CLOUD_H diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_ffi.h b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_ffi.h new file mode 100644 index 0000000..163e180 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_ffi.h @@ -0,0 +1,454 @@ +#ifndef CACTUS_FFI_H +#define CACTUS_FFI_H + +#include +#include +#include + +#if __GNUC__ >= 4 + #define CACTUS_FFI_EXPORT __attribute__((visibility("default"))) + #define CACTUS_FFI_LOCAL __attribute__((visibility("hidden"))) +#else + #define CACTUS_FFI_EXPORT + #define CACTUS_FFI_LOCAL +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* cactus_model_t; +typedef void* cactus_index_t; +typedef void* cactus_stream_transcribe_t; + +typedef void (*cactus_token_callback)(const char* token, uint32_t token_id, void* user_data); + +CACTUS_FFI_EXPORT cactus_model_t cactus_init( + const char* model_path, + const char* corpus_dir, // optional: NULL if no RAG corpus + bool cache_index // false = always rebuild index, true = load cached if available +); + +CACTUS_FFI_EXPORT void cactus_destroy(cactus_model_t model); +CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model); +CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model); + +CACTUS_FFI_EXPORT int cactus_complete( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const char* tools_json, // optional + cactus_token_callback callback, // optional + void* user_data, // optional + const uint8_t* pcm_buffer, // optional: NULL when not used + size_t pcm_buffer_size // optional: 0 when not used +); + +CACTUS_FFI_EXPORT int cactus_prefill( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const char* tools_json, // optional + const uint8_t* pcm_buffer, // optional: NULL when not used + size_t pcm_buffer_size // optional: 0 when not used +); + +CACTUS_FFI_EXPORT int cactus_tokenize( + cactus_model_t model, + const char* text, + uint32_t* token_buffer, + size_t token_buffer_len, + size_t* out_token_len +); + +CACTUS_FFI_EXPORT int cactus_score_window( + cactus_model_t model, + const uint32_t* tokens, + size_t token_len, + size_t start, + size_t end, + size_t context, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_transcribe( + cactus_model_t model, + const char* audio_file_path, // NULL if using pcm_buffer + const char* prompt, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + cactus_token_callback callback, // optional + void* user_data, // optional + const uint8_t* pcm_buffer, // NULL if using audio_file_path + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_detect_language( + cactus_model_t model, + const char* audio_file_path, // NULL if using pcm_buffer + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const uint8_t* pcm_buffer, // NULL if using audio_file_path + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT cactus_stream_transcribe_t cactus_stream_transcribe_start( + cactus_model_t model, + const char* options_json // optional +); + +CACTUS_FFI_EXPORT int cactus_stream_transcribe_process( + cactus_stream_transcribe_t stream, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_stream_transcribe_stop( + cactus_stream_transcribe_t stream, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_embed( + cactus_model_t model, + const char* text, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim, + bool normalize +); + +CACTUS_FFI_EXPORT int cactus_image_embed( + cactus_model_t model, + const char* image_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_audio_embed( + cactus_model_t model, + const char* audio_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_vad( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_diarize( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_embed_speaker( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + const float* mask_weights, + size_t mask_num_frames +); + +CACTUS_FFI_EXPORT int cactus_rag_query( + cactus_model_t model, + const char* query, + char* response_buffer, + size_t buffer_size, + size_t top_k +); + +CACTUS_FFI_EXPORT cactus_index_t cactus_index_init( + const char* index_dir, + size_t embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_index_add( + cactus_index_t index, + const int* ids, + const char** documents, + const char** metadatas, // optional: can be NULL + const float** embeddings, + size_t count, + size_t embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_index_delete( + cactus_index_t index, + const int* ids, + size_t ids_count +); + +CACTUS_FFI_EXPORT int cactus_index_get( + cactus_index_t index, + const int* ids, + size_t ids_count, + char** document_buffers, + size_t* document_buffer_sizes, + char** metadata_buffers, + size_t* metadata_buffer_sizes, + float** embedding_buffers, + size_t* embedding_buffer_sizes +); + +CACTUS_FFI_EXPORT int cactus_index_query( + cactus_index_t index, + const float** embeddings, + size_t embeddings_count, + size_t embedding_dim, + const char* options_json, // optional + int** id_buffers, + size_t* id_buffer_sizes, + float** score_buffers, + size_t* score_buffer_sizes +); + +CACTUS_FFI_EXPORT int cactus_index_compact(cactus_index_t index); +CACTUS_FFI_EXPORT void cactus_index_destroy(cactus_index_t index); + +CACTUS_FFI_EXPORT const char* cactus_get_last_error(void); + +// level: 0=DEBUG, 1=INFO, 2=WARN (default), 3=ERROR, 4=NONE +CACTUS_FFI_EXPORT void cactus_log_set_level(int level); + +typedef void (*cactus_log_callback_t)(int level, const char* component, const char* message, void* user_data); +CACTUS_FFI_EXPORT void cactus_log_set_callback(cactus_log_callback_t callback, void* user_data); + +CACTUS_FFI_EXPORT void cactus_set_telemetry_environment(const char* framework, const char* cache_location, const char* version); +CACTUS_FFI_EXPORT void cactus_set_app_id(const char* app_id); +CACTUS_FFI_EXPORT void cactus_telemetry_flush(void); +CACTUS_FFI_EXPORT void cactus_telemetry_shutdown(void); + +// cactus graph export +typedef void* cactus_graph_t; +typedef uint64_t cactus_node_t; + +typedef struct { + int32_t precision; + size_t rank; + size_t shape[8]; + size_t num_elements; + size_t byte_size; +} cactus_tensor_info_t; + +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_create(void); +CACTUS_FFI_EXPORT void cactus_graph_destroy(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_hard_reset(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_save(cactus_graph_t graph, const char* filename); +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_load(const char* filename); + +CACTUS_FFI_EXPORT int cactus_graph_input( + cactus_graph_t graph, const size_t* shape, size_t rank, int32_t precision, +cactus_node_t* out_node); + +CACTUS_FFI_EXPORT int cactus_graph_set_input( + cactus_graph_t graph, cactus_node_t node, const void* data, int32_t +precision); +CACTUS_FFI_EXPORT int cactus_graph_set_external_input( + cactus_graph_t graph, cactus_node_t node, void* data, int32_t precision); + +CACTUS_FFI_EXPORT int cactus_graph_precision_cast( + cactus_graph_t graph, cactus_node_t input, int32_t target_precision, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_quantize_activations( + cactus_graph_t graph, cactus_node_t input, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_add(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_add_clipped(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_subtract(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_multiply(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_divide(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_scalar_add(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_subtract(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_multiply(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_exp(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sqrt(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_cos(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sin(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_log(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_abs(cactus_graph_t graph, cactus_node_t x, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_pow(cactus_graph_t graph, cactus_node_t x, +float exponent, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_view( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_flatten( + cactus_graph_t graph, cactus_node_t x, int32_t start_dim, int32_t end_dim, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_reshape( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose( + cactus_graph_t graph, cactus_node_t x, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose_n( + cactus_graph_t graph, cactus_node_t x, const size_t* permutation, size_t rank, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_slice( + cactus_graph_t graph, cactus_node_t x, int32_t axis, size_t start, size_t length, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_index( + cactus_graph_t graph, cactus_node_t x, size_t index_value, int32_t dim, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_concat( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, +cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_cat( + cactus_graph_t graph, const cactus_node_t* nodes, size_t count, int32_t +axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_matmul( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, bool pretransposed_rhs, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gather( + cactus_graph_t graph, cactus_node_t tensor, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_tensor( + cactus_graph_t graph, cactus_node_t embedding_tensor, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_file( + cactus_graph_t graph, const char* filename, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_embeddings( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_weights( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_bilinear_interpolation( + cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_set_grouped_scales( + cactus_graph_t graph, cactus_node_t node, size_t group_size, size_t num_groups, void* scales_ptr); +CACTUS_FFI_EXPORT int cactus_graph_set_interleaved( + cactus_graph_t graph, cactus_node_t node, bool interleaved, size_t original_n); +CACTUS_FFI_EXPORT int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_prefetch_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_release_all_weight_pages(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_relu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_silu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu_erf(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sigmoid(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_tanh(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_glu(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_layernorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, float epsilon, bool has_bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_groupnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, size_t num_groups, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_batchnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t running_mean, cactus_node_t running_var, int32_t axis, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_topk(cactus_graph_t graph, cactus_node_t input, size_t k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rms_norm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope_gptj( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, size_t rot_dim, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_softmax(cactus_graph_t graph, cactus_node_t input, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, float scale, bool is_causal, size_t position_offset, size_t window_size, int32_t backend, bool use_mask, cactus_node_t mask, bool additive_mask, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rel_pos_bias( + cactus_graph_t graph, cactus_node_t query, cactus_node_t relative_key, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention_int8_hybrid( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_conv1d_causal( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t kernel_size, size_t dilation, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k7s3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_same_depthwise_k9( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_pointwise( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_depthwise_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_pointwise_1x1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_lstm_cell( + cactus_graph_t graph, cactus_node_t input, cactus_node_t h_prev, cactus_node_t c_prev, cactus_node_t weight_ih, cactus_node_t weight_hh, cactus_node_t bias_ih, cactus_node_t bias_hh, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_decode( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_prefill( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, size_t chunk_size, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_stft( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, size_t num_fft_bins, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_altup_predict( + cactus_graph_t graph, cactus_node_t coefs, const cactus_node_t* streams, size_t num_streams, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_altup_correct( + cactus_graph_t graph, cactus_node_t coefs, cactus_node_t innovation, const cactus_node_t* predictions, size_t num_predictions, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gaussian_topk( + cactus_graph_t graph, cactus_node_t input, float ppf, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_gated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_ungated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_sample( + cactus_graph_t graph, cactus_node_t logits, float temperature, float top_p, size_t top_k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scatter_topk( + cactus_graph_t graph, cactus_node_t indices, cactus_node_t values, size_t num_classes, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_persistent( + cactus_graph_t graph, cactus_node_t source_node, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_is_populated( + cactus_graph_t graph, cactus_node_t persistent_node, int32_t* out_is_populated); +CACTUS_FFI_EXPORT int cactus_graph_invalidate_persistent( + cactus_graph_t graph, cactus_node_t persistent_node); + +CACTUS_FFI_EXPORT int cactus_graph_execute(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_get_output_ptr(cactus_graph_t graph, +cactus_node_t node, void** out_ptr); +CACTUS_FFI_EXPORT int cactus_graph_get_output_info(cactus_graph_t graph, +cactus_node_t node, cactus_tensor_info_t* out_info); + +#ifdef __cplusplus +} +#endif + +#endif // CACTUS_FFI_H diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_utils.h b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_utils.h new file mode 100644 index 0000000..aad98d8 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_utils.h @@ -0,0 +1,1869 @@ +#ifndef CACTUS_UTILS_H +#define CACTUS_UTILS_H + +#include "../engine/engine.h" +#include "../models/model.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#include +#elif defined(_WIN32) +#include +#include +#elif defined(__linux__) || defined(__ANDROID__) +#include +#endif + +inline size_t get_memory_footprint_bytes() { +#ifdef __APPLE__ + task_vm_info_data_t vm_info; + mach_msg_type_number_t count = TASK_VM_INFO_COUNT; + if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&vm_info, &count) == KERN_SUCCESS) + return vm_info.phys_footprint; + +#elif defined(_WIN32) + PROCESS_MEMORY_COUNTERS_EX pmc; + if (GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc))) + return pmc.PrivateUsage; + +#elif defined(__linux__) || defined(__ANDROID__) + std::ifstream statm("/proc/self/statm"); + if (statm.is_open()) { + size_t size, resident; + statm >> size >> resident; + return resident * sysconf(_SC_PAGESIZE); + } +#endif + return 0; +} + +inline double get_ram_usage_mb() { + return get_memory_footprint_bytes() / (1024.0 * 1024.0); +} + +struct CactusModelHandle { + std::unique_ptr model; + std::unique_ptr vad_model; + std::atomic should_stop; + std::vector processed_tokens; + struct ProcessedImage { + std::string path; + long long last_modified_timestamp = 0; + + bool operator==(const ProcessedImage& other) const { + return path == other.path && last_modified_timestamp == other.last_modified_timestamp; + } + }; + + std::vector> processed_images; + std::vector user_audio_counts; + std::mutex model_mutex; + std::string model_name; + std::unique_ptr corpus_index; + std::string corpus_dir; + size_t corpus_embedding_dim = 0; + std::vector> tool_embeddings; + std::vector tool_texts; + + CactusModelHandle() : should_stop(false) {} +}; + +extern std::string last_error_message; + +bool matches_stop_sequence(const std::vector& generated_tokens, + const std::vector>& stop_sequences); + +std::string retrieve_rag_context(CactusModelHandle* handle, const std::string& query); + +namespace cactus { +namespace audio { + +static constexpr size_t WHISPER_TARGET_FRAMES = 3000; +static constexpr int WHISPER_SAMPLE_RATE = 16000; + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_whisper_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 400; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = true; + cfg.pad_mode = "reflect"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 1e-10f; + cfg.log_mel = "log10"; + cfg.reference = 1.0f; + cfg.min_value = 1e-10f; + cfg.remove_dc_offset = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_parakeet_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 512; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = true; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 5.960464477539063e-08f; // 2^-24 guard value used by HF Parakeet. + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 1e-10f; + cfg.remove_dc_offset = false; + cfg.hann_periodic = false; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_htk_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 321; + cfg.frame_length = 320; + cfg.fft_override = 1024; + cfg.hop_length = 160; + cfg.power = 1.0f; + cfg.center = false; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 0.001f; + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 0.001f; + cfg.remove_dc_offset = false; + cfg.hann_periodic = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_gemma4_audio_spectrogram_config( + const cactus::engine::Config& model_config) { + auto cfg = get_htk_spectrogram_config(); + cfg.fft_override = model_config.audio_fft_length; + cfg.mel_floor_additive = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_wespeaker_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 512; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = false; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 1.1754944e-38f; + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 1.1754944e-38f; + cfg.remove_dc_offset = true; + cfg.preemphasis = 0.97f; + cfg.hann_periodic = false; + cfg.window_a0 = 0.54f; + return cfg; +} + +// Whisper v1/v2: 80 mel bins, HTK. Whisper v3: 128 mel bins, Slaney, 512-FFT, no DC removal. +inline void init_whisper_mel_filters(cactus::engine::AudioProcessor& ap, + cactus::engine::AudioProcessor::SpectrogramConfig& cfg, + size_t mel_bins) { + const size_t num_mel_filters = std::max(1, mel_bins); + const bool is_v3 = mel_bins > 80; + if (is_v3) { + cfg.fft_override = 512; + cfg.remove_dc_offset = false; + } + const size_t fft_len = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; + const size_t num_frequency_bins = fft_len / 2 + 1; + if (is_v3) { + ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, + WHISPER_SAMPLE_RATE, "slaney", "slaney"); + } else { + ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, + WHISPER_SAMPLE_RATE); + } +} + +// use_mel_floor_padding=true pads short audio with the normalized mel floor (required for v3). +inline std::vector normalize_whisper_mel(std::vector& mel, size_t n_mels, + bool use_mel_floor_padding = false) { + if (mel.empty() || n_mels == 0) return mel; + size_t n_frames = mel.size() / n_mels; + + float max_val = -std::numeric_limits::infinity(); + for (float v : mel) if (v > max_val) max_val = v; + + float min_allowed = max_val - 8.0f; + for (float& v : mel) { + if (v < min_allowed) v = min_allowed; + v = (v + 4.0f) * 0.25f; + } + + if (n_frames != WHISPER_TARGET_FRAMES) { + float pad_val = use_mel_floor_padding ? (min_allowed + 4.0f) * 0.25f : 0.0f; + std::vector fixed(n_mels * WHISPER_TARGET_FRAMES, pad_val); + size_t copy_frames = std::min(n_frames, WHISPER_TARGET_FRAMES); + for (size_t m = 0; m < n_mels; ++m) { + const float* src = &mel[m * n_frames]; + float* dst = &fixed[m * WHISPER_TARGET_FRAMES]; + std::copy(src, src + copy_frames, dst); + } + return fixed; + } + return std::move(mel); +} + +inline std::vector transpose_mel_to_frame_major(const std::vector& mel, + size_t num_mels, size_t num_frames) { + std::vector transposed(num_frames * num_mels); + for (size_t m = 0; m < num_mels; m++) { + for (size_t t = 0; t < num_frames; t++) { + transposed[t * num_mels + m] = mel[m * num_frames + t]; + } + } + return transposed; +} + +inline void apply_preemphasis(std::vector& waveform, float coefficient = 0.97f) { + if (waveform.size() < 2 || coefficient == 0.0f) { + return; + } + for (size_t i = waveform.size() - 1; i > 0; --i) { + waveform[i] -= coefficient * waveform[i - 1]; + } +} + +inline void normalize_parakeet_log_mel(std::vector& mel, size_t num_mels, float epsilon = 1e-5f) { + if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { + return; + } + const size_t num_frames = mel.size() / num_mels; + if (num_frames == 0) { + return; + } + + for (size_t m = 0; m < num_mels; ++m) { + const size_t base = m * num_frames; + float mean = 0.0f; + for (size_t t = 0; t < num_frames; ++t) { + mean += mel[base + t]; + } + mean /= static_cast(num_frames); + + float variance = 0.0f; + for (size_t t = 0; t < num_frames; ++t) { + const float d = mel[base + t] - mean; + variance += d * d; + } + const float denom = static_cast(std::max(1, num_frames - 1)); + const float inv_std = 1.0f / std::sqrt((variance / denom) + epsilon); + for (size_t t = 0; t < num_frames; ++t) { + mel[base + t] = (mel[base + t] - mean) * inv_std; + } + } +} + +inline void trim_mel_frames(std::vector& mel, size_t num_mels, size_t valid_frames) { + if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { + return; + } + size_t total_frames = mel.size() / num_mels; + if (valid_frames == 0 || valid_frames >= total_frames) { + return; + } + std::vector trimmed(num_mels * valid_frames); + for (size_t m = 0; m < num_mels; ++m) { + const float* src = &mel[m * total_frames]; + float* dst = &trimmed[m * valid_frames]; + std::copy(src, src + valid_frames, dst); + } + mel.swap(trimmed); +} + +struct AudioPreprocessResult { + std::vector features; + size_t num_frames = 0; + size_t num_soft_tokens = 0; +}; + +inline AudioPreprocessResult preprocess_audio_for_gemma4( + std::vector audio_samples, + const cactus::engine::Config& model_config +) { + AudioPreprocessResult result; + if (audio_samples.empty()) return result; + + size_t pad_amt = 320 - (audio_samples.size() % 320); + if (pad_amt < 320) + audio_samples.resize(audio_samples.size() + pad_amt, 0.0f); + + size_t mel_bins = model_config.audio_input_feat_size; + auto cfg = get_gemma4_audio_spectrogram_config(model_config); + + size_t semicausal_pad = cfg.frame_length / 2; + audio_samples.insert(audio_samples.begin(), semicausal_pad, 0.0f); + + cactus::engine::AudioProcessor ap; + size_t fft_for_mel = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; + ap.init_mel_filters(fft_for_mel / 2 + 1, mel_bins, 0.0f, 8000.0f, 16000, + nullptr, "htk"); + std::vector mel = ap.compute_spectrogram(audio_samples, cfg); + + result.num_frames = mel.size() / mel_bins; + result.features = transpose_mel_to_frame_major(mel, mel_bins, result.num_frames); + + size_t after_stage1 = (result.num_frames + 1) / 2; + result.num_soft_tokens = (after_stage1 + 1) / 2; + + return result; +} + +inline std::vector pcm_buffer_to_float_samples( + const uint8_t* pcm_buffer, size_t pcm_buffer_size +) { + const int16_t* pcm_samples = reinterpret_cast(pcm_buffer); + size_t num_samples = pcm_buffer_size / 2; + std::vector waveform_fp32(num_samples); + constexpr float inv_32768 = 1.0f / 32768.0f; + for (size_t i = 0; i < num_samples; i++) + waveform_fp32[i] = static_cast(pcm_samples[i]) * inv_32768; + return waveform_fp32; +} + +} // namespace audio +} // namespace cactus + +namespace cactus { +namespace ffi { + +inline bool env_flag_enabled(const char* key) { + const char* value = std::getenv(key); + return value && value[0] != '\0' && !(value[0] == '0' && value[1] == '\0'); +} + +inline std::string generateUUID() { +#ifdef __APPLE__ + uuid_t uuid; + uuid_generate_random(uuid); + char uuid_str[37]; + uuid_unparse_lower(uuid, uuid_str); + return std::string(uuid_str); +#else + static std::random_device rd; + static std::mt19937 gen(rd()); + static std::uniform_int_distribution<> dis(0, 15); + static std::uniform_int_distribution<> dis2(8, 11); + + std::stringstream ss; + ss << std::hex; + for (int i = 0; i < 8; i++) ss << dis(gen); + ss << "-"; + for (int i = 0; i < 4; i++) ss << dis(gen); + ss << "-4"; + for (int i = 0; i < 3; i++) ss << dis(gen); + ss << "-"; + ss << dis2(gen); + for (int i = 0; i < 3; i++) ss << dis(gen); + ss << "-"; + for (int i = 0; i < 12; i++) ss << dis(gen); + return ss.str(); +#endif +} + +struct ToolFunction { + std::string name; + std::string description; + std::unordered_map parameters; +}; + +struct InferenceOptions { + float temperature = 0.0f; + float top_p = 0.0f; + float min_p = 0.15f; + float repetition_penalty = 1.1f; + float confidence_threshold = -1.0f; + size_t top_k = 0; + size_t max_tokens = 100; + size_t tool_rag_top_k = 2; + size_t cloud_timeout_ms = 15000; + std::vector stop_sequences; + bool force_tools = false; + bool include_stop_sequences = false; + bool use_vad = true; + bool telemetry_enabled = true; + bool auto_handoff = true; + bool handoff_with_images = true; + bool enable_thinking_if_supported = false; +}; + +} // namespace ffi +} // namespace cactus + +std::vector select_relevant_tools( + CactusModelHandle* handle, + const std::string& query, + const std::vector& all_tools, + size_t top_k); + +#include "gemma_tools.h" + +namespace cactus { +namespace ffi { + +inline std::string escape_json_string(const std::string& s) { + std::ostringstream o; + for (char c : s) { + if (c == '"') o << "\\\""; + else if (c == '\n') o << "\\n"; + else if (c == '\r') o << "\\r"; + else if (c == '\t') o << "\\t"; + else if (c == '\\') o << "\\\\"; + else o << c; + } + return o.str(); +} + + +inline std::string trim_string(const std::string& s) { + size_t start = 0; + while (start < s.size() && std::isspace(static_cast(s[start]))) ++start; + size_t end = s.size(); + while (end > start && std::isspace(static_cast(s[end - 1]))) --end; + return s.substr(start, end - start); +} + +inline size_t find_matching_delimiter(const std::string& s, size_t pos, char open, char close) { + int depth = 1; + pos++; + while (pos < s.length() && depth > 0) { + if (s[pos] == open) depth++; + else if (s[pos] == close) depth--; + else if (s[pos] == '"') { + pos++; + while (pos < s.length() && s[pos] != '"') { + if (s[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + return pos; +} + +inline std::string env_or_default(const char* key, const char* fallback) { + const char* v = std::getenv(key); + if (v && v[0] != '\0') return std::string(v); + return std::string(fallback); +} + +inline std::string json_string_field(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return {}; + + size_t i = pos + pattern.size(); + while (i < json.size() && std::isspace(static_cast(json[i]))) i++; + if (i >= json.size() || json[i] != '"') return {}; + ++i; + + std::string out; + out.reserve(128); + while (i < json.size()) { + char c = json[i++]; + if (c == '"') return out; + if (c == '\\' && i < json.size()) { + char e = json[i++]; + switch (e) { + case '"': out.push_back('"'); break; + case '\\': out.push_back('\\'); break; + case '/': out.push_back('/'); break; + case 'b': out.push_back('\b'); break; + case 'f': out.push_back('\f'); break; + case 'n': out.push_back('\n'); break; + case 'r': out.push_back('\r'); break; + case 't': out.push_back('\t'); break; + default: out.push_back(e); break; + } + continue; + } + out.push_back(c); + } + return {}; +} + +inline std::string json_array_field(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return "[]"; + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + if (start >= json.size() || json[start] != '[') return "[]"; + + int depth = 1; + size_t end = start + 1; + while (end < json.size() && depth > 0) { + if (json[end] == '[') depth++; + else if (json[end] == ']') depth--; + end++; + } + return json.substr(start, end - start); +} + +inline std::vector split_json_array(const std::string& array_json) { + std::vector out; + if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; + + size_t i = 1; + while (i + 1 < array_json.size()) { + while (i + 1 < array_json.size() && + (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) i++; + if (i + 1 >= array_json.size() || array_json[i] != '{') break; + + size_t start = i; + int depth = 0; + bool in_str = false; + bool esc = false; + for (; i < array_json.size(); ++i) { + char c = array_json[i]; + if (in_str) { + if (esc) esc = false; + else if (c == '\\') esc = true; + else if (c == '"') in_str = false; + continue; + } + if (c == '"') { in_str = true; continue; } + if (c == '{') depth++; + if (c == '}') { + depth--; + if (depth == 0) { + out.push_back(array_json.substr(start, i - start + 1)); + i++; + break; + } + } + } + } + return out; +} + +inline std::string serialize_tools_json(const std::vector& tools) { + if (tools.empty()) return ""; + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < tools.size(); ++i) { + if (i > 0) oss << ","; + oss << "{\"type\":\"function\",\"function\":{"; + oss << "\"name\":\"" << escape_json_string(tools[i].name) << "\","; + oss << "\"description\":\"" << escape_json_string(tools[i].description) << "\""; + auto it = tools[i].parameters.find("schema"); + if (it != tools[i].parameters.end()) { + oss << ",\"parameters\":" << it->second; + } + oss << "}}"; + } + oss << "]"; + return oss.str(); +} + +namespace json_sorted { + +inline void skip_ws(const std::string& s, size_t& p) { + while (p < s.size() && std::isspace(static_cast(s[p]))) p++; +} + +inline std::string parse_string(const std::string& s, size_t& p) { + std::string r = "\""; + p++; + while (p < s.size()) { + if (s[p] == '\\') { + r += s[p++]; + if (p < s.size()) r += s[p++]; + } else if (s[p] == '"') { + r += '"'; + p++; + return r; + } else { + r += s[p++]; + } + } + return r; +} + +inline std::string parse_value(const std::string& s, size_t& p); + +inline std::string parse_object(const std::string& s, size_t& p) { + p++; + std::map entries; + skip_ws(s, p); + while (p < s.size() && s[p] != '}') { + if (s[p] == ',') { p++; skip_ws(s, p); continue; } + std::string key = parse_string(s, p); + skip_ws(s, p); + if (p < s.size() && s[p] == ':') p++; + skip_ws(s, p); + std::string val = parse_value(s, p); + entries[key] = val; + skip_ws(s, p); + } + if (p < s.size()) p++; + std::string r = "{"; + bool first = true; + for (const auto& kv : entries) { + if (!first) r += ", "; + r += kv.first + ": " + kv.second; + first = false; + } + r += "}"; + return r; +} + +inline std::string parse_array(const std::string& s, size_t& p) { + p++; + std::vector items; + skip_ws(s, p); + while (p < s.size() && s[p] != ']') { + if (s[p] == ',') { p++; skip_ws(s, p); continue; } + items.push_back(parse_value(s, p)); + skip_ws(s, p); + } + if (p < s.size()) p++; + std::string r = "["; + for (size_t i = 0; i < items.size(); i++) { + if (i > 0) r += ", "; + r += items[i]; + } + r += "]"; + return r; +} + +inline std::string parse_value(const std::string& s, size_t& p) { + skip_ws(s, p); + if (p >= s.size()) return ""; + if (s[p] == '"') return parse_string(s, p); + if (s[p] == '{') return parse_object(s, p); + if (s[p] == '[') return parse_array(s, p); + size_t start = p; + while (p < s.size() && s[p] != ',' && s[p] != '}' && s[p] != ']' && !std::isspace(static_cast(s[p]))) p++; + return s.substr(start, p - start); +} + +inline std::string reformat(const std::string& json) { + size_t p = 0; + return parse_value(json, p); +} + +} // namespace json_sorted + +inline std::string serialize_tools_for_template(const std::vector& tools) { + if (tools.empty()) return ""; + std::string result; + for (const auto& tool : tools) { + std::map func_fields; + func_fields["\"description\""] = "\"" + escape_json_string(tool.description) + "\""; + func_fields["\"name\""] = "\"" + escape_json_string(tool.name) + "\""; + auto it = tool.parameters.find("schema"); + if (it != tool.parameters.end()) { + func_fields["\"parameters\""] = json_sorted::reformat(it->second); + } + std::string func_json = "{"; + bool first = true; + for (const auto& kv : func_fields) { + if (!first) func_json += ", "; + func_json += kv.first + ": " + kv.second; + first = false; + } + func_json += "}"; + result += "\n{\"function\": " + func_json + ", \"type\": \"function\"}"; + } + return result; +} + +inline void handle_error_response(const std::string& error_message, char* response_buffer, size_t buffer_size) { + std::ostringstream json; + json << "{"; + json << "\"success\":false,"; + json << "\"error\":\"" << escape_json_string(error_message) << "\","; + json << "\"cloud_handoff\":false,"; + json << "\"response\":null,"; + json << "\"function_calls\":[],"; + json << "\"confidence\":0.0,"; + json << "\"time_to_first_token_ms\":0.0,"; + json << "\"total_time_ms\":0.0,"; + json << "\"prefill_tps\":0.0,"; + json << "\"decode_tps\":0.0,"; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; + json << "\"prefill_tokens\":0,"; + json << "\"decode_tokens\":0,"; + json << "\"total_tokens\":0"; + json << "}"; + std::string error_json = json.str(); + if (response_buffer && error_json.length() < buffer_size) { + std::strcpy(response_buffer, error_json.c_str()); + } +} + +inline std::vector parse_messages_json(const std::string& json, + std::vector& out_image_paths, + std::vector* out_audio_paths = nullptr) { + std::vector messages; + out_image_paths.clear(); + if (out_audio_paths) out_audio_paths->clear(); + + size_t pos = json.find('['); + if (pos == std::string::npos) { + throw std::runtime_error("Invalid JSON: expected array"); + } + + pos = json.find('{', pos); + while (pos != std::string::npos) { + cactus::engine::ChatMessage msg; + + size_t obj_start = pos; + int brace_count = 1; + size_t obj_end = obj_start + 1; + while (obj_end < json.length() && brace_count > 0) { + if (json[obj_end] == '{') brace_count++; + else if (json[obj_end] == '}') brace_count--; + obj_end++; + } + + size_t role_pos = json.find("\"role\"", pos); + if (role_pos == std::string::npos || role_pos >= obj_end) break; + + size_t role_start = json.find('"', role_pos + 6) + 1; + size_t role_end = json.find('"', role_start); + msg.role = json.substr(role_start, role_end - role_start); + + size_t content_pos = json.find("\"content\"", role_end); + if (content_pos != std::string::npos && content_pos < obj_end) { + size_t content_start = json.find('"', content_pos + 9) + 1; + size_t content_end = content_start; + + while (content_end < json.length()) { + content_end = json.find('"', content_end); + if (content_end == std::string::npos) break; + if (json[content_end - 1] != '\\') break; + content_end++; + } + + msg.content = json.substr(content_start, content_end - content_start); + + size_t escape_pos = 0; + while ((escape_pos = msg.content.find("\\n", escape_pos)) != std::string::npos) { + msg.content.replace(escape_pos, 2, "\n"); + escape_pos += 1; + } + escape_pos = 0; + while ((escape_pos = msg.content.find("\\\"", escape_pos)) != std::string::npos) { + msg.content.replace(escape_pos, 2, "\""); + escape_pos += 1; + } + } + + auto parse_path_array = [&](const char* key, std::vector& dest, + std::vector* out_paths) { + size_t key_pos = json.find(key, pos); + if (key_pos == std::string::npos || key_pos >= obj_end) return; + size_t array_start = json.find('[', key_pos); + if (array_start == std::string::npos || array_start >= obj_end) return; + size_t array_end = json.find(']', array_start); + if (array_end == std::string::npos || array_end >= obj_end) return; + size_t cur = array_start; + while (true) { + cur = json.find('"', cur + 1); + if (cur == std::string::npos || cur >= array_end) break; + size_t str_start = cur + 1; + size_t str_end = json.find('"', str_start); + if (str_end == std::string::npos || str_end > array_end) break; + std::string path = std::filesystem::absolute( + std::filesystem::path(json.substr(str_start, str_end - str_start))).string(); + dest.push_back(path); + if (out_paths) out_paths->push_back(path); + cur = str_end; + } + }; + + parse_path_array("\"images\"", msg.images, &out_image_paths); + parse_path_array("\"audio\"", msg.audio, out_audio_paths); + + if (msg.role == "tool") { + size_t name_pos = json.find("\"name\"", obj_start); + if (name_pos != std::string::npos && name_pos < obj_end) { + size_t name_quote = json.find('"', name_pos + 6); + if (name_quote != std::string::npos && name_quote < obj_end) { + size_t name_start = name_quote + 1; + size_t name_end = json.find('"', name_start); + if (name_end != std::string::npos && name_end < obj_end) { + msg.name = json.substr(name_start, name_end - name_start); + } + } + } + } + + size_t tool_calls_pos = json.find("\"tool_calls\"", obj_start); + if (tool_calls_pos != std::string::npos && tool_calls_pos < obj_end) { + size_t tool_calls_arr_start = json.find('[', tool_calls_pos); + if (tool_calls_arr_start != std::string::npos && tool_calls_arr_start < obj_end) { + size_t tool_calls_arr_end = find_matching_delimiter(json, tool_calls_arr_start, '[', ']'); + + size_t search_pos = tool_calls_arr_start; + while (true) { + size_t func_pos = json.find("\"function\"", search_pos); + if (func_pos == std::string::npos || func_pos >= tool_calls_arr_end) break; + + size_t func_obj_start = json.find('{', func_pos + 10); + if (func_obj_start == std::string::npos || func_obj_start >= tool_calls_arr_end) break; + + size_t func_obj_end = find_matching_delimiter(json, func_obj_start, '{', '}'); + + cactus::engine::ToolCallInfo tool_call; + + size_t fn_name_pos = json.find("\"name\"", func_obj_start); + if (fn_name_pos != std::string::npos && fn_name_pos < func_obj_end) { + size_t fn_name_quote = json.find('"', fn_name_pos + 6); + if (fn_name_quote != std::string::npos && fn_name_quote < func_obj_end) { + size_t fn_name_start = fn_name_quote + 1; + size_t fn_name_end = json.find('"', fn_name_start); + if (fn_name_end != std::string::npos && fn_name_end < func_obj_end) { + tool_call.name = json.substr(fn_name_start, fn_name_end - fn_name_start); + } + } + } + + size_t args_pos = json.find("\"arguments\"", func_obj_start); + if (args_pos != std::string::npos && args_pos < func_obj_end) { + size_t colon_pos = json.find(':', args_pos + 11); + if (colon_pos != std::string::npos && colon_pos < func_obj_end) { + size_t args_start = colon_pos + 1; + while (args_start < json.length() && std::isspace(static_cast(json[args_start]))) args_start++; + + if (args_start < func_obj_end && json[args_start] == '{') { + size_t args_end = find_matching_delimiter(json, args_start, '{', '}'); + tool_call.arguments = json.substr(args_start, args_end - args_start); + } else if (args_start < func_obj_end && json[args_start] == '"') { + size_t str_start = args_start + 1; + size_t str_end = str_start; + while (str_end < json.length() && json[str_end] != '"') { + if (json[str_end] == '\\') str_end++; + str_end++; + } + tool_call.arguments = json.substr(str_start, str_end - str_start); + } + } + } + + if (!tool_call.name.empty()) { + msg.tool_calls.push_back(tool_call); + } + search_pos = func_obj_end; + } + } + } + + messages.push_back(msg); + + pos = json.find('{', obj_end); + } + + return messages; +} + +inline std::vector parse_tools_json(const std::string& json) { + std::vector tools; + + if (json.empty()) return tools; + + size_t pos = json.find('['); + if (pos == std::string::npos) return tools; + + pos = json.find("\"function\"", pos); + while (pos != std::string::npos) { + ToolFunction tool; + size_t next_search = pos + 1; + + size_t name_pos = json.find("\"name\"", pos); + if (name_pos != std::string::npos) { + size_t name_start = json.find('"', name_pos + 6) + 1; + size_t name_end = json.find('"', name_start); + tool.name = json.substr(name_start, name_end - name_start); + } + + size_t desc_pos = json.find("\"description\"", pos); + if (desc_pos != std::string::npos) { + size_t desc_start = json.find('"', desc_pos + 13) + 1; + size_t desc_end = json.find('"', desc_start); + tool.description = json.substr(desc_start, desc_end - desc_start); + } + + size_t params_pos = json.find("\"parameters\"", pos); + if (params_pos != std::string::npos) { + size_t params_start = json.find('{', params_pos); + if (params_start != std::string::npos) { + int brace_count = 1; + size_t params_end = params_start + 1; + while (params_end < json.length() && brace_count > 0) { + if (json[params_end] == '{') brace_count++; + else if (json[params_end] == '}') brace_count--; + params_end++; + } + tool.parameters["schema"] = json.substr(params_start, params_end - params_start); + next_search = params_end; + } + } + + if (!tool.name.empty()) { + tools.push_back(tool); + } + + pos = json.find("\"function\"", next_search); + } + + return tools; +} + +inline bool try_parse_json_float(const std::string& json, const std::string& key, float& out_value) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return false; + + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + + size_t end = start; + while (end < json.size() && std::string(",}] \t\n\r").find(json[end]) == std::string::npos) ++end; + + try { + out_value = std::stof(json.substr(start, end - start)); + return true; + } catch (...) { + return false; + } +} + +inline std::vector parse_json_string_array_field(const std::string& json, const std::string& key) { + std::vector out; + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return out; + + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + if (start >= json.size() || json[start] != '[') return out; + + int depth = 1; + bool in_string = false; + bool escaped = false; + size_t end = start + 1; + + while (end < json.size() && depth > 0) { + char c = json[end]; + if (in_string) { + if (escaped) escaped = false; + else if (c == '\\') escaped = true; + else if (c == '"') in_string = false; + } else { + if (c == '"') in_string = true; + else if (c == '[') depth++; + else if (c == ']') depth--; + } + ++end; + } + + if (depth != 0) return out; + const std::string array_json = json.substr(start, end - start); + if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; + + size_t i = 1; + while (i + 1 < array_json.size()) { + while (i + 1 < array_json.size() && + (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) { + ++i; + } + if (i + 1 >= array_json.size() || array_json[i] == ']') break; + if (array_json[i] != '"') break; + + ++i; + std::string value; + bool escaped = false; + while (i < array_json.size()) { + char c = array_json[i++]; + if (escaped) { + switch (c) { + case '"': value.push_back('"'); break; + case '\\': value.push_back('\\'); break; + case '/': value.push_back('/'); break; + case 'b': value.push_back('\b'); break; + case 'f': value.push_back('\f'); break; + case 'n': value.push_back('\n'); break; + case 'r': value.push_back('\r'); break; + case 't': value.push_back('\t'); break; + default: value.push_back(c); break; + } + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == '"') { + out.push_back(value); + break; + } + value.push_back(c); + } + } + + return out; +} + +inline void parse_custom_vocabulary_options(const std::string& json, + std::vector& custom_vocabulary, + float& vocabulary_boost) { + custom_vocabulary.clear(); + vocabulary_boost = 5.0f; + if (json.empty()) return; + + float parsed_boost = vocabulary_boost; + if (try_parse_json_float(json, "vocabulary_boost", parsed_boost)) { + vocabulary_boost = std::clamp(parsed_boost, 0.0f, 20.0f); + } + + custom_vocabulary = parse_json_string_array_field(json, "custom_vocabulary"); +} + +inline std::unordered_map build_token_bias_map(const std::vector>& tokenized_entries, + float vocabulary_boost) { + std::unordered_map vocab_bias; + const float clamped_boost = std::clamp(vocabulary_boost, 0.0f, 20.0f); + if (clamped_boost == 0.0f) return vocab_bias; + + for (const auto& token_ids : tokenized_entries) { + for (uint32_t token_id : token_ids) { + float& entry = vocab_bias[token_id]; + if (entry < clamped_boost) { + entry = clamped_boost; + } + } + } + + return vocab_bias; +} + +inline std::unordered_map build_custom_vocabulary_bias(cactus::engine::Tokenizer* tokenizer, + const std::vector& custom_vocabulary, + float vocabulary_boost) { + if (!tokenizer || custom_vocabulary.empty()) return {}; + std::vector> tokenized_entries; + tokenized_entries.reserve(custom_vocabulary.size()); + + for (const auto& word : custom_vocabulary) { + if (word.empty()) continue; + tokenized_entries.push_back(tokenizer->encode(word)); + } + + return build_token_bias_map(tokenized_entries, vocabulary_boost); +} + +inline void apply_custom_vocabulary_options(cactus::engine::Model* model, const std::string& json) { + if (!model) return; + + std::vector custom_vocabulary; + float vocabulary_boost = 5.0f; + parse_custom_vocabulary_options(json, custom_vocabulary, vocabulary_boost); + model->set_vocab_bias(build_custom_vocabulary_bias(model->get_tokenizer(), custom_vocabulary, vocabulary_boost)); +} + +inline size_t levenshtein_ci(const std::string& a, const std::string& b) { + const size_t m = a.size(), n = b.size(); + std::vector prev(n + 1), curr(n + 1); + for (size_t j = 0; j <= n; ++j) prev[j] = j; + for (size_t i = 1; i <= m; ++i) { + curr[0] = i; + for (size_t j = 1; j <= n; ++j) { + const bool match = std::tolower(static_cast(a[i - 1])) == + std::tolower(static_cast(b[j - 1])); + curr[j] = std::min({prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + (match ? 0 : 1)}); + } + std::swap(prev, curr); + } + return prev[n]; +} + +inline std::string collapse_spaces(const std::string& s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + if (c != ' ') out += c; + } + return out; +} + +inline void apply_vocabulary_spelling_correction( + std::string& text, + const std::vector& custom_vocabulary) +{ + if (custom_vocabulary.empty() || text.empty()) return; + + struct VocabEntry { + const std::string* original; + std::string collapsed; + }; + std::vector vocab_entries; + vocab_entries.reserve(custom_vocabulary.size()); + for (const auto& v : custom_vocabulary) { + vocab_entries.push_back({&v, collapse_spaces(v)}); + } + + struct Token { std::string text; bool is_word; }; + std::vector tokens; + size_t pos = 0; + while (pos < text.size()) { + if (std::isalnum(static_cast(text[pos])) || + text[pos] == '\'' || text[pos] == '-') { + size_t start = pos; + while (pos < text.size() && (std::isalnum(static_cast(text[pos])) || + text[pos] == '\'' || text[pos] == '-')) { + ++pos; + } + tokens.push_back({text.substr(start, pos - start), true}); + } else { + size_t start = pos; + while (pos < text.size() && !std::isalnum(static_cast(text[pos])) && + text[pos] != '\'' && text[pos] != '-') { + ++pos; + } + tokens.push_back({text.substr(start, pos - start), false}); + } + } + + std::vector word_indices; + for (size_t i = 0; i < tokens.size(); ++i) { + if (tokens[i].is_word) word_indices.push_back(i); + } + + std::vector consumed(tokens.size(), false); + + auto strip_suffix = [](const std::string& word) -> std::pair { + if (word.size() >= 3 && word.substr(word.size() - 2) == "'s") { + return {word.substr(0, word.size() - 2), "'s"}; + } + if (word.size() >= 3 && word.substr(word.size() - 2) == "'t") { + return {word.substr(0, word.size() - 2), "'t"}; + } + if (word.size() >= 4 && word.back() == 's' && + word[word.size() - 2] != 's' && // avoid stripping from "boss", "class" + std::isalpha(static_cast(word[word.size() - 2]))) { + return {word.substr(0, word.size() - 1), "s"}; + } + return {word, ""}; + }; + + size_t wi = 0; + while (wi < word_indices.size()) { + size_t best_dist = std::numeric_limits::max(); + const std::string* best_match = nullptr; + size_t best_window = 0; + size_t best_first_token = 0; + size_t best_last_token = 0; + std::string best_suffix; + + for (size_t window = std::min(3, word_indices.size() - wi); window >= 1; --window) { + std::string window_collapsed; + const size_t first_tok = word_indices[wi]; + const size_t last_tok = word_indices[wi + window - 1]; + for (size_t w = 0; w < window; ++w) { + window_collapsed += tokens[word_indices[wi + w]].text; + } + + if (window == 1 && window_collapsed.size() < 3) break; + + auto [stem, suffix] = strip_suffix(window_collapsed); + const std::string* candidates[] = {&window_collapsed, &stem}; + const std::string suffixes[] = {"", suffix}; + const size_t num_candidates = suffix.empty() ? 1 : 2; + + for (size_t ci = 0; ci < num_candidates; ++ci) { + const std::string& candidate = *candidates[ci]; + if (candidate.empty()) continue; + + for (const auto& entry : vocab_entries) { + const size_t wlen = candidate.size(); + const size_t vlen = entry.collapsed.size(); + + const size_t len_diff = wlen > vlen ? wlen - vlen : vlen - wlen; + const size_t max_dist = std::max(1, std::min(wlen, vlen) / 3); + if (len_diff > max_dist) continue; + + const size_t dist = levenshtein_ci(candidate, entry.collapsed); + + // For single-edit corrections, require first char match to prevent + // false positives like "vortex" → "Cortex". + if (dist == 1 && window == 1) { + const bool first_char_match = + std::tolower(static_cast(candidate[0])) == + std::tolower(static_cast(entry.collapsed[0])); + if (!first_char_match) continue; + } + + if (dist <= max_dist && dist < best_dist) { + best_dist = dist; + best_match = entry.original; + best_window = window; + best_first_token = first_tok; + best_last_token = last_tok; + best_suffix = suffixes[ci]; + } + } + } + + if (best_dist == 0) break; + } + + // Allow dist==0 for multi-word merges where word boundaries changed. + const bool should_replace = best_match && + best_dist != std::numeric_limits::max() && + (best_dist > 0 || best_window > 1); + + if (should_replace) { + tokens[best_first_token].text = *best_match + best_suffix; + for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { + consumed[t] = true; + } + for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { + if (t > 0) consumed[t - 1] = consumed[t - 1] || !tokens[t - 1].is_word; + } + wi += best_window; + } else { + ++wi; + } + } + + std::string result; + result.reserve(text.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + if (!consumed[i]) { + result += tokens[i].text; + } + } + + text = std::move(result); +} + +inline InferenceOptions parse_inference_options_json(const std::string& json) { + InferenceOptions options; + + if (json.empty()) return options; + + size_t pos = json.find("\"temperature\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.temperature = std::stof(json.substr(pos)); + } + + pos = json.find("\"top_p\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.top_p = std::stof(json.substr(pos)); + } + + float parsed_min_p = options.min_p; + if (try_parse_json_float(json, "min_p", parsed_min_p)) { + options.min_p = std::clamp(parsed_min_p, 0.0f, 1.0f); + } + + float parsed_rep_penalty = options.repetition_penalty; + if (try_parse_json_float(json, "repetition_penalty", parsed_rep_penalty)) { + if (std::isfinite(parsed_rep_penalty) && parsed_rep_penalty > 0.0f) { + options.repetition_penalty = parsed_rep_penalty; + } + } + + pos = json.find("\"top_k\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.top_k = std::stoul(json.substr(pos)); + } + + pos = json.find("\"max_tokens\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.max_tokens = std::stoul(json.substr(pos)); + } + + pos = json.find("\"force_tools\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.force_tools = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"tool_rag_top_k\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.tool_rag_top_k = std::stoul(json.substr(pos)); + } + + pos = json.find("\"confidence_threshold\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.confidence_threshold = std::stof(json.substr(pos)); + } + + pos = json.find("\"include_stop_sequences\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.include_stop_sequences = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"use_vad\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.use_vad = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"telemetry_enabled\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.telemetry_enabled = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"auto_handoff\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.auto_handoff = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"cloud_timeout_ms\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.cloud_timeout_ms = std::stoul(json.substr(pos)); + } + + pos = json.find("\"handoff_with_images\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.handoff_with_images = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"enable_thinking_if_supported\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.enable_thinking_if_supported = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"stop_sequences\""); + if (pos != std::string::npos) { + pos = json.find('[', pos); + if (pos != std::string::npos) { + size_t end_pos = json.find(']', pos); + size_t seq_pos = json.find('"', pos); + + while (seq_pos != std::string::npos && seq_pos < end_pos) { + size_t seq_start = seq_pos + 1; + size_t seq_end = json.find('"', seq_start); + if (seq_end != std::string::npos) { + options.stop_sequences.push_back(json.substr(seq_start, seq_end - seq_start)); + } + seq_pos = json.find('"', seq_end + 1); + } + } + } + + return options; +} + +static inline std::string trim_lfm2_slice(const std::string& value, size_t begin, size_t end) { + return trim_string(value.substr(begin, end - begin)); +} + +static inline void append_lfm2_call(const std::string& entry, + std::vector& function_calls) { + if (entry.empty()) return; + + std::string trimmed_entry = trim_lfm2_slice(entry, 0, entry.size()); + if (trimmed_entry.empty()) return; + + size_t paren_pos = trimmed_entry.find('('); + if (paren_pos == std::string::npos) return; + + std::string func_name = trim_lfm2_slice(trimmed_entry, 0, paren_pos); + std::string args_str = trim_lfm2_slice(trimmed_entry, paren_pos + 1, trimmed_entry.size()); + + while (!args_str.empty() && (args_str.back() == ')' || args_str.back() == ']')) { + args_str.pop_back(); + } + args_str = trim_lfm2_slice(args_str, 0, args_str.size()); + + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":{"; + + size_t arg_pos = 0; + bool first_arg = true; + while (arg_pos < args_str.length()) { + while (arg_pos < args_str.length() && std::isspace(static_cast(args_str[arg_pos]))) { + arg_pos++; + } + + size_t eq_pos = args_str.find('=', arg_pos); + if (eq_pos == std::string::npos) break; + + std::string arg_name = args_str.substr(arg_pos, eq_pos - arg_pos); + + size_t val_start = eq_pos + 1; + size_t val_end = val_start; + bool quoted = false; + + if (val_start < args_str.length() && args_str[val_start] == '"') { + quoted = true; + val_start++; + val_end = args_str.find('"', val_start); + if (val_end == std::string::npos) break; + } else { + int depth = 0; + for (val_end = val_start; val_end < args_str.length(); val_end++) { + char c = args_str[val_end]; + if (c == '[' || c == '{') depth++; + else if (c == ']' || c == '}') depth--; + else if (c == ',' && depth == 0) break; + } + } + + std::string arg_value = args_str.substr(val_start, val_end - val_start); + + if (!quoted) { + if (arg_value == "True") arg_value = "true"; + else if (arg_value == "False") arg_value = "false"; + else if (arg_value == "None") arg_value = "null"; + } + + if (!first_arg) json_call += ","; + json_call += "\"" + arg_name + "\":"; + json_call += quoted ? ("\"" + arg_value + "\"") : arg_value; + first_arg = false; + + arg_pos = args_str.find(',', val_end); + if (arg_pos != std::string::npos) { + arg_pos++; + } else { + break; + } + } + + json_call += "}}"; + function_calls.push_back(json_call); +} + +inline void parse_function_calls_from_response(const std::string& response_text, + std::string& regular_response, + std::vector& function_calls) { + regular_response = response_text; + function_calls.clear(); + + gemma::parse_function_calls(regular_response, function_calls); + + const std::string QWEN_TOOL_START = ""; + const std::string QWEN_TOOL_END = ""; + size_t qwen_start_pos = 0; + + while ((qwen_start_pos = regular_response.find(QWEN_TOOL_START, qwen_start_pos)) != std::string::npos) { + size_t content_start = qwen_start_pos + QWEN_TOOL_START.length(); + size_t qwen_end_pos = regular_response.find(QWEN_TOOL_END, content_start); + + size_t erase_end; + std::string json_content; + + if (qwen_end_pos != std::string::npos) { + json_content = regular_response.substr(content_start, qwen_end_pos - content_start); + erase_end = qwen_end_pos + QWEN_TOOL_END.length(); + } else { + json_content = regular_response.substr(content_start); + erase_end = regular_response.length(); + } + + size_t first = json_content.find_first_not_of(" \t\n\r"); + size_t last = json_content.find_last_not_of(" \t\n\r"); + if (first != std::string::npos && last != std::string::npos) { + json_content = json_content.substr(first, last - first + 1); + } + + if (json_content.size() > 2 && json_content.find("\"name\"") != std::string::npos) { + // Unwrap array wrapper if present: [{"name":...}] -> {"name":...} + if (json_content[0] == '[') { + size_t obj_start = json_content.find('{'); + size_t obj_end = json_content.rfind('}'); + if (obj_start != std::string::npos && obj_end != std::string::npos && obj_end > obj_start) { + json_content = json_content.substr(obj_start, obj_end - obj_start + 1); + } + } + if (json_content[0] == '{') { + size_t depth = 0; + bool in_string = false; + bool escaped = false; + size_t end_pos = 0; + for (size_t c = 0; c < json_content.size(); c++) { + char ch = json_content[c]; + if (escaped) { escaped = false; continue; } + if (ch == '\\' && in_string) { escaped = true; continue; } + if (ch == '"') { in_string = !in_string; continue; } + if (!in_string) { + if (ch == '{') depth++; + else if (ch == '}' && --depth == 0) { end_pos = c + 1; break; } + } + } + if (end_pos > 0) { + function_calls.push_back(json_content.substr(0, end_pos)); + } + } + } + + regular_response.erase(qwen_start_pos, erase_end - qwen_start_pos); + } + + const std::string TOOL_CALL_START = "<|tool_call_start|>"; + const std::string TOOL_CALL_END = "<|tool_call_end|>"; + size_t lfm2_start_pos = 0; + + while ((lfm2_start_pos = regular_response.find(TOOL_CALL_START, lfm2_start_pos)) != std::string::npos) { + size_t content_start = lfm2_start_pos + TOOL_CALL_START.length(); + size_t tool_end_pos = regular_response.find(TOOL_CALL_END, content_start); + + if (tool_end_pos != std::string::npos) { + std::string tool_content = regular_response.substr(content_start, tool_end_pos - content_start); + std::string content = tool_content; + size_t trim_start = 0; + while (trim_start < content.size() && std::isspace(static_cast(content[trim_start]))) { + trim_start++; + } + + if (trim_start < content.size()) { + size_t trim_end = content.size() - 1; + while (trim_end > trim_start && std::isspace(static_cast(content[trim_end]))) { + trim_end--; + } + content = content.substr(trim_start, trim_end - trim_start + 1); + } else { + content.clear(); + } + + if (!content.empty() && content.front() == '[' && content.back() == ']') { + std::string inner = content.substr(1, content.size() - 2); + + size_t inner_first = inner.find_first_not_of(" \t\n\r"); + if (inner_first != std::string::npos && inner[inner_first] == '{') { + size_t pos = inner_first; + while (pos < inner.size()) { + if (inner[pos] == '{') { + int brace_depth = 1; + size_t obj_start = pos; + pos++; + while (pos < inner.size() && brace_depth > 0) { + if (inner[pos] == '{') brace_depth++; + else if (inner[pos] == '}') brace_depth--; + pos++; + } + if (brace_depth == 0) { + std::string json_obj = inner.substr(obj_start, pos - obj_start); + if (json_obj.find("\"name\"") != std::string::npos) { + function_calls.push_back(json_obj); + } + } + } else { + pos++; + } + } + } else { + size_t start = 0; + int paren_depth = 0; + + for (size_t i = 0; i < inner.size(); ++i) { + char c = inner[i]; + if (c == '(') { + paren_depth++; + } else if (c == ')' && paren_depth > 0) { + paren_depth--; + } else if (c == ',' && paren_depth == 0) { + append_lfm2_call(inner.substr(start, i - start), function_calls); + start = i + 1; + } + } + + if (start < inner.size()) { + append_lfm2_call(inner.substr(start), function_calls); + } + } + } else if (!content.empty()) { + append_lfm2_call(content, function_calls); + } + + regular_response.erase(lfm2_start_pos, tool_end_pos + TOOL_CALL_END.length() - lfm2_start_pos); + } else { + break; + } + } + + const char* FUNCTION_CALL_MARKER = "\"function_call\""; + size_t search_pos = 0; + const size_t text_len = regular_response.length(); + + while (search_pos < text_len) { + size_t marker_pos = regular_response.find(FUNCTION_CALL_MARKER, search_pos); + if (marker_pos == std::string::npos) break; + + size_t json_start = regular_response.find('{', marker_pos); + if (json_start == std::string::npos) break; + + int brace_count = 1; + size_t json_end = json_start + 1; + while (json_end < text_len && brace_count > 0) { + char c = regular_response[json_end]; + brace_count += (c == '{') - (c == '}'); + json_end++; + } + + if (brace_count == 0) { + function_calls.push_back(regular_response.substr(json_start, json_end - json_start)); + regular_response = regular_response.substr(0, marker_pos); + size_t last_bracket = regular_response.rfind('{'); + if(last_bracket != std::string::npos) { + regular_response = regular_response.substr(0, last_bracket); + } + } + search_pos = json_end; + } +} + +inline std::vector> find_channel_token_ranges( + const std::vector& tokens, size_t offset, + uint32_t channel_open_id, uint32_t channel_close_id) { + std::vector> ranges; + size_t pos = 0; + while (pos < tokens.size()) { + if (tokens[pos] != channel_open_id) { + pos++; + continue; + } + + size_t block_start = pos; + pos++; + while (pos < tokens.size() && tokens[pos] != channel_close_id) { + pos++; + } + if (pos < tokens.size()) { + pos++; + } + ranges.push_back({offset + block_start, pos - block_start}); + } + return ranges; +} + +inline void strip_tag_blocks(std::string& text, std::string& extracted, + const std::string& open_tag, const std::string& close_tag) { + std::string result; + size_t pos = 0; + + size_t first_close = text.find(close_tag); + size_t first_open = text.find(open_tag); + if (first_close != std::string::npos && + (first_open == std::string::npos || first_close < first_open)) { + extracted += text.substr(0, first_close); + pos = first_close + close_tag.size(); + } + + while (pos < text.size()) { + size_t open_pos = text.find(open_tag, pos); + if (open_pos == std::string::npos) { + result += text.substr(pos); + break; + } + result += text.substr(pos, open_pos - pos); + size_t content_start = open_pos + open_tag.size(); + size_t close_pos = text.find(close_tag, content_start); + if (close_pos == std::string::npos) { + if (!extracted.empty()) extracted += "\n"; + extracted += text.substr(content_start); + break; + } + if (!extracted.empty()) extracted += "\n"; + extracted += text.substr(content_start, close_pos - content_start); + pos = close_pos + close_tag.size(); + } + text = result; +} + +inline void strip_thinking_block(const std::string& input, std::string& thinking, std::string& content) { + thinking.clear(); + content = input; + + auto trim = [](std::string& s) { + size_t first = s.find_first_not_of(" \t\n\r"); + size_t last = s.find_last_not_of(" \t\n\r"); + if (first != std::string::npos && last != std::string::npos) + s = s.substr(first, last - first + 1); + else + s.clear(); + }; + + if (content.find("<|channel>") != std::string::npos || content.find("") != std::string::npos) { + strip_tag_blocks(content, thinking, "<|channel>", ""); + } else if (content.find("") != std::string::npos || content.find("") != std::string::npos) { + strip_tag_blocks(content, thinking, "", ""); + } else { + return; + } + + trim(thinking); + trim(content); +} + +struct TranscriptSegment { + float start; + float end; + std::string text; +}; + +inline std::string construct_response_json(const std::string& regular_response, + const std::vector& function_calls, + double time_to_first_token, + double total_time_ms, + double prefill_tps, + double decode_tps, + size_t prompt_tokens, + size_t completion_tokens, + float confidence = 0.0f, + bool cloud_handoff = false, + const std::string& thinking = "", + const std::vector& segments = {}) { + std::ostringstream json; + json << "{"; + json << "\"success\":true,"; + json << "\"error\":null,"; + json << "\"cloud_handoff\":" << (cloud_handoff ? "true" : "false") << ","; + json << "\"response\":\"" << escape_json_string(regular_response) << "\","; + if (!thinking.empty()) { + json << "\"thinking\":\"" << escape_json_string(thinking) << "\","; + } + json << "\"function_calls\":["; + for (size_t i = 0; i < function_calls.size(); ++i) { + if (i > 0) json << ","; + json << function_calls[i]; + } + json << "],"; + json << "\"segments\":["; + for (size_t i = 0; i < segments.size(); ++i) { + if (i > 0) json << ","; + json << "{\"start\":" << std::fixed << std::setprecision(3) << segments[i].start + << ",\"end\":" << std::fixed << std::setprecision(3) << segments[i].end + << ",\"text\":\"" << escape_json_string(segments[i].text) << "\"}"; + } + json << "],"; + json << "\"confidence\":" << std::fixed << std::setprecision(4) << confidence << ","; + json << "\"time_to_first_token_ms\":" << std::fixed << std::setprecision(2) << time_to_first_token << ","; + json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; + json << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << ","; + json << "\"decode_tps\":" << std::fixed << std::setprecision(2) << decode_tps << ","; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; + json << "\"prefill_tokens\":" << prompt_tokens << ","; + json << "\"decode_tokens\":" << completion_tokens << ","; + json << "\"total_tokens\":" << (prompt_tokens + completion_tokens); + json << "}"; + return json.str(); +} + +inline std::string serialize_function_calls(const std::vector& calls) { + if (calls.empty()) return "[]"; + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < calls.size(); ++i) { + if (i > 0) oss << ","; + oss << calls[i]; + } + oss << "]"; + return oss.str(); +} + +inline int validate_audio_params( + const char* component, + void* model, + char* response_buffer, size_t buffer_size, + const char* audio_file_path, + const uint8_t* pcm_buffer, size_t pcm_buffer_size) { + if (!model) { + std::string err = last_error_message.empty() ? "Model not initialized." : last_error_message; + CACTUS_LOG_ERROR(component, err); + handle_error_response(err, response_buffer, buffer_size); + return -1; + } + if (!response_buffer || buffer_size == 0) { + CACTUS_LOG_ERROR(component, "Invalid parameters: response_buffer or buffer_size"); + handle_error_response("Invalid parameters", response_buffer, buffer_size); + return -1; + } + if (!audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { + CACTUS_LOG_ERROR(component, "No audio input provided"); + handle_error_response("Either audio_file_path or pcm_buffer must be provided", response_buffer, buffer_size); + return -1; + } + if (audio_file_path && pcm_buffer && pcm_buffer_size > 0) { + CACTUS_LOG_ERROR(component, "Both audio_file_path and pcm_buffer provided"); + handle_error_response("Cannot provide both audio_file_path and pcm_buffer", response_buffer, buffer_size); + return -1; + } + if (pcm_buffer && pcm_buffer_size > 0 && (pcm_buffer_size < 2 || pcm_buffer_size % 2 != 0)) { + CACTUS_LOG_ERROR(component, "Invalid pcm_buffer_size"); + handle_error_response("pcm_buffer_size must be even and at least 2 bytes", response_buffer, buffer_size); + return -1; + } + return 0; +} + +inline std::vector pcm_to_float(const uint8_t* pcm_buffer, size_t pcm_buffer_size) { + const int16_t* samples = reinterpret_cast(pcm_buffer); + size_t n = pcm_buffer_size / 2; + std::vector out(n); + for (size_t i = 0; i < n; ++i) + out[i] = static_cast(samples[i]) / 32768.0f; + return out; +} + +} // namespace ffi +} // namespace cactus + +#ifdef __cplusplus +extern "C" { +#endif + +const char* cactus_get_last_error(); + +#ifdef __cplusplus +} +#endif + +#endif // CACTUS_UTILS_H diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/engine.h b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/engine.h new file mode 100644 index 0000000..afebfeb --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/engine.h @@ -0,0 +1,1104 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../graph/graph.h" + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc99-extensions" +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +extern "C" { + #include "../../libs/stb/stb_image.h" + #include "../../libs/stb/stb_image_resize2.h" +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + +class CactusGraph; + +namespace cactus { +namespace npu { + class NPUPrefill; +} +namespace engine { + +class Siglip2Preprocessor; + +struct Config { + uint32_t vocab_size = 151936; + uint32_t bos_token_id = 151643; + uint32_t eos_token_id = 151645; + uint32_t num_layers = 28; + uint32_t hidden_dim = 1024; + uint32_t ffn_intermediate_dim = 3072; + uint32_t attention_heads = 16; + uint32_t attention_kv_heads = 8; + uint32_t attention_head_dim = 128; + float layer_norm_eps = 1e-6f; + float rope_theta = 1000000.0f; + uint32_t num_experts = 0; + uint32_t num_shared_experts = 0; + uint32_t num_top_experts = 0; + uint32_t moe_every_n_layers = 0; + uint32_t moe_intermediate_dim = 0; + uint32_t num_dense_layers = 0; + uint32_t num_experts_per_tok = 0; + bool norm_topk_prob = false; + bool use_expert_bias = false; + float routed_scaling_factor = 1.0f; + bool tie_word_embeddings = true; + + uint32_t vision_hidden_dim = 0; + uint32_t vision_num_layers = 0; + uint32_t vision_attention_heads = 0; + uint32_t vision_image_size = 0; + uint32_t vision_patch_size = 0; + uint32_t vision_num_channels = 3; + uint32_t vision_embed_dim = 0; + uint32_t visual_tokens_per_img = 0; + bool use_pixel_shuffle = false; + uint32_t pixel_shuffle_factor = 1; + bool use_image_tokens = false; + uint32_t image_token_id = 0; + bool use_layout_tags = false; + uint32_t image_seq_len = 64; + + uint32_t global_image_size = 2048; + uint32_t max_tile_size = 512; + float rescale_factor = 0.00392156862745098f; + float image_mean = 0.5f; + float image_std = 0.5f; + + uint32_t downsample_factor = 2; + uint32_t min_tiles = 2; + uint32_t max_tiles = 10; + bool use_thumbnail = true; + uint32_t min_image_tokens = 64; + uint32_t max_image_tokens = 256; + uint32_t max_num_patches = 1024; + uint32_t tile_size = 512; + float max_pixels_tolerance = 2.0f; + bool do_image_splitting = true; + bool encoder_act_gelu = false; + bool decoder_act_gelu = false; + uint32_t num_encoder_layers = 0; + uint32_t num_decoder_layers = 0; + float partial_rotary_factor = 0.0f; + uint32_t pad_token_id = 0; + uint32_t conv_kernel_size = 0; + uint32_t subsampling_conv_kernel_size = 0; + uint32_t subsampling_conv_stride = 0; + uint32_t subsampling_conv_channels = 0; + uint32_t subsampling_factor = 0; + uint32_t num_mel_bins = 80; + std::string encoder_hidden_act = "silu"; + uint32_t linear_num_key_heads = 0; + uint32_t linear_key_head_dim = 0; + uint32_t linear_num_value_heads = 0; + uint32_t linear_value_head_dim = 0; + uint32_t linear_q_proj_dim = 0; + uint32_t linear_k_proj_dim = 0; + uint32_t linear_v_proj_dim = 0; + + uint32_t kv_lora_rank = 0; + uint32_t q_lora_rank = 0; + uint32_t qk_head_dim = 0; + uint32_t qk_nope_head_dim = 0; + uint32_t qk_rope_head_dim = 0; + uint32_t v_head_dim = 0; + uint32_t rope_interleave = 0; + bool attention_bias = false; + float rope_scaling_factor = 1.0f; + float rope_mscale_all_dim = 0.0f; + + enum class ModelType {QWEN = 0, GEMMA = 1, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7, MOONSHINE = 8, SILERO_VAD = 9, PARAKEET = 10, QWEN3P5 = 11, PARAKEET_TDT = 12, GEMMA3N = 13, YOUTU = 14, GEMMA4 = 15, PYANNOTE = 16, WESPEAKER = 17, NEEDLE = 18}; + uint32_t predictor_hidden_dim = 0; + uint32_t predictor_num_layers = 0; + uint32_t tdt_joint_dim = 0; + uint32_t tdt_num_durations = 0; + uint32_t tdt_blank_id = 0; + std::vector tdt_durations; + + ModelType model_type = ModelType::QWEN; + + enum class ModelVariant {DEFAULT = 0, VLM = 1, EXTRACT = 2, RAG = 3}; + ModelVariant model_variant = ModelVariant::DEFAULT; + + enum class Activation {GELU = 0, SILU = 1}; + Activation activation = Activation::SILU; + + enum class Backend {CPU = 0, NPU = 1}; + Backend default_backend = Backend::CPU; + + enum class Precision {INT8 = 0, FP16 = 1, FP32 = 2}; + Precision precision = Precision::FP32; + + float default_temperature = 0.6f; + float default_top_p = 0.95f; + size_t default_top_k = 20; + float default_max_tps = -1.0f; + float default_cloud_handoff_threshold = 0.0f; + size_t default_rolling_entropy_window = 10; + + std::vector layer_types; + size_t conv_L_cache = 0; + + uint32_t altup_num_inputs = 4; + uint32_t laurel_rank = 64; + uint32_t hidden_size_per_layer_input = 256; + uint32_t num_kv_shared_layers = 0; + uint32_t sliding_window = 512; + float rope_local_base_freq = 10000.0f; + float final_logit_softcapping = 0.0f; + float global_partial_rotary_factor = 1.0f; + uint32_t expert_intermediate_size = 0; + uint32_t global_head_dim = 0; + uint32_t num_global_kv_heads = 0; + bool attention_k_eq_v = false; + bool enable_moe_block = false; + std::vector activation_sparsity_ppf; + + uint32_t vision_head_dim = 64; + uint32_t vision_kv_heads = 12; + uint32_t vision_intermediate_size = 3072; + uint32_t vision_position_embedding_size = 10240; + uint32_t vision_pooling_kernel_size = 3; + uint32_t vision_default_output_length = 280; + float vision_rope_theta = 100.0f; + + uint32_t audio_hidden_dim = 0; + uint32_t audio_num_layers = 0; + uint32_t audio_num_heads = 0; + uint32_t audio_head_dim = 0; + uint32_t audio_input_feat_size = 128; + uint32_t audio_conf_conv_kernel_size = 5; + uint32_t audio_chunk_size = 12; + uint32_t audio_context_left = 13; + uint32_t audio_context_right = 0; + float audio_logit_cap = 50.0f; + float audio_residual_weight = 0.5f; + uint32_t audio_output_proj_dims = 0; + uint32_t audio_vocab_size = 128; + uint32_t audio_vocab_offset = 0; + uint32_t audio_soft_tokens = 188; + uint32_t audio_sscp_conv0_channels = 128; + uint32_t audio_sscp_conv1_channels = 32; + float audio_sscp_conv_eps = 1e-3f; + float audio_rms_norm_eps = 1e-6f; + uint32_t audio_fft_length = 1024; + uint32_t audio_token_id = 0; + bool audio_fft_overdrive = false; + uint32_t channel_open_token_id = 100; + uint32_t channel_close_token_id = 101; + + static bool is_gemma_family(ModelType t) { + return t == ModelType::GEMMA || t == ModelType::GEMMA3N || t == ModelType::GEMMA4; + } + + bool from_json(const std::string& json_path); + std::string to_json() const; +}; + + + +struct MergeRule { + std::string first; + std::string second; + std::string merged; + uint32_t priority; + + MergeRule(const std::string& f, const std::string& s, const std::string& m, uint32_t p) + : first(f), second(s), merged(m), priority(p) {} +}; + + +struct ToolCallInfo { + std::string name; + std::string arguments; +}; + +struct ChatMessage { + std::string role; + std::string content; + std::string name; + std::vector images; + std::vector audio; + size_t audio_soft_token_count = 0; + std::vector tool_calls; +}; + +inline std::string format_needle_query_text(const std::vector& messages) { + std::string system_text; + std::string user_query; + + for (const auto& msg : messages) { + if (msg.role == "system") { + if (!system_text.empty()) { + system_text += "\n"; + } + system_text += msg.content; + } else if (msg.role == "user") { + user_query = msg.content; + } + } + + if (user_query.empty() && !messages.empty()) { + user_query = messages.back().content; + } + if (system_text.empty()) { + return user_query; + } + if (user_query.empty()) { + return system_text; + } + return system_text + "\n\n" + user_query; +} + +struct ToolConstraintSpec { + std::string name; + std::vector parameter_names; + std::vector required_parameter_names; +}; + +struct TokenizerRuntimeConfig { + enum class TokenizerType { UNKNOWN, BPE, SENTENCEPIECE }; + enum class VocabFormat { UNKNOWN, ID_TAB_TOKEN, LINE_TOKEN }; + enum class Normalizer { NONE, METASPACE, BYTE_LEVEL }; + enum class Decoder { NONE, REPLACE_METASPACE, BYTE_LEVEL }; + + TokenizerType tokenizer_type = TokenizerType::UNKNOWN; + VocabFormat vocab_format = VocabFormat::UNKNOWN; + Normalizer normalizer = Normalizer::NONE; + Decoder decoder = Decoder::NONE; + bool byte_fallback = false; + bool has_chat_template = false; +}; + +TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file); +void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens); +std::vector split_with_special_tokens(const std::string& text, const std::unordered_map& special_tokens); + +inline std::string extract_json_string(const std::string& json, size_t& pos) { + std::string value; + while (pos < json.size() && json[pos] != '"') { + if (json[pos] == '\\' && pos + 1 < json.size()) { + pos++; + if (json[pos] == 'n') value += '\n'; + else if (json[pos] == 't') value += '\t'; + else if (json[pos] == 'r') value += '\r'; + else if (json[pos] == '"') value += '"'; + else if (json[pos] == '\\') value += '\\'; + else value += json[pos]; + } else { + value += json[pos]; + } + pos++; + } + if (pos < json.size()) pos++; + return value; +} + +class Tokenizer { +public: + virtual ~Tokenizer() = default; + + virtual std::vector encode(const std::string& text) const = 0; + virtual std::string decode(const std::vector& tokens) const = 0; + + virtual std::vector apply_chat_template(const std::vector& messages, bool add_generation_prompt = true) const; + virtual std::string format_chat_prompt(const std::vector& messages, bool add_generation_prompt = true, const std::string& tools_json = "", bool enable_thinking_if_supported = false) const; + + virtual uint32_t get_vocab_size() const = 0; + virtual uint32_t get_unk_token() const = 0; + virtual uint32_t get_bos_token() const = 0; + virtual uint32_t get_eos_token() const = 0; + virtual bool has_chat_template() const { return has_chat_template_; } + std::string get_default_stop_sequence() const; + + virtual bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) = 0; + + uint32_t get_image_token_id() const { return image_token_id_; } + uint32_t get_fake_token_id() const { return fake_token_id_; } + uint32_t get_global_img_token_id() const { return global_img_token_id_; } + +protected: + enum class ModelType { UNKNOWN, QWEN, QWEN3P5, GEMMA, GEMMA4, LFM2, BERT, WHISPER, PARAKEET, YOUTU, NEEDLE}; + ModelType model_type_ = ModelType::UNKNOWN; + enum class ModelVariant { DEFAULT, VLM, EXTRACT, RAG}; + ModelVariant model_variant_ = ModelVariant::DEFAULT; + bool has_chat_template_ = false; + std::string chat_template_; + + uint32_t image_token_id_ = 396; + uint32_t fake_token_id_ = 49189; + uint32_t global_img_token_id_ = 49152; + + + uint32_t vision_patch_size_ = 16; + uint32_t vision_pooling_kernel_size_ = 3; + uint32_t vision_default_output_length_ = 280; + uint32_t vision_image_size_ = 768; + TokenizerRuntimeConfig runtime_config_; + + void detect_model_type(const std::string& config_path); + void load_chat_template(const std::string& template_file); + std::string format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; + std::string format_gemma_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_gemma4_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; + std::string format_lfm2_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_lfm2_vl_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_needle_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_youtu_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; +}; + +class BPETokenizer : public Tokenizer { +public: + BPETokenizer(); + ~BPETokenizer(); + + bool load_vocabulary_mmap(const std::string& vocab_file, const std::string& merges_file); + bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; + + std::vector encode(const std::string& text) const override; + std::string decode(const std::vector& tokens) const override; + + uint32_t get_vocab_size() const override { return vocab_size_; } + uint32_t get_unk_token() const override { return unk_token_id_; } + uint32_t get_bos_token() const override { return bos_token_id_; } + uint32_t get_eos_token() const override { return eos_token_id_; } + +private: + std::unordered_map token_to_id_; + std::vector id_to_token_; + std::vector merge_rules_; + std::unordered_map merge_map_; + + uint32_t vocab_size_; + uint32_t unk_token_id_; + uint32_t bos_token_id_; + uint32_t eos_token_id_; + + void* vocab_mmap_ptr_; + size_t vocab_mmap_size_; + + void* merges_mmap_ptr_; + size_t merges_mmap_size_; + + std::vector apply_bpe(const std::vector& tokens) const; + std::pair find_best_merge_fast(const std::vector& tokens) const; + + std::string bytes_to_unicode(const std::string& text) const; + std::string unicode_to_bytes(const std::string& text) const; + std::vector byte_level_split(const std::string& text) const; + std::vector utf8_split(const std::string& text) const; + + void cleanup_mmap(); + +private: + mutable std::unordered_map byte_to_unicode_; + mutable std::unordered_map unicode_to_byte_; + void init_byte_mappings() const; + + std::unordered_map special_tokens_; + std::vector split_with_special_tokens(const std::string& text) const; + void load_special_tokens(const std::string& config_file); +}; + +class SPTokenizer : public Tokenizer { +public: + SPTokenizer(); + ~SPTokenizer(); + + bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; + + std::vector encode(const std::string& text) const override; + std::string decode(const std::vector& tokens) const override; + + uint32_t get_vocab_size() const override { return vocab_size_; } + uint32_t get_unk_token() const override { return unk_token_id_; } + uint32_t get_bos_token() const override { return bos_token_id_; } + uint32_t get_eos_token() const override { return eos_token_id_; } + +private: + struct TrieNode { + std::unordered_map> children; + int32_t token_id = -1; + float score = 0.0f; + }; + + std::unique_ptr trie_root_; + std::unordered_map token_to_id_; + std::vector id_to_token_; + std::vector token_scores_; + + uint32_t vocab_size_; + uint32_t unk_token_id_; + uint32_t bos_token_id_; + uint32_t eos_token_id_; + uint32_t pad_token_id_; + + bool sp_bpe_mode_ = false; + bool sp_add_dummy_prefix_ = false; + bool sp_byte_fallback_ = false; + + void* vocab_mmap_ptr_; + size_t vocab_mmap_size_; + + void build_trie(); + std::vector> tokenize_with_trie(const std::string& text) const; + std::vector tokenize_with_bpe(const std::string& text) const; + std::string preprocess_text(const std::string& text) const; + std::string postprocess_text(const std::string& text) const; + std::vector split_by_unicode_spaces(const std::string& text) const; + + void cleanup_mmap(); + + std::unordered_map special_tokens_; + std::vector split_with_special_tokens(const std::string& text) const; + void load_special_tokens(const std::string& config_file); +}; + +class ConvCache { +public: + struct CircularView { + const void* ptr1; + size_t len1; + const void* ptr2; + size_t len2; + size_t total_len; + }; + + void init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision); + CircularView get_window(size_t layer) const; + void update(CactusGraph* gb, size_t layer, const size_t latest_token); + void reset(); + + bool is_empty() const { return num_layers == 0; } + + size_t num_layers = 0; + size_t hidden_size = 0; + size_t window_size = 0; + Precision precision = Precision::FP32; + size_t element_size = 4; + +private: + struct LayerState { + std::vector data; + size_t head = 0; + size_t count = 0; + }; + + std::vector layer_states; +}; + +struct KVCache { + static constexpr size_t DEFAULT_WINDOW_SIZE = 1024; + static constexpr size_t DEFAULT_SINK_SIZE = 4; + + struct LayerCache { + std::vector keys; + std::vector values; + std::vector key_scales; + std::vector value_scales; + size_t head_dim = 0; + size_t kv_heads = 0; + }; + + std::vector layer_caches; + + size_t window_size = DEFAULT_WINDOW_SIZE; + size_t sink_size = DEFAULT_SINK_SIZE; + size_t current_seq_len = 0; + size_t total_seq_len = 0; + size_t max_seq_len = 2048; + size_t num_layers = 0; + Precision precision; + size_t element_size = 4; + + void set_window_size(size_t window, size_t sink = DEFAULT_SINK_SIZE); + size_t get_effective_seq_len() const { return current_seq_len; } + size_t get_total_seq_len() const { return total_seq_len; } + size_t get_layer_head_dim(size_t layer_idx) const { return layer_caches[layer_idx].head_dim; } + size_t get_layer_kv_heads(size_t layer_idx) const { return layer_caches[layer_idx].kv_heads; } + + void init(size_t num_layers, size_t max_seq, const std::vector& layer_dims, const std::vector& layer_kv_heads, Precision model_precision); + void reset(); + void update_from_graph(CactusGraph* gb, const std::vector& k_nodes, + const std::vector& v_nodes, size_t seq_len, + size_t num_layers); + + void update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data, + size_t num_tokens, size_t kv_heads, size_t head_dim); + + bool is_empty() const { return current_seq_len == 0; } + void* get_key_ptr(size_t layer); + void* get_value_ptr(size_t layer); + + struct CircularView { + const void* ptr1; + const void* ptr2; + size_t len1; + size_t len2; + size_t total_len; + }; + + CircularView get_key_view(size_t layer); + CircularView get_value_view(size_t layer); + + const int8_t* get_keys_int8(size_t layer) const; + const int8_t* get_values_int8(size_t layer) const; + const float* get_key_scales(size_t layer) const; + const float* get_value_scales(size_t layer) const; + + void remove_token_range(size_t start, size_t count); + void compact_to_windows(const std::vector& target_windows); +}; + +class ToolCallConstrainer { +public: + enum class State { + DONE, + + QWEN_START, + QWEN_EXPECT_OPEN_BRACE, + QWEN_EXPECT_NAME_KEY, + QWEN_EXPECT_NAME_COLON, + QWEN_EXPECT_NAME_VALUE, + QWEN_EXPECT_COMMA, + QWEN_EXPECT_ARGS_KEY, + QWEN_EXPECT_ARGS_COLON, + QWEN_IN_ARGUMENTS, + QWEN_EXPECT_CLOSE_BRACE, + QWEN_EXPECT_END, + + NEEDLE_START, + + LFM_START, + LFM_EXPECT_BRACKET, + LFM_IN_FUNC_NAME, + LFM_EXPECT_PAREN, + LFM_IN_ARGUMENTS, + LFM_EXPECT_BRACKET_CLOSE, + LFM_EXPECT_END, + + GEMMA_START, + GEMMA_EXPECT_CALL, + GEMMA_IN_FUNC_NAME, + GEMMA_EXPECT_BRACE, + GEMMA_IN_ARGUMENTS, + GEMMA_EXPECT_END + }; + + void init(Config::ModelType model_type, + const std::vector& tools, + Tokenizer* tokenizer); + + const std::unordered_map& get_bias() const { return current_bias_; } + + void update(uint32_t token_id, const std::string& decoded_text); + + void reset(); + + bool is_active() const { return active_; } + +private: + bool active_ = false; + State state_ = State::QWEN_START; + Config::ModelType model_type_ = Config::ModelType::QWEN; + Tokenizer* tokenizer_ = nullptr; + + bool is_gemma_family() const { return Config::is_gemma_family(model_type_); } + bool is_needle() const { return model_type_ == Config::ModelType::NEEDLE; } + + enum class NeedleJsonState { + FREE, + IN_NAME, + IN_ARG_KEY, + }; + + struct NeedleTrieNode { + std::unordered_map> children; + bool is_terminal = false; + }; + + std::vector tool_specs_; + std::vector function_names_; + std::string generated_text_; + int brace_depth_ = 0; + + std::string call_start_tag_; + std::string call_end_tag_; + + std::unordered_set qwen_tool_call_start_tokens_; + std::unordered_set qwen_tool_call_end_tokens_; + std::unordered_set open_brace_tokens_; + std::unordered_set close_brace_tokens_; + std::unordered_set colon_tokens_; + std::unordered_set comma_tokens_; + std::unordered_set name_key_tokens_; + std::unordered_set args_key_tokens_; + std::unordered_set quote_tokens_; + std::unordered_set backtick_tokens_; + std::unordered_set all_func_name_tokens_; + std::unordered_map> func_name_sequences_; + NeedleJsonState needle_json_state_ = NeedleJsonState::FREE; + std::string needle_buffer_; + std::string needle_constrained_buf_; + std::string needle_current_function_; + bool needle_in_arguments_ = false; + int needle_arguments_depth_ = 0; + int needle_nesting_depth_ = 0; + bool needle_in_string_value_ = false; + bool needle_prev_char_escape_ = false; + std::unique_ptr needle_name_trie_; + std::unordered_map> needle_param_tries_; + std::vector needle_token_strings_; + std::unordered_map> needle_token_index_; + + std::unordered_set tool_start_tokens_; + std::unordered_set tool_end_tokens_; + std::unordered_set bracket_open_tokens_; + std::unordered_set bracket_close_tokens_; + std::unordered_set paren_open_tokens_; + std::unordered_set paren_close_tokens_; + std::unordered_set equals_tokens_; + + std::string lfm_current_function_; + std::string lfm_args_buffer_; + std::unordered_set lfm_seen_arg_keys_; + std::unordered_map> lfm_required_params_; + std::unordered_map> lfm_all_params_; + + std::unordered_set gemma_call_start_tokens_; + std::unordered_set gemma_call_end_tokens_; + std::unordered_set gemma_response_start_tokens_; + std::unordered_set gemma_call_prefix_tokens_; + std::unordered_set escape_tokens_; + + std::unordered_map current_bias_; + + void compute_bias(); + void tokenize_grammar_elements(); + void add_tokens_for_string(const std::string& str, std::unordered_set& token_set); + void add_tokens_for_prefix_string(const std::string& prefix, std::unordered_set& token_set); + void add_tokens_containing(char needle, std::unordered_set& token_set); + void tokenize_function_names(bool quote_names); + void init_common_tokens(); + void init_needle_constraints(); + void reset_needle_constraints(); + void feed_needle_text(const std::string& text); + void feed_needle_char(char ch); + bool needle_at_arg_key_start() const; + bool needle_is_value_string_start() const; + void needle_insert_word(NeedleTrieNode* root, const std::string& word); + const NeedleTrieNode* needle_get_trie_node(const NeedleTrieNode* root, const std::string& prefix) const; + bool needle_check_token_valid(const std::string& token_text, const NeedleTrieNode* trie_node) const; +}; + +class Model { +public: + struct DebugNode { + uint32_t layer_idx; + std::string name; + size_t node_id; + }; + + Model(); + explicit Model(const Config& config); + virtual ~Model(); + + const Config& get_config() const { return config_; } + Tokenizer* get_tokenizer() const { return tokenizer_.get(); } + const std::vector& get_debug_nodes() const; + + virtual bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true); + + virtual bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, + const std::string& system_prompt = "", bool do_warmup = true); + + virtual uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f); + + virtual void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = ""); + + virtual void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, + const std::string& profile_file = ""); + + virtual uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, + float temperature = -1.0f, float top_p = -1.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f); + + virtual uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, float temperature = 0.0f, float top_p = 0.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f, + float* out_token_time_start = nullptr, float* out_token_time_end = nullptr); + + std::vector get_embeddings(const std::vector& tokens, bool pooled = true, bool normalize = false, const std::string& profile_file = ""); + + virtual std::vector get_image_embeddings(const std::string& image_path); + + virtual std::vector get_audio_embeddings(const std::vector& audio_features); + + virtual void reset_cache() { kv_cache_.reset(); token_history_.clear(); } + void record_sampled_token(uint32_t token) { + if (token_history_.size() >= MAX_TOKEN_HISTORY) { + token_history_.erase(token_history_.begin(), token_history_.begin() + (MAX_TOKEN_HISTORY / 2)); + } + token_history_.push_back(token); + } + + double score_tokens_window_logprob(const std::vector& tokens, size_t start, size_t end, size_t context, size_t* tokens_scored); + + + + void set_cache_window(size_t window_size, size_t sink_size = 4) { kv_cache_.set_window_size(window_size, sink_size); } + size_t get_cache_size() const { return kv_cache_.current_seq_len; } + + bool load_npu_prefill(const std::string& model_path); + bool has_npu_prefill() const; + size_t get_prefill_chunk_size() const; + + virtual void remove_thinking_tokens(const std::vector>& ranges); + virtual void compact_kv_cache() {} + + virtual void set_tool_constraints(const std::vector& tools); + virtual void clear_tool_constraints(); + virtual void update_tool_constraints(uint32_t token_id); + + void* graph_handle_; + + void set_vocab_bias(const std::unordered_map& bias) { + vocab_bias_ = bias; + } + + void clear_vocab_bias() { + vocab_bias_.clear(); + } + + bool has_vocab_bias() const { + return !vocab_bias_.empty(); + } + + const std::unordered_map& get_vocab_bias() const { + return vocab_bias_; + } + +protected: + size_t sample_token(CactusGraph* gb, size_t logits_node_id, float temperature, float top_p, size_t top_k, + float min_p, float repetition_penalty, + const std::unordered_map* extra_bias = nullptr) const; + + static void compute_entropy(CactusGraph* gb, size_t logits_node_id, float* out_entropy); + + virtual size_t forward(const std::vector& tokens, bool use_cache = false) = 0; + + virtual size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false); + + virtual void load_weights_to_graph(CactusGraph* gb) = 0; + + virtual size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; + + virtual size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const = 0; + virtual size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; + void update_kv_cache(CactusGraph* gb, size_t seq_len); + virtual std::vector get_kv_layer_dims() const { + return std::vector(config_.num_layers, config_.attention_head_dim); + } + virtual std::vector get_kv_layer_heads() const { + return std::vector(config_.num_layers, config_.attention_kv_heads); + } + virtual void post_init() {} + virtual void post_execute_updates(CactusGraph*, size_t) {} + Config config_; + std::unique_ptr tokenizer_; + + bool initialized_; + float attention_scale_; + +protected: + KVCache kv_cache_; + std::vector cache_k_output_nodes_; + std::vector cache_v_output_nodes_; + + std::string embedding_file_path_; + size_t embedding_node_id_; + std::string model_folder_path_; + size_t output_weight_node_id_; + + mutable std::vector debug_nodes_; + + void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) const; + void clear_debug_nodes(); + + bool init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup); + bool owns_graph_; + + std::unique_ptr npu_prefill_; + void prefill_npu(const std::vector& tokens); + virtual std::vector<__fp16> get_token_embeddings(const std::vector& tokens); + + static constexpr size_t MAX_TOKEN_HISTORY = 128; + ToolCallConstrainer tool_constrainer_; + std::vector token_history_; + +private: + std::unordered_map vocab_bias_; +}; + +std::unique_ptr create_model(const std::string& model_folder); + +class Siglip2Preprocessor { +public: + struct Config { + int patch_size = 16; + int downsample_factor = 2; + int min_tiles = 2; + int max_tiles = 10; + bool use_thumbnail = true; + int min_image_tokens = 64; + int max_image_tokens = 256; + int max_num_patches = 1024; + int tile_size = 512; + float max_pixels_tolerance = 2.0f; + bool do_resize = true; + bool do_rescale = true; + bool do_normalize = true; + bool do_convert_rgb = true; + bool do_image_splitting = true; + float rescale_factor = 1.0f / 255.0f; + float image_mean[3] = {0.5f, 0.5f, 0.5f}; + float image_std[3] = {0.5f, 0.5f, 0.5f}; + }; + + struct PreprocessedImage { + std::vector pixel_values; + std::vector pixel_attention_mask; + std::vector> spatial_shapes; + std::vector pixel_values_shape; + std::vector pixel_attention_mask_shape; + std::vector spatial_shapes_shape; + int num_patches_height; + int num_patches_width; + int actual_num_patches; + int num_tiles; + int patch_dim; + int max_patches_per_tile; + + int image_rows; + int image_cols; + int image_height; + int image_width; + int tokens_per_tile; + int thumbnail_tokens; + + ~PreprocessedImage(); + }; + + struct SpatialShapeResult { + std::vector> shapes; + int grid_rows; + int grid_cols; + }; + + explicit Siglip2Preprocessor(const Config& config); + Siglip2Preprocessor(); + ~Siglip2Preprocessor(); + + PreprocessedImage preprocess_from_file(const std::string& image_path); + PreprocessedImage preprocess_from_memory(const unsigned char* img_data, int width, int height, int channels); + SpatialShapeResult compute_spatial_shapes(int height, int width); + +private: + Config config_; + + std::pair compute_pixel_limits() const; + std::vector convert_to_rgb(const unsigned char* img_data, int width, int height, int channels); + std::pair smart_resize(int height, int width); + bool is_image_too_large(int height, int width); + std::pair get_grid_layout(int height, int width); + std::pair find_closest_aspect_ratio(float aspect_ratio, int width, int height); + std::vector resize_image(const unsigned char* img_data, int src_width, int src_height, + int dst_width, int dst_height, int channels); + std::vector normalize_image(const float* img_data, int width, int height, int channels); + std::vector> convert_image_to_patches( + const std::vector& image, int width, int height, int channels, int patch_size); + PreprocessedImage pad_patches(const std::vector>& tile_patches, + const std::vector>& spatial_shapes, + int patch_dim, + int max_patches_per_tile); + int round_by_factor(int number, int factor); +}; + +class AudioProcessor { +public: + struct SpectrogramConfig { + size_t n_fft = 400; + size_t hop_length = 160; + size_t frame_length = 400; + float power = 2.0f; + bool center = true; + const char* pad_mode = "reflect"; + bool onesided = true; + float dither = 0.0f; + float mel_floor = 1e-10f; + const char* log_mel = nullptr; + float reference = 1.0f; + float min_value = 1e-10f; + bool remove_dc_offset = false; + float preemphasis = 0.0f; + bool hann_periodic = true; + float window_a0 = 0.5f; + size_t fft_override = 0; + bool mel_floor_additive = false; + }; + + AudioProcessor(); + ~AudioProcessor(); + + void init_mel_filters(size_t num_frequency_bins, size_t num_mel_filters, + float min_freq, float max_freq, size_t sampling_rate, + const char* norm = "slaney", const char* mel_scale = "slaney"); + + std::vector compute_spectrogram( + const std::vector& waveform, + const SpectrogramConfig& config); + + static std::vector compute_irfft( + const std::vector& complex_input, + size_t n, + const char* norm = "backward"); + + const std::vector& get_mel_filters() const { return mel_filters_; } + + size_t get_num_mel_filters() const { return num_mel_filters_; } + size_t get_num_frequency_bins() const { return num_frequency_bins_; } + +private: + std::vector mel_filters_; + size_t num_frequency_bins_; + size_t num_mel_filters_; +}; + +namespace index { + constexpr uint32_t MAGIC = 0x43414354; + constexpr uint32_t VERSION = 1; + + struct Document { + int id; + std::vector embedding; + std::string content; + std::string metadata; + }; + + struct QueryResult { + int doc_id; + float score; + + QueryResult(int doc_id, float score) : doc_id(doc_id), score(score) {} + }; + + struct QueryOptions { + size_t top_k = 10; + float score_threshold = -1.0f; + }; + + class Index { + public: + Index(const std::string& index_path, const std::string& data_path, size_t embedding_dim); + ~Index(); + + Index(const Index&) = delete; + Index& operator=(const Index&) = delete; + Index(Index&&) = delete; + Index& operator=(Index&&) = delete; + + void add_documents(const std::vector& documents); + void delete_documents(const std::vector& doc_ids); + std::vector get_documents(const std::vector& doc_ids); + std::vector> query(const std::vector>& embeddings, const QueryOptions& options); + void compact(); + + private: + struct IndexHeader { + uint32_t magic; + uint32_t version; + uint32_t embedding_dim; + uint32_t num_documents; + }; + + struct IndexEntry { + int32_t doc_id; + uint64_t data_offset; + uint8_t flags; // bit 0: tombstone + + const __fp16* embedding() const { + return reinterpret_cast(this + 1); + } + + static size_t size(size_t embedding_dim) { + return sizeof(IndexEntry) + embedding_dim * sizeof(__fp16); + } + }; + + struct DataHeader { + uint32_t magic; + uint32_t version; + }; + + struct DataEntry { + uint16_t content_len; + uint16_t metadata_len; + + const char* content() const { + return reinterpret_cast(this + 1); + } + + const char* metadata() const { + return content() + content_len; + } + }; + + void parse_index_header(); + void parse_data_header(); + void build_doc_id_map(); + void validate_documents(const std::vector& documents); + void validate_doc_ids(const std::vector& doc_ids); + ssize_t write_full(int fd, const void* buf, size_t count); + + std::unordered_map doc_id_map_; + + std::string index_path_, data_path_; + size_t embedding_dim_; + size_t index_entry_size_; + uint32_t num_documents_; + + int index_fd_, data_fd_; + void *mapped_index_, *mapped_data_; + size_t index_file_size_, data_file_size_; + }; +} // namespace index + +} +} diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/gemma_tools.h b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/gemma_tools.h new file mode 100644 index 0000000..f0f9fe2 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/gemma_tools.h @@ -0,0 +1,576 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace gemma { + +inline std::string to_upper(const std::string& s) { + std::string result = s; + for (auto& c : result) c = std::toupper(c); + return result; +} + +inline std::string escape(const std::string& s) { + return "" + s + ""; +} + +inline void skip_whitespace(const std::string& json, size_t& pos) { + while (pos < json.length() && std::isspace(json[pos])) pos++; +} + +inline std::string extract_json_string(const std::string& json, size_t& pos) { + std::string value; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\' && pos + 1 < json.length()) { + pos++; + if (json[pos] == 'n') value += '\n'; + else if (json[pos] == 't') value += '\t'; + else if (json[pos] == 'r') value += '\r'; + else if (json[pos] == '"') value += '"'; + else if (json[pos] == '\\') value += '\\'; + else value += json[pos]; + } else { + value += json[pos]; + } + pos++; + } + if (pos < json.length()) pos++; + return value; +} + +std::string format_argument(const std::string& json, size_t& pos, bool escape_keys); +std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/); + +inline std::string format_argument(const std::string& json, size_t& pos, bool escape_keys = true) { + skip_whitespace(json, pos); + if (pos >= json.length()) return ""; + + char c = json[pos]; + + if (c == '"') { + pos++; + std::string value = extract_json_string(json, pos); + return escape(value); + } else if (c == '{') { + std::string result = "{"; + pos++; + bool first = true; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == '}') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + if (json[pos] != '"') break; + pos++; + std::string key = extract_json_string(json, pos); + + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == ':') pos++; + + std::string value = format_argument(json, pos, escape_keys); + + if (!first) result += ","; + first = false; + if (escape_keys) { + result += escape(key) + ":" + value; + } else { + result += key + ":" + value; + } + } + result += "}"; + return result; + } else if (c == '[') { + std::string result = "["; + pos++; + bool first = true; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == ']') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + std::string value = format_argument(json, pos, escape_keys); + + if (!first) result += ","; + first = false; + result += value; + } + result += "]"; + return result; + } else if (json.compare(pos, 4, "true") == 0) { + pos += 4; + return "true"; + } else if (json.compare(pos, 5, "false") == 0) { + pos += 5; + return "false"; + } else if (json.compare(pos, 4, "null") == 0) { + pos += 4; + return "null"; + } else { + size_t start = pos; + while (pos < json.length() && (std::isdigit(json[pos]) || json[pos] == '.' || + json[pos] == '-' || json[pos] == '+' || json[pos] == 'e' || json[pos] == 'E')) { + pos++; + } + return json.substr(start, pos - start); + } +} + +inline std::map parse_json_object_raw(const std::string& json, size_t& pos) { + std::map result; + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] != '{') return result; + pos++; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == '}') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + if (json[pos] != '"') break; + pos++; + std::string key = extract_json_string(json, pos); + + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == ':') pos++; + skip_whitespace(json, pos); + + size_t value_start = pos; + if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + pos++; + } else if (json[pos] == '{') { + int depth = 1; + pos++; + while (pos < json.length() && depth > 0) { + if (json[pos] == '{') depth++; + else if (json[pos] == '}') depth--; + else if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + } else if (json[pos] == '[') { + int depth = 1; + pos++; + while (pos < json.length() && depth > 0) { + if (json[pos] == '[') depth++; + else if (json[pos] == ']') depth--; + else if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + } else { + while (pos < json.length() && json[pos] != ',' && json[pos] != '}') pos++; + } + result[key] = json.substr(value_start, pos - value_start); + } + return result; +} + +inline std::string get_json_string_value(const std::string& json, size_t pos) { + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == '"') { + pos++; + return extract_json_string(json, pos); + } + return ""; +} + +inline std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/) { + static const std::set standard_keys = {"description", "type", "properties", "required", "nullable"}; + + size_t pos = 0; + auto properties = parse_json_object_raw(properties_json, pos); + + std::string result; + bool first = true; + + for (const auto& [key, value_json] : properties) { + if (standard_keys.count(key)) continue; + + if (!first) result += ","; + first = false; + + size_t prop_pos = 0; + auto prop_obj = parse_json_object_raw(value_json, prop_pos); + + result += key + ":{"; + + if (prop_obj.count("description")) { + std::string desc = get_json_string_value(prop_obj["description"], 0); + result += "description:" + escape(desc); + } + + std::string type_val; + if (prop_obj.count("type")) { + type_val = get_json_string_value(prop_obj["type"], 0); + } + + if (to_upper(type_val) == "STRING") { + if (prop_obj.count("enum")) { + size_t enum_pos = 0; + std::string enum_formatted = format_argument(prop_obj["enum"], enum_pos, true); + result += ",enum:" + enum_formatted; + } + } else if (to_upper(type_val) == "OBJECT") { + if (prop_obj.count("properties")) { + std::string nested_required; + if (prop_obj.count("required")) { + nested_required = prop_obj["required"]; + } + result += ",properties:{" + format_parameters(prop_obj["properties"], nested_required) + "}"; + } + if (prop_obj.count("required")) { + std::string req_items; + size_t req_pos = 0; + skip_whitespace(prop_obj["required"], req_pos); + if (req_pos < prop_obj["required"].length() && prop_obj["required"][req_pos] == '[') { + req_pos++; + bool req_first = true; + while (req_pos < prop_obj["required"].length()) { + skip_whitespace(prop_obj["required"], req_pos); + if (prop_obj["required"][req_pos] == ']') break; + if (prop_obj["required"][req_pos] == ',') { req_pos++; continue; } + if (prop_obj["required"][req_pos] == '"') { + req_pos++; + std::string req_item = extract_json_string(prop_obj["required"], req_pos); + if (!req_first) req_items += ","; + req_first = false; + req_items += escape(req_item); + } + } + } + if (!req_items.empty()) { + result += ",required:[" + req_items + "]"; + } + } + } else if (to_upper(type_val) == "ARRAY") { + if (prop_obj.count("items")) { + result += ",items:{"; + size_t items_pos = 0; + auto items_obj = parse_json_object_raw(prop_obj["items"], items_pos); + bool items_first = true; + + for (const auto& [item_key, item_value] : items_obj) { + if (!items_first) result += ","; + items_first = false; + + if (item_key == "properties") { + std::string items_required; + if (items_obj.count("required")) { + items_required = items_obj["required"]; + } + result += "properties:{" + format_parameters(item_value, items_required) + "}"; + } else if (item_key == "required") { + result += "required:["; + size_t req_pos = 0; + skip_whitespace(item_value, req_pos); + if (req_pos < item_value.length() && item_value[req_pos] == '[') { + req_pos++; + bool req_first = true; + while (req_pos < item_value.length()) { + skip_whitespace(item_value, req_pos); + if (item_value[req_pos] == ']') break; + if (item_value[req_pos] == ',') { req_pos++; continue; } + if (item_value[req_pos] == '"') { + req_pos++; + std::string req_item = extract_json_string(item_value, req_pos); + if (!req_first) result += ","; + req_first = false; + result += escape(req_item); + } + } + } + result += "]"; + } else if (item_key == "type") { + std::string item_type = get_json_string_value(item_value, 0); + result += "type:" + escape(to_upper(item_type)); + } else { + size_t val_pos = 0; + result += item_key + ":" + format_argument(item_value, val_pos, true); + } + } + result += "}"; + } + } + + if (!type_val.empty()) { + result += ",type:" + escape(to_upper(type_val)); + } + + result += "}"; + } + + return result; +} + +inline std::string format_function_declaration(const std::string& name, + const std::string& description, + const std::string& params_json) { + std::string result = "declaration:" + name + "{"; + result += "description:" + escape(description); + + if (!params_json.empty()) { + result += ",parameters:{"; + + size_t pos = 0; + auto params = parse_json_object_raw(params_json, pos); + + if (params.count("properties")) { + std::string required_json; + if (params.count("required")) { + required_json = params["required"]; + } + result += "properties:{" + format_parameters(params["properties"], required_json) + "}"; + } + + if (params.count("required")) { + std::string req_items; + size_t req_pos = 0; + skip_whitespace(params["required"], req_pos); + if (req_pos < params["required"].length() && params["required"][req_pos] == '[') { + req_pos++; + bool first = true; + while (req_pos < params["required"].length()) { + skip_whitespace(params["required"], req_pos); + if (params["required"][req_pos] == ']') break; + if (params["required"][req_pos] == ',') { req_pos++; continue; } + if (params["required"][req_pos] == '"') { + req_pos++; + std::string item = extract_json_string(params["required"], req_pos); + if (!first) req_items += ","; + first = false; + req_items += escape(item); + } + } + } + if (!req_items.empty()) { + result += ",required:[" + req_items + "]"; + } + } + + if (params.count("type")) { + std::string type_val = get_json_string_value(params["type"], 0); + result += ",type:" + escape(to_upper(type_val)); + } + + result += "}"; + } + + result += "}"; + return result; +} + +template +inline std::string format_tools(const std::vector& tools, bool use_pipe_tags = false) { + if (tools.empty()) return ""; + + const char* decl_start = use_pipe_tags ? "<|tool>" : ""; + const char* decl_end = use_pipe_tags ? "" : ""; + + std::string result; + for (const auto& tool : tools) { + result += decl_start; + std::string params_json; + auto it = tool.parameters.find("schema"); + if (it != tool.parameters.end()) { + params_json = it->second; + } + + result += format_function_declaration(tool.name, tool.description, params_json); + result += decl_end; + } + return result; +} + + +inline size_t match_quote_tag(const std::string& s, size_t pos) { + if (s.compare(pos, 8, "") == 0) return 8; + if (s.compare(pos, 5, "<|\"|>") == 0) return 5; + return 0; +} + +inline size_t find_quote_tag(const std::string& s, size_t pos) { + size_t e = s.find("", pos); + size_t t = s.find("<|\"|>", pos); + if (e == std::string::npos) return t; + if (t == std::string::npos) return e; + return std::min(e, t); +} + +inline std::string unescape(const std::string& s) { + const std::string ESCAPE_TAG = ""; + std::string result = s; + size_t pos = 0; + while ((pos = result.find(ESCAPE_TAG, pos)) != std::string::npos) { + result.erase(pos, ESCAPE_TAG.length()); + } + return result; +} + +inline std::string args_to_json(const std::string& args_content) { + std::string result = "{"; + size_t pos = 0; + bool first = true; + + if (!args_content.empty() && args_content[0] == '{') pos = 1; + + while (pos < args_content.length()) { + while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; + if (pos >= args_content.length() || args_content[pos] == '}') break; + if (args_content[pos] == ',') { pos++; continue; } + + size_t key_start = pos; + while (pos < args_content.length() && args_content[pos] != ':') pos++; + std::string key = args_content.substr(key_start, pos - key_start); + if (pos < args_content.length()) pos++; + + std::string value; + while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; + + if (pos < args_content.length()) { + size_t qtag_len = match_quote_tag(args_content, pos); + if (qtag_len > 0) { + pos += qtag_len; + size_t val_end = find_quote_tag(args_content, pos); + if (val_end != std::string::npos) { + value = "\"" + args_content.substr(pos, val_end - pos) + "\""; + pos = val_end + match_quote_tag(args_content, val_end); + } + } else if (args_content[pos] == '{') { + int depth = 1; + size_t start = pos; + pos++; + while (pos < args_content.length() && depth > 0) { + if (args_content[pos] == '{') depth++; + else if (args_content[pos] == '}') depth--; + pos++; + } + value = args_to_json(args_content.substr(start, pos - start)); + } else if (args_content[pos] == '[') { + int depth = 1; + size_t start = pos; + pos++; + while (pos < args_content.length() && depth > 0) { + if (args_content[pos] == '[') depth++; + else if (args_content[pos] == ']') depth--; + pos++; + } + std::string arr_content = args_content.substr(start + 1, pos - start - 2); + value = "["; + size_t arr_pos = 0; + bool first_item = true; + while (arr_pos < arr_content.length()) { + while (arr_pos < arr_content.length() && (std::isspace(arr_content[arr_pos]) || arr_content[arr_pos] == ',')) arr_pos++; + if (arr_pos >= arr_content.length()) break; + + if (!first_item) value += ","; + first_item = false; + + size_t aq_len = match_quote_tag(arr_content, arr_pos); + if (aq_len > 0) { + arr_pos += aq_len; + size_t end = find_quote_tag(arr_content, arr_pos); + if (end != std::string::npos) { + value += "\"" + arr_content.substr(arr_pos, end - arr_pos) + "\""; + arr_pos = end + match_quote_tag(arr_content, end); + } + } else { + size_t end = arr_content.find_first_of(",]", arr_pos); + if (end == std::string::npos) end = arr_content.length(); + value += arr_content.substr(arr_pos, end - arr_pos); + arr_pos = end; + } + } + value += "]"; + } else { + size_t val_start = pos; + while (pos < args_content.length() && args_content[pos] != ',' && args_content[pos] != '}') { + pos++; + } + value = args_content.substr(val_start, pos - val_start); + while (!value.empty() && std::isspace(value.back())) value.pop_back(); + } + } + + if (!first) result += ","; + first = false; + result += "\"" + key + "\":" + value; + } + + result += "}"; + return result; +} + +inline void parse_function_calls(std::string& response, std::vector& function_calls) { + + const std::string CALL_START = (response.find("<|tool_call>") != std::string::npos) + ? "<|tool_call>" : ""; + const std::string CALL_END = (CALL_START == "<|tool_call>") + ? "" : ""; + size_t pos = 0; + + while ((pos = response.find(CALL_START, pos)) != std::string::npos) { + size_t content_start = pos + CALL_START.length(); + size_t call_end_pos = response.find(CALL_END, content_start); + + size_t content_end = (call_end_pos != std::string::npos) ? call_end_pos : response.length(); + std::string call_content = response.substr(content_start, content_end - content_start); + + if (call_content.compare(0, 5, "call:") == 0) { + size_t brace_pos = call_content.find('{'); + + if (brace_pos == std::string::npos) { + size_t sep_pos = call_content.find_first_of(", ", 5); + if (sep_pos != std::string::npos) { + std::string func_name = call_content.substr(5, sep_pos - 5); + size_t args_start = sep_pos + 1; + while (args_start < call_content.length() && + (call_content[args_start] == ' ' || call_content[args_start] == ',')) { + args_start++; + } + std::string args_content = "{" + call_content.substr(args_start); + if (args_content.back() != '}') args_content += "}"; + + std::string args_json = args_to_json(args_content); + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; + function_calls.push_back(json_call); + } + } else { + std::string func_name = call_content.substr(5, brace_pos - 5); + std::string args_content = call_content.substr(brace_pos); + if (args_content.back() != '}') args_content += "}"; + + std::string args_json = args_to_json(args_content); + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; + function_calls.push_back(json_call); + } + } + + size_t erase_end = (call_end_pos != std::string::npos) ? + call_end_pos + CALL_END.length() : response.length(); + response.erase(pos, erase_end - pos); + } +} + +} // namespace gemma \ No newline at end of file diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/graph.h b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/graph.h new file mode 100644 index 0000000..b436177 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/graph.h @@ -0,0 +1,779 @@ +#ifndef GRAPH_H +#define GRAPH_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { + +enum class LogLevel { + DEBUG = 0, + INFO = 1, + WARN = 2, + ERROR = 3, + NONE = 4 +}; + +class Logger { +public: + static Logger& instance() { + static Logger logger; + return logger; + } + + void set_level(LogLevel level) { min_level_ = level; } + LogLevel get_level() const { return min_level_; } + + void set_callback(std::function cb) { + std::lock_guard lock(mutex_); + callback_ = cb; + } + + void log(LogLevel level, const std::string& component, const std::string& message) { + if (level < min_level_) return; + + std::lock_guard lock(mutex_); + + if (callback_) { + callback_(level, component, message); + } else { + std::cerr << "[" << level_string(level) << "] [" << component << "] " << message << std::endl; + } + + if (level == LogLevel::ERROR) { + last_error_ = "[" + component + "] " + message; + } + } + + const std::string& last_error() const { return last_error_; } + void clear_error() { last_error_.clear(); } + +private: + Logger() : min_level_(LogLevel::WARN) {} + + static const char* level_string(LogLevel level) { + switch (level) { + case LogLevel::DEBUG: return "DEBUG"; + case LogLevel::INFO: return "INFO"; + case LogLevel::WARN: return "WARN"; + case LogLevel::ERROR: return "ERROR"; + default: return "?"; + } + } + + LogLevel min_level_; + std::mutex mutex_; + std::string last_error_; + std::function callback_; +}; + +} // namespace cactus + +#define CACTUS_LOG(level, component, msg) \ + do { \ + if (static_cast(level) >= static_cast(cactus::Logger::instance().get_level())) { \ + std::ostringstream _cactus_log_ss; \ + _cactus_log_ss << msg; \ + cactus::Logger::instance().log(level, component, _cactus_log_ss.str()); \ + } \ + } while(0) + +#define CACTUS_LOG_DEBUG(component, msg) CACTUS_LOG(cactus::LogLevel::DEBUG, component, msg) +#define CACTUS_LOG_INFO(component, msg) CACTUS_LOG(cactus::LogLevel::INFO, component, msg) +#define CACTUS_LOG_WARN(component, msg) CACTUS_LOG(cactus::LogLevel::WARN, component, msg) +#define CACTUS_LOG_ERROR(component, msg) CACTUS_LOG(cactus::LogLevel::ERROR, component, msg) + +namespace GraphFile { + class MappedFile; + struct SerializedGraph; +} + +enum class Precision { + INT8, + FP16, + FP32, + INT4 +}; + +enum class ComputeBackend { + CPU, + NPU +}; + +enum class Activation { + SILU, + GELU, + GELU_ERF, + RELU, + SIGMOID, + TANH +}; + +enum class OpType { + INPUT, PRECISION_CAST, + ADD, ADD_CLIPPED, SUBTRACT, MULTIPLY, DIVIDE, + ABS, POW, FLATTEN, VIEW, + MATMUL, TRANSPOSE, RESHAPE, SLICE, GATHER, EMBEDDING, + BILINEAR_INTERPOLATION, + SUM, MEAN, VARIANCE, MIN, MAX, + RMS_NORM, ROPE, ROPE_GPTJ, SOFTMAX, ATTENTION, ATTENTION_INT8_HYBRID, REL_POS_BIAS, CONV1D_CAUSAL, CONV1D_K3, CONV1D_K7S3, CONV1D, CONV1D_SAME_DEPTHWISE_K9, CONV1D_POINTWISE, CONV2D_K3S2P1, CONV2D_DEPTHWISE_K3S2P1, CONV2D_POINTWISE_1X1, GLU, BATCHNORM, + SCALAR_ADD, SCALAR_SUBTRACT, SCALAR_MULTIPLY, SCALAR_DIVIDE, SCALAR_EXP, SCALAR_SQRT, SCALAR_COS, SCALAR_SIN, SCALAR_LOG, + RELU, SILU, GELU, GELU_ERF, SIGMOID, TANH, + SAMPLE, CONCAT, CAT, + SCATTER_TOPK, + TOPK, LAYERNORM, GROUPNORM, + MOE_LAYER, + INDEX, + PERSISTENT, + QUANTIZE_ACTIVATIONS, + LSTM_CELL, + GATED_DELTANET_DECODE, + GATED_DELTANET_PREFILL, + STFT, + ALTUP_PREDICT, + ALTUP_CORRECT, + GAUSSIAN_TOPK, + MAXPOOL1D, + BILSTM_SEQUENCE, + LEAKY_RELU, + CONV2D_K3S1P1, + STATS_POOL, + WEIGHTED_STATS_POOL, + DENSE_MLP_INT4_FUSED +}; + +struct PrecisionTraits { + static constexpr size_t size_of(Precision prec) { + switch (prec) { + case Precision::INT8: return 1; + case Precision::FP16: return 2; + case Precision::FP32: return 4; + case Precision::INT4: return 1; + } + return 1; + } + + static constexpr size_t packed_size_of(Precision prec, size_t count) { + switch (prec) { + case Precision::INT4: return (count + 1) / 2; + default: return count * size_of(prec); + } + } + + static size_t byte_offset_of(Precision prec, size_t element_offset) { + switch (prec) { + case Precision::INT4: + assert(element_offset % 32 == 0 && "INT4 byte offset must be group-aligned (multiple of 32)"); + return element_offset / 2; + default: return element_offset * size_of(prec); + } + } + + static constexpr bool is_integer(Precision prec) { + switch (prec) { + case Precision::INT8: return true; + case Precision::INT4: return true; + case Precision::FP16: return false; + case Precision::FP32: return false; + } + return true; + } + + static constexpr bool is_floating_point(Precision prec) { + switch (prec) { + case Precision::INT8: return false; + case Precision::INT4: return false; + case Precision::FP16: return true; + case Precision::FP32: return true; + } + return false; + } +}; + +namespace Quantization { + void int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); + void fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); + void fp16_to_fp32(const __fp16* src, float* dst, size_t count); + void fp32_to_fp16(const float* src, __fp16* dst, size_t count); + void int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); + void fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); +} + +struct TensorConfig { + Precision default_precision = Precision::INT8; + Precision compute_precision = Precision::INT8; + Precision output_precision = Precision::INT8; + bool auto_mixed_precision = false; + + static TensorConfig& global(); +}; + +struct BroadcastInfo { + std::vector output_shape; + bool needs_broadcasting; + + static BroadcastInfo compute(const std::vector& lhs, const std::vector& rhs); +}; + +class BufferPool; + +struct BufferDesc { + std::vector shape; + size_t total_size; + size_t byte_size; + std::unique_ptr data; + void* external_data; + char* pooled_data; + Precision precision; + + size_t group_size = 0; + size_t num_groups = 0; + void* scales_data = nullptr; + std::unique_ptr owned_scales; + + bool is_interleaved = false; + size_t original_N = 0; + + void* activation_scales_data = nullptr; + std::unique_ptr owned_activation_scales; + size_t num_rows_for_activation_scales = 0; + + BufferDesc(); + BufferDesc(const std::vector& s, Precision prec = Precision::INT8); + ~BufferDesc(); + + BufferDesc(BufferDesc&& other) noexcept; + BufferDesc& operator=(BufferDesc&& other) noexcept; + + BufferDesc(const BufferDesc&) = delete; + BufferDesc& operator=(const BufferDesc&) = delete; + + void* get_data(); + const void* get_data() const; + + template + T* data_as() { return static_cast(get_data()); } + + template + const T* data_as() const { return static_cast(get_data()); } + + const __fp16* scales_as_fp16() const { + return reinterpret_cast(scales_data); + } + + bool is_grouped_int8() const { + return precision == Precision::INT8 && group_size > 0; + } + + bool is_grouped_int4() const { + return precision == Precision::INT4 && group_size > 0; + } + + void set_grouped_scales(size_t gs, size_t ng, void* scales_ptr) { + group_size = gs; + num_groups = ng; + scales_data = scales_ptr; + } + + void set_interleaved(bool interleaved, size_t orig_n) { + is_interleaved = interleaved; + original_N = orig_n; + } + + bool has_activation_scales() const { + return activation_scales_data != nullptr && num_rows_for_activation_scales > 0; + } + const float* activation_scales_as_float() const { + return reinterpret_cast(activation_scales_data); + } + float* activation_scales_as_float() { + return reinterpret_cast(activation_scales_data); + } + void allocate_activation_scales(size_t num_rows) { + num_rows_for_activation_scales = num_rows; + owned_activation_scales = std::make_unique(num_rows * sizeof(float)); + activation_scales_data = owned_activation_scales.get(); + } + void set_activation_scales(void* scales_ptr, size_t num_rows) { + activation_scales_data = scales_ptr; + num_rows_for_activation_scales = num_rows; + } + + void allocate(); + void allocate_from_pool(BufferPool& pool); + void release_to_pool(BufferPool& pool); + void set_external(void* ptr); +}; + +struct OpParams { + float scalar = 0.0f; + float scale = 1.0f; + float theta = 10000.0f; + float epsilon = 1e-6f; + int axis = -1; + bool pretransposed_rhs = false; + size_t position_offset = 0; + size_t slice_start = 0; + size_t slice_length = 0; + size_t window_size = 0; + bool is_causal = true; + bool attention_mask_is_additive = false; + float logit_cap = 0.0f; + std::vector new_shape; + std::vector permutation; + Precision output_precision = Precision::INT8; + BroadcastInfo broadcast_info; + ComputeBackend backend = ComputeBackend::CPU; + + size_t dilation = 1; + size_t stride = 1; + float temperature = 1.0f; + float top_p = 1.0f; + float min_p = 0.15f; + float repetition_penalty = 1.1f; + size_t top_k = 0; + size_t random_seed = 0; + + size_t index_value = 0; + size_t num_classes = 0; + size_t num_groups = 0; + size_t dst_height = 0; + size_t dst_width = 0; + bool align_corners = true; + bool normalize_routing = false; + size_t num_experts = 0; + size_t num_experts_per_tok = 0; + bool moe_gated = true; + Activation activation = Activation::SILU; + + std::vector bias_values; + std::vector bias_indices; + + const int8_t* cached_keys_int8 = nullptr; + const int8_t* cached_values_int8 = nullptr; + const float* cached_k_scales = nullptr; + const float* cached_v_scales = nullptr; + size_t cache_seq_len = 0; + size_t num_kv_heads = 0; + size_t head_dim = 0; + size_t num_fft_bins = 0; + size_t chunk_size = 0; + size_t num_altup_inputs = 0; + size_t v_head_dim = 0; + size_t kernel_size = 0; +}; + +struct GraphNode { + size_t id; + OpType op_type; + std::vector input_ids; + BufferDesc output_buffer; + OpParams params; + + GraphNode(size_t node_id, OpType type); +}; + +using nodes_vector = std::vector>; +using node_index_map_t = std::unordered_map; + +inline const BufferDesc& get_input(const GraphNode& node, size_t idx, + const nodes_vector& nodes, + const node_index_map_t& node_index_map) { + return nodes[node_index_map.at(node.input_ids[idx])]->output_buffer; +} + +struct AxisDims { + size_t outer, axis_size, inner; + static AxisDims from_shape(const std::vector& shape, size_t axis) { + AxisDims d; + d.outer = 1; + for (size_t i = 0; i < axis; i++) d.outer *= shape[i]; + d.axis_size = shape[axis]; + d.inner = 1; + for (size_t i = axis + 1; i < shape.size(); i++) d.inner *= shape[i]; + return d; + } +}; + +template +void dispatch_binary_op(OpType op, const T* lhs, const T* rhs, T* output, size_t count); + +template +void dispatch_unary_op(OpType op, const T* input, T* output, size_t count, float param = 0.0f); + +void compute_node_optimized(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_matmul_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_transpose_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_reduce_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_reshape_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_precision_cast_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_sample_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_scatter_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_layernorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_groupnorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_persistent_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_dense_mlp_int4_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_index_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_lstm_cell_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_gated_deltanet_decode_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_gated_deltanet_prefill_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_altup_predict_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_altup_correct_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_maxpool1d_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_bilstm_sequence_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_conv2d_k3s1p1_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_weighted_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); + +void shrink_thread_local_buffers(); +class BufferPool { +public: + BufferPool() = default; + ~BufferPool() = default; + + BufferPool(const BufferPool&) = delete; + BufferPool& operator=(const BufferPool&) = delete; + BufferPool(BufferPool&&) noexcept = default; + BufferPool& operator=(BufferPool&&) noexcept = default; + + char* acquire(size_t byte_size); + void release(char* ptr, size_t byte_size); + void clear(); + + size_t active_bytes() const { return active_bytes_; } + size_t pool_bytes() const { return pool_bytes_; } + size_t peak_bytes() const { return peak_bytes_; } + +private: + std::unordered_map>> free_buffers_; + size_t active_bytes_ = 0; + size_t pool_bytes_ = 0; + size_t peak_bytes_ = 0; + + size_t round_up_size(size_t size) const; +}; + +namespace ValidationUtils { + void validate_tensor_dims(const std::vector& shape, size_t required_dims, const std::string& op_name); + void validate_precision(Precision actual, Precision required, const std::string& op_name); + void validate_input_count(size_t actual, size_t required, const std::string& op_name); +} + + +class CactusGraph { +public: + CactusGraph(); + ~CactusGraph() = default; + + CactusGraph(const CactusGraph&) = delete; + CactusGraph& operator=(const CactusGraph&) = delete; + CactusGraph(CactusGraph&&) noexcept = default; + CactusGraph& operator=(CactusGraph&&) noexcept = default; + + struct DebugNodeEntry { + uint32_t layer_idx; + std::string name; + size_t node_id; + }; + + void save(const std::string& path); + static CactusGraph load(const std::string& path); + + size_t input(const std::vector& shape, Precision precision = Precision::INT8); + size_t precision_cast(size_t input, Precision target_precision); + size_t quantize_activations(size_t input); + + size_t add(size_t input1, size_t input2); + size_t add_clipped(size_t input1, size_t input2); + size_t subtract(size_t input1, size_t input2); + size_t multiply(size_t input1, size_t input2); + size_t divide(size_t input1, size_t input2); + + size_t scalar_add(size_t input, float value); + size_t scalar_subtract(size_t input, float value); + size_t scalar_multiply(size_t input, float value); + size_t scalar_divide(size_t input, float value); + size_t scalar_exp(size_t input); + size_t scalar_sqrt(size_t input); + size_t scalar_cos(size_t input); + size_t scalar_sin(size_t input); + size_t scalar_log(size_t input); + + size_t relu(size_t input); + size_t silu(size_t input); + size_t gelu(size_t input); + size_t gelu_erf(size_t input); + size_t sigmoid(size_t input); + size_t tanh(size_t input); + size_t glu(size_t input, int axis = -1); + + size_t abs(size_t input); + size_t pow(size_t input, float exponent); + size_t view(size_t input, const std::vector& new_shape); + size_t flatten(size_t input, int start_dim = 0, int end_dim = -1); + + size_t matmul(size_t input1, size_t input2, bool pretransposed_rhs = false, ComputeBackend backend = ComputeBackend::CPU); + size_t transpose(size_t input, ComputeBackend backend = ComputeBackend::CPU); + size_t transposeN(size_t input, const std::vector& permutation, ComputeBackend backend = ComputeBackend::CPU); + size_t reshape(size_t input, const std::vector& new_shape); + size_t slice(size_t input, int axis, size_t start, size_t length); + size_t index(size_t input, size_t index_value, int dim); + + size_t sum(size_t input, int axis); + size_t mean(size_t input, int axis); + size_t variance(size_t input, int axis); + size_t min(size_t input, int axis); + size_t max(size_t input, int axis); + + size_t gather(size_t embeddings, size_t indices); + size_t mmap_embeddings(const std::string& filename); + size_t mmap_weights(const std::string& filename); + void set_grouped_scales(size_t node_id, size_t group_size, size_t num_groups, void* scales_ptr); + void set_interleaved(size_t node_id, bool interleaved, size_t original_N); + + void release_weight_pages(size_t node_id); + void prefetch_weight_pages(size_t node_id); + void release_all_weight_pages(); + size_t embedding(const std::string& filename, size_t indices); + size_t embedding(size_t embedding_tensor, size_t indices); + size_t bilinear_interpolation(size_t pos_embeds, size_t dst_height, size_t dst_width, bool align_corners = true); + + size_t layernorm(size_t input, size_t weight, size_t bias, float epsilon = 1e-5f); + size_t layernorm(size_t input, size_t weight, float epsilon = 1e-5f); // No bias version + size_t groupnorm(size_t input, size_t weight, size_t bias, size_t num_groups = 32, float epsilon = 1e-5f); + size_t batchnorm(size_t input, size_t weight, size_t bias, size_t running_mean, size_t running_var, int axis = 1, float epsilon = 1e-5f); + size_t topk(size_t input, size_t k); + size_t dense_mlp_int4_fused(size_t hidden, size_t gate_weight, + size_t up_weight, size_t down_weight); + size_t moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w3_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation = Activation::SILU, + size_t per_expert_scale = 0); + size_t moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation); + size_t rms_norm(size_t input, size_t weight, float epsilon = 1e-5f); + size_t rope(size_t input, float theta, size_t position_offset = 0, ComputeBackend backend = ComputeBackend::CPU); + size_t rope_gptj(size_t input, float theta, size_t position_offset = 0, size_t rot_dim = 0, ComputeBackend backend = ComputeBackend::CPU); + size_t softmax(size_t input, int axis = -1); + size_t attention(size_t query, size_t key, size_t value, float scale, bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU); + size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, ComputeBackend backend = ComputeBackend::CPU); + size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, size_t window_size, ComputeBackend backend = ComputeBackend::CPU); + size_t attention_masked(size_t query, size_t key, size_t value, size_t mask, float scale, + bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU, + bool additive_mask = false, size_t position_offset = 0, size_t window_size = 0, + float logit_cap = 0.0f); + size_t rel_pos_bias(size_t query, size_t relative_key, float scale); + + size_t attention_int8_hybrid(size_t query, size_t key_new, size_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, + const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, + size_t window_size = 0, size_t v_head_dim = 0); + + size_t conv1d_causal(size_t input, size_t weight, size_t kernel_size, size_t dilation = 1); + size_t conv1d_k3(size_t input, size_t weight, size_t stride); + size_t conv1d_k7s3(size_t input, size_t weight, size_t bias); + size_t conv1d(size_t input, size_t weight, size_t stride); + size_t conv1d(size_t input, size_t weight, size_t bias, size_t stride); + size_t conv1d_same_depthwise_k9(size_t input, size_t weight); + size_t conv1d_same_depthwise_k9(size_t input, size_t weight, size_t bias); + size_t conv1d_pointwise(size_t input, size_t weight); + size_t conv1d_pointwise(size_t input, size_t weight, size_t bias); + size_t conv2d_k3s2p1(size_t input, size_t weight); + size_t conv2d_k3s2p1(size_t input, size_t weight, size_t bias); + size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight); + size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight, size_t bias); + size_t conv2d_pointwise_1x1(size_t input, size_t weight); + size_t conv2d_pointwise_1x1(size_t input, size_t weight, size_t bias); + + size_t lstm_cell(size_t input, size_t h_prev, size_t c_prev, size_t weight_ih, size_t weight_hh, size_t bias_ih, size_t bias_hh); + size_t gated_deltanet_decode(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, float scale = 0.0f); + size_t gated_deltanet_prefill(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, size_t chunk_size = 64, float scale = 0.0f); + size_t stft(size_t input, size_t weight, size_t stride, size_t num_fft_bins); + + size_t altup_predict(size_t coefs, const size_t* streams, size_t num_streams); + size_t altup_correct(size_t coefs, size_t innovation, const size_t* predictions, size_t num_predictions); + + size_t gaussian_topk(size_t input, float ppf); + + size_t maxpool1d(size_t input, size_t kernel_size, size_t stride); + size_t leaky_relu(size_t input, float negative_slope = 0.01f); + size_t bilstm_sequence(size_t input, size_t w_ih_fwd, size_t w_hh_fwd, size_t b_ih_fwd, size_t b_hh_fwd, + size_t w_ih_bwd, size_t w_hh_bwd, size_t b_ih_bwd, size_t b_hh_bwd); + size_t conv2d_k3s1p1(size_t input, size_t weight); + size_t conv2d_k3s1p1(size_t input, size_t weight, size_t bias); + size_t stats_pool(size_t input); + size_t weighted_stats_pool(size_t input, size_t weights); + + size_t sample(size_t logits, float temperature = 0.6f, float top_p = 0.95f, size_t top_k = 20, + const std::unordered_map& logit_bias = {}); + size_t sample_with_options(size_t logits, float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, const std::unordered_map& logit_bias = {}); + + size_t concat(size_t input1, size_t input2, int axis = 0); + size_t cat(const std::vector& inputs, int axis); + size_t scatter_topk(size_t indices, size_t values, size_t num_classes); + + void set_input(size_t node_id, const void* data, Precision precision); + void set_external_input(size_t node_id, void* data, Precision precision); + void* get_output(size_t node_id); + + void execute(const std::string& profile_file = ""); + void hard_reset(); + void soft_reset(); + void soft_reset_keep_pool(); + void set_prefill_mode(bool enabled) { prefill_mode_ = enabled; } + + void register_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); + void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); + const std::vector& get_debug_nodes() const; + void clear_debug_nodes(); + + size_t add_node(OpType op_type, const std::vector& inputs, const std::vector& output_shape, const OpParams& params = {}); + const BufferDesc& get_output_buffer(size_t node_id) const; + void allocate_buffers(); + size_t get_node_count() const; + + size_t persistent(size_t source_node); + bool is_populated(size_t persistent_node_id) const; + void invalidate_persistent(size_t persistent_node_id); + + std::vector> nodes_; + std::unordered_map node_index_map_; + +private: + static CactusGraph from_serialized(const GraphFile::SerializedGraph& serialized); + size_t next_node_id_; + std::vector> mapped_files_; + std::unordered_map weight_cache_; + std::unordered_map node_to_mapped_file_; + std::vector debug_nodes_; + BufferPool buffer_pool_; + bool prefill_mode_ = false; + + std::unordered_set persistent_node_ids_; + std::unordered_set populated_node_ids_; +}; + + +namespace GraphFile { + struct LoadedNode { + size_t node_id; + std::vector shape; + Precision precision; + size_t byte_size; + }; + + struct GraphHeader { + uint32_t magic; + uint32_t version; + uint32_t node_count; + uint32_t flags = 0; + }; + + struct NodeEntry { + uint32_t index; // serialized node index 0..n-1 + OpType op_type; + std::vector inputs; + std::vector output_shape; + Precision precision; + OpParams params; + }; + + struct SerializedGraph { + GraphHeader header; + std::vector nodes; + std::vector graph_inputs; // IDs of serialized inputs + std::vector graph_outputs; // IDs of serialized outputs + }; + + SerializedGraph load_graph(const std::string& filename); + void save_graph(const CactusGraph& graph, const std::string& filename); + + void save_node(CactusGraph& graph, size_t node_id, const std::string& filename); + + class MappedFile { + public: + MappedFile(const std::string& filename); + ~MappedFile(); + + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + MappedFile(MappedFile&& other) noexcept; + MappedFile& operator=(MappedFile&& other) noexcept; + + const std::vector& shape() const; + Precision precision() const; + size_t byte_size() const; + + size_t group_size() const { return group_size_; } + size_t num_groups() const { return num_groups_; } + const void* scales_data() const; + + bool is_interleaved() const { return is_interleaved_; } + size_t original_N() const { return original_N_; } + + void* data(); + const void* data() const; + + template + const T* typed_data() const; + + void release_pages(); + void prefetch_pages(); + + private: + int fd_; + void* mapped_data_; + size_t file_size_, data_offset_; + std::vector shape_; + Precision precision_; + size_t byte_size_; + size_t group_size_ = 0; + size_t num_groups_ = 0; + size_t scales_offset_ = 0; + size_t scales_bytes_ = 0; + uint32_t alignment_ = 32; + + bool is_interleaved_ = false; + size_t original_N_ = 0; + + void parse_header(); + void apply_madvise_hints(); + }; +} + +#endif diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/graph_param_io.h b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/graph_param_io.h new file mode 100644 index 0000000..0362ea8 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/graph_param_io.h @@ -0,0 +1,11 @@ +#pragma once + +#include "graph.h" +#include + +namespace GraphParamIO { + +void write_op_params(std::ostream& out, OpType op_type, const OpParams& params); +void read_op_params(std::istream& in, OpType op_type, OpParams& params); + +} // namespace GraphParamIO diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/kernel.h b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/kernel.h new file mode 100644 index 0000000..13f4a00 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/kernel.h @@ -0,0 +1,475 @@ +#ifndef KERNEL_H +#define KERNEL_H + +#include +#include + +enum class Precision; + +enum class ScalarOpType { + ADD, + SUBTRACT, + MULTIPLY, + DIVIDE, + ABS, + EXP, + POW, + SQRT, + COS, + SIN, + LOG +}; + +constexpr size_t KV_QUANT_GROUP_SIZE = 32; + +void cactus_add_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_add_f16_clipped(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_subtract_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_multiply_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_add_scaled_f16(const __fp16* base, const __fp16* src, __fp16* output, size_t num_elements, float scale); +void cactus_divide_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); + +void cactus_add_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_subtract_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_multiply_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_divide_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); + +void cactus_scalar_op_f16(const __fp16* input, __fp16* output, size_t num_elements, float scalar_value, ScalarOpType op_type); + +void cactus_gemv_int8(const int8_t* A, float A_scale, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemm_int8(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_int8(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int8_i8mm(const int8_t* A, float A_scale, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemm_int8_i8mm(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int4(const int8_t* A, float A_scale, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int4_block_range(const int8_t* A, float A_scale, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size, + size_t block_start, size_t block_end); + +void cactus_gemm_int4(const int8_t* A, const float* A_scales, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_int4(const int8_t* A, const float* A_scales, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_integer(Precision precision, + const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_f16(const __fp16* a, const __fp16* b_transposed, __fp16* c, + size_t M, size_t K, size_t N); + +void cactus_transpose_2d_f16(const __fp16* source, __fp16* destination, + size_t num_rows, size_t num_cols, size_t start_row, size_t end_row); +void cactus_transpose_f16(const __fp16* source, __fp16* destination, const size_t* shape, + const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx); + +double cactus_sum_all_f16(const __fp16* data, size_t num_elements); +void cactus_sum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +double cactus_mean_all_f16(const __fp16* data, size_t num_elements); +void cactus_mean_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +double cactus_variance_all_f16(const __fp16* data, size_t num_elements); +void cactus_variance_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +__fp16 cactus_min_all_f16(const __fp16* data, size_t num_elements); +void cactus_min_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +__fp16 cactus_max_all_f16(const __fp16* data, size_t num_elements); +void cactus_max_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +void cactus_rms_norm_f16(const __fp16* input, const __fp16* weight, __fp16* output, + size_t batch_size, size_t dims, float eps); + +void cactus_layer_norm_f16(const __fp16* input, const __fp16* weight, const __fp16* bias, + __fp16* output, size_t batch_size, size_t dims, float eps); + +void cactus_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, + size_t num_heads, size_t head_dim, size_t start_pos, float theta); + +void cactus_gpt_j_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, + size_t num_heads, size_t head_dim, size_t rot_dim, size_t start_pos, float theta); + +void cactus_softmax_f16(const __fp16* input, __fp16* output, size_t batch_size, + size_t seq_len, size_t vocab_size); + +void cactus_relu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_leaky_relu_f16(const __fp16* input, __fp16* output, size_t num_elements, float negative_slope); + +void cactus_silu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_gelu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_gelu_f16_erf(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_sigmoid_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_tanh_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_glu_f16( + const __fp16* input, + __fp16* output, + size_t outer_size, + size_t split_size, + size_t inner_size +); + +void cactus_glu_f32( + const float* input, + float* output, + size_t outer_size, + size_t split_size, + size_t inner_size +); + +void cactus_batchnorm_f16( + const __fp16* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + __fp16* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +); + +void cactus_batchnorm_f32( + const float* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + float* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +); + +void cactus_attention_f16(const __fp16* queries, const __fp16* keys, const __fp16* values, __fp16* output, + size_t batch_size, size_t seq_len, size_t kv_seq_len, size_t num_q_heads, size_t num_kv_heads, + size_t head_dim, float scale, const __fp16* mask, size_t position_offset = 0, size_t window_size = 0, + bool is_causal = true, bool mask_is_additive = false, bool mask_per_head = false, + size_t v_head_dim = 0, float logit_cap = 0.0f); + +void cactus_attention_hybrid_int8_fp16( + const __fp16* queries, + const int8_t* keys_cached, + const int8_t* values_cached, + const float* k_scales, + const float* v_scales, + const __fp16* keys_new, + const __fp16* values_new, + __fp16* output, + size_t batch_size, size_t seq_len, size_t cache_len, size_t new_len, + size_t num_q_heads, size_t num_kv_heads, size_t head_dim, + float scale, size_t position_offset = 0, bool is_causal = true, size_t window_size = 0, + size_t group_size = KV_QUANT_GROUP_SIZE, size_t v_head_dim = 0); + +void cactus_gated_deltanet_decode_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + float scale); + +void cactus_gated_deltanet_prefill_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t T, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + size_t requested_chunk_size, + float scale); + +void cactus_conv1d_causal_depthwise_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, + size_t L, + size_t C, + size_t K, + size_t dilation); + +void cactus_conv1d_f16_k3( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out, + size_t stride +); + +void cactus_conv1d_f16( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out, + size_t K, + size_t stride +); + +void cactus_stft_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, size_t L, + size_t C_in, size_t C_out, + size_t K, size_t stride, + size_t num_fft_bins +); + +void cactus_conv1d_f16_k7s3_oc8( + const __fp16* input, + const __fp16* Wpack, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out +); + +void cactus_conv1d_same_depthwise_f16_k9( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C +); + +void cactus_conv2d_f16_k3s1p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv2d_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv2d_depthwise_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C, + size_t H, + size_t W +); + +void cactus_conv2d_pointwise_f16_1x1_nchw_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv1d_pointwise_f16_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out +); + +void cactus_bilinear_interpolation_f16(const __fp16* input, __fp16* output, size_t src_height, size_t src_width, size_t embed_dim, + size_t dst_height, size_t dst_width, bool align_corners = true); + +void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); +void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); + +void cactus_sample_f32_ex(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); +void cactus_sample_f16_ex(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); + +void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* output, + const size_t* shape1, const size_t* shape2, const size_t* output_shape, + size_t ndims, int axis); +void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_shapes, + const size_t* output_shape, size_t num_inputs, size_t rank, int axis); + +void cactus_int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); +void cactus_fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); +void cactus_fp16_to_fp32(const __fp16* src, float* dst, size_t count); +void cactus_fp32_to_fp16(const float* src, __fp16* dst, size_t count); +void cactus_int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); +void cactus_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); +float cactus_fp16_max_abs(const __fp16* src, size_t count); + +void cactus_quantize_kv_fp16_to_int8( + const __fp16* src, + int8_t* dst, + float* scales, + size_t seq_len, size_t kv_heads, size_t head_dim, + size_t group_size = KV_QUANT_GROUP_SIZE); + +inline size_t kv_scales_count(size_t seq_len, size_t kv_heads, size_t head_dim, size_t group_size = KV_QUANT_GROUP_SIZE) { + size_t num_groups = (head_dim + group_size - 1) / group_size; + return seq_len * kv_heads * num_groups; +} + +void cactus_unpack_int4_to_int8(const uint8_t* packed, int8_t* unpacked, size_t unpacked_count); + +void cactus_gaussian_topk_f16( + const __fp16* input, + __fp16* output, + size_t rows, + size_t cols, + float ppf); + +void cactus_altup_predict_f16( + const __fp16* coefs, + const __fp16* const* streams, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim); + +void cactus_altup_correct_f16( + const __fp16* coefs, + const __fp16* innovation, + const __fp16* const* predictions, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim); + +void cactus_lstm_cell_f16( + const __fp16* x_input, + const __fp16* h_prev, + const __fp16* c_prev, + const __fp16* weight_ih, + const __fp16* weight_hh, + const __fp16* bias_ih, + const __fp16* bias_hh, + __fp16* h_new, + __fp16* c_new, + size_t batch_size, + size_t input_size, + size_t hidden_size +); + +void cactus_bilstm_sequence_f16( + const __fp16* input, + const __fp16* weight_ih_fwd, + const __fp16* weight_hh_fwd, + const __fp16* bias_ih_fwd, + const __fp16* bias_hh_fwd, + const __fp16* weight_ih_bwd, + const __fp16* weight_hh_bwd, + const __fp16* bias_ih_bwd, + const __fp16* bias_hh_bwd, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t input_size, + size_t hidden_size +); + +void cactus_maxpool1d_f16( + const __fp16* input, + __fp16* output, + size_t batch_size, + size_t channels, + size_t input_length, + size_t kernel_size, + size_t stride +); + +#endif diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/kernel_utils.h b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/kernel_utils.h new file mode 100644 index 0000000..b202158 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/kernel_utils.h @@ -0,0 +1,681 @@ +#ifndef KERNEL_UTILS_H +#define KERNEL_UTILS_H + +#include +#if defined(__APPLE__) +#include +#include +#endif +#if defined(__ANDROID__) +#include +#include +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr size_t NEON_VECTOR_SIZE = 16; +constexpr size_t STREAMING_STORE_THRESHOLD = 32768; + +inline void stream_store_f16x8(__fp16* dst, float16x8_t val) { +#if defined(__aarch64__) + float16x4_t lo = vget_low_f16(val); + float16x4_t hi = vget_high_f16(val); + __asm__ __volatile__( + "stnp %d0, %d1, [%2]" + : + : "w"(lo), "w"(hi), "r"(dst) + : "memory" + ); +#else + vst1q_f16(dst, val); +#endif +} + +inline bool cpu_has_i8mm() { +#if defined(__aarch64__) + static std::once_flag once; + static bool has = false; + + std::call_once(once, []() { +#if defined(__APPLE__) + int ret = 0; + size_t size = sizeof(ret); + if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &ret, &size, nullptr, 0) == 0) { + has = (ret == 1); + } +#elif defined(__ANDROID__) + unsigned long hwcap2 = getauxval(AT_HWCAP2); + #ifndef HWCAP2_I8MM + #define HWCAP2_I8MM (1 << 13) + #endif + has = (hwcap2 & HWCAP2_I8MM) != 0; +#endif + }); + + return has; +#else + return false; +#endif +} + +inline float32x4_t fast_exp_f32x4(float32x4_t x) { + const float32x4_t log2e = vdupq_n_f32(1.4426950408889634f); + const float32x4_t ln2 = vdupq_n_f32(0.6931471805599453f); + + const float32x4_t c0 = vdupq_n_f32(1.0f); + const float32x4_t c1 = vdupq_n_f32(0.6931471805599453f); + const float32x4_t c2 = vdupq_n_f32(0.2402265069591007f); + const float32x4_t c3 = vdupq_n_f32(0.05550410866482158f); + const float32x4_t c4 = vdupq_n_f32(0.009618129842071803f); + + x = vmaxq_f32(x, vdupq_n_f32(-87.0f)); + x = vminq_f32(x, vdupq_n_f32(87.0f)); + + float32x4_t z = vmulq_f32(x, log2e); + + int32x4_t zi = vcvtq_s32_f32(z); + float32x4_t zf = vsubq_f32(z, vcvtq_f32_s32(zi)); + + uint32x4_t neg_mask = vcltq_f32(zf, vdupq_n_f32(0.0f)); + zi = vsubq_s32(zi, vandq_s32(vreinterpretq_s32_u32(neg_mask), vdupq_n_s32(1))); + zf = vaddq_f32(zf, vreinterpretq_f32_u32(vandq_u32(neg_mask, vreinterpretq_u32_f32(vdupq_n_f32(1.0f))))); + + float32x4_t zf_ln2 = vmulq_f32(zf, ln2); + float32x4_t p = c4; + p = vfmaq_f32(c3, p, zf_ln2); + p = vfmaq_f32(c2, p, zf_ln2); + p = vfmaq_f32(c1, p, zf_ln2); + p = vfmaq_f32(c0, p, zf_ln2); + + int32x4_t exp_bits = vshlq_n_s32(vaddq_s32(zi, vdupq_n_s32(127)), 23); + float32x4_t scale = vreinterpretq_f32_s32(exp_bits); + + return vmulq_f32(p, scale); +} + +// Cephes-style 13/6 rational tanh approximation (same coefficients as Eigen). +// Constants are stored as static splatted arrays so the compiler emits a single +// pc-relative `ldr q` per load. +alignas(16) inline constexpr float kFastTanhAlpha[7][4] = { + { 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f }, + { 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f }, + { 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f }, + { 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f }, + {-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f }, + { 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f }, + {-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f }, +}; +alignas(16) inline constexpr float kFastTanhBeta[4][4] = { + { 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f }, + { 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f }, + { 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f }, + { 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f }, +}; +alignas(16) inline constexpr float kFastTanhClampHi[4] = { 9.0f, 9.0f, 9.0f, 9.0f }; +alignas(16) inline constexpr float kFastTanhClampLo[4] = {-9.0f,-9.0f,-9.0f,-9.0f }; + +inline float32x4_t fast_tanh_f32x4(float32x4_t x) { + x = vmaxq_f32(vld1q_f32(kFastTanhClampLo), vminq_f32(vld1q_f32(kFastTanhClampHi), x)); + float32x4_t x2 = vmulq_f32(x, x); + float32x4_t p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[5]), vld1q_f32(kFastTanhAlpha[6]), x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[4]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[3]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[2]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[1]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[0]), p, x2); + p = vmulq_f32(p, x); + float32x4_t q = vfmaq_f32(vld1q_f32(kFastTanhBeta[2]), vld1q_f32(kFastTanhBeta[3]), x2); + q = vfmaq_f32(vld1q_f32(kFastTanhBeta[1]), q, x2); + q = vfmaq_f32(vld1q_f32(kFastTanhBeta[0]), q, x2); + return vdivq_f32(p, q); +} + +constexpr size_t SIMD_F16_WIDTH = 8; + +inline size_t simd_align(size_t count, size_t width = SIMD_F16_WIDTH) { + return (count / width) * width; +} + +inline void f16x8_split_f32(float16x8_t v, float32x4_t& lo, float32x4_t& hi) { + lo = vcvt_f32_f16(vget_low_f16(v)); + hi = vcvt_f32_f16(vget_high_f16(v)); +} + +inline float16x8_t f32_merge_f16(float32x4_t lo, float32x4_t hi) { + return vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi)); +} + +inline float32x4_t fast_sigmoid_f32x4(float32x4_t x) { + const float32x4_t one = vdupq_n_f32(1.0f); + return vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(x)))); +} + +template +inline float16x8_t apply_f32_op_on_f16x8(float16x8_t v, F32x4Op op) { + float32x4_t lo, hi; + f16x8_split_f32(v, lo, hi); + return f32_merge_f16(op(lo), op(hi)); +} + +inline void unpack_int4_as_int8x16x2(const uint8_t* ptr, int8x16_t& high_decoded, int8x16_t& low_decoded) { + int8x16_t packed = vreinterpretq_s8_u8(vld1q_u8(ptr)); + high_decoded = vshrq_n_s8(packed, 4); + low_decoded = vshrq_n_s8(vshlq_n_s8(packed, 4), 4); +} + +namespace CactusThreading { + +#if defined(__ANDROID__) + struct CoreTopology { + std::vector performance_cores; + std::vector all_cores; + + static CoreTopology& get() { + static CoreTopology topo = detect(); + return topo; + } + + private: + static int read_sysfs_int(const char* path) { + std::ifstream f(path); + if (!f.is_open()) return -1; + int val = -1; + f >> val; + return val; + } + + static CoreTopology detect() { + CoreTopology topo; + constexpr int MAX_CPUS = 16; + std::vector> core_caps; + + for (int i = 0; i < MAX_CPUS; ++i) { + char path[128]; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpu_capacity", i); + int cap = read_sysfs_int(path); + if (cap > 0) { + core_caps.push_back({i, cap}); + topo.all_cores.push_back(i); + continue; + } + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", i); + int freq = read_sysfs_int(path); + if (freq > 0) { + core_caps.push_back({i, freq}); + topo.all_cores.push_back(i); + } + } + + if (core_caps.empty()) return topo; + + int max_cap = 0; + for (auto& [id, cap] : core_caps) { + max_cap = std::max(max_cap, cap); + } + + int threshold = static_cast(max_cap * 0.70); + for (auto& [id, cap] : core_caps) { + if (cap >= threshold) { + topo.performance_cores.push_back(id); + } + } + + return topo; + } + }; + + inline bool pin_current_thread_to_cores(const std::vector& cores) { + if (cores.empty()) return false; + cpu_set_t mask; + CPU_ZERO(&mask); + for (int core : cores) { + CPU_SET(core, &mask); + } + return sched_setaffinity(0, sizeof(mask), &mask) == 0; + } +#endif + + class ThreadPool { + private: + static constexpr size_t MAX_WORKERS = 16; + + std::vector workers; + std::deque> tasks; + + std::mutex mutex; + std::condition_variable work_available; + std::condition_variable work_done; + + bool stop{false}; + std::atomic pending_tasks{0}; + size_t num_workers_; + + void worker_thread() { + while (true) { + std::function task; + { + std::unique_lock lock(mutex); + work_available.wait(lock, [this] { + return stop || !tasks.empty(); + }); + + if (stop && tasks.empty()) { + return; + } + + task = std::move(tasks.front()); + tasks.pop_front(); + } + + task(); + + if (pending_tasks.fetch_sub(1, std::memory_order_acq_rel) == 1) { + std::lock_guard lock(mutex); + work_done.notify_one(); + } + } + } + + public: + explicit ThreadPool(size_t num_threads = std::thread::hardware_concurrency()) + : stop(false), pending_tasks(0) { + num_workers_ = std::min(num_threads, MAX_WORKERS); + if (num_workers_ == 0) num_workers_ = 1; + +#if defined(__ANDROID__) + auto& topo = CoreTopology::get(); + if (!topo.performance_cores.empty()) { + num_workers_ = std::min(num_workers_, topo.performance_cores.size()); + } +#endif + + workers.reserve(num_workers_); + for (size_t i = 0; i < num_workers_; ++i) { + workers.emplace_back([this]() { +#if defined(__ANDROID__) + auto& perf = CoreTopology::get().performance_cores; + if (!perf.empty()) { + pin_current_thread_to_cores(perf); + } +#endif + worker_thread(); + }); + } + } + + ~ThreadPool() { + { + std::lock_guard lock(mutex); + stop = true; + } + work_available.notify_all(); + for (auto& worker : workers) { + if (worker.joinable()) { + worker.join(); + } + } + } + + template + auto enqueue(F&& f) -> std::future { + using return_type = decltype(f()); + + auto task = std::make_shared>( + std::forward(f) + ); + + std::future res = task->get_future(); + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(1, std::memory_order_relaxed); + tasks.emplace_back([task](){ (*task)(); }); + } + work_available.notify_one(); + + return res; + } + + template + void enqueue_batch(size_t total_work, F task_func) { + if (total_work == 0) return; + + const size_t num_tasks = std::min(num_workers_, total_work); + const size_t per_worker = total_work / num_tasks; + const size_t remainder = total_work % num_tasks; + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(num_tasks, std::memory_order_relaxed); + + for (size_t w = 0; w < num_tasks; ++w) { + size_t start = w * per_worker + std::min(w, remainder); + size_t end = start + per_worker + (w < remainder ? 1 : 0); + tasks.emplace_back([=]() { task_func(start, end); }); + } + } + work_available.notify_all(); + } + + void wait_all() { + std::unique_lock lock(mutex); + work_done.wait(lock, [this] { + return pending_tasks.load(std::memory_order_acquire) == 0; + }); + } + + template + void enqueue_n_threads(size_t total_work, size_t num_threads, F task_func) { + if (total_work == 0 || num_threads == 0) return; + + num_threads = std::min(num_threads, std::min(num_workers_, total_work)); + const size_t per_thread = total_work / num_threads; + const size_t remainder = total_work % num_threads; + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(num_threads, std::memory_order_relaxed); + + for (size_t t = 0; t < num_threads; ++t) { + size_t start = t * per_thread + std::min(t, remainder); + size_t end = start + per_thread + (t < remainder ? 1 : 0); + tasks.emplace_back([=]() { task_func(start, end); }); + } + } + work_available.notify_all(); + } + + size_t num_workers() const { return num_workers_; } + }; + + inline ThreadPool& get_thread_pool() { + static ThreadPool pool; + return pool; + } + + struct ParallelConfig { + size_t min_work_gate; + size_t work_per_thread; + + constexpr ParallelConfig(size_t gate, size_t per_thread) + : min_work_gate(gate), work_per_thread(per_thread) {} + }; + + inline size_t get_optimal_thread_count(size_t total_work, ParallelConfig config) { + if (total_work < config.min_work_gate) return 1; + + size_t pool_size = get_thread_pool().num_workers(); + size_t num_threads = (total_work + config.work_per_thread - 1) / config.work_per_thread; + return std::min(pool_size, std::max(static_cast(1), num_threads)); + } + + struct Thresholds { + #if defined(__ANDROID__) + static constexpr ParallelConfig ATTENTION{64, 32}; + static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; + static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; + static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; + static constexpr ParallelConfig SCALAR_BASIC{30000, 15000}; + static constexpr ParallelConfig SCALAR_EXPENSIVE{10000, 5000}; + #else // Apple + static constexpr ParallelConfig ATTENTION{32, 16}; + static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; + static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; + static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; + static constexpr ParallelConfig SCALAR_BASIC{5000, 2500}; + static constexpr ParallelConfig SCALAR_EXPENSIVE{2500, 1250}; + #endif + }; + + struct GemmThreading { + #if defined(__ANDROID__) + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return 1; + return pool_size; + } + static size_t get_gemv_threads(size_t /*N_blocks*/, size_t /*pool_size*/) { + return 1; + } + #elif defined(__APPLE__) && TARGET_OS_IPHONE + static constexpr size_t GEMV_MIN_N_BLOCKS = 512; + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return std::min(pool_size, static_cast(2)); + return pool_size; + } + static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { + if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; + return std::min(pool_size, static_cast(3)); + } + #else + static constexpr size_t GEMV_MIN_N_BLOCKS = 256; + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return std::min(pool_size, static_cast(4)); + return pool_size; + } + static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { + if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; + if (N_blocks < 512) return std::min(pool_size, static_cast(2)); + return std::min(pool_size, static_cast(5)); + } + #endif + }; + + inline size_t& get_gemm_thread_override() { + static size_t override_threads = 0; + return override_threads; + } + + inline void set_gemm_threads(size_t num_threads) { + get_gemm_thread_override() = num_threads; + } + + inline void reset_gemm_threads() { + get_gemm_thread_override() = 0; + } + + class TaskHandle { + private: + std::vector> futures_; + bool auto_wait_; + + public: + TaskHandle(bool auto_wait = true) : auto_wait_(auto_wait) {} + + ~TaskHandle() { + if (auto_wait_) { + wait(); + } + } + + TaskHandle(TaskHandle&&) = default; + TaskHandle& operator=(TaskHandle&&) = default; + TaskHandle(const TaskHandle&) = delete; + TaskHandle& operator=(const TaskHandle&) = delete; + + void add_future(std::future&& f) { + futures_.push_back(std::move(f)); + } + + void wait() { + for (auto& f : futures_) { + if (f.valid()) { + f.wait(); + } + } + futures_.clear(); + } + + bool is_ready() const { + for (const auto& f : futures_) { + if (f.valid() && f.wait_for(std::chrono::seconds(0)) != std::future_status::ready) { + return false; + } + } + return true; + } + + size_t task_count() const { return futures_.size(); } + }; + + template + TaskHandle parallel_for(size_t total_work, ParallelConfig config, WorkFunc work_func, bool wait = true) { + const size_t num_threads = get_optimal_thread_count(total_work, config); + TaskHandle handle(!wait); + + if (num_threads == 1) { + if (wait) { + work_func(0, total_work); + return handle; + } + auto& pool = get_thread_pool(); + handle.add_future(pool.enqueue([work_func, total_work]() { + work_func(0, total_work); + })); + return handle; + } + + auto& pool = get_thread_pool(); + const size_t work_per_thread = total_work / num_threads; + + for (size_t t = 0; t < num_threads; ++t) { + handle.add_future(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() { + const size_t start_idx = t * work_per_thread; + const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; + work_func(start_idx, end_idx); + })); + } + + if (wait) { + handle.wait(); + } + return handle; + } + + template + void parallel_for_2d(size_t outer_size, size_t inner_size, ParallelConfig config, WorkFunc work_func) { + const size_t total_work = outer_size * inner_size; + parallel_for(total_work, config, [&](size_t start_idx, size_t end_idx) { + for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { + const size_t outer = work_idx / inner_size; + const size_t inner = work_idx % inner_size; + work_func(outer, inner); + } + }); + } + + template + ResultType parallel_reduce(size_t total_work, ParallelConfig config, + WorkFunc work_func, ResultType init_value, CombineFunc combine_func) { + const size_t num_threads = get_optimal_thread_count(total_work, config); + + if (num_threads == 1) { + return work_func(0, total_work); + } + + auto& pool = get_thread_pool(); + std::vector> futures; + std::vector partial_results(num_threads, init_value); + const size_t work_per_thread = total_work / num_threads; + + for (size_t t = 0; t < num_threads; ++t) { + futures.push_back(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() -> ResultType { + const size_t start_idx = t * work_per_thread; + const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; + return work_func(start_idx, end_idx); + })); + } + + ResultType result = init_value; + for (auto& future : futures) { + result = combine_func(result, future.get()); + } + return result; + } + + template + void parallel_gemm_tiles(size_t M, size_t total_tiles, WorkFunc work_func) { + auto& pool = get_thread_pool(); + + size_t override = get_gemm_thread_override(); + size_t num_threads = (override > 0) ? override : GemmThreading::get_num_threads(M, pool.num_workers()); + num_threads = std::min(num_threads, total_tiles); + + if (num_threads <= 1) { + work_func(0, total_tiles); + return; + } + + pool.enqueue_n_threads(total_tiles, num_threads, work_func); + pool.wait_all(); + } + +} + +template +void elementwise_op_f16(const __fp16* input, __fp16* output, size_t num_elements, + bool use_streaming, CactusThreading::ParallelConfig config, + SimdOp simd_op, ScalarOp scalar_op, size_t unroll = 4) { + CactusThreading::parallel_for(num_elements, config, + [&](size_t start, size_t end) { + const size_t n = end - start; + const size_t vec_end = start + simd_align(n); + + if (use_streaming && unroll >= 4) { + const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 4); + for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 4) { + __builtin_prefetch(&input[i + 256], 0, 0); + float16x8_t v0 = simd_op(vld1q_f16(&input[i])); + float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); + float16x8_t v2 = simd_op(vld1q_f16(&input[i + 16])); + float16x8_t v3 = simd_op(vld1q_f16(&input[i + 24])); + stream_store_f16x8(&output[i], v0); + stream_store_f16x8(&output[i + 8], v1); + stream_store_f16x8(&output[i + 16], v2); + stream_store_f16x8(&output[i + 24], v3); + } + for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { + stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } else if (use_streaming && unroll >= 2) { + const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 2); + for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 2) { + __builtin_prefetch(&input[i + 128], 0, 0); + float16x8_t v0 = simd_op(vld1q_f16(&input[i])); + float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); + stream_store_f16x8(&output[i], v0); + stream_store_f16x8(&output[i + 8], v1); + } + for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { + stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } else { + for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { + vst1q_f16(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } + for (size_t i = vec_end; i < end; ++i) { + output[i] = scalar_op(input[i]); + } + }); +} + +#endif // KERNEL_UTILS_H diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/needle_tools.h b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/needle_tools.h new file mode 100644 index 0000000..4330cff --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/needle_tools.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace needle { + +inline std::string to_snake_case(const std::string& name) { + std::string s; + s.reserve(name.size() * 2); + for (char c : name) { + if (std::isalnum(static_cast(c)) || c == '_') { + s += c; + } else { + if (s.empty() || s.back() != '_') s += '_'; + } + } + + std::string s2; + s2.reserve(s.size() * 2); + for (size_t i = 0; i < s.size(); i++) { + char c = s[i]; + if (i > 0 && std::isupper(static_cast(c))) { + char prev = s[i - 1]; + if (std::islower(static_cast(prev)) || std::isdigit(static_cast(prev))) { + s2 += '_'; + } + } + s2 += c; + } + + std::string s3; + s3.reserve(s2.size() * 2); + for (size_t i = 0; i < s2.size(); i++) { + s3 += s2[i]; + if (i + 1 < s2.size() && + std::isupper(static_cast(s2[i])) && + std::isupper(static_cast(s2[i + 1]))) { + if (i + 2 < s2.size() && std::islower(static_cast(s2[i + 2]))) { + s3 += '_'; + } + } + } + + std::string result; + result.reserve(s3.size()); + bool prev_underscore = false; + for (char c : s3) { + if (c == '_') { + if (!prev_underscore) result += '_'; + prev_underscore = true; + } else { + result += static_cast(std::tolower(static_cast(c))); + prev_underscore = false; + } + } + + size_t start = result.find_first_not_of('_'); + if (start == std::string::npos) return result; + size_t end = result.find_last_not_of('_'); + return result.substr(start, end - start + 1); +} + +inline void restore_tool_names(std::vector& function_calls, + const std::unordered_map& name_map) { + if (name_map.empty()) return; + for (auto& call : function_calls) { + for (const auto& [snake, orig] : name_map) { + std::string from = "\"name\":\"" + snake + "\""; + size_t pos = call.find(from); + if (pos == std::string::npos) { + from = "\"name\": \"" + snake + "\""; + pos = call.find(from); + } + if (pos != std::string::npos) { + std::string to = from.substr(0, from.size() - snake.size() - 1) + orig + "\""; + call.replace(pos, from.size(), to); + break; + } + } + } +} + +} // namespace needle diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Info.plist b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Info.plist new file mode 100644 index 0000000..eead57b Binary files /dev/null and b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Info.plist differ diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/_CodeSignature/CodeResources b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/_CodeSignature/CodeResources new file mode 100644 index 0000000..70da95d --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/_CodeSignature/CodeResources @@ -0,0 +1,101 @@ + + + + + files + + Info.plist + + wPyLfOfBtc39RdGXZwsNMxGnEQg= + + + files2 + + rules + + ^.* + + ^.*\.lproj/ + + optional + + weight + 1000 + + ^.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Base\.lproj/ + + weight + 1010 + + ^version.plist$ + + + rules2 + + .*\.dSYM($|/) + + weight + 11 + + ^(.*/)?\.DS_Store$ + + omit + + weight + 2000 + + ^.* + + ^.*\.lproj/ + + optional + + weight + 1000 + + ^.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Base\.lproj/ + + weight + 1010 + + ^Info\.plist$ + + omit + + weight + 20 + + ^PkgInfo$ + + omit + + weight + 20 + + ^embedded\.provisionprofile$ + + weight + 20 + + ^version\.plist$ + + weight + 20 + + + + diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/cactus b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/cactus new file mode 100755 index 0000000..b8c8004 Binary files /dev/null and b/ios/Runner/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/cactus differ diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus.h b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus.h new file mode 100644 index 0000000..b5b6d80 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus.h @@ -0,0 +1,12 @@ +#ifndef CACTUS_H +#define CACTUS_H + +#include "graph/graph.h" +#include "kernel/kernel.h" +#include "kernel/kernel_utils.h" +#include "engine/engine.h" +#include "models/model.h" +#include "ffi/cactus_ffi.h" +#include "npu/npu.h" + +#endif // CACTUS_H \ No newline at end of file diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_cloud.h b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_cloud.h new file mode 100644 index 0000000..6911d9a --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_cloud.h @@ -0,0 +1,50 @@ +#ifndef CACTUS_CLOUD_H +#define CACTUS_CLOUD_H + +#include "cactus_utils.h" +#include +#include + +namespace cactus { +namespace ffi { + +struct CloudResponse { + std::string transcript; + std::string api_key_hash; + bool used_cloud = false; + std::string error; +}; + +struct CloudCompletionRequest { + std::vector messages; + std::vector tools; + std::string local_output; + std::vector local_function_calls; + bool has_images = false; + bool has_audio = false; + std::vector audio_pcm; + std::string cloud_key; +}; + +struct CloudCompletionResult { + bool ok = false; + bool used_cloud = false; + std::string response; + std::vector function_calls; + std::string error; +}; + +std::string cloud_base64_encode(const uint8_t* data, size_t len); +std::vector cloud_build_wav(const uint8_t* pcm, size_t pcm_bytes); +std::string resolve_cloud_api_key(const char* cloud_key_param); +CloudResponse cloud_transcribe_request(const std::string& audio_b64, + const std::string& fallback_text, + long timeout_seconds = 15L, + const char* cloud_key = nullptr); +CloudCompletionResult cloud_complete_request(const CloudCompletionRequest& request, + long timeout_ms); + +} // namespace ffi +} // namespace cactus + +#endif // CACTUS_CLOUD_H diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_ffi.h b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_ffi.h new file mode 100644 index 0000000..163e180 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_ffi.h @@ -0,0 +1,454 @@ +#ifndef CACTUS_FFI_H +#define CACTUS_FFI_H + +#include +#include +#include + +#if __GNUC__ >= 4 + #define CACTUS_FFI_EXPORT __attribute__((visibility("default"))) + #define CACTUS_FFI_LOCAL __attribute__((visibility("hidden"))) +#else + #define CACTUS_FFI_EXPORT + #define CACTUS_FFI_LOCAL +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* cactus_model_t; +typedef void* cactus_index_t; +typedef void* cactus_stream_transcribe_t; + +typedef void (*cactus_token_callback)(const char* token, uint32_t token_id, void* user_data); + +CACTUS_FFI_EXPORT cactus_model_t cactus_init( + const char* model_path, + const char* corpus_dir, // optional: NULL if no RAG corpus + bool cache_index // false = always rebuild index, true = load cached if available +); + +CACTUS_FFI_EXPORT void cactus_destroy(cactus_model_t model); +CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model); +CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model); + +CACTUS_FFI_EXPORT int cactus_complete( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const char* tools_json, // optional + cactus_token_callback callback, // optional + void* user_data, // optional + const uint8_t* pcm_buffer, // optional: NULL when not used + size_t pcm_buffer_size // optional: 0 when not used +); + +CACTUS_FFI_EXPORT int cactus_prefill( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const char* tools_json, // optional + const uint8_t* pcm_buffer, // optional: NULL when not used + size_t pcm_buffer_size // optional: 0 when not used +); + +CACTUS_FFI_EXPORT int cactus_tokenize( + cactus_model_t model, + const char* text, + uint32_t* token_buffer, + size_t token_buffer_len, + size_t* out_token_len +); + +CACTUS_FFI_EXPORT int cactus_score_window( + cactus_model_t model, + const uint32_t* tokens, + size_t token_len, + size_t start, + size_t end, + size_t context, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_transcribe( + cactus_model_t model, + const char* audio_file_path, // NULL if using pcm_buffer + const char* prompt, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + cactus_token_callback callback, // optional + void* user_data, // optional + const uint8_t* pcm_buffer, // NULL if using audio_file_path + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_detect_language( + cactus_model_t model, + const char* audio_file_path, // NULL if using pcm_buffer + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const uint8_t* pcm_buffer, // NULL if using audio_file_path + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT cactus_stream_transcribe_t cactus_stream_transcribe_start( + cactus_model_t model, + const char* options_json // optional +); + +CACTUS_FFI_EXPORT int cactus_stream_transcribe_process( + cactus_stream_transcribe_t stream, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_stream_transcribe_stop( + cactus_stream_transcribe_t stream, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_embed( + cactus_model_t model, + const char* text, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim, + bool normalize +); + +CACTUS_FFI_EXPORT int cactus_image_embed( + cactus_model_t model, + const char* image_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_audio_embed( + cactus_model_t model, + const char* audio_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_vad( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_diarize( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_embed_speaker( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + const float* mask_weights, + size_t mask_num_frames +); + +CACTUS_FFI_EXPORT int cactus_rag_query( + cactus_model_t model, + const char* query, + char* response_buffer, + size_t buffer_size, + size_t top_k +); + +CACTUS_FFI_EXPORT cactus_index_t cactus_index_init( + const char* index_dir, + size_t embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_index_add( + cactus_index_t index, + const int* ids, + const char** documents, + const char** metadatas, // optional: can be NULL + const float** embeddings, + size_t count, + size_t embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_index_delete( + cactus_index_t index, + const int* ids, + size_t ids_count +); + +CACTUS_FFI_EXPORT int cactus_index_get( + cactus_index_t index, + const int* ids, + size_t ids_count, + char** document_buffers, + size_t* document_buffer_sizes, + char** metadata_buffers, + size_t* metadata_buffer_sizes, + float** embedding_buffers, + size_t* embedding_buffer_sizes +); + +CACTUS_FFI_EXPORT int cactus_index_query( + cactus_index_t index, + const float** embeddings, + size_t embeddings_count, + size_t embedding_dim, + const char* options_json, // optional + int** id_buffers, + size_t* id_buffer_sizes, + float** score_buffers, + size_t* score_buffer_sizes +); + +CACTUS_FFI_EXPORT int cactus_index_compact(cactus_index_t index); +CACTUS_FFI_EXPORT void cactus_index_destroy(cactus_index_t index); + +CACTUS_FFI_EXPORT const char* cactus_get_last_error(void); + +// level: 0=DEBUG, 1=INFO, 2=WARN (default), 3=ERROR, 4=NONE +CACTUS_FFI_EXPORT void cactus_log_set_level(int level); + +typedef void (*cactus_log_callback_t)(int level, const char* component, const char* message, void* user_data); +CACTUS_FFI_EXPORT void cactus_log_set_callback(cactus_log_callback_t callback, void* user_data); + +CACTUS_FFI_EXPORT void cactus_set_telemetry_environment(const char* framework, const char* cache_location, const char* version); +CACTUS_FFI_EXPORT void cactus_set_app_id(const char* app_id); +CACTUS_FFI_EXPORT void cactus_telemetry_flush(void); +CACTUS_FFI_EXPORT void cactus_telemetry_shutdown(void); + +// cactus graph export +typedef void* cactus_graph_t; +typedef uint64_t cactus_node_t; + +typedef struct { + int32_t precision; + size_t rank; + size_t shape[8]; + size_t num_elements; + size_t byte_size; +} cactus_tensor_info_t; + +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_create(void); +CACTUS_FFI_EXPORT void cactus_graph_destroy(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_hard_reset(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_save(cactus_graph_t graph, const char* filename); +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_load(const char* filename); + +CACTUS_FFI_EXPORT int cactus_graph_input( + cactus_graph_t graph, const size_t* shape, size_t rank, int32_t precision, +cactus_node_t* out_node); + +CACTUS_FFI_EXPORT int cactus_graph_set_input( + cactus_graph_t graph, cactus_node_t node, const void* data, int32_t +precision); +CACTUS_FFI_EXPORT int cactus_graph_set_external_input( + cactus_graph_t graph, cactus_node_t node, void* data, int32_t precision); + +CACTUS_FFI_EXPORT int cactus_graph_precision_cast( + cactus_graph_t graph, cactus_node_t input, int32_t target_precision, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_quantize_activations( + cactus_graph_t graph, cactus_node_t input, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_add(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_add_clipped(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_subtract(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_multiply(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_divide(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_scalar_add(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_subtract(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_multiply(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_exp(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sqrt(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_cos(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sin(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_log(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_abs(cactus_graph_t graph, cactus_node_t x, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_pow(cactus_graph_t graph, cactus_node_t x, +float exponent, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_view( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_flatten( + cactus_graph_t graph, cactus_node_t x, int32_t start_dim, int32_t end_dim, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_reshape( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose( + cactus_graph_t graph, cactus_node_t x, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose_n( + cactus_graph_t graph, cactus_node_t x, const size_t* permutation, size_t rank, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_slice( + cactus_graph_t graph, cactus_node_t x, int32_t axis, size_t start, size_t length, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_index( + cactus_graph_t graph, cactus_node_t x, size_t index_value, int32_t dim, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_concat( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, +cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_cat( + cactus_graph_t graph, const cactus_node_t* nodes, size_t count, int32_t +axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_matmul( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, bool pretransposed_rhs, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gather( + cactus_graph_t graph, cactus_node_t tensor, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_tensor( + cactus_graph_t graph, cactus_node_t embedding_tensor, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_file( + cactus_graph_t graph, const char* filename, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_embeddings( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_weights( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_bilinear_interpolation( + cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_set_grouped_scales( + cactus_graph_t graph, cactus_node_t node, size_t group_size, size_t num_groups, void* scales_ptr); +CACTUS_FFI_EXPORT int cactus_graph_set_interleaved( + cactus_graph_t graph, cactus_node_t node, bool interleaved, size_t original_n); +CACTUS_FFI_EXPORT int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_prefetch_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_release_all_weight_pages(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_relu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_silu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu_erf(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sigmoid(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_tanh(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_glu(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_layernorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, float epsilon, bool has_bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_groupnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, size_t num_groups, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_batchnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t running_mean, cactus_node_t running_var, int32_t axis, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_topk(cactus_graph_t graph, cactus_node_t input, size_t k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rms_norm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope_gptj( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, size_t rot_dim, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_softmax(cactus_graph_t graph, cactus_node_t input, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, float scale, bool is_causal, size_t position_offset, size_t window_size, int32_t backend, bool use_mask, cactus_node_t mask, bool additive_mask, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rel_pos_bias( + cactus_graph_t graph, cactus_node_t query, cactus_node_t relative_key, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention_int8_hybrid( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_conv1d_causal( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t kernel_size, size_t dilation, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k7s3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_same_depthwise_k9( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_pointwise( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_depthwise_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_pointwise_1x1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_lstm_cell( + cactus_graph_t graph, cactus_node_t input, cactus_node_t h_prev, cactus_node_t c_prev, cactus_node_t weight_ih, cactus_node_t weight_hh, cactus_node_t bias_ih, cactus_node_t bias_hh, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_decode( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_prefill( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, size_t chunk_size, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_stft( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, size_t num_fft_bins, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_altup_predict( + cactus_graph_t graph, cactus_node_t coefs, const cactus_node_t* streams, size_t num_streams, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_altup_correct( + cactus_graph_t graph, cactus_node_t coefs, cactus_node_t innovation, const cactus_node_t* predictions, size_t num_predictions, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gaussian_topk( + cactus_graph_t graph, cactus_node_t input, float ppf, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_gated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_ungated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_sample( + cactus_graph_t graph, cactus_node_t logits, float temperature, float top_p, size_t top_k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scatter_topk( + cactus_graph_t graph, cactus_node_t indices, cactus_node_t values, size_t num_classes, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_persistent( + cactus_graph_t graph, cactus_node_t source_node, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_is_populated( + cactus_graph_t graph, cactus_node_t persistent_node, int32_t* out_is_populated); +CACTUS_FFI_EXPORT int cactus_graph_invalidate_persistent( + cactus_graph_t graph, cactus_node_t persistent_node); + +CACTUS_FFI_EXPORT int cactus_graph_execute(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_get_output_ptr(cactus_graph_t graph, +cactus_node_t node, void** out_ptr); +CACTUS_FFI_EXPORT int cactus_graph_get_output_info(cactus_graph_t graph, +cactus_node_t node, cactus_tensor_info_t* out_info); + +#ifdef __cplusplus +} +#endif + +#endif // CACTUS_FFI_H diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_utils.h b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_utils.h new file mode 100644 index 0000000..aad98d8 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_utils.h @@ -0,0 +1,1869 @@ +#ifndef CACTUS_UTILS_H +#define CACTUS_UTILS_H + +#include "../engine/engine.h" +#include "../models/model.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#include +#elif defined(_WIN32) +#include +#include +#elif defined(__linux__) || defined(__ANDROID__) +#include +#endif + +inline size_t get_memory_footprint_bytes() { +#ifdef __APPLE__ + task_vm_info_data_t vm_info; + mach_msg_type_number_t count = TASK_VM_INFO_COUNT; + if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&vm_info, &count) == KERN_SUCCESS) + return vm_info.phys_footprint; + +#elif defined(_WIN32) + PROCESS_MEMORY_COUNTERS_EX pmc; + if (GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc))) + return pmc.PrivateUsage; + +#elif defined(__linux__) || defined(__ANDROID__) + std::ifstream statm("/proc/self/statm"); + if (statm.is_open()) { + size_t size, resident; + statm >> size >> resident; + return resident * sysconf(_SC_PAGESIZE); + } +#endif + return 0; +} + +inline double get_ram_usage_mb() { + return get_memory_footprint_bytes() / (1024.0 * 1024.0); +} + +struct CactusModelHandle { + std::unique_ptr model; + std::unique_ptr vad_model; + std::atomic should_stop; + std::vector processed_tokens; + struct ProcessedImage { + std::string path; + long long last_modified_timestamp = 0; + + bool operator==(const ProcessedImage& other) const { + return path == other.path && last_modified_timestamp == other.last_modified_timestamp; + } + }; + + std::vector> processed_images; + std::vector user_audio_counts; + std::mutex model_mutex; + std::string model_name; + std::unique_ptr corpus_index; + std::string corpus_dir; + size_t corpus_embedding_dim = 0; + std::vector> tool_embeddings; + std::vector tool_texts; + + CactusModelHandle() : should_stop(false) {} +}; + +extern std::string last_error_message; + +bool matches_stop_sequence(const std::vector& generated_tokens, + const std::vector>& stop_sequences); + +std::string retrieve_rag_context(CactusModelHandle* handle, const std::string& query); + +namespace cactus { +namespace audio { + +static constexpr size_t WHISPER_TARGET_FRAMES = 3000; +static constexpr int WHISPER_SAMPLE_RATE = 16000; + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_whisper_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 400; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = true; + cfg.pad_mode = "reflect"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 1e-10f; + cfg.log_mel = "log10"; + cfg.reference = 1.0f; + cfg.min_value = 1e-10f; + cfg.remove_dc_offset = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_parakeet_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 512; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = true; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 5.960464477539063e-08f; // 2^-24 guard value used by HF Parakeet. + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 1e-10f; + cfg.remove_dc_offset = false; + cfg.hann_periodic = false; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_htk_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 321; + cfg.frame_length = 320; + cfg.fft_override = 1024; + cfg.hop_length = 160; + cfg.power = 1.0f; + cfg.center = false; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 0.001f; + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 0.001f; + cfg.remove_dc_offset = false; + cfg.hann_periodic = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_gemma4_audio_spectrogram_config( + const cactus::engine::Config& model_config) { + auto cfg = get_htk_spectrogram_config(); + cfg.fft_override = model_config.audio_fft_length; + cfg.mel_floor_additive = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_wespeaker_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 512; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = false; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 1.1754944e-38f; + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 1.1754944e-38f; + cfg.remove_dc_offset = true; + cfg.preemphasis = 0.97f; + cfg.hann_periodic = false; + cfg.window_a0 = 0.54f; + return cfg; +} + +// Whisper v1/v2: 80 mel bins, HTK. Whisper v3: 128 mel bins, Slaney, 512-FFT, no DC removal. +inline void init_whisper_mel_filters(cactus::engine::AudioProcessor& ap, + cactus::engine::AudioProcessor::SpectrogramConfig& cfg, + size_t mel_bins) { + const size_t num_mel_filters = std::max(1, mel_bins); + const bool is_v3 = mel_bins > 80; + if (is_v3) { + cfg.fft_override = 512; + cfg.remove_dc_offset = false; + } + const size_t fft_len = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; + const size_t num_frequency_bins = fft_len / 2 + 1; + if (is_v3) { + ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, + WHISPER_SAMPLE_RATE, "slaney", "slaney"); + } else { + ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, + WHISPER_SAMPLE_RATE); + } +} + +// use_mel_floor_padding=true pads short audio with the normalized mel floor (required for v3). +inline std::vector normalize_whisper_mel(std::vector& mel, size_t n_mels, + bool use_mel_floor_padding = false) { + if (mel.empty() || n_mels == 0) return mel; + size_t n_frames = mel.size() / n_mels; + + float max_val = -std::numeric_limits::infinity(); + for (float v : mel) if (v > max_val) max_val = v; + + float min_allowed = max_val - 8.0f; + for (float& v : mel) { + if (v < min_allowed) v = min_allowed; + v = (v + 4.0f) * 0.25f; + } + + if (n_frames != WHISPER_TARGET_FRAMES) { + float pad_val = use_mel_floor_padding ? (min_allowed + 4.0f) * 0.25f : 0.0f; + std::vector fixed(n_mels * WHISPER_TARGET_FRAMES, pad_val); + size_t copy_frames = std::min(n_frames, WHISPER_TARGET_FRAMES); + for (size_t m = 0; m < n_mels; ++m) { + const float* src = &mel[m * n_frames]; + float* dst = &fixed[m * WHISPER_TARGET_FRAMES]; + std::copy(src, src + copy_frames, dst); + } + return fixed; + } + return std::move(mel); +} + +inline std::vector transpose_mel_to_frame_major(const std::vector& mel, + size_t num_mels, size_t num_frames) { + std::vector transposed(num_frames * num_mels); + for (size_t m = 0; m < num_mels; m++) { + for (size_t t = 0; t < num_frames; t++) { + transposed[t * num_mels + m] = mel[m * num_frames + t]; + } + } + return transposed; +} + +inline void apply_preemphasis(std::vector& waveform, float coefficient = 0.97f) { + if (waveform.size() < 2 || coefficient == 0.0f) { + return; + } + for (size_t i = waveform.size() - 1; i > 0; --i) { + waveform[i] -= coefficient * waveform[i - 1]; + } +} + +inline void normalize_parakeet_log_mel(std::vector& mel, size_t num_mels, float epsilon = 1e-5f) { + if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { + return; + } + const size_t num_frames = mel.size() / num_mels; + if (num_frames == 0) { + return; + } + + for (size_t m = 0; m < num_mels; ++m) { + const size_t base = m * num_frames; + float mean = 0.0f; + for (size_t t = 0; t < num_frames; ++t) { + mean += mel[base + t]; + } + mean /= static_cast(num_frames); + + float variance = 0.0f; + for (size_t t = 0; t < num_frames; ++t) { + const float d = mel[base + t] - mean; + variance += d * d; + } + const float denom = static_cast(std::max(1, num_frames - 1)); + const float inv_std = 1.0f / std::sqrt((variance / denom) + epsilon); + for (size_t t = 0; t < num_frames; ++t) { + mel[base + t] = (mel[base + t] - mean) * inv_std; + } + } +} + +inline void trim_mel_frames(std::vector& mel, size_t num_mels, size_t valid_frames) { + if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { + return; + } + size_t total_frames = mel.size() / num_mels; + if (valid_frames == 0 || valid_frames >= total_frames) { + return; + } + std::vector trimmed(num_mels * valid_frames); + for (size_t m = 0; m < num_mels; ++m) { + const float* src = &mel[m * total_frames]; + float* dst = &trimmed[m * valid_frames]; + std::copy(src, src + valid_frames, dst); + } + mel.swap(trimmed); +} + +struct AudioPreprocessResult { + std::vector features; + size_t num_frames = 0; + size_t num_soft_tokens = 0; +}; + +inline AudioPreprocessResult preprocess_audio_for_gemma4( + std::vector audio_samples, + const cactus::engine::Config& model_config +) { + AudioPreprocessResult result; + if (audio_samples.empty()) return result; + + size_t pad_amt = 320 - (audio_samples.size() % 320); + if (pad_amt < 320) + audio_samples.resize(audio_samples.size() + pad_amt, 0.0f); + + size_t mel_bins = model_config.audio_input_feat_size; + auto cfg = get_gemma4_audio_spectrogram_config(model_config); + + size_t semicausal_pad = cfg.frame_length / 2; + audio_samples.insert(audio_samples.begin(), semicausal_pad, 0.0f); + + cactus::engine::AudioProcessor ap; + size_t fft_for_mel = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; + ap.init_mel_filters(fft_for_mel / 2 + 1, mel_bins, 0.0f, 8000.0f, 16000, + nullptr, "htk"); + std::vector mel = ap.compute_spectrogram(audio_samples, cfg); + + result.num_frames = mel.size() / mel_bins; + result.features = transpose_mel_to_frame_major(mel, mel_bins, result.num_frames); + + size_t after_stage1 = (result.num_frames + 1) / 2; + result.num_soft_tokens = (after_stage1 + 1) / 2; + + return result; +} + +inline std::vector pcm_buffer_to_float_samples( + const uint8_t* pcm_buffer, size_t pcm_buffer_size +) { + const int16_t* pcm_samples = reinterpret_cast(pcm_buffer); + size_t num_samples = pcm_buffer_size / 2; + std::vector waveform_fp32(num_samples); + constexpr float inv_32768 = 1.0f / 32768.0f; + for (size_t i = 0; i < num_samples; i++) + waveform_fp32[i] = static_cast(pcm_samples[i]) * inv_32768; + return waveform_fp32; +} + +} // namespace audio +} // namespace cactus + +namespace cactus { +namespace ffi { + +inline bool env_flag_enabled(const char* key) { + const char* value = std::getenv(key); + return value && value[0] != '\0' && !(value[0] == '0' && value[1] == '\0'); +} + +inline std::string generateUUID() { +#ifdef __APPLE__ + uuid_t uuid; + uuid_generate_random(uuid); + char uuid_str[37]; + uuid_unparse_lower(uuid, uuid_str); + return std::string(uuid_str); +#else + static std::random_device rd; + static std::mt19937 gen(rd()); + static std::uniform_int_distribution<> dis(0, 15); + static std::uniform_int_distribution<> dis2(8, 11); + + std::stringstream ss; + ss << std::hex; + for (int i = 0; i < 8; i++) ss << dis(gen); + ss << "-"; + for (int i = 0; i < 4; i++) ss << dis(gen); + ss << "-4"; + for (int i = 0; i < 3; i++) ss << dis(gen); + ss << "-"; + ss << dis2(gen); + for (int i = 0; i < 3; i++) ss << dis(gen); + ss << "-"; + for (int i = 0; i < 12; i++) ss << dis(gen); + return ss.str(); +#endif +} + +struct ToolFunction { + std::string name; + std::string description; + std::unordered_map parameters; +}; + +struct InferenceOptions { + float temperature = 0.0f; + float top_p = 0.0f; + float min_p = 0.15f; + float repetition_penalty = 1.1f; + float confidence_threshold = -1.0f; + size_t top_k = 0; + size_t max_tokens = 100; + size_t tool_rag_top_k = 2; + size_t cloud_timeout_ms = 15000; + std::vector stop_sequences; + bool force_tools = false; + bool include_stop_sequences = false; + bool use_vad = true; + bool telemetry_enabled = true; + bool auto_handoff = true; + bool handoff_with_images = true; + bool enable_thinking_if_supported = false; +}; + +} // namespace ffi +} // namespace cactus + +std::vector select_relevant_tools( + CactusModelHandle* handle, + const std::string& query, + const std::vector& all_tools, + size_t top_k); + +#include "gemma_tools.h" + +namespace cactus { +namespace ffi { + +inline std::string escape_json_string(const std::string& s) { + std::ostringstream o; + for (char c : s) { + if (c == '"') o << "\\\""; + else if (c == '\n') o << "\\n"; + else if (c == '\r') o << "\\r"; + else if (c == '\t') o << "\\t"; + else if (c == '\\') o << "\\\\"; + else o << c; + } + return o.str(); +} + + +inline std::string trim_string(const std::string& s) { + size_t start = 0; + while (start < s.size() && std::isspace(static_cast(s[start]))) ++start; + size_t end = s.size(); + while (end > start && std::isspace(static_cast(s[end - 1]))) --end; + return s.substr(start, end - start); +} + +inline size_t find_matching_delimiter(const std::string& s, size_t pos, char open, char close) { + int depth = 1; + pos++; + while (pos < s.length() && depth > 0) { + if (s[pos] == open) depth++; + else if (s[pos] == close) depth--; + else if (s[pos] == '"') { + pos++; + while (pos < s.length() && s[pos] != '"') { + if (s[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + return pos; +} + +inline std::string env_or_default(const char* key, const char* fallback) { + const char* v = std::getenv(key); + if (v && v[0] != '\0') return std::string(v); + return std::string(fallback); +} + +inline std::string json_string_field(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return {}; + + size_t i = pos + pattern.size(); + while (i < json.size() && std::isspace(static_cast(json[i]))) i++; + if (i >= json.size() || json[i] != '"') return {}; + ++i; + + std::string out; + out.reserve(128); + while (i < json.size()) { + char c = json[i++]; + if (c == '"') return out; + if (c == '\\' && i < json.size()) { + char e = json[i++]; + switch (e) { + case '"': out.push_back('"'); break; + case '\\': out.push_back('\\'); break; + case '/': out.push_back('/'); break; + case 'b': out.push_back('\b'); break; + case 'f': out.push_back('\f'); break; + case 'n': out.push_back('\n'); break; + case 'r': out.push_back('\r'); break; + case 't': out.push_back('\t'); break; + default: out.push_back(e); break; + } + continue; + } + out.push_back(c); + } + return {}; +} + +inline std::string json_array_field(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return "[]"; + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + if (start >= json.size() || json[start] != '[') return "[]"; + + int depth = 1; + size_t end = start + 1; + while (end < json.size() && depth > 0) { + if (json[end] == '[') depth++; + else if (json[end] == ']') depth--; + end++; + } + return json.substr(start, end - start); +} + +inline std::vector split_json_array(const std::string& array_json) { + std::vector out; + if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; + + size_t i = 1; + while (i + 1 < array_json.size()) { + while (i + 1 < array_json.size() && + (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) i++; + if (i + 1 >= array_json.size() || array_json[i] != '{') break; + + size_t start = i; + int depth = 0; + bool in_str = false; + bool esc = false; + for (; i < array_json.size(); ++i) { + char c = array_json[i]; + if (in_str) { + if (esc) esc = false; + else if (c == '\\') esc = true; + else if (c == '"') in_str = false; + continue; + } + if (c == '"') { in_str = true; continue; } + if (c == '{') depth++; + if (c == '}') { + depth--; + if (depth == 0) { + out.push_back(array_json.substr(start, i - start + 1)); + i++; + break; + } + } + } + } + return out; +} + +inline std::string serialize_tools_json(const std::vector& tools) { + if (tools.empty()) return ""; + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < tools.size(); ++i) { + if (i > 0) oss << ","; + oss << "{\"type\":\"function\",\"function\":{"; + oss << "\"name\":\"" << escape_json_string(tools[i].name) << "\","; + oss << "\"description\":\"" << escape_json_string(tools[i].description) << "\""; + auto it = tools[i].parameters.find("schema"); + if (it != tools[i].parameters.end()) { + oss << ",\"parameters\":" << it->second; + } + oss << "}}"; + } + oss << "]"; + return oss.str(); +} + +namespace json_sorted { + +inline void skip_ws(const std::string& s, size_t& p) { + while (p < s.size() && std::isspace(static_cast(s[p]))) p++; +} + +inline std::string parse_string(const std::string& s, size_t& p) { + std::string r = "\""; + p++; + while (p < s.size()) { + if (s[p] == '\\') { + r += s[p++]; + if (p < s.size()) r += s[p++]; + } else if (s[p] == '"') { + r += '"'; + p++; + return r; + } else { + r += s[p++]; + } + } + return r; +} + +inline std::string parse_value(const std::string& s, size_t& p); + +inline std::string parse_object(const std::string& s, size_t& p) { + p++; + std::map entries; + skip_ws(s, p); + while (p < s.size() && s[p] != '}') { + if (s[p] == ',') { p++; skip_ws(s, p); continue; } + std::string key = parse_string(s, p); + skip_ws(s, p); + if (p < s.size() && s[p] == ':') p++; + skip_ws(s, p); + std::string val = parse_value(s, p); + entries[key] = val; + skip_ws(s, p); + } + if (p < s.size()) p++; + std::string r = "{"; + bool first = true; + for (const auto& kv : entries) { + if (!first) r += ", "; + r += kv.first + ": " + kv.second; + first = false; + } + r += "}"; + return r; +} + +inline std::string parse_array(const std::string& s, size_t& p) { + p++; + std::vector items; + skip_ws(s, p); + while (p < s.size() && s[p] != ']') { + if (s[p] == ',') { p++; skip_ws(s, p); continue; } + items.push_back(parse_value(s, p)); + skip_ws(s, p); + } + if (p < s.size()) p++; + std::string r = "["; + for (size_t i = 0; i < items.size(); i++) { + if (i > 0) r += ", "; + r += items[i]; + } + r += "]"; + return r; +} + +inline std::string parse_value(const std::string& s, size_t& p) { + skip_ws(s, p); + if (p >= s.size()) return ""; + if (s[p] == '"') return parse_string(s, p); + if (s[p] == '{') return parse_object(s, p); + if (s[p] == '[') return parse_array(s, p); + size_t start = p; + while (p < s.size() && s[p] != ',' && s[p] != '}' && s[p] != ']' && !std::isspace(static_cast(s[p]))) p++; + return s.substr(start, p - start); +} + +inline std::string reformat(const std::string& json) { + size_t p = 0; + return parse_value(json, p); +} + +} // namespace json_sorted + +inline std::string serialize_tools_for_template(const std::vector& tools) { + if (tools.empty()) return ""; + std::string result; + for (const auto& tool : tools) { + std::map func_fields; + func_fields["\"description\""] = "\"" + escape_json_string(tool.description) + "\""; + func_fields["\"name\""] = "\"" + escape_json_string(tool.name) + "\""; + auto it = tool.parameters.find("schema"); + if (it != tool.parameters.end()) { + func_fields["\"parameters\""] = json_sorted::reformat(it->second); + } + std::string func_json = "{"; + bool first = true; + for (const auto& kv : func_fields) { + if (!first) func_json += ", "; + func_json += kv.first + ": " + kv.second; + first = false; + } + func_json += "}"; + result += "\n{\"function\": " + func_json + ", \"type\": \"function\"}"; + } + return result; +} + +inline void handle_error_response(const std::string& error_message, char* response_buffer, size_t buffer_size) { + std::ostringstream json; + json << "{"; + json << "\"success\":false,"; + json << "\"error\":\"" << escape_json_string(error_message) << "\","; + json << "\"cloud_handoff\":false,"; + json << "\"response\":null,"; + json << "\"function_calls\":[],"; + json << "\"confidence\":0.0,"; + json << "\"time_to_first_token_ms\":0.0,"; + json << "\"total_time_ms\":0.0,"; + json << "\"prefill_tps\":0.0,"; + json << "\"decode_tps\":0.0,"; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; + json << "\"prefill_tokens\":0,"; + json << "\"decode_tokens\":0,"; + json << "\"total_tokens\":0"; + json << "}"; + std::string error_json = json.str(); + if (response_buffer && error_json.length() < buffer_size) { + std::strcpy(response_buffer, error_json.c_str()); + } +} + +inline std::vector parse_messages_json(const std::string& json, + std::vector& out_image_paths, + std::vector* out_audio_paths = nullptr) { + std::vector messages; + out_image_paths.clear(); + if (out_audio_paths) out_audio_paths->clear(); + + size_t pos = json.find('['); + if (pos == std::string::npos) { + throw std::runtime_error("Invalid JSON: expected array"); + } + + pos = json.find('{', pos); + while (pos != std::string::npos) { + cactus::engine::ChatMessage msg; + + size_t obj_start = pos; + int brace_count = 1; + size_t obj_end = obj_start + 1; + while (obj_end < json.length() && brace_count > 0) { + if (json[obj_end] == '{') brace_count++; + else if (json[obj_end] == '}') brace_count--; + obj_end++; + } + + size_t role_pos = json.find("\"role\"", pos); + if (role_pos == std::string::npos || role_pos >= obj_end) break; + + size_t role_start = json.find('"', role_pos + 6) + 1; + size_t role_end = json.find('"', role_start); + msg.role = json.substr(role_start, role_end - role_start); + + size_t content_pos = json.find("\"content\"", role_end); + if (content_pos != std::string::npos && content_pos < obj_end) { + size_t content_start = json.find('"', content_pos + 9) + 1; + size_t content_end = content_start; + + while (content_end < json.length()) { + content_end = json.find('"', content_end); + if (content_end == std::string::npos) break; + if (json[content_end - 1] != '\\') break; + content_end++; + } + + msg.content = json.substr(content_start, content_end - content_start); + + size_t escape_pos = 0; + while ((escape_pos = msg.content.find("\\n", escape_pos)) != std::string::npos) { + msg.content.replace(escape_pos, 2, "\n"); + escape_pos += 1; + } + escape_pos = 0; + while ((escape_pos = msg.content.find("\\\"", escape_pos)) != std::string::npos) { + msg.content.replace(escape_pos, 2, "\""); + escape_pos += 1; + } + } + + auto parse_path_array = [&](const char* key, std::vector& dest, + std::vector* out_paths) { + size_t key_pos = json.find(key, pos); + if (key_pos == std::string::npos || key_pos >= obj_end) return; + size_t array_start = json.find('[', key_pos); + if (array_start == std::string::npos || array_start >= obj_end) return; + size_t array_end = json.find(']', array_start); + if (array_end == std::string::npos || array_end >= obj_end) return; + size_t cur = array_start; + while (true) { + cur = json.find('"', cur + 1); + if (cur == std::string::npos || cur >= array_end) break; + size_t str_start = cur + 1; + size_t str_end = json.find('"', str_start); + if (str_end == std::string::npos || str_end > array_end) break; + std::string path = std::filesystem::absolute( + std::filesystem::path(json.substr(str_start, str_end - str_start))).string(); + dest.push_back(path); + if (out_paths) out_paths->push_back(path); + cur = str_end; + } + }; + + parse_path_array("\"images\"", msg.images, &out_image_paths); + parse_path_array("\"audio\"", msg.audio, out_audio_paths); + + if (msg.role == "tool") { + size_t name_pos = json.find("\"name\"", obj_start); + if (name_pos != std::string::npos && name_pos < obj_end) { + size_t name_quote = json.find('"', name_pos + 6); + if (name_quote != std::string::npos && name_quote < obj_end) { + size_t name_start = name_quote + 1; + size_t name_end = json.find('"', name_start); + if (name_end != std::string::npos && name_end < obj_end) { + msg.name = json.substr(name_start, name_end - name_start); + } + } + } + } + + size_t tool_calls_pos = json.find("\"tool_calls\"", obj_start); + if (tool_calls_pos != std::string::npos && tool_calls_pos < obj_end) { + size_t tool_calls_arr_start = json.find('[', tool_calls_pos); + if (tool_calls_arr_start != std::string::npos && tool_calls_arr_start < obj_end) { + size_t tool_calls_arr_end = find_matching_delimiter(json, tool_calls_arr_start, '[', ']'); + + size_t search_pos = tool_calls_arr_start; + while (true) { + size_t func_pos = json.find("\"function\"", search_pos); + if (func_pos == std::string::npos || func_pos >= tool_calls_arr_end) break; + + size_t func_obj_start = json.find('{', func_pos + 10); + if (func_obj_start == std::string::npos || func_obj_start >= tool_calls_arr_end) break; + + size_t func_obj_end = find_matching_delimiter(json, func_obj_start, '{', '}'); + + cactus::engine::ToolCallInfo tool_call; + + size_t fn_name_pos = json.find("\"name\"", func_obj_start); + if (fn_name_pos != std::string::npos && fn_name_pos < func_obj_end) { + size_t fn_name_quote = json.find('"', fn_name_pos + 6); + if (fn_name_quote != std::string::npos && fn_name_quote < func_obj_end) { + size_t fn_name_start = fn_name_quote + 1; + size_t fn_name_end = json.find('"', fn_name_start); + if (fn_name_end != std::string::npos && fn_name_end < func_obj_end) { + tool_call.name = json.substr(fn_name_start, fn_name_end - fn_name_start); + } + } + } + + size_t args_pos = json.find("\"arguments\"", func_obj_start); + if (args_pos != std::string::npos && args_pos < func_obj_end) { + size_t colon_pos = json.find(':', args_pos + 11); + if (colon_pos != std::string::npos && colon_pos < func_obj_end) { + size_t args_start = colon_pos + 1; + while (args_start < json.length() && std::isspace(static_cast(json[args_start]))) args_start++; + + if (args_start < func_obj_end && json[args_start] == '{') { + size_t args_end = find_matching_delimiter(json, args_start, '{', '}'); + tool_call.arguments = json.substr(args_start, args_end - args_start); + } else if (args_start < func_obj_end && json[args_start] == '"') { + size_t str_start = args_start + 1; + size_t str_end = str_start; + while (str_end < json.length() && json[str_end] != '"') { + if (json[str_end] == '\\') str_end++; + str_end++; + } + tool_call.arguments = json.substr(str_start, str_end - str_start); + } + } + } + + if (!tool_call.name.empty()) { + msg.tool_calls.push_back(tool_call); + } + search_pos = func_obj_end; + } + } + } + + messages.push_back(msg); + + pos = json.find('{', obj_end); + } + + return messages; +} + +inline std::vector parse_tools_json(const std::string& json) { + std::vector tools; + + if (json.empty()) return tools; + + size_t pos = json.find('['); + if (pos == std::string::npos) return tools; + + pos = json.find("\"function\"", pos); + while (pos != std::string::npos) { + ToolFunction tool; + size_t next_search = pos + 1; + + size_t name_pos = json.find("\"name\"", pos); + if (name_pos != std::string::npos) { + size_t name_start = json.find('"', name_pos + 6) + 1; + size_t name_end = json.find('"', name_start); + tool.name = json.substr(name_start, name_end - name_start); + } + + size_t desc_pos = json.find("\"description\"", pos); + if (desc_pos != std::string::npos) { + size_t desc_start = json.find('"', desc_pos + 13) + 1; + size_t desc_end = json.find('"', desc_start); + tool.description = json.substr(desc_start, desc_end - desc_start); + } + + size_t params_pos = json.find("\"parameters\"", pos); + if (params_pos != std::string::npos) { + size_t params_start = json.find('{', params_pos); + if (params_start != std::string::npos) { + int brace_count = 1; + size_t params_end = params_start + 1; + while (params_end < json.length() && brace_count > 0) { + if (json[params_end] == '{') brace_count++; + else if (json[params_end] == '}') brace_count--; + params_end++; + } + tool.parameters["schema"] = json.substr(params_start, params_end - params_start); + next_search = params_end; + } + } + + if (!tool.name.empty()) { + tools.push_back(tool); + } + + pos = json.find("\"function\"", next_search); + } + + return tools; +} + +inline bool try_parse_json_float(const std::string& json, const std::string& key, float& out_value) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return false; + + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + + size_t end = start; + while (end < json.size() && std::string(",}] \t\n\r").find(json[end]) == std::string::npos) ++end; + + try { + out_value = std::stof(json.substr(start, end - start)); + return true; + } catch (...) { + return false; + } +} + +inline std::vector parse_json_string_array_field(const std::string& json, const std::string& key) { + std::vector out; + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return out; + + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + if (start >= json.size() || json[start] != '[') return out; + + int depth = 1; + bool in_string = false; + bool escaped = false; + size_t end = start + 1; + + while (end < json.size() && depth > 0) { + char c = json[end]; + if (in_string) { + if (escaped) escaped = false; + else if (c == '\\') escaped = true; + else if (c == '"') in_string = false; + } else { + if (c == '"') in_string = true; + else if (c == '[') depth++; + else if (c == ']') depth--; + } + ++end; + } + + if (depth != 0) return out; + const std::string array_json = json.substr(start, end - start); + if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; + + size_t i = 1; + while (i + 1 < array_json.size()) { + while (i + 1 < array_json.size() && + (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) { + ++i; + } + if (i + 1 >= array_json.size() || array_json[i] == ']') break; + if (array_json[i] != '"') break; + + ++i; + std::string value; + bool escaped = false; + while (i < array_json.size()) { + char c = array_json[i++]; + if (escaped) { + switch (c) { + case '"': value.push_back('"'); break; + case '\\': value.push_back('\\'); break; + case '/': value.push_back('/'); break; + case 'b': value.push_back('\b'); break; + case 'f': value.push_back('\f'); break; + case 'n': value.push_back('\n'); break; + case 'r': value.push_back('\r'); break; + case 't': value.push_back('\t'); break; + default: value.push_back(c); break; + } + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == '"') { + out.push_back(value); + break; + } + value.push_back(c); + } + } + + return out; +} + +inline void parse_custom_vocabulary_options(const std::string& json, + std::vector& custom_vocabulary, + float& vocabulary_boost) { + custom_vocabulary.clear(); + vocabulary_boost = 5.0f; + if (json.empty()) return; + + float parsed_boost = vocabulary_boost; + if (try_parse_json_float(json, "vocabulary_boost", parsed_boost)) { + vocabulary_boost = std::clamp(parsed_boost, 0.0f, 20.0f); + } + + custom_vocabulary = parse_json_string_array_field(json, "custom_vocabulary"); +} + +inline std::unordered_map build_token_bias_map(const std::vector>& tokenized_entries, + float vocabulary_boost) { + std::unordered_map vocab_bias; + const float clamped_boost = std::clamp(vocabulary_boost, 0.0f, 20.0f); + if (clamped_boost == 0.0f) return vocab_bias; + + for (const auto& token_ids : tokenized_entries) { + for (uint32_t token_id : token_ids) { + float& entry = vocab_bias[token_id]; + if (entry < clamped_boost) { + entry = clamped_boost; + } + } + } + + return vocab_bias; +} + +inline std::unordered_map build_custom_vocabulary_bias(cactus::engine::Tokenizer* tokenizer, + const std::vector& custom_vocabulary, + float vocabulary_boost) { + if (!tokenizer || custom_vocabulary.empty()) return {}; + std::vector> tokenized_entries; + tokenized_entries.reserve(custom_vocabulary.size()); + + for (const auto& word : custom_vocabulary) { + if (word.empty()) continue; + tokenized_entries.push_back(tokenizer->encode(word)); + } + + return build_token_bias_map(tokenized_entries, vocabulary_boost); +} + +inline void apply_custom_vocabulary_options(cactus::engine::Model* model, const std::string& json) { + if (!model) return; + + std::vector custom_vocabulary; + float vocabulary_boost = 5.0f; + parse_custom_vocabulary_options(json, custom_vocabulary, vocabulary_boost); + model->set_vocab_bias(build_custom_vocabulary_bias(model->get_tokenizer(), custom_vocabulary, vocabulary_boost)); +} + +inline size_t levenshtein_ci(const std::string& a, const std::string& b) { + const size_t m = a.size(), n = b.size(); + std::vector prev(n + 1), curr(n + 1); + for (size_t j = 0; j <= n; ++j) prev[j] = j; + for (size_t i = 1; i <= m; ++i) { + curr[0] = i; + for (size_t j = 1; j <= n; ++j) { + const bool match = std::tolower(static_cast(a[i - 1])) == + std::tolower(static_cast(b[j - 1])); + curr[j] = std::min({prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + (match ? 0 : 1)}); + } + std::swap(prev, curr); + } + return prev[n]; +} + +inline std::string collapse_spaces(const std::string& s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + if (c != ' ') out += c; + } + return out; +} + +inline void apply_vocabulary_spelling_correction( + std::string& text, + const std::vector& custom_vocabulary) +{ + if (custom_vocabulary.empty() || text.empty()) return; + + struct VocabEntry { + const std::string* original; + std::string collapsed; + }; + std::vector vocab_entries; + vocab_entries.reserve(custom_vocabulary.size()); + for (const auto& v : custom_vocabulary) { + vocab_entries.push_back({&v, collapse_spaces(v)}); + } + + struct Token { std::string text; bool is_word; }; + std::vector tokens; + size_t pos = 0; + while (pos < text.size()) { + if (std::isalnum(static_cast(text[pos])) || + text[pos] == '\'' || text[pos] == '-') { + size_t start = pos; + while (pos < text.size() && (std::isalnum(static_cast(text[pos])) || + text[pos] == '\'' || text[pos] == '-')) { + ++pos; + } + tokens.push_back({text.substr(start, pos - start), true}); + } else { + size_t start = pos; + while (pos < text.size() && !std::isalnum(static_cast(text[pos])) && + text[pos] != '\'' && text[pos] != '-') { + ++pos; + } + tokens.push_back({text.substr(start, pos - start), false}); + } + } + + std::vector word_indices; + for (size_t i = 0; i < tokens.size(); ++i) { + if (tokens[i].is_word) word_indices.push_back(i); + } + + std::vector consumed(tokens.size(), false); + + auto strip_suffix = [](const std::string& word) -> std::pair { + if (word.size() >= 3 && word.substr(word.size() - 2) == "'s") { + return {word.substr(0, word.size() - 2), "'s"}; + } + if (word.size() >= 3 && word.substr(word.size() - 2) == "'t") { + return {word.substr(0, word.size() - 2), "'t"}; + } + if (word.size() >= 4 && word.back() == 's' && + word[word.size() - 2] != 's' && // avoid stripping from "boss", "class" + std::isalpha(static_cast(word[word.size() - 2]))) { + return {word.substr(0, word.size() - 1), "s"}; + } + return {word, ""}; + }; + + size_t wi = 0; + while (wi < word_indices.size()) { + size_t best_dist = std::numeric_limits::max(); + const std::string* best_match = nullptr; + size_t best_window = 0; + size_t best_first_token = 0; + size_t best_last_token = 0; + std::string best_suffix; + + for (size_t window = std::min(3, word_indices.size() - wi); window >= 1; --window) { + std::string window_collapsed; + const size_t first_tok = word_indices[wi]; + const size_t last_tok = word_indices[wi + window - 1]; + for (size_t w = 0; w < window; ++w) { + window_collapsed += tokens[word_indices[wi + w]].text; + } + + if (window == 1 && window_collapsed.size() < 3) break; + + auto [stem, suffix] = strip_suffix(window_collapsed); + const std::string* candidates[] = {&window_collapsed, &stem}; + const std::string suffixes[] = {"", suffix}; + const size_t num_candidates = suffix.empty() ? 1 : 2; + + for (size_t ci = 0; ci < num_candidates; ++ci) { + const std::string& candidate = *candidates[ci]; + if (candidate.empty()) continue; + + for (const auto& entry : vocab_entries) { + const size_t wlen = candidate.size(); + const size_t vlen = entry.collapsed.size(); + + const size_t len_diff = wlen > vlen ? wlen - vlen : vlen - wlen; + const size_t max_dist = std::max(1, std::min(wlen, vlen) / 3); + if (len_diff > max_dist) continue; + + const size_t dist = levenshtein_ci(candidate, entry.collapsed); + + // For single-edit corrections, require first char match to prevent + // false positives like "vortex" → "Cortex". + if (dist == 1 && window == 1) { + const bool first_char_match = + std::tolower(static_cast(candidate[0])) == + std::tolower(static_cast(entry.collapsed[0])); + if (!first_char_match) continue; + } + + if (dist <= max_dist && dist < best_dist) { + best_dist = dist; + best_match = entry.original; + best_window = window; + best_first_token = first_tok; + best_last_token = last_tok; + best_suffix = suffixes[ci]; + } + } + } + + if (best_dist == 0) break; + } + + // Allow dist==0 for multi-word merges where word boundaries changed. + const bool should_replace = best_match && + best_dist != std::numeric_limits::max() && + (best_dist > 0 || best_window > 1); + + if (should_replace) { + tokens[best_first_token].text = *best_match + best_suffix; + for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { + consumed[t] = true; + } + for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { + if (t > 0) consumed[t - 1] = consumed[t - 1] || !tokens[t - 1].is_word; + } + wi += best_window; + } else { + ++wi; + } + } + + std::string result; + result.reserve(text.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + if (!consumed[i]) { + result += tokens[i].text; + } + } + + text = std::move(result); +} + +inline InferenceOptions parse_inference_options_json(const std::string& json) { + InferenceOptions options; + + if (json.empty()) return options; + + size_t pos = json.find("\"temperature\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.temperature = std::stof(json.substr(pos)); + } + + pos = json.find("\"top_p\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.top_p = std::stof(json.substr(pos)); + } + + float parsed_min_p = options.min_p; + if (try_parse_json_float(json, "min_p", parsed_min_p)) { + options.min_p = std::clamp(parsed_min_p, 0.0f, 1.0f); + } + + float parsed_rep_penalty = options.repetition_penalty; + if (try_parse_json_float(json, "repetition_penalty", parsed_rep_penalty)) { + if (std::isfinite(parsed_rep_penalty) && parsed_rep_penalty > 0.0f) { + options.repetition_penalty = parsed_rep_penalty; + } + } + + pos = json.find("\"top_k\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.top_k = std::stoul(json.substr(pos)); + } + + pos = json.find("\"max_tokens\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.max_tokens = std::stoul(json.substr(pos)); + } + + pos = json.find("\"force_tools\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.force_tools = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"tool_rag_top_k\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.tool_rag_top_k = std::stoul(json.substr(pos)); + } + + pos = json.find("\"confidence_threshold\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.confidence_threshold = std::stof(json.substr(pos)); + } + + pos = json.find("\"include_stop_sequences\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.include_stop_sequences = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"use_vad\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.use_vad = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"telemetry_enabled\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.telemetry_enabled = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"auto_handoff\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.auto_handoff = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"cloud_timeout_ms\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.cloud_timeout_ms = std::stoul(json.substr(pos)); + } + + pos = json.find("\"handoff_with_images\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.handoff_with_images = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"enable_thinking_if_supported\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.enable_thinking_if_supported = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"stop_sequences\""); + if (pos != std::string::npos) { + pos = json.find('[', pos); + if (pos != std::string::npos) { + size_t end_pos = json.find(']', pos); + size_t seq_pos = json.find('"', pos); + + while (seq_pos != std::string::npos && seq_pos < end_pos) { + size_t seq_start = seq_pos + 1; + size_t seq_end = json.find('"', seq_start); + if (seq_end != std::string::npos) { + options.stop_sequences.push_back(json.substr(seq_start, seq_end - seq_start)); + } + seq_pos = json.find('"', seq_end + 1); + } + } + } + + return options; +} + +static inline std::string trim_lfm2_slice(const std::string& value, size_t begin, size_t end) { + return trim_string(value.substr(begin, end - begin)); +} + +static inline void append_lfm2_call(const std::string& entry, + std::vector& function_calls) { + if (entry.empty()) return; + + std::string trimmed_entry = trim_lfm2_slice(entry, 0, entry.size()); + if (trimmed_entry.empty()) return; + + size_t paren_pos = trimmed_entry.find('('); + if (paren_pos == std::string::npos) return; + + std::string func_name = trim_lfm2_slice(trimmed_entry, 0, paren_pos); + std::string args_str = trim_lfm2_slice(trimmed_entry, paren_pos + 1, trimmed_entry.size()); + + while (!args_str.empty() && (args_str.back() == ')' || args_str.back() == ']')) { + args_str.pop_back(); + } + args_str = trim_lfm2_slice(args_str, 0, args_str.size()); + + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":{"; + + size_t arg_pos = 0; + bool first_arg = true; + while (arg_pos < args_str.length()) { + while (arg_pos < args_str.length() && std::isspace(static_cast(args_str[arg_pos]))) { + arg_pos++; + } + + size_t eq_pos = args_str.find('=', arg_pos); + if (eq_pos == std::string::npos) break; + + std::string arg_name = args_str.substr(arg_pos, eq_pos - arg_pos); + + size_t val_start = eq_pos + 1; + size_t val_end = val_start; + bool quoted = false; + + if (val_start < args_str.length() && args_str[val_start] == '"') { + quoted = true; + val_start++; + val_end = args_str.find('"', val_start); + if (val_end == std::string::npos) break; + } else { + int depth = 0; + for (val_end = val_start; val_end < args_str.length(); val_end++) { + char c = args_str[val_end]; + if (c == '[' || c == '{') depth++; + else if (c == ']' || c == '}') depth--; + else if (c == ',' && depth == 0) break; + } + } + + std::string arg_value = args_str.substr(val_start, val_end - val_start); + + if (!quoted) { + if (arg_value == "True") arg_value = "true"; + else if (arg_value == "False") arg_value = "false"; + else if (arg_value == "None") arg_value = "null"; + } + + if (!first_arg) json_call += ","; + json_call += "\"" + arg_name + "\":"; + json_call += quoted ? ("\"" + arg_value + "\"") : arg_value; + first_arg = false; + + arg_pos = args_str.find(',', val_end); + if (arg_pos != std::string::npos) { + arg_pos++; + } else { + break; + } + } + + json_call += "}}"; + function_calls.push_back(json_call); +} + +inline void parse_function_calls_from_response(const std::string& response_text, + std::string& regular_response, + std::vector& function_calls) { + regular_response = response_text; + function_calls.clear(); + + gemma::parse_function_calls(regular_response, function_calls); + + const std::string QWEN_TOOL_START = ""; + const std::string QWEN_TOOL_END = ""; + size_t qwen_start_pos = 0; + + while ((qwen_start_pos = regular_response.find(QWEN_TOOL_START, qwen_start_pos)) != std::string::npos) { + size_t content_start = qwen_start_pos + QWEN_TOOL_START.length(); + size_t qwen_end_pos = regular_response.find(QWEN_TOOL_END, content_start); + + size_t erase_end; + std::string json_content; + + if (qwen_end_pos != std::string::npos) { + json_content = regular_response.substr(content_start, qwen_end_pos - content_start); + erase_end = qwen_end_pos + QWEN_TOOL_END.length(); + } else { + json_content = regular_response.substr(content_start); + erase_end = regular_response.length(); + } + + size_t first = json_content.find_first_not_of(" \t\n\r"); + size_t last = json_content.find_last_not_of(" \t\n\r"); + if (first != std::string::npos && last != std::string::npos) { + json_content = json_content.substr(first, last - first + 1); + } + + if (json_content.size() > 2 && json_content.find("\"name\"") != std::string::npos) { + // Unwrap array wrapper if present: [{"name":...}] -> {"name":...} + if (json_content[0] == '[') { + size_t obj_start = json_content.find('{'); + size_t obj_end = json_content.rfind('}'); + if (obj_start != std::string::npos && obj_end != std::string::npos && obj_end > obj_start) { + json_content = json_content.substr(obj_start, obj_end - obj_start + 1); + } + } + if (json_content[0] == '{') { + size_t depth = 0; + bool in_string = false; + bool escaped = false; + size_t end_pos = 0; + for (size_t c = 0; c < json_content.size(); c++) { + char ch = json_content[c]; + if (escaped) { escaped = false; continue; } + if (ch == '\\' && in_string) { escaped = true; continue; } + if (ch == '"') { in_string = !in_string; continue; } + if (!in_string) { + if (ch == '{') depth++; + else if (ch == '}' && --depth == 0) { end_pos = c + 1; break; } + } + } + if (end_pos > 0) { + function_calls.push_back(json_content.substr(0, end_pos)); + } + } + } + + regular_response.erase(qwen_start_pos, erase_end - qwen_start_pos); + } + + const std::string TOOL_CALL_START = "<|tool_call_start|>"; + const std::string TOOL_CALL_END = "<|tool_call_end|>"; + size_t lfm2_start_pos = 0; + + while ((lfm2_start_pos = regular_response.find(TOOL_CALL_START, lfm2_start_pos)) != std::string::npos) { + size_t content_start = lfm2_start_pos + TOOL_CALL_START.length(); + size_t tool_end_pos = regular_response.find(TOOL_CALL_END, content_start); + + if (tool_end_pos != std::string::npos) { + std::string tool_content = regular_response.substr(content_start, tool_end_pos - content_start); + std::string content = tool_content; + size_t trim_start = 0; + while (trim_start < content.size() && std::isspace(static_cast(content[trim_start]))) { + trim_start++; + } + + if (trim_start < content.size()) { + size_t trim_end = content.size() - 1; + while (trim_end > trim_start && std::isspace(static_cast(content[trim_end]))) { + trim_end--; + } + content = content.substr(trim_start, trim_end - trim_start + 1); + } else { + content.clear(); + } + + if (!content.empty() && content.front() == '[' && content.back() == ']') { + std::string inner = content.substr(1, content.size() - 2); + + size_t inner_first = inner.find_first_not_of(" \t\n\r"); + if (inner_first != std::string::npos && inner[inner_first] == '{') { + size_t pos = inner_first; + while (pos < inner.size()) { + if (inner[pos] == '{') { + int brace_depth = 1; + size_t obj_start = pos; + pos++; + while (pos < inner.size() && brace_depth > 0) { + if (inner[pos] == '{') brace_depth++; + else if (inner[pos] == '}') brace_depth--; + pos++; + } + if (brace_depth == 0) { + std::string json_obj = inner.substr(obj_start, pos - obj_start); + if (json_obj.find("\"name\"") != std::string::npos) { + function_calls.push_back(json_obj); + } + } + } else { + pos++; + } + } + } else { + size_t start = 0; + int paren_depth = 0; + + for (size_t i = 0; i < inner.size(); ++i) { + char c = inner[i]; + if (c == '(') { + paren_depth++; + } else if (c == ')' && paren_depth > 0) { + paren_depth--; + } else if (c == ',' && paren_depth == 0) { + append_lfm2_call(inner.substr(start, i - start), function_calls); + start = i + 1; + } + } + + if (start < inner.size()) { + append_lfm2_call(inner.substr(start), function_calls); + } + } + } else if (!content.empty()) { + append_lfm2_call(content, function_calls); + } + + regular_response.erase(lfm2_start_pos, tool_end_pos + TOOL_CALL_END.length() - lfm2_start_pos); + } else { + break; + } + } + + const char* FUNCTION_CALL_MARKER = "\"function_call\""; + size_t search_pos = 0; + const size_t text_len = regular_response.length(); + + while (search_pos < text_len) { + size_t marker_pos = regular_response.find(FUNCTION_CALL_MARKER, search_pos); + if (marker_pos == std::string::npos) break; + + size_t json_start = regular_response.find('{', marker_pos); + if (json_start == std::string::npos) break; + + int brace_count = 1; + size_t json_end = json_start + 1; + while (json_end < text_len && brace_count > 0) { + char c = regular_response[json_end]; + brace_count += (c == '{') - (c == '}'); + json_end++; + } + + if (brace_count == 0) { + function_calls.push_back(regular_response.substr(json_start, json_end - json_start)); + regular_response = regular_response.substr(0, marker_pos); + size_t last_bracket = regular_response.rfind('{'); + if(last_bracket != std::string::npos) { + regular_response = regular_response.substr(0, last_bracket); + } + } + search_pos = json_end; + } +} + +inline std::vector> find_channel_token_ranges( + const std::vector& tokens, size_t offset, + uint32_t channel_open_id, uint32_t channel_close_id) { + std::vector> ranges; + size_t pos = 0; + while (pos < tokens.size()) { + if (tokens[pos] != channel_open_id) { + pos++; + continue; + } + + size_t block_start = pos; + pos++; + while (pos < tokens.size() && tokens[pos] != channel_close_id) { + pos++; + } + if (pos < tokens.size()) { + pos++; + } + ranges.push_back({offset + block_start, pos - block_start}); + } + return ranges; +} + +inline void strip_tag_blocks(std::string& text, std::string& extracted, + const std::string& open_tag, const std::string& close_tag) { + std::string result; + size_t pos = 0; + + size_t first_close = text.find(close_tag); + size_t first_open = text.find(open_tag); + if (first_close != std::string::npos && + (first_open == std::string::npos || first_close < first_open)) { + extracted += text.substr(0, first_close); + pos = first_close + close_tag.size(); + } + + while (pos < text.size()) { + size_t open_pos = text.find(open_tag, pos); + if (open_pos == std::string::npos) { + result += text.substr(pos); + break; + } + result += text.substr(pos, open_pos - pos); + size_t content_start = open_pos + open_tag.size(); + size_t close_pos = text.find(close_tag, content_start); + if (close_pos == std::string::npos) { + if (!extracted.empty()) extracted += "\n"; + extracted += text.substr(content_start); + break; + } + if (!extracted.empty()) extracted += "\n"; + extracted += text.substr(content_start, close_pos - content_start); + pos = close_pos + close_tag.size(); + } + text = result; +} + +inline void strip_thinking_block(const std::string& input, std::string& thinking, std::string& content) { + thinking.clear(); + content = input; + + auto trim = [](std::string& s) { + size_t first = s.find_first_not_of(" \t\n\r"); + size_t last = s.find_last_not_of(" \t\n\r"); + if (first != std::string::npos && last != std::string::npos) + s = s.substr(first, last - first + 1); + else + s.clear(); + }; + + if (content.find("<|channel>") != std::string::npos || content.find("") != std::string::npos) { + strip_tag_blocks(content, thinking, "<|channel>", ""); + } else if (content.find("") != std::string::npos || content.find("") != std::string::npos) { + strip_tag_blocks(content, thinking, "", ""); + } else { + return; + } + + trim(thinking); + trim(content); +} + +struct TranscriptSegment { + float start; + float end; + std::string text; +}; + +inline std::string construct_response_json(const std::string& regular_response, + const std::vector& function_calls, + double time_to_first_token, + double total_time_ms, + double prefill_tps, + double decode_tps, + size_t prompt_tokens, + size_t completion_tokens, + float confidence = 0.0f, + bool cloud_handoff = false, + const std::string& thinking = "", + const std::vector& segments = {}) { + std::ostringstream json; + json << "{"; + json << "\"success\":true,"; + json << "\"error\":null,"; + json << "\"cloud_handoff\":" << (cloud_handoff ? "true" : "false") << ","; + json << "\"response\":\"" << escape_json_string(regular_response) << "\","; + if (!thinking.empty()) { + json << "\"thinking\":\"" << escape_json_string(thinking) << "\","; + } + json << "\"function_calls\":["; + for (size_t i = 0; i < function_calls.size(); ++i) { + if (i > 0) json << ","; + json << function_calls[i]; + } + json << "],"; + json << "\"segments\":["; + for (size_t i = 0; i < segments.size(); ++i) { + if (i > 0) json << ","; + json << "{\"start\":" << std::fixed << std::setprecision(3) << segments[i].start + << ",\"end\":" << std::fixed << std::setprecision(3) << segments[i].end + << ",\"text\":\"" << escape_json_string(segments[i].text) << "\"}"; + } + json << "],"; + json << "\"confidence\":" << std::fixed << std::setprecision(4) << confidence << ","; + json << "\"time_to_first_token_ms\":" << std::fixed << std::setprecision(2) << time_to_first_token << ","; + json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; + json << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << ","; + json << "\"decode_tps\":" << std::fixed << std::setprecision(2) << decode_tps << ","; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; + json << "\"prefill_tokens\":" << prompt_tokens << ","; + json << "\"decode_tokens\":" << completion_tokens << ","; + json << "\"total_tokens\":" << (prompt_tokens + completion_tokens); + json << "}"; + return json.str(); +} + +inline std::string serialize_function_calls(const std::vector& calls) { + if (calls.empty()) return "[]"; + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < calls.size(); ++i) { + if (i > 0) oss << ","; + oss << calls[i]; + } + oss << "]"; + return oss.str(); +} + +inline int validate_audio_params( + const char* component, + void* model, + char* response_buffer, size_t buffer_size, + const char* audio_file_path, + const uint8_t* pcm_buffer, size_t pcm_buffer_size) { + if (!model) { + std::string err = last_error_message.empty() ? "Model not initialized." : last_error_message; + CACTUS_LOG_ERROR(component, err); + handle_error_response(err, response_buffer, buffer_size); + return -1; + } + if (!response_buffer || buffer_size == 0) { + CACTUS_LOG_ERROR(component, "Invalid parameters: response_buffer or buffer_size"); + handle_error_response("Invalid parameters", response_buffer, buffer_size); + return -1; + } + if (!audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { + CACTUS_LOG_ERROR(component, "No audio input provided"); + handle_error_response("Either audio_file_path or pcm_buffer must be provided", response_buffer, buffer_size); + return -1; + } + if (audio_file_path && pcm_buffer && pcm_buffer_size > 0) { + CACTUS_LOG_ERROR(component, "Both audio_file_path and pcm_buffer provided"); + handle_error_response("Cannot provide both audio_file_path and pcm_buffer", response_buffer, buffer_size); + return -1; + } + if (pcm_buffer && pcm_buffer_size > 0 && (pcm_buffer_size < 2 || pcm_buffer_size % 2 != 0)) { + CACTUS_LOG_ERROR(component, "Invalid pcm_buffer_size"); + handle_error_response("pcm_buffer_size must be even and at least 2 bytes", response_buffer, buffer_size); + return -1; + } + return 0; +} + +inline std::vector pcm_to_float(const uint8_t* pcm_buffer, size_t pcm_buffer_size) { + const int16_t* samples = reinterpret_cast(pcm_buffer); + size_t n = pcm_buffer_size / 2; + std::vector out(n); + for (size_t i = 0; i < n; ++i) + out[i] = static_cast(samples[i]) / 32768.0f; + return out; +} + +} // namespace ffi +} // namespace cactus + +#ifdef __cplusplus +extern "C" { +#endif + +const char* cactus_get_last_error(); + +#ifdef __cplusplus +} +#endif + +#endif // CACTUS_UTILS_H diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/engine.h b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/engine.h new file mode 100644 index 0000000..afebfeb --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/engine.h @@ -0,0 +1,1104 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../graph/graph.h" + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc99-extensions" +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +extern "C" { + #include "../../libs/stb/stb_image.h" + #include "../../libs/stb/stb_image_resize2.h" +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + +class CactusGraph; + +namespace cactus { +namespace npu { + class NPUPrefill; +} +namespace engine { + +class Siglip2Preprocessor; + +struct Config { + uint32_t vocab_size = 151936; + uint32_t bos_token_id = 151643; + uint32_t eos_token_id = 151645; + uint32_t num_layers = 28; + uint32_t hidden_dim = 1024; + uint32_t ffn_intermediate_dim = 3072; + uint32_t attention_heads = 16; + uint32_t attention_kv_heads = 8; + uint32_t attention_head_dim = 128; + float layer_norm_eps = 1e-6f; + float rope_theta = 1000000.0f; + uint32_t num_experts = 0; + uint32_t num_shared_experts = 0; + uint32_t num_top_experts = 0; + uint32_t moe_every_n_layers = 0; + uint32_t moe_intermediate_dim = 0; + uint32_t num_dense_layers = 0; + uint32_t num_experts_per_tok = 0; + bool norm_topk_prob = false; + bool use_expert_bias = false; + float routed_scaling_factor = 1.0f; + bool tie_word_embeddings = true; + + uint32_t vision_hidden_dim = 0; + uint32_t vision_num_layers = 0; + uint32_t vision_attention_heads = 0; + uint32_t vision_image_size = 0; + uint32_t vision_patch_size = 0; + uint32_t vision_num_channels = 3; + uint32_t vision_embed_dim = 0; + uint32_t visual_tokens_per_img = 0; + bool use_pixel_shuffle = false; + uint32_t pixel_shuffle_factor = 1; + bool use_image_tokens = false; + uint32_t image_token_id = 0; + bool use_layout_tags = false; + uint32_t image_seq_len = 64; + + uint32_t global_image_size = 2048; + uint32_t max_tile_size = 512; + float rescale_factor = 0.00392156862745098f; + float image_mean = 0.5f; + float image_std = 0.5f; + + uint32_t downsample_factor = 2; + uint32_t min_tiles = 2; + uint32_t max_tiles = 10; + bool use_thumbnail = true; + uint32_t min_image_tokens = 64; + uint32_t max_image_tokens = 256; + uint32_t max_num_patches = 1024; + uint32_t tile_size = 512; + float max_pixels_tolerance = 2.0f; + bool do_image_splitting = true; + bool encoder_act_gelu = false; + bool decoder_act_gelu = false; + uint32_t num_encoder_layers = 0; + uint32_t num_decoder_layers = 0; + float partial_rotary_factor = 0.0f; + uint32_t pad_token_id = 0; + uint32_t conv_kernel_size = 0; + uint32_t subsampling_conv_kernel_size = 0; + uint32_t subsampling_conv_stride = 0; + uint32_t subsampling_conv_channels = 0; + uint32_t subsampling_factor = 0; + uint32_t num_mel_bins = 80; + std::string encoder_hidden_act = "silu"; + uint32_t linear_num_key_heads = 0; + uint32_t linear_key_head_dim = 0; + uint32_t linear_num_value_heads = 0; + uint32_t linear_value_head_dim = 0; + uint32_t linear_q_proj_dim = 0; + uint32_t linear_k_proj_dim = 0; + uint32_t linear_v_proj_dim = 0; + + uint32_t kv_lora_rank = 0; + uint32_t q_lora_rank = 0; + uint32_t qk_head_dim = 0; + uint32_t qk_nope_head_dim = 0; + uint32_t qk_rope_head_dim = 0; + uint32_t v_head_dim = 0; + uint32_t rope_interleave = 0; + bool attention_bias = false; + float rope_scaling_factor = 1.0f; + float rope_mscale_all_dim = 0.0f; + + enum class ModelType {QWEN = 0, GEMMA = 1, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7, MOONSHINE = 8, SILERO_VAD = 9, PARAKEET = 10, QWEN3P5 = 11, PARAKEET_TDT = 12, GEMMA3N = 13, YOUTU = 14, GEMMA4 = 15, PYANNOTE = 16, WESPEAKER = 17, NEEDLE = 18}; + uint32_t predictor_hidden_dim = 0; + uint32_t predictor_num_layers = 0; + uint32_t tdt_joint_dim = 0; + uint32_t tdt_num_durations = 0; + uint32_t tdt_blank_id = 0; + std::vector tdt_durations; + + ModelType model_type = ModelType::QWEN; + + enum class ModelVariant {DEFAULT = 0, VLM = 1, EXTRACT = 2, RAG = 3}; + ModelVariant model_variant = ModelVariant::DEFAULT; + + enum class Activation {GELU = 0, SILU = 1}; + Activation activation = Activation::SILU; + + enum class Backend {CPU = 0, NPU = 1}; + Backend default_backend = Backend::CPU; + + enum class Precision {INT8 = 0, FP16 = 1, FP32 = 2}; + Precision precision = Precision::FP32; + + float default_temperature = 0.6f; + float default_top_p = 0.95f; + size_t default_top_k = 20; + float default_max_tps = -1.0f; + float default_cloud_handoff_threshold = 0.0f; + size_t default_rolling_entropy_window = 10; + + std::vector layer_types; + size_t conv_L_cache = 0; + + uint32_t altup_num_inputs = 4; + uint32_t laurel_rank = 64; + uint32_t hidden_size_per_layer_input = 256; + uint32_t num_kv_shared_layers = 0; + uint32_t sliding_window = 512; + float rope_local_base_freq = 10000.0f; + float final_logit_softcapping = 0.0f; + float global_partial_rotary_factor = 1.0f; + uint32_t expert_intermediate_size = 0; + uint32_t global_head_dim = 0; + uint32_t num_global_kv_heads = 0; + bool attention_k_eq_v = false; + bool enable_moe_block = false; + std::vector activation_sparsity_ppf; + + uint32_t vision_head_dim = 64; + uint32_t vision_kv_heads = 12; + uint32_t vision_intermediate_size = 3072; + uint32_t vision_position_embedding_size = 10240; + uint32_t vision_pooling_kernel_size = 3; + uint32_t vision_default_output_length = 280; + float vision_rope_theta = 100.0f; + + uint32_t audio_hidden_dim = 0; + uint32_t audio_num_layers = 0; + uint32_t audio_num_heads = 0; + uint32_t audio_head_dim = 0; + uint32_t audio_input_feat_size = 128; + uint32_t audio_conf_conv_kernel_size = 5; + uint32_t audio_chunk_size = 12; + uint32_t audio_context_left = 13; + uint32_t audio_context_right = 0; + float audio_logit_cap = 50.0f; + float audio_residual_weight = 0.5f; + uint32_t audio_output_proj_dims = 0; + uint32_t audio_vocab_size = 128; + uint32_t audio_vocab_offset = 0; + uint32_t audio_soft_tokens = 188; + uint32_t audio_sscp_conv0_channels = 128; + uint32_t audio_sscp_conv1_channels = 32; + float audio_sscp_conv_eps = 1e-3f; + float audio_rms_norm_eps = 1e-6f; + uint32_t audio_fft_length = 1024; + uint32_t audio_token_id = 0; + bool audio_fft_overdrive = false; + uint32_t channel_open_token_id = 100; + uint32_t channel_close_token_id = 101; + + static bool is_gemma_family(ModelType t) { + return t == ModelType::GEMMA || t == ModelType::GEMMA3N || t == ModelType::GEMMA4; + } + + bool from_json(const std::string& json_path); + std::string to_json() const; +}; + + + +struct MergeRule { + std::string first; + std::string second; + std::string merged; + uint32_t priority; + + MergeRule(const std::string& f, const std::string& s, const std::string& m, uint32_t p) + : first(f), second(s), merged(m), priority(p) {} +}; + + +struct ToolCallInfo { + std::string name; + std::string arguments; +}; + +struct ChatMessage { + std::string role; + std::string content; + std::string name; + std::vector images; + std::vector audio; + size_t audio_soft_token_count = 0; + std::vector tool_calls; +}; + +inline std::string format_needle_query_text(const std::vector& messages) { + std::string system_text; + std::string user_query; + + for (const auto& msg : messages) { + if (msg.role == "system") { + if (!system_text.empty()) { + system_text += "\n"; + } + system_text += msg.content; + } else if (msg.role == "user") { + user_query = msg.content; + } + } + + if (user_query.empty() && !messages.empty()) { + user_query = messages.back().content; + } + if (system_text.empty()) { + return user_query; + } + if (user_query.empty()) { + return system_text; + } + return system_text + "\n\n" + user_query; +} + +struct ToolConstraintSpec { + std::string name; + std::vector parameter_names; + std::vector required_parameter_names; +}; + +struct TokenizerRuntimeConfig { + enum class TokenizerType { UNKNOWN, BPE, SENTENCEPIECE }; + enum class VocabFormat { UNKNOWN, ID_TAB_TOKEN, LINE_TOKEN }; + enum class Normalizer { NONE, METASPACE, BYTE_LEVEL }; + enum class Decoder { NONE, REPLACE_METASPACE, BYTE_LEVEL }; + + TokenizerType tokenizer_type = TokenizerType::UNKNOWN; + VocabFormat vocab_format = VocabFormat::UNKNOWN; + Normalizer normalizer = Normalizer::NONE; + Decoder decoder = Decoder::NONE; + bool byte_fallback = false; + bool has_chat_template = false; +}; + +TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file); +void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens); +std::vector split_with_special_tokens(const std::string& text, const std::unordered_map& special_tokens); + +inline std::string extract_json_string(const std::string& json, size_t& pos) { + std::string value; + while (pos < json.size() && json[pos] != '"') { + if (json[pos] == '\\' && pos + 1 < json.size()) { + pos++; + if (json[pos] == 'n') value += '\n'; + else if (json[pos] == 't') value += '\t'; + else if (json[pos] == 'r') value += '\r'; + else if (json[pos] == '"') value += '"'; + else if (json[pos] == '\\') value += '\\'; + else value += json[pos]; + } else { + value += json[pos]; + } + pos++; + } + if (pos < json.size()) pos++; + return value; +} + +class Tokenizer { +public: + virtual ~Tokenizer() = default; + + virtual std::vector encode(const std::string& text) const = 0; + virtual std::string decode(const std::vector& tokens) const = 0; + + virtual std::vector apply_chat_template(const std::vector& messages, bool add_generation_prompt = true) const; + virtual std::string format_chat_prompt(const std::vector& messages, bool add_generation_prompt = true, const std::string& tools_json = "", bool enable_thinking_if_supported = false) const; + + virtual uint32_t get_vocab_size() const = 0; + virtual uint32_t get_unk_token() const = 0; + virtual uint32_t get_bos_token() const = 0; + virtual uint32_t get_eos_token() const = 0; + virtual bool has_chat_template() const { return has_chat_template_; } + std::string get_default_stop_sequence() const; + + virtual bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) = 0; + + uint32_t get_image_token_id() const { return image_token_id_; } + uint32_t get_fake_token_id() const { return fake_token_id_; } + uint32_t get_global_img_token_id() const { return global_img_token_id_; } + +protected: + enum class ModelType { UNKNOWN, QWEN, QWEN3P5, GEMMA, GEMMA4, LFM2, BERT, WHISPER, PARAKEET, YOUTU, NEEDLE}; + ModelType model_type_ = ModelType::UNKNOWN; + enum class ModelVariant { DEFAULT, VLM, EXTRACT, RAG}; + ModelVariant model_variant_ = ModelVariant::DEFAULT; + bool has_chat_template_ = false; + std::string chat_template_; + + uint32_t image_token_id_ = 396; + uint32_t fake_token_id_ = 49189; + uint32_t global_img_token_id_ = 49152; + + + uint32_t vision_patch_size_ = 16; + uint32_t vision_pooling_kernel_size_ = 3; + uint32_t vision_default_output_length_ = 280; + uint32_t vision_image_size_ = 768; + TokenizerRuntimeConfig runtime_config_; + + void detect_model_type(const std::string& config_path); + void load_chat_template(const std::string& template_file); + std::string format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; + std::string format_gemma_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_gemma4_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; + std::string format_lfm2_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_lfm2_vl_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_needle_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_youtu_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; +}; + +class BPETokenizer : public Tokenizer { +public: + BPETokenizer(); + ~BPETokenizer(); + + bool load_vocabulary_mmap(const std::string& vocab_file, const std::string& merges_file); + bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; + + std::vector encode(const std::string& text) const override; + std::string decode(const std::vector& tokens) const override; + + uint32_t get_vocab_size() const override { return vocab_size_; } + uint32_t get_unk_token() const override { return unk_token_id_; } + uint32_t get_bos_token() const override { return bos_token_id_; } + uint32_t get_eos_token() const override { return eos_token_id_; } + +private: + std::unordered_map token_to_id_; + std::vector id_to_token_; + std::vector merge_rules_; + std::unordered_map merge_map_; + + uint32_t vocab_size_; + uint32_t unk_token_id_; + uint32_t bos_token_id_; + uint32_t eos_token_id_; + + void* vocab_mmap_ptr_; + size_t vocab_mmap_size_; + + void* merges_mmap_ptr_; + size_t merges_mmap_size_; + + std::vector apply_bpe(const std::vector& tokens) const; + std::pair find_best_merge_fast(const std::vector& tokens) const; + + std::string bytes_to_unicode(const std::string& text) const; + std::string unicode_to_bytes(const std::string& text) const; + std::vector byte_level_split(const std::string& text) const; + std::vector utf8_split(const std::string& text) const; + + void cleanup_mmap(); + +private: + mutable std::unordered_map byte_to_unicode_; + mutable std::unordered_map unicode_to_byte_; + void init_byte_mappings() const; + + std::unordered_map special_tokens_; + std::vector split_with_special_tokens(const std::string& text) const; + void load_special_tokens(const std::string& config_file); +}; + +class SPTokenizer : public Tokenizer { +public: + SPTokenizer(); + ~SPTokenizer(); + + bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; + + std::vector encode(const std::string& text) const override; + std::string decode(const std::vector& tokens) const override; + + uint32_t get_vocab_size() const override { return vocab_size_; } + uint32_t get_unk_token() const override { return unk_token_id_; } + uint32_t get_bos_token() const override { return bos_token_id_; } + uint32_t get_eos_token() const override { return eos_token_id_; } + +private: + struct TrieNode { + std::unordered_map> children; + int32_t token_id = -1; + float score = 0.0f; + }; + + std::unique_ptr trie_root_; + std::unordered_map token_to_id_; + std::vector id_to_token_; + std::vector token_scores_; + + uint32_t vocab_size_; + uint32_t unk_token_id_; + uint32_t bos_token_id_; + uint32_t eos_token_id_; + uint32_t pad_token_id_; + + bool sp_bpe_mode_ = false; + bool sp_add_dummy_prefix_ = false; + bool sp_byte_fallback_ = false; + + void* vocab_mmap_ptr_; + size_t vocab_mmap_size_; + + void build_trie(); + std::vector> tokenize_with_trie(const std::string& text) const; + std::vector tokenize_with_bpe(const std::string& text) const; + std::string preprocess_text(const std::string& text) const; + std::string postprocess_text(const std::string& text) const; + std::vector split_by_unicode_spaces(const std::string& text) const; + + void cleanup_mmap(); + + std::unordered_map special_tokens_; + std::vector split_with_special_tokens(const std::string& text) const; + void load_special_tokens(const std::string& config_file); +}; + +class ConvCache { +public: + struct CircularView { + const void* ptr1; + size_t len1; + const void* ptr2; + size_t len2; + size_t total_len; + }; + + void init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision); + CircularView get_window(size_t layer) const; + void update(CactusGraph* gb, size_t layer, const size_t latest_token); + void reset(); + + bool is_empty() const { return num_layers == 0; } + + size_t num_layers = 0; + size_t hidden_size = 0; + size_t window_size = 0; + Precision precision = Precision::FP32; + size_t element_size = 4; + +private: + struct LayerState { + std::vector data; + size_t head = 0; + size_t count = 0; + }; + + std::vector layer_states; +}; + +struct KVCache { + static constexpr size_t DEFAULT_WINDOW_SIZE = 1024; + static constexpr size_t DEFAULT_SINK_SIZE = 4; + + struct LayerCache { + std::vector keys; + std::vector values; + std::vector key_scales; + std::vector value_scales; + size_t head_dim = 0; + size_t kv_heads = 0; + }; + + std::vector layer_caches; + + size_t window_size = DEFAULT_WINDOW_SIZE; + size_t sink_size = DEFAULT_SINK_SIZE; + size_t current_seq_len = 0; + size_t total_seq_len = 0; + size_t max_seq_len = 2048; + size_t num_layers = 0; + Precision precision; + size_t element_size = 4; + + void set_window_size(size_t window, size_t sink = DEFAULT_SINK_SIZE); + size_t get_effective_seq_len() const { return current_seq_len; } + size_t get_total_seq_len() const { return total_seq_len; } + size_t get_layer_head_dim(size_t layer_idx) const { return layer_caches[layer_idx].head_dim; } + size_t get_layer_kv_heads(size_t layer_idx) const { return layer_caches[layer_idx].kv_heads; } + + void init(size_t num_layers, size_t max_seq, const std::vector& layer_dims, const std::vector& layer_kv_heads, Precision model_precision); + void reset(); + void update_from_graph(CactusGraph* gb, const std::vector& k_nodes, + const std::vector& v_nodes, size_t seq_len, + size_t num_layers); + + void update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data, + size_t num_tokens, size_t kv_heads, size_t head_dim); + + bool is_empty() const { return current_seq_len == 0; } + void* get_key_ptr(size_t layer); + void* get_value_ptr(size_t layer); + + struct CircularView { + const void* ptr1; + const void* ptr2; + size_t len1; + size_t len2; + size_t total_len; + }; + + CircularView get_key_view(size_t layer); + CircularView get_value_view(size_t layer); + + const int8_t* get_keys_int8(size_t layer) const; + const int8_t* get_values_int8(size_t layer) const; + const float* get_key_scales(size_t layer) const; + const float* get_value_scales(size_t layer) const; + + void remove_token_range(size_t start, size_t count); + void compact_to_windows(const std::vector& target_windows); +}; + +class ToolCallConstrainer { +public: + enum class State { + DONE, + + QWEN_START, + QWEN_EXPECT_OPEN_BRACE, + QWEN_EXPECT_NAME_KEY, + QWEN_EXPECT_NAME_COLON, + QWEN_EXPECT_NAME_VALUE, + QWEN_EXPECT_COMMA, + QWEN_EXPECT_ARGS_KEY, + QWEN_EXPECT_ARGS_COLON, + QWEN_IN_ARGUMENTS, + QWEN_EXPECT_CLOSE_BRACE, + QWEN_EXPECT_END, + + NEEDLE_START, + + LFM_START, + LFM_EXPECT_BRACKET, + LFM_IN_FUNC_NAME, + LFM_EXPECT_PAREN, + LFM_IN_ARGUMENTS, + LFM_EXPECT_BRACKET_CLOSE, + LFM_EXPECT_END, + + GEMMA_START, + GEMMA_EXPECT_CALL, + GEMMA_IN_FUNC_NAME, + GEMMA_EXPECT_BRACE, + GEMMA_IN_ARGUMENTS, + GEMMA_EXPECT_END + }; + + void init(Config::ModelType model_type, + const std::vector& tools, + Tokenizer* tokenizer); + + const std::unordered_map& get_bias() const { return current_bias_; } + + void update(uint32_t token_id, const std::string& decoded_text); + + void reset(); + + bool is_active() const { return active_; } + +private: + bool active_ = false; + State state_ = State::QWEN_START; + Config::ModelType model_type_ = Config::ModelType::QWEN; + Tokenizer* tokenizer_ = nullptr; + + bool is_gemma_family() const { return Config::is_gemma_family(model_type_); } + bool is_needle() const { return model_type_ == Config::ModelType::NEEDLE; } + + enum class NeedleJsonState { + FREE, + IN_NAME, + IN_ARG_KEY, + }; + + struct NeedleTrieNode { + std::unordered_map> children; + bool is_terminal = false; + }; + + std::vector tool_specs_; + std::vector function_names_; + std::string generated_text_; + int brace_depth_ = 0; + + std::string call_start_tag_; + std::string call_end_tag_; + + std::unordered_set qwen_tool_call_start_tokens_; + std::unordered_set qwen_tool_call_end_tokens_; + std::unordered_set open_brace_tokens_; + std::unordered_set close_brace_tokens_; + std::unordered_set colon_tokens_; + std::unordered_set comma_tokens_; + std::unordered_set name_key_tokens_; + std::unordered_set args_key_tokens_; + std::unordered_set quote_tokens_; + std::unordered_set backtick_tokens_; + std::unordered_set all_func_name_tokens_; + std::unordered_map> func_name_sequences_; + NeedleJsonState needle_json_state_ = NeedleJsonState::FREE; + std::string needle_buffer_; + std::string needle_constrained_buf_; + std::string needle_current_function_; + bool needle_in_arguments_ = false; + int needle_arguments_depth_ = 0; + int needle_nesting_depth_ = 0; + bool needle_in_string_value_ = false; + bool needle_prev_char_escape_ = false; + std::unique_ptr needle_name_trie_; + std::unordered_map> needle_param_tries_; + std::vector needle_token_strings_; + std::unordered_map> needle_token_index_; + + std::unordered_set tool_start_tokens_; + std::unordered_set tool_end_tokens_; + std::unordered_set bracket_open_tokens_; + std::unordered_set bracket_close_tokens_; + std::unordered_set paren_open_tokens_; + std::unordered_set paren_close_tokens_; + std::unordered_set equals_tokens_; + + std::string lfm_current_function_; + std::string lfm_args_buffer_; + std::unordered_set lfm_seen_arg_keys_; + std::unordered_map> lfm_required_params_; + std::unordered_map> lfm_all_params_; + + std::unordered_set gemma_call_start_tokens_; + std::unordered_set gemma_call_end_tokens_; + std::unordered_set gemma_response_start_tokens_; + std::unordered_set gemma_call_prefix_tokens_; + std::unordered_set escape_tokens_; + + std::unordered_map current_bias_; + + void compute_bias(); + void tokenize_grammar_elements(); + void add_tokens_for_string(const std::string& str, std::unordered_set& token_set); + void add_tokens_for_prefix_string(const std::string& prefix, std::unordered_set& token_set); + void add_tokens_containing(char needle, std::unordered_set& token_set); + void tokenize_function_names(bool quote_names); + void init_common_tokens(); + void init_needle_constraints(); + void reset_needle_constraints(); + void feed_needle_text(const std::string& text); + void feed_needle_char(char ch); + bool needle_at_arg_key_start() const; + bool needle_is_value_string_start() const; + void needle_insert_word(NeedleTrieNode* root, const std::string& word); + const NeedleTrieNode* needle_get_trie_node(const NeedleTrieNode* root, const std::string& prefix) const; + bool needle_check_token_valid(const std::string& token_text, const NeedleTrieNode* trie_node) const; +}; + +class Model { +public: + struct DebugNode { + uint32_t layer_idx; + std::string name; + size_t node_id; + }; + + Model(); + explicit Model(const Config& config); + virtual ~Model(); + + const Config& get_config() const { return config_; } + Tokenizer* get_tokenizer() const { return tokenizer_.get(); } + const std::vector& get_debug_nodes() const; + + virtual bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true); + + virtual bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, + const std::string& system_prompt = "", bool do_warmup = true); + + virtual uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f); + + virtual void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = ""); + + virtual void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, + const std::string& profile_file = ""); + + virtual uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, + float temperature = -1.0f, float top_p = -1.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f); + + virtual uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, float temperature = 0.0f, float top_p = 0.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f, + float* out_token_time_start = nullptr, float* out_token_time_end = nullptr); + + std::vector get_embeddings(const std::vector& tokens, bool pooled = true, bool normalize = false, const std::string& profile_file = ""); + + virtual std::vector get_image_embeddings(const std::string& image_path); + + virtual std::vector get_audio_embeddings(const std::vector& audio_features); + + virtual void reset_cache() { kv_cache_.reset(); token_history_.clear(); } + void record_sampled_token(uint32_t token) { + if (token_history_.size() >= MAX_TOKEN_HISTORY) { + token_history_.erase(token_history_.begin(), token_history_.begin() + (MAX_TOKEN_HISTORY / 2)); + } + token_history_.push_back(token); + } + + double score_tokens_window_logprob(const std::vector& tokens, size_t start, size_t end, size_t context, size_t* tokens_scored); + + + + void set_cache_window(size_t window_size, size_t sink_size = 4) { kv_cache_.set_window_size(window_size, sink_size); } + size_t get_cache_size() const { return kv_cache_.current_seq_len; } + + bool load_npu_prefill(const std::string& model_path); + bool has_npu_prefill() const; + size_t get_prefill_chunk_size() const; + + virtual void remove_thinking_tokens(const std::vector>& ranges); + virtual void compact_kv_cache() {} + + virtual void set_tool_constraints(const std::vector& tools); + virtual void clear_tool_constraints(); + virtual void update_tool_constraints(uint32_t token_id); + + void* graph_handle_; + + void set_vocab_bias(const std::unordered_map& bias) { + vocab_bias_ = bias; + } + + void clear_vocab_bias() { + vocab_bias_.clear(); + } + + bool has_vocab_bias() const { + return !vocab_bias_.empty(); + } + + const std::unordered_map& get_vocab_bias() const { + return vocab_bias_; + } + +protected: + size_t sample_token(CactusGraph* gb, size_t logits_node_id, float temperature, float top_p, size_t top_k, + float min_p, float repetition_penalty, + const std::unordered_map* extra_bias = nullptr) const; + + static void compute_entropy(CactusGraph* gb, size_t logits_node_id, float* out_entropy); + + virtual size_t forward(const std::vector& tokens, bool use_cache = false) = 0; + + virtual size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false); + + virtual void load_weights_to_graph(CactusGraph* gb) = 0; + + virtual size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; + + virtual size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const = 0; + virtual size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; + void update_kv_cache(CactusGraph* gb, size_t seq_len); + virtual std::vector get_kv_layer_dims() const { + return std::vector(config_.num_layers, config_.attention_head_dim); + } + virtual std::vector get_kv_layer_heads() const { + return std::vector(config_.num_layers, config_.attention_kv_heads); + } + virtual void post_init() {} + virtual void post_execute_updates(CactusGraph*, size_t) {} + Config config_; + std::unique_ptr tokenizer_; + + bool initialized_; + float attention_scale_; + +protected: + KVCache kv_cache_; + std::vector cache_k_output_nodes_; + std::vector cache_v_output_nodes_; + + std::string embedding_file_path_; + size_t embedding_node_id_; + std::string model_folder_path_; + size_t output_weight_node_id_; + + mutable std::vector debug_nodes_; + + void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) const; + void clear_debug_nodes(); + + bool init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup); + bool owns_graph_; + + std::unique_ptr npu_prefill_; + void prefill_npu(const std::vector& tokens); + virtual std::vector<__fp16> get_token_embeddings(const std::vector& tokens); + + static constexpr size_t MAX_TOKEN_HISTORY = 128; + ToolCallConstrainer tool_constrainer_; + std::vector token_history_; + +private: + std::unordered_map vocab_bias_; +}; + +std::unique_ptr create_model(const std::string& model_folder); + +class Siglip2Preprocessor { +public: + struct Config { + int patch_size = 16; + int downsample_factor = 2; + int min_tiles = 2; + int max_tiles = 10; + bool use_thumbnail = true; + int min_image_tokens = 64; + int max_image_tokens = 256; + int max_num_patches = 1024; + int tile_size = 512; + float max_pixels_tolerance = 2.0f; + bool do_resize = true; + bool do_rescale = true; + bool do_normalize = true; + bool do_convert_rgb = true; + bool do_image_splitting = true; + float rescale_factor = 1.0f / 255.0f; + float image_mean[3] = {0.5f, 0.5f, 0.5f}; + float image_std[3] = {0.5f, 0.5f, 0.5f}; + }; + + struct PreprocessedImage { + std::vector pixel_values; + std::vector pixel_attention_mask; + std::vector> spatial_shapes; + std::vector pixel_values_shape; + std::vector pixel_attention_mask_shape; + std::vector spatial_shapes_shape; + int num_patches_height; + int num_patches_width; + int actual_num_patches; + int num_tiles; + int patch_dim; + int max_patches_per_tile; + + int image_rows; + int image_cols; + int image_height; + int image_width; + int tokens_per_tile; + int thumbnail_tokens; + + ~PreprocessedImage(); + }; + + struct SpatialShapeResult { + std::vector> shapes; + int grid_rows; + int grid_cols; + }; + + explicit Siglip2Preprocessor(const Config& config); + Siglip2Preprocessor(); + ~Siglip2Preprocessor(); + + PreprocessedImage preprocess_from_file(const std::string& image_path); + PreprocessedImage preprocess_from_memory(const unsigned char* img_data, int width, int height, int channels); + SpatialShapeResult compute_spatial_shapes(int height, int width); + +private: + Config config_; + + std::pair compute_pixel_limits() const; + std::vector convert_to_rgb(const unsigned char* img_data, int width, int height, int channels); + std::pair smart_resize(int height, int width); + bool is_image_too_large(int height, int width); + std::pair get_grid_layout(int height, int width); + std::pair find_closest_aspect_ratio(float aspect_ratio, int width, int height); + std::vector resize_image(const unsigned char* img_data, int src_width, int src_height, + int dst_width, int dst_height, int channels); + std::vector normalize_image(const float* img_data, int width, int height, int channels); + std::vector> convert_image_to_patches( + const std::vector& image, int width, int height, int channels, int patch_size); + PreprocessedImage pad_patches(const std::vector>& tile_patches, + const std::vector>& spatial_shapes, + int patch_dim, + int max_patches_per_tile); + int round_by_factor(int number, int factor); +}; + +class AudioProcessor { +public: + struct SpectrogramConfig { + size_t n_fft = 400; + size_t hop_length = 160; + size_t frame_length = 400; + float power = 2.0f; + bool center = true; + const char* pad_mode = "reflect"; + bool onesided = true; + float dither = 0.0f; + float mel_floor = 1e-10f; + const char* log_mel = nullptr; + float reference = 1.0f; + float min_value = 1e-10f; + bool remove_dc_offset = false; + float preemphasis = 0.0f; + bool hann_periodic = true; + float window_a0 = 0.5f; + size_t fft_override = 0; + bool mel_floor_additive = false; + }; + + AudioProcessor(); + ~AudioProcessor(); + + void init_mel_filters(size_t num_frequency_bins, size_t num_mel_filters, + float min_freq, float max_freq, size_t sampling_rate, + const char* norm = "slaney", const char* mel_scale = "slaney"); + + std::vector compute_spectrogram( + const std::vector& waveform, + const SpectrogramConfig& config); + + static std::vector compute_irfft( + const std::vector& complex_input, + size_t n, + const char* norm = "backward"); + + const std::vector& get_mel_filters() const { return mel_filters_; } + + size_t get_num_mel_filters() const { return num_mel_filters_; } + size_t get_num_frequency_bins() const { return num_frequency_bins_; } + +private: + std::vector mel_filters_; + size_t num_frequency_bins_; + size_t num_mel_filters_; +}; + +namespace index { + constexpr uint32_t MAGIC = 0x43414354; + constexpr uint32_t VERSION = 1; + + struct Document { + int id; + std::vector embedding; + std::string content; + std::string metadata; + }; + + struct QueryResult { + int doc_id; + float score; + + QueryResult(int doc_id, float score) : doc_id(doc_id), score(score) {} + }; + + struct QueryOptions { + size_t top_k = 10; + float score_threshold = -1.0f; + }; + + class Index { + public: + Index(const std::string& index_path, const std::string& data_path, size_t embedding_dim); + ~Index(); + + Index(const Index&) = delete; + Index& operator=(const Index&) = delete; + Index(Index&&) = delete; + Index& operator=(Index&&) = delete; + + void add_documents(const std::vector& documents); + void delete_documents(const std::vector& doc_ids); + std::vector get_documents(const std::vector& doc_ids); + std::vector> query(const std::vector>& embeddings, const QueryOptions& options); + void compact(); + + private: + struct IndexHeader { + uint32_t magic; + uint32_t version; + uint32_t embedding_dim; + uint32_t num_documents; + }; + + struct IndexEntry { + int32_t doc_id; + uint64_t data_offset; + uint8_t flags; // bit 0: tombstone + + const __fp16* embedding() const { + return reinterpret_cast(this + 1); + } + + static size_t size(size_t embedding_dim) { + return sizeof(IndexEntry) + embedding_dim * sizeof(__fp16); + } + }; + + struct DataHeader { + uint32_t magic; + uint32_t version; + }; + + struct DataEntry { + uint16_t content_len; + uint16_t metadata_len; + + const char* content() const { + return reinterpret_cast(this + 1); + } + + const char* metadata() const { + return content() + content_len; + } + }; + + void parse_index_header(); + void parse_data_header(); + void build_doc_id_map(); + void validate_documents(const std::vector& documents); + void validate_doc_ids(const std::vector& doc_ids); + ssize_t write_full(int fd, const void* buf, size_t count); + + std::unordered_map doc_id_map_; + + std::string index_path_, data_path_; + size_t embedding_dim_; + size_t index_entry_size_; + uint32_t num_documents_; + + int index_fd_, data_fd_; + void *mapped_index_, *mapped_data_; + size_t index_file_size_, data_file_size_; + }; +} // namespace index + +} +} diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/gemma_tools.h b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/gemma_tools.h new file mode 100644 index 0000000..f0f9fe2 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/gemma_tools.h @@ -0,0 +1,576 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace gemma { + +inline std::string to_upper(const std::string& s) { + std::string result = s; + for (auto& c : result) c = std::toupper(c); + return result; +} + +inline std::string escape(const std::string& s) { + return "" + s + ""; +} + +inline void skip_whitespace(const std::string& json, size_t& pos) { + while (pos < json.length() && std::isspace(json[pos])) pos++; +} + +inline std::string extract_json_string(const std::string& json, size_t& pos) { + std::string value; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\' && pos + 1 < json.length()) { + pos++; + if (json[pos] == 'n') value += '\n'; + else if (json[pos] == 't') value += '\t'; + else if (json[pos] == 'r') value += '\r'; + else if (json[pos] == '"') value += '"'; + else if (json[pos] == '\\') value += '\\'; + else value += json[pos]; + } else { + value += json[pos]; + } + pos++; + } + if (pos < json.length()) pos++; + return value; +} + +std::string format_argument(const std::string& json, size_t& pos, bool escape_keys); +std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/); + +inline std::string format_argument(const std::string& json, size_t& pos, bool escape_keys = true) { + skip_whitespace(json, pos); + if (pos >= json.length()) return ""; + + char c = json[pos]; + + if (c == '"') { + pos++; + std::string value = extract_json_string(json, pos); + return escape(value); + } else if (c == '{') { + std::string result = "{"; + pos++; + bool first = true; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == '}') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + if (json[pos] != '"') break; + pos++; + std::string key = extract_json_string(json, pos); + + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == ':') pos++; + + std::string value = format_argument(json, pos, escape_keys); + + if (!first) result += ","; + first = false; + if (escape_keys) { + result += escape(key) + ":" + value; + } else { + result += key + ":" + value; + } + } + result += "}"; + return result; + } else if (c == '[') { + std::string result = "["; + pos++; + bool first = true; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == ']') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + std::string value = format_argument(json, pos, escape_keys); + + if (!first) result += ","; + first = false; + result += value; + } + result += "]"; + return result; + } else if (json.compare(pos, 4, "true") == 0) { + pos += 4; + return "true"; + } else if (json.compare(pos, 5, "false") == 0) { + pos += 5; + return "false"; + } else if (json.compare(pos, 4, "null") == 0) { + pos += 4; + return "null"; + } else { + size_t start = pos; + while (pos < json.length() && (std::isdigit(json[pos]) || json[pos] == '.' || + json[pos] == '-' || json[pos] == '+' || json[pos] == 'e' || json[pos] == 'E')) { + pos++; + } + return json.substr(start, pos - start); + } +} + +inline std::map parse_json_object_raw(const std::string& json, size_t& pos) { + std::map result; + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] != '{') return result; + pos++; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == '}') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + if (json[pos] != '"') break; + pos++; + std::string key = extract_json_string(json, pos); + + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == ':') pos++; + skip_whitespace(json, pos); + + size_t value_start = pos; + if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + pos++; + } else if (json[pos] == '{') { + int depth = 1; + pos++; + while (pos < json.length() && depth > 0) { + if (json[pos] == '{') depth++; + else if (json[pos] == '}') depth--; + else if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + } else if (json[pos] == '[') { + int depth = 1; + pos++; + while (pos < json.length() && depth > 0) { + if (json[pos] == '[') depth++; + else if (json[pos] == ']') depth--; + else if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + } else { + while (pos < json.length() && json[pos] != ',' && json[pos] != '}') pos++; + } + result[key] = json.substr(value_start, pos - value_start); + } + return result; +} + +inline std::string get_json_string_value(const std::string& json, size_t pos) { + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == '"') { + pos++; + return extract_json_string(json, pos); + } + return ""; +} + +inline std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/) { + static const std::set standard_keys = {"description", "type", "properties", "required", "nullable"}; + + size_t pos = 0; + auto properties = parse_json_object_raw(properties_json, pos); + + std::string result; + bool first = true; + + for (const auto& [key, value_json] : properties) { + if (standard_keys.count(key)) continue; + + if (!first) result += ","; + first = false; + + size_t prop_pos = 0; + auto prop_obj = parse_json_object_raw(value_json, prop_pos); + + result += key + ":{"; + + if (prop_obj.count("description")) { + std::string desc = get_json_string_value(prop_obj["description"], 0); + result += "description:" + escape(desc); + } + + std::string type_val; + if (prop_obj.count("type")) { + type_val = get_json_string_value(prop_obj["type"], 0); + } + + if (to_upper(type_val) == "STRING") { + if (prop_obj.count("enum")) { + size_t enum_pos = 0; + std::string enum_formatted = format_argument(prop_obj["enum"], enum_pos, true); + result += ",enum:" + enum_formatted; + } + } else if (to_upper(type_val) == "OBJECT") { + if (prop_obj.count("properties")) { + std::string nested_required; + if (prop_obj.count("required")) { + nested_required = prop_obj["required"]; + } + result += ",properties:{" + format_parameters(prop_obj["properties"], nested_required) + "}"; + } + if (prop_obj.count("required")) { + std::string req_items; + size_t req_pos = 0; + skip_whitespace(prop_obj["required"], req_pos); + if (req_pos < prop_obj["required"].length() && prop_obj["required"][req_pos] == '[') { + req_pos++; + bool req_first = true; + while (req_pos < prop_obj["required"].length()) { + skip_whitespace(prop_obj["required"], req_pos); + if (prop_obj["required"][req_pos] == ']') break; + if (prop_obj["required"][req_pos] == ',') { req_pos++; continue; } + if (prop_obj["required"][req_pos] == '"') { + req_pos++; + std::string req_item = extract_json_string(prop_obj["required"], req_pos); + if (!req_first) req_items += ","; + req_first = false; + req_items += escape(req_item); + } + } + } + if (!req_items.empty()) { + result += ",required:[" + req_items + "]"; + } + } + } else if (to_upper(type_val) == "ARRAY") { + if (prop_obj.count("items")) { + result += ",items:{"; + size_t items_pos = 0; + auto items_obj = parse_json_object_raw(prop_obj["items"], items_pos); + bool items_first = true; + + for (const auto& [item_key, item_value] : items_obj) { + if (!items_first) result += ","; + items_first = false; + + if (item_key == "properties") { + std::string items_required; + if (items_obj.count("required")) { + items_required = items_obj["required"]; + } + result += "properties:{" + format_parameters(item_value, items_required) + "}"; + } else if (item_key == "required") { + result += "required:["; + size_t req_pos = 0; + skip_whitespace(item_value, req_pos); + if (req_pos < item_value.length() && item_value[req_pos] == '[') { + req_pos++; + bool req_first = true; + while (req_pos < item_value.length()) { + skip_whitespace(item_value, req_pos); + if (item_value[req_pos] == ']') break; + if (item_value[req_pos] == ',') { req_pos++; continue; } + if (item_value[req_pos] == '"') { + req_pos++; + std::string req_item = extract_json_string(item_value, req_pos); + if (!req_first) result += ","; + req_first = false; + result += escape(req_item); + } + } + } + result += "]"; + } else if (item_key == "type") { + std::string item_type = get_json_string_value(item_value, 0); + result += "type:" + escape(to_upper(item_type)); + } else { + size_t val_pos = 0; + result += item_key + ":" + format_argument(item_value, val_pos, true); + } + } + result += "}"; + } + } + + if (!type_val.empty()) { + result += ",type:" + escape(to_upper(type_val)); + } + + result += "}"; + } + + return result; +} + +inline std::string format_function_declaration(const std::string& name, + const std::string& description, + const std::string& params_json) { + std::string result = "declaration:" + name + "{"; + result += "description:" + escape(description); + + if (!params_json.empty()) { + result += ",parameters:{"; + + size_t pos = 0; + auto params = parse_json_object_raw(params_json, pos); + + if (params.count("properties")) { + std::string required_json; + if (params.count("required")) { + required_json = params["required"]; + } + result += "properties:{" + format_parameters(params["properties"], required_json) + "}"; + } + + if (params.count("required")) { + std::string req_items; + size_t req_pos = 0; + skip_whitespace(params["required"], req_pos); + if (req_pos < params["required"].length() && params["required"][req_pos] == '[') { + req_pos++; + bool first = true; + while (req_pos < params["required"].length()) { + skip_whitespace(params["required"], req_pos); + if (params["required"][req_pos] == ']') break; + if (params["required"][req_pos] == ',') { req_pos++; continue; } + if (params["required"][req_pos] == '"') { + req_pos++; + std::string item = extract_json_string(params["required"], req_pos); + if (!first) req_items += ","; + first = false; + req_items += escape(item); + } + } + } + if (!req_items.empty()) { + result += ",required:[" + req_items + "]"; + } + } + + if (params.count("type")) { + std::string type_val = get_json_string_value(params["type"], 0); + result += ",type:" + escape(to_upper(type_val)); + } + + result += "}"; + } + + result += "}"; + return result; +} + +template +inline std::string format_tools(const std::vector& tools, bool use_pipe_tags = false) { + if (tools.empty()) return ""; + + const char* decl_start = use_pipe_tags ? "<|tool>" : ""; + const char* decl_end = use_pipe_tags ? "" : ""; + + std::string result; + for (const auto& tool : tools) { + result += decl_start; + std::string params_json; + auto it = tool.parameters.find("schema"); + if (it != tool.parameters.end()) { + params_json = it->second; + } + + result += format_function_declaration(tool.name, tool.description, params_json); + result += decl_end; + } + return result; +} + + +inline size_t match_quote_tag(const std::string& s, size_t pos) { + if (s.compare(pos, 8, "") == 0) return 8; + if (s.compare(pos, 5, "<|\"|>") == 0) return 5; + return 0; +} + +inline size_t find_quote_tag(const std::string& s, size_t pos) { + size_t e = s.find("", pos); + size_t t = s.find("<|\"|>", pos); + if (e == std::string::npos) return t; + if (t == std::string::npos) return e; + return std::min(e, t); +} + +inline std::string unescape(const std::string& s) { + const std::string ESCAPE_TAG = ""; + std::string result = s; + size_t pos = 0; + while ((pos = result.find(ESCAPE_TAG, pos)) != std::string::npos) { + result.erase(pos, ESCAPE_TAG.length()); + } + return result; +} + +inline std::string args_to_json(const std::string& args_content) { + std::string result = "{"; + size_t pos = 0; + bool first = true; + + if (!args_content.empty() && args_content[0] == '{') pos = 1; + + while (pos < args_content.length()) { + while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; + if (pos >= args_content.length() || args_content[pos] == '}') break; + if (args_content[pos] == ',') { pos++; continue; } + + size_t key_start = pos; + while (pos < args_content.length() && args_content[pos] != ':') pos++; + std::string key = args_content.substr(key_start, pos - key_start); + if (pos < args_content.length()) pos++; + + std::string value; + while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; + + if (pos < args_content.length()) { + size_t qtag_len = match_quote_tag(args_content, pos); + if (qtag_len > 0) { + pos += qtag_len; + size_t val_end = find_quote_tag(args_content, pos); + if (val_end != std::string::npos) { + value = "\"" + args_content.substr(pos, val_end - pos) + "\""; + pos = val_end + match_quote_tag(args_content, val_end); + } + } else if (args_content[pos] == '{') { + int depth = 1; + size_t start = pos; + pos++; + while (pos < args_content.length() && depth > 0) { + if (args_content[pos] == '{') depth++; + else if (args_content[pos] == '}') depth--; + pos++; + } + value = args_to_json(args_content.substr(start, pos - start)); + } else if (args_content[pos] == '[') { + int depth = 1; + size_t start = pos; + pos++; + while (pos < args_content.length() && depth > 0) { + if (args_content[pos] == '[') depth++; + else if (args_content[pos] == ']') depth--; + pos++; + } + std::string arr_content = args_content.substr(start + 1, pos - start - 2); + value = "["; + size_t arr_pos = 0; + bool first_item = true; + while (arr_pos < arr_content.length()) { + while (arr_pos < arr_content.length() && (std::isspace(arr_content[arr_pos]) || arr_content[arr_pos] == ',')) arr_pos++; + if (arr_pos >= arr_content.length()) break; + + if (!first_item) value += ","; + first_item = false; + + size_t aq_len = match_quote_tag(arr_content, arr_pos); + if (aq_len > 0) { + arr_pos += aq_len; + size_t end = find_quote_tag(arr_content, arr_pos); + if (end != std::string::npos) { + value += "\"" + arr_content.substr(arr_pos, end - arr_pos) + "\""; + arr_pos = end + match_quote_tag(arr_content, end); + } + } else { + size_t end = arr_content.find_first_of(",]", arr_pos); + if (end == std::string::npos) end = arr_content.length(); + value += arr_content.substr(arr_pos, end - arr_pos); + arr_pos = end; + } + } + value += "]"; + } else { + size_t val_start = pos; + while (pos < args_content.length() && args_content[pos] != ',' && args_content[pos] != '}') { + pos++; + } + value = args_content.substr(val_start, pos - val_start); + while (!value.empty() && std::isspace(value.back())) value.pop_back(); + } + } + + if (!first) result += ","; + first = false; + result += "\"" + key + "\":" + value; + } + + result += "}"; + return result; +} + +inline void parse_function_calls(std::string& response, std::vector& function_calls) { + + const std::string CALL_START = (response.find("<|tool_call>") != std::string::npos) + ? "<|tool_call>" : ""; + const std::string CALL_END = (CALL_START == "<|tool_call>") + ? "" : ""; + size_t pos = 0; + + while ((pos = response.find(CALL_START, pos)) != std::string::npos) { + size_t content_start = pos + CALL_START.length(); + size_t call_end_pos = response.find(CALL_END, content_start); + + size_t content_end = (call_end_pos != std::string::npos) ? call_end_pos : response.length(); + std::string call_content = response.substr(content_start, content_end - content_start); + + if (call_content.compare(0, 5, "call:") == 0) { + size_t brace_pos = call_content.find('{'); + + if (brace_pos == std::string::npos) { + size_t sep_pos = call_content.find_first_of(", ", 5); + if (sep_pos != std::string::npos) { + std::string func_name = call_content.substr(5, sep_pos - 5); + size_t args_start = sep_pos + 1; + while (args_start < call_content.length() && + (call_content[args_start] == ' ' || call_content[args_start] == ',')) { + args_start++; + } + std::string args_content = "{" + call_content.substr(args_start); + if (args_content.back() != '}') args_content += "}"; + + std::string args_json = args_to_json(args_content); + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; + function_calls.push_back(json_call); + } + } else { + std::string func_name = call_content.substr(5, brace_pos - 5); + std::string args_content = call_content.substr(brace_pos); + if (args_content.back() != '}') args_content += "}"; + + std::string args_json = args_to_json(args_content); + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; + function_calls.push_back(json_call); + } + } + + size_t erase_end = (call_end_pos != std::string::npos) ? + call_end_pos + CALL_END.length() : response.length(); + response.erase(pos, erase_end - pos); + } +} + +} // namespace gemma \ No newline at end of file diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/graph.h b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/graph.h new file mode 100644 index 0000000..b436177 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/graph.h @@ -0,0 +1,779 @@ +#ifndef GRAPH_H +#define GRAPH_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { + +enum class LogLevel { + DEBUG = 0, + INFO = 1, + WARN = 2, + ERROR = 3, + NONE = 4 +}; + +class Logger { +public: + static Logger& instance() { + static Logger logger; + return logger; + } + + void set_level(LogLevel level) { min_level_ = level; } + LogLevel get_level() const { return min_level_; } + + void set_callback(std::function cb) { + std::lock_guard lock(mutex_); + callback_ = cb; + } + + void log(LogLevel level, const std::string& component, const std::string& message) { + if (level < min_level_) return; + + std::lock_guard lock(mutex_); + + if (callback_) { + callback_(level, component, message); + } else { + std::cerr << "[" << level_string(level) << "] [" << component << "] " << message << std::endl; + } + + if (level == LogLevel::ERROR) { + last_error_ = "[" + component + "] " + message; + } + } + + const std::string& last_error() const { return last_error_; } + void clear_error() { last_error_.clear(); } + +private: + Logger() : min_level_(LogLevel::WARN) {} + + static const char* level_string(LogLevel level) { + switch (level) { + case LogLevel::DEBUG: return "DEBUG"; + case LogLevel::INFO: return "INFO"; + case LogLevel::WARN: return "WARN"; + case LogLevel::ERROR: return "ERROR"; + default: return "?"; + } + } + + LogLevel min_level_; + std::mutex mutex_; + std::string last_error_; + std::function callback_; +}; + +} // namespace cactus + +#define CACTUS_LOG(level, component, msg) \ + do { \ + if (static_cast(level) >= static_cast(cactus::Logger::instance().get_level())) { \ + std::ostringstream _cactus_log_ss; \ + _cactus_log_ss << msg; \ + cactus::Logger::instance().log(level, component, _cactus_log_ss.str()); \ + } \ + } while(0) + +#define CACTUS_LOG_DEBUG(component, msg) CACTUS_LOG(cactus::LogLevel::DEBUG, component, msg) +#define CACTUS_LOG_INFO(component, msg) CACTUS_LOG(cactus::LogLevel::INFO, component, msg) +#define CACTUS_LOG_WARN(component, msg) CACTUS_LOG(cactus::LogLevel::WARN, component, msg) +#define CACTUS_LOG_ERROR(component, msg) CACTUS_LOG(cactus::LogLevel::ERROR, component, msg) + +namespace GraphFile { + class MappedFile; + struct SerializedGraph; +} + +enum class Precision { + INT8, + FP16, + FP32, + INT4 +}; + +enum class ComputeBackend { + CPU, + NPU +}; + +enum class Activation { + SILU, + GELU, + GELU_ERF, + RELU, + SIGMOID, + TANH +}; + +enum class OpType { + INPUT, PRECISION_CAST, + ADD, ADD_CLIPPED, SUBTRACT, MULTIPLY, DIVIDE, + ABS, POW, FLATTEN, VIEW, + MATMUL, TRANSPOSE, RESHAPE, SLICE, GATHER, EMBEDDING, + BILINEAR_INTERPOLATION, + SUM, MEAN, VARIANCE, MIN, MAX, + RMS_NORM, ROPE, ROPE_GPTJ, SOFTMAX, ATTENTION, ATTENTION_INT8_HYBRID, REL_POS_BIAS, CONV1D_CAUSAL, CONV1D_K3, CONV1D_K7S3, CONV1D, CONV1D_SAME_DEPTHWISE_K9, CONV1D_POINTWISE, CONV2D_K3S2P1, CONV2D_DEPTHWISE_K3S2P1, CONV2D_POINTWISE_1X1, GLU, BATCHNORM, + SCALAR_ADD, SCALAR_SUBTRACT, SCALAR_MULTIPLY, SCALAR_DIVIDE, SCALAR_EXP, SCALAR_SQRT, SCALAR_COS, SCALAR_SIN, SCALAR_LOG, + RELU, SILU, GELU, GELU_ERF, SIGMOID, TANH, + SAMPLE, CONCAT, CAT, + SCATTER_TOPK, + TOPK, LAYERNORM, GROUPNORM, + MOE_LAYER, + INDEX, + PERSISTENT, + QUANTIZE_ACTIVATIONS, + LSTM_CELL, + GATED_DELTANET_DECODE, + GATED_DELTANET_PREFILL, + STFT, + ALTUP_PREDICT, + ALTUP_CORRECT, + GAUSSIAN_TOPK, + MAXPOOL1D, + BILSTM_SEQUENCE, + LEAKY_RELU, + CONV2D_K3S1P1, + STATS_POOL, + WEIGHTED_STATS_POOL, + DENSE_MLP_INT4_FUSED +}; + +struct PrecisionTraits { + static constexpr size_t size_of(Precision prec) { + switch (prec) { + case Precision::INT8: return 1; + case Precision::FP16: return 2; + case Precision::FP32: return 4; + case Precision::INT4: return 1; + } + return 1; + } + + static constexpr size_t packed_size_of(Precision prec, size_t count) { + switch (prec) { + case Precision::INT4: return (count + 1) / 2; + default: return count * size_of(prec); + } + } + + static size_t byte_offset_of(Precision prec, size_t element_offset) { + switch (prec) { + case Precision::INT4: + assert(element_offset % 32 == 0 && "INT4 byte offset must be group-aligned (multiple of 32)"); + return element_offset / 2; + default: return element_offset * size_of(prec); + } + } + + static constexpr bool is_integer(Precision prec) { + switch (prec) { + case Precision::INT8: return true; + case Precision::INT4: return true; + case Precision::FP16: return false; + case Precision::FP32: return false; + } + return true; + } + + static constexpr bool is_floating_point(Precision prec) { + switch (prec) { + case Precision::INT8: return false; + case Precision::INT4: return false; + case Precision::FP16: return true; + case Precision::FP32: return true; + } + return false; + } +}; + +namespace Quantization { + void int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); + void fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); + void fp16_to_fp32(const __fp16* src, float* dst, size_t count); + void fp32_to_fp16(const float* src, __fp16* dst, size_t count); + void int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); + void fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); +} + +struct TensorConfig { + Precision default_precision = Precision::INT8; + Precision compute_precision = Precision::INT8; + Precision output_precision = Precision::INT8; + bool auto_mixed_precision = false; + + static TensorConfig& global(); +}; + +struct BroadcastInfo { + std::vector output_shape; + bool needs_broadcasting; + + static BroadcastInfo compute(const std::vector& lhs, const std::vector& rhs); +}; + +class BufferPool; + +struct BufferDesc { + std::vector shape; + size_t total_size; + size_t byte_size; + std::unique_ptr data; + void* external_data; + char* pooled_data; + Precision precision; + + size_t group_size = 0; + size_t num_groups = 0; + void* scales_data = nullptr; + std::unique_ptr owned_scales; + + bool is_interleaved = false; + size_t original_N = 0; + + void* activation_scales_data = nullptr; + std::unique_ptr owned_activation_scales; + size_t num_rows_for_activation_scales = 0; + + BufferDesc(); + BufferDesc(const std::vector& s, Precision prec = Precision::INT8); + ~BufferDesc(); + + BufferDesc(BufferDesc&& other) noexcept; + BufferDesc& operator=(BufferDesc&& other) noexcept; + + BufferDesc(const BufferDesc&) = delete; + BufferDesc& operator=(const BufferDesc&) = delete; + + void* get_data(); + const void* get_data() const; + + template + T* data_as() { return static_cast(get_data()); } + + template + const T* data_as() const { return static_cast(get_data()); } + + const __fp16* scales_as_fp16() const { + return reinterpret_cast(scales_data); + } + + bool is_grouped_int8() const { + return precision == Precision::INT8 && group_size > 0; + } + + bool is_grouped_int4() const { + return precision == Precision::INT4 && group_size > 0; + } + + void set_grouped_scales(size_t gs, size_t ng, void* scales_ptr) { + group_size = gs; + num_groups = ng; + scales_data = scales_ptr; + } + + void set_interleaved(bool interleaved, size_t orig_n) { + is_interleaved = interleaved; + original_N = orig_n; + } + + bool has_activation_scales() const { + return activation_scales_data != nullptr && num_rows_for_activation_scales > 0; + } + const float* activation_scales_as_float() const { + return reinterpret_cast(activation_scales_data); + } + float* activation_scales_as_float() { + return reinterpret_cast(activation_scales_data); + } + void allocate_activation_scales(size_t num_rows) { + num_rows_for_activation_scales = num_rows; + owned_activation_scales = std::make_unique(num_rows * sizeof(float)); + activation_scales_data = owned_activation_scales.get(); + } + void set_activation_scales(void* scales_ptr, size_t num_rows) { + activation_scales_data = scales_ptr; + num_rows_for_activation_scales = num_rows; + } + + void allocate(); + void allocate_from_pool(BufferPool& pool); + void release_to_pool(BufferPool& pool); + void set_external(void* ptr); +}; + +struct OpParams { + float scalar = 0.0f; + float scale = 1.0f; + float theta = 10000.0f; + float epsilon = 1e-6f; + int axis = -1; + bool pretransposed_rhs = false; + size_t position_offset = 0; + size_t slice_start = 0; + size_t slice_length = 0; + size_t window_size = 0; + bool is_causal = true; + bool attention_mask_is_additive = false; + float logit_cap = 0.0f; + std::vector new_shape; + std::vector permutation; + Precision output_precision = Precision::INT8; + BroadcastInfo broadcast_info; + ComputeBackend backend = ComputeBackend::CPU; + + size_t dilation = 1; + size_t stride = 1; + float temperature = 1.0f; + float top_p = 1.0f; + float min_p = 0.15f; + float repetition_penalty = 1.1f; + size_t top_k = 0; + size_t random_seed = 0; + + size_t index_value = 0; + size_t num_classes = 0; + size_t num_groups = 0; + size_t dst_height = 0; + size_t dst_width = 0; + bool align_corners = true; + bool normalize_routing = false; + size_t num_experts = 0; + size_t num_experts_per_tok = 0; + bool moe_gated = true; + Activation activation = Activation::SILU; + + std::vector bias_values; + std::vector bias_indices; + + const int8_t* cached_keys_int8 = nullptr; + const int8_t* cached_values_int8 = nullptr; + const float* cached_k_scales = nullptr; + const float* cached_v_scales = nullptr; + size_t cache_seq_len = 0; + size_t num_kv_heads = 0; + size_t head_dim = 0; + size_t num_fft_bins = 0; + size_t chunk_size = 0; + size_t num_altup_inputs = 0; + size_t v_head_dim = 0; + size_t kernel_size = 0; +}; + +struct GraphNode { + size_t id; + OpType op_type; + std::vector input_ids; + BufferDesc output_buffer; + OpParams params; + + GraphNode(size_t node_id, OpType type); +}; + +using nodes_vector = std::vector>; +using node_index_map_t = std::unordered_map; + +inline const BufferDesc& get_input(const GraphNode& node, size_t idx, + const nodes_vector& nodes, + const node_index_map_t& node_index_map) { + return nodes[node_index_map.at(node.input_ids[idx])]->output_buffer; +} + +struct AxisDims { + size_t outer, axis_size, inner; + static AxisDims from_shape(const std::vector& shape, size_t axis) { + AxisDims d; + d.outer = 1; + for (size_t i = 0; i < axis; i++) d.outer *= shape[i]; + d.axis_size = shape[axis]; + d.inner = 1; + for (size_t i = axis + 1; i < shape.size(); i++) d.inner *= shape[i]; + return d; + } +}; + +template +void dispatch_binary_op(OpType op, const T* lhs, const T* rhs, T* output, size_t count); + +template +void dispatch_unary_op(OpType op, const T* input, T* output, size_t count, float param = 0.0f); + +void compute_node_optimized(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_matmul_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_transpose_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_reduce_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_reshape_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_precision_cast_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_sample_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_scatter_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_layernorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_groupnorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_persistent_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_dense_mlp_int4_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_index_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_lstm_cell_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_gated_deltanet_decode_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_gated_deltanet_prefill_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_altup_predict_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_altup_correct_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_maxpool1d_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_bilstm_sequence_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_conv2d_k3s1p1_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_weighted_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); + +void shrink_thread_local_buffers(); +class BufferPool { +public: + BufferPool() = default; + ~BufferPool() = default; + + BufferPool(const BufferPool&) = delete; + BufferPool& operator=(const BufferPool&) = delete; + BufferPool(BufferPool&&) noexcept = default; + BufferPool& operator=(BufferPool&&) noexcept = default; + + char* acquire(size_t byte_size); + void release(char* ptr, size_t byte_size); + void clear(); + + size_t active_bytes() const { return active_bytes_; } + size_t pool_bytes() const { return pool_bytes_; } + size_t peak_bytes() const { return peak_bytes_; } + +private: + std::unordered_map>> free_buffers_; + size_t active_bytes_ = 0; + size_t pool_bytes_ = 0; + size_t peak_bytes_ = 0; + + size_t round_up_size(size_t size) const; +}; + +namespace ValidationUtils { + void validate_tensor_dims(const std::vector& shape, size_t required_dims, const std::string& op_name); + void validate_precision(Precision actual, Precision required, const std::string& op_name); + void validate_input_count(size_t actual, size_t required, const std::string& op_name); +} + + +class CactusGraph { +public: + CactusGraph(); + ~CactusGraph() = default; + + CactusGraph(const CactusGraph&) = delete; + CactusGraph& operator=(const CactusGraph&) = delete; + CactusGraph(CactusGraph&&) noexcept = default; + CactusGraph& operator=(CactusGraph&&) noexcept = default; + + struct DebugNodeEntry { + uint32_t layer_idx; + std::string name; + size_t node_id; + }; + + void save(const std::string& path); + static CactusGraph load(const std::string& path); + + size_t input(const std::vector& shape, Precision precision = Precision::INT8); + size_t precision_cast(size_t input, Precision target_precision); + size_t quantize_activations(size_t input); + + size_t add(size_t input1, size_t input2); + size_t add_clipped(size_t input1, size_t input2); + size_t subtract(size_t input1, size_t input2); + size_t multiply(size_t input1, size_t input2); + size_t divide(size_t input1, size_t input2); + + size_t scalar_add(size_t input, float value); + size_t scalar_subtract(size_t input, float value); + size_t scalar_multiply(size_t input, float value); + size_t scalar_divide(size_t input, float value); + size_t scalar_exp(size_t input); + size_t scalar_sqrt(size_t input); + size_t scalar_cos(size_t input); + size_t scalar_sin(size_t input); + size_t scalar_log(size_t input); + + size_t relu(size_t input); + size_t silu(size_t input); + size_t gelu(size_t input); + size_t gelu_erf(size_t input); + size_t sigmoid(size_t input); + size_t tanh(size_t input); + size_t glu(size_t input, int axis = -1); + + size_t abs(size_t input); + size_t pow(size_t input, float exponent); + size_t view(size_t input, const std::vector& new_shape); + size_t flatten(size_t input, int start_dim = 0, int end_dim = -1); + + size_t matmul(size_t input1, size_t input2, bool pretransposed_rhs = false, ComputeBackend backend = ComputeBackend::CPU); + size_t transpose(size_t input, ComputeBackend backend = ComputeBackend::CPU); + size_t transposeN(size_t input, const std::vector& permutation, ComputeBackend backend = ComputeBackend::CPU); + size_t reshape(size_t input, const std::vector& new_shape); + size_t slice(size_t input, int axis, size_t start, size_t length); + size_t index(size_t input, size_t index_value, int dim); + + size_t sum(size_t input, int axis); + size_t mean(size_t input, int axis); + size_t variance(size_t input, int axis); + size_t min(size_t input, int axis); + size_t max(size_t input, int axis); + + size_t gather(size_t embeddings, size_t indices); + size_t mmap_embeddings(const std::string& filename); + size_t mmap_weights(const std::string& filename); + void set_grouped_scales(size_t node_id, size_t group_size, size_t num_groups, void* scales_ptr); + void set_interleaved(size_t node_id, bool interleaved, size_t original_N); + + void release_weight_pages(size_t node_id); + void prefetch_weight_pages(size_t node_id); + void release_all_weight_pages(); + size_t embedding(const std::string& filename, size_t indices); + size_t embedding(size_t embedding_tensor, size_t indices); + size_t bilinear_interpolation(size_t pos_embeds, size_t dst_height, size_t dst_width, bool align_corners = true); + + size_t layernorm(size_t input, size_t weight, size_t bias, float epsilon = 1e-5f); + size_t layernorm(size_t input, size_t weight, float epsilon = 1e-5f); // No bias version + size_t groupnorm(size_t input, size_t weight, size_t bias, size_t num_groups = 32, float epsilon = 1e-5f); + size_t batchnorm(size_t input, size_t weight, size_t bias, size_t running_mean, size_t running_var, int axis = 1, float epsilon = 1e-5f); + size_t topk(size_t input, size_t k); + size_t dense_mlp_int4_fused(size_t hidden, size_t gate_weight, + size_t up_weight, size_t down_weight); + size_t moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w3_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation = Activation::SILU, + size_t per_expert_scale = 0); + size_t moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation); + size_t rms_norm(size_t input, size_t weight, float epsilon = 1e-5f); + size_t rope(size_t input, float theta, size_t position_offset = 0, ComputeBackend backend = ComputeBackend::CPU); + size_t rope_gptj(size_t input, float theta, size_t position_offset = 0, size_t rot_dim = 0, ComputeBackend backend = ComputeBackend::CPU); + size_t softmax(size_t input, int axis = -1); + size_t attention(size_t query, size_t key, size_t value, float scale, bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU); + size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, ComputeBackend backend = ComputeBackend::CPU); + size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, size_t window_size, ComputeBackend backend = ComputeBackend::CPU); + size_t attention_masked(size_t query, size_t key, size_t value, size_t mask, float scale, + bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU, + bool additive_mask = false, size_t position_offset = 0, size_t window_size = 0, + float logit_cap = 0.0f); + size_t rel_pos_bias(size_t query, size_t relative_key, float scale); + + size_t attention_int8_hybrid(size_t query, size_t key_new, size_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, + const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, + size_t window_size = 0, size_t v_head_dim = 0); + + size_t conv1d_causal(size_t input, size_t weight, size_t kernel_size, size_t dilation = 1); + size_t conv1d_k3(size_t input, size_t weight, size_t stride); + size_t conv1d_k7s3(size_t input, size_t weight, size_t bias); + size_t conv1d(size_t input, size_t weight, size_t stride); + size_t conv1d(size_t input, size_t weight, size_t bias, size_t stride); + size_t conv1d_same_depthwise_k9(size_t input, size_t weight); + size_t conv1d_same_depthwise_k9(size_t input, size_t weight, size_t bias); + size_t conv1d_pointwise(size_t input, size_t weight); + size_t conv1d_pointwise(size_t input, size_t weight, size_t bias); + size_t conv2d_k3s2p1(size_t input, size_t weight); + size_t conv2d_k3s2p1(size_t input, size_t weight, size_t bias); + size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight); + size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight, size_t bias); + size_t conv2d_pointwise_1x1(size_t input, size_t weight); + size_t conv2d_pointwise_1x1(size_t input, size_t weight, size_t bias); + + size_t lstm_cell(size_t input, size_t h_prev, size_t c_prev, size_t weight_ih, size_t weight_hh, size_t bias_ih, size_t bias_hh); + size_t gated_deltanet_decode(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, float scale = 0.0f); + size_t gated_deltanet_prefill(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, size_t chunk_size = 64, float scale = 0.0f); + size_t stft(size_t input, size_t weight, size_t stride, size_t num_fft_bins); + + size_t altup_predict(size_t coefs, const size_t* streams, size_t num_streams); + size_t altup_correct(size_t coefs, size_t innovation, const size_t* predictions, size_t num_predictions); + + size_t gaussian_topk(size_t input, float ppf); + + size_t maxpool1d(size_t input, size_t kernel_size, size_t stride); + size_t leaky_relu(size_t input, float negative_slope = 0.01f); + size_t bilstm_sequence(size_t input, size_t w_ih_fwd, size_t w_hh_fwd, size_t b_ih_fwd, size_t b_hh_fwd, + size_t w_ih_bwd, size_t w_hh_bwd, size_t b_ih_bwd, size_t b_hh_bwd); + size_t conv2d_k3s1p1(size_t input, size_t weight); + size_t conv2d_k3s1p1(size_t input, size_t weight, size_t bias); + size_t stats_pool(size_t input); + size_t weighted_stats_pool(size_t input, size_t weights); + + size_t sample(size_t logits, float temperature = 0.6f, float top_p = 0.95f, size_t top_k = 20, + const std::unordered_map& logit_bias = {}); + size_t sample_with_options(size_t logits, float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, const std::unordered_map& logit_bias = {}); + + size_t concat(size_t input1, size_t input2, int axis = 0); + size_t cat(const std::vector& inputs, int axis); + size_t scatter_topk(size_t indices, size_t values, size_t num_classes); + + void set_input(size_t node_id, const void* data, Precision precision); + void set_external_input(size_t node_id, void* data, Precision precision); + void* get_output(size_t node_id); + + void execute(const std::string& profile_file = ""); + void hard_reset(); + void soft_reset(); + void soft_reset_keep_pool(); + void set_prefill_mode(bool enabled) { prefill_mode_ = enabled; } + + void register_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); + void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); + const std::vector& get_debug_nodes() const; + void clear_debug_nodes(); + + size_t add_node(OpType op_type, const std::vector& inputs, const std::vector& output_shape, const OpParams& params = {}); + const BufferDesc& get_output_buffer(size_t node_id) const; + void allocate_buffers(); + size_t get_node_count() const; + + size_t persistent(size_t source_node); + bool is_populated(size_t persistent_node_id) const; + void invalidate_persistent(size_t persistent_node_id); + + std::vector> nodes_; + std::unordered_map node_index_map_; + +private: + static CactusGraph from_serialized(const GraphFile::SerializedGraph& serialized); + size_t next_node_id_; + std::vector> mapped_files_; + std::unordered_map weight_cache_; + std::unordered_map node_to_mapped_file_; + std::vector debug_nodes_; + BufferPool buffer_pool_; + bool prefill_mode_ = false; + + std::unordered_set persistent_node_ids_; + std::unordered_set populated_node_ids_; +}; + + +namespace GraphFile { + struct LoadedNode { + size_t node_id; + std::vector shape; + Precision precision; + size_t byte_size; + }; + + struct GraphHeader { + uint32_t magic; + uint32_t version; + uint32_t node_count; + uint32_t flags = 0; + }; + + struct NodeEntry { + uint32_t index; // serialized node index 0..n-1 + OpType op_type; + std::vector inputs; + std::vector output_shape; + Precision precision; + OpParams params; + }; + + struct SerializedGraph { + GraphHeader header; + std::vector nodes; + std::vector graph_inputs; // IDs of serialized inputs + std::vector graph_outputs; // IDs of serialized outputs + }; + + SerializedGraph load_graph(const std::string& filename); + void save_graph(const CactusGraph& graph, const std::string& filename); + + void save_node(CactusGraph& graph, size_t node_id, const std::string& filename); + + class MappedFile { + public: + MappedFile(const std::string& filename); + ~MappedFile(); + + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + MappedFile(MappedFile&& other) noexcept; + MappedFile& operator=(MappedFile&& other) noexcept; + + const std::vector& shape() const; + Precision precision() const; + size_t byte_size() const; + + size_t group_size() const { return group_size_; } + size_t num_groups() const { return num_groups_; } + const void* scales_data() const; + + bool is_interleaved() const { return is_interleaved_; } + size_t original_N() const { return original_N_; } + + void* data(); + const void* data() const; + + template + const T* typed_data() const; + + void release_pages(); + void prefetch_pages(); + + private: + int fd_; + void* mapped_data_; + size_t file_size_, data_offset_; + std::vector shape_; + Precision precision_; + size_t byte_size_; + size_t group_size_ = 0; + size_t num_groups_ = 0; + size_t scales_offset_ = 0; + size_t scales_bytes_ = 0; + uint32_t alignment_ = 32; + + bool is_interleaved_ = false; + size_t original_N_ = 0; + + void parse_header(); + void apply_madvise_hints(); + }; +} + +#endif diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/graph_param_io.h b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/graph_param_io.h new file mode 100644 index 0000000..0362ea8 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/graph_param_io.h @@ -0,0 +1,11 @@ +#pragma once + +#include "graph.h" +#include + +namespace GraphParamIO { + +void write_op_params(std::ostream& out, OpType op_type, const OpParams& params); +void read_op_params(std::istream& in, OpType op_type, OpParams& params); + +} // namespace GraphParamIO diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/kernel.h b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/kernel.h new file mode 100644 index 0000000..13f4a00 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/kernel.h @@ -0,0 +1,475 @@ +#ifndef KERNEL_H +#define KERNEL_H + +#include +#include + +enum class Precision; + +enum class ScalarOpType { + ADD, + SUBTRACT, + MULTIPLY, + DIVIDE, + ABS, + EXP, + POW, + SQRT, + COS, + SIN, + LOG +}; + +constexpr size_t KV_QUANT_GROUP_SIZE = 32; + +void cactus_add_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_add_f16_clipped(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_subtract_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_multiply_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_add_scaled_f16(const __fp16* base, const __fp16* src, __fp16* output, size_t num_elements, float scale); +void cactus_divide_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); + +void cactus_add_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_subtract_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_multiply_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_divide_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); + +void cactus_scalar_op_f16(const __fp16* input, __fp16* output, size_t num_elements, float scalar_value, ScalarOpType op_type); + +void cactus_gemv_int8(const int8_t* A, float A_scale, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemm_int8(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_int8(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int8_i8mm(const int8_t* A, float A_scale, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemm_int8_i8mm(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int4(const int8_t* A, float A_scale, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int4_block_range(const int8_t* A, float A_scale, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size, + size_t block_start, size_t block_end); + +void cactus_gemm_int4(const int8_t* A, const float* A_scales, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_int4(const int8_t* A, const float* A_scales, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_integer(Precision precision, + const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_f16(const __fp16* a, const __fp16* b_transposed, __fp16* c, + size_t M, size_t K, size_t N); + +void cactus_transpose_2d_f16(const __fp16* source, __fp16* destination, + size_t num_rows, size_t num_cols, size_t start_row, size_t end_row); +void cactus_transpose_f16(const __fp16* source, __fp16* destination, const size_t* shape, + const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx); + +double cactus_sum_all_f16(const __fp16* data, size_t num_elements); +void cactus_sum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +double cactus_mean_all_f16(const __fp16* data, size_t num_elements); +void cactus_mean_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +double cactus_variance_all_f16(const __fp16* data, size_t num_elements); +void cactus_variance_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +__fp16 cactus_min_all_f16(const __fp16* data, size_t num_elements); +void cactus_min_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +__fp16 cactus_max_all_f16(const __fp16* data, size_t num_elements); +void cactus_max_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +void cactus_rms_norm_f16(const __fp16* input, const __fp16* weight, __fp16* output, + size_t batch_size, size_t dims, float eps); + +void cactus_layer_norm_f16(const __fp16* input, const __fp16* weight, const __fp16* bias, + __fp16* output, size_t batch_size, size_t dims, float eps); + +void cactus_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, + size_t num_heads, size_t head_dim, size_t start_pos, float theta); + +void cactus_gpt_j_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, + size_t num_heads, size_t head_dim, size_t rot_dim, size_t start_pos, float theta); + +void cactus_softmax_f16(const __fp16* input, __fp16* output, size_t batch_size, + size_t seq_len, size_t vocab_size); + +void cactus_relu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_leaky_relu_f16(const __fp16* input, __fp16* output, size_t num_elements, float negative_slope); + +void cactus_silu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_gelu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_gelu_f16_erf(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_sigmoid_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_tanh_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_glu_f16( + const __fp16* input, + __fp16* output, + size_t outer_size, + size_t split_size, + size_t inner_size +); + +void cactus_glu_f32( + const float* input, + float* output, + size_t outer_size, + size_t split_size, + size_t inner_size +); + +void cactus_batchnorm_f16( + const __fp16* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + __fp16* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +); + +void cactus_batchnorm_f32( + const float* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + float* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +); + +void cactus_attention_f16(const __fp16* queries, const __fp16* keys, const __fp16* values, __fp16* output, + size_t batch_size, size_t seq_len, size_t kv_seq_len, size_t num_q_heads, size_t num_kv_heads, + size_t head_dim, float scale, const __fp16* mask, size_t position_offset = 0, size_t window_size = 0, + bool is_causal = true, bool mask_is_additive = false, bool mask_per_head = false, + size_t v_head_dim = 0, float logit_cap = 0.0f); + +void cactus_attention_hybrid_int8_fp16( + const __fp16* queries, + const int8_t* keys_cached, + const int8_t* values_cached, + const float* k_scales, + const float* v_scales, + const __fp16* keys_new, + const __fp16* values_new, + __fp16* output, + size_t batch_size, size_t seq_len, size_t cache_len, size_t new_len, + size_t num_q_heads, size_t num_kv_heads, size_t head_dim, + float scale, size_t position_offset = 0, bool is_causal = true, size_t window_size = 0, + size_t group_size = KV_QUANT_GROUP_SIZE, size_t v_head_dim = 0); + +void cactus_gated_deltanet_decode_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + float scale); + +void cactus_gated_deltanet_prefill_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t T, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + size_t requested_chunk_size, + float scale); + +void cactus_conv1d_causal_depthwise_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, + size_t L, + size_t C, + size_t K, + size_t dilation); + +void cactus_conv1d_f16_k3( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out, + size_t stride +); + +void cactus_conv1d_f16( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out, + size_t K, + size_t stride +); + +void cactus_stft_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, size_t L, + size_t C_in, size_t C_out, + size_t K, size_t stride, + size_t num_fft_bins +); + +void cactus_conv1d_f16_k7s3_oc8( + const __fp16* input, + const __fp16* Wpack, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out +); + +void cactus_conv1d_same_depthwise_f16_k9( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C +); + +void cactus_conv2d_f16_k3s1p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv2d_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv2d_depthwise_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C, + size_t H, + size_t W +); + +void cactus_conv2d_pointwise_f16_1x1_nchw_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv1d_pointwise_f16_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out +); + +void cactus_bilinear_interpolation_f16(const __fp16* input, __fp16* output, size_t src_height, size_t src_width, size_t embed_dim, + size_t dst_height, size_t dst_width, bool align_corners = true); + +void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); +void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); + +void cactus_sample_f32_ex(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); +void cactus_sample_f16_ex(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); + +void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* output, + const size_t* shape1, const size_t* shape2, const size_t* output_shape, + size_t ndims, int axis); +void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_shapes, + const size_t* output_shape, size_t num_inputs, size_t rank, int axis); + +void cactus_int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); +void cactus_fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); +void cactus_fp16_to_fp32(const __fp16* src, float* dst, size_t count); +void cactus_fp32_to_fp16(const float* src, __fp16* dst, size_t count); +void cactus_int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); +void cactus_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); +float cactus_fp16_max_abs(const __fp16* src, size_t count); + +void cactus_quantize_kv_fp16_to_int8( + const __fp16* src, + int8_t* dst, + float* scales, + size_t seq_len, size_t kv_heads, size_t head_dim, + size_t group_size = KV_QUANT_GROUP_SIZE); + +inline size_t kv_scales_count(size_t seq_len, size_t kv_heads, size_t head_dim, size_t group_size = KV_QUANT_GROUP_SIZE) { + size_t num_groups = (head_dim + group_size - 1) / group_size; + return seq_len * kv_heads * num_groups; +} + +void cactus_unpack_int4_to_int8(const uint8_t* packed, int8_t* unpacked, size_t unpacked_count); + +void cactus_gaussian_topk_f16( + const __fp16* input, + __fp16* output, + size_t rows, + size_t cols, + float ppf); + +void cactus_altup_predict_f16( + const __fp16* coefs, + const __fp16* const* streams, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim); + +void cactus_altup_correct_f16( + const __fp16* coefs, + const __fp16* innovation, + const __fp16* const* predictions, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim); + +void cactus_lstm_cell_f16( + const __fp16* x_input, + const __fp16* h_prev, + const __fp16* c_prev, + const __fp16* weight_ih, + const __fp16* weight_hh, + const __fp16* bias_ih, + const __fp16* bias_hh, + __fp16* h_new, + __fp16* c_new, + size_t batch_size, + size_t input_size, + size_t hidden_size +); + +void cactus_bilstm_sequence_f16( + const __fp16* input, + const __fp16* weight_ih_fwd, + const __fp16* weight_hh_fwd, + const __fp16* bias_ih_fwd, + const __fp16* bias_hh_fwd, + const __fp16* weight_ih_bwd, + const __fp16* weight_hh_bwd, + const __fp16* bias_ih_bwd, + const __fp16* bias_hh_bwd, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t input_size, + size_t hidden_size +); + +void cactus_maxpool1d_f16( + const __fp16* input, + __fp16* output, + size_t batch_size, + size_t channels, + size_t input_length, + size_t kernel_size, + size_t stride +); + +#endif diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/kernel_utils.h b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/kernel_utils.h new file mode 100644 index 0000000..b202158 --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/kernel_utils.h @@ -0,0 +1,681 @@ +#ifndef KERNEL_UTILS_H +#define KERNEL_UTILS_H + +#include +#if defined(__APPLE__) +#include +#include +#endif +#if defined(__ANDROID__) +#include +#include +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr size_t NEON_VECTOR_SIZE = 16; +constexpr size_t STREAMING_STORE_THRESHOLD = 32768; + +inline void stream_store_f16x8(__fp16* dst, float16x8_t val) { +#if defined(__aarch64__) + float16x4_t lo = vget_low_f16(val); + float16x4_t hi = vget_high_f16(val); + __asm__ __volatile__( + "stnp %d0, %d1, [%2]" + : + : "w"(lo), "w"(hi), "r"(dst) + : "memory" + ); +#else + vst1q_f16(dst, val); +#endif +} + +inline bool cpu_has_i8mm() { +#if defined(__aarch64__) + static std::once_flag once; + static bool has = false; + + std::call_once(once, []() { +#if defined(__APPLE__) + int ret = 0; + size_t size = sizeof(ret); + if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &ret, &size, nullptr, 0) == 0) { + has = (ret == 1); + } +#elif defined(__ANDROID__) + unsigned long hwcap2 = getauxval(AT_HWCAP2); + #ifndef HWCAP2_I8MM + #define HWCAP2_I8MM (1 << 13) + #endif + has = (hwcap2 & HWCAP2_I8MM) != 0; +#endif + }); + + return has; +#else + return false; +#endif +} + +inline float32x4_t fast_exp_f32x4(float32x4_t x) { + const float32x4_t log2e = vdupq_n_f32(1.4426950408889634f); + const float32x4_t ln2 = vdupq_n_f32(0.6931471805599453f); + + const float32x4_t c0 = vdupq_n_f32(1.0f); + const float32x4_t c1 = vdupq_n_f32(0.6931471805599453f); + const float32x4_t c2 = vdupq_n_f32(0.2402265069591007f); + const float32x4_t c3 = vdupq_n_f32(0.05550410866482158f); + const float32x4_t c4 = vdupq_n_f32(0.009618129842071803f); + + x = vmaxq_f32(x, vdupq_n_f32(-87.0f)); + x = vminq_f32(x, vdupq_n_f32(87.0f)); + + float32x4_t z = vmulq_f32(x, log2e); + + int32x4_t zi = vcvtq_s32_f32(z); + float32x4_t zf = vsubq_f32(z, vcvtq_f32_s32(zi)); + + uint32x4_t neg_mask = vcltq_f32(zf, vdupq_n_f32(0.0f)); + zi = vsubq_s32(zi, vandq_s32(vreinterpretq_s32_u32(neg_mask), vdupq_n_s32(1))); + zf = vaddq_f32(zf, vreinterpretq_f32_u32(vandq_u32(neg_mask, vreinterpretq_u32_f32(vdupq_n_f32(1.0f))))); + + float32x4_t zf_ln2 = vmulq_f32(zf, ln2); + float32x4_t p = c4; + p = vfmaq_f32(c3, p, zf_ln2); + p = vfmaq_f32(c2, p, zf_ln2); + p = vfmaq_f32(c1, p, zf_ln2); + p = vfmaq_f32(c0, p, zf_ln2); + + int32x4_t exp_bits = vshlq_n_s32(vaddq_s32(zi, vdupq_n_s32(127)), 23); + float32x4_t scale = vreinterpretq_f32_s32(exp_bits); + + return vmulq_f32(p, scale); +} + +// Cephes-style 13/6 rational tanh approximation (same coefficients as Eigen). +// Constants are stored as static splatted arrays so the compiler emits a single +// pc-relative `ldr q` per load. +alignas(16) inline constexpr float kFastTanhAlpha[7][4] = { + { 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f }, + { 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f }, + { 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f }, + { 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f }, + {-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f }, + { 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f }, + {-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f }, +}; +alignas(16) inline constexpr float kFastTanhBeta[4][4] = { + { 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f }, + { 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f }, + { 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f }, + { 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f }, +}; +alignas(16) inline constexpr float kFastTanhClampHi[4] = { 9.0f, 9.0f, 9.0f, 9.0f }; +alignas(16) inline constexpr float kFastTanhClampLo[4] = {-9.0f,-9.0f,-9.0f,-9.0f }; + +inline float32x4_t fast_tanh_f32x4(float32x4_t x) { + x = vmaxq_f32(vld1q_f32(kFastTanhClampLo), vminq_f32(vld1q_f32(kFastTanhClampHi), x)); + float32x4_t x2 = vmulq_f32(x, x); + float32x4_t p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[5]), vld1q_f32(kFastTanhAlpha[6]), x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[4]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[3]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[2]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[1]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[0]), p, x2); + p = vmulq_f32(p, x); + float32x4_t q = vfmaq_f32(vld1q_f32(kFastTanhBeta[2]), vld1q_f32(kFastTanhBeta[3]), x2); + q = vfmaq_f32(vld1q_f32(kFastTanhBeta[1]), q, x2); + q = vfmaq_f32(vld1q_f32(kFastTanhBeta[0]), q, x2); + return vdivq_f32(p, q); +} + +constexpr size_t SIMD_F16_WIDTH = 8; + +inline size_t simd_align(size_t count, size_t width = SIMD_F16_WIDTH) { + return (count / width) * width; +} + +inline void f16x8_split_f32(float16x8_t v, float32x4_t& lo, float32x4_t& hi) { + lo = vcvt_f32_f16(vget_low_f16(v)); + hi = vcvt_f32_f16(vget_high_f16(v)); +} + +inline float16x8_t f32_merge_f16(float32x4_t lo, float32x4_t hi) { + return vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi)); +} + +inline float32x4_t fast_sigmoid_f32x4(float32x4_t x) { + const float32x4_t one = vdupq_n_f32(1.0f); + return vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(x)))); +} + +template +inline float16x8_t apply_f32_op_on_f16x8(float16x8_t v, F32x4Op op) { + float32x4_t lo, hi; + f16x8_split_f32(v, lo, hi); + return f32_merge_f16(op(lo), op(hi)); +} + +inline void unpack_int4_as_int8x16x2(const uint8_t* ptr, int8x16_t& high_decoded, int8x16_t& low_decoded) { + int8x16_t packed = vreinterpretq_s8_u8(vld1q_u8(ptr)); + high_decoded = vshrq_n_s8(packed, 4); + low_decoded = vshrq_n_s8(vshlq_n_s8(packed, 4), 4); +} + +namespace CactusThreading { + +#if defined(__ANDROID__) + struct CoreTopology { + std::vector performance_cores; + std::vector all_cores; + + static CoreTopology& get() { + static CoreTopology topo = detect(); + return topo; + } + + private: + static int read_sysfs_int(const char* path) { + std::ifstream f(path); + if (!f.is_open()) return -1; + int val = -1; + f >> val; + return val; + } + + static CoreTopology detect() { + CoreTopology topo; + constexpr int MAX_CPUS = 16; + std::vector> core_caps; + + for (int i = 0; i < MAX_CPUS; ++i) { + char path[128]; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpu_capacity", i); + int cap = read_sysfs_int(path); + if (cap > 0) { + core_caps.push_back({i, cap}); + topo.all_cores.push_back(i); + continue; + } + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", i); + int freq = read_sysfs_int(path); + if (freq > 0) { + core_caps.push_back({i, freq}); + topo.all_cores.push_back(i); + } + } + + if (core_caps.empty()) return topo; + + int max_cap = 0; + for (auto& [id, cap] : core_caps) { + max_cap = std::max(max_cap, cap); + } + + int threshold = static_cast(max_cap * 0.70); + for (auto& [id, cap] : core_caps) { + if (cap >= threshold) { + topo.performance_cores.push_back(id); + } + } + + return topo; + } + }; + + inline bool pin_current_thread_to_cores(const std::vector& cores) { + if (cores.empty()) return false; + cpu_set_t mask; + CPU_ZERO(&mask); + for (int core : cores) { + CPU_SET(core, &mask); + } + return sched_setaffinity(0, sizeof(mask), &mask) == 0; + } +#endif + + class ThreadPool { + private: + static constexpr size_t MAX_WORKERS = 16; + + std::vector workers; + std::deque> tasks; + + std::mutex mutex; + std::condition_variable work_available; + std::condition_variable work_done; + + bool stop{false}; + std::atomic pending_tasks{0}; + size_t num_workers_; + + void worker_thread() { + while (true) { + std::function task; + { + std::unique_lock lock(mutex); + work_available.wait(lock, [this] { + return stop || !tasks.empty(); + }); + + if (stop && tasks.empty()) { + return; + } + + task = std::move(tasks.front()); + tasks.pop_front(); + } + + task(); + + if (pending_tasks.fetch_sub(1, std::memory_order_acq_rel) == 1) { + std::lock_guard lock(mutex); + work_done.notify_one(); + } + } + } + + public: + explicit ThreadPool(size_t num_threads = std::thread::hardware_concurrency()) + : stop(false), pending_tasks(0) { + num_workers_ = std::min(num_threads, MAX_WORKERS); + if (num_workers_ == 0) num_workers_ = 1; + +#if defined(__ANDROID__) + auto& topo = CoreTopology::get(); + if (!topo.performance_cores.empty()) { + num_workers_ = std::min(num_workers_, topo.performance_cores.size()); + } +#endif + + workers.reserve(num_workers_); + for (size_t i = 0; i < num_workers_; ++i) { + workers.emplace_back([this]() { +#if defined(__ANDROID__) + auto& perf = CoreTopology::get().performance_cores; + if (!perf.empty()) { + pin_current_thread_to_cores(perf); + } +#endif + worker_thread(); + }); + } + } + + ~ThreadPool() { + { + std::lock_guard lock(mutex); + stop = true; + } + work_available.notify_all(); + for (auto& worker : workers) { + if (worker.joinable()) { + worker.join(); + } + } + } + + template + auto enqueue(F&& f) -> std::future { + using return_type = decltype(f()); + + auto task = std::make_shared>( + std::forward(f) + ); + + std::future res = task->get_future(); + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(1, std::memory_order_relaxed); + tasks.emplace_back([task](){ (*task)(); }); + } + work_available.notify_one(); + + return res; + } + + template + void enqueue_batch(size_t total_work, F task_func) { + if (total_work == 0) return; + + const size_t num_tasks = std::min(num_workers_, total_work); + const size_t per_worker = total_work / num_tasks; + const size_t remainder = total_work % num_tasks; + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(num_tasks, std::memory_order_relaxed); + + for (size_t w = 0; w < num_tasks; ++w) { + size_t start = w * per_worker + std::min(w, remainder); + size_t end = start + per_worker + (w < remainder ? 1 : 0); + tasks.emplace_back([=]() { task_func(start, end); }); + } + } + work_available.notify_all(); + } + + void wait_all() { + std::unique_lock lock(mutex); + work_done.wait(lock, [this] { + return pending_tasks.load(std::memory_order_acquire) == 0; + }); + } + + template + void enqueue_n_threads(size_t total_work, size_t num_threads, F task_func) { + if (total_work == 0 || num_threads == 0) return; + + num_threads = std::min(num_threads, std::min(num_workers_, total_work)); + const size_t per_thread = total_work / num_threads; + const size_t remainder = total_work % num_threads; + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(num_threads, std::memory_order_relaxed); + + for (size_t t = 0; t < num_threads; ++t) { + size_t start = t * per_thread + std::min(t, remainder); + size_t end = start + per_thread + (t < remainder ? 1 : 0); + tasks.emplace_back([=]() { task_func(start, end); }); + } + } + work_available.notify_all(); + } + + size_t num_workers() const { return num_workers_; } + }; + + inline ThreadPool& get_thread_pool() { + static ThreadPool pool; + return pool; + } + + struct ParallelConfig { + size_t min_work_gate; + size_t work_per_thread; + + constexpr ParallelConfig(size_t gate, size_t per_thread) + : min_work_gate(gate), work_per_thread(per_thread) {} + }; + + inline size_t get_optimal_thread_count(size_t total_work, ParallelConfig config) { + if (total_work < config.min_work_gate) return 1; + + size_t pool_size = get_thread_pool().num_workers(); + size_t num_threads = (total_work + config.work_per_thread - 1) / config.work_per_thread; + return std::min(pool_size, std::max(static_cast(1), num_threads)); + } + + struct Thresholds { + #if defined(__ANDROID__) + static constexpr ParallelConfig ATTENTION{64, 32}; + static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; + static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; + static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; + static constexpr ParallelConfig SCALAR_BASIC{30000, 15000}; + static constexpr ParallelConfig SCALAR_EXPENSIVE{10000, 5000}; + #else // Apple + static constexpr ParallelConfig ATTENTION{32, 16}; + static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; + static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; + static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; + static constexpr ParallelConfig SCALAR_BASIC{5000, 2500}; + static constexpr ParallelConfig SCALAR_EXPENSIVE{2500, 1250}; + #endif + }; + + struct GemmThreading { + #if defined(__ANDROID__) + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return 1; + return pool_size; + } + static size_t get_gemv_threads(size_t /*N_blocks*/, size_t /*pool_size*/) { + return 1; + } + #elif defined(__APPLE__) && TARGET_OS_IPHONE + static constexpr size_t GEMV_MIN_N_BLOCKS = 512; + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return std::min(pool_size, static_cast(2)); + return pool_size; + } + static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { + if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; + return std::min(pool_size, static_cast(3)); + } + #else + static constexpr size_t GEMV_MIN_N_BLOCKS = 256; + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return std::min(pool_size, static_cast(4)); + return pool_size; + } + static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { + if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; + if (N_blocks < 512) return std::min(pool_size, static_cast(2)); + return std::min(pool_size, static_cast(5)); + } + #endif + }; + + inline size_t& get_gemm_thread_override() { + static size_t override_threads = 0; + return override_threads; + } + + inline void set_gemm_threads(size_t num_threads) { + get_gemm_thread_override() = num_threads; + } + + inline void reset_gemm_threads() { + get_gemm_thread_override() = 0; + } + + class TaskHandle { + private: + std::vector> futures_; + bool auto_wait_; + + public: + TaskHandle(bool auto_wait = true) : auto_wait_(auto_wait) {} + + ~TaskHandle() { + if (auto_wait_) { + wait(); + } + } + + TaskHandle(TaskHandle&&) = default; + TaskHandle& operator=(TaskHandle&&) = default; + TaskHandle(const TaskHandle&) = delete; + TaskHandle& operator=(const TaskHandle&) = delete; + + void add_future(std::future&& f) { + futures_.push_back(std::move(f)); + } + + void wait() { + for (auto& f : futures_) { + if (f.valid()) { + f.wait(); + } + } + futures_.clear(); + } + + bool is_ready() const { + for (const auto& f : futures_) { + if (f.valid() && f.wait_for(std::chrono::seconds(0)) != std::future_status::ready) { + return false; + } + } + return true; + } + + size_t task_count() const { return futures_.size(); } + }; + + template + TaskHandle parallel_for(size_t total_work, ParallelConfig config, WorkFunc work_func, bool wait = true) { + const size_t num_threads = get_optimal_thread_count(total_work, config); + TaskHandle handle(!wait); + + if (num_threads == 1) { + if (wait) { + work_func(0, total_work); + return handle; + } + auto& pool = get_thread_pool(); + handle.add_future(pool.enqueue([work_func, total_work]() { + work_func(0, total_work); + })); + return handle; + } + + auto& pool = get_thread_pool(); + const size_t work_per_thread = total_work / num_threads; + + for (size_t t = 0; t < num_threads; ++t) { + handle.add_future(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() { + const size_t start_idx = t * work_per_thread; + const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; + work_func(start_idx, end_idx); + })); + } + + if (wait) { + handle.wait(); + } + return handle; + } + + template + void parallel_for_2d(size_t outer_size, size_t inner_size, ParallelConfig config, WorkFunc work_func) { + const size_t total_work = outer_size * inner_size; + parallel_for(total_work, config, [&](size_t start_idx, size_t end_idx) { + for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { + const size_t outer = work_idx / inner_size; + const size_t inner = work_idx % inner_size; + work_func(outer, inner); + } + }); + } + + template + ResultType parallel_reduce(size_t total_work, ParallelConfig config, + WorkFunc work_func, ResultType init_value, CombineFunc combine_func) { + const size_t num_threads = get_optimal_thread_count(total_work, config); + + if (num_threads == 1) { + return work_func(0, total_work); + } + + auto& pool = get_thread_pool(); + std::vector> futures; + std::vector partial_results(num_threads, init_value); + const size_t work_per_thread = total_work / num_threads; + + for (size_t t = 0; t < num_threads; ++t) { + futures.push_back(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() -> ResultType { + const size_t start_idx = t * work_per_thread; + const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; + return work_func(start_idx, end_idx); + })); + } + + ResultType result = init_value; + for (auto& future : futures) { + result = combine_func(result, future.get()); + } + return result; + } + + template + void parallel_gemm_tiles(size_t M, size_t total_tiles, WorkFunc work_func) { + auto& pool = get_thread_pool(); + + size_t override = get_gemm_thread_override(); + size_t num_threads = (override > 0) ? override : GemmThreading::get_num_threads(M, pool.num_workers()); + num_threads = std::min(num_threads, total_tiles); + + if (num_threads <= 1) { + work_func(0, total_tiles); + return; + } + + pool.enqueue_n_threads(total_tiles, num_threads, work_func); + pool.wait_all(); + } + +} + +template +void elementwise_op_f16(const __fp16* input, __fp16* output, size_t num_elements, + bool use_streaming, CactusThreading::ParallelConfig config, + SimdOp simd_op, ScalarOp scalar_op, size_t unroll = 4) { + CactusThreading::parallel_for(num_elements, config, + [&](size_t start, size_t end) { + const size_t n = end - start; + const size_t vec_end = start + simd_align(n); + + if (use_streaming && unroll >= 4) { + const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 4); + for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 4) { + __builtin_prefetch(&input[i + 256], 0, 0); + float16x8_t v0 = simd_op(vld1q_f16(&input[i])); + float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); + float16x8_t v2 = simd_op(vld1q_f16(&input[i + 16])); + float16x8_t v3 = simd_op(vld1q_f16(&input[i + 24])); + stream_store_f16x8(&output[i], v0); + stream_store_f16x8(&output[i + 8], v1); + stream_store_f16x8(&output[i + 16], v2); + stream_store_f16x8(&output[i + 24], v3); + } + for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { + stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } else if (use_streaming && unroll >= 2) { + const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 2); + for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 2) { + __builtin_prefetch(&input[i + 128], 0, 0); + float16x8_t v0 = simd_op(vld1q_f16(&input[i])); + float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); + stream_store_f16x8(&output[i], v0); + stream_store_f16x8(&output[i + 8], v1); + } + for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { + stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } else { + for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { + vst1q_f16(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } + for (size_t i = vec_end; i < end; ++i) { + output[i] = scalar_op(input[i]); + } + }); +} + +#endif // KERNEL_UTILS_H diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/needle_tools.h b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/needle_tools.h new file mode 100644 index 0000000..4330cff --- /dev/null +++ b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/needle_tools.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace needle { + +inline std::string to_snake_case(const std::string& name) { + std::string s; + s.reserve(name.size() * 2); + for (char c : name) { + if (std::isalnum(static_cast(c)) || c == '_') { + s += c; + } else { + if (s.empty() || s.back() != '_') s += '_'; + } + } + + std::string s2; + s2.reserve(s.size() * 2); + for (size_t i = 0; i < s.size(); i++) { + char c = s[i]; + if (i > 0 && std::isupper(static_cast(c))) { + char prev = s[i - 1]; + if (std::islower(static_cast(prev)) || std::isdigit(static_cast(prev))) { + s2 += '_'; + } + } + s2 += c; + } + + std::string s3; + s3.reserve(s2.size() * 2); + for (size_t i = 0; i < s2.size(); i++) { + s3 += s2[i]; + if (i + 1 < s2.size() && + std::isupper(static_cast(s2[i])) && + std::isupper(static_cast(s2[i + 1]))) { + if (i + 2 < s2.size() && std::islower(static_cast(s2[i + 2]))) { + s3 += '_'; + } + } + } + + std::string result; + result.reserve(s3.size()); + bool prev_underscore = false; + for (char c : s3) { + if (c == '_') { + if (!prev_underscore) result += '_'; + prev_underscore = true; + } else { + result += static_cast(std::tolower(static_cast(c))); + prev_underscore = false; + } + } + + size_t start = result.find_first_not_of('_'); + if (start == std::string::npos) return result; + size_t end = result.find_last_not_of('_'); + return result.substr(start, end - start + 1); +} + +inline void restore_tool_names(std::vector& function_calls, + const std::unordered_map& name_map) { + if (name_map.empty()) return; + for (auto& call : function_calls) { + for (const auto& [snake, orig] : name_map) { + std::string from = "\"name\":\"" + snake + "\""; + size_t pos = call.find(from); + if (pos == std::string::npos) { + from = "\"name\": \"" + snake + "\""; + pos = call.find(from); + } + if (pos != std::string::npos) { + std::string to = from.substr(0, from.size() - snake.size() - 1) + orig + "\""; + call.replace(pos, from.size(), to); + break; + } + } + } +} + +} // namespace needle diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Info.plist b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Info.plist new file mode 100644 index 0000000..8104ada Binary files /dev/null and b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/Info.plist differ diff --git a/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/cactus b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/cactus new file mode 100755 index 0000000..cf91649 Binary files /dev/null and b/ios/Runner/cactus-ios.xcframework/ios-arm64/cactus.framework/cactus differ diff --git a/ios/RunnerTests/RunnerTests.swift b/ios/RunnerTests/RunnerTests.swift new file mode 100644 index 0000000..86a7c3b --- /dev/null +++ b/ios/RunnerTests/RunnerTests.swift @@ -0,0 +1,12 @@ +import Flutter +import UIKit +import XCTest + +class RunnerTests: XCTestCase { + + func testExample() { + // If you add code to the Runner application, consider adding tests here. + // See https://developer.apple.com/documentation/xctest for more information about using XCTest. + } + +} diff --git a/ios/cactus-ios.xcframework/Info.plist b/ios/cactus-ios.xcframework/Info.plist new file mode 100644 index 0000000..55424d6 --- /dev/null +++ b/ios/cactus-ios.xcframework/Info.plist @@ -0,0 +1,39 @@ + + + + + AvailableLibraries + + + LibraryIdentifier + ios-arm64 + LibraryPath + cactus.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + ios + + + LibraryIdentifier + ios-arm64-simulator + LibraryPath + cactus.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + ios + SupportedPlatformVariant + simulator + + + CFBundlePackageType + XFWK + XCFrameworkFormatVersion + 1.0 + + diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus.h b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus.h new file mode 100644 index 0000000..b5b6d80 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus.h @@ -0,0 +1,12 @@ +#ifndef CACTUS_H +#define CACTUS_H + +#include "graph/graph.h" +#include "kernel/kernel.h" +#include "kernel/kernel_utils.h" +#include "engine/engine.h" +#include "models/model.h" +#include "ffi/cactus_ffi.h" +#include "npu/npu.h" + +#endif // CACTUS_H \ No newline at end of file diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_cloud.h b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_cloud.h new file mode 100644 index 0000000..6911d9a --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_cloud.h @@ -0,0 +1,50 @@ +#ifndef CACTUS_CLOUD_H +#define CACTUS_CLOUD_H + +#include "cactus_utils.h" +#include +#include + +namespace cactus { +namespace ffi { + +struct CloudResponse { + std::string transcript; + std::string api_key_hash; + bool used_cloud = false; + std::string error; +}; + +struct CloudCompletionRequest { + std::vector messages; + std::vector tools; + std::string local_output; + std::vector local_function_calls; + bool has_images = false; + bool has_audio = false; + std::vector audio_pcm; + std::string cloud_key; +}; + +struct CloudCompletionResult { + bool ok = false; + bool used_cloud = false; + std::string response; + std::vector function_calls; + std::string error; +}; + +std::string cloud_base64_encode(const uint8_t* data, size_t len); +std::vector cloud_build_wav(const uint8_t* pcm, size_t pcm_bytes); +std::string resolve_cloud_api_key(const char* cloud_key_param); +CloudResponse cloud_transcribe_request(const std::string& audio_b64, + const std::string& fallback_text, + long timeout_seconds = 15L, + const char* cloud_key = nullptr); +CloudCompletionResult cloud_complete_request(const CloudCompletionRequest& request, + long timeout_ms); + +} // namespace ffi +} // namespace cactus + +#endif // CACTUS_CLOUD_H diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_ffi.h b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_ffi.h new file mode 100644 index 0000000..163e180 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_ffi.h @@ -0,0 +1,454 @@ +#ifndef CACTUS_FFI_H +#define CACTUS_FFI_H + +#include +#include +#include + +#if __GNUC__ >= 4 + #define CACTUS_FFI_EXPORT __attribute__((visibility("default"))) + #define CACTUS_FFI_LOCAL __attribute__((visibility("hidden"))) +#else + #define CACTUS_FFI_EXPORT + #define CACTUS_FFI_LOCAL +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* cactus_model_t; +typedef void* cactus_index_t; +typedef void* cactus_stream_transcribe_t; + +typedef void (*cactus_token_callback)(const char* token, uint32_t token_id, void* user_data); + +CACTUS_FFI_EXPORT cactus_model_t cactus_init( + const char* model_path, + const char* corpus_dir, // optional: NULL if no RAG corpus + bool cache_index // false = always rebuild index, true = load cached if available +); + +CACTUS_FFI_EXPORT void cactus_destroy(cactus_model_t model); +CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model); +CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model); + +CACTUS_FFI_EXPORT int cactus_complete( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const char* tools_json, // optional + cactus_token_callback callback, // optional + void* user_data, // optional + const uint8_t* pcm_buffer, // optional: NULL when not used + size_t pcm_buffer_size // optional: 0 when not used +); + +CACTUS_FFI_EXPORT int cactus_prefill( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const char* tools_json, // optional + const uint8_t* pcm_buffer, // optional: NULL when not used + size_t pcm_buffer_size // optional: 0 when not used +); + +CACTUS_FFI_EXPORT int cactus_tokenize( + cactus_model_t model, + const char* text, + uint32_t* token_buffer, + size_t token_buffer_len, + size_t* out_token_len +); + +CACTUS_FFI_EXPORT int cactus_score_window( + cactus_model_t model, + const uint32_t* tokens, + size_t token_len, + size_t start, + size_t end, + size_t context, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_transcribe( + cactus_model_t model, + const char* audio_file_path, // NULL if using pcm_buffer + const char* prompt, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + cactus_token_callback callback, // optional + void* user_data, // optional + const uint8_t* pcm_buffer, // NULL if using audio_file_path + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_detect_language( + cactus_model_t model, + const char* audio_file_path, // NULL if using pcm_buffer + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const uint8_t* pcm_buffer, // NULL if using audio_file_path + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT cactus_stream_transcribe_t cactus_stream_transcribe_start( + cactus_model_t model, + const char* options_json // optional +); + +CACTUS_FFI_EXPORT int cactus_stream_transcribe_process( + cactus_stream_transcribe_t stream, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_stream_transcribe_stop( + cactus_stream_transcribe_t stream, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_embed( + cactus_model_t model, + const char* text, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim, + bool normalize +); + +CACTUS_FFI_EXPORT int cactus_image_embed( + cactus_model_t model, + const char* image_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_audio_embed( + cactus_model_t model, + const char* audio_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_vad( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_diarize( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_embed_speaker( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + const float* mask_weights, + size_t mask_num_frames +); + +CACTUS_FFI_EXPORT int cactus_rag_query( + cactus_model_t model, + const char* query, + char* response_buffer, + size_t buffer_size, + size_t top_k +); + +CACTUS_FFI_EXPORT cactus_index_t cactus_index_init( + const char* index_dir, + size_t embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_index_add( + cactus_index_t index, + const int* ids, + const char** documents, + const char** metadatas, // optional: can be NULL + const float** embeddings, + size_t count, + size_t embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_index_delete( + cactus_index_t index, + const int* ids, + size_t ids_count +); + +CACTUS_FFI_EXPORT int cactus_index_get( + cactus_index_t index, + const int* ids, + size_t ids_count, + char** document_buffers, + size_t* document_buffer_sizes, + char** metadata_buffers, + size_t* metadata_buffer_sizes, + float** embedding_buffers, + size_t* embedding_buffer_sizes +); + +CACTUS_FFI_EXPORT int cactus_index_query( + cactus_index_t index, + const float** embeddings, + size_t embeddings_count, + size_t embedding_dim, + const char* options_json, // optional + int** id_buffers, + size_t* id_buffer_sizes, + float** score_buffers, + size_t* score_buffer_sizes +); + +CACTUS_FFI_EXPORT int cactus_index_compact(cactus_index_t index); +CACTUS_FFI_EXPORT void cactus_index_destroy(cactus_index_t index); + +CACTUS_FFI_EXPORT const char* cactus_get_last_error(void); + +// level: 0=DEBUG, 1=INFO, 2=WARN (default), 3=ERROR, 4=NONE +CACTUS_FFI_EXPORT void cactus_log_set_level(int level); + +typedef void (*cactus_log_callback_t)(int level, const char* component, const char* message, void* user_data); +CACTUS_FFI_EXPORT void cactus_log_set_callback(cactus_log_callback_t callback, void* user_data); + +CACTUS_FFI_EXPORT void cactus_set_telemetry_environment(const char* framework, const char* cache_location, const char* version); +CACTUS_FFI_EXPORT void cactus_set_app_id(const char* app_id); +CACTUS_FFI_EXPORT void cactus_telemetry_flush(void); +CACTUS_FFI_EXPORT void cactus_telemetry_shutdown(void); + +// cactus graph export +typedef void* cactus_graph_t; +typedef uint64_t cactus_node_t; + +typedef struct { + int32_t precision; + size_t rank; + size_t shape[8]; + size_t num_elements; + size_t byte_size; +} cactus_tensor_info_t; + +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_create(void); +CACTUS_FFI_EXPORT void cactus_graph_destroy(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_hard_reset(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_save(cactus_graph_t graph, const char* filename); +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_load(const char* filename); + +CACTUS_FFI_EXPORT int cactus_graph_input( + cactus_graph_t graph, const size_t* shape, size_t rank, int32_t precision, +cactus_node_t* out_node); + +CACTUS_FFI_EXPORT int cactus_graph_set_input( + cactus_graph_t graph, cactus_node_t node, const void* data, int32_t +precision); +CACTUS_FFI_EXPORT int cactus_graph_set_external_input( + cactus_graph_t graph, cactus_node_t node, void* data, int32_t precision); + +CACTUS_FFI_EXPORT int cactus_graph_precision_cast( + cactus_graph_t graph, cactus_node_t input, int32_t target_precision, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_quantize_activations( + cactus_graph_t graph, cactus_node_t input, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_add(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_add_clipped(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_subtract(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_multiply(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_divide(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_scalar_add(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_subtract(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_multiply(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_exp(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sqrt(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_cos(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sin(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_log(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_abs(cactus_graph_t graph, cactus_node_t x, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_pow(cactus_graph_t graph, cactus_node_t x, +float exponent, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_view( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_flatten( + cactus_graph_t graph, cactus_node_t x, int32_t start_dim, int32_t end_dim, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_reshape( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose( + cactus_graph_t graph, cactus_node_t x, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose_n( + cactus_graph_t graph, cactus_node_t x, const size_t* permutation, size_t rank, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_slice( + cactus_graph_t graph, cactus_node_t x, int32_t axis, size_t start, size_t length, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_index( + cactus_graph_t graph, cactus_node_t x, size_t index_value, int32_t dim, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_concat( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, +cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_cat( + cactus_graph_t graph, const cactus_node_t* nodes, size_t count, int32_t +axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_matmul( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, bool pretransposed_rhs, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gather( + cactus_graph_t graph, cactus_node_t tensor, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_tensor( + cactus_graph_t graph, cactus_node_t embedding_tensor, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_file( + cactus_graph_t graph, const char* filename, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_embeddings( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_weights( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_bilinear_interpolation( + cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_set_grouped_scales( + cactus_graph_t graph, cactus_node_t node, size_t group_size, size_t num_groups, void* scales_ptr); +CACTUS_FFI_EXPORT int cactus_graph_set_interleaved( + cactus_graph_t graph, cactus_node_t node, bool interleaved, size_t original_n); +CACTUS_FFI_EXPORT int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_prefetch_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_release_all_weight_pages(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_relu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_silu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu_erf(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sigmoid(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_tanh(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_glu(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_layernorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, float epsilon, bool has_bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_groupnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, size_t num_groups, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_batchnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t running_mean, cactus_node_t running_var, int32_t axis, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_topk(cactus_graph_t graph, cactus_node_t input, size_t k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rms_norm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope_gptj( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, size_t rot_dim, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_softmax(cactus_graph_t graph, cactus_node_t input, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, float scale, bool is_causal, size_t position_offset, size_t window_size, int32_t backend, bool use_mask, cactus_node_t mask, bool additive_mask, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rel_pos_bias( + cactus_graph_t graph, cactus_node_t query, cactus_node_t relative_key, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention_int8_hybrid( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_conv1d_causal( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t kernel_size, size_t dilation, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k7s3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_same_depthwise_k9( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_pointwise( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_depthwise_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_pointwise_1x1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_lstm_cell( + cactus_graph_t graph, cactus_node_t input, cactus_node_t h_prev, cactus_node_t c_prev, cactus_node_t weight_ih, cactus_node_t weight_hh, cactus_node_t bias_ih, cactus_node_t bias_hh, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_decode( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_prefill( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, size_t chunk_size, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_stft( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, size_t num_fft_bins, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_altup_predict( + cactus_graph_t graph, cactus_node_t coefs, const cactus_node_t* streams, size_t num_streams, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_altup_correct( + cactus_graph_t graph, cactus_node_t coefs, cactus_node_t innovation, const cactus_node_t* predictions, size_t num_predictions, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gaussian_topk( + cactus_graph_t graph, cactus_node_t input, float ppf, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_gated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_ungated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_sample( + cactus_graph_t graph, cactus_node_t logits, float temperature, float top_p, size_t top_k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scatter_topk( + cactus_graph_t graph, cactus_node_t indices, cactus_node_t values, size_t num_classes, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_persistent( + cactus_graph_t graph, cactus_node_t source_node, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_is_populated( + cactus_graph_t graph, cactus_node_t persistent_node, int32_t* out_is_populated); +CACTUS_FFI_EXPORT int cactus_graph_invalidate_persistent( + cactus_graph_t graph, cactus_node_t persistent_node); + +CACTUS_FFI_EXPORT int cactus_graph_execute(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_get_output_ptr(cactus_graph_t graph, +cactus_node_t node, void** out_ptr); +CACTUS_FFI_EXPORT int cactus_graph_get_output_info(cactus_graph_t graph, +cactus_node_t node, cactus_tensor_info_t* out_info); + +#ifdef __cplusplus +} +#endif + +#endif // CACTUS_FFI_H diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_utils.h b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_utils.h new file mode 100644 index 0000000..aad98d8 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_utils.h @@ -0,0 +1,1869 @@ +#ifndef CACTUS_UTILS_H +#define CACTUS_UTILS_H + +#include "../engine/engine.h" +#include "../models/model.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#include +#elif defined(_WIN32) +#include +#include +#elif defined(__linux__) || defined(__ANDROID__) +#include +#endif + +inline size_t get_memory_footprint_bytes() { +#ifdef __APPLE__ + task_vm_info_data_t vm_info; + mach_msg_type_number_t count = TASK_VM_INFO_COUNT; + if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&vm_info, &count) == KERN_SUCCESS) + return vm_info.phys_footprint; + +#elif defined(_WIN32) + PROCESS_MEMORY_COUNTERS_EX pmc; + if (GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc))) + return pmc.PrivateUsage; + +#elif defined(__linux__) || defined(__ANDROID__) + std::ifstream statm("/proc/self/statm"); + if (statm.is_open()) { + size_t size, resident; + statm >> size >> resident; + return resident * sysconf(_SC_PAGESIZE); + } +#endif + return 0; +} + +inline double get_ram_usage_mb() { + return get_memory_footprint_bytes() / (1024.0 * 1024.0); +} + +struct CactusModelHandle { + std::unique_ptr model; + std::unique_ptr vad_model; + std::atomic should_stop; + std::vector processed_tokens; + struct ProcessedImage { + std::string path; + long long last_modified_timestamp = 0; + + bool operator==(const ProcessedImage& other) const { + return path == other.path && last_modified_timestamp == other.last_modified_timestamp; + } + }; + + std::vector> processed_images; + std::vector user_audio_counts; + std::mutex model_mutex; + std::string model_name; + std::unique_ptr corpus_index; + std::string corpus_dir; + size_t corpus_embedding_dim = 0; + std::vector> tool_embeddings; + std::vector tool_texts; + + CactusModelHandle() : should_stop(false) {} +}; + +extern std::string last_error_message; + +bool matches_stop_sequence(const std::vector& generated_tokens, + const std::vector>& stop_sequences); + +std::string retrieve_rag_context(CactusModelHandle* handle, const std::string& query); + +namespace cactus { +namespace audio { + +static constexpr size_t WHISPER_TARGET_FRAMES = 3000; +static constexpr int WHISPER_SAMPLE_RATE = 16000; + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_whisper_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 400; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = true; + cfg.pad_mode = "reflect"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 1e-10f; + cfg.log_mel = "log10"; + cfg.reference = 1.0f; + cfg.min_value = 1e-10f; + cfg.remove_dc_offset = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_parakeet_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 512; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = true; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 5.960464477539063e-08f; // 2^-24 guard value used by HF Parakeet. + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 1e-10f; + cfg.remove_dc_offset = false; + cfg.hann_periodic = false; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_htk_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 321; + cfg.frame_length = 320; + cfg.fft_override = 1024; + cfg.hop_length = 160; + cfg.power = 1.0f; + cfg.center = false; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 0.001f; + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 0.001f; + cfg.remove_dc_offset = false; + cfg.hann_periodic = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_gemma4_audio_spectrogram_config( + const cactus::engine::Config& model_config) { + auto cfg = get_htk_spectrogram_config(); + cfg.fft_override = model_config.audio_fft_length; + cfg.mel_floor_additive = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_wespeaker_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 512; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = false; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 1.1754944e-38f; + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 1.1754944e-38f; + cfg.remove_dc_offset = true; + cfg.preemphasis = 0.97f; + cfg.hann_periodic = false; + cfg.window_a0 = 0.54f; + return cfg; +} + +// Whisper v1/v2: 80 mel bins, HTK. Whisper v3: 128 mel bins, Slaney, 512-FFT, no DC removal. +inline void init_whisper_mel_filters(cactus::engine::AudioProcessor& ap, + cactus::engine::AudioProcessor::SpectrogramConfig& cfg, + size_t mel_bins) { + const size_t num_mel_filters = std::max(1, mel_bins); + const bool is_v3 = mel_bins > 80; + if (is_v3) { + cfg.fft_override = 512; + cfg.remove_dc_offset = false; + } + const size_t fft_len = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; + const size_t num_frequency_bins = fft_len / 2 + 1; + if (is_v3) { + ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, + WHISPER_SAMPLE_RATE, "slaney", "slaney"); + } else { + ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, + WHISPER_SAMPLE_RATE); + } +} + +// use_mel_floor_padding=true pads short audio with the normalized mel floor (required for v3). +inline std::vector normalize_whisper_mel(std::vector& mel, size_t n_mels, + bool use_mel_floor_padding = false) { + if (mel.empty() || n_mels == 0) return mel; + size_t n_frames = mel.size() / n_mels; + + float max_val = -std::numeric_limits::infinity(); + for (float v : mel) if (v > max_val) max_val = v; + + float min_allowed = max_val - 8.0f; + for (float& v : mel) { + if (v < min_allowed) v = min_allowed; + v = (v + 4.0f) * 0.25f; + } + + if (n_frames != WHISPER_TARGET_FRAMES) { + float pad_val = use_mel_floor_padding ? (min_allowed + 4.0f) * 0.25f : 0.0f; + std::vector fixed(n_mels * WHISPER_TARGET_FRAMES, pad_val); + size_t copy_frames = std::min(n_frames, WHISPER_TARGET_FRAMES); + for (size_t m = 0; m < n_mels; ++m) { + const float* src = &mel[m * n_frames]; + float* dst = &fixed[m * WHISPER_TARGET_FRAMES]; + std::copy(src, src + copy_frames, dst); + } + return fixed; + } + return std::move(mel); +} + +inline std::vector transpose_mel_to_frame_major(const std::vector& mel, + size_t num_mels, size_t num_frames) { + std::vector transposed(num_frames * num_mels); + for (size_t m = 0; m < num_mels; m++) { + for (size_t t = 0; t < num_frames; t++) { + transposed[t * num_mels + m] = mel[m * num_frames + t]; + } + } + return transposed; +} + +inline void apply_preemphasis(std::vector& waveform, float coefficient = 0.97f) { + if (waveform.size() < 2 || coefficient == 0.0f) { + return; + } + for (size_t i = waveform.size() - 1; i > 0; --i) { + waveform[i] -= coefficient * waveform[i - 1]; + } +} + +inline void normalize_parakeet_log_mel(std::vector& mel, size_t num_mels, float epsilon = 1e-5f) { + if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { + return; + } + const size_t num_frames = mel.size() / num_mels; + if (num_frames == 0) { + return; + } + + for (size_t m = 0; m < num_mels; ++m) { + const size_t base = m * num_frames; + float mean = 0.0f; + for (size_t t = 0; t < num_frames; ++t) { + mean += mel[base + t]; + } + mean /= static_cast(num_frames); + + float variance = 0.0f; + for (size_t t = 0; t < num_frames; ++t) { + const float d = mel[base + t] - mean; + variance += d * d; + } + const float denom = static_cast(std::max(1, num_frames - 1)); + const float inv_std = 1.0f / std::sqrt((variance / denom) + epsilon); + for (size_t t = 0; t < num_frames; ++t) { + mel[base + t] = (mel[base + t] - mean) * inv_std; + } + } +} + +inline void trim_mel_frames(std::vector& mel, size_t num_mels, size_t valid_frames) { + if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { + return; + } + size_t total_frames = mel.size() / num_mels; + if (valid_frames == 0 || valid_frames >= total_frames) { + return; + } + std::vector trimmed(num_mels * valid_frames); + for (size_t m = 0; m < num_mels; ++m) { + const float* src = &mel[m * total_frames]; + float* dst = &trimmed[m * valid_frames]; + std::copy(src, src + valid_frames, dst); + } + mel.swap(trimmed); +} + +struct AudioPreprocessResult { + std::vector features; + size_t num_frames = 0; + size_t num_soft_tokens = 0; +}; + +inline AudioPreprocessResult preprocess_audio_for_gemma4( + std::vector audio_samples, + const cactus::engine::Config& model_config +) { + AudioPreprocessResult result; + if (audio_samples.empty()) return result; + + size_t pad_amt = 320 - (audio_samples.size() % 320); + if (pad_amt < 320) + audio_samples.resize(audio_samples.size() + pad_amt, 0.0f); + + size_t mel_bins = model_config.audio_input_feat_size; + auto cfg = get_gemma4_audio_spectrogram_config(model_config); + + size_t semicausal_pad = cfg.frame_length / 2; + audio_samples.insert(audio_samples.begin(), semicausal_pad, 0.0f); + + cactus::engine::AudioProcessor ap; + size_t fft_for_mel = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; + ap.init_mel_filters(fft_for_mel / 2 + 1, mel_bins, 0.0f, 8000.0f, 16000, + nullptr, "htk"); + std::vector mel = ap.compute_spectrogram(audio_samples, cfg); + + result.num_frames = mel.size() / mel_bins; + result.features = transpose_mel_to_frame_major(mel, mel_bins, result.num_frames); + + size_t after_stage1 = (result.num_frames + 1) / 2; + result.num_soft_tokens = (after_stage1 + 1) / 2; + + return result; +} + +inline std::vector pcm_buffer_to_float_samples( + const uint8_t* pcm_buffer, size_t pcm_buffer_size +) { + const int16_t* pcm_samples = reinterpret_cast(pcm_buffer); + size_t num_samples = pcm_buffer_size / 2; + std::vector waveform_fp32(num_samples); + constexpr float inv_32768 = 1.0f / 32768.0f; + for (size_t i = 0; i < num_samples; i++) + waveform_fp32[i] = static_cast(pcm_samples[i]) * inv_32768; + return waveform_fp32; +} + +} // namespace audio +} // namespace cactus + +namespace cactus { +namespace ffi { + +inline bool env_flag_enabled(const char* key) { + const char* value = std::getenv(key); + return value && value[0] != '\0' && !(value[0] == '0' && value[1] == '\0'); +} + +inline std::string generateUUID() { +#ifdef __APPLE__ + uuid_t uuid; + uuid_generate_random(uuid); + char uuid_str[37]; + uuid_unparse_lower(uuid, uuid_str); + return std::string(uuid_str); +#else + static std::random_device rd; + static std::mt19937 gen(rd()); + static std::uniform_int_distribution<> dis(0, 15); + static std::uniform_int_distribution<> dis2(8, 11); + + std::stringstream ss; + ss << std::hex; + for (int i = 0; i < 8; i++) ss << dis(gen); + ss << "-"; + for (int i = 0; i < 4; i++) ss << dis(gen); + ss << "-4"; + for (int i = 0; i < 3; i++) ss << dis(gen); + ss << "-"; + ss << dis2(gen); + for (int i = 0; i < 3; i++) ss << dis(gen); + ss << "-"; + for (int i = 0; i < 12; i++) ss << dis(gen); + return ss.str(); +#endif +} + +struct ToolFunction { + std::string name; + std::string description; + std::unordered_map parameters; +}; + +struct InferenceOptions { + float temperature = 0.0f; + float top_p = 0.0f; + float min_p = 0.15f; + float repetition_penalty = 1.1f; + float confidence_threshold = -1.0f; + size_t top_k = 0; + size_t max_tokens = 100; + size_t tool_rag_top_k = 2; + size_t cloud_timeout_ms = 15000; + std::vector stop_sequences; + bool force_tools = false; + bool include_stop_sequences = false; + bool use_vad = true; + bool telemetry_enabled = true; + bool auto_handoff = true; + bool handoff_with_images = true; + bool enable_thinking_if_supported = false; +}; + +} // namespace ffi +} // namespace cactus + +std::vector select_relevant_tools( + CactusModelHandle* handle, + const std::string& query, + const std::vector& all_tools, + size_t top_k); + +#include "gemma_tools.h" + +namespace cactus { +namespace ffi { + +inline std::string escape_json_string(const std::string& s) { + std::ostringstream o; + for (char c : s) { + if (c == '"') o << "\\\""; + else if (c == '\n') o << "\\n"; + else if (c == '\r') o << "\\r"; + else if (c == '\t') o << "\\t"; + else if (c == '\\') o << "\\\\"; + else o << c; + } + return o.str(); +} + + +inline std::string trim_string(const std::string& s) { + size_t start = 0; + while (start < s.size() && std::isspace(static_cast(s[start]))) ++start; + size_t end = s.size(); + while (end > start && std::isspace(static_cast(s[end - 1]))) --end; + return s.substr(start, end - start); +} + +inline size_t find_matching_delimiter(const std::string& s, size_t pos, char open, char close) { + int depth = 1; + pos++; + while (pos < s.length() && depth > 0) { + if (s[pos] == open) depth++; + else if (s[pos] == close) depth--; + else if (s[pos] == '"') { + pos++; + while (pos < s.length() && s[pos] != '"') { + if (s[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + return pos; +} + +inline std::string env_or_default(const char* key, const char* fallback) { + const char* v = std::getenv(key); + if (v && v[0] != '\0') return std::string(v); + return std::string(fallback); +} + +inline std::string json_string_field(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return {}; + + size_t i = pos + pattern.size(); + while (i < json.size() && std::isspace(static_cast(json[i]))) i++; + if (i >= json.size() || json[i] != '"') return {}; + ++i; + + std::string out; + out.reserve(128); + while (i < json.size()) { + char c = json[i++]; + if (c == '"') return out; + if (c == '\\' && i < json.size()) { + char e = json[i++]; + switch (e) { + case '"': out.push_back('"'); break; + case '\\': out.push_back('\\'); break; + case '/': out.push_back('/'); break; + case 'b': out.push_back('\b'); break; + case 'f': out.push_back('\f'); break; + case 'n': out.push_back('\n'); break; + case 'r': out.push_back('\r'); break; + case 't': out.push_back('\t'); break; + default: out.push_back(e); break; + } + continue; + } + out.push_back(c); + } + return {}; +} + +inline std::string json_array_field(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return "[]"; + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + if (start >= json.size() || json[start] != '[') return "[]"; + + int depth = 1; + size_t end = start + 1; + while (end < json.size() && depth > 0) { + if (json[end] == '[') depth++; + else if (json[end] == ']') depth--; + end++; + } + return json.substr(start, end - start); +} + +inline std::vector split_json_array(const std::string& array_json) { + std::vector out; + if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; + + size_t i = 1; + while (i + 1 < array_json.size()) { + while (i + 1 < array_json.size() && + (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) i++; + if (i + 1 >= array_json.size() || array_json[i] != '{') break; + + size_t start = i; + int depth = 0; + bool in_str = false; + bool esc = false; + for (; i < array_json.size(); ++i) { + char c = array_json[i]; + if (in_str) { + if (esc) esc = false; + else if (c == '\\') esc = true; + else if (c == '"') in_str = false; + continue; + } + if (c == '"') { in_str = true; continue; } + if (c == '{') depth++; + if (c == '}') { + depth--; + if (depth == 0) { + out.push_back(array_json.substr(start, i - start + 1)); + i++; + break; + } + } + } + } + return out; +} + +inline std::string serialize_tools_json(const std::vector& tools) { + if (tools.empty()) return ""; + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < tools.size(); ++i) { + if (i > 0) oss << ","; + oss << "{\"type\":\"function\",\"function\":{"; + oss << "\"name\":\"" << escape_json_string(tools[i].name) << "\","; + oss << "\"description\":\"" << escape_json_string(tools[i].description) << "\""; + auto it = tools[i].parameters.find("schema"); + if (it != tools[i].parameters.end()) { + oss << ",\"parameters\":" << it->second; + } + oss << "}}"; + } + oss << "]"; + return oss.str(); +} + +namespace json_sorted { + +inline void skip_ws(const std::string& s, size_t& p) { + while (p < s.size() && std::isspace(static_cast(s[p]))) p++; +} + +inline std::string parse_string(const std::string& s, size_t& p) { + std::string r = "\""; + p++; + while (p < s.size()) { + if (s[p] == '\\') { + r += s[p++]; + if (p < s.size()) r += s[p++]; + } else if (s[p] == '"') { + r += '"'; + p++; + return r; + } else { + r += s[p++]; + } + } + return r; +} + +inline std::string parse_value(const std::string& s, size_t& p); + +inline std::string parse_object(const std::string& s, size_t& p) { + p++; + std::map entries; + skip_ws(s, p); + while (p < s.size() && s[p] != '}') { + if (s[p] == ',') { p++; skip_ws(s, p); continue; } + std::string key = parse_string(s, p); + skip_ws(s, p); + if (p < s.size() && s[p] == ':') p++; + skip_ws(s, p); + std::string val = parse_value(s, p); + entries[key] = val; + skip_ws(s, p); + } + if (p < s.size()) p++; + std::string r = "{"; + bool first = true; + for (const auto& kv : entries) { + if (!first) r += ", "; + r += kv.first + ": " + kv.second; + first = false; + } + r += "}"; + return r; +} + +inline std::string parse_array(const std::string& s, size_t& p) { + p++; + std::vector items; + skip_ws(s, p); + while (p < s.size() && s[p] != ']') { + if (s[p] == ',') { p++; skip_ws(s, p); continue; } + items.push_back(parse_value(s, p)); + skip_ws(s, p); + } + if (p < s.size()) p++; + std::string r = "["; + for (size_t i = 0; i < items.size(); i++) { + if (i > 0) r += ", "; + r += items[i]; + } + r += "]"; + return r; +} + +inline std::string parse_value(const std::string& s, size_t& p) { + skip_ws(s, p); + if (p >= s.size()) return ""; + if (s[p] == '"') return parse_string(s, p); + if (s[p] == '{') return parse_object(s, p); + if (s[p] == '[') return parse_array(s, p); + size_t start = p; + while (p < s.size() && s[p] != ',' && s[p] != '}' && s[p] != ']' && !std::isspace(static_cast(s[p]))) p++; + return s.substr(start, p - start); +} + +inline std::string reformat(const std::string& json) { + size_t p = 0; + return parse_value(json, p); +} + +} // namespace json_sorted + +inline std::string serialize_tools_for_template(const std::vector& tools) { + if (tools.empty()) return ""; + std::string result; + for (const auto& tool : tools) { + std::map func_fields; + func_fields["\"description\""] = "\"" + escape_json_string(tool.description) + "\""; + func_fields["\"name\""] = "\"" + escape_json_string(tool.name) + "\""; + auto it = tool.parameters.find("schema"); + if (it != tool.parameters.end()) { + func_fields["\"parameters\""] = json_sorted::reformat(it->second); + } + std::string func_json = "{"; + bool first = true; + for (const auto& kv : func_fields) { + if (!first) func_json += ", "; + func_json += kv.first + ": " + kv.second; + first = false; + } + func_json += "}"; + result += "\n{\"function\": " + func_json + ", \"type\": \"function\"}"; + } + return result; +} + +inline void handle_error_response(const std::string& error_message, char* response_buffer, size_t buffer_size) { + std::ostringstream json; + json << "{"; + json << "\"success\":false,"; + json << "\"error\":\"" << escape_json_string(error_message) << "\","; + json << "\"cloud_handoff\":false,"; + json << "\"response\":null,"; + json << "\"function_calls\":[],"; + json << "\"confidence\":0.0,"; + json << "\"time_to_first_token_ms\":0.0,"; + json << "\"total_time_ms\":0.0,"; + json << "\"prefill_tps\":0.0,"; + json << "\"decode_tps\":0.0,"; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; + json << "\"prefill_tokens\":0,"; + json << "\"decode_tokens\":0,"; + json << "\"total_tokens\":0"; + json << "}"; + std::string error_json = json.str(); + if (response_buffer && error_json.length() < buffer_size) { + std::strcpy(response_buffer, error_json.c_str()); + } +} + +inline std::vector parse_messages_json(const std::string& json, + std::vector& out_image_paths, + std::vector* out_audio_paths = nullptr) { + std::vector messages; + out_image_paths.clear(); + if (out_audio_paths) out_audio_paths->clear(); + + size_t pos = json.find('['); + if (pos == std::string::npos) { + throw std::runtime_error("Invalid JSON: expected array"); + } + + pos = json.find('{', pos); + while (pos != std::string::npos) { + cactus::engine::ChatMessage msg; + + size_t obj_start = pos; + int brace_count = 1; + size_t obj_end = obj_start + 1; + while (obj_end < json.length() && brace_count > 0) { + if (json[obj_end] == '{') brace_count++; + else if (json[obj_end] == '}') brace_count--; + obj_end++; + } + + size_t role_pos = json.find("\"role\"", pos); + if (role_pos == std::string::npos || role_pos >= obj_end) break; + + size_t role_start = json.find('"', role_pos + 6) + 1; + size_t role_end = json.find('"', role_start); + msg.role = json.substr(role_start, role_end - role_start); + + size_t content_pos = json.find("\"content\"", role_end); + if (content_pos != std::string::npos && content_pos < obj_end) { + size_t content_start = json.find('"', content_pos + 9) + 1; + size_t content_end = content_start; + + while (content_end < json.length()) { + content_end = json.find('"', content_end); + if (content_end == std::string::npos) break; + if (json[content_end - 1] != '\\') break; + content_end++; + } + + msg.content = json.substr(content_start, content_end - content_start); + + size_t escape_pos = 0; + while ((escape_pos = msg.content.find("\\n", escape_pos)) != std::string::npos) { + msg.content.replace(escape_pos, 2, "\n"); + escape_pos += 1; + } + escape_pos = 0; + while ((escape_pos = msg.content.find("\\\"", escape_pos)) != std::string::npos) { + msg.content.replace(escape_pos, 2, "\""); + escape_pos += 1; + } + } + + auto parse_path_array = [&](const char* key, std::vector& dest, + std::vector* out_paths) { + size_t key_pos = json.find(key, pos); + if (key_pos == std::string::npos || key_pos >= obj_end) return; + size_t array_start = json.find('[', key_pos); + if (array_start == std::string::npos || array_start >= obj_end) return; + size_t array_end = json.find(']', array_start); + if (array_end == std::string::npos || array_end >= obj_end) return; + size_t cur = array_start; + while (true) { + cur = json.find('"', cur + 1); + if (cur == std::string::npos || cur >= array_end) break; + size_t str_start = cur + 1; + size_t str_end = json.find('"', str_start); + if (str_end == std::string::npos || str_end > array_end) break; + std::string path = std::filesystem::absolute( + std::filesystem::path(json.substr(str_start, str_end - str_start))).string(); + dest.push_back(path); + if (out_paths) out_paths->push_back(path); + cur = str_end; + } + }; + + parse_path_array("\"images\"", msg.images, &out_image_paths); + parse_path_array("\"audio\"", msg.audio, out_audio_paths); + + if (msg.role == "tool") { + size_t name_pos = json.find("\"name\"", obj_start); + if (name_pos != std::string::npos && name_pos < obj_end) { + size_t name_quote = json.find('"', name_pos + 6); + if (name_quote != std::string::npos && name_quote < obj_end) { + size_t name_start = name_quote + 1; + size_t name_end = json.find('"', name_start); + if (name_end != std::string::npos && name_end < obj_end) { + msg.name = json.substr(name_start, name_end - name_start); + } + } + } + } + + size_t tool_calls_pos = json.find("\"tool_calls\"", obj_start); + if (tool_calls_pos != std::string::npos && tool_calls_pos < obj_end) { + size_t tool_calls_arr_start = json.find('[', tool_calls_pos); + if (tool_calls_arr_start != std::string::npos && tool_calls_arr_start < obj_end) { + size_t tool_calls_arr_end = find_matching_delimiter(json, tool_calls_arr_start, '[', ']'); + + size_t search_pos = tool_calls_arr_start; + while (true) { + size_t func_pos = json.find("\"function\"", search_pos); + if (func_pos == std::string::npos || func_pos >= tool_calls_arr_end) break; + + size_t func_obj_start = json.find('{', func_pos + 10); + if (func_obj_start == std::string::npos || func_obj_start >= tool_calls_arr_end) break; + + size_t func_obj_end = find_matching_delimiter(json, func_obj_start, '{', '}'); + + cactus::engine::ToolCallInfo tool_call; + + size_t fn_name_pos = json.find("\"name\"", func_obj_start); + if (fn_name_pos != std::string::npos && fn_name_pos < func_obj_end) { + size_t fn_name_quote = json.find('"', fn_name_pos + 6); + if (fn_name_quote != std::string::npos && fn_name_quote < func_obj_end) { + size_t fn_name_start = fn_name_quote + 1; + size_t fn_name_end = json.find('"', fn_name_start); + if (fn_name_end != std::string::npos && fn_name_end < func_obj_end) { + tool_call.name = json.substr(fn_name_start, fn_name_end - fn_name_start); + } + } + } + + size_t args_pos = json.find("\"arguments\"", func_obj_start); + if (args_pos != std::string::npos && args_pos < func_obj_end) { + size_t colon_pos = json.find(':', args_pos + 11); + if (colon_pos != std::string::npos && colon_pos < func_obj_end) { + size_t args_start = colon_pos + 1; + while (args_start < json.length() && std::isspace(static_cast(json[args_start]))) args_start++; + + if (args_start < func_obj_end && json[args_start] == '{') { + size_t args_end = find_matching_delimiter(json, args_start, '{', '}'); + tool_call.arguments = json.substr(args_start, args_end - args_start); + } else if (args_start < func_obj_end && json[args_start] == '"') { + size_t str_start = args_start + 1; + size_t str_end = str_start; + while (str_end < json.length() && json[str_end] != '"') { + if (json[str_end] == '\\') str_end++; + str_end++; + } + tool_call.arguments = json.substr(str_start, str_end - str_start); + } + } + } + + if (!tool_call.name.empty()) { + msg.tool_calls.push_back(tool_call); + } + search_pos = func_obj_end; + } + } + } + + messages.push_back(msg); + + pos = json.find('{', obj_end); + } + + return messages; +} + +inline std::vector parse_tools_json(const std::string& json) { + std::vector tools; + + if (json.empty()) return tools; + + size_t pos = json.find('['); + if (pos == std::string::npos) return tools; + + pos = json.find("\"function\"", pos); + while (pos != std::string::npos) { + ToolFunction tool; + size_t next_search = pos + 1; + + size_t name_pos = json.find("\"name\"", pos); + if (name_pos != std::string::npos) { + size_t name_start = json.find('"', name_pos + 6) + 1; + size_t name_end = json.find('"', name_start); + tool.name = json.substr(name_start, name_end - name_start); + } + + size_t desc_pos = json.find("\"description\"", pos); + if (desc_pos != std::string::npos) { + size_t desc_start = json.find('"', desc_pos + 13) + 1; + size_t desc_end = json.find('"', desc_start); + tool.description = json.substr(desc_start, desc_end - desc_start); + } + + size_t params_pos = json.find("\"parameters\"", pos); + if (params_pos != std::string::npos) { + size_t params_start = json.find('{', params_pos); + if (params_start != std::string::npos) { + int brace_count = 1; + size_t params_end = params_start + 1; + while (params_end < json.length() && brace_count > 0) { + if (json[params_end] == '{') brace_count++; + else if (json[params_end] == '}') brace_count--; + params_end++; + } + tool.parameters["schema"] = json.substr(params_start, params_end - params_start); + next_search = params_end; + } + } + + if (!tool.name.empty()) { + tools.push_back(tool); + } + + pos = json.find("\"function\"", next_search); + } + + return tools; +} + +inline bool try_parse_json_float(const std::string& json, const std::string& key, float& out_value) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return false; + + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + + size_t end = start; + while (end < json.size() && std::string(",}] \t\n\r").find(json[end]) == std::string::npos) ++end; + + try { + out_value = std::stof(json.substr(start, end - start)); + return true; + } catch (...) { + return false; + } +} + +inline std::vector parse_json_string_array_field(const std::string& json, const std::string& key) { + std::vector out; + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return out; + + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + if (start >= json.size() || json[start] != '[') return out; + + int depth = 1; + bool in_string = false; + bool escaped = false; + size_t end = start + 1; + + while (end < json.size() && depth > 0) { + char c = json[end]; + if (in_string) { + if (escaped) escaped = false; + else if (c == '\\') escaped = true; + else if (c == '"') in_string = false; + } else { + if (c == '"') in_string = true; + else if (c == '[') depth++; + else if (c == ']') depth--; + } + ++end; + } + + if (depth != 0) return out; + const std::string array_json = json.substr(start, end - start); + if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; + + size_t i = 1; + while (i + 1 < array_json.size()) { + while (i + 1 < array_json.size() && + (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) { + ++i; + } + if (i + 1 >= array_json.size() || array_json[i] == ']') break; + if (array_json[i] != '"') break; + + ++i; + std::string value; + bool escaped = false; + while (i < array_json.size()) { + char c = array_json[i++]; + if (escaped) { + switch (c) { + case '"': value.push_back('"'); break; + case '\\': value.push_back('\\'); break; + case '/': value.push_back('/'); break; + case 'b': value.push_back('\b'); break; + case 'f': value.push_back('\f'); break; + case 'n': value.push_back('\n'); break; + case 'r': value.push_back('\r'); break; + case 't': value.push_back('\t'); break; + default: value.push_back(c); break; + } + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == '"') { + out.push_back(value); + break; + } + value.push_back(c); + } + } + + return out; +} + +inline void parse_custom_vocabulary_options(const std::string& json, + std::vector& custom_vocabulary, + float& vocabulary_boost) { + custom_vocabulary.clear(); + vocabulary_boost = 5.0f; + if (json.empty()) return; + + float parsed_boost = vocabulary_boost; + if (try_parse_json_float(json, "vocabulary_boost", parsed_boost)) { + vocabulary_boost = std::clamp(parsed_boost, 0.0f, 20.0f); + } + + custom_vocabulary = parse_json_string_array_field(json, "custom_vocabulary"); +} + +inline std::unordered_map build_token_bias_map(const std::vector>& tokenized_entries, + float vocabulary_boost) { + std::unordered_map vocab_bias; + const float clamped_boost = std::clamp(vocabulary_boost, 0.0f, 20.0f); + if (clamped_boost == 0.0f) return vocab_bias; + + for (const auto& token_ids : tokenized_entries) { + for (uint32_t token_id : token_ids) { + float& entry = vocab_bias[token_id]; + if (entry < clamped_boost) { + entry = clamped_boost; + } + } + } + + return vocab_bias; +} + +inline std::unordered_map build_custom_vocabulary_bias(cactus::engine::Tokenizer* tokenizer, + const std::vector& custom_vocabulary, + float vocabulary_boost) { + if (!tokenizer || custom_vocabulary.empty()) return {}; + std::vector> tokenized_entries; + tokenized_entries.reserve(custom_vocabulary.size()); + + for (const auto& word : custom_vocabulary) { + if (word.empty()) continue; + tokenized_entries.push_back(tokenizer->encode(word)); + } + + return build_token_bias_map(tokenized_entries, vocabulary_boost); +} + +inline void apply_custom_vocabulary_options(cactus::engine::Model* model, const std::string& json) { + if (!model) return; + + std::vector custom_vocabulary; + float vocabulary_boost = 5.0f; + parse_custom_vocabulary_options(json, custom_vocabulary, vocabulary_boost); + model->set_vocab_bias(build_custom_vocabulary_bias(model->get_tokenizer(), custom_vocabulary, vocabulary_boost)); +} + +inline size_t levenshtein_ci(const std::string& a, const std::string& b) { + const size_t m = a.size(), n = b.size(); + std::vector prev(n + 1), curr(n + 1); + for (size_t j = 0; j <= n; ++j) prev[j] = j; + for (size_t i = 1; i <= m; ++i) { + curr[0] = i; + for (size_t j = 1; j <= n; ++j) { + const bool match = std::tolower(static_cast(a[i - 1])) == + std::tolower(static_cast(b[j - 1])); + curr[j] = std::min({prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + (match ? 0 : 1)}); + } + std::swap(prev, curr); + } + return prev[n]; +} + +inline std::string collapse_spaces(const std::string& s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + if (c != ' ') out += c; + } + return out; +} + +inline void apply_vocabulary_spelling_correction( + std::string& text, + const std::vector& custom_vocabulary) +{ + if (custom_vocabulary.empty() || text.empty()) return; + + struct VocabEntry { + const std::string* original; + std::string collapsed; + }; + std::vector vocab_entries; + vocab_entries.reserve(custom_vocabulary.size()); + for (const auto& v : custom_vocabulary) { + vocab_entries.push_back({&v, collapse_spaces(v)}); + } + + struct Token { std::string text; bool is_word; }; + std::vector tokens; + size_t pos = 0; + while (pos < text.size()) { + if (std::isalnum(static_cast(text[pos])) || + text[pos] == '\'' || text[pos] == '-') { + size_t start = pos; + while (pos < text.size() && (std::isalnum(static_cast(text[pos])) || + text[pos] == '\'' || text[pos] == '-')) { + ++pos; + } + tokens.push_back({text.substr(start, pos - start), true}); + } else { + size_t start = pos; + while (pos < text.size() && !std::isalnum(static_cast(text[pos])) && + text[pos] != '\'' && text[pos] != '-') { + ++pos; + } + tokens.push_back({text.substr(start, pos - start), false}); + } + } + + std::vector word_indices; + for (size_t i = 0; i < tokens.size(); ++i) { + if (tokens[i].is_word) word_indices.push_back(i); + } + + std::vector consumed(tokens.size(), false); + + auto strip_suffix = [](const std::string& word) -> std::pair { + if (word.size() >= 3 && word.substr(word.size() - 2) == "'s") { + return {word.substr(0, word.size() - 2), "'s"}; + } + if (word.size() >= 3 && word.substr(word.size() - 2) == "'t") { + return {word.substr(0, word.size() - 2), "'t"}; + } + if (word.size() >= 4 && word.back() == 's' && + word[word.size() - 2] != 's' && // avoid stripping from "boss", "class" + std::isalpha(static_cast(word[word.size() - 2]))) { + return {word.substr(0, word.size() - 1), "s"}; + } + return {word, ""}; + }; + + size_t wi = 0; + while (wi < word_indices.size()) { + size_t best_dist = std::numeric_limits::max(); + const std::string* best_match = nullptr; + size_t best_window = 0; + size_t best_first_token = 0; + size_t best_last_token = 0; + std::string best_suffix; + + for (size_t window = std::min(3, word_indices.size() - wi); window >= 1; --window) { + std::string window_collapsed; + const size_t first_tok = word_indices[wi]; + const size_t last_tok = word_indices[wi + window - 1]; + for (size_t w = 0; w < window; ++w) { + window_collapsed += tokens[word_indices[wi + w]].text; + } + + if (window == 1 && window_collapsed.size() < 3) break; + + auto [stem, suffix] = strip_suffix(window_collapsed); + const std::string* candidates[] = {&window_collapsed, &stem}; + const std::string suffixes[] = {"", suffix}; + const size_t num_candidates = suffix.empty() ? 1 : 2; + + for (size_t ci = 0; ci < num_candidates; ++ci) { + const std::string& candidate = *candidates[ci]; + if (candidate.empty()) continue; + + for (const auto& entry : vocab_entries) { + const size_t wlen = candidate.size(); + const size_t vlen = entry.collapsed.size(); + + const size_t len_diff = wlen > vlen ? wlen - vlen : vlen - wlen; + const size_t max_dist = std::max(1, std::min(wlen, vlen) / 3); + if (len_diff > max_dist) continue; + + const size_t dist = levenshtein_ci(candidate, entry.collapsed); + + // For single-edit corrections, require first char match to prevent + // false positives like "vortex" → "Cortex". + if (dist == 1 && window == 1) { + const bool first_char_match = + std::tolower(static_cast(candidate[0])) == + std::tolower(static_cast(entry.collapsed[0])); + if (!first_char_match) continue; + } + + if (dist <= max_dist && dist < best_dist) { + best_dist = dist; + best_match = entry.original; + best_window = window; + best_first_token = first_tok; + best_last_token = last_tok; + best_suffix = suffixes[ci]; + } + } + } + + if (best_dist == 0) break; + } + + // Allow dist==0 for multi-word merges where word boundaries changed. + const bool should_replace = best_match && + best_dist != std::numeric_limits::max() && + (best_dist > 0 || best_window > 1); + + if (should_replace) { + tokens[best_first_token].text = *best_match + best_suffix; + for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { + consumed[t] = true; + } + for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { + if (t > 0) consumed[t - 1] = consumed[t - 1] || !tokens[t - 1].is_word; + } + wi += best_window; + } else { + ++wi; + } + } + + std::string result; + result.reserve(text.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + if (!consumed[i]) { + result += tokens[i].text; + } + } + + text = std::move(result); +} + +inline InferenceOptions parse_inference_options_json(const std::string& json) { + InferenceOptions options; + + if (json.empty()) return options; + + size_t pos = json.find("\"temperature\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.temperature = std::stof(json.substr(pos)); + } + + pos = json.find("\"top_p\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.top_p = std::stof(json.substr(pos)); + } + + float parsed_min_p = options.min_p; + if (try_parse_json_float(json, "min_p", parsed_min_p)) { + options.min_p = std::clamp(parsed_min_p, 0.0f, 1.0f); + } + + float parsed_rep_penalty = options.repetition_penalty; + if (try_parse_json_float(json, "repetition_penalty", parsed_rep_penalty)) { + if (std::isfinite(parsed_rep_penalty) && parsed_rep_penalty > 0.0f) { + options.repetition_penalty = parsed_rep_penalty; + } + } + + pos = json.find("\"top_k\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.top_k = std::stoul(json.substr(pos)); + } + + pos = json.find("\"max_tokens\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.max_tokens = std::stoul(json.substr(pos)); + } + + pos = json.find("\"force_tools\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.force_tools = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"tool_rag_top_k\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.tool_rag_top_k = std::stoul(json.substr(pos)); + } + + pos = json.find("\"confidence_threshold\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.confidence_threshold = std::stof(json.substr(pos)); + } + + pos = json.find("\"include_stop_sequences\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.include_stop_sequences = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"use_vad\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.use_vad = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"telemetry_enabled\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.telemetry_enabled = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"auto_handoff\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.auto_handoff = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"cloud_timeout_ms\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.cloud_timeout_ms = std::stoul(json.substr(pos)); + } + + pos = json.find("\"handoff_with_images\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.handoff_with_images = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"enable_thinking_if_supported\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.enable_thinking_if_supported = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"stop_sequences\""); + if (pos != std::string::npos) { + pos = json.find('[', pos); + if (pos != std::string::npos) { + size_t end_pos = json.find(']', pos); + size_t seq_pos = json.find('"', pos); + + while (seq_pos != std::string::npos && seq_pos < end_pos) { + size_t seq_start = seq_pos + 1; + size_t seq_end = json.find('"', seq_start); + if (seq_end != std::string::npos) { + options.stop_sequences.push_back(json.substr(seq_start, seq_end - seq_start)); + } + seq_pos = json.find('"', seq_end + 1); + } + } + } + + return options; +} + +static inline std::string trim_lfm2_slice(const std::string& value, size_t begin, size_t end) { + return trim_string(value.substr(begin, end - begin)); +} + +static inline void append_lfm2_call(const std::string& entry, + std::vector& function_calls) { + if (entry.empty()) return; + + std::string trimmed_entry = trim_lfm2_slice(entry, 0, entry.size()); + if (trimmed_entry.empty()) return; + + size_t paren_pos = trimmed_entry.find('('); + if (paren_pos == std::string::npos) return; + + std::string func_name = trim_lfm2_slice(trimmed_entry, 0, paren_pos); + std::string args_str = trim_lfm2_slice(trimmed_entry, paren_pos + 1, trimmed_entry.size()); + + while (!args_str.empty() && (args_str.back() == ')' || args_str.back() == ']')) { + args_str.pop_back(); + } + args_str = trim_lfm2_slice(args_str, 0, args_str.size()); + + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":{"; + + size_t arg_pos = 0; + bool first_arg = true; + while (arg_pos < args_str.length()) { + while (arg_pos < args_str.length() && std::isspace(static_cast(args_str[arg_pos]))) { + arg_pos++; + } + + size_t eq_pos = args_str.find('=', arg_pos); + if (eq_pos == std::string::npos) break; + + std::string arg_name = args_str.substr(arg_pos, eq_pos - arg_pos); + + size_t val_start = eq_pos + 1; + size_t val_end = val_start; + bool quoted = false; + + if (val_start < args_str.length() && args_str[val_start] == '"') { + quoted = true; + val_start++; + val_end = args_str.find('"', val_start); + if (val_end == std::string::npos) break; + } else { + int depth = 0; + for (val_end = val_start; val_end < args_str.length(); val_end++) { + char c = args_str[val_end]; + if (c == '[' || c == '{') depth++; + else if (c == ']' || c == '}') depth--; + else if (c == ',' && depth == 0) break; + } + } + + std::string arg_value = args_str.substr(val_start, val_end - val_start); + + if (!quoted) { + if (arg_value == "True") arg_value = "true"; + else if (arg_value == "False") arg_value = "false"; + else if (arg_value == "None") arg_value = "null"; + } + + if (!first_arg) json_call += ","; + json_call += "\"" + arg_name + "\":"; + json_call += quoted ? ("\"" + arg_value + "\"") : arg_value; + first_arg = false; + + arg_pos = args_str.find(',', val_end); + if (arg_pos != std::string::npos) { + arg_pos++; + } else { + break; + } + } + + json_call += "}}"; + function_calls.push_back(json_call); +} + +inline void parse_function_calls_from_response(const std::string& response_text, + std::string& regular_response, + std::vector& function_calls) { + regular_response = response_text; + function_calls.clear(); + + gemma::parse_function_calls(regular_response, function_calls); + + const std::string QWEN_TOOL_START = ""; + const std::string QWEN_TOOL_END = ""; + size_t qwen_start_pos = 0; + + while ((qwen_start_pos = regular_response.find(QWEN_TOOL_START, qwen_start_pos)) != std::string::npos) { + size_t content_start = qwen_start_pos + QWEN_TOOL_START.length(); + size_t qwen_end_pos = regular_response.find(QWEN_TOOL_END, content_start); + + size_t erase_end; + std::string json_content; + + if (qwen_end_pos != std::string::npos) { + json_content = regular_response.substr(content_start, qwen_end_pos - content_start); + erase_end = qwen_end_pos + QWEN_TOOL_END.length(); + } else { + json_content = regular_response.substr(content_start); + erase_end = regular_response.length(); + } + + size_t first = json_content.find_first_not_of(" \t\n\r"); + size_t last = json_content.find_last_not_of(" \t\n\r"); + if (first != std::string::npos && last != std::string::npos) { + json_content = json_content.substr(first, last - first + 1); + } + + if (json_content.size() > 2 && json_content.find("\"name\"") != std::string::npos) { + // Unwrap array wrapper if present: [{"name":...}] -> {"name":...} + if (json_content[0] == '[') { + size_t obj_start = json_content.find('{'); + size_t obj_end = json_content.rfind('}'); + if (obj_start != std::string::npos && obj_end != std::string::npos && obj_end > obj_start) { + json_content = json_content.substr(obj_start, obj_end - obj_start + 1); + } + } + if (json_content[0] == '{') { + size_t depth = 0; + bool in_string = false; + bool escaped = false; + size_t end_pos = 0; + for (size_t c = 0; c < json_content.size(); c++) { + char ch = json_content[c]; + if (escaped) { escaped = false; continue; } + if (ch == '\\' && in_string) { escaped = true; continue; } + if (ch == '"') { in_string = !in_string; continue; } + if (!in_string) { + if (ch == '{') depth++; + else if (ch == '}' && --depth == 0) { end_pos = c + 1; break; } + } + } + if (end_pos > 0) { + function_calls.push_back(json_content.substr(0, end_pos)); + } + } + } + + regular_response.erase(qwen_start_pos, erase_end - qwen_start_pos); + } + + const std::string TOOL_CALL_START = "<|tool_call_start|>"; + const std::string TOOL_CALL_END = "<|tool_call_end|>"; + size_t lfm2_start_pos = 0; + + while ((lfm2_start_pos = regular_response.find(TOOL_CALL_START, lfm2_start_pos)) != std::string::npos) { + size_t content_start = lfm2_start_pos + TOOL_CALL_START.length(); + size_t tool_end_pos = regular_response.find(TOOL_CALL_END, content_start); + + if (tool_end_pos != std::string::npos) { + std::string tool_content = regular_response.substr(content_start, tool_end_pos - content_start); + std::string content = tool_content; + size_t trim_start = 0; + while (trim_start < content.size() && std::isspace(static_cast(content[trim_start]))) { + trim_start++; + } + + if (trim_start < content.size()) { + size_t trim_end = content.size() - 1; + while (trim_end > trim_start && std::isspace(static_cast(content[trim_end]))) { + trim_end--; + } + content = content.substr(trim_start, trim_end - trim_start + 1); + } else { + content.clear(); + } + + if (!content.empty() && content.front() == '[' && content.back() == ']') { + std::string inner = content.substr(1, content.size() - 2); + + size_t inner_first = inner.find_first_not_of(" \t\n\r"); + if (inner_first != std::string::npos && inner[inner_first] == '{') { + size_t pos = inner_first; + while (pos < inner.size()) { + if (inner[pos] == '{') { + int brace_depth = 1; + size_t obj_start = pos; + pos++; + while (pos < inner.size() && brace_depth > 0) { + if (inner[pos] == '{') brace_depth++; + else if (inner[pos] == '}') brace_depth--; + pos++; + } + if (brace_depth == 0) { + std::string json_obj = inner.substr(obj_start, pos - obj_start); + if (json_obj.find("\"name\"") != std::string::npos) { + function_calls.push_back(json_obj); + } + } + } else { + pos++; + } + } + } else { + size_t start = 0; + int paren_depth = 0; + + for (size_t i = 0; i < inner.size(); ++i) { + char c = inner[i]; + if (c == '(') { + paren_depth++; + } else if (c == ')' && paren_depth > 0) { + paren_depth--; + } else if (c == ',' && paren_depth == 0) { + append_lfm2_call(inner.substr(start, i - start), function_calls); + start = i + 1; + } + } + + if (start < inner.size()) { + append_lfm2_call(inner.substr(start), function_calls); + } + } + } else if (!content.empty()) { + append_lfm2_call(content, function_calls); + } + + regular_response.erase(lfm2_start_pos, tool_end_pos + TOOL_CALL_END.length() - lfm2_start_pos); + } else { + break; + } + } + + const char* FUNCTION_CALL_MARKER = "\"function_call\""; + size_t search_pos = 0; + const size_t text_len = regular_response.length(); + + while (search_pos < text_len) { + size_t marker_pos = regular_response.find(FUNCTION_CALL_MARKER, search_pos); + if (marker_pos == std::string::npos) break; + + size_t json_start = regular_response.find('{', marker_pos); + if (json_start == std::string::npos) break; + + int brace_count = 1; + size_t json_end = json_start + 1; + while (json_end < text_len && brace_count > 0) { + char c = regular_response[json_end]; + brace_count += (c == '{') - (c == '}'); + json_end++; + } + + if (brace_count == 0) { + function_calls.push_back(regular_response.substr(json_start, json_end - json_start)); + regular_response = regular_response.substr(0, marker_pos); + size_t last_bracket = regular_response.rfind('{'); + if(last_bracket != std::string::npos) { + regular_response = regular_response.substr(0, last_bracket); + } + } + search_pos = json_end; + } +} + +inline std::vector> find_channel_token_ranges( + const std::vector& tokens, size_t offset, + uint32_t channel_open_id, uint32_t channel_close_id) { + std::vector> ranges; + size_t pos = 0; + while (pos < tokens.size()) { + if (tokens[pos] != channel_open_id) { + pos++; + continue; + } + + size_t block_start = pos; + pos++; + while (pos < tokens.size() && tokens[pos] != channel_close_id) { + pos++; + } + if (pos < tokens.size()) { + pos++; + } + ranges.push_back({offset + block_start, pos - block_start}); + } + return ranges; +} + +inline void strip_tag_blocks(std::string& text, std::string& extracted, + const std::string& open_tag, const std::string& close_tag) { + std::string result; + size_t pos = 0; + + size_t first_close = text.find(close_tag); + size_t first_open = text.find(open_tag); + if (first_close != std::string::npos && + (first_open == std::string::npos || first_close < first_open)) { + extracted += text.substr(0, first_close); + pos = first_close + close_tag.size(); + } + + while (pos < text.size()) { + size_t open_pos = text.find(open_tag, pos); + if (open_pos == std::string::npos) { + result += text.substr(pos); + break; + } + result += text.substr(pos, open_pos - pos); + size_t content_start = open_pos + open_tag.size(); + size_t close_pos = text.find(close_tag, content_start); + if (close_pos == std::string::npos) { + if (!extracted.empty()) extracted += "\n"; + extracted += text.substr(content_start); + break; + } + if (!extracted.empty()) extracted += "\n"; + extracted += text.substr(content_start, close_pos - content_start); + pos = close_pos + close_tag.size(); + } + text = result; +} + +inline void strip_thinking_block(const std::string& input, std::string& thinking, std::string& content) { + thinking.clear(); + content = input; + + auto trim = [](std::string& s) { + size_t first = s.find_first_not_of(" \t\n\r"); + size_t last = s.find_last_not_of(" \t\n\r"); + if (first != std::string::npos && last != std::string::npos) + s = s.substr(first, last - first + 1); + else + s.clear(); + }; + + if (content.find("<|channel>") != std::string::npos || content.find("") != std::string::npos) { + strip_tag_blocks(content, thinking, "<|channel>", ""); + } else if (content.find("") != std::string::npos || content.find("") != std::string::npos) { + strip_tag_blocks(content, thinking, "", ""); + } else { + return; + } + + trim(thinking); + trim(content); +} + +struct TranscriptSegment { + float start; + float end; + std::string text; +}; + +inline std::string construct_response_json(const std::string& regular_response, + const std::vector& function_calls, + double time_to_first_token, + double total_time_ms, + double prefill_tps, + double decode_tps, + size_t prompt_tokens, + size_t completion_tokens, + float confidence = 0.0f, + bool cloud_handoff = false, + const std::string& thinking = "", + const std::vector& segments = {}) { + std::ostringstream json; + json << "{"; + json << "\"success\":true,"; + json << "\"error\":null,"; + json << "\"cloud_handoff\":" << (cloud_handoff ? "true" : "false") << ","; + json << "\"response\":\"" << escape_json_string(regular_response) << "\","; + if (!thinking.empty()) { + json << "\"thinking\":\"" << escape_json_string(thinking) << "\","; + } + json << "\"function_calls\":["; + for (size_t i = 0; i < function_calls.size(); ++i) { + if (i > 0) json << ","; + json << function_calls[i]; + } + json << "],"; + json << "\"segments\":["; + for (size_t i = 0; i < segments.size(); ++i) { + if (i > 0) json << ","; + json << "{\"start\":" << std::fixed << std::setprecision(3) << segments[i].start + << ",\"end\":" << std::fixed << std::setprecision(3) << segments[i].end + << ",\"text\":\"" << escape_json_string(segments[i].text) << "\"}"; + } + json << "],"; + json << "\"confidence\":" << std::fixed << std::setprecision(4) << confidence << ","; + json << "\"time_to_first_token_ms\":" << std::fixed << std::setprecision(2) << time_to_first_token << ","; + json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; + json << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << ","; + json << "\"decode_tps\":" << std::fixed << std::setprecision(2) << decode_tps << ","; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; + json << "\"prefill_tokens\":" << prompt_tokens << ","; + json << "\"decode_tokens\":" << completion_tokens << ","; + json << "\"total_tokens\":" << (prompt_tokens + completion_tokens); + json << "}"; + return json.str(); +} + +inline std::string serialize_function_calls(const std::vector& calls) { + if (calls.empty()) return "[]"; + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < calls.size(); ++i) { + if (i > 0) oss << ","; + oss << calls[i]; + } + oss << "]"; + return oss.str(); +} + +inline int validate_audio_params( + const char* component, + void* model, + char* response_buffer, size_t buffer_size, + const char* audio_file_path, + const uint8_t* pcm_buffer, size_t pcm_buffer_size) { + if (!model) { + std::string err = last_error_message.empty() ? "Model not initialized." : last_error_message; + CACTUS_LOG_ERROR(component, err); + handle_error_response(err, response_buffer, buffer_size); + return -1; + } + if (!response_buffer || buffer_size == 0) { + CACTUS_LOG_ERROR(component, "Invalid parameters: response_buffer or buffer_size"); + handle_error_response("Invalid parameters", response_buffer, buffer_size); + return -1; + } + if (!audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { + CACTUS_LOG_ERROR(component, "No audio input provided"); + handle_error_response("Either audio_file_path or pcm_buffer must be provided", response_buffer, buffer_size); + return -1; + } + if (audio_file_path && pcm_buffer && pcm_buffer_size > 0) { + CACTUS_LOG_ERROR(component, "Both audio_file_path and pcm_buffer provided"); + handle_error_response("Cannot provide both audio_file_path and pcm_buffer", response_buffer, buffer_size); + return -1; + } + if (pcm_buffer && pcm_buffer_size > 0 && (pcm_buffer_size < 2 || pcm_buffer_size % 2 != 0)) { + CACTUS_LOG_ERROR(component, "Invalid pcm_buffer_size"); + handle_error_response("pcm_buffer_size must be even and at least 2 bytes", response_buffer, buffer_size); + return -1; + } + return 0; +} + +inline std::vector pcm_to_float(const uint8_t* pcm_buffer, size_t pcm_buffer_size) { + const int16_t* samples = reinterpret_cast(pcm_buffer); + size_t n = pcm_buffer_size / 2; + std::vector out(n); + for (size_t i = 0; i < n; ++i) + out[i] = static_cast(samples[i]) / 32768.0f; + return out; +} + +} // namespace ffi +} // namespace cactus + +#ifdef __cplusplus +extern "C" { +#endif + +const char* cactus_get_last_error(); + +#ifdef __cplusplus +} +#endif + +#endif // CACTUS_UTILS_H diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/engine.h b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/engine.h new file mode 100644 index 0000000..afebfeb --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/engine.h @@ -0,0 +1,1104 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../graph/graph.h" + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc99-extensions" +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +extern "C" { + #include "../../libs/stb/stb_image.h" + #include "../../libs/stb/stb_image_resize2.h" +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + +class CactusGraph; + +namespace cactus { +namespace npu { + class NPUPrefill; +} +namespace engine { + +class Siglip2Preprocessor; + +struct Config { + uint32_t vocab_size = 151936; + uint32_t bos_token_id = 151643; + uint32_t eos_token_id = 151645; + uint32_t num_layers = 28; + uint32_t hidden_dim = 1024; + uint32_t ffn_intermediate_dim = 3072; + uint32_t attention_heads = 16; + uint32_t attention_kv_heads = 8; + uint32_t attention_head_dim = 128; + float layer_norm_eps = 1e-6f; + float rope_theta = 1000000.0f; + uint32_t num_experts = 0; + uint32_t num_shared_experts = 0; + uint32_t num_top_experts = 0; + uint32_t moe_every_n_layers = 0; + uint32_t moe_intermediate_dim = 0; + uint32_t num_dense_layers = 0; + uint32_t num_experts_per_tok = 0; + bool norm_topk_prob = false; + bool use_expert_bias = false; + float routed_scaling_factor = 1.0f; + bool tie_word_embeddings = true; + + uint32_t vision_hidden_dim = 0; + uint32_t vision_num_layers = 0; + uint32_t vision_attention_heads = 0; + uint32_t vision_image_size = 0; + uint32_t vision_patch_size = 0; + uint32_t vision_num_channels = 3; + uint32_t vision_embed_dim = 0; + uint32_t visual_tokens_per_img = 0; + bool use_pixel_shuffle = false; + uint32_t pixel_shuffle_factor = 1; + bool use_image_tokens = false; + uint32_t image_token_id = 0; + bool use_layout_tags = false; + uint32_t image_seq_len = 64; + + uint32_t global_image_size = 2048; + uint32_t max_tile_size = 512; + float rescale_factor = 0.00392156862745098f; + float image_mean = 0.5f; + float image_std = 0.5f; + + uint32_t downsample_factor = 2; + uint32_t min_tiles = 2; + uint32_t max_tiles = 10; + bool use_thumbnail = true; + uint32_t min_image_tokens = 64; + uint32_t max_image_tokens = 256; + uint32_t max_num_patches = 1024; + uint32_t tile_size = 512; + float max_pixels_tolerance = 2.0f; + bool do_image_splitting = true; + bool encoder_act_gelu = false; + bool decoder_act_gelu = false; + uint32_t num_encoder_layers = 0; + uint32_t num_decoder_layers = 0; + float partial_rotary_factor = 0.0f; + uint32_t pad_token_id = 0; + uint32_t conv_kernel_size = 0; + uint32_t subsampling_conv_kernel_size = 0; + uint32_t subsampling_conv_stride = 0; + uint32_t subsampling_conv_channels = 0; + uint32_t subsampling_factor = 0; + uint32_t num_mel_bins = 80; + std::string encoder_hidden_act = "silu"; + uint32_t linear_num_key_heads = 0; + uint32_t linear_key_head_dim = 0; + uint32_t linear_num_value_heads = 0; + uint32_t linear_value_head_dim = 0; + uint32_t linear_q_proj_dim = 0; + uint32_t linear_k_proj_dim = 0; + uint32_t linear_v_proj_dim = 0; + + uint32_t kv_lora_rank = 0; + uint32_t q_lora_rank = 0; + uint32_t qk_head_dim = 0; + uint32_t qk_nope_head_dim = 0; + uint32_t qk_rope_head_dim = 0; + uint32_t v_head_dim = 0; + uint32_t rope_interleave = 0; + bool attention_bias = false; + float rope_scaling_factor = 1.0f; + float rope_mscale_all_dim = 0.0f; + + enum class ModelType {QWEN = 0, GEMMA = 1, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7, MOONSHINE = 8, SILERO_VAD = 9, PARAKEET = 10, QWEN3P5 = 11, PARAKEET_TDT = 12, GEMMA3N = 13, YOUTU = 14, GEMMA4 = 15, PYANNOTE = 16, WESPEAKER = 17, NEEDLE = 18}; + uint32_t predictor_hidden_dim = 0; + uint32_t predictor_num_layers = 0; + uint32_t tdt_joint_dim = 0; + uint32_t tdt_num_durations = 0; + uint32_t tdt_blank_id = 0; + std::vector tdt_durations; + + ModelType model_type = ModelType::QWEN; + + enum class ModelVariant {DEFAULT = 0, VLM = 1, EXTRACT = 2, RAG = 3}; + ModelVariant model_variant = ModelVariant::DEFAULT; + + enum class Activation {GELU = 0, SILU = 1}; + Activation activation = Activation::SILU; + + enum class Backend {CPU = 0, NPU = 1}; + Backend default_backend = Backend::CPU; + + enum class Precision {INT8 = 0, FP16 = 1, FP32 = 2}; + Precision precision = Precision::FP32; + + float default_temperature = 0.6f; + float default_top_p = 0.95f; + size_t default_top_k = 20; + float default_max_tps = -1.0f; + float default_cloud_handoff_threshold = 0.0f; + size_t default_rolling_entropy_window = 10; + + std::vector layer_types; + size_t conv_L_cache = 0; + + uint32_t altup_num_inputs = 4; + uint32_t laurel_rank = 64; + uint32_t hidden_size_per_layer_input = 256; + uint32_t num_kv_shared_layers = 0; + uint32_t sliding_window = 512; + float rope_local_base_freq = 10000.0f; + float final_logit_softcapping = 0.0f; + float global_partial_rotary_factor = 1.0f; + uint32_t expert_intermediate_size = 0; + uint32_t global_head_dim = 0; + uint32_t num_global_kv_heads = 0; + bool attention_k_eq_v = false; + bool enable_moe_block = false; + std::vector activation_sparsity_ppf; + + uint32_t vision_head_dim = 64; + uint32_t vision_kv_heads = 12; + uint32_t vision_intermediate_size = 3072; + uint32_t vision_position_embedding_size = 10240; + uint32_t vision_pooling_kernel_size = 3; + uint32_t vision_default_output_length = 280; + float vision_rope_theta = 100.0f; + + uint32_t audio_hidden_dim = 0; + uint32_t audio_num_layers = 0; + uint32_t audio_num_heads = 0; + uint32_t audio_head_dim = 0; + uint32_t audio_input_feat_size = 128; + uint32_t audio_conf_conv_kernel_size = 5; + uint32_t audio_chunk_size = 12; + uint32_t audio_context_left = 13; + uint32_t audio_context_right = 0; + float audio_logit_cap = 50.0f; + float audio_residual_weight = 0.5f; + uint32_t audio_output_proj_dims = 0; + uint32_t audio_vocab_size = 128; + uint32_t audio_vocab_offset = 0; + uint32_t audio_soft_tokens = 188; + uint32_t audio_sscp_conv0_channels = 128; + uint32_t audio_sscp_conv1_channels = 32; + float audio_sscp_conv_eps = 1e-3f; + float audio_rms_norm_eps = 1e-6f; + uint32_t audio_fft_length = 1024; + uint32_t audio_token_id = 0; + bool audio_fft_overdrive = false; + uint32_t channel_open_token_id = 100; + uint32_t channel_close_token_id = 101; + + static bool is_gemma_family(ModelType t) { + return t == ModelType::GEMMA || t == ModelType::GEMMA3N || t == ModelType::GEMMA4; + } + + bool from_json(const std::string& json_path); + std::string to_json() const; +}; + + + +struct MergeRule { + std::string first; + std::string second; + std::string merged; + uint32_t priority; + + MergeRule(const std::string& f, const std::string& s, const std::string& m, uint32_t p) + : first(f), second(s), merged(m), priority(p) {} +}; + + +struct ToolCallInfo { + std::string name; + std::string arguments; +}; + +struct ChatMessage { + std::string role; + std::string content; + std::string name; + std::vector images; + std::vector audio; + size_t audio_soft_token_count = 0; + std::vector tool_calls; +}; + +inline std::string format_needle_query_text(const std::vector& messages) { + std::string system_text; + std::string user_query; + + for (const auto& msg : messages) { + if (msg.role == "system") { + if (!system_text.empty()) { + system_text += "\n"; + } + system_text += msg.content; + } else if (msg.role == "user") { + user_query = msg.content; + } + } + + if (user_query.empty() && !messages.empty()) { + user_query = messages.back().content; + } + if (system_text.empty()) { + return user_query; + } + if (user_query.empty()) { + return system_text; + } + return system_text + "\n\n" + user_query; +} + +struct ToolConstraintSpec { + std::string name; + std::vector parameter_names; + std::vector required_parameter_names; +}; + +struct TokenizerRuntimeConfig { + enum class TokenizerType { UNKNOWN, BPE, SENTENCEPIECE }; + enum class VocabFormat { UNKNOWN, ID_TAB_TOKEN, LINE_TOKEN }; + enum class Normalizer { NONE, METASPACE, BYTE_LEVEL }; + enum class Decoder { NONE, REPLACE_METASPACE, BYTE_LEVEL }; + + TokenizerType tokenizer_type = TokenizerType::UNKNOWN; + VocabFormat vocab_format = VocabFormat::UNKNOWN; + Normalizer normalizer = Normalizer::NONE; + Decoder decoder = Decoder::NONE; + bool byte_fallback = false; + bool has_chat_template = false; +}; + +TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file); +void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens); +std::vector split_with_special_tokens(const std::string& text, const std::unordered_map& special_tokens); + +inline std::string extract_json_string(const std::string& json, size_t& pos) { + std::string value; + while (pos < json.size() && json[pos] != '"') { + if (json[pos] == '\\' && pos + 1 < json.size()) { + pos++; + if (json[pos] == 'n') value += '\n'; + else if (json[pos] == 't') value += '\t'; + else if (json[pos] == 'r') value += '\r'; + else if (json[pos] == '"') value += '"'; + else if (json[pos] == '\\') value += '\\'; + else value += json[pos]; + } else { + value += json[pos]; + } + pos++; + } + if (pos < json.size()) pos++; + return value; +} + +class Tokenizer { +public: + virtual ~Tokenizer() = default; + + virtual std::vector encode(const std::string& text) const = 0; + virtual std::string decode(const std::vector& tokens) const = 0; + + virtual std::vector apply_chat_template(const std::vector& messages, bool add_generation_prompt = true) const; + virtual std::string format_chat_prompt(const std::vector& messages, bool add_generation_prompt = true, const std::string& tools_json = "", bool enable_thinking_if_supported = false) const; + + virtual uint32_t get_vocab_size() const = 0; + virtual uint32_t get_unk_token() const = 0; + virtual uint32_t get_bos_token() const = 0; + virtual uint32_t get_eos_token() const = 0; + virtual bool has_chat_template() const { return has_chat_template_; } + std::string get_default_stop_sequence() const; + + virtual bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) = 0; + + uint32_t get_image_token_id() const { return image_token_id_; } + uint32_t get_fake_token_id() const { return fake_token_id_; } + uint32_t get_global_img_token_id() const { return global_img_token_id_; } + +protected: + enum class ModelType { UNKNOWN, QWEN, QWEN3P5, GEMMA, GEMMA4, LFM2, BERT, WHISPER, PARAKEET, YOUTU, NEEDLE}; + ModelType model_type_ = ModelType::UNKNOWN; + enum class ModelVariant { DEFAULT, VLM, EXTRACT, RAG}; + ModelVariant model_variant_ = ModelVariant::DEFAULT; + bool has_chat_template_ = false; + std::string chat_template_; + + uint32_t image_token_id_ = 396; + uint32_t fake_token_id_ = 49189; + uint32_t global_img_token_id_ = 49152; + + + uint32_t vision_patch_size_ = 16; + uint32_t vision_pooling_kernel_size_ = 3; + uint32_t vision_default_output_length_ = 280; + uint32_t vision_image_size_ = 768; + TokenizerRuntimeConfig runtime_config_; + + void detect_model_type(const std::string& config_path); + void load_chat_template(const std::string& template_file); + std::string format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; + std::string format_gemma_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_gemma4_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; + std::string format_lfm2_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_lfm2_vl_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_needle_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_youtu_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; +}; + +class BPETokenizer : public Tokenizer { +public: + BPETokenizer(); + ~BPETokenizer(); + + bool load_vocabulary_mmap(const std::string& vocab_file, const std::string& merges_file); + bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; + + std::vector encode(const std::string& text) const override; + std::string decode(const std::vector& tokens) const override; + + uint32_t get_vocab_size() const override { return vocab_size_; } + uint32_t get_unk_token() const override { return unk_token_id_; } + uint32_t get_bos_token() const override { return bos_token_id_; } + uint32_t get_eos_token() const override { return eos_token_id_; } + +private: + std::unordered_map token_to_id_; + std::vector id_to_token_; + std::vector merge_rules_; + std::unordered_map merge_map_; + + uint32_t vocab_size_; + uint32_t unk_token_id_; + uint32_t bos_token_id_; + uint32_t eos_token_id_; + + void* vocab_mmap_ptr_; + size_t vocab_mmap_size_; + + void* merges_mmap_ptr_; + size_t merges_mmap_size_; + + std::vector apply_bpe(const std::vector& tokens) const; + std::pair find_best_merge_fast(const std::vector& tokens) const; + + std::string bytes_to_unicode(const std::string& text) const; + std::string unicode_to_bytes(const std::string& text) const; + std::vector byte_level_split(const std::string& text) const; + std::vector utf8_split(const std::string& text) const; + + void cleanup_mmap(); + +private: + mutable std::unordered_map byte_to_unicode_; + mutable std::unordered_map unicode_to_byte_; + void init_byte_mappings() const; + + std::unordered_map special_tokens_; + std::vector split_with_special_tokens(const std::string& text) const; + void load_special_tokens(const std::string& config_file); +}; + +class SPTokenizer : public Tokenizer { +public: + SPTokenizer(); + ~SPTokenizer(); + + bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; + + std::vector encode(const std::string& text) const override; + std::string decode(const std::vector& tokens) const override; + + uint32_t get_vocab_size() const override { return vocab_size_; } + uint32_t get_unk_token() const override { return unk_token_id_; } + uint32_t get_bos_token() const override { return bos_token_id_; } + uint32_t get_eos_token() const override { return eos_token_id_; } + +private: + struct TrieNode { + std::unordered_map> children; + int32_t token_id = -1; + float score = 0.0f; + }; + + std::unique_ptr trie_root_; + std::unordered_map token_to_id_; + std::vector id_to_token_; + std::vector token_scores_; + + uint32_t vocab_size_; + uint32_t unk_token_id_; + uint32_t bos_token_id_; + uint32_t eos_token_id_; + uint32_t pad_token_id_; + + bool sp_bpe_mode_ = false; + bool sp_add_dummy_prefix_ = false; + bool sp_byte_fallback_ = false; + + void* vocab_mmap_ptr_; + size_t vocab_mmap_size_; + + void build_trie(); + std::vector> tokenize_with_trie(const std::string& text) const; + std::vector tokenize_with_bpe(const std::string& text) const; + std::string preprocess_text(const std::string& text) const; + std::string postprocess_text(const std::string& text) const; + std::vector split_by_unicode_spaces(const std::string& text) const; + + void cleanup_mmap(); + + std::unordered_map special_tokens_; + std::vector split_with_special_tokens(const std::string& text) const; + void load_special_tokens(const std::string& config_file); +}; + +class ConvCache { +public: + struct CircularView { + const void* ptr1; + size_t len1; + const void* ptr2; + size_t len2; + size_t total_len; + }; + + void init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision); + CircularView get_window(size_t layer) const; + void update(CactusGraph* gb, size_t layer, const size_t latest_token); + void reset(); + + bool is_empty() const { return num_layers == 0; } + + size_t num_layers = 0; + size_t hidden_size = 0; + size_t window_size = 0; + Precision precision = Precision::FP32; + size_t element_size = 4; + +private: + struct LayerState { + std::vector data; + size_t head = 0; + size_t count = 0; + }; + + std::vector layer_states; +}; + +struct KVCache { + static constexpr size_t DEFAULT_WINDOW_SIZE = 1024; + static constexpr size_t DEFAULT_SINK_SIZE = 4; + + struct LayerCache { + std::vector keys; + std::vector values; + std::vector key_scales; + std::vector value_scales; + size_t head_dim = 0; + size_t kv_heads = 0; + }; + + std::vector layer_caches; + + size_t window_size = DEFAULT_WINDOW_SIZE; + size_t sink_size = DEFAULT_SINK_SIZE; + size_t current_seq_len = 0; + size_t total_seq_len = 0; + size_t max_seq_len = 2048; + size_t num_layers = 0; + Precision precision; + size_t element_size = 4; + + void set_window_size(size_t window, size_t sink = DEFAULT_SINK_SIZE); + size_t get_effective_seq_len() const { return current_seq_len; } + size_t get_total_seq_len() const { return total_seq_len; } + size_t get_layer_head_dim(size_t layer_idx) const { return layer_caches[layer_idx].head_dim; } + size_t get_layer_kv_heads(size_t layer_idx) const { return layer_caches[layer_idx].kv_heads; } + + void init(size_t num_layers, size_t max_seq, const std::vector& layer_dims, const std::vector& layer_kv_heads, Precision model_precision); + void reset(); + void update_from_graph(CactusGraph* gb, const std::vector& k_nodes, + const std::vector& v_nodes, size_t seq_len, + size_t num_layers); + + void update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data, + size_t num_tokens, size_t kv_heads, size_t head_dim); + + bool is_empty() const { return current_seq_len == 0; } + void* get_key_ptr(size_t layer); + void* get_value_ptr(size_t layer); + + struct CircularView { + const void* ptr1; + const void* ptr2; + size_t len1; + size_t len2; + size_t total_len; + }; + + CircularView get_key_view(size_t layer); + CircularView get_value_view(size_t layer); + + const int8_t* get_keys_int8(size_t layer) const; + const int8_t* get_values_int8(size_t layer) const; + const float* get_key_scales(size_t layer) const; + const float* get_value_scales(size_t layer) const; + + void remove_token_range(size_t start, size_t count); + void compact_to_windows(const std::vector& target_windows); +}; + +class ToolCallConstrainer { +public: + enum class State { + DONE, + + QWEN_START, + QWEN_EXPECT_OPEN_BRACE, + QWEN_EXPECT_NAME_KEY, + QWEN_EXPECT_NAME_COLON, + QWEN_EXPECT_NAME_VALUE, + QWEN_EXPECT_COMMA, + QWEN_EXPECT_ARGS_KEY, + QWEN_EXPECT_ARGS_COLON, + QWEN_IN_ARGUMENTS, + QWEN_EXPECT_CLOSE_BRACE, + QWEN_EXPECT_END, + + NEEDLE_START, + + LFM_START, + LFM_EXPECT_BRACKET, + LFM_IN_FUNC_NAME, + LFM_EXPECT_PAREN, + LFM_IN_ARGUMENTS, + LFM_EXPECT_BRACKET_CLOSE, + LFM_EXPECT_END, + + GEMMA_START, + GEMMA_EXPECT_CALL, + GEMMA_IN_FUNC_NAME, + GEMMA_EXPECT_BRACE, + GEMMA_IN_ARGUMENTS, + GEMMA_EXPECT_END + }; + + void init(Config::ModelType model_type, + const std::vector& tools, + Tokenizer* tokenizer); + + const std::unordered_map& get_bias() const { return current_bias_; } + + void update(uint32_t token_id, const std::string& decoded_text); + + void reset(); + + bool is_active() const { return active_; } + +private: + bool active_ = false; + State state_ = State::QWEN_START; + Config::ModelType model_type_ = Config::ModelType::QWEN; + Tokenizer* tokenizer_ = nullptr; + + bool is_gemma_family() const { return Config::is_gemma_family(model_type_); } + bool is_needle() const { return model_type_ == Config::ModelType::NEEDLE; } + + enum class NeedleJsonState { + FREE, + IN_NAME, + IN_ARG_KEY, + }; + + struct NeedleTrieNode { + std::unordered_map> children; + bool is_terminal = false; + }; + + std::vector tool_specs_; + std::vector function_names_; + std::string generated_text_; + int brace_depth_ = 0; + + std::string call_start_tag_; + std::string call_end_tag_; + + std::unordered_set qwen_tool_call_start_tokens_; + std::unordered_set qwen_tool_call_end_tokens_; + std::unordered_set open_brace_tokens_; + std::unordered_set close_brace_tokens_; + std::unordered_set colon_tokens_; + std::unordered_set comma_tokens_; + std::unordered_set name_key_tokens_; + std::unordered_set args_key_tokens_; + std::unordered_set quote_tokens_; + std::unordered_set backtick_tokens_; + std::unordered_set all_func_name_tokens_; + std::unordered_map> func_name_sequences_; + NeedleJsonState needle_json_state_ = NeedleJsonState::FREE; + std::string needle_buffer_; + std::string needle_constrained_buf_; + std::string needle_current_function_; + bool needle_in_arguments_ = false; + int needle_arguments_depth_ = 0; + int needle_nesting_depth_ = 0; + bool needle_in_string_value_ = false; + bool needle_prev_char_escape_ = false; + std::unique_ptr needle_name_trie_; + std::unordered_map> needle_param_tries_; + std::vector needle_token_strings_; + std::unordered_map> needle_token_index_; + + std::unordered_set tool_start_tokens_; + std::unordered_set tool_end_tokens_; + std::unordered_set bracket_open_tokens_; + std::unordered_set bracket_close_tokens_; + std::unordered_set paren_open_tokens_; + std::unordered_set paren_close_tokens_; + std::unordered_set equals_tokens_; + + std::string lfm_current_function_; + std::string lfm_args_buffer_; + std::unordered_set lfm_seen_arg_keys_; + std::unordered_map> lfm_required_params_; + std::unordered_map> lfm_all_params_; + + std::unordered_set gemma_call_start_tokens_; + std::unordered_set gemma_call_end_tokens_; + std::unordered_set gemma_response_start_tokens_; + std::unordered_set gemma_call_prefix_tokens_; + std::unordered_set escape_tokens_; + + std::unordered_map current_bias_; + + void compute_bias(); + void tokenize_grammar_elements(); + void add_tokens_for_string(const std::string& str, std::unordered_set& token_set); + void add_tokens_for_prefix_string(const std::string& prefix, std::unordered_set& token_set); + void add_tokens_containing(char needle, std::unordered_set& token_set); + void tokenize_function_names(bool quote_names); + void init_common_tokens(); + void init_needle_constraints(); + void reset_needle_constraints(); + void feed_needle_text(const std::string& text); + void feed_needle_char(char ch); + bool needle_at_arg_key_start() const; + bool needle_is_value_string_start() const; + void needle_insert_word(NeedleTrieNode* root, const std::string& word); + const NeedleTrieNode* needle_get_trie_node(const NeedleTrieNode* root, const std::string& prefix) const; + bool needle_check_token_valid(const std::string& token_text, const NeedleTrieNode* trie_node) const; +}; + +class Model { +public: + struct DebugNode { + uint32_t layer_idx; + std::string name; + size_t node_id; + }; + + Model(); + explicit Model(const Config& config); + virtual ~Model(); + + const Config& get_config() const { return config_; } + Tokenizer* get_tokenizer() const { return tokenizer_.get(); } + const std::vector& get_debug_nodes() const; + + virtual bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true); + + virtual bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, + const std::string& system_prompt = "", bool do_warmup = true); + + virtual uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f); + + virtual void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = ""); + + virtual void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, + const std::string& profile_file = ""); + + virtual uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, + float temperature = -1.0f, float top_p = -1.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f); + + virtual uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, float temperature = 0.0f, float top_p = 0.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f, + float* out_token_time_start = nullptr, float* out_token_time_end = nullptr); + + std::vector get_embeddings(const std::vector& tokens, bool pooled = true, bool normalize = false, const std::string& profile_file = ""); + + virtual std::vector get_image_embeddings(const std::string& image_path); + + virtual std::vector get_audio_embeddings(const std::vector& audio_features); + + virtual void reset_cache() { kv_cache_.reset(); token_history_.clear(); } + void record_sampled_token(uint32_t token) { + if (token_history_.size() >= MAX_TOKEN_HISTORY) { + token_history_.erase(token_history_.begin(), token_history_.begin() + (MAX_TOKEN_HISTORY / 2)); + } + token_history_.push_back(token); + } + + double score_tokens_window_logprob(const std::vector& tokens, size_t start, size_t end, size_t context, size_t* tokens_scored); + + + + void set_cache_window(size_t window_size, size_t sink_size = 4) { kv_cache_.set_window_size(window_size, sink_size); } + size_t get_cache_size() const { return kv_cache_.current_seq_len; } + + bool load_npu_prefill(const std::string& model_path); + bool has_npu_prefill() const; + size_t get_prefill_chunk_size() const; + + virtual void remove_thinking_tokens(const std::vector>& ranges); + virtual void compact_kv_cache() {} + + virtual void set_tool_constraints(const std::vector& tools); + virtual void clear_tool_constraints(); + virtual void update_tool_constraints(uint32_t token_id); + + void* graph_handle_; + + void set_vocab_bias(const std::unordered_map& bias) { + vocab_bias_ = bias; + } + + void clear_vocab_bias() { + vocab_bias_.clear(); + } + + bool has_vocab_bias() const { + return !vocab_bias_.empty(); + } + + const std::unordered_map& get_vocab_bias() const { + return vocab_bias_; + } + +protected: + size_t sample_token(CactusGraph* gb, size_t logits_node_id, float temperature, float top_p, size_t top_k, + float min_p, float repetition_penalty, + const std::unordered_map* extra_bias = nullptr) const; + + static void compute_entropy(CactusGraph* gb, size_t logits_node_id, float* out_entropy); + + virtual size_t forward(const std::vector& tokens, bool use_cache = false) = 0; + + virtual size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false); + + virtual void load_weights_to_graph(CactusGraph* gb) = 0; + + virtual size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; + + virtual size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const = 0; + virtual size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; + void update_kv_cache(CactusGraph* gb, size_t seq_len); + virtual std::vector get_kv_layer_dims() const { + return std::vector(config_.num_layers, config_.attention_head_dim); + } + virtual std::vector get_kv_layer_heads() const { + return std::vector(config_.num_layers, config_.attention_kv_heads); + } + virtual void post_init() {} + virtual void post_execute_updates(CactusGraph*, size_t) {} + Config config_; + std::unique_ptr tokenizer_; + + bool initialized_; + float attention_scale_; + +protected: + KVCache kv_cache_; + std::vector cache_k_output_nodes_; + std::vector cache_v_output_nodes_; + + std::string embedding_file_path_; + size_t embedding_node_id_; + std::string model_folder_path_; + size_t output_weight_node_id_; + + mutable std::vector debug_nodes_; + + void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) const; + void clear_debug_nodes(); + + bool init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup); + bool owns_graph_; + + std::unique_ptr npu_prefill_; + void prefill_npu(const std::vector& tokens); + virtual std::vector<__fp16> get_token_embeddings(const std::vector& tokens); + + static constexpr size_t MAX_TOKEN_HISTORY = 128; + ToolCallConstrainer tool_constrainer_; + std::vector token_history_; + +private: + std::unordered_map vocab_bias_; +}; + +std::unique_ptr create_model(const std::string& model_folder); + +class Siglip2Preprocessor { +public: + struct Config { + int patch_size = 16; + int downsample_factor = 2; + int min_tiles = 2; + int max_tiles = 10; + bool use_thumbnail = true; + int min_image_tokens = 64; + int max_image_tokens = 256; + int max_num_patches = 1024; + int tile_size = 512; + float max_pixels_tolerance = 2.0f; + bool do_resize = true; + bool do_rescale = true; + bool do_normalize = true; + bool do_convert_rgb = true; + bool do_image_splitting = true; + float rescale_factor = 1.0f / 255.0f; + float image_mean[3] = {0.5f, 0.5f, 0.5f}; + float image_std[3] = {0.5f, 0.5f, 0.5f}; + }; + + struct PreprocessedImage { + std::vector pixel_values; + std::vector pixel_attention_mask; + std::vector> spatial_shapes; + std::vector pixel_values_shape; + std::vector pixel_attention_mask_shape; + std::vector spatial_shapes_shape; + int num_patches_height; + int num_patches_width; + int actual_num_patches; + int num_tiles; + int patch_dim; + int max_patches_per_tile; + + int image_rows; + int image_cols; + int image_height; + int image_width; + int tokens_per_tile; + int thumbnail_tokens; + + ~PreprocessedImage(); + }; + + struct SpatialShapeResult { + std::vector> shapes; + int grid_rows; + int grid_cols; + }; + + explicit Siglip2Preprocessor(const Config& config); + Siglip2Preprocessor(); + ~Siglip2Preprocessor(); + + PreprocessedImage preprocess_from_file(const std::string& image_path); + PreprocessedImage preprocess_from_memory(const unsigned char* img_data, int width, int height, int channels); + SpatialShapeResult compute_spatial_shapes(int height, int width); + +private: + Config config_; + + std::pair compute_pixel_limits() const; + std::vector convert_to_rgb(const unsigned char* img_data, int width, int height, int channels); + std::pair smart_resize(int height, int width); + bool is_image_too_large(int height, int width); + std::pair get_grid_layout(int height, int width); + std::pair find_closest_aspect_ratio(float aspect_ratio, int width, int height); + std::vector resize_image(const unsigned char* img_data, int src_width, int src_height, + int dst_width, int dst_height, int channels); + std::vector normalize_image(const float* img_data, int width, int height, int channels); + std::vector> convert_image_to_patches( + const std::vector& image, int width, int height, int channels, int patch_size); + PreprocessedImage pad_patches(const std::vector>& tile_patches, + const std::vector>& spatial_shapes, + int patch_dim, + int max_patches_per_tile); + int round_by_factor(int number, int factor); +}; + +class AudioProcessor { +public: + struct SpectrogramConfig { + size_t n_fft = 400; + size_t hop_length = 160; + size_t frame_length = 400; + float power = 2.0f; + bool center = true; + const char* pad_mode = "reflect"; + bool onesided = true; + float dither = 0.0f; + float mel_floor = 1e-10f; + const char* log_mel = nullptr; + float reference = 1.0f; + float min_value = 1e-10f; + bool remove_dc_offset = false; + float preemphasis = 0.0f; + bool hann_periodic = true; + float window_a0 = 0.5f; + size_t fft_override = 0; + bool mel_floor_additive = false; + }; + + AudioProcessor(); + ~AudioProcessor(); + + void init_mel_filters(size_t num_frequency_bins, size_t num_mel_filters, + float min_freq, float max_freq, size_t sampling_rate, + const char* norm = "slaney", const char* mel_scale = "slaney"); + + std::vector compute_spectrogram( + const std::vector& waveform, + const SpectrogramConfig& config); + + static std::vector compute_irfft( + const std::vector& complex_input, + size_t n, + const char* norm = "backward"); + + const std::vector& get_mel_filters() const { return mel_filters_; } + + size_t get_num_mel_filters() const { return num_mel_filters_; } + size_t get_num_frequency_bins() const { return num_frequency_bins_; } + +private: + std::vector mel_filters_; + size_t num_frequency_bins_; + size_t num_mel_filters_; +}; + +namespace index { + constexpr uint32_t MAGIC = 0x43414354; + constexpr uint32_t VERSION = 1; + + struct Document { + int id; + std::vector embedding; + std::string content; + std::string metadata; + }; + + struct QueryResult { + int doc_id; + float score; + + QueryResult(int doc_id, float score) : doc_id(doc_id), score(score) {} + }; + + struct QueryOptions { + size_t top_k = 10; + float score_threshold = -1.0f; + }; + + class Index { + public: + Index(const std::string& index_path, const std::string& data_path, size_t embedding_dim); + ~Index(); + + Index(const Index&) = delete; + Index& operator=(const Index&) = delete; + Index(Index&&) = delete; + Index& operator=(Index&&) = delete; + + void add_documents(const std::vector& documents); + void delete_documents(const std::vector& doc_ids); + std::vector get_documents(const std::vector& doc_ids); + std::vector> query(const std::vector>& embeddings, const QueryOptions& options); + void compact(); + + private: + struct IndexHeader { + uint32_t magic; + uint32_t version; + uint32_t embedding_dim; + uint32_t num_documents; + }; + + struct IndexEntry { + int32_t doc_id; + uint64_t data_offset; + uint8_t flags; // bit 0: tombstone + + const __fp16* embedding() const { + return reinterpret_cast(this + 1); + } + + static size_t size(size_t embedding_dim) { + return sizeof(IndexEntry) + embedding_dim * sizeof(__fp16); + } + }; + + struct DataHeader { + uint32_t magic; + uint32_t version; + }; + + struct DataEntry { + uint16_t content_len; + uint16_t metadata_len; + + const char* content() const { + return reinterpret_cast(this + 1); + } + + const char* metadata() const { + return content() + content_len; + } + }; + + void parse_index_header(); + void parse_data_header(); + void build_doc_id_map(); + void validate_documents(const std::vector& documents); + void validate_doc_ids(const std::vector& doc_ids); + ssize_t write_full(int fd, const void* buf, size_t count); + + std::unordered_map doc_id_map_; + + std::string index_path_, data_path_; + size_t embedding_dim_; + size_t index_entry_size_; + uint32_t num_documents_; + + int index_fd_, data_fd_; + void *mapped_index_, *mapped_data_; + size_t index_file_size_, data_file_size_; + }; +} // namespace index + +} +} diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/gemma_tools.h b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/gemma_tools.h new file mode 100644 index 0000000..f0f9fe2 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/gemma_tools.h @@ -0,0 +1,576 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace gemma { + +inline std::string to_upper(const std::string& s) { + std::string result = s; + for (auto& c : result) c = std::toupper(c); + return result; +} + +inline std::string escape(const std::string& s) { + return "" + s + ""; +} + +inline void skip_whitespace(const std::string& json, size_t& pos) { + while (pos < json.length() && std::isspace(json[pos])) pos++; +} + +inline std::string extract_json_string(const std::string& json, size_t& pos) { + std::string value; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\' && pos + 1 < json.length()) { + pos++; + if (json[pos] == 'n') value += '\n'; + else if (json[pos] == 't') value += '\t'; + else if (json[pos] == 'r') value += '\r'; + else if (json[pos] == '"') value += '"'; + else if (json[pos] == '\\') value += '\\'; + else value += json[pos]; + } else { + value += json[pos]; + } + pos++; + } + if (pos < json.length()) pos++; + return value; +} + +std::string format_argument(const std::string& json, size_t& pos, bool escape_keys); +std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/); + +inline std::string format_argument(const std::string& json, size_t& pos, bool escape_keys = true) { + skip_whitespace(json, pos); + if (pos >= json.length()) return ""; + + char c = json[pos]; + + if (c == '"') { + pos++; + std::string value = extract_json_string(json, pos); + return escape(value); + } else if (c == '{') { + std::string result = "{"; + pos++; + bool first = true; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == '}') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + if (json[pos] != '"') break; + pos++; + std::string key = extract_json_string(json, pos); + + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == ':') pos++; + + std::string value = format_argument(json, pos, escape_keys); + + if (!first) result += ","; + first = false; + if (escape_keys) { + result += escape(key) + ":" + value; + } else { + result += key + ":" + value; + } + } + result += "}"; + return result; + } else if (c == '[') { + std::string result = "["; + pos++; + bool first = true; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == ']') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + std::string value = format_argument(json, pos, escape_keys); + + if (!first) result += ","; + first = false; + result += value; + } + result += "]"; + return result; + } else if (json.compare(pos, 4, "true") == 0) { + pos += 4; + return "true"; + } else if (json.compare(pos, 5, "false") == 0) { + pos += 5; + return "false"; + } else if (json.compare(pos, 4, "null") == 0) { + pos += 4; + return "null"; + } else { + size_t start = pos; + while (pos < json.length() && (std::isdigit(json[pos]) || json[pos] == '.' || + json[pos] == '-' || json[pos] == '+' || json[pos] == 'e' || json[pos] == 'E')) { + pos++; + } + return json.substr(start, pos - start); + } +} + +inline std::map parse_json_object_raw(const std::string& json, size_t& pos) { + std::map result; + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] != '{') return result; + pos++; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == '}') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + if (json[pos] != '"') break; + pos++; + std::string key = extract_json_string(json, pos); + + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == ':') pos++; + skip_whitespace(json, pos); + + size_t value_start = pos; + if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + pos++; + } else if (json[pos] == '{') { + int depth = 1; + pos++; + while (pos < json.length() && depth > 0) { + if (json[pos] == '{') depth++; + else if (json[pos] == '}') depth--; + else if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + } else if (json[pos] == '[') { + int depth = 1; + pos++; + while (pos < json.length() && depth > 0) { + if (json[pos] == '[') depth++; + else if (json[pos] == ']') depth--; + else if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + } else { + while (pos < json.length() && json[pos] != ',' && json[pos] != '}') pos++; + } + result[key] = json.substr(value_start, pos - value_start); + } + return result; +} + +inline std::string get_json_string_value(const std::string& json, size_t pos) { + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == '"') { + pos++; + return extract_json_string(json, pos); + } + return ""; +} + +inline std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/) { + static const std::set standard_keys = {"description", "type", "properties", "required", "nullable"}; + + size_t pos = 0; + auto properties = parse_json_object_raw(properties_json, pos); + + std::string result; + bool first = true; + + for (const auto& [key, value_json] : properties) { + if (standard_keys.count(key)) continue; + + if (!first) result += ","; + first = false; + + size_t prop_pos = 0; + auto prop_obj = parse_json_object_raw(value_json, prop_pos); + + result += key + ":{"; + + if (prop_obj.count("description")) { + std::string desc = get_json_string_value(prop_obj["description"], 0); + result += "description:" + escape(desc); + } + + std::string type_val; + if (prop_obj.count("type")) { + type_val = get_json_string_value(prop_obj["type"], 0); + } + + if (to_upper(type_val) == "STRING") { + if (prop_obj.count("enum")) { + size_t enum_pos = 0; + std::string enum_formatted = format_argument(prop_obj["enum"], enum_pos, true); + result += ",enum:" + enum_formatted; + } + } else if (to_upper(type_val) == "OBJECT") { + if (prop_obj.count("properties")) { + std::string nested_required; + if (prop_obj.count("required")) { + nested_required = prop_obj["required"]; + } + result += ",properties:{" + format_parameters(prop_obj["properties"], nested_required) + "}"; + } + if (prop_obj.count("required")) { + std::string req_items; + size_t req_pos = 0; + skip_whitespace(prop_obj["required"], req_pos); + if (req_pos < prop_obj["required"].length() && prop_obj["required"][req_pos] == '[') { + req_pos++; + bool req_first = true; + while (req_pos < prop_obj["required"].length()) { + skip_whitespace(prop_obj["required"], req_pos); + if (prop_obj["required"][req_pos] == ']') break; + if (prop_obj["required"][req_pos] == ',') { req_pos++; continue; } + if (prop_obj["required"][req_pos] == '"') { + req_pos++; + std::string req_item = extract_json_string(prop_obj["required"], req_pos); + if (!req_first) req_items += ","; + req_first = false; + req_items += escape(req_item); + } + } + } + if (!req_items.empty()) { + result += ",required:[" + req_items + "]"; + } + } + } else if (to_upper(type_val) == "ARRAY") { + if (prop_obj.count("items")) { + result += ",items:{"; + size_t items_pos = 0; + auto items_obj = parse_json_object_raw(prop_obj["items"], items_pos); + bool items_first = true; + + for (const auto& [item_key, item_value] : items_obj) { + if (!items_first) result += ","; + items_first = false; + + if (item_key == "properties") { + std::string items_required; + if (items_obj.count("required")) { + items_required = items_obj["required"]; + } + result += "properties:{" + format_parameters(item_value, items_required) + "}"; + } else if (item_key == "required") { + result += "required:["; + size_t req_pos = 0; + skip_whitespace(item_value, req_pos); + if (req_pos < item_value.length() && item_value[req_pos] == '[') { + req_pos++; + bool req_first = true; + while (req_pos < item_value.length()) { + skip_whitespace(item_value, req_pos); + if (item_value[req_pos] == ']') break; + if (item_value[req_pos] == ',') { req_pos++; continue; } + if (item_value[req_pos] == '"') { + req_pos++; + std::string req_item = extract_json_string(item_value, req_pos); + if (!req_first) result += ","; + req_first = false; + result += escape(req_item); + } + } + } + result += "]"; + } else if (item_key == "type") { + std::string item_type = get_json_string_value(item_value, 0); + result += "type:" + escape(to_upper(item_type)); + } else { + size_t val_pos = 0; + result += item_key + ":" + format_argument(item_value, val_pos, true); + } + } + result += "}"; + } + } + + if (!type_val.empty()) { + result += ",type:" + escape(to_upper(type_val)); + } + + result += "}"; + } + + return result; +} + +inline std::string format_function_declaration(const std::string& name, + const std::string& description, + const std::string& params_json) { + std::string result = "declaration:" + name + "{"; + result += "description:" + escape(description); + + if (!params_json.empty()) { + result += ",parameters:{"; + + size_t pos = 0; + auto params = parse_json_object_raw(params_json, pos); + + if (params.count("properties")) { + std::string required_json; + if (params.count("required")) { + required_json = params["required"]; + } + result += "properties:{" + format_parameters(params["properties"], required_json) + "}"; + } + + if (params.count("required")) { + std::string req_items; + size_t req_pos = 0; + skip_whitespace(params["required"], req_pos); + if (req_pos < params["required"].length() && params["required"][req_pos] == '[') { + req_pos++; + bool first = true; + while (req_pos < params["required"].length()) { + skip_whitespace(params["required"], req_pos); + if (params["required"][req_pos] == ']') break; + if (params["required"][req_pos] == ',') { req_pos++; continue; } + if (params["required"][req_pos] == '"') { + req_pos++; + std::string item = extract_json_string(params["required"], req_pos); + if (!first) req_items += ","; + first = false; + req_items += escape(item); + } + } + } + if (!req_items.empty()) { + result += ",required:[" + req_items + "]"; + } + } + + if (params.count("type")) { + std::string type_val = get_json_string_value(params["type"], 0); + result += ",type:" + escape(to_upper(type_val)); + } + + result += "}"; + } + + result += "}"; + return result; +} + +template +inline std::string format_tools(const std::vector& tools, bool use_pipe_tags = false) { + if (tools.empty()) return ""; + + const char* decl_start = use_pipe_tags ? "<|tool>" : ""; + const char* decl_end = use_pipe_tags ? "" : ""; + + std::string result; + for (const auto& tool : tools) { + result += decl_start; + std::string params_json; + auto it = tool.parameters.find("schema"); + if (it != tool.parameters.end()) { + params_json = it->second; + } + + result += format_function_declaration(tool.name, tool.description, params_json); + result += decl_end; + } + return result; +} + + +inline size_t match_quote_tag(const std::string& s, size_t pos) { + if (s.compare(pos, 8, "") == 0) return 8; + if (s.compare(pos, 5, "<|\"|>") == 0) return 5; + return 0; +} + +inline size_t find_quote_tag(const std::string& s, size_t pos) { + size_t e = s.find("", pos); + size_t t = s.find("<|\"|>", pos); + if (e == std::string::npos) return t; + if (t == std::string::npos) return e; + return std::min(e, t); +} + +inline std::string unescape(const std::string& s) { + const std::string ESCAPE_TAG = ""; + std::string result = s; + size_t pos = 0; + while ((pos = result.find(ESCAPE_TAG, pos)) != std::string::npos) { + result.erase(pos, ESCAPE_TAG.length()); + } + return result; +} + +inline std::string args_to_json(const std::string& args_content) { + std::string result = "{"; + size_t pos = 0; + bool first = true; + + if (!args_content.empty() && args_content[0] == '{') pos = 1; + + while (pos < args_content.length()) { + while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; + if (pos >= args_content.length() || args_content[pos] == '}') break; + if (args_content[pos] == ',') { pos++; continue; } + + size_t key_start = pos; + while (pos < args_content.length() && args_content[pos] != ':') pos++; + std::string key = args_content.substr(key_start, pos - key_start); + if (pos < args_content.length()) pos++; + + std::string value; + while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; + + if (pos < args_content.length()) { + size_t qtag_len = match_quote_tag(args_content, pos); + if (qtag_len > 0) { + pos += qtag_len; + size_t val_end = find_quote_tag(args_content, pos); + if (val_end != std::string::npos) { + value = "\"" + args_content.substr(pos, val_end - pos) + "\""; + pos = val_end + match_quote_tag(args_content, val_end); + } + } else if (args_content[pos] == '{') { + int depth = 1; + size_t start = pos; + pos++; + while (pos < args_content.length() && depth > 0) { + if (args_content[pos] == '{') depth++; + else if (args_content[pos] == '}') depth--; + pos++; + } + value = args_to_json(args_content.substr(start, pos - start)); + } else if (args_content[pos] == '[') { + int depth = 1; + size_t start = pos; + pos++; + while (pos < args_content.length() && depth > 0) { + if (args_content[pos] == '[') depth++; + else if (args_content[pos] == ']') depth--; + pos++; + } + std::string arr_content = args_content.substr(start + 1, pos - start - 2); + value = "["; + size_t arr_pos = 0; + bool first_item = true; + while (arr_pos < arr_content.length()) { + while (arr_pos < arr_content.length() && (std::isspace(arr_content[arr_pos]) || arr_content[arr_pos] == ',')) arr_pos++; + if (arr_pos >= arr_content.length()) break; + + if (!first_item) value += ","; + first_item = false; + + size_t aq_len = match_quote_tag(arr_content, arr_pos); + if (aq_len > 0) { + arr_pos += aq_len; + size_t end = find_quote_tag(arr_content, arr_pos); + if (end != std::string::npos) { + value += "\"" + arr_content.substr(arr_pos, end - arr_pos) + "\""; + arr_pos = end + match_quote_tag(arr_content, end); + } + } else { + size_t end = arr_content.find_first_of(",]", arr_pos); + if (end == std::string::npos) end = arr_content.length(); + value += arr_content.substr(arr_pos, end - arr_pos); + arr_pos = end; + } + } + value += "]"; + } else { + size_t val_start = pos; + while (pos < args_content.length() && args_content[pos] != ',' && args_content[pos] != '}') { + pos++; + } + value = args_content.substr(val_start, pos - val_start); + while (!value.empty() && std::isspace(value.back())) value.pop_back(); + } + } + + if (!first) result += ","; + first = false; + result += "\"" + key + "\":" + value; + } + + result += "}"; + return result; +} + +inline void parse_function_calls(std::string& response, std::vector& function_calls) { + + const std::string CALL_START = (response.find("<|tool_call>") != std::string::npos) + ? "<|tool_call>" : ""; + const std::string CALL_END = (CALL_START == "<|tool_call>") + ? "" : ""; + size_t pos = 0; + + while ((pos = response.find(CALL_START, pos)) != std::string::npos) { + size_t content_start = pos + CALL_START.length(); + size_t call_end_pos = response.find(CALL_END, content_start); + + size_t content_end = (call_end_pos != std::string::npos) ? call_end_pos : response.length(); + std::string call_content = response.substr(content_start, content_end - content_start); + + if (call_content.compare(0, 5, "call:") == 0) { + size_t brace_pos = call_content.find('{'); + + if (brace_pos == std::string::npos) { + size_t sep_pos = call_content.find_first_of(", ", 5); + if (sep_pos != std::string::npos) { + std::string func_name = call_content.substr(5, sep_pos - 5); + size_t args_start = sep_pos + 1; + while (args_start < call_content.length() && + (call_content[args_start] == ' ' || call_content[args_start] == ',')) { + args_start++; + } + std::string args_content = "{" + call_content.substr(args_start); + if (args_content.back() != '}') args_content += "}"; + + std::string args_json = args_to_json(args_content); + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; + function_calls.push_back(json_call); + } + } else { + std::string func_name = call_content.substr(5, brace_pos - 5); + std::string args_content = call_content.substr(brace_pos); + if (args_content.back() != '}') args_content += "}"; + + std::string args_json = args_to_json(args_content); + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; + function_calls.push_back(json_call); + } + } + + size_t erase_end = (call_end_pos != std::string::npos) ? + call_end_pos + CALL_END.length() : response.length(); + response.erase(pos, erase_end - pos); + } +} + +} // namespace gemma \ No newline at end of file diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/graph.h b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/graph.h new file mode 100644 index 0000000..b436177 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/graph.h @@ -0,0 +1,779 @@ +#ifndef GRAPH_H +#define GRAPH_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { + +enum class LogLevel { + DEBUG = 0, + INFO = 1, + WARN = 2, + ERROR = 3, + NONE = 4 +}; + +class Logger { +public: + static Logger& instance() { + static Logger logger; + return logger; + } + + void set_level(LogLevel level) { min_level_ = level; } + LogLevel get_level() const { return min_level_; } + + void set_callback(std::function cb) { + std::lock_guard lock(mutex_); + callback_ = cb; + } + + void log(LogLevel level, const std::string& component, const std::string& message) { + if (level < min_level_) return; + + std::lock_guard lock(mutex_); + + if (callback_) { + callback_(level, component, message); + } else { + std::cerr << "[" << level_string(level) << "] [" << component << "] " << message << std::endl; + } + + if (level == LogLevel::ERROR) { + last_error_ = "[" + component + "] " + message; + } + } + + const std::string& last_error() const { return last_error_; } + void clear_error() { last_error_.clear(); } + +private: + Logger() : min_level_(LogLevel::WARN) {} + + static const char* level_string(LogLevel level) { + switch (level) { + case LogLevel::DEBUG: return "DEBUG"; + case LogLevel::INFO: return "INFO"; + case LogLevel::WARN: return "WARN"; + case LogLevel::ERROR: return "ERROR"; + default: return "?"; + } + } + + LogLevel min_level_; + std::mutex mutex_; + std::string last_error_; + std::function callback_; +}; + +} // namespace cactus + +#define CACTUS_LOG(level, component, msg) \ + do { \ + if (static_cast(level) >= static_cast(cactus::Logger::instance().get_level())) { \ + std::ostringstream _cactus_log_ss; \ + _cactus_log_ss << msg; \ + cactus::Logger::instance().log(level, component, _cactus_log_ss.str()); \ + } \ + } while(0) + +#define CACTUS_LOG_DEBUG(component, msg) CACTUS_LOG(cactus::LogLevel::DEBUG, component, msg) +#define CACTUS_LOG_INFO(component, msg) CACTUS_LOG(cactus::LogLevel::INFO, component, msg) +#define CACTUS_LOG_WARN(component, msg) CACTUS_LOG(cactus::LogLevel::WARN, component, msg) +#define CACTUS_LOG_ERROR(component, msg) CACTUS_LOG(cactus::LogLevel::ERROR, component, msg) + +namespace GraphFile { + class MappedFile; + struct SerializedGraph; +} + +enum class Precision { + INT8, + FP16, + FP32, + INT4 +}; + +enum class ComputeBackend { + CPU, + NPU +}; + +enum class Activation { + SILU, + GELU, + GELU_ERF, + RELU, + SIGMOID, + TANH +}; + +enum class OpType { + INPUT, PRECISION_CAST, + ADD, ADD_CLIPPED, SUBTRACT, MULTIPLY, DIVIDE, + ABS, POW, FLATTEN, VIEW, + MATMUL, TRANSPOSE, RESHAPE, SLICE, GATHER, EMBEDDING, + BILINEAR_INTERPOLATION, + SUM, MEAN, VARIANCE, MIN, MAX, + RMS_NORM, ROPE, ROPE_GPTJ, SOFTMAX, ATTENTION, ATTENTION_INT8_HYBRID, REL_POS_BIAS, CONV1D_CAUSAL, CONV1D_K3, CONV1D_K7S3, CONV1D, CONV1D_SAME_DEPTHWISE_K9, CONV1D_POINTWISE, CONV2D_K3S2P1, CONV2D_DEPTHWISE_K3S2P1, CONV2D_POINTWISE_1X1, GLU, BATCHNORM, + SCALAR_ADD, SCALAR_SUBTRACT, SCALAR_MULTIPLY, SCALAR_DIVIDE, SCALAR_EXP, SCALAR_SQRT, SCALAR_COS, SCALAR_SIN, SCALAR_LOG, + RELU, SILU, GELU, GELU_ERF, SIGMOID, TANH, + SAMPLE, CONCAT, CAT, + SCATTER_TOPK, + TOPK, LAYERNORM, GROUPNORM, + MOE_LAYER, + INDEX, + PERSISTENT, + QUANTIZE_ACTIVATIONS, + LSTM_CELL, + GATED_DELTANET_DECODE, + GATED_DELTANET_PREFILL, + STFT, + ALTUP_PREDICT, + ALTUP_CORRECT, + GAUSSIAN_TOPK, + MAXPOOL1D, + BILSTM_SEQUENCE, + LEAKY_RELU, + CONV2D_K3S1P1, + STATS_POOL, + WEIGHTED_STATS_POOL, + DENSE_MLP_INT4_FUSED +}; + +struct PrecisionTraits { + static constexpr size_t size_of(Precision prec) { + switch (prec) { + case Precision::INT8: return 1; + case Precision::FP16: return 2; + case Precision::FP32: return 4; + case Precision::INT4: return 1; + } + return 1; + } + + static constexpr size_t packed_size_of(Precision prec, size_t count) { + switch (prec) { + case Precision::INT4: return (count + 1) / 2; + default: return count * size_of(prec); + } + } + + static size_t byte_offset_of(Precision prec, size_t element_offset) { + switch (prec) { + case Precision::INT4: + assert(element_offset % 32 == 0 && "INT4 byte offset must be group-aligned (multiple of 32)"); + return element_offset / 2; + default: return element_offset * size_of(prec); + } + } + + static constexpr bool is_integer(Precision prec) { + switch (prec) { + case Precision::INT8: return true; + case Precision::INT4: return true; + case Precision::FP16: return false; + case Precision::FP32: return false; + } + return true; + } + + static constexpr bool is_floating_point(Precision prec) { + switch (prec) { + case Precision::INT8: return false; + case Precision::INT4: return false; + case Precision::FP16: return true; + case Precision::FP32: return true; + } + return false; + } +}; + +namespace Quantization { + void int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); + void fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); + void fp16_to_fp32(const __fp16* src, float* dst, size_t count); + void fp32_to_fp16(const float* src, __fp16* dst, size_t count); + void int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); + void fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); +} + +struct TensorConfig { + Precision default_precision = Precision::INT8; + Precision compute_precision = Precision::INT8; + Precision output_precision = Precision::INT8; + bool auto_mixed_precision = false; + + static TensorConfig& global(); +}; + +struct BroadcastInfo { + std::vector output_shape; + bool needs_broadcasting; + + static BroadcastInfo compute(const std::vector& lhs, const std::vector& rhs); +}; + +class BufferPool; + +struct BufferDesc { + std::vector shape; + size_t total_size; + size_t byte_size; + std::unique_ptr data; + void* external_data; + char* pooled_data; + Precision precision; + + size_t group_size = 0; + size_t num_groups = 0; + void* scales_data = nullptr; + std::unique_ptr owned_scales; + + bool is_interleaved = false; + size_t original_N = 0; + + void* activation_scales_data = nullptr; + std::unique_ptr owned_activation_scales; + size_t num_rows_for_activation_scales = 0; + + BufferDesc(); + BufferDesc(const std::vector& s, Precision prec = Precision::INT8); + ~BufferDesc(); + + BufferDesc(BufferDesc&& other) noexcept; + BufferDesc& operator=(BufferDesc&& other) noexcept; + + BufferDesc(const BufferDesc&) = delete; + BufferDesc& operator=(const BufferDesc&) = delete; + + void* get_data(); + const void* get_data() const; + + template + T* data_as() { return static_cast(get_data()); } + + template + const T* data_as() const { return static_cast(get_data()); } + + const __fp16* scales_as_fp16() const { + return reinterpret_cast(scales_data); + } + + bool is_grouped_int8() const { + return precision == Precision::INT8 && group_size > 0; + } + + bool is_grouped_int4() const { + return precision == Precision::INT4 && group_size > 0; + } + + void set_grouped_scales(size_t gs, size_t ng, void* scales_ptr) { + group_size = gs; + num_groups = ng; + scales_data = scales_ptr; + } + + void set_interleaved(bool interleaved, size_t orig_n) { + is_interleaved = interleaved; + original_N = orig_n; + } + + bool has_activation_scales() const { + return activation_scales_data != nullptr && num_rows_for_activation_scales > 0; + } + const float* activation_scales_as_float() const { + return reinterpret_cast(activation_scales_data); + } + float* activation_scales_as_float() { + return reinterpret_cast(activation_scales_data); + } + void allocate_activation_scales(size_t num_rows) { + num_rows_for_activation_scales = num_rows; + owned_activation_scales = std::make_unique(num_rows * sizeof(float)); + activation_scales_data = owned_activation_scales.get(); + } + void set_activation_scales(void* scales_ptr, size_t num_rows) { + activation_scales_data = scales_ptr; + num_rows_for_activation_scales = num_rows; + } + + void allocate(); + void allocate_from_pool(BufferPool& pool); + void release_to_pool(BufferPool& pool); + void set_external(void* ptr); +}; + +struct OpParams { + float scalar = 0.0f; + float scale = 1.0f; + float theta = 10000.0f; + float epsilon = 1e-6f; + int axis = -1; + bool pretransposed_rhs = false; + size_t position_offset = 0; + size_t slice_start = 0; + size_t slice_length = 0; + size_t window_size = 0; + bool is_causal = true; + bool attention_mask_is_additive = false; + float logit_cap = 0.0f; + std::vector new_shape; + std::vector permutation; + Precision output_precision = Precision::INT8; + BroadcastInfo broadcast_info; + ComputeBackend backend = ComputeBackend::CPU; + + size_t dilation = 1; + size_t stride = 1; + float temperature = 1.0f; + float top_p = 1.0f; + float min_p = 0.15f; + float repetition_penalty = 1.1f; + size_t top_k = 0; + size_t random_seed = 0; + + size_t index_value = 0; + size_t num_classes = 0; + size_t num_groups = 0; + size_t dst_height = 0; + size_t dst_width = 0; + bool align_corners = true; + bool normalize_routing = false; + size_t num_experts = 0; + size_t num_experts_per_tok = 0; + bool moe_gated = true; + Activation activation = Activation::SILU; + + std::vector bias_values; + std::vector bias_indices; + + const int8_t* cached_keys_int8 = nullptr; + const int8_t* cached_values_int8 = nullptr; + const float* cached_k_scales = nullptr; + const float* cached_v_scales = nullptr; + size_t cache_seq_len = 0; + size_t num_kv_heads = 0; + size_t head_dim = 0; + size_t num_fft_bins = 0; + size_t chunk_size = 0; + size_t num_altup_inputs = 0; + size_t v_head_dim = 0; + size_t kernel_size = 0; +}; + +struct GraphNode { + size_t id; + OpType op_type; + std::vector input_ids; + BufferDesc output_buffer; + OpParams params; + + GraphNode(size_t node_id, OpType type); +}; + +using nodes_vector = std::vector>; +using node_index_map_t = std::unordered_map; + +inline const BufferDesc& get_input(const GraphNode& node, size_t idx, + const nodes_vector& nodes, + const node_index_map_t& node_index_map) { + return nodes[node_index_map.at(node.input_ids[idx])]->output_buffer; +} + +struct AxisDims { + size_t outer, axis_size, inner; + static AxisDims from_shape(const std::vector& shape, size_t axis) { + AxisDims d; + d.outer = 1; + for (size_t i = 0; i < axis; i++) d.outer *= shape[i]; + d.axis_size = shape[axis]; + d.inner = 1; + for (size_t i = axis + 1; i < shape.size(); i++) d.inner *= shape[i]; + return d; + } +}; + +template +void dispatch_binary_op(OpType op, const T* lhs, const T* rhs, T* output, size_t count); + +template +void dispatch_unary_op(OpType op, const T* input, T* output, size_t count, float param = 0.0f); + +void compute_node_optimized(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_matmul_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_transpose_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_reduce_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_reshape_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_precision_cast_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_sample_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_scatter_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_layernorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_groupnorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_persistent_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_dense_mlp_int4_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_index_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_lstm_cell_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_gated_deltanet_decode_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_gated_deltanet_prefill_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_altup_predict_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_altup_correct_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_maxpool1d_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_bilstm_sequence_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_conv2d_k3s1p1_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_weighted_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); + +void shrink_thread_local_buffers(); +class BufferPool { +public: + BufferPool() = default; + ~BufferPool() = default; + + BufferPool(const BufferPool&) = delete; + BufferPool& operator=(const BufferPool&) = delete; + BufferPool(BufferPool&&) noexcept = default; + BufferPool& operator=(BufferPool&&) noexcept = default; + + char* acquire(size_t byte_size); + void release(char* ptr, size_t byte_size); + void clear(); + + size_t active_bytes() const { return active_bytes_; } + size_t pool_bytes() const { return pool_bytes_; } + size_t peak_bytes() const { return peak_bytes_; } + +private: + std::unordered_map>> free_buffers_; + size_t active_bytes_ = 0; + size_t pool_bytes_ = 0; + size_t peak_bytes_ = 0; + + size_t round_up_size(size_t size) const; +}; + +namespace ValidationUtils { + void validate_tensor_dims(const std::vector& shape, size_t required_dims, const std::string& op_name); + void validate_precision(Precision actual, Precision required, const std::string& op_name); + void validate_input_count(size_t actual, size_t required, const std::string& op_name); +} + + +class CactusGraph { +public: + CactusGraph(); + ~CactusGraph() = default; + + CactusGraph(const CactusGraph&) = delete; + CactusGraph& operator=(const CactusGraph&) = delete; + CactusGraph(CactusGraph&&) noexcept = default; + CactusGraph& operator=(CactusGraph&&) noexcept = default; + + struct DebugNodeEntry { + uint32_t layer_idx; + std::string name; + size_t node_id; + }; + + void save(const std::string& path); + static CactusGraph load(const std::string& path); + + size_t input(const std::vector& shape, Precision precision = Precision::INT8); + size_t precision_cast(size_t input, Precision target_precision); + size_t quantize_activations(size_t input); + + size_t add(size_t input1, size_t input2); + size_t add_clipped(size_t input1, size_t input2); + size_t subtract(size_t input1, size_t input2); + size_t multiply(size_t input1, size_t input2); + size_t divide(size_t input1, size_t input2); + + size_t scalar_add(size_t input, float value); + size_t scalar_subtract(size_t input, float value); + size_t scalar_multiply(size_t input, float value); + size_t scalar_divide(size_t input, float value); + size_t scalar_exp(size_t input); + size_t scalar_sqrt(size_t input); + size_t scalar_cos(size_t input); + size_t scalar_sin(size_t input); + size_t scalar_log(size_t input); + + size_t relu(size_t input); + size_t silu(size_t input); + size_t gelu(size_t input); + size_t gelu_erf(size_t input); + size_t sigmoid(size_t input); + size_t tanh(size_t input); + size_t glu(size_t input, int axis = -1); + + size_t abs(size_t input); + size_t pow(size_t input, float exponent); + size_t view(size_t input, const std::vector& new_shape); + size_t flatten(size_t input, int start_dim = 0, int end_dim = -1); + + size_t matmul(size_t input1, size_t input2, bool pretransposed_rhs = false, ComputeBackend backend = ComputeBackend::CPU); + size_t transpose(size_t input, ComputeBackend backend = ComputeBackend::CPU); + size_t transposeN(size_t input, const std::vector& permutation, ComputeBackend backend = ComputeBackend::CPU); + size_t reshape(size_t input, const std::vector& new_shape); + size_t slice(size_t input, int axis, size_t start, size_t length); + size_t index(size_t input, size_t index_value, int dim); + + size_t sum(size_t input, int axis); + size_t mean(size_t input, int axis); + size_t variance(size_t input, int axis); + size_t min(size_t input, int axis); + size_t max(size_t input, int axis); + + size_t gather(size_t embeddings, size_t indices); + size_t mmap_embeddings(const std::string& filename); + size_t mmap_weights(const std::string& filename); + void set_grouped_scales(size_t node_id, size_t group_size, size_t num_groups, void* scales_ptr); + void set_interleaved(size_t node_id, bool interleaved, size_t original_N); + + void release_weight_pages(size_t node_id); + void prefetch_weight_pages(size_t node_id); + void release_all_weight_pages(); + size_t embedding(const std::string& filename, size_t indices); + size_t embedding(size_t embedding_tensor, size_t indices); + size_t bilinear_interpolation(size_t pos_embeds, size_t dst_height, size_t dst_width, bool align_corners = true); + + size_t layernorm(size_t input, size_t weight, size_t bias, float epsilon = 1e-5f); + size_t layernorm(size_t input, size_t weight, float epsilon = 1e-5f); // No bias version + size_t groupnorm(size_t input, size_t weight, size_t bias, size_t num_groups = 32, float epsilon = 1e-5f); + size_t batchnorm(size_t input, size_t weight, size_t bias, size_t running_mean, size_t running_var, int axis = 1, float epsilon = 1e-5f); + size_t topk(size_t input, size_t k); + size_t dense_mlp_int4_fused(size_t hidden, size_t gate_weight, + size_t up_weight, size_t down_weight); + size_t moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w3_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation = Activation::SILU, + size_t per_expert_scale = 0); + size_t moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation); + size_t rms_norm(size_t input, size_t weight, float epsilon = 1e-5f); + size_t rope(size_t input, float theta, size_t position_offset = 0, ComputeBackend backend = ComputeBackend::CPU); + size_t rope_gptj(size_t input, float theta, size_t position_offset = 0, size_t rot_dim = 0, ComputeBackend backend = ComputeBackend::CPU); + size_t softmax(size_t input, int axis = -1); + size_t attention(size_t query, size_t key, size_t value, float scale, bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU); + size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, ComputeBackend backend = ComputeBackend::CPU); + size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, size_t window_size, ComputeBackend backend = ComputeBackend::CPU); + size_t attention_masked(size_t query, size_t key, size_t value, size_t mask, float scale, + bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU, + bool additive_mask = false, size_t position_offset = 0, size_t window_size = 0, + float logit_cap = 0.0f); + size_t rel_pos_bias(size_t query, size_t relative_key, float scale); + + size_t attention_int8_hybrid(size_t query, size_t key_new, size_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, + const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, + size_t window_size = 0, size_t v_head_dim = 0); + + size_t conv1d_causal(size_t input, size_t weight, size_t kernel_size, size_t dilation = 1); + size_t conv1d_k3(size_t input, size_t weight, size_t stride); + size_t conv1d_k7s3(size_t input, size_t weight, size_t bias); + size_t conv1d(size_t input, size_t weight, size_t stride); + size_t conv1d(size_t input, size_t weight, size_t bias, size_t stride); + size_t conv1d_same_depthwise_k9(size_t input, size_t weight); + size_t conv1d_same_depthwise_k9(size_t input, size_t weight, size_t bias); + size_t conv1d_pointwise(size_t input, size_t weight); + size_t conv1d_pointwise(size_t input, size_t weight, size_t bias); + size_t conv2d_k3s2p1(size_t input, size_t weight); + size_t conv2d_k3s2p1(size_t input, size_t weight, size_t bias); + size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight); + size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight, size_t bias); + size_t conv2d_pointwise_1x1(size_t input, size_t weight); + size_t conv2d_pointwise_1x1(size_t input, size_t weight, size_t bias); + + size_t lstm_cell(size_t input, size_t h_prev, size_t c_prev, size_t weight_ih, size_t weight_hh, size_t bias_ih, size_t bias_hh); + size_t gated_deltanet_decode(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, float scale = 0.0f); + size_t gated_deltanet_prefill(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, size_t chunk_size = 64, float scale = 0.0f); + size_t stft(size_t input, size_t weight, size_t stride, size_t num_fft_bins); + + size_t altup_predict(size_t coefs, const size_t* streams, size_t num_streams); + size_t altup_correct(size_t coefs, size_t innovation, const size_t* predictions, size_t num_predictions); + + size_t gaussian_topk(size_t input, float ppf); + + size_t maxpool1d(size_t input, size_t kernel_size, size_t stride); + size_t leaky_relu(size_t input, float negative_slope = 0.01f); + size_t bilstm_sequence(size_t input, size_t w_ih_fwd, size_t w_hh_fwd, size_t b_ih_fwd, size_t b_hh_fwd, + size_t w_ih_bwd, size_t w_hh_bwd, size_t b_ih_bwd, size_t b_hh_bwd); + size_t conv2d_k3s1p1(size_t input, size_t weight); + size_t conv2d_k3s1p1(size_t input, size_t weight, size_t bias); + size_t stats_pool(size_t input); + size_t weighted_stats_pool(size_t input, size_t weights); + + size_t sample(size_t logits, float temperature = 0.6f, float top_p = 0.95f, size_t top_k = 20, + const std::unordered_map& logit_bias = {}); + size_t sample_with_options(size_t logits, float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, const std::unordered_map& logit_bias = {}); + + size_t concat(size_t input1, size_t input2, int axis = 0); + size_t cat(const std::vector& inputs, int axis); + size_t scatter_topk(size_t indices, size_t values, size_t num_classes); + + void set_input(size_t node_id, const void* data, Precision precision); + void set_external_input(size_t node_id, void* data, Precision precision); + void* get_output(size_t node_id); + + void execute(const std::string& profile_file = ""); + void hard_reset(); + void soft_reset(); + void soft_reset_keep_pool(); + void set_prefill_mode(bool enabled) { prefill_mode_ = enabled; } + + void register_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); + void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); + const std::vector& get_debug_nodes() const; + void clear_debug_nodes(); + + size_t add_node(OpType op_type, const std::vector& inputs, const std::vector& output_shape, const OpParams& params = {}); + const BufferDesc& get_output_buffer(size_t node_id) const; + void allocate_buffers(); + size_t get_node_count() const; + + size_t persistent(size_t source_node); + bool is_populated(size_t persistent_node_id) const; + void invalidate_persistent(size_t persistent_node_id); + + std::vector> nodes_; + std::unordered_map node_index_map_; + +private: + static CactusGraph from_serialized(const GraphFile::SerializedGraph& serialized); + size_t next_node_id_; + std::vector> mapped_files_; + std::unordered_map weight_cache_; + std::unordered_map node_to_mapped_file_; + std::vector debug_nodes_; + BufferPool buffer_pool_; + bool prefill_mode_ = false; + + std::unordered_set persistent_node_ids_; + std::unordered_set populated_node_ids_; +}; + + +namespace GraphFile { + struct LoadedNode { + size_t node_id; + std::vector shape; + Precision precision; + size_t byte_size; + }; + + struct GraphHeader { + uint32_t magic; + uint32_t version; + uint32_t node_count; + uint32_t flags = 0; + }; + + struct NodeEntry { + uint32_t index; // serialized node index 0..n-1 + OpType op_type; + std::vector inputs; + std::vector output_shape; + Precision precision; + OpParams params; + }; + + struct SerializedGraph { + GraphHeader header; + std::vector nodes; + std::vector graph_inputs; // IDs of serialized inputs + std::vector graph_outputs; // IDs of serialized outputs + }; + + SerializedGraph load_graph(const std::string& filename); + void save_graph(const CactusGraph& graph, const std::string& filename); + + void save_node(CactusGraph& graph, size_t node_id, const std::string& filename); + + class MappedFile { + public: + MappedFile(const std::string& filename); + ~MappedFile(); + + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + MappedFile(MappedFile&& other) noexcept; + MappedFile& operator=(MappedFile&& other) noexcept; + + const std::vector& shape() const; + Precision precision() const; + size_t byte_size() const; + + size_t group_size() const { return group_size_; } + size_t num_groups() const { return num_groups_; } + const void* scales_data() const; + + bool is_interleaved() const { return is_interleaved_; } + size_t original_N() const { return original_N_; } + + void* data(); + const void* data() const; + + template + const T* typed_data() const; + + void release_pages(); + void prefetch_pages(); + + private: + int fd_; + void* mapped_data_; + size_t file_size_, data_offset_; + std::vector shape_; + Precision precision_; + size_t byte_size_; + size_t group_size_ = 0; + size_t num_groups_ = 0; + size_t scales_offset_ = 0; + size_t scales_bytes_ = 0; + uint32_t alignment_ = 32; + + bool is_interleaved_ = false; + size_t original_N_ = 0; + + void parse_header(); + void apply_madvise_hints(); + }; +} + +#endif diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/graph_param_io.h b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/graph_param_io.h new file mode 100644 index 0000000..0362ea8 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/graph_param_io.h @@ -0,0 +1,11 @@ +#pragma once + +#include "graph.h" +#include + +namespace GraphParamIO { + +void write_op_params(std::ostream& out, OpType op_type, const OpParams& params); +void read_op_params(std::istream& in, OpType op_type, OpParams& params); + +} // namespace GraphParamIO diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/kernel.h b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/kernel.h new file mode 100644 index 0000000..13f4a00 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/kernel.h @@ -0,0 +1,475 @@ +#ifndef KERNEL_H +#define KERNEL_H + +#include +#include + +enum class Precision; + +enum class ScalarOpType { + ADD, + SUBTRACT, + MULTIPLY, + DIVIDE, + ABS, + EXP, + POW, + SQRT, + COS, + SIN, + LOG +}; + +constexpr size_t KV_QUANT_GROUP_SIZE = 32; + +void cactus_add_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_add_f16_clipped(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_subtract_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_multiply_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_add_scaled_f16(const __fp16* base, const __fp16* src, __fp16* output, size_t num_elements, float scale); +void cactus_divide_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); + +void cactus_add_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_subtract_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_multiply_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_divide_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); + +void cactus_scalar_op_f16(const __fp16* input, __fp16* output, size_t num_elements, float scalar_value, ScalarOpType op_type); + +void cactus_gemv_int8(const int8_t* A, float A_scale, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemm_int8(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_int8(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int8_i8mm(const int8_t* A, float A_scale, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemm_int8_i8mm(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int4(const int8_t* A, float A_scale, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int4_block_range(const int8_t* A, float A_scale, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size, + size_t block_start, size_t block_end); + +void cactus_gemm_int4(const int8_t* A, const float* A_scales, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_int4(const int8_t* A, const float* A_scales, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_integer(Precision precision, + const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_f16(const __fp16* a, const __fp16* b_transposed, __fp16* c, + size_t M, size_t K, size_t N); + +void cactus_transpose_2d_f16(const __fp16* source, __fp16* destination, + size_t num_rows, size_t num_cols, size_t start_row, size_t end_row); +void cactus_transpose_f16(const __fp16* source, __fp16* destination, const size_t* shape, + const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx); + +double cactus_sum_all_f16(const __fp16* data, size_t num_elements); +void cactus_sum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +double cactus_mean_all_f16(const __fp16* data, size_t num_elements); +void cactus_mean_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +double cactus_variance_all_f16(const __fp16* data, size_t num_elements); +void cactus_variance_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +__fp16 cactus_min_all_f16(const __fp16* data, size_t num_elements); +void cactus_min_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +__fp16 cactus_max_all_f16(const __fp16* data, size_t num_elements); +void cactus_max_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +void cactus_rms_norm_f16(const __fp16* input, const __fp16* weight, __fp16* output, + size_t batch_size, size_t dims, float eps); + +void cactus_layer_norm_f16(const __fp16* input, const __fp16* weight, const __fp16* bias, + __fp16* output, size_t batch_size, size_t dims, float eps); + +void cactus_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, + size_t num_heads, size_t head_dim, size_t start_pos, float theta); + +void cactus_gpt_j_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, + size_t num_heads, size_t head_dim, size_t rot_dim, size_t start_pos, float theta); + +void cactus_softmax_f16(const __fp16* input, __fp16* output, size_t batch_size, + size_t seq_len, size_t vocab_size); + +void cactus_relu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_leaky_relu_f16(const __fp16* input, __fp16* output, size_t num_elements, float negative_slope); + +void cactus_silu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_gelu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_gelu_f16_erf(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_sigmoid_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_tanh_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_glu_f16( + const __fp16* input, + __fp16* output, + size_t outer_size, + size_t split_size, + size_t inner_size +); + +void cactus_glu_f32( + const float* input, + float* output, + size_t outer_size, + size_t split_size, + size_t inner_size +); + +void cactus_batchnorm_f16( + const __fp16* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + __fp16* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +); + +void cactus_batchnorm_f32( + const float* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + float* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +); + +void cactus_attention_f16(const __fp16* queries, const __fp16* keys, const __fp16* values, __fp16* output, + size_t batch_size, size_t seq_len, size_t kv_seq_len, size_t num_q_heads, size_t num_kv_heads, + size_t head_dim, float scale, const __fp16* mask, size_t position_offset = 0, size_t window_size = 0, + bool is_causal = true, bool mask_is_additive = false, bool mask_per_head = false, + size_t v_head_dim = 0, float logit_cap = 0.0f); + +void cactus_attention_hybrid_int8_fp16( + const __fp16* queries, + const int8_t* keys_cached, + const int8_t* values_cached, + const float* k_scales, + const float* v_scales, + const __fp16* keys_new, + const __fp16* values_new, + __fp16* output, + size_t batch_size, size_t seq_len, size_t cache_len, size_t new_len, + size_t num_q_heads, size_t num_kv_heads, size_t head_dim, + float scale, size_t position_offset = 0, bool is_causal = true, size_t window_size = 0, + size_t group_size = KV_QUANT_GROUP_SIZE, size_t v_head_dim = 0); + +void cactus_gated_deltanet_decode_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + float scale); + +void cactus_gated_deltanet_prefill_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t T, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + size_t requested_chunk_size, + float scale); + +void cactus_conv1d_causal_depthwise_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, + size_t L, + size_t C, + size_t K, + size_t dilation); + +void cactus_conv1d_f16_k3( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out, + size_t stride +); + +void cactus_conv1d_f16( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out, + size_t K, + size_t stride +); + +void cactus_stft_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, size_t L, + size_t C_in, size_t C_out, + size_t K, size_t stride, + size_t num_fft_bins +); + +void cactus_conv1d_f16_k7s3_oc8( + const __fp16* input, + const __fp16* Wpack, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out +); + +void cactus_conv1d_same_depthwise_f16_k9( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C +); + +void cactus_conv2d_f16_k3s1p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv2d_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv2d_depthwise_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C, + size_t H, + size_t W +); + +void cactus_conv2d_pointwise_f16_1x1_nchw_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv1d_pointwise_f16_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out +); + +void cactus_bilinear_interpolation_f16(const __fp16* input, __fp16* output, size_t src_height, size_t src_width, size_t embed_dim, + size_t dst_height, size_t dst_width, bool align_corners = true); + +void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); +void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); + +void cactus_sample_f32_ex(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); +void cactus_sample_f16_ex(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); + +void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* output, + const size_t* shape1, const size_t* shape2, const size_t* output_shape, + size_t ndims, int axis); +void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_shapes, + const size_t* output_shape, size_t num_inputs, size_t rank, int axis); + +void cactus_int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); +void cactus_fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); +void cactus_fp16_to_fp32(const __fp16* src, float* dst, size_t count); +void cactus_fp32_to_fp16(const float* src, __fp16* dst, size_t count); +void cactus_int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); +void cactus_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); +float cactus_fp16_max_abs(const __fp16* src, size_t count); + +void cactus_quantize_kv_fp16_to_int8( + const __fp16* src, + int8_t* dst, + float* scales, + size_t seq_len, size_t kv_heads, size_t head_dim, + size_t group_size = KV_QUANT_GROUP_SIZE); + +inline size_t kv_scales_count(size_t seq_len, size_t kv_heads, size_t head_dim, size_t group_size = KV_QUANT_GROUP_SIZE) { + size_t num_groups = (head_dim + group_size - 1) / group_size; + return seq_len * kv_heads * num_groups; +} + +void cactus_unpack_int4_to_int8(const uint8_t* packed, int8_t* unpacked, size_t unpacked_count); + +void cactus_gaussian_topk_f16( + const __fp16* input, + __fp16* output, + size_t rows, + size_t cols, + float ppf); + +void cactus_altup_predict_f16( + const __fp16* coefs, + const __fp16* const* streams, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim); + +void cactus_altup_correct_f16( + const __fp16* coefs, + const __fp16* innovation, + const __fp16* const* predictions, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim); + +void cactus_lstm_cell_f16( + const __fp16* x_input, + const __fp16* h_prev, + const __fp16* c_prev, + const __fp16* weight_ih, + const __fp16* weight_hh, + const __fp16* bias_ih, + const __fp16* bias_hh, + __fp16* h_new, + __fp16* c_new, + size_t batch_size, + size_t input_size, + size_t hidden_size +); + +void cactus_bilstm_sequence_f16( + const __fp16* input, + const __fp16* weight_ih_fwd, + const __fp16* weight_hh_fwd, + const __fp16* bias_ih_fwd, + const __fp16* bias_hh_fwd, + const __fp16* weight_ih_bwd, + const __fp16* weight_hh_bwd, + const __fp16* bias_ih_bwd, + const __fp16* bias_hh_bwd, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t input_size, + size_t hidden_size +); + +void cactus_maxpool1d_f16( + const __fp16* input, + __fp16* output, + size_t batch_size, + size_t channels, + size_t input_length, + size_t kernel_size, + size_t stride +); + +#endif diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/kernel_utils.h b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/kernel_utils.h new file mode 100644 index 0000000..b202158 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/kernel_utils.h @@ -0,0 +1,681 @@ +#ifndef KERNEL_UTILS_H +#define KERNEL_UTILS_H + +#include +#if defined(__APPLE__) +#include +#include +#endif +#if defined(__ANDROID__) +#include +#include +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr size_t NEON_VECTOR_SIZE = 16; +constexpr size_t STREAMING_STORE_THRESHOLD = 32768; + +inline void stream_store_f16x8(__fp16* dst, float16x8_t val) { +#if defined(__aarch64__) + float16x4_t lo = vget_low_f16(val); + float16x4_t hi = vget_high_f16(val); + __asm__ __volatile__( + "stnp %d0, %d1, [%2]" + : + : "w"(lo), "w"(hi), "r"(dst) + : "memory" + ); +#else + vst1q_f16(dst, val); +#endif +} + +inline bool cpu_has_i8mm() { +#if defined(__aarch64__) + static std::once_flag once; + static bool has = false; + + std::call_once(once, []() { +#if defined(__APPLE__) + int ret = 0; + size_t size = sizeof(ret); + if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &ret, &size, nullptr, 0) == 0) { + has = (ret == 1); + } +#elif defined(__ANDROID__) + unsigned long hwcap2 = getauxval(AT_HWCAP2); + #ifndef HWCAP2_I8MM + #define HWCAP2_I8MM (1 << 13) + #endif + has = (hwcap2 & HWCAP2_I8MM) != 0; +#endif + }); + + return has; +#else + return false; +#endif +} + +inline float32x4_t fast_exp_f32x4(float32x4_t x) { + const float32x4_t log2e = vdupq_n_f32(1.4426950408889634f); + const float32x4_t ln2 = vdupq_n_f32(0.6931471805599453f); + + const float32x4_t c0 = vdupq_n_f32(1.0f); + const float32x4_t c1 = vdupq_n_f32(0.6931471805599453f); + const float32x4_t c2 = vdupq_n_f32(0.2402265069591007f); + const float32x4_t c3 = vdupq_n_f32(0.05550410866482158f); + const float32x4_t c4 = vdupq_n_f32(0.009618129842071803f); + + x = vmaxq_f32(x, vdupq_n_f32(-87.0f)); + x = vminq_f32(x, vdupq_n_f32(87.0f)); + + float32x4_t z = vmulq_f32(x, log2e); + + int32x4_t zi = vcvtq_s32_f32(z); + float32x4_t zf = vsubq_f32(z, vcvtq_f32_s32(zi)); + + uint32x4_t neg_mask = vcltq_f32(zf, vdupq_n_f32(0.0f)); + zi = vsubq_s32(zi, vandq_s32(vreinterpretq_s32_u32(neg_mask), vdupq_n_s32(1))); + zf = vaddq_f32(zf, vreinterpretq_f32_u32(vandq_u32(neg_mask, vreinterpretq_u32_f32(vdupq_n_f32(1.0f))))); + + float32x4_t zf_ln2 = vmulq_f32(zf, ln2); + float32x4_t p = c4; + p = vfmaq_f32(c3, p, zf_ln2); + p = vfmaq_f32(c2, p, zf_ln2); + p = vfmaq_f32(c1, p, zf_ln2); + p = vfmaq_f32(c0, p, zf_ln2); + + int32x4_t exp_bits = vshlq_n_s32(vaddq_s32(zi, vdupq_n_s32(127)), 23); + float32x4_t scale = vreinterpretq_f32_s32(exp_bits); + + return vmulq_f32(p, scale); +} + +// Cephes-style 13/6 rational tanh approximation (same coefficients as Eigen). +// Constants are stored as static splatted arrays so the compiler emits a single +// pc-relative `ldr q` per load. +alignas(16) inline constexpr float kFastTanhAlpha[7][4] = { + { 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f }, + { 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f }, + { 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f }, + { 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f }, + {-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f }, + { 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f }, + {-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f }, +}; +alignas(16) inline constexpr float kFastTanhBeta[4][4] = { + { 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f }, + { 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f }, + { 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f }, + { 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f }, +}; +alignas(16) inline constexpr float kFastTanhClampHi[4] = { 9.0f, 9.0f, 9.0f, 9.0f }; +alignas(16) inline constexpr float kFastTanhClampLo[4] = {-9.0f,-9.0f,-9.0f,-9.0f }; + +inline float32x4_t fast_tanh_f32x4(float32x4_t x) { + x = vmaxq_f32(vld1q_f32(kFastTanhClampLo), vminq_f32(vld1q_f32(kFastTanhClampHi), x)); + float32x4_t x2 = vmulq_f32(x, x); + float32x4_t p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[5]), vld1q_f32(kFastTanhAlpha[6]), x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[4]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[3]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[2]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[1]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[0]), p, x2); + p = vmulq_f32(p, x); + float32x4_t q = vfmaq_f32(vld1q_f32(kFastTanhBeta[2]), vld1q_f32(kFastTanhBeta[3]), x2); + q = vfmaq_f32(vld1q_f32(kFastTanhBeta[1]), q, x2); + q = vfmaq_f32(vld1q_f32(kFastTanhBeta[0]), q, x2); + return vdivq_f32(p, q); +} + +constexpr size_t SIMD_F16_WIDTH = 8; + +inline size_t simd_align(size_t count, size_t width = SIMD_F16_WIDTH) { + return (count / width) * width; +} + +inline void f16x8_split_f32(float16x8_t v, float32x4_t& lo, float32x4_t& hi) { + lo = vcvt_f32_f16(vget_low_f16(v)); + hi = vcvt_f32_f16(vget_high_f16(v)); +} + +inline float16x8_t f32_merge_f16(float32x4_t lo, float32x4_t hi) { + return vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi)); +} + +inline float32x4_t fast_sigmoid_f32x4(float32x4_t x) { + const float32x4_t one = vdupq_n_f32(1.0f); + return vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(x)))); +} + +template +inline float16x8_t apply_f32_op_on_f16x8(float16x8_t v, F32x4Op op) { + float32x4_t lo, hi; + f16x8_split_f32(v, lo, hi); + return f32_merge_f16(op(lo), op(hi)); +} + +inline void unpack_int4_as_int8x16x2(const uint8_t* ptr, int8x16_t& high_decoded, int8x16_t& low_decoded) { + int8x16_t packed = vreinterpretq_s8_u8(vld1q_u8(ptr)); + high_decoded = vshrq_n_s8(packed, 4); + low_decoded = vshrq_n_s8(vshlq_n_s8(packed, 4), 4); +} + +namespace CactusThreading { + +#if defined(__ANDROID__) + struct CoreTopology { + std::vector performance_cores; + std::vector all_cores; + + static CoreTopology& get() { + static CoreTopology topo = detect(); + return topo; + } + + private: + static int read_sysfs_int(const char* path) { + std::ifstream f(path); + if (!f.is_open()) return -1; + int val = -1; + f >> val; + return val; + } + + static CoreTopology detect() { + CoreTopology topo; + constexpr int MAX_CPUS = 16; + std::vector> core_caps; + + for (int i = 0; i < MAX_CPUS; ++i) { + char path[128]; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpu_capacity", i); + int cap = read_sysfs_int(path); + if (cap > 0) { + core_caps.push_back({i, cap}); + topo.all_cores.push_back(i); + continue; + } + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", i); + int freq = read_sysfs_int(path); + if (freq > 0) { + core_caps.push_back({i, freq}); + topo.all_cores.push_back(i); + } + } + + if (core_caps.empty()) return topo; + + int max_cap = 0; + for (auto& [id, cap] : core_caps) { + max_cap = std::max(max_cap, cap); + } + + int threshold = static_cast(max_cap * 0.70); + for (auto& [id, cap] : core_caps) { + if (cap >= threshold) { + topo.performance_cores.push_back(id); + } + } + + return topo; + } + }; + + inline bool pin_current_thread_to_cores(const std::vector& cores) { + if (cores.empty()) return false; + cpu_set_t mask; + CPU_ZERO(&mask); + for (int core : cores) { + CPU_SET(core, &mask); + } + return sched_setaffinity(0, sizeof(mask), &mask) == 0; + } +#endif + + class ThreadPool { + private: + static constexpr size_t MAX_WORKERS = 16; + + std::vector workers; + std::deque> tasks; + + std::mutex mutex; + std::condition_variable work_available; + std::condition_variable work_done; + + bool stop{false}; + std::atomic pending_tasks{0}; + size_t num_workers_; + + void worker_thread() { + while (true) { + std::function task; + { + std::unique_lock lock(mutex); + work_available.wait(lock, [this] { + return stop || !tasks.empty(); + }); + + if (stop && tasks.empty()) { + return; + } + + task = std::move(tasks.front()); + tasks.pop_front(); + } + + task(); + + if (pending_tasks.fetch_sub(1, std::memory_order_acq_rel) == 1) { + std::lock_guard lock(mutex); + work_done.notify_one(); + } + } + } + + public: + explicit ThreadPool(size_t num_threads = std::thread::hardware_concurrency()) + : stop(false), pending_tasks(0) { + num_workers_ = std::min(num_threads, MAX_WORKERS); + if (num_workers_ == 0) num_workers_ = 1; + +#if defined(__ANDROID__) + auto& topo = CoreTopology::get(); + if (!topo.performance_cores.empty()) { + num_workers_ = std::min(num_workers_, topo.performance_cores.size()); + } +#endif + + workers.reserve(num_workers_); + for (size_t i = 0; i < num_workers_; ++i) { + workers.emplace_back([this]() { +#if defined(__ANDROID__) + auto& perf = CoreTopology::get().performance_cores; + if (!perf.empty()) { + pin_current_thread_to_cores(perf); + } +#endif + worker_thread(); + }); + } + } + + ~ThreadPool() { + { + std::lock_guard lock(mutex); + stop = true; + } + work_available.notify_all(); + for (auto& worker : workers) { + if (worker.joinable()) { + worker.join(); + } + } + } + + template + auto enqueue(F&& f) -> std::future { + using return_type = decltype(f()); + + auto task = std::make_shared>( + std::forward(f) + ); + + std::future res = task->get_future(); + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(1, std::memory_order_relaxed); + tasks.emplace_back([task](){ (*task)(); }); + } + work_available.notify_one(); + + return res; + } + + template + void enqueue_batch(size_t total_work, F task_func) { + if (total_work == 0) return; + + const size_t num_tasks = std::min(num_workers_, total_work); + const size_t per_worker = total_work / num_tasks; + const size_t remainder = total_work % num_tasks; + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(num_tasks, std::memory_order_relaxed); + + for (size_t w = 0; w < num_tasks; ++w) { + size_t start = w * per_worker + std::min(w, remainder); + size_t end = start + per_worker + (w < remainder ? 1 : 0); + tasks.emplace_back([=]() { task_func(start, end); }); + } + } + work_available.notify_all(); + } + + void wait_all() { + std::unique_lock lock(mutex); + work_done.wait(lock, [this] { + return pending_tasks.load(std::memory_order_acquire) == 0; + }); + } + + template + void enqueue_n_threads(size_t total_work, size_t num_threads, F task_func) { + if (total_work == 0 || num_threads == 0) return; + + num_threads = std::min(num_threads, std::min(num_workers_, total_work)); + const size_t per_thread = total_work / num_threads; + const size_t remainder = total_work % num_threads; + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(num_threads, std::memory_order_relaxed); + + for (size_t t = 0; t < num_threads; ++t) { + size_t start = t * per_thread + std::min(t, remainder); + size_t end = start + per_thread + (t < remainder ? 1 : 0); + tasks.emplace_back([=]() { task_func(start, end); }); + } + } + work_available.notify_all(); + } + + size_t num_workers() const { return num_workers_; } + }; + + inline ThreadPool& get_thread_pool() { + static ThreadPool pool; + return pool; + } + + struct ParallelConfig { + size_t min_work_gate; + size_t work_per_thread; + + constexpr ParallelConfig(size_t gate, size_t per_thread) + : min_work_gate(gate), work_per_thread(per_thread) {} + }; + + inline size_t get_optimal_thread_count(size_t total_work, ParallelConfig config) { + if (total_work < config.min_work_gate) return 1; + + size_t pool_size = get_thread_pool().num_workers(); + size_t num_threads = (total_work + config.work_per_thread - 1) / config.work_per_thread; + return std::min(pool_size, std::max(static_cast(1), num_threads)); + } + + struct Thresholds { + #if defined(__ANDROID__) + static constexpr ParallelConfig ATTENTION{64, 32}; + static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; + static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; + static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; + static constexpr ParallelConfig SCALAR_BASIC{30000, 15000}; + static constexpr ParallelConfig SCALAR_EXPENSIVE{10000, 5000}; + #else // Apple + static constexpr ParallelConfig ATTENTION{32, 16}; + static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; + static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; + static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; + static constexpr ParallelConfig SCALAR_BASIC{5000, 2500}; + static constexpr ParallelConfig SCALAR_EXPENSIVE{2500, 1250}; + #endif + }; + + struct GemmThreading { + #if defined(__ANDROID__) + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return 1; + return pool_size; + } + static size_t get_gemv_threads(size_t /*N_blocks*/, size_t /*pool_size*/) { + return 1; + } + #elif defined(__APPLE__) && TARGET_OS_IPHONE + static constexpr size_t GEMV_MIN_N_BLOCKS = 512; + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return std::min(pool_size, static_cast(2)); + return pool_size; + } + static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { + if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; + return std::min(pool_size, static_cast(3)); + } + #else + static constexpr size_t GEMV_MIN_N_BLOCKS = 256; + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return std::min(pool_size, static_cast(4)); + return pool_size; + } + static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { + if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; + if (N_blocks < 512) return std::min(pool_size, static_cast(2)); + return std::min(pool_size, static_cast(5)); + } + #endif + }; + + inline size_t& get_gemm_thread_override() { + static size_t override_threads = 0; + return override_threads; + } + + inline void set_gemm_threads(size_t num_threads) { + get_gemm_thread_override() = num_threads; + } + + inline void reset_gemm_threads() { + get_gemm_thread_override() = 0; + } + + class TaskHandle { + private: + std::vector> futures_; + bool auto_wait_; + + public: + TaskHandle(bool auto_wait = true) : auto_wait_(auto_wait) {} + + ~TaskHandle() { + if (auto_wait_) { + wait(); + } + } + + TaskHandle(TaskHandle&&) = default; + TaskHandle& operator=(TaskHandle&&) = default; + TaskHandle(const TaskHandle&) = delete; + TaskHandle& operator=(const TaskHandle&) = delete; + + void add_future(std::future&& f) { + futures_.push_back(std::move(f)); + } + + void wait() { + for (auto& f : futures_) { + if (f.valid()) { + f.wait(); + } + } + futures_.clear(); + } + + bool is_ready() const { + for (const auto& f : futures_) { + if (f.valid() && f.wait_for(std::chrono::seconds(0)) != std::future_status::ready) { + return false; + } + } + return true; + } + + size_t task_count() const { return futures_.size(); } + }; + + template + TaskHandle parallel_for(size_t total_work, ParallelConfig config, WorkFunc work_func, bool wait = true) { + const size_t num_threads = get_optimal_thread_count(total_work, config); + TaskHandle handle(!wait); + + if (num_threads == 1) { + if (wait) { + work_func(0, total_work); + return handle; + } + auto& pool = get_thread_pool(); + handle.add_future(pool.enqueue([work_func, total_work]() { + work_func(0, total_work); + })); + return handle; + } + + auto& pool = get_thread_pool(); + const size_t work_per_thread = total_work / num_threads; + + for (size_t t = 0; t < num_threads; ++t) { + handle.add_future(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() { + const size_t start_idx = t * work_per_thread; + const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; + work_func(start_idx, end_idx); + })); + } + + if (wait) { + handle.wait(); + } + return handle; + } + + template + void parallel_for_2d(size_t outer_size, size_t inner_size, ParallelConfig config, WorkFunc work_func) { + const size_t total_work = outer_size * inner_size; + parallel_for(total_work, config, [&](size_t start_idx, size_t end_idx) { + for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { + const size_t outer = work_idx / inner_size; + const size_t inner = work_idx % inner_size; + work_func(outer, inner); + } + }); + } + + template + ResultType parallel_reduce(size_t total_work, ParallelConfig config, + WorkFunc work_func, ResultType init_value, CombineFunc combine_func) { + const size_t num_threads = get_optimal_thread_count(total_work, config); + + if (num_threads == 1) { + return work_func(0, total_work); + } + + auto& pool = get_thread_pool(); + std::vector> futures; + std::vector partial_results(num_threads, init_value); + const size_t work_per_thread = total_work / num_threads; + + for (size_t t = 0; t < num_threads; ++t) { + futures.push_back(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() -> ResultType { + const size_t start_idx = t * work_per_thread; + const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; + return work_func(start_idx, end_idx); + })); + } + + ResultType result = init_value; + for (auto& future : futures) { + result = combine_func(result, future.get()); + } + return result; + } + + template + void parallel_gemm_tiles(size_t M, size_t total_tiles, WorkFunc work_func) { + auto& pool = get_thread_pool(); + + size_t override = get_gemm_thread_override(); + size_t num_threads = (override > 0) ? override : GemmThreading::get_num_threads(M, pool.num_workers()); + num_threads = std::min(num_threads, total_tiles); + + if (num_threads <= 1) { + work_func(0, total_tiles); + return; + } + + pool.enqueue_n_threads(total_tiles, num_threads, work_func); + pool.wait_all(); + } + +} + +template +void elementwise_op_f16(const __fp16* input, __fp16* output, size_t num_elements, + bool use_streaming, CactusThreading::ParallelConfig config, + SimdOp simd_op, ScalarOp scalar_op, size_t unroll = 4) { + CactusThreading::parallel_for(num_elements, config, + [&](size_t start, size_t end) { + const size_t n = end - start; + const size_t vec_end = start + simd_align(n); + + if (use_streaming && unroll >= 4) { + const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 4); + for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 4) { + __builtin_prefetch(&input[i + 256], 0, 0); + float16x8_t v0 = simd_op(vld1q_f16(&input[i])); + float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); + float16x8_t v2 = simd_op(vld1q_f16(&input[i + 16])); + float16x8_t v3 = simd_op(vld1q_f16(&input[i + 24])); + stream_store_f16x8(&output[i], v0); + stream_store_f16x8(&output[i + 8], v1); + stream_store_f16x8(&output[i + 16], v2); + stream_store_f16x8(&output[i + 24], v3); + } + for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { + stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } else if (use_streaming && unroll >= 2) { + const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 2); + for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 2) { + __builtin_prefetch(&input[i + 128], 0, 0); + float16x8_t v0 = simd_op(vld1q_f16(&input[i])); + float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); + stream_store_f16x8(&output[i], v0); + stream_store_f16x8(&output[i + 8], v1); + } + for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { + stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } else { + for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { + vst1q_f16(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } + for (size_t i = vec_end; i < end; ++i) { + output[i] = scalar_op(input[i]); + } + }); +} + +#endif // KERNEL_UTILS_H diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/needle_tools.h b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/needle_tools.h new file mode 100644 index 0000000..4330cff --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Headers/needle_tools.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace needle { + +inline std::string to_snake_case(const std::string& name) { + std::string s; + s.reserve(name.size() * 2); + for (char c : name) { + if (std::isalnum(static_cast(c)) || c == '_') { + s += c; + } else { + if (s.empty() || s.back() != '_') s += '_'; + } + } + + std::string s2; + s2.reserve(s.size() * 2); + for (size_t i = 0; i < s.size(); i++) { + char c = s[i]; + if (i > 0 && std::isupper(static_cast(c))) { + char prev = s[i - 1]; + if (std::islower(static_cast(prev)) || std::isdigit(static_cast(prev))) { + s2 += '_'; + } + } + s2 += c; + } + + std::string s3; + s3.reserve(s2.size() * 2); + for (size_t i = 0; i < s2.size(); i++) { + s3 += s2[i]; + if (i + 1 < s2.size() && + std::isupper(static_cast(s2[i])) && + std::isupper(static_cast(s2[i + 1]))) { + if (i + 2 < s2.size() && std::islower(static_cast(s2[i + 2]))) { + s3 += '_'; + } + } + } + + std::string result; + result.reserve(s3.size()); + bool prev_underscore = false; + for (char c : s3) { + if (c == '_') { + if (!prev_underscore) result += '_'; + prev_underscore = true; + } else { + result += static_cast(std::tolower(static_cast(c))); + prev_underscore = false; + } + } + + size_t start = result.find_first_not_of('_'); + if (start == std::string::npos) return result; + size_t end = result.find_last_not_of('_'); + return result.substr(start, end - start + 1); +} + +inline void restore_tool_names(std::vector& function_calls, + const std::unordered_map& name_map) { + if (name_map.empty()) return; + for (auto& call : function_calls) { + for (const auto& [snake, orig] : name_map) { + std::string from = "\"name\":\"" + snake + "\""; + size_t pos = call.find(from); + if (pos == std::string::npos) { + from = "\"name\": \"" + snake + "\""; + pos = call.find(from); + } + if (pos != std::string::npos) { + std::string to = from.substr(0, from.size() - snake.size() - 1) + orig + "\""; + call.replace(pos, from.size(), to); + break; + } + } + } +} + +} // namespace needle diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Info.plist b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Info.plist new file mode 100644 index 0000000..eead57b Binary files /dev/null and b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/Info.plist differ diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/_CodeSignature/CodeResources b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/_CodeSignature/CodeResources new file mode 100644 index 0000000..70da95d --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/_CodeSignature/CodeResources @@ -0,0 +1,101 @@ + + + + + files + + Info.plist + + wPyLfOfBtc39RdGXZwsNMxGnEQg= + + + files2 + + rules + + ^.* + + ^.*\.lproj/ + + optional + + weight + 1000 + + ^.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Base\.lproj/ + + weight + 1010 + + ^version.plist$ + + + rules2 + + .*\.dSYM($|/) + + weight + 11 + + ^(.*/)?\.DS_Store$ + + omit + + weight + 2000 + + ^.* + + ^.*\.lproj/ + + optional + + weight + 1000 + + ^.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Base\.lproj/ + + weight + 1010 + + ^Info\.plist$ + + omit + + weight + 20 + + ^PkgInfo$ + + omit + + weight + 20 + + ^embedded\.provisionprofile$ + + weight + 20 + + ^version\.plist$ + + weight + 20 + + + + diff --git a/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/cactus b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/cactus new file mode 100755 index 0000000..b8c8004 Binary files /dev/null and b/ios/cactus-ios.xcframework/ios-arm64-simulator/cactus.framework/cactus differ diff --git a/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus.h b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus.h new file mode 100644 index 0000000..b5b6d80 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus.h @@ -0,0 +1,12 @@ +#ifndef CACTUS_H +#define CACTUS_H + +#include "graph/graph.h" +#include "kernel/kernel.h" +#include "kernel/kernel_utils.h" +#include "engine/engine.h" +#include "models/model.h" +#include "ffi/cactus_ffi.h" +#include "npu/npu.h" + +#endif // CACTUS_H \ No newline at end of file diff --git a/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_cloud.h b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_cloud.h new file mode 100644 index 0000000..6911d9a --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_cloud.h @@ -0,0 +1,50 @@ +#ifndef CACTUS_CLOUD_H +#define CACTUS_CLOUD_H + +#include "cactus_utils.h" +#include +#include + +namespace cactus { +namespace ffi { + +struct CloudResponse { + std::string transcript; + std::string api_key_hash; + bool used_cloud = false; + std::string error; +}; + +struct CloudCompletionRequest { + std::vector messages; + std::vector tools; + std::string local_output; + std::vector local_function_calls; + bool has_images = false; + bool has_audio = false; + std::vector audio_pcm; + std::string cloud_key; +}; + +struct CloudCompletionResult { + bool ok = false; + bool used_cloud = false; + std::string response; + std::vector function_calls; + std::string error; +}; + +std::string cloud_base64_encode(const uint8_t* data, size_t len); +std::vector cloud_build_wav(const uint8_t* pcm, size_t pcm_bytes); +std::string resolve_cloud_api_key(const char* cloud_key_param); +CloudResponse cloud_transcribe_request(const std::string& audio_b64, + const std::string& fallback_text, + long timeout_seconds = 15L, + const char* cloud_key = nullptr); +CloudCompletionResult cloud_complete_request(const CloudCompletionRequest& request, + long timeout_ms); + +} // namespace ffi +} // namespace cactus + +#endif // CACTUS_CLOUD_H diff --git a/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_ffi.h b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_ffi.h new file mode 100644 index 0000000..163e180 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_ffi.h @@ -0,0 +1,454 @@ +#ifndef CACTUS_FFI_H +#define CACTUS_FFI_H + +#include +#include +#include + +#if __GNUC__ >= 4 + #define CACTUS_FFI_EXPORT __attribute__((visibility("default"))) + #define CACTUS_FFI_LOCAL __attribute__((visibility("hidden"))) +#else + #define CACTUS_FFI_EXPORT + #define CACTUS_FFI_LOCAL +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* cactus_model_t; +typedef void* cactus_index_t; +typedef void* cactus_stream_transcribe_t; + +typedef void (*cactus_token_callback)(const char* token, uint32_t token_id, void* user_data); + +CACTUS_FFI_EXPORT cactus_model_t cactus_init( + const char* model_path, + const char* corpus_dir, // optional: NULL if no RAG corpus + bool cache_index // false = always rebuild index, true = load cached if available +); + +CACTUS_FFI_EXPORT void cactus_destroy(cactus_model_t model); +CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model); +CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model); + +CACTUS_FFI_EXPORT int cactus_complete( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const char* tools_json, // optional + cactus_token_callback callback, // optional + void* user_data, // optional + const uint8_t* pcm_buffer, // optional: NULL when not used + size_t pcm_buffer_size // optional: 0 when not used +); + +CACTUS_FFI_EXPORT int cactus_prefill( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const char* tools_json, // optional + const uint8_t* pcm_buffer, // optional: NULL when not used + size_t pcm_buffer_size // optional: 0 when not used +); + +CACTUS_FFI_EXPORT int cactus_tokenize( + cactus_model_t model, + const char* text, + uint32_t* token_buffer, + size_t token_buffer_len, + size_t* out_token_len +); + +CACTUS_FFI_EXPORT int cactus_score_window( + cactus_model_t model, + const uint32_t* tokens, + size_t token_len, + size_t start, + size_t end, + size_t context, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_transcribe( + cactus_model_t model, + const char* audio_file_path, // NULL if using pcm_buffer + const char* prompt, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + cactus_token_callback callback, // optional + void* user_data, // optional + const uint8_t* pcm_buffer, // NULL if using audio_file_path + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_detect_language( + cactus_model_t model, + const char* audio_file_path, // NULL if using pcm_buffer + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const uint8_t* pcm_buffer, // NULL if using audio_file_path + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT cactus_stream_transcribe_t cactus_stream_transcribe_start( + cactus_model_t model, + const char* options_json // optional +); + +CACTUS_FFI_EXPORT int cactus_stream_transcribe_process( + cactus_stream_transcribe_t stream, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_stream_transcribe_stop( + cactus_stream_transcribe_t stream, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_embed( + cactus_model_t model, + const char* text, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim, + bool normalize +); + +CACTUS_FFI_EXPORT int cactus_image_embed( + cactus_model_t model, + const char* image_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_audio_embed( + cactus_model_t model, + const char* audio_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_vad( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_diarize( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_embed_speaker( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + const float* mask_weights, + size_t mask_num_frames +); + +CACTUS_FFI_EXPORT int cactus_rag_query( + cactus_model_t model, + const char* query, + char* response_buffer, + size_t buffer_size, + size_t top_k +); + +CACTUS_FFI_EXPORT cactus_index_t cactus_index_init( + const char* index_dir, + size_t embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_index_add( + cactus_index_t index, + const int* ids, + const char** documents, + const char** metadatas, // optional: can be NULL + const float** embeddings, + size_t count, + size_t embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_index_delete( + cactus_index_t index, + const int* ids, + size_t ids_count +); + +CACTUS_FFI_EXPORT int cactus_index_get( + cactus_index_t index, + const int* ids, + size_t ids_count, + char** document_buffers, + size_t* document_buffer_sizes, + char** metadata_buffers, + size_t* metadata_buffer_sizes, + float** embedding_buffers, + size_t* embedding_buffer_sizes +); + +CACTUS_FFI_EXPORT int cactus_index_query( + cactus_index_t index, + const float** embeddings, + size_t embeddings_count, + size_t embedding_dim, + const char* options_json, // optional + int** id_buffers, + size_t* id_buffer_sizes, + float** score_buffers, + size_t* score_buffer_sizes +); + +CACTUS_FFI_EXPORT int cactus_index_compact(cactus_index_t index); +CACTUS_FFI_EXPORT void cactus_index_destroy(cactus_index_t index); + +CACTUS_FFI_EXPORT const char* cactus_get_last_error(void); + +// level: 0=DEBUG, 1=INFO, 2=WARN (default), 3=ERROR, 4=NONE +CACTUS_FFI_EXPORT void cactus_log_set_level(int level); + +typedef void (*cactus_log_callback_t)(int level, const char* component, const char* message, void* user_data); +CACTUS_FFI_EXPORT void cactus_log_set_callback(cactus_log_callback_t callback, void* user_data); + +CACTUS_FFI_EXPORT void cactus_set_telemetry_environment(const char* framework, const char* cache_location, const char* version); +CACTUS_FFI_EXPORT void cactus_set_app_id(const char* app_id); +CACTUS_FFI_EXPORT void cactus_telemetry_flush(void); +CACTUS_FFI_EXPORT void cactus_telemetry_shutdown(void); + +// cactus graph export +typedef void* cactus_graph_t; +typedef uint64_t cactus_node_t; + +typedef struct { + int32_t precision; + size_t rank; + size_t shape[8]; + size_t num_elements; + size_t byte_size; +} cactus_tensor_info_t; + +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_create(void); +CACTUS_FFI_EXPORT void cactus_graph_destroy(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_hard_reset(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_save(cactus_graph_t graph, const char* filename); +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_load(const char* filename); + +CACTUS_FFI_EXPORT int cactus_graph_input( + cactus_graph_t graph, const size_t* shape, size_t rank, int32_t precision, +cactus_node_t* out_node); + +CACTUS_FFI_EXPORT int cactus_graph_set_input( + cactus_graph_t graph, cactus_node_t node, const void* data, int32_t +precision); +CACTUS_FFI_EXPORT int cactus_graph_set_external_input( + cactus_graph_t graph, cactus_node_t node, void* data, int32_t precision); + +CACTUS_FFI_EXPORT int cactus_graph_precision_cast( + cactus_graph_t graph, cactus_node_t input, int32_t target_precision, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_quantize_activations( + cactus_graph_t graph, cactus_node_t input, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_add(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_add_clipped(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_subtract(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_multiply(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_divide(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_scalar_add(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_subtract(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_multiply(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_exp(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sqrt(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_cos(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sin(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_log(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_abs(cactus_graph_t graph, cactus_node_t x, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_pow(cactus_graph_t graph, cactus_node_t x, +float exponent, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_view( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_flatten( + cactus_graph_t graph, cactus_node_t x, int32_t start_dim, int32_t end_dim, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_reshape( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose( + cactus_graph_t graph, cactus_node_t x, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose_n( + cactus_graph_t graph, cactus_node_t x, const size_t* permutation, size_t rank, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_slice( + cactus_graph_t graph, cactus_node_t x, int32_t axis, size_t start, size_t length, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_index( + cactus_graph_t graph, cactus_node_t x, size_t index_value, int32_t dim, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_concat( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, +cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_cat( + cactus_graph_t graph, const cactus_node_t* nodes, size_t count, int32_t +axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_matmul( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, bool pretransposed_rhs, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gather( + cactus_graph_t graph, cactus_node_t tensor, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_tensor( + cactus_graph_t graph, cactus_node_t embedding_tensor, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_file( + cactus_graph_t graph, const char* filename, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_embeddings( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_weights( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_bilinear_interpolation( + cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_set_grouped_scales( + cactus_graph_t graph, cactus_node_t node, size_t group_size, size_t num_groups, void* scales_ptr); +CACTUS_FFI_EXPORT int cactus_graph_set_interleaved( + cactus_graph_t graph, cactus_node_t node, bool interleaved, size_t original_n); +CACTUS_FFI_EXPORT int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_prefetch_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_release_all_weight_pages(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_relu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_silu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu_erf(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sigmoid(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_tanh(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_glu(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_layernorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, float epsilon, bool has_bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_groupnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, size_t num_groups, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_batchnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t running_mean, cactus_node_t running_var, int32_t axis, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_topk(cactus_graph_t graph, cactus_node_t input, size_t k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rms_norm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope_gptj( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, size_t rot_dim, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_softmax(cactus_graph_t graph, cactus_node_t input, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, float scale, bool is_causal, size_t position_offset, size_t window_size, int32_t backend, bool use_mask, cactus_node_t mask, bool additive_mask, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rel_pos_bias( + cactus_graph_t graph, cactus_node_t query, cactus_node_t relative_key, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention_int8_hybrid( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_conv1d_causal( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t kernel_size, size_t dilation, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k7s3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_same_depthwise_k9( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_pointwise( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_depthwise_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_pointwise_1x1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_lstm_cell( + cactus_graph_t graph, cactus_node_t input, cactus_node_t h_prev, cactus_node_t c_prev, cactus_node_t weight_ih, cactus_node_t weight_hh, cactus_node_t bias_ih, cactus_node_t bias_hh, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_decode( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_prefill( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, size_t chunk_size, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_stft( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, size_t num_fft_bins, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_altup_predict( + cactus_graph_t graph, cactus_node_t coefs, const cactus_node_t* streams, size_t num_streams, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_altup_correct( + cactus_graph_t graph, cactus_node_t coefs, cactus_node_t innovation, const cactus_node_t* predictions, size_t num_predictions, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gaussian_topk( + cactus_graph_t graph, cactus_node_t input, float ppf, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_gated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_ungated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_sample( + cactus_graph_t graph, cactus_node_t logits, float temperature, float top_p, size_t top_k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scatter_topk( + cactus_graph_t graph, cactus_node_t indices, cactus_node_t values, size_t num_classes, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_persistent( + cactus_graph_t graph, cactus_node_t source_node, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_is_populated( + cactus_graph_t graph, cactus_node_t persistent_node, int32_t* out_is_populated); +CACTUS_FFI_EXPORT int cactus_graph_invalidate_persistent( + cactus_graph_t graph, cactus_node_t persistent_node); + +CACTUS_FFI_EXPORT int cactus_graph_execute(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_get_output_ptr(cactus_graph_t graph, +cactus_node_t node, void** out_ptr); +CACTUS_FFI_EXPORT int cactus_graph_get_output_info(cactus_graph_t graph, +cactus_node_t node, cactus_tensor_info_t* out_info); + +#ifdef __cplusplus +} +#endif + +#endif // CACTUS_FFI_H diff --git a/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_utils.h b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_utils.h new file mode 100644 index 0000000..aad98d8 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/cactus_utils.h @@ -0,0 +1,1869 @@ +#ifndef CACTUS_UTILS_H +#define CACTUS_UTILS_H + +#include "../engine/engine.h" +#include "../models/model.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#include +#elif defined(_WIN32) +#include +#include +#elif defined(__linux__) || defined(__ANDROID__) +#include +#endif + +inline size_t get_memory_footprint_bytes() { +#ifdef __APPLE__ + task_vm_info_data_t vm_info; + mach_msg_type_number_t count = TASK_VM_INFO_COUNT; + if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&vm_info, &count) == KERN_SUCCESS) + return vm_info.phys_footprint; + +#elif defined(_WIN32) + PROCESS_MEMORY_COUNTERS_EX pmc; + if (GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc))) + return pmc.PrivateUsage; + +#elif defined(__linux__) || defined(__ANDROID__) + std::ifstream statm("/proc/self/statm"); + if (statm.is_open()) { + size_t size, resident; + statm >> size >> resident; + return resident * sysconf(_SC_PAGESIZE); + } +#endif + return 0; +} + +inline double get_ram_usage_mb() { + return get_memory_footprint_bytes() / (1024.0 * 1024.0); +} + +struct CactusModelHandle { + std::unique_ptr model; + std::unique_ptr vad_model; + std::atomic should_stop; + std::vector processed_tokens; + struct ProcessedImage { + std::string path; + long long last_modified_timestamp = 0; + + bool operator==(const ProcessedImage& other) const { + return path == other.path && last_modified_timestamp == other.last_modified_timestamp; + } + }; + + std::vector> processed_images; + std::vector user_audio_counts; + std::mutex model_mutex; + std::string model_name; + std::unique_ptr corpus_index; + std::string corpus_dir; + size_t corpus_embedding_dim = 0; + std::vector> tool_embeddings; + std::vector tool_texts; + + CactusModelHandle() : should_stop(false) {} +}; + +extern std::string last_error_message; + +bool matches_stop_sequence(const std::vector& generated_tokens, + const std::vector>& stop_sequences); + +std::string retrieve_rag_context(CactusModelHandle* handle, const std::string& query); + +namespace cactus { +namespace audio { + +static constexpr size_t WHISPER_TARGET_FRAMES = 3000; +static constexpr int WHISPER_SAMPLE_RATE = 16000; + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_whisper_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 400; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = true; + cfg.pad_mode = "reflect"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 1e-10f; + cfg.log_mel = "log10"; + cfg.reference = 1.0f; + cfg.min_value = 1e-10f; + cfg.remove_dc_offset = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_parakeet_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 512; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = true; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 5.960464477539063e-08f; // 2^-24 guard value used by HF Parakeet. + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 1e-10f; + cfg.remove_dc_offset = false; + cfg.hann_periodic = false; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_htk_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 321; + cfg.frame_length = 320; + cfg.fft_override = 1024; + cfg.hop_length = 160; + cfg.power = 1.0f; + cfg.center = false; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 0.001f; + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 0.001f; + cfg.remove_dc_offset = false; + cfg.hann_periodic = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_gemma4_audio_spectrogram_config( + const cactus::engine::Config& model_config) { + auto cfg = get_htk_spectrogram_config(); + cfg.fft_override = model_config.audio_fft_length; + cfg.mel_floor_additive = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_wespeaker_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 512; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = false; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 1.1754944e-38f; + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 1.1754944e-38f; + cfg.remove_dc_offset = true; + cfg.preemphasis = 0.97f; + cfg.hann_periodic = false; + cfg.window_a0 = 0.54f; + return cfg; +} + +// Whisper v1/v2: 80 mel bins, HTK. Whisper v3: 128 mel bins, Slaney, 512-FFT, no DC removal. +inline void init_whisper_mel_filters(cactus::engine::AudioProcessor& ap, + cactus::engine::AudioProcessor::SpectrogramConfig& cfg, + size_t mel_bins) { + const size_t num_mel_filters = std::max(1, mel_bins); + const bool is_v3 = mel_bins > 80; + if (is_v3) { + cfg.fft_override = 512; + cfg.remove_dc_offset = false; + } + const size_t fft_len = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; + const size_t num_frequency_bins = fft_len / 2 + 1; + if (is_v3) { + ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, + WHISPER_SAMPLE_RATE, "slaney", "slaney"); + } else { + ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, + WHISPER_SAMPLE_RATE); + } +} + +// use_mel_floor_padding=true pads short audio with the normalized mel floor (required for v3). +inline std::vector normalize_whisper_mel(std::vector& mel, size_t n_mels, + bool use_mel_floor_padding = false) { + if (mel.empty() || n_mels == 0) return mel; + size_t n_frames = mel.size() / n_mels; + + float max_val = -std::numeric_limits::infinity(); + for (float v : mel) if (v > max_val) max_val = v; + + float min_allowed = max_val - 8.0f; + for (float& v : mel) { + if (v < min_allowed) v = min_allowed; + v = (v + 4.0f) * 0.25f; + } + + if (n_frames != WHISPER_TARGET_FRAMES) { + float pad_val = use_mel_floor_padding ? (min_allowed + 4.0f) * 0.25f : 0.0f; + std::vector fixed(n_mels * WHISPER_TARGET_FRAMES, pad_val); + size_t copy_frames = std::min(n_frames, WHISPER_TARGET_FRAMES); + for (size_t m = 0; m < n_mels; ++m) { + const float* src = &mel[m * n_frames]; + float* dst = &fixed[m * WHISPER_TARGET_FRAMES]; + std::copy(src, src + copy_frames, dst); + } + return fixed; + } + return std::move(mel); +} + +inline std::vector transpose_mel_to_frame_major(const std::vector& mel, + size_t num_mels, size_t num_frames) { + std::vector transposed(num_frames * num_mels); + for (size_t m = 0; m < num_mels; m++) { + for (size_t t = 0; t < num_frames; t++) { + transposed[t * num_mels + m] = mel[m * num_frames + t]; + } + } + return transposed; +} + +inline void apply_preemphasis(std::vector& waveform, float coefficient = 0.97f) { + if (waveform.size() < 2 || coefficient == 0.0f) { + return; + } + for (size_t i = waveform.size() - 1; i > 0; --i) { + waveform[i] -= coefficient * waveform[i - 1]; + } +} + +inline void normalize_parakeet_log_mel(std::vector& mel, size_t num_mels, float epsilon = 1e-5f) { + if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { + return; + } + const size_t num_frames = mel.size() / num_mels; + if (num_frames == 0) { + return; + } + + for (size_t m = 0; m < num_mels; ++m) { + const size_t base = m * num_frames; + float mean = 0.0f; + for (size_t t = 0; t < num_frames; ++t) { + mean += mel[base + t]; + } + mean /= static_cast(num_frames); + + float variance = 0.0f; + for (size_t t = 0; t < num_frames; ++t) { + const float d = mel[base + t] - mean; + variance += d * d; + } + const float denom = static_cast(std::max(1, num_frames - 1)); + const float inv_std = 1.0f / std::sqrt((variance / denom) + epsilon); + for (size_t t = 0; t < num_frames; ++t) { + mel[base + t] = (mel[base + t] - mean) * inv_std; + } + } +} + +inline void trim_mel_frames(std::vector& mel, size_t num_mels, size_t valid_frames) { + if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { + return; + } + size_t total_frames = mel.size() / num_mels; + if (valid_frames == 0 || valid_frames >= total_frames) { + return; + } + std::vector trimmed(num_mels * valid_frames); + for (size_t m = 0; m < num_mels; ++m) { + const float* src = &mel[m * total_frames]; + float* dst = &trimmed[m * valid_frames]; + std::copy(src, src + valid_frames, dst); + } + mel.swap(trimmed); +} + +struct AudioPreprocessResult { + std::vector features; + size_t num_frames = 0; + size_t num_soft_tokens = 0; +}; + +inline AudioPreprocessResult preprocess_audio_for_gemma4( + std::vector audio_samples, + const cactus::engine::Config& model_config +) { + AudioPreprocessResult result; + if (audio_samples.empty()) return result; + + size_t pad_amt = 320 - (audio_samples.size() % 320); + if (pad_amt < 320) + audio_samples.resize(audio_samples.size() + pad_amt, 0.0f); + + size_t mel_bins = model_config.audio_input_feat_size; + auto cfg = get_gemma4_audio_spectrogram_config(model_config); + + size_t semicausal_pad = cfg.frame_length / 2; + audio_samples.insert(audio_samples.begin(), semicausal_pad, 0.0f); + + cactus::engine::AudioProcessor ap; + size_t fft_for_mel = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; + ap.init_mel_filters(fft_for_mel / 2 + 1, mel_bins, 0.0f, 8000.0f, 16000, + nullptr, "htk"); + std::vector mel = ap.compute_spectrogram(audio_samples, cfg); + + result.num_frames = mel.size() / mel_bins; + result.features = transpose_mel_to_frame_major(mel, mel_bins, result.num_frames); + + size_t after_stage1 = (result.num_frames + 1) / 2; + result.num_soft_tokens = (after_stage1 + 1) / 2; + + return result; +} + +inline std::vector pcm_buffer_to_float_samples( + const uint8_t* pcm_buffer, size_t pcm_buffer_size +) { + const int16_t* pcm_samples = reinterpret_cast(pcm_buffer); + size_t num_samples = pcm_buffer_size / 2; + std::vector waveform_fp32(num_samples); + constexpr float inv_32768 = 1.0f / 32768.0f; + for (size_t i = 0; i < num_samples; i++) + waveform_fp32[i] = static_cast(pcm_samples[i]) * inv_32768; + return waveform_fp32; +} + +} // namespace audio +} // namespace cactus + +namespace cactus { +namespace ffi { + +inline bool env_flag_enabled(const char* key) { + const char* value = std::getenv(key); + return value && value[0] != '\0' && !(value[0] == '0' && value[1] == '\0'); +} + +inline std::string generateUUID() { +#ifdef __APPLE__ + uuid_t uuid; + uuid_generate_random(uuid); + char uuid_str[37]; + uuid_unparse_lower(uuid, uuid_str); + return std::string(uuid_str); +#else + static std::random_device rd; + static std::mt19937 gen(rd()); + static std::uniform_int_distribution<> dis(0, 15); + static std::uniform_int_distribution<> dis2(8, 11); + + std::stringstream ss; + ss << std::hex; + for (int i = 0; i < 8; i++) ss << dis(gen); + ss << "-"; + for (int i = 0; i < 4; i++) ss << dis(gen); + ss << "-4"; + for (int i = 0; i < 3; i++) ss << dis(gen); + ss << "-"; + ss << dis2(gen); + for (int i = 0; i < 3; i++) ss << dis(gen); + ss << "-"; + for (int i = 0; i < 12; i++) ss << dis(gen); + return ss.str(); +#endif +} + +struct ToolFunction { + std::string name; + std::string description; + std::unordered_map parameters; +}; + +struct InferenceOptions { + float temperature = 0.0f; + float top_p = 0.0f; + float min_p = 0.15f; + float repetition_penalty = 1.1f; + float confidence_threshold = -1.0f; + size_t top_k = 0; + size_t max_tokens = 100; + size_t tool_rag_top_k = 2; + size_t cloud_timeout_ms = 15000; + std::vector stop_sequences; + bool force_tools = false; + bool include_stop_sequences = false; + bool use_vad = true; + bool telemetry_enabled = true; + bool auto_handoff = true; + bool handoff_with_images = true; + bool enable_thinking_if_supported = false; +}; + +} // namespace ffi +} // namespace cactus + +std::vector select_relevant_tools( + CactusModelHandle* handle, + const std::string& query, + const std::vector& all_tools, + size_t top_k); + +#include "gemma_tools.h" + +namespace cactus { +namespace ffi { + +inline std::string escape_json_string(const std::string& s) { + std::ostringstream o; + for (char c : s) { + if (c == '"') o << "\\\""; + else if (c == '\n') o << "\\n"; + else if (c == '\r') o << "\\r"; + else if (c == '\t') o << "\\t"; + else if (c == '\\') o << "\\\\"; + else o << c; + } + return o.str(); +} + + +inline std::string trim_string(const std::string& s) { + size_t start = 0; + while (start < s.size() && std::isspace(static_cast(s[start]))) ++start; + size_t end = s.size(); + while (end > start && std::isspace(static_cast(s[end - 1]))) --end; + return s.substr(start, end - start); +} + +inline size_t find_matching_delimiter(const std::string& s, size_t pos, char open, char close) { + int depth = 1; + pos++; + while (pos < s.length() && depth > 0) { + if (s[pos] == open) depth++; + else if (s[pos] == close) depth--; + else if (s[pos] == '"') { + pos++; + while (pos < s.length() && s[pos] != '"') { + if (s[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + return pos; +} + +inline std::string env_or_default(const char* key, const char* fallback) { + const char* v = std::getenv(key); + if (v && v[0] != '\0') return std::string(v); + return std::string(fallback); +} + +inline std::string json_string_field(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return {}; + + size_t i = pos + pattern.size(); + while (i < json.size() && std::isspace(static_cast(json[i]))) i++; + if (i >= json.size() || json[i] != '"') return {}; + ++i; + + std::string out; + out.reserve(128); + while (i < json.size()) { + char c = json[i++]; + if (c == '"') return out; + if (c == '\\' && i < json.size()) { + char e = json[i++]; + switch (e) { + case '"': out.push_back('"'); break; + case '\\': out.push_back('\\'); break; + case '/': out.push_back('/'); break; + case 'b': out.push_back('\b'); break; + case 'f': out.push_back('\f'); break; + case 'n': out.push_back('\n'); break; + case 'r': out.push_back('\r'); break; + case 't': out.push_back('\t'); break; + default: out.push_back(e); break; + } + continue; + } + out.push_back(c); + } + return {}; +} + +inline std::string json_array_field(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return "[]"; + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + if (start >= json.size() || json[start] != '[') return "[]"; + + int depth = 1; + size_t end = start + 1; + while (end < json.size() && depth > 0) { + if (json[end] == '[') depth++; + else if (json[end] == ']') depth--; + end++; + } + return json.substr(start, end - start); +} + +inline std::vector split_json_array(const std::string& array_json) { + std::vector out; + if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; + + size_t i = 1; + while (i + 1 < array_json.size()) { + while (i + 1 < array_json.size() && + (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) i++; + if (i + 1 >= array_json.size() || array_json[i] != '{') break; + + size_t start = i; + int depth = 0; + bool in_str = false; + bool esc = false; + for (; i < array_json.size(); ++i) { + char c = array_json[i]; + if (in_str) { + if (esc) esc = false; + else if (c == '\\') esc = true; + else if (c == '"') in_str = false; + continue; + } + if (c == '"') { in_str = true; continue; } + if (c == '{') depth++; + if (c == '}') { + depth--; + if (depth == 0) { + out.push_back(array_json.substr(start, i - start + 1)); + i++; + break; + } + } + } + } + return out; +} + +inline std::string serialize_tools_json(const std::vector& tools) { + if (tools.empty()) return ""; + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < tools.size(); ++i) { + if (i > 0) oss << ","; + oss << "{\"type\":\"function\",\"function\":{"; + oss << "\"name\":\"" << escape_json_string(tools[i].name) << "\","; + oss << "\"description\":\"" << escape_json_string(tools[i].description) << "\""; + auto it = tools[i].parameters.find("schema"); + if (it != tools[i].parameters.end()) { + oss << ",\"parameters\":" << it->second; + } + oss << "}}"; + } + oss << "]"; + return oss.str(); +} + +namespace json_sorted { + +inline void skip_ws(const std::string& s, size_t& p) { + while (p < s.size() && std::isspace(static_cast(s[p]))) p++; +} + +inline std::string parse_string(const std::string& s, size_t& p) { + std::string r = "\""; + p++; + while (p < s.size()) { + if (s[p] == '\\') { + r += s[p++]; + if (p < s.size()) r += s[p++]; + } else if (s[p] == '"') { + r += '"'; + p++; + return r; + } else { + r += s[p++]; + } + } + return r; +} + +inline std::string parse_value(const std::string& s, size_t& p); + +inline std::string parse_object(const std::string& s, size_t& p) { + p++; + std::map entries; + skip_ws(s, p); + while (p < s.size() && s[p] != '}') { + if (s[p] == ',') { p++; skip_ws(s, p); continue; } + std::string key = parse_string(s, p); + skip_ws(s, p); + if (p < s.size() && s[p] == ':') p++; + skip_ws(s, p); + std::string val = parse_value(s, p); + entries[key] = val; + skip_ws(s, p); + } + if (p < s.size()) p++; + std::string r = "{"; + bool first = true; + for (const auto& kv : entries) { + if (!first) r += ", "; + r += kv.first + ": " + kv.second; + first = false; + } + r += "}"; + return r; +} + +inline std::string parse_array(const std::string& s, size_t& p) { + p++; + std::vector items; + skip_ws(s, p); + while (p < s.size() && s[p] != ']') { + if (s[p] == ',') { p++; skip_ws(s, p); continue; } + items.push_back(parse_value(s, p)); + skip_ws(s, p); + } + if (p < s.size()) p++; + std::string r = "["; + for (size_t i = 0; i < items.size(); i++) { + if (i > 0) r += ", "; + r += items[i]; + } + r += "]"; + return r; +} + +inline std::string parse_value(const std::string& s, size_t& p) { + skip_ws(s, p); + if (p >= s.size()) return ""; + if (s[p] == '"') return parse_string(s, p); + if (s[p] == '{') return parse_object(s, p); + if (s[p] == '[') return parse_array(s, p); + size_t start = p; + while (p < s.size() && s[p] != ',' && s[p] != '}' && s[p] != ']' && !std::isspace(static_cast(s[p]))) p++; + return s.substr(start, p - start); +} + +inline std::string reformat(const std::string& json) { + size_t p = 0; + return parse_value(json, p); +} + +} // namespace json_sorted + +inline std::string serialize_tools_for_template(const std::vector& tools) { + if (tools.empty()) return ""; + std::string result; + for (const auto& tool : tools) { + std::map func_fields; + func_fields["\"description\""] = "\"" + escape_json_string(tool.description) + "\""; + func_fields["\"name\""] = "\"" + escape_json_string(tool.name) + "\""; + auto it = tool.parameters.find("schema"); + if (it != tool.parameters.end()) { + func_fields["\"parameters\""] = json_sorted::reformat(it->second); + } + std::string func_json = "{"; + bool first = true; + for (const auto& kv : func_fields) { + if (!first) func_json += ", "; + func_json += kv.first + ": " + kv.second; + first = false; + } + func_json += "}"; + result += "\n{\"function\": " + func_json + ", \"type\": \"function\"}"; + } + return result; +} + +inline void handle_error_response(const std::string& error_message, char* response_buffer, size_t buffer_size) { + std::ostringstream json; + json << "{"; + json << "\"success\":false,"; + json << "\"error\":\"" << escape_json_string(error_message) << "\","; + json << "\"cloud_handoff\":false,"; + json << "\"response\":null,"; + json << "\"function_calls\":[],"; + json << "\"confidence\":0.0,"; + json << "\"time_to_first_token_ms\":0.0,"; + json << "\"total_time_ms\":0.0,"; + json << "\"prefill_tps\":0.0,"; + json << "\"decode_tps\":0.0,"; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; + json << "\"prefill_tokens\":0,"; + json << "\"decode_tokens\":0,"; + json << "\"total_tokens\":0"; + json << "}"; + std::string error_json = json.str(); + if (response_buffer && error_json.length() < buffer_size) { + std::strcpy(response_buffer, error_json.c_str()); + } +} + +inline std::vector parse_messages_json(const std::string& json, + std::vector& out_image_paths, + std::vector* out_audio_paths = nullptr) { + std::vector messages; + out_image_paths.clear(); + if (out_audio_paths) out_audio_paths->clear(); + + size_t pos = json.find('['); + if (pos == std::string::npos) { + throw std::runtime_error("Invalid JSON: expected array"); + } + + pos = json.find('{', pos); + while (pos != std::string::npos) { + cactus::engine::ChatMessage msg; + + size_t obj_start = pos; + int brace_count = 1; + size_t obj_end = obj_start + 1; + while (obj_end < json.length() && brace_count > 0) { + if (json[obj_end] == '{') brace_count++; + else if (json[obj_end] == '}') brace_count--; + obj_end++; + } + + size_t role_pos = json.find("\"role\"", pos); + if (role_pos == std::string::npos || role_pos >= obj_end) break; + + size_t role_start = json.find('"', role_pos + 6) + 1; + size_t role_end = json.find('"', role_start); + msg.role = json.substr(role_start, role_end - role_start); + + size_t content_pos = json.find("\"content\"", role_end); + if (content_pos != std::string::npos && content_pos < obj_end) { + size_t content_start = json.find('"', content_pos + 9) + 1; + size_t content_end = content_start; + + while (content_end < json.length()) { + content_end = json.find('"', content_end); + if (content_end == std::string::npos) break; + if (json[content_end - 1] != '\\') break; + content_end++; + } + + msg.content = json.substr(content_start, content_end - content_start); + + size_t escape_pos = 0; + while ((escape_pos = msg.content.find("\\n", escape_pos)) != std::string::npos) { + msg.content.replace(escape_pos, 2, "\n"); + escape_pos += 1; + } + escape_pos = 0; + while ((escape_pos = msg.content.find("\\\"", escape_pos)) != std::string::npos) { + msg.content.replace(escape_pos, 2, "\""); + escape_pos += 1; + } + } + + auto parse_path_array = [&](const char* key, std::vector& dest, + std::vector* out_paths) { + size_t key_pos = json.find(key, pos); + if (key_pos == std::string::npos || key_pos >= obj_end) return; + size_t array_start = json.find('[', key_pos); + if (array_start == std::string::npos || array_start >= obj_end) return; + size_t array_end = json.find(']', array_start); + if (array_end == std::string::npos || array_end >= obj_end) return; + size_t cur = array_start; + while (true) { + cur = json.find('"', cur + 1); + if (cur == std::string::npos || cur >= array_end) break; + size_t str_start = cur + 1; + size_t str_end = json.find('"', str_start); + if (str_end == std::string::npos || str_end > array_end) break; + std::string path = std::filesystem::absolute( + std::filesystem::path(json.substr(str_start, str_end - str_start))).string(); + dest.push_back(path); + if (out_paths) out_paths->push_back(path); + cur = str_end; + } + }; + + parse_path_array("\"images\"", msg.images, &out_image_paths); + parse_path_array("\"audio\"", msg.audio, out_audio_paths); + + if (msg.role == "tool") { + size_t name_pos = json.find("\"name\"", obj_start); + if (name_pos != std::string::npos && name_pos < obj_end) { + size_t name_quote = json.find('"', name_pos + 6); + if (name_quote != std::string::npos && name_quote < obj_end) { + size_t name_start = name_quote + 1; + size_t name_end = json.find('"', name_start); + if (name_end != std::string::npos && name_end < obj_end) { + msg.name = json.substr(name_start, name_end - name_start); + } + } + } + } + + size_t tool_calls_pos = json.find("\"tool_calls\"", obj_start); + if (tool_calls_pos != std::string::npos && tool_calls_pos < obj_end) { + size_t tool_calls_arr_start = json.find('[', tool_calls_pos); + if (tool_calls_arr_start != std::string::npos && tool_calls_arr_start < obj_end) { + size_t tool_calls_arr_end = find_matching_delimiter(json, tool_calls_arr_start, '[', ']'); + + size_t search_pos = tool_calls_arr_start; + while (true) { + size_t func_pos = json.find("\"function\"", search_pos); + if (func_pos == std::string::npos || func_pos >= tool_calls_arr_end) break; + + size_t func_obj_start = json.find('{', func_pos + 10); + if (func_obj_start == std::string::npos || func_obj_start >= tool_calls_arr_end) break; + + size_t func_obj_end = find_matching_delimiter(json, func_obj_start, '{', '}'); + + cactus::engine::ToolCallInfo tool_call; + + size_t fn_name_pos = json.find("\"name\"", func_obj_start); + if (fn_name_pos != std::string::npos && fn_name_pos < func_obj_end) { + size_t fn_name_quote = json.find('"', fn_name_pos + 6); + if (fn_name_quote != std::string::npos && fn_name_quote < func_obj_end) { + size_t fn_name_start = fn_name_quote + 1; + size_t fn_name_end = json.find('"', fn_name_start); + if (fn_name_end != std::string::npos && fn_name_end < func_obj_end) { + tool_call.name = json.substr(fn_name_start, fn_name_end - fn_name_start); + } + } + } + + size_t args_pos = json.find("\"arguments\"", func_obj_start); + if (args_pos != std::string::npos && args_pos < func_obj_end) { + size_t colon_pos = json.find(':', args_pos + 11); + if (colon_pos != std::string::npos && colon_pos < func_obj_end) { + size_t args_start = colon_pos + 1; + while (args_start < json.length() && std::isspace(static_cast(json[args_start]))) args_start++; + + if (args_start < func_obj_end && json[args_start] == '{') { + size_t args_end = find_matching_delimiter(json, args_start, '{', '}'); + tool_call.arguments = json.substr(args_start, args_end - args_start); + } else if (args_start < func_obj_end && json[args_start] == '"') { + size_t str_start = args_start + 1; + size_t str_end = str_start; + while (str_end < json.length() && json[str_end] != '"') { + if (json[str_end] == '\\') str_end++; + str_end++; + } + tool_call.arguments = json.substr(str_start, str_end - str_start); + } + } + } + + if (!tool_call.name.empty()) { + msg.tool_calls.push_back(tool_call); + } + search_pos = func_obj_end; + } + } + } + + messages.push_back(msg); + + pos = json.find('{', obj_end); + } + + return messages; +} + +inline std::vector parse_tools_json(const std::string& json) { + std::vector tools; + + if (json.empty()) return tools; + + size_t pos = json.find('['); + if (pos == std::string::npos) return tools; + + pos = json.find("\"function\"", pos); + while (pos != std::string::npos) { + ToolFunction tool; + size_t next_search = pos + 1; + + size_t name_pos = json.find("\"name\"", pos); + if (name_pos != std::string::npos) { + size_t name_start = json.find('"', name_pos + 6) + 1; + size_t name_end = json.find('"', name_start); + tool.name = json.substr(name_start, name_end - name_start); + } + + size_t desc_pos = json.find("\"description\"", pos); + if (desc_pos != std::string::npos) { + size_t desc_start = json.find('"', desc_pos + 13) + 1; + size_t desc_end = json.find('"', desc_start); + tool.description = json.substr(desc_start, desc_end - desc_start); + } + + size_t params_pos = json.find("\"parameters\"", pos); + if (params_pos != std::string::npos) { + size_t params_start = json.find('{', params_pos); + if (params_start != std::string::npos) { + int brace_count = 1; + size_t params_end = params_start + 1; + while (params_end < json.length() && brace_count > 0) { + if (json[params_end] == '{') brace_count++; + else if (json[params_end] == '}') brace_count--; + params_end++; + } + tool.parameters["schema"] = json.substr(params_start, params_end - params_start); + next_search = params_end; + } + } + + if (!tool.name.empty()) { + tools.push_back(tool); + } + + pos = json.find("\"function\"", next_search); + } + + return tools; +} + +inline bool try_parse_json_float(const std::string& json, const std::string& key, float& out_value) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return false; + + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + + size_t end = start; + while (end < json.size() && std::string(",}] \t\n\r").find(json[end]) == std::string::npos) ++end; + + try { + out_value = std::stof(json.substr(start, end - start)); + return true; + } catch (...) { + return false; + } +} + +inline std::vector parse_json_string_array_field(const std::string& json, const std::string& key) { + std::vector out; + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return out; + + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + if (start >= json.size() || json[start] != '[') return out; + + int depth = 1; + bool in_string = false; + bool escaped = false; + size_t end = start + 1; + + while (end < json.size() && depth > 0) { + char c = json[end]; + if (in_string) { + if (escaped) escaped = false; + else if (c == '\\') escaped = true; + else if (c == '"') in_string = false; + } else { + if (c == '"') in_string = true; + else if (c == '[') depth++; + else if (c == ']') depth--; + } + ++end; + } + + if (depth != 0) return out; + const std::string array_json = json.substr(start, end - start); + if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; + + size_t i = 1; + while (i + 1 < array_json.size()) { + while (i + 1 < array_json.size() && + (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) { + ++i; + } + if (i + 1 >= array_json.size() || array_json[i] == ']') break; + if (array_json[i] != '"') break; + + ++i; + std::string value; + bool escaped = false; + while (i < array_json.size()) { + char c = array_json[i++]; + if (escaped) { + switch (c) { + case '"': value.push_back('"'); break; + case '\\': value.push_back('\\'); break; + case '/': value.push_back('/'); break; + case 'b': value.push_back('\b'); break; + case 'f': value.push_back('\f'); break; + case 'n': value.push_back('\n'); break; + case 'r': value.push_back('\r'); break; + case 't': value.push_back('\t'); break; + default: value.push_back(c); break; + } + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == '"') { + out.push_back(value); + break; + } + value.push_back(c); + } + } + + return out; +} + +inline void parse_custom_vocabulary_options(const std::string& json, + std::vector& custom_vocabulary, + float& vocabulary_boost) { + custom_vocabulary.clear(); + vocabulary_boost = 5.0f; + if (json.empty()) return; + + float parsed_boost = vocabulary_boost; + if (try_parse_json_float(json, "vocabulary_boost", parsed_boost)) { + vocabulary_boost = std::clamp(parsed_boost, 0.0f, 20.0f); + } + + custom_vocabulary = parse_json_string_array_field(json, "custom_vocabulary"); +} + +inline std::unordered_map build_token_bias_map(const std::vector>& tokenized_entries, + float vocabulary_boost) { + std::unordered_map vocab_bias; + const float clamped_boost = std::clamp(vocabulary_boost, 0.0f, 20.0f); + if (clamped_boost == 0.0f) return vocab_bias; + + for (const auto& token_ids : tokenized_entries) { + for (uint32_t token_id : token_ids) { + float& entry = vocab_bias[token_id]; + if (entry < clamped_boost) { + entry = clamped_boost; + } + } + } + + return vocab_bias; +} + +inline std::unordered_map build_custom_vocabulary_bias(cactus::engine::Tokenizer* tokenizer, + const std::vector& custom_vocabulary, + float vocabulary_boost) { + if (!tokenizer || custom_vocabulary.empty()) return {}; + std::vector> tokenized_entries; + tokenized_entries.reserve(custom_vocabulary.size()); + + for (const auto& word : custom_vocabulary) { + if (word.empty()) continue; + tokenized_entries.push_back(tokenizer->encode(word)); + } + + return build_token_bias_map(tokenized_entries, vocabulary_boost); +} + +inline void apply_custom_vocabulary_options(cactus::engine::Model* model, const std::string& json) { + if (!model) return; + + std::vector custom_vocabulary; + float vocabulary_boost = 5.0f; + parse_custom_vocabulary_options(json, custom_vocabulary, vocabulary_boost); + model->set_vocab_bias(build_custom_vocabulary_bias(model->get_tokenizer(), custom_vocabulary, vocabulary_boost)); +} + +inline size_t levenshtein_ci(const std::string& a, const std::string& b) { + const size_t m = a.size(), n = b.size(); + std::vector prev(n + 1), curr(n + 1); + for (size_t j = 0; j <= n; ++j) prev[j] = j; + for (size_t i = 1; i <= m; ++i) { + curr[0] = i; + for (size_t j = 1; j <= n; ++j) { + const bool match = std::tolower(static_cast(a[i - 1])) == + std::tolower(static_cast(b[j - 1])); + curr[j] = std::min({prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + (match ? 0 : 1)}); + } + std::swap(prev, curr); + } + return prev[n]; +} + +inline std::string collapse_spaces(const std::string& s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + if (c != ' ') out += c; + } + return out; +} + +inline void apply_vocabulary_spelling_correction( + std::string& text, + const std::vector& custom_vocabulary) +{ + if (custom_vocabulary.empty() || text.empty()) return; + + struct VocabEntry { + const std::string* original; + std::string collapsed; + }; + std::vector vocab_entries; + vocab_entries.reserve(custom_vocabulary.size()); + for (const auto& v : custom_vocabulary) { + vocab_entries.push_back({&v, collapse_spaces(v)}); + } + + struct Token { std::string text; bool is_word; }; + std::vector tokens; + size_t pos = 0; + while (pos < text.size()) { + if (std::isalnum(static_cast(text[pos])) || + text[pos] == '\'' || text[pos] == '-') { + size_t start = pos; + while (pos < text.size() && (std::isalnum(static_cast(text[pos])) || + text[pos] == '\'' || text[pos] == '-')) { + ++pos; + } + tokens.push_back({text.substr(start, pos - start), true}); + } else { + size_t start = pos; + while (pos < text.size() && !std::isalnum(static_cast(text[pos])) && + text[pos] != '\'' && text[pos] != '-') { + ++pos; + } + tokens.push_back({text.substr(start, pos - start), false}); + } + } + + std::vector word_indices; + for (size_t i = 0; i < tokens.size(); ++i) { + if (tokens[i].is_word) word_indices.push_back(i); + } + + std::vector consumed(tokens.size(), false); + + auto strip_suffix = [](const std::string& word) -> std::pair { + if (word.size() >= 3 && word.substr(word.size() - 2) == "'s") { + return {word.substr(0, word.size() - 2), "'s"}; + } + if (word.size() >= 3 && word.substr(word.size() - 2) == "'t") { + return {word.substr(0, word.size() - 2), "'t"}; + } + if (word.size() >= 4 && word.back() == 's' && + word[word.size() - 2] != 's' && // avoid stripping from "boss", "class" + std::isalpha(static_cast(word[word.size() - 2]))) { + return {word.substr(0, word.size() - 1), "s"}; + } + return {word, ""}; + }; + + size_t wi = 0; + while (wi < word_indices.size()) { + size_t best_dist = std::numeric_limits::max(); + const std::string* best_match = nullptr; + size_t best_window = 0; + size_t best_first_token = 0; + size_t best_last_token = 0; + std::string best_suffix; + + for (size_t window = std::min(3, word_indices.size() - wi); window >= 1; --window) { + std::string window_collapsed; + const size_t first_tok = word_indices[wi]; + const size_t last_tok = word_indices[wi + window - 1]; + for (size_t w = 0; w < window; ++w) { + window_collapsed += tokens[word_indices[wi + w]].text; + } + + if (window == 1 && window_collapsed.size() < 3) break; + + auto [stem, suffix] = strip_suffix(window_collapsed); + const std::string* candidates[] = {&window_collapsed, &stem}; + const std::string suffixes[] = {"", suffix}; + const size_t num_candidates = suffix.empty() ? 1 : 2; + + for (size_t ci = 0; ci < num_candidates; ++ci) { + const std::string& candidate = *candidates[ci]; + if (candidate.empty()) continue; + + for (const auto& entry : vocab_entries) { + const size_t wlen = candidate.size(); + const size_t vlen = entry.collapsed.size(); + + const size_t len_diff = wlen > vlen ? wlen - vlen : vlen - wlen; + const size_t max_dist = std::max(1, std::min(wlen, vlen) / 3); + if (len_diff > max_dist) continue; + + const size_t dist = levenshtein_ci(candidate, entry.collapsed); + + // For single-edit corrections, require first char match to prevent + // false positives like "vortex" → "Cortex". + if (dist == 1 && window == 1) { + const bool first_char_match = + std::tolower(static_cast(candidate[0])) == + std::tolower(static_cast(entry.collapsed[0])); + if (!first_char_match) continue; + } + + if (dist <= max_dist && dist < best_dist) { + best_dist = dist; + best_match = entry.original; + best_window = window; + best_first_token = first_tok; + best_last_token = last_tok; + best_suffix = suffixes[ci]; + } + } + } + + if (best_dist == 0) break; + } + + // Allow dist==0 for multi-word merges where word boundaries changed. + const bool should_replace = best_match && + best_dist != std::numeric_limits::max() && + (best_dist > 0 || best_window > 1); + + if (should_replace) { + tokens[best_first_token].text = *best_match + best_suffix; + for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { + consumed[t] = true; + } + for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { + if (t > 0) consumed[t - 1] = consumed[t - 1] || !tokens[t - 1].is_word; + } + wi += best_window; + } else { + ++wi; + } + } + + std::string result; + result.reserve(text.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + if (!consumed[i]) { + result += tokens[i].text; + } + } + + text = std::move(result); +} + +inline InferenceOptions parse_inference_options_json(const std::string& json) { + InferenceOptions options; + + if (json.empty()) return options; + + size_t pos = json.find("\"temperature\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.temperature = std::stof(json.substr(pos)); + } + + pos = json.find("\"top_p\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.top_p = std::stof(json.substr(pos)); + } + + float parsed_min_p = options.min_p; + if (try_parse_json_float(json, "min_p", parsed_min_p)) { + options.min_p = std::clamp(parsed_min_p, 0.0f, 1.0f); + } + + float parsed_rep_penalty = options.repetition_penalty; + if (try_parse_json_float(json, "repetition_penalty", parsed_rep_penalty)) { + if (std::isfinite(parsed_rep_penalty) && parsed_rep_penalty > 0.0f) { + options.repetition_penalty = parsed_rep_penalty; + } + } + + pos = json.find("\"top_k\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.top_k = std::stoul(json.substr(pos)); + } + + pos = json.find("\"max_tokens\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.max_tokens = std::stoul(json.substr(pos)); + } + + pos = json.find("\"force_tools\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.force_tools = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"tool_rag_top_k\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.tool_rag_top_k = std::stoul(json.substr(pos)); + } + + pos = json.find("\"confidence_threshold\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.confidence_threshold = std::stof(json.substr(pos)); + } + + pos = json.find("\"include_stop_sequences\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.include_stop_sequences = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"use_vad\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.use_vad = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"telemetry_enabled\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.telemetry_enabled = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"auto_handoff\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.auto_handoff = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"cloud_timeout_ms\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.cloud_timeout_ms = std::stoul(json.substr(pos)); + } + + pos = json.find("\"handoff_with_images\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.handoff_with_images = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"enable_thinking_if_supported\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.enable_thinking_if_supported = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"stop_sequences\""); + if (pos != std::string::npos) { + pos = json.find('[', pos); + if (pos != std::string::npos) { + size_t end_pos = json.find(']', pos); + size_t seq_pos = json.find('"', pos); + + while (seq_pos != std::string::npos && seq_pos < end_pos) { + size_t seq_start = seq_pos + 1; + size_t seq_end = json.find('"', seq_start); + if (seq_end != std::string::npos) { + options.stop_sequences.push_back(json.substr(seq_start, seq_end - seq_start)); + } + seq_pos = json.find('"', seq_end + 1); + } + } + } + + return options; +} + +static inline std::string trim_lfm2_slice(const std::string& value, size_t begin, size_t end) { + return trim_string(value.substr(begin, end - begin)); +} + +static inline void append_lfm2_call(const std::string& entry, + std::vector& function_calls) { + if (entry.empty()) return; + + std::string trimmed_entry = trim_lfm2_slice(entry, 0, entry.size()); + if (trimmed_entry.empty()) return; + + size_t paren_pos = trimmed_entry.find('('); + if (paren_pos == std::string::npos) return; + + std::string func_name = trim_lfm2_slice(trimmed_entry, 0, paren_pos); + std::string args_str = trim_lfm2_slice(trimmed_entry, paren_pos + 1, trimmed_entry.size()); + + while (!args_str.empty() && (args_str.back() == ')' || args_str.back() == ']')) { + args_str.pop_back(); + } + args_str = trim_lfm2_slice(args_str, 0, args_str.size()); + + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":{"; + + size_t arg_pos = 0; + bool first_arg = true; + while (arg_pos < args_str.length()) { + while (arg_pos < args_str.length() && std::isspace(static_cast(args_str[arg_pos]))) { + arg_pos++; + } + + size_t eq_pos = args_str.find('=', arg_pos); + if (eq_pos == std::string::npos) break; + + std::string arg_name = args_str.substr(arg_pos, eq_pos - arg_pos); + + size_t val_start = eq_pos + 1; + size_t val_end = val_start; + bool quoted = false; + + if (val_start < args_str.length() && args_str[val_start] == '"') { + quoted = true; + val_start++; + val_end = args_str.find('"', val_start); + if (val_end == std::string::npos) break; + } else { + int depth = 0; + for (val_end = val_start; val_end < args_str.length(); val_end++) { + char c = args_str[val_end]; + if (c == '[' || c == '{') depth++; + else if (c == ']' || c == '}') depth--; + else if (c == ',' && depth == 0) break; + } + } + + std::string arg_value = args_str.substr(val_start, val_end - val_start); + + if (!quoted) { + if (arg_value == "True") arg_value = "true"; + else if (arg_value == "False") arg_value = "false"; + else if (arg_value == "None") arg_value = "null"; + } + + if (!first_arg) json_call += ","; + json_call += "\"" + arg_name + "\":"; + json_call += quoted ? ("\"" + arg_value + "\"") : arg_value; + first_arg = false; + + arg_pos = args_str.find(',', val_end); + if (arg_pos != std::string::npos) { + arg_pos++; + } else { + break; + } + } + + json_call += "}}"; + function_calls.push_back(json_call); +} + +inline void parse_function_calls_from_response(const std::string& response_text, + std::string& regular_response, + std::vector& function_calls) { + regular_response = response_text; + function_calls.clear(); + + gemma::parse_function_calls(regular_response, function_calls); + + const std::string QWEN_TOOL_START = ""; + const std::string QWEN_TOOL_END = ""; + size_t qwen_start_pos = 0; + + while ((qwen_start_pos = regular_response.find(QWEN_TOOL_START, qwen_start_pos)) != std::string::npos) { + size_t content_start = qwen_start_pos + QWEN_TOOL_START.length(); + size_t qwen_end_pos = regular_response.find(QWEN_TOOL_END, content_start); + + size_t erase_end; + std::string json_content; + + if (qwen_end_pos != std::string::npos) { + json_content = regular_response.substr(content_start, qwen_end_pos - content_start); + erase_end = qwen_end_pos + QWEN_TOOL_END.length(); + } else { + json_content = regular_response.substr(content_start); + erase_end = regular_response.length(); + } + + size_t first = json_content.find_first_not_of(" \t\n\r"); + size_t last = json_content.find_last_not_of(" \t\n\r"); + if (first != std::string::npos && last != std::string::npos) { + json_content = json_content.substr(first, last - first + 1); + } + + if (json_content.size() > 2 && json_content.find("\"name\"") != std::string::npos) { + // Unwrap array wrapper if present: [{"name":...}] -> {"name":...} + if (json_content[0] == '[') { + size_t obj_start = json_content.find('{'); + size_t obj_end = json_content.rfind('}'); + if (obj_start != std::string::npos && obj_end != std::string::npos && obj_end > obj_start) { + json_content = json_content.substr(obj_start, obj_end - obj_start + 1); + } + } + if (json_content[0] == '{') { + size_t depth = 0; + bool in_string = false; + bool escaped = false; + size_t end_pos = 0; + for (size_t c = 0; c < json_content.size(); c++) { + char ch = json_content[c]; + if (escaped) { escaped = false; continue; } + if (ch == '\\' && in_string) { escaped = true; continue; } + if (ch == '"') { in_string = !in_string; continue; } + if (!in_string) { + if (ch == '{') depth++; + else if (ch == '}' && --depth == 0) { end_pos = c + 1; break; } + } + } + if (end_pos > 0) { + function_calls.push_back(json_content.substr(0, end_pos)); + } + } + } + + regular_response.erase(qwen_start_pos, erase_end - qwen_start_pos); + } + + const std::string TOOL_CALL_START = "<|tool_call_start|>"; + const std::string TOOL_CALL_END = "<|tool_call_end|>"; + size_t lfm2_start_pos = 0; + + while ((lfm2_start_pos = regular_response.find(TOOL_CALL_START, lfm2_start_pos)) != std::string::npos) { + size_t content_start = lfm2_start_pos + TOOL_CALL_START.length(); + size_t tool_end_pos = regular_response.find(TOOL_CALL_END, content_start); + + if (tool_end_pos != std::string::npos) { + std::string tool_content = regular_response.substr(content_start, tool_end_pos - content_start); + std::string content = tool_content; + size_t trim_start = 0; + while (trim_start < content.size() && std::isspace(static_cast(content[trim_start]))) { + trim_start++; + } + + if (trim_start < content.size()) { + size_t trim_end = content.size() - 1; + while (trim_end > trim_start && std::isspace(static_cast(content[trim_end]))) { + trim_end--; + } + content = content.substr(trim_start, trim_end - trim_start + 1); + } else { + content.clear(); + } + + if (!content.empty() && content.front() == '[' && content.back() == ']') { + std::string inner = content.substr(1, content.size() - 2); + + size_t inner_first = inner.find_first_not_of(" \t\n\r"); + if (inner_first != std::string::npos && inner[inner_first] == '{') { + size_t pos = inner_first; + while (pos < inner.size()) { + if (inner[pos] == '{') { + int brace_depth = 1; + size_t obj_start = pos; + pos++; + while (pos < inner.size() && brace_depth > 0) { + if (inner[pos] == '{') brace_depth++; + else if (inner[pos] == '}') brace_depth--; + pos++; + } + if (brace_depth == 0) { + std::string json_obj = inner.substr(obj_start, pos - obj_start); + if (json_obj.find("\"name\"") != std::string::npos) { + function_calls.push_back(json_obj); + } + } + } else { + pos++; + } + } + } else { + size_t start = 0; + int paren_depth = 0; + + for (size_t i = 0; i < inner.size(); ++i) { + char c = inner[i]; + if (c == '(') { + paren_depth++; + } else if (c == ')' && paren_depth > 0) { + paren_depth--; + } else if (c == ',' && paren_depth == 0) { + append_lfm2_call(inner.substr(start, i - start), function_calls); + start = i + 1; + } + } + + if (start < inner.size()) { + append_lfm2_call(inner.substr(start), function_calls); + } + } + } else if (!content.empty()) { + append_lfm2_call(content, function_calls); + } + + regular_response.erase(lfm2_start_pos, tool_end_pos + TOOL_CALL_END.length() - lfm2_start_pos); + } else { + break; + } + } + + const char* FUNCTION_CALL_MARKER = "\"function_call\""; + size_t search_pos = 0; + const size_t text_len = regular_response.length(); + + while (search_pos < text_len) { + size_t marker_pos = regular_response.find(FUNCTION_CALL_MARKER, search_pos); + if (marker_pos == std::string::npos) break; + + size_t json_start = regular_response.find('{', marker_pos); + if (json_start == std::string::npos) break; + + int brace_count = 1; + size_t json_end = json_start + 1; + while (json_end < text_len && brace_count > 0) { + char c = regular_response[json_end]; + brace_count += (c == '{') - (c == '}'); + json_end++; + } + + if (brace_count == 0) { + function_calls.push_back(regular_response.substr(json_start, json_end - json_start)); + regular_response = regular_response.substr(0, marker_pos); + size_t last_bracket = regular_response.rfind('{'); + if(last_bracket != std::string::npos) { + regular_response = regular_response.substr(0, last_bracket); + } + } + search_pos = json_end; + } +} + +inline std::vector> find_channel_token_ranges( + const std::vector& tokens, size_t offset, + uint32_t channel_open_id, uint32_t channel_close_id) { + std::vector> ranges; + size_t pos = 0; + while (pos < tokens.size()) { + if (tokens[pos] != channel_open_id) { + pos++; + continue; + } + + size_t block_start = pos; + pos++; + while (pos < tokens.size() && tokens[pos] != channel_close_id) { + pos++; + } + if (pos < tokens.size()) { + pos++; + } + ranges.push_back({offset + block_start, pos - block_start}); + } + return ranges; +} + +inline void strip_tag_blocks(std::string& text, std::string& extracted, + const std::string& open_tag, const std::string& close_tag) { + std::string result; + size_t pos = 0; + + size_t first_close = text.find(close_tag); + size_t first_open = text.find(open_tag); + if (first_close != std::string::npos && + (first_open == std::string::npos || first_close < first_open)) { + extracted += text.substr(0, first_close); + pos = first_close + close_tag.size(); + } + + while (pos < text.size()) { + size_t open_pos = text.find(open_tag, pos); + if (open_pos == std::string::npos) { + result += text.substr(pos); + break; + } + result += text.substr(pos, open_pos - pos); + size_t content_start = open_pos + open_tag.size(); + size_t close_pos = text.find(close_tag, content_start); + if (close_pos == std::string::npos) { + if (!extracted.empty()) extracted += "\n"; + extracted += text.substr(content_start); + break; + } + if (!extracted.empty()) extracted += "\n"; + extracted += text.substr(content_start, close_pos - content_start); + pos = close_pos + close_tag.size(); + } + text = result; +} + +inline void strip_thinking_block(const std::string& input, std::string& thinking, std::string& content) { + thinking.clear(); + content = input; + + auto trim = [](std::string& s) { + size_t first = s.find_first_not_of(" \t\n\r"); + size_t last = s.find_last_not_of(" \t\n\r"); + if (first != std::string::npos && last != std::string::npos) + s = s.substr(first, last - first + 1); + else + s.clear(); + }; + + if (content.find("<|channel>") != std::string::npos || content.find("") != std::string::npos) { + strip_tag_blocks(content, thinking, "<|channel>", ""); + } else if (content.find("") != std::string::npos || content.find("") != std::string::npos) { + strip_tag_blocks(content, thinking, "", ""); + } else { + return; + } + + trim(thinking); + trim(content); +} + +struct TranscriptSegment { + float start; + float end; + std::string text; +}; + +inline std::string construct_response_json(const std::string& regular_response, + const std::vector& function_calls, + double time_to_first_token, + double total_time_ms, + double prefill_tps, + double decode_tps, + size_t prompt_tokens, + size_t completion_tokens, + float confidence = 0.0f, + bool cloud_handoff = false, + const std::string& thinking = "", + const std::vector& segments = {}) { + std::ostringstream json; + json << "{"; + json << "\"success\":true,"; + json << "\"error\":null,"; + json << "\"cloud_handoff\":" << (cloud_handoff ? "true" : "false") << ","; + json << "\"response\":\"" << escape_json_string(regular_response) << "\","; + if (!thinking.empty()) { + json << "\"thinking\":\"" << escape_json_string(thinking) << "\","; + } + json << "\"function_calls\":["; + for (size_t i = 0; i < function_calls.size(); ++i) { + if (i > 0) json << ","; + json << function_calls[i]; + } + json << "],"; + json << "\"segments\":["; + for (size_t i = 0; i < segments.size(); ++i) { + if (i > 0) json << ","; + json << "{\"start\":" << std::fixed << std::setprecision(3) << segments[i].start + << ",\"end\":" << std::fixed << std::setprecision(3) << segments[i].end + << ",\"text\":\"" << escape_json_string(segments[i].text) << "\"}"; + } + json << "],"; + json << "\"confidence\":" << std::fixed << std::setprecision(4) << confidence << ","; + json << "\"time_to_first_token_ms\":" << std::fixed << std::setprecision(2) << time_to_first_token << ","; + json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; + json << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << ","; + json << "\"decode_tps\":" << std::fixed << std::setprecision(2) << decode_tps << ","; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; + json << "\"prefill_tokens\":" << prompt_tokens << ","; + json << "\"decode_tokens\":" << completion_tokens << ","; + json << "\"total_tokens\":" << (prompt_tokens + completion_tokens); + json << "}"; + return json.str(); +} + +inline std::string serialize_function_calls(const std::vector& calls) { + if (calls.empty()) return "[]"; + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < calls.size(); ++i) { + if (i > 0) oss << ","; + oss << calls[i]; + } + oss << "]"; + return oss.str(); +} + +inline int validate_audio_params( + const char* component, + void* model, + char* response_buffer, size_t buffer_size, + const char* audio_file_path, + const uint8_t* pcm_buffer, size_t pcm_buffer_size) { + if (!model) { + std::string err = last_error_message.empty() ? "Model not initialized." : last_error_message; + CACTUS_LOG_ERROR(component, err); + handle_error_response(err, response_buffer, buffer_size); + return -1; + } + if (!response_buffer || buffer_size == 0) { + CACTUS_LOG_ERROR(component, "Invalid parameters: response_buffer or buffer_size"); + handle_error_response("Invalid parameters", response_buffer, buffer_size); + return -1; + } + if (!audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { + CACTUS_LOG_ERROR(component, "No audio input provided"); + handle_error_response("Either audio_file_path or pcm_buffer must be provided", response_buffer, buffer_size); + return -1; + } + if (audio_file_path && pcm_buffer && pcm_buffer_size > 0) { + CACTUS_LOG_ERROR(component, "Both audio_file_path and pcm_buffer provided"); + handle_error_response("Cannot provide both audio_file_path and pcm_buffer", response_buffer, buffer_size); + return -1; + } + if (pcm_buffer && pcm_buffer_size > 0 && (pcm_buffer_size < 2 || pcm_buffer_size % 2 != 0)) { + CACTUS_LOG_ERROR(component, "Invalid pcm_buffer_size"); + handle_error_response("pcm_buffer_size must be even and at least 2 bytes", response_buffer, buffer_size); + return -1; + } + return 0; +} + +inline std::vector pcm_to_float(const uint8_t* pcm_buffer, size_t pcm_buffer_size) { + const int16_t* samples = reinterpret_cast(pcm_buffer); + size_t n = pcm_buffer_size / 2; + std::vector out(n); + for (size_t i = 0; i < n; ++i) + out[i] = static_cast(samples[i]) / 32768.0f; + return out; +} + +} // namespace ffi +} // namespace cactus + +#ifdef __cplusplus +extern "C" { +#endif + +const char* cactus_get_last_error(); + +#ifdef __cplusplus +} +#endif + +#endif // CACTUS_UTILS_H diff --git a/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/engine.h b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/engine.h new file mode 100644 index 0000000..afebfeb --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/engine.h @@ -0,0 +1,1104 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../graph/graph.h" + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc99-extensions" +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +extern "C" { + #include "../../libs/stb/stb_image.h" + #include "../../libs/stb/stb_image_resize2.h" +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + +class CactusGraph; + +namespace cactus { +namespace npu { + class NPUPrefill; +} +namespace engine { + +class Siglip2Preprocessor; + +struct Config { + uint32_t vocab_size = 151936; + uint32_t bos_token_id = 151643; + uint32_t eos_token_id = 151645; + uint32_t num_layers = 28; + uint32_t hidden_dim = 1024; + uint32_t ffn_intermediate_dim = 3072; + uint32_t attention_heads = 16; + uint32_t attention_kv_heads = 8; + uint32_t attention_head_dim = 128; + float layer_norm_eps = 1e-6f; + float rope_theta = 1000000.0f; + uint32_t num_experts = 0; + uint32_t num_shared_experts = 0; + uint32_t num_top_experts = 0; + uint32_t moe_every_n_layers = 0; + uint32_t moe_intermediate_dim = 0; + uint32_t num_dense_layers = 0; + uint32_t num_experts_per_tok = 0; + bool norm_topk_prob = false; + bool use_expert_bias = false; + float routed_scaling_factor = 1.0f; + bool tie_word_embeddings = true; + + uint32_t vision_hidden_dim = 0; + uint32_t vision_num_layers = 0; + uint32_t vision_attention_heads = 0; + uint32_t vision_image_size = 0; + uint32_t vision_patch_size = 0; + uint32_t vision_num_channels = 3; + uint32_t vision_embed_dim = 0; + uint32_t visual_tokens_per_img = 0; + bool use_pixel_shuffle = false; + uint32_t pixel_shuffle_factor = 1; + bool use_image_tokens = false; + uint32_t image_token_id = 0; + bool use_layout_tags = false; + uint32_t image_seq_len = 64; + + uint32_t global_image_size = 2048; + uint32_t max_tile_size = 512; + float rescale_factor = 0.00392156862745098f; + float image_mean = 0.5f; + float image_std = 0.5f; + + uint32_t downsample_factor = 2; + uint32_t min_tiles = 2; + uint32_t max_tiles = 10; + bool use_thumbnail = true; + uint32_t min_image_tokens = 64; + uint32_t max_image_tokens = 256; + uint32_t max_num_patches = 1024; + uint32_t tile_size = 512; + float max_pixels_tolerance = 2.0f; + bool do_image_splitting = true; + bool encoder_act_gelu = false; + bool decoder_act_gelu = false; + uint32_t num_encoder_layers = 0; + uint32_t num_decoder_layers = 0; + float partial_rotary_factor = 0.0f; + uint32_t pad_token_id = 0; + uint32_t conv_kernel_size = 0; + uint32_t subsampling_conv_kernel_size = 0; + uint32_t subsampling_conv_stride = 0; + uint32_t subsampling_conv_channels = 0; + uint32_t subsampling_factor = 0; + uint32_t num_mel_bins = 80; + std::string encoder_hidden_act = "silu"; + uint32_t linear_num_key_heads = 0; + uint32_t linear_key_head_dim = 0; + uint32_t linear_num_value_heads = 0; + uint32_t linear_value_head_dim = 0; + uint32_t linear_q_proj_dim = 0; + uint32_t linear_k_proj_dim = 0; + uint32_t linear_v_proj_dim = 0; + + uint32_t kv_lora_rank = 0; + uint32_t q_lora_rank = 0; + uint32_t qk_head_dim = 0; + uint32_t qk_nope_head_dim = 0; + uint32_t qk_rope_head_dim = 0; + uint32_t v_head_dim = 0; + uint32_t rope_interleave = 0; + bool attention_bias = false; + float rope_scaling_factor = 1.0f; + float rope_mscale_all_dim = 0.0f; + + enum class ModelType {QWEN = 0, GEMMA = 1, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7, MOONSHINE = 8, SILERO_VAD = 9, PARAKEET = 10, QWEN3P5 = 11, PARAKEET_TDT = 12, GEMMA3N = 13, YOUTU = 14, GEMMA4 = 15, PYANNOTE = 16, WESPEAKER = 17, NEEDLE = 18}; + uint32_t predictor_hidden_dim = 0; + uint32_t predictor_num_layers = 0; + uint32_t tdt_joint_dim = 0; + uint32_t tdt_num_durations = 0; + uint32_t tdt_blank_id = 0; + std::vector tdt_durations; + + ModelType model_type = ModelType::QWEN; + + enum class ModelVariant {DEFAULT = 0, VLM = 1, EXTRACT = 2, RAG = 3}; + ModelVariant model_variant = ModelVariant::DEFAULT; + + enum class Activation {GELU = 0, SILU = 1}; + Activation activation = Activation::SILU; + + enum class Backend {CPU = 0, NPU = 1}; + Backend default_backend = Backend::CPU; + + enum class Precision {INT8 = 0, FP16 = 1, FP32 = 2}; + Precision precision = Precision::FP32; + + float default_temperature = 0.6f; + float default_top_p = 0.95f; + size_t default_top_k = 20; + float default_max_tps = -1.0f; + float default_cloud_handoff_threshold = 0.0f; + size_t default_rolling_entropy_window = 10; + + std::vector layer_types; + size_t conv_L_cache = 0; + + uint32_t altup_num_inputs = 4; + uint32_t laurel_rank = 64; + uint32_t hidden_size_per_layer_input = 256; + uint32_t num_kv_shared_layers = 0; + uint32_t sliding_window = 512; + float rope_local_base_freq = 10000.0f; + float final_logit_softcapping = 0.0f; + float global_partial_rotary_factor = 1.0f; + uint32_t expert_intermediate_size = 0; + uint32_t global_head_dim = 0; + uint32_t num_global_kv_heads = 0; + bool attention_k_eq_v = false; + bool enable_moe_block = false; + std::vector activation_sparsity_ppf; + + uint32_t vision_head_dim = 64; + uint32_t vision_kv_heads = 12; + uint32_t vision_intermediate_size = 3072; + uint32_t vision_position_embedding_size = 10240; + uint32_t vision_pooling_kernel_size = 3; + uint32_t vision_default_output_length = 280; + float vision_rope_theta = 100.0f; + + uint32_t audio_hidden_dim = 0; + uint32_t audio_num_layers = 0; + uint32_t audio_num_heads = 0; + uint32_t audio_head_dim = 0; + uint32_t audio_input_feat_size = 128; + uint32_t audio_conf_conv_kernel_size = 5; + uint32_t audio_chunk_size = 12; + uint32_t audio_context_left = 13; + uint32_t audio_context_right = 0; + float audio_logit_cap = 50.0f; + float audio_residual_weight = 0.5f; + uint32_t audio_output_proj_dims = 0; + uint32_t audio_vocab_size = 128; + uint32_t audio_vocab_offset = 0; + uint32_t audio_soft_tokens = 188; + uint32_t audio_sscp_conv0_channels = 128; + uint32_t audio_sscp_conv1_channels = 32; + float audio_sscp_conv_eps = 1e-3f; + float audio_rms_norm_eps = 1e-6f; + uint32_t audio_fft_length = 1024; + uint32_t audio_token_id = 0; + bool audio_fft_overdrive = false; + uint32_t channel_open_token_id = 100; + uint32_t channel_close_token_id = 101; + + static bool is_gemma_family(ModelType t) { + return t == ModelType::GEMMA || t == ModelType::GEMMA3N || t == ModelType::GEMMA4; + } + + bool from_json(const std::string& json_path); + std::string to_json() const; +}; + + + +struct MergeRule { + std::string first; + std::string second; + std::string merged; + uint32_t priority; + + MergeRule(const std::string& f, const std::string& s, const std::string& m, uint32_t p) + : first(f), second(s), merged(m), priority(p) {} +}; + + +struct ToolCallInfo { + std::string name; + std::string arguments; +}; + +struct ChatMessage { + std::string role; + std::string content; + std::string name; + std::vector images; + std::vector audio; + size_t audio_soft_token_count = 0; + std::vector tool_calls; +}; + +inline std::string format_needle_query_text(const std::vector& messages) { + std::string system_text; + std::string user_query; + + for (const auto& msg : messages) { + if (msg.role == "system") { + if (!system_text.empty()) { + system_text += "\n"; + } + system_text += msg.content; + } else if (msg.role == "user") { + user_query = msg.content; + } + } + + if (user_query.empty() && !messages.empty()) { + user_query = messages.back().content; + } + if (system_text.empty()) { + return user_query; + } + if (user_query.empty()) { + return system_text; + } + return system_text + "\n\n" + user_query; +} + +struct ToolConstraintSpec { + std::string name; + std::vector parameter_names; + std::vector required_parameter_names; +}; + +struct TokenizerRuntimeConfig { + enum class TokenizerType { UNKNOWN, BPE, SENTENCEPIECE }; + enum class VocabFormat { UNKNOWN, ID_TAB_TOKEN, LINE_TOKEN }; + enum class Normalizer { NONE, METASPACE, BYTE_LEVEL }; + enum class Decoder { NONE, REPLACE_METASPACE, BYTE_LEVEL }; + + TokenizerType tokenizer_type = TokenizerType::UNKNOWN; + VocabFormat vocab_format = VocabFormat::UNKNOWN; + Normalizer normalizer = Normalizer::NONE; + Decoder decoder = Decoder::NONE; + bool byte_fallback = false; + bool has_chat_template = false; +}; + +TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file); +void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens); +std::vector split_with_special_tokens(const std::string& text, const std::unordered_map& special_tokens); + +inline std::string extract_json_string(const std::string& json, size_t& pos) { + std::string value; + while (pos < json.size() && json[pos] != '"') { + if (json[pos] == '\\' && pos + 1 < json.size()) { + pos++; + if (json[pos] == 'n') value += '\n'; + else if (json[pos] == 't') value += '\t'; + else if (json[pos] == 'r') value += '\r'; + else if (json[pos] == '"') value += '"'; + else if (json[pos] == '\\') value += '\\'; + else value += json[pos]; + } else { + value += json[pos]; + } + pos++; + } + if (pos < json.size()) pos++; + return value; +} + +class Tokenizer { +public: + virtual ~Tokenizer() = default; + + virtual std::vector encode(const std::string& text) const = 0; + virtual std::string decode(const std::vector& tokens) const = 0; + + virtual std::vector apply_chat_template(const std::vector& messages, bool add_generation_prompt = true) const; + virtual std::string format_chat_prompt(const std::vector& messages, bool add_generation_prompt = true, const std::string& tools_json = "", bool enable_thinking_if_supported = false) const; + + virtual uint32_t get_vocab_size() const = 0; + virtual uint32_t get_unk_token() const = 0; + virtual uint32_t get_bos_token() const = 0; + virtual uint32_t get_eos_token() const = 0; + virtual bool has_chat_template() const { return has_chat_template_; } + std::string get_default_stop_sequence() const; + + virtual bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) = 0; + + uint32_t get_image_token_id() const { return image_token_id_; } + uint32_t get_fake_token_id() const { return fake_token_id_; } + uint32_t get_global_img_token_id() const { return global_img_token_id_; } + +protected: + enum class ModelType { UNKNOWN, QWEN, QWEN3P5, GEMMA, GEMMA4, LFM2, BERT, WHISPER, PARAKEET, YOUTU, NEEDLE}; + ModelType model_type_ = ModelType::UNKNOWN; + enum class ModelVariant { DEFAULT, VLM, EXTRACT, RAG}; + ModelVariant model_variant_ = ModelVariant::DEFAULT; + bool has_chat_template_ = false; + std::string chat_template_; + + uint32_t image_token_id_ = 396; + uint32_t fake_token_id_ = 49189; + uint32_t global_img_token_id_ = 49152; + + + uint32_t vision_patch_size_ = 16; + uint32_t vision_pooling_kernel_size_ = 3; + uint32_t vision_default_output_length_ = 280; + uint32_t vision_image_size_ = 768; + TokenizerRuntimeConfig runtime_config_; + + void detect_model_type(const std::string& config_path); + void load_chat_template(const std::string& template_file); + std::string format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; + std::string format_gemma_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_gemma4_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; + std::string format_lfm2_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_lfm2_vl_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_needle_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_youtu_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; +}; + +class BPETokenizer : public Tokenizer { +public: + BPETokenizer(); + ~BPETokenizer(); + + bool load_vocabulary_mmap(const std::string& vocab_file, const std::string& merges_file); + bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; + + std::vector encode(const std::string& text) const override; + std::string decode(const std::vector& tokens) const override; + + uint32_t get_vocab_size() const override { return vocab_size_; } + uint32_t get_unk_token() const override { return unk_token_id_; } + uint32_t get_bos_token() const override { return bos_token_id_; } + uint32_t get_eos_token() const override { return eos_token_id_; } + +private: + std::unordered_map token_to_id_; + std::vector id_to_token_; + std::vector merge_rules_; + std::unordered_map merge_map_; + + uint32_t vocab_size_; + uint32_t unk_token_id_; + uint32_t bos_token_id_; + uint32_t eos_token_id_; + + void* vocab_mmap_ptr_; + size_t vocab_mmap_size_; + + void* merges_mmap_ptr_; + size_t merges_mmap_size_; + + std::vector apply_bpe(const std::vector& tokens) const; + std::pair find_best_merge_fast(const std::vector& tokens) const; + + std::string bytes_to_unicode(const std::string& text) const; + std::string unicode_to_bytes(const std::string& text) const; + std::vector byte_level_split(const std::string& text) const; + std::vector utf8_split(const std::string& text) const; + + void cleanup_mmap(); + +private: + mutable std::unordered_map byte_to_unicode_; + mutable std::unordered_map unicode_to_byte_; + void init_byte_mappings() const; + + std::unordered_map special_tokens_; + std::vector split_with_special_tokens(const std::string& text) const; + void load_special_tokens(const std::string& config_file); +}; + +class SPTokenizer : public Tokenizer { +public: + SPTokenizer(); + ~SPTokenizer(); + + bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; + + std::vector encode(const std::string& text) const override; + std::string decode(const std::vector& tokens) const override; + + uint32_t get_vocab_size() const override { return vocab_size_; } + uint32_t get_unk_token() const override { return unk_token_id_; } + uint32_t get_bos_token() const override { return bos_token_id_; } + uint32_t get_eos_token() const override { return eos_token_id_; } + +private: + struct TrieNode { + std::unordered_map> children; + int32_t token_id = -1; + float score = 0.0f; + }; + + std::unique_ptr trie_root_; + std::unordered_map token_to_id_; + std::vector id_to_token_; + std::vector token_scores_; + + uint32_t vocab_size_; + uint32_t unk_token_id_; + uint32_t bos_token_id_; + uint32_t eos_token_id_; + uint32_t pad_token_id_; + + bool sp_bpe_mode_ = false; + bool sp_add_dummy_prefix_ = false; + bool sp_byte_fallback_ = false; + + void* vocab_mmap_ptr_; + size_t vocab_mmap_size_; + + void build_trie(); + std::vector> tokenize_with_trie(const std::string& text) const; + std::vector tokenize_with_bpe(const std::string& text) const; + std::string preprocess_text(const std::string& text) const; + std::string postprocess_text(const std::string& text) const; + std::vector split_by_unicode_spaces(const std::string& text) const; + + void cleanup_mmap(); + + std::unordered_map special_tokens_; + std::vector split_with_special_tokens(const std::string& text) const; + void load_special_tokens(const std::string& config_file); +}; + +class ConvCache { +public: + struct CircularView { + const void* ptr1; + size_t len1; + const void* ptr2; + size_t len2; + size_t total_len; + }; + + void init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision); + CircularView get_window(size_t layer) const; + void update(CactusGraph* gb, size_t layer, const size_t latest_token); + void reset(); + + bool is_empty() const { return num_layers == 0; } + + size_t num_layers = 0; + size_t hidden_size = 0; + size_t window_size = 0; + Precision precision = Precision::FP32; + size_t element_size = 4; + +private: + struct LayerState { + std::vector data; + size_t head = 0; + size_t count = 0; + }; + + std::vector layer_states; +}; + +struct KVCache { + static constexpr size_t DEFAULT_WINDOW_SIZE = 1024; + static constexpr size_t DEFAULT_SINK_SIZE = 4; + + struct LayerCache { + std::vector keys; + std::vector values; + std::vector key_scales; + std::vector value_scales; + size_t head_dim = 0; + size_t kv_heads = 0; + }; + + std::vector layer_caches; + + size_t window_size = DEFAULT_WINDOW_SIZE; + size_t sink_size = DEFAULT_SINK_SIZE; + size_t current_seq_len = 0; + size_t total_seq_len = 0; + size_t max_seq_len = 2048; + size_t num_layers = 0; + Precision precision; + size_t element_size = 4; + + void set_window_size(size_t window, size_t sink = DEFAULT_SINK_SIZE); + size_t get_effective_seq_len() const { return current_seq_len; } + size_t get_total_seq_len() const { return total_seq_len; } + size_t get_layer_head_dim(size_t layer_idx) const { return layer_caches[layer_idx].head_dim; } + size_t get_layer_kv_heads(size_t layer_idx) const { return layer_caches[layer_idx].kv_heads; } + + void init(size_t num_layers, size_t max_seq, const std::vector& layer_dims, const std::vector& layer_kv_heads, Precision model_precision); + void reset(); + void update_from_graph(CactusGraph* gb, const std::vector& k_nodes, + const std::vector& v_nodes, size_t seq_len, + size_t num_layers); + + void update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data, + size_t num_tokens, size_t kv_heads, size_t head_dim); + + bool is_empty() const { return current_seq_len == 0; } + void* get_key_ptr(size_t layer); + void* get_value_ptr(size_t layer); + + struct CircularView { + const void* ptr1; + const void* ptr2; + size_t len1; + size_t len2; + size_t total_len; + }; + + CircularView get_key_view(size_t layer); + CircularView get_value_view(size_t layer); + + const int8_t* get_keys_int8(size_t layer) const; + const int8_t* get_values_int8(size_t layer) const; + const float* get_key_scales(size_t layer) const; + const float* get_value_scales(size_t layer) const; + + void remove_token_range(size_t start, size_t count); + void compact_to_windows(const std::vector& target_windows); +}; + +class ToolCallConstrainer { +public: + enum class State { + DONE, + + QWEN_START, + QWEN_EXPECT_OPEN_BRACE, + QWEN_EXPECT_NAME_KEY, + QWEN_EXPECT_NAME_COLON, + QWEN_EXPECT_NAME_VALUE, + QWEN_EXPECT_COMMA, + QWEN_EXPECT_ARGS_KEY, + QWEN_EXPECT_ARGS_COLON, + QWEN_IN_ARGUMENTS, + QWEN_EXPECT_CLOSE_BRACE, + QWEN_EXPECT_END, + + NEEDLE_START, + + LFM_START, + LFM_EXPECT_BRACKET, + LFM_IN_FUNC_NAME, + LFM_EXPECT_PAREN, + LFM_IN_ARGUMENTS, + LFM_EXPECT_BRACKET_CLOSE, + LFM_EXPECT_END, + + GEMMA_START, + GEMMA_EXPECT_CALL, + GEMMA_IN_FUNC_NAME, + GEMMA_EXPECT_BRACE, + GEMMA_IN_ARGUMENTS, + GEMMA_EXPECT_END + }; + + void init(Config::ModelType model_type, + const std::vector& tools, + Tokenizer* tokenizer); + + const std::unordered_map& get_bias() const { return current_bias_; } + + void update(uint32_t token_id, const std::string& decoded_text); + + void reset(); + + bool is_active() const { return active_; } + +private: + bool active_ = false; + State state_ = State::QWEN_START; + Config::ModelType model_type_ = Config::ModelType::QWEN; + Tokenizer* tokenizer_ = nullptr; + + bool is_gemma_family() const { return Config::is_gemma_family(model_type_); } + bool is_needle() const { return model_type_ == Config::ModelType::NEEDLE; } + + enum class NeedleJsonState { + FREE, + IN_NAME, + IN_ARG_KEY, + }; + + struct NeedleTrieNode { + std::unordered_map> children; + bool is_terminal = false; + }; + + std::vector tool_specs_; + std::vector function_names_; + std::string generated_text_; + int brace_depth_ = 0; + + std::string call_start_tag_; + std::string call_end_tag_; + + std::unordered_set qwen_tool_call_start_tokens_; + std::unordered_set qwen_tool_call_end_tokens_; + std::unordered_set open_brace_tokens_; + std::unordered_set close_brace_tokens_; + std::unordered_set colon_tokens_; + std::unordered_set comma_tokens_; + std::unordered_set name_key_tokens_; + std::unordered_set args_key_tokens_; + std::unordered_set quote_tokens_; + std::unordered_set backtick_tokens_; + std::unordered_set all_func_name_tokens_; + std::unordered_map> func_name_sequences_; + NeedleJsonState needle_json_state_ = NeedleJsonState::FREE; + std::string needle_buffer_; + std::string needle_constrained_buf_; + std::string needle_current_function_; + bool needle_in_arguments_ = false; + int needle_arguments_depth_ = 0; + int needle_nesting_depth_ = 0; + bool needle_in_string_value_ = false; + bool needle_prev_char_escape_ = false; + std::unique_ptr needle_name_trie_; + std::unordered_map> needle_param_tries_; + std::vector needle_token_strings_; + std::unordered_map> needle_token_index_; + + std::unordered_set tool_start_tokens_; + std::unordered_set tool_end_tokens_; + std::unordered_set bracket_open_tokens_; + std::unordered_set bracket_close_tokens_; + std::unordered_set paren_open_tokens_; + std::unordered_set paren_close_tokens_; + std::unordered_set equals_tokens_; + + std::string lfm_current_function_; + std::string lfm_args_buffer_; + std::unordered_set lfm_seen_arg_keys_; + std::unordered_map> lfm_required_params_; + std::unordered_map> lfm_all_params_; + + std::unordered_set gemma_call_start_tokens_; + std::unordered_set gemma_call_end_tokens_; + std::unordered_set gemma_response_start_tokens_; + std::unordered_set gemma_call_prefix_tokens_; + std::unordered_set escape_tokens_; + + std::unordered_map current_bias_; + + void compute_bias(); + void tokenize_grammar_elements(); + void add_tokens_for_string(const std::string& str, std::unordered_set& token_set); + void add_tokens_for_prefix_string(const std::string& prefix, std::unordered_set& token_set); + void add_tokens_containing(char needle, std::unordered_set& token_set); + void tokenize_function_names(bool quote_names); + void init_common_tokens(); + void init_needle_constraints(); + void reset_needle_constraints(); + void feed_needle_text(const std::string& text); + void feed_needle_char(char ch); + bool needle_at_arg_key_start() const; + bool needle_is_value_string_start() const; + void needle_insert_word(NeedleTrieNode* root, const std::string& word); + const NeedleTrieNode* needle_get_trie_node(const NeedleTrieNode* root, const std::string& prefix) const; + bool needle_check_token_valid(const std::string& token_text, const NeedleTrieNode* trie_node) const; +}; + +class Model { +public: + struct DebugNode { + uint32_t layer_idx; + std::string name; + size_t node_id; + }; + + Model(); + explicit Model(const Config& config); + virtual ~Model(); + + const Config& get_config() const { return config_; } + Tokenizer* get_tokenizer() const { return tokenizer_.get(); } + const std::vector& get_debug_nodes() const; + + virtual bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true); + + virtual bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, + const std::string& system_prompt = "", bool do_warmup = true); + + virtual uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f); + + virtual void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = ""); + + virtual void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, + const std::string& profile_file = ""); + + virtual uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, + float temperature = -1.0f, float top_p = -1.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f); + + virtual uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, float temperature = 0.0f, float top_p = 0.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f, + float* out_token_time_start = nullptr, float* out_token_time_end = nullptr); + + std::vector get_embeddings(const std::vector& tokens, bool pooled = true, bool normalize = false, const std::string& profile_file = ""); + + virtual std::vector get_image_embeddings(const std::string& image_path); + + virtual std::vector get_audio_embeddings(const std::vector& audio_features); + + virtual void reset_cache() { kv_cache_.reset(); token_history_.clear(); } + void record_sampled_token(uint32_t token) { + if (token_history_.size() >= MAX_TOKEN_HISTORY) { + token_history_.erase(token_history_.begin(), token_history_.begin() + (MAX_TOKEN_HISTORY / 2)); + } + token_history_.push_back(token); + } + + double score_tokens_window_logprob(const std::vector& tokens, size_t start, size_t end, size_t context, size_t* tokens_scored); + + + + void set_cache_window(size_t window_size, size_t sink_size = 4) { kv_cache_.set_window_size(window_size, sink_size); } + size_t get_cache_size() const { return kv_cache_.current_seq_len; } + + bool load_npu_prefill(const std::string& model_path); + bool has_npu_prefill() const; + size_t get_prefill_chunk_size() const; + + virtual void remove_thinking_tokens(const std::vector>& ranges); + virtual void compact_kv_cache() {} + + virtual void set_tool_constraints(const std::vector& tools); + virtual void clear_tool_constraints(); + virtual void update_tool_constraints(uint32_t token_id); + + void* graph_handle_; + + void set_vocab_bias(const std::unordered_map& bias) { + vocab_bias_ = bias; + } + + void clear_vocab_bias() { + vocab_bias_.clear(); + } + + bool has_vocab_bias() const { + return !vocab_bias_.empty(); + } + + const std::unordered_map& get_vocab_bias() const { + return vocab_bias_; + } + +protected: + size_t sample_token(CactusGraph* gb, size_t logits_node_id, float temperature, float top_p, size_t top_k, + float min_p, float repetition_penalty, + const std::unordered_map* extra_bias = nullptr) const; + + static void compute_entropy(CactusGraph* gb, size_t logits_node_id, float* out_entropy); + + virtual size_t forward(const std::vector& tokens, bool use_cache = false) = 0; + + virtual size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false); + + virtual void load_weights_to_graph(CactusGraph* gb) = 0; + + virtual size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; + + virtual size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const = 0; + virtual size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; + void update_kv_cache(CactusGraph* gb, size_t seq_len); + virtual std::vector get_kv_layer_dims() const { + return std::vector(config_.num_layers, config_.attention_head_dim); + } + virtual std::vector get_kv_layer_heads() const { + return std::vector(config_.num_layers, config_.attention_kv_heads); + } + virtual void post_init() {} + virtual void post_execute_updates(CactusGraph*, size_t) {} + Config config_; + std::unique_ptr tokenizer_; + + bool initialized_; + float attention_scale_; + +protected: + KVCache kv_cache_; + std::vector cache_k_output_nodes_; + std::vector cache_v_output_nodes_; + + std::string embedding_file_path_; + size_t embedding_node_id_; + std::string model_folder_path_; + size_t output_weight_node_id_; + + mutable std::vector debug_nodes_; + + void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) const; + void clear_debug_nodes(); + + bool init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup); + bool owns_graph_; + + std::unique_ptr npu_prefill_; + void prefill_npu(const std::vector& tokens); + virtual std::vector<__fp16> get_token_embeddings(const std::vector& tokens); + + static constexpr size_t MAX_TOKEN_HISTORY = 128; + ToolCallConstrainer tool_constrainer_; + std::vector token_history_; + +private: + std::unordered_map vocab_bias_; +}; + +std::unique_ptr create_model(const std::string& model_folder); + +class Siglip2Preprocessor { +public: + struct Config { + int patch_size = 16; + int downsample_factor = 2; + int min_tiles = 2; + int max_tiles = 10; + bool use_thumbnail = true; + int min_image_tokens = 64; + int max_image_tokens = 256; + int max_num_patches = 1024; + int tile_size = 512; + float max_pixels_tolerance = 2.0f; + bool do_resize = true; + bool do_rescale = true; + bool do_normalize = true; + bool do_convert_rgb = true; + bool do_image_splitting = true; + float rescale_factor = 1.0f / 255.0f; + float image_mean[3] = {0.5f, 0.5f, 0.5f}; + float image_std[3] = {0.5f, 0.5f, 0.5f}; + }; + + struct PreprocessedImage { + std::vector pixel_values; + std::vector pixel_attention_mask; + std::vector> spatial_shapes; + std::vector pixel_values_shape; + std::vector pixel_attention_mask_shape; + std::vector spatial_shapes_shape; + int num_patches_height; + int num_patches_width; + int actual_num_patches; + int num_tiles; + int patch_dim; + int max_patches_per_tile; + + int image_rows; + int image_cols; + int image_height; + int image_width; + int tokens_per_tile; + int thumbnail_tokens; + + ~PreprocessedImage(); + }; + + struct SpatialShapeResult { + std::vector> shapes; + int grid_rows; + int grid_cols; + }; + + explicit Siglip2Preprocessor(const Config& config); + Siglip2Preprocessor(); + ~Siglip2Preprocessor(); + + PreprocessedImage preprocess_from_file(const std::string& image_path); + PreprocessedImage preprocess_from_memory(const unsigned char* img_data, int width, int height, int channels); + SpatialShapeResult compute_spatial_shapes(int height, int width); + +private: + Config config_; + + std::pair compute_pixel_limits() const; + std::vector convert_to_rgb(const unsigned char* img_data, int width, int height, int channels); + std::pair smart_resize(int height, int width); + bool is_image_too_large(int height, int width); + std::pair get_grid_layout(int height, int width); + std::pair find_closest_aspect_ratio(float aspect_ratio, int width, int height); + std::vector resize_image(const unsigned char* img_data, int src_width, int src_height, + int dst_width, int dst_height, int channels); + std::vector normalize_image(const float* img_data, int width, int height, int channels); + std::vector> convert_image_to_patches( + const std::vector& image, int width, int height, int channels, int patch_size); + PreprocessedImage pad_patches(const std::vector>& tile_patches, + const std::vector>& spatial_shapes, + int patch_dim, + int max_patches_per_tile); + int round_by_factor(int number, int factor); +}; + +class AudioProcessor { +public: + struct SpectrogramConfig { + size_t n_fft = 400; + size_t hop_length = 160; + size_t frame_length = 400; + float power = 2.0f; + bool center = true; + const char* pad_mode = "reflect"; + bool onesided = true; + float dither = 0.0f; + float mel_floor = 1e-10f; + const char* log_mel = nullptr; + float reference = 1.0f; + float min_value = 1e-10f; + bool remove_dc_offset = false; + float preemphasis = 0.0f; + bool hann_periodic = true; + float window_a0 = 0.5f; + size_t fft_override = 0; + bool mel_floor_additive = false; + }; + + AudioProcessor(); + ~AudioProcessor(); + + void init_mel_filters(size_t num_frequency_bins, size_t num_mel_filters, + float min_freq, float max_freq, size_t sampling_rate, + const char* norm = "slaney", const char* mel_scale = "slaney"); + + std::vector compute_spectrogram( + const std::vector& waveform, + const SpectrogramConfig& config); + + static std::vector compute_irfft( + const std::vector& complex_input, + size_t n, + const char* norm = "backward"); + + const std::vector& get_mel_filters() const { return mel_filters_; } + + size_t get_num_mel_filters() const { return num_mel_filters_; } + size_t get_num_frequency_bins() const { return num_frequency_bins_; } + +private: + std::vector mel_filters_; + size_t num_frequency_bins_; + size_t num_mel_filters_; +}; + +namespace index { + constexpr uint32_t MAGIC = 0x43414354; + constexpr uint32_t VERSION = 1; + + struct Document { + int id; + std::vector embedding; + std::string content; + std::string metadata; + }; + + struct QueryResult { + int doc_id; + float score; + + QueryResult(int doc_id, float score) : doc_id(doc_id), score(score) {} + }; + + struct QueryOptions { + size_t top_k = 10; + float score_threshold = -1.0f; + }; + + class Index { + public: + Index(const std::string& index_path, const std::string& data_path, size_t embedding_dim); + ~Index(); + + Index(const Index&) = delete; + Index& operator=(const Index&) = delete; + Index(Index&&) = delete; + Index& operator=(Index&&) = delete; + + void add_documents(const std::vector& documents); + void delete_documents(const std::vector& doc_ids); + std::vector get_documents(const std::vector& doc_ids); + std::vector> query(const std::vector>& embeddings, const QueryOptions& options); + void compact(); + + private: + struct IndexHeader { + uint32_t magic; + uint32_t version; + uint32_t embedding_dim; + uint32_t num_documents; + }; + + struct IndexEntry { + int32_t doc_id; + uint64_t data_offset; + uint8_t flags; // bit 0: tombstone + + const __fp16* embedding() const { + return reinterpret_cast(this + 1); + } + + static size_t size(size_t embedding_dim) { + return sizeof(IndexEntry) + embedding_dim * sizeof(__fp16); + } + }; + + struct DataHeader { + uint32_t magic; + uint32_t version; + }; + + struct DataEntry { + uint16_t content_len; + uint16_t metadata_len; + + const char* content() const { + return reinterpret_cast(this + 1); + } + + const char* metadata() const { + return content() + content_len; + } + }; + + void parse_index_header(); + void parse_data_header(); + void build_doc_id_map(); + void validate_documents(const std::vector& documents); + void validate_doc_ids(const std::vector& doc_ids); + ssize_t write_full(int fd, const void* buf, size_t count); + + std::unordered_map doc_id_map_; + + std::string index_path_, data_path_; + size_t embedding_dim_; + size_t index_entry_size_; + uint32_t num_documents_; + + int index_fd_, data_fd_; + void *mapped_index_, *mapped_data_; + size_t index_file_size_, data_file_size_; + }; +} // namespace index + +} +} diff --git a/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/gemma_tools.h b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/gemma_tools.h new file mode 100644 index 0000000..f0f9fe2 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/gemma_tools.h @@ -0,0 +1,576 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace gemma { + +inline std::string to_upper(const std::string& s) { + std::string result = s; + for (auto& c : result) c = std::toupper(c); + return result; +} + +inline std::string escape(const std::string& s) { + return "" + s + ""; +} + +inline void skip_whitespace(const std::string& json, size_t& pos) { + while (pos < json.length() && std::isspace(json[pos])) pos++; +} + +inline std::string extract_json_string(const std::string& json, size_t& pos) { + std::string value; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\' && pos + 1 < json.length()) { + pos++; + if (json[pos] == 'n') value += '\n'; + else if (json[pos] == 't') value += '\t'; + else if (json[pos] == 'r') value += '\r'; + else if (json[pos] == '"') value += '"'; + else if (json[pos] == '\\') value += '\\'; + else value += json[pos]; + } else { + value += json[pos]; + } + pos++; + } + if (pos < json.length()) pos++; + return value; +} + +std::string format_argument(const std::string& json, size_t& pos, bool escape_keys); +std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/); + +inline std::string format_argument(const std::string& json, size_t& pos, bool escape_keys = true) { + skip_whitespace(json, pos); + if (pos >= json.length()) return ""; + + char c = json[pos]; + + if (c == '"') { + pos++; + std::string value = extract_json_string(json, pos); + return escape(value); + } else if (c == '{') { + std::string result = "{"; + pos++; + bool first = true; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == '}') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + if (json[pos] != '"') break; + pos++; + std::string key = extract_json_string(json, pos); + + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == ':') pos++; + + std::string value = format_argument(json, pos, escape_keys); + + if (!first) result += ","; + first = false; + if (escape_keys) { + result += escape(key) + ":" + value; + } else { + result += key + ":" + value; + } + } + result += "}"; + return result; + } else if (c == '[') { + std::string result = "["; + pos++; + bool first = true; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == ']') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + std::string value = format_argument(json, pos, escape_keys); + + if (!first) result += ","; + first = false; + result += value; + } + result += "]"; + return result; + } else if (json.compare(pos, 4, "true") == 0) { + pos += 4; + return "true"; + } else if (json.compare(pos, 5, "false") == 0) { + pos += 5; + return "false"; + } else if (json.compare(pos, 4, "null") == 0) { + pos += 4; + return "null"; + } else { + size_t start = pos; + while (pos < json.length() && (std::isdigit(json[pos]) || json[pos] == '.' || + json[pos] == '-' || json[pos] == '+' || json[pos] == 'e' || json[pos] == 'E')) { + pos++; + } + return json.substr(start, pos - start); + } +} + +inline std::map parse_json_object_raw(const std::string& json, size_t& pos) { + std::map result; + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] != '{') return result; + pos++; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == '}') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + if (json[pos] != '"') break; + pos++; + std::string key = extract_json_string(json, pos); + + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == ':') pos++; + skip_whitespace(json, pos); + + size_t value_start = pos; + if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + pos++; + } else if (json[pos] == '{') { + int depth = 1; + pos++; + while (pos < json.length() && depth > 0) { + if (json[pos] == '{') depth++; + else if (json[pos] == '}') depth--; + else if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + } else if (json[pos] == '[') { + int depth = 1; + pos++; + while (pos < json.length() && depth > 0) { + if (json[pos] == '[') depth++; + else if (json[pos] == ']') depth--; + else if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + } else { + while (pos < json.length() && json[pos] != ',' && json[pos] != '}') pos++; + } + result[key] = json.substr(value_start, pos - value_start); + } + return result; +} + +inline std::string get_json_string_value(const std::string& json, size_t pos) { + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == '"') { + pos++; + return extract_json_string(json, pos); + } + return ""; +} + +inline std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/) { + static const std::set standard_keys = {"description", "type", "properties", "required", "nullable"}; + + size_t pos = 0; + auto properties = parse_json_object_raw(properties_json, pos); + + std::string result; + bool first = true; + + for (const auto& [key, value_json] : properties) { + if (standard_keys.count(key)) continue; + + if (!first) result += ","; + first = false; + + size_t prop_pos = 0; + auto prop_obj = parse_json_object_raw(value_json, prop_pos); + + result += key + ":{"; + + if (prop_obj.count("description")) { + std::string desc = get_json_string_value(prop_obj["description"], 0); + result += "description:" + escape(desc); + } + + std::string type_val; + if (prop_obj.count("type")) { + type_val = get_json_string_value(prop_obj["type"], 0); + } + + if (to_upper(type_val) == "STRING") { + if (prop_obj.count("enum")) { + size_t enum_pos = 0; + std::string enum_formatted = format_argument(prop_obj["enum"], enum_pos, true); + result += ",enum:" + enum_formatted; + } + } else if (to_upper(type_val) == "OBJECT") { + if (prop_obj.count("properties")) { + std::string nested_required; + if (prop_obj.count("required")) { + nested_required = prop_obj["required"]; + } + result += ",properties:{" + format_parameters(prop_obj["properties"], nested_required) + "}"; + } + if (prop_obj.count("required")) { + std::string req_items; + size_t req_pos = 0; + skip_whitespace(prop_obj["required"], req_pos); + if (req_pos < prop_obj["required"].length() && prop_obj["required"][req_pos] == '[') { + req_pos++; + bool req_first = true; + while (req_pos < prop_obj["required"].length()) { + skip_whitespace(prop_obj["required"], req_pos); + if (prop_obj["required"][req_pos] == ']') break; + if (prop_obj["required"][req_pos] == ',') { req_pos++; continue; } + if (prop_obj["required"][req_pos] == '"') { + req_pos++; + std::string req_item = extract_json_string(prop_obj["required"], req_pos); + if (!req_first) req_items += ","; + req_first = false; + req_items += escape(req_item); + } + } + } + if (!req_items.empty()) { + result += ",required:[" + req_items + "]"; + } + } + } else if (to_upper(type_val) == "ARRAY") { + if (prop_obj.count("items")) { + result += ",items:{"; + size_t items_pos = 0; + auto items_obj = parse_json_object_raw(prop_obj["items"], items_pos); + bool items_first = true; + + for (const auto& [item_key, item_value] : items_obj) { + if (!items_first) result += ","; + items_first = false; + + if (item_key == "properties") { + std::string items_required; + if (items_obj.count("required")) { + items_required = items_obj["required"]; + } + result += "properties:{" + format_parameters(item_value, items_required) + "}"; + } else if (item_key == "required") { + result += "required:["; + size_t req_pos = 0; + skip_whitespace(item_value, req_pos); + if (req_pos < item_value.length() && item_value[req_pos] == '[') { + req_pos++; + bool req_first = true; + while (req_pos < item_value.length()) { + skip_whitespace(item_value, req_pos); + if (item_value[req_pos] == ']') break; + if (item_value[req_pos] == ',') { req_pos++; continue; } + if (item_value[req_pos] == '"') { + req_pos++; + std::string req_item = extract_json_string(item_value, req_pos); + if (!req_first) result += ","; + req_first = false; + result += escape(req_item); + } + } + } + result += "]"; + } else if (item_key == "type") { + std::string item_type = get_json_string_value(item_value, 0); + result += "type:" + escape(to_upper(item_type)); + } else { + size_t val_pos = 0; + result += item_key + ":" + format_argument(item_value, val_pos, true); + } + } + result += "}"; + } + } + + if (!type_val.empty()) { + result += ",type:" + escape(to_upper(type_val)); + } + + result += "}"; + } + + return result; +} + +inline std::string format_function_declaration(const std::string& name, + const std::string& description, + const std::string& params_json) { + std::string result = "declaration:" + name + "{"; + result += "description:" + escape(description); + + if (!params_json.empty()) { + result += ",parameters:{"; + + size_t pos = 0; + auto params = parse_json_object_raw(params_json, pos); + + if (params.count("properties")) { + std::string required_json; + if (params.count("required")) { + required_json = params["required"]; + } + result += "properties:{" + format_parameters(params["properties"], required_json) + "}"; + } + + if (params.count("required")) { + std::string req_items; + size_t req_pos = 0; + skip_whitespace(params["required"], req_pos); + if (req_pos < params["required"].length() && params["required"][req_pos] == '[') { + req_pos++; + bool first = true; + while (req_pos < params["required"].length()) { + skip_whitespace(params["required"], req_pos); + if (params["required"][req_pos] == ']') break; + if (params["required"][req_pos] == ',') { req_pos++; continue; } + if (params["required"][req_pos] == '"') { + req_pos++; + std::string item = extract_json_string(params["required"], req_pos); + if (!first) req_items += ","; + first = false; + req_items += escape(item); + } + } + } + if (!req_items.empty()) { + result += ",required:[" + req_items + "]"; + } + } + + if (params.count("type")) { + std::string type_val = get_json_string_value(params["type"], 0); + result += ",type:" + escape(to_upper(type_val)); + } + + result += "}"; + } + + result += "}"; + return result; +} + +template +inline std::string format_tools(const std::vector& tools, bool use_pipe_tags = false) { + if (tools.empty()) return ""; + + const char* decl_start = use_pipe_tags ? "<|tool>" : ""; + const char* decl_end = use_pipe_tags ? "" : ""; + + std::string result; + for (const auto& tool : tools) { + result += decl_start; + std::string params_json; + auto it = tool.parameters.find("schema"); + if (it != tool.parameters.end()) { + params_json = it->second; + } + + result += format_function_declaration(tool.name, tool.description, params_json); + result += decl_end; + } + return result; +} + + +inline size_t match_quote_tag(const std::string& s, size_t pos) { + if (s.compare(pos, 8, "") == 0) return 8; + if (s.compare(pos, 5, "<|\"|>") == 0) return 5; + return 0; +} + +inline size_t find_quote_tag(const std::string& s, size_t pos) { + size_t e = s.find("", pos); + size_t t = s.find("<|\"|>", pos); + if (e == std::string::npos) return t; + if (t == std::string::npos) return e; + return std::min(e, t); +} + +inline std::string unescape(const std::string& s) { + const std::string ESCAPE_TAG = ""; + std::string result = s; + size_t pos = 0; + while ((pos = result.find(ESCAPE_TAG, pos)) != std::string::npos) { + result.erase(pos, ESCAPE_TAG.length()); + } + return result; +} + +inline std::string args_to_json(const std::string& args_content) { + std::string result = "{"; + size_t pos = 0; + bool first = true; + + if (!args_content.empty() && args_content[0] == '{') pos = 1; + + while (pos < args_content.length()) { + while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; + if (pos >= args_content.length() || args_content[pos] == '}') break; + if (args_content[pos] == ',') { pos++; continue; } + + size_t key_start = pos; + while (pos < args_content.length() && args_content[pos] != ':') pos++; + std::string key = args_content.substr(key_start, pos - key_start); + if (pos < args_content.length()) pos++; + + std::string value; + while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; + + if (pos < args_content.length()) { + size_t qtag_len = match_quote_tag(args_content, pos); + if (qtag_len > 0) { + pos += qtag_len; + size_t val_end = find_quote_tag(args_content, pos); + if (val_end != std::string::npos) { + value = "\"" + args_content.substr(pos, val_end - pos) + "\""; + pos = val_end + match_quote_tag(args_content, val_end); + } + } else if (args_content[pos] == '{') { + int depth = 1; + size_t start = pos; + pos++; + while (pos < args_content.length() && depth > 0) { + if (args_content[pos] == '{') depth++; + else if (args_content[pos] == '}') depth--; + pos++; + } + value = args_to_json(args_content.substr(start, pos - start)); + } else if (args_content[pos] == '[') { + int depth = 1; + size_t start = pos; + pos++; + while (pos < args_content.length() && depth > 0) { + if (args_content[pos] == '[') depth++; + else if (args_content[pos] == ']') depth--; + pos++; + } + std::string arr_content = args_content.substr(start + 1, pos - start - 2); + value = "["; + size_t arr_pos = 0; + bool first_item = true; + while (arr_pos < arr_content.length()) { + while (arr_pos < arr_content.length() && (std::isspace(arr_content[arr_pos]) || arr_content[arr_pos] == ',')) arr_pos++; + if (arr_pos >= arr_content.length()) break; + + if (!first_item) value += ","; + first_item = false; + + size_t aq_len = match_quote_tag(arr_content, arr_pos); + if (aq_len > 0) { + arr_pos += aq_len; + size_t end = find_quote_tag(arr_content, arr_pos); + if (end != std::string::npos) { + value += "\"" + arr_content.substr(arr_pos, end - arr_pos) + "\""; + arr_pos = end + match_quote_tag(arr_content, end); + } + } else { + size_t end = arr_content.find_first_of(",]", arr_pos); + if (end == std::string::npos) end = arr_content.length(); + value += arr_content.substr(arr_pos, end - arr_pos); + arr_pos = end; + } + } + value += "]"; + } else { + size_t val_start = pos; + while (pos < args_content.length() && args_content[pos] != ',' && args_content[pos] != '}') { + pos++; + } + value = args_content.substr(val_start, pos - val_start); + while (!value.empty() && std::isspace(value.back())) value.pop_back(); + } + } + + if (!first) result += ","; + first = false; + result += "\"" + key + "\":" + value; + } + + result += "}"; + return result; +} + +inline void parse_function_calls(std::string& response, std::vector& function_calls) { + + const std::string CALL_START = (response.find("<|tool_call>") != std::string::npos) + ? "<|tool_call>" : ""; + const std::string CALL_END = (CALL_START == "<|tool_call>") + ? "" : ""; + size_t pos = 0; + + while ((pos = response.find(CALL_START, pos)) != std::string::npos) { + size_t content_start = pos + CALL_START.length(); + size_t call_end_pos = response.find(CALL_END, content_start); + + size_t content_end = (call_end_pos != std::string::npos) ? call_end_pos : response.length(); + std::string call_content = response.substr(content_start, content_end - content_start); + + if (call_content.compare(0, 5, "call:") == 0) { + size_t brace_pos = call_content.find('{'); + + if (brace_pos == std::string::npos) { + size_t sep_pos = call_content.find_first_of(", ", 5); + if (sep_pos != std::string::npos) { + std::string func_name = call_content.substr(5, sep_pos - 5); + size_t args_start = sep_pos + 1; + while (args_start < call_content.length() && + (call_content[args_start] == ' ' || call_content[args_start] == ',')) { + args_start++; + } + std::string args_content = "{" + call_content.substr(args_start); + if (args_content.back() != '}') args_content += "}"; + + std::string args_json = args_to_json(args_content); + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; + function_calls.push_back(json_call); + } + } else { + std::string func_name = call_content.substr(5, brace_pos - 5); + std::string args_content = call_content.substr(brace_pos); + if (args_content.back() != '}') args_content += "}"; + + std::string args_json = args_to_json(args_content); + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; + function_calls.push_back(json_call); + } + } + + size_t erase_end = (call_end_pos != std::string::npos) ? + call_end_pos + CALL_END.length() : response.length(); + response.erase(pos, erase_end - pos); + } +} + +} // namespace gemma \ No newline at end of file diff --git a/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/graph.h b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/graph.h new file mode 100644 index 0000000..b436177 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/graph.h @@ -0,0 +1,779 @@ +#ifndef GRAPH_H +#define GRAPH_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { + +enum class LogLevel { + DEBUG = 0, + INFO = 1, + WARN = 2, + ERROR = 3, + NONE = 4 +}; + +class Logger { +public: + static Logger& instance() { + static Logger logger; + return logger; + } + + void set_level(LogLevel level) { min_level_ = level; } + LogLevel get_level() const { return min_level_; } + + void set_callback(std::function cb) { + std::lock_guard lock(mutex_); + callback_ = cb; + } + + void log(LogLevel level, const std::string& component, const std::string& message) { + if (level < min_level_) return; + + std::lock_guard lock(mutex_); + + if (callback_) { + callback_(level, component, message); + } else { + std::cerr << "[" << level_string(level) << "] [" << component << "] " << message << std::endl; + } + + if (level == LogLevel::ERROR) { + last_error_ = "[" + component + "] " + message; + } + } + + const std::string& last_error() const { return last_error_; } + void clear_error() { last_error_.clear(); } + +private: + Logger() : min_level_(LogLevel::WARN) {} + + static const char* level_string(LogLevel level) { + switch (level) { + case LogLevel::DEBUG: return "DEBUG"; + case LogLevel::INFO: return "INFO"; + case LogLevel::WARN: return "WARN"; + case LogLevel::ERROR: return "ERROR"; + default: return "?"; + } + } + + LogLevel min_level_; + std::mutex mutex_; + std::string last_error_; + std::function callback_; +}; + +} // namespace cactus + +#define CACTUS_LOG(level, component, msg) \ + do { \ + if (static_cast(level) >= static_cast(cactus::Logger::instance().get_level())) { \ + std::ostringstream _cactus_log_ss; \ + _cactus_log_ss << msg; \ + cactus::Logger::instance().log(level, component, _cactus_log_ss.str()); \ + } \ + } while(0) + +#define CACTUS_LOG_DEBUG(component, msg) CACTUS_LOG(cactus::LogLevel::DEBUG, component, msg) +#define CACTUS_LOG_INFO(component, msg) CACTUS_LOG(cactus::LogLevel::INFO, component, msg) +#define CACTUS_LOG_WARN(component, msg) CACTUS_LOG(cactus::LogLevel::WARN, component, msg) +#define CACTUS_LOG_ERROR(component, msg) CACTUS_LOG(cactus::LogLevel::ERROR, component, msg) + +namespace GraphFile { + class MappedFile; + struct SerializedGraph; +} + +enum class Precision { + INT8, + FP16, + FP32, + INT4 +}; + +enum class ComputeBackend { + CPU, + NPU +}; + +enum class Activation { + SILU, + GELU, + GELU_ERF, + RELU, + SIGMOID, + TANH +}; + +enum class OpType { + INPUT, PRECISION_CAST, + ADD, ADD_CLIPPED, SUBTRACT, MULTIPLY, DIVIDE, + ABS, POW, FLATTEN, VIEW, + MATMUL, TRANSPOSE, RESHAPE, SLICE, GATHER, EMBEDDING, + BILINEAR_INTERPOLATION, + SUM, MEAN, VARIANCE, MIN, MAX, + RMS_NORM, ROPE, ROPE_GPTJ, SOFTMAX, ATTENTION, ATTENTION_INT8_HYBRID, REL_POS_BIAS, CONV1D_CAUSAL, CONV1D_K3, CONV1D_K7S3, CONV1D, CONV1D_SAME_DEPTHWISE_K9, CONV1D_POINTWISE, CONV2D_K3S2P1, CONV2D_DEPTHWISE_K3S2P1, CONV2D_POINTWISE_1X1, GLU, BATCHNORM, + SCALAR_ADD, SCALAR_SUBTRACT, SCALAR_MULTIPLY, SCALAR_DIVIDE, SCALAR_EXP, SCALAR_SQRT, SCALAR_COS, SCALAR_SIN, SCALAR_LOG, + RELU, SILU, GELU, GELU_ERF, SIGMOID, TANH, + SAMPLE, CONCAT, CAT, + SCATTER_TOPK, + TOPK, LAYERNORM, GROUPNORM, + MOE_LAYER, + INDEX, + PERSISTENT, + QUANTIZE_ACTIVATIONS, + LSTM_CELL, + GATED_DELTANET_DECODE, + GATED_DELTANET_PREFILL, + STFT, + ALTUP_PREDICT, + ALTUP_CORRECT, + GAUSSIAN_TOPK, + MAXPOOL1D, + BILSTM_SEQUENCE, + LEAKY_RELU, + CONV2D_K3S1P1, + STATS_POOL, + WEIGHTED_STATS_POOL, + DENSE_MLP_INT4_FUSED +}; + +struct PrecisionTraits { + static constexpr size_t size_of(Precision prec) { + switch (prec) { + case Precision::INT8: return 1; + case Precision::FP16: return 2; + case Precision::FP32: return 4; + case Precision::INT4: return 1; + } + return 1; + } + + static constexpr size_t packed_size_of(Precision prec, size_t count) { + switch (prec) { + case Precision::INT4: return (count + 1) / 2; + default: return count * size_of(prec); + } + } + + static size_t byte_offset_of(Precision prec, size_t element_offset) { + switch (prec) { + case Precision::INT4: + assert(element_offset % 32 == 0 && "INT4 byte offset must be group-aligned (multiple of 32)"); + return element_offset / 2; + default: return element_offset * size_of(prec); + } + } + + static constexpr bool is_integer(Precision prec) { + switch (prec) { + case Precision::INT8: return true; + case Precision::INT4: return true; + case Precision::FP16: return false; + case Precision::FP32: return false; + } + return true; + } + + static constexpr bool is_floating_point(Precision prec) { + switch (prec) { + case Precision::INT8: return false; + case Precision::INT4: return false; + case Precision::FP16: return true; + case Precision::FP32: return true; + } + return false; + } +}; + +namespace Quantization { + void int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); + void fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); + void fp16_to_fp32(const __fp16* src, float* dst, size_t count); + void fp32_to_fp16(const float* src, __fp16* dst, size_t count); + void int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); + void fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); +} + +struct TensorConfig { + Precision default_precision = Precision::INT8; + Precision compute_precision = Precision::INT8; + Precision output_precision = Precision::INT8; + bool auto_mixed_precision = false; + + static TensorConfig& global(); +}; + +struct BroadcastInfo { + std::vector output_shape; + bool needs_broadcasting; + + static BroadcastInfo compute(const std::vector& lhs, const std::vector& rhs); +}; + +class BufferPool; + +struct BufferDesc { + std::vector shape; + size_t total_size; + size_t byte_size; + std::unique_ptr data; + void* external_data; + char* pooled_data; + Precision precision; + + size_t group_size = 0; + size_t num_groups = 0; + void* scales_data = nullptr; + std::unique_ptr owned_scales; + + bool is_interleaved = false; + size_t original_N = 0; + + void* activation_scales_data = nullptr; + std::unique_ptr owned_activation_scales; + size_t num_rows_for_activation_scales = 0; + + BufferDesc(); + BufferDesc(const std::vector& s, Precision prec = Precision::INT8); + ~BufferDesc(); + + BufferDesc(BufferDesc&& other) noexcept; + BufferDesc& operator=(BufferDesc&& other) noexcept; + + BufferDesc(const BufferDesc&) = delete; + BufferDesc& operator=(const BufferDesc&) = delete; + + void* get_data(); + const void* get_data() const; + + template + T* data_as() { return static_cast(get_data()); } + + template + const T* data_as() const { return static_cast(get_data()); } + + const __fp16* scales_as_fp16() const { + return reinterpret_cast(scales_data); + } + + bool is_grouped_int8() const { + return precision == Precision::INT8 && group_size > 0; + } + + bool is_grouped_int4() const { + return precision == Precision::INT4 && group_size > 0; + } + + void set_grouped_scales(size_t gs, size_t ng, void* scales_ptr) { + group_size = gs; + num_groups = ng; + scales_data = scales_ptr; + } + + void set_interleaved(bool interleaved, size_t orig_n) { + is_interleaved = interleaved; + original_N = orig_n; + } + + bool has_activation_scales() const { + return activation_scales_data != nullptr && num_rows_for_activation_scales > 0; + } + const float* activation_scales_as_float() const { + return reinterpret_cast(activation_scales_data); + } + float* activation_scales_as_float() { + return reinterpret_cast(activation_scales_data); + } + void allocate_activation_scales(size_t num_rows) { + num_rows_for_activation_scales = num_rows; + owned_activation_scales = std::make_unique(num_rows * sizeof(float)); + activation_scales_data = owned_activation_scales.get(); + } + void set_activation_scales(void* scales_ptr, size_t num_rows) { + activation_scales_data = scales_ptr; + num_rows_for_activation_scales = num_rows; + } + + void allocate(); + void allocate_from_pool(BufferPool& pool); + void release_to_pool(BufferPool& pool); + void set_external(void* ptr); +}; + +struct OpParams { + float scalar = 0.0f; + float scale = 1.0f; + float theta = 10000.0f; + float epsilon = 1e-6f; + int axis = -1; + bool pretransposed_rhs = false; + size_t position_offset = 0; + size_t slice_start = 0; + size_t slice_length = 0; + size_t window_size = 0; + bool is_causal = true; + bool attention_mask_is_additive = false; + float logit_cap = 0.0f; + std::vector new_shape; + std::vector permutation; + Precision output_precision = Precision::INT8; + BroadcastInfo broadcast_info; + ComputeBackend backend = ComputeBackend::CPU; + + size_t dilation = 1; + size_t stride = 1; + float temperature = 1.0f; + float top_p = 1.0f; + float min_p = 0.15f; + float repetition_penalty = 1.1f; + size_t top_k = 0; + size_t random_seed = 0; + + size_t index_value = 0; + size_t num_classes = 0; + size_t num_groups = 0; + size_t dst_height = 0; + size_t dst_width = 0; + bool align_corners = true; + bool normalize_routing = false; + size_t num_experts = 0; + size_t num_experts_per_tok = 0; + bool moe_gated = true; + Activation activation = Activation::SILU; + + std::vector bias_values; + std::vector bias_indices; + + const int8_t* cached_keys_int8 = nullptr; + const int8_t* cached_values_int8 = nullptr; + const float* cached_k_scales = nullptr; + const float* cached_v_scales = nullptr; + size_t cache_seq_len = 0; + size_t num_kv_heads = 0; + size_t head_dim = 0; + size_t num_fft_bins = 0; + size_t chunk_size = 0; + size_t num_altup_inputs = 0; + size_t v_head_dim = 0; + size_t kernel_size = 0; +}; + +struct GraphNode { + size_t id; + OpType op_type; + std::vector input_ids; + BufferDesc output_buffer; + OpParams params; + + GraphNode(size_t node_id, OpType type); +}; + +using nodes_vector = std::vector>; +using node_index_map_t = std::unordered_map; + +inline const BufferDesc& get_input(const GraphNode& node, size_t idx, + const nodes_vector& nodes, + const node_index_map_t& node_index_map) { + return nodes[node_index_map.at(node.input_ids[idx])]->output_buffer; +} + +struct AxisDims { + size_t outer, axis_size, inner; + static AxisDims from_shape(const std::vector& shape, size_t axis) { + AxisDims d; + d.outer = 1; + for (size_t i = 0; i < axis; i++) d.outer *= shape[i]; + d.axis_size = shape[axis]; + d.inner = 1; + for (size_t i = axis + 1; i < shape.size(); i++) d.inner *= shape[i]; + return d; + } +}; + +template +void dispatch_binary_op(OpType op, const T* lhs, const T* rhs, T* output, size_t count); + +template +void dispatch_unary_op(OpType op, const T* input, T* output, size_t count, float param = 0.0f); + +void compute_node_optimized(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_matmul_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_transpose_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_reduce_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_reshape_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_precision_cast_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_sample_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_scatter_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_layernorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_groupnorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_persistent_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_dense_mlp_int4_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_index_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_lstm_cell_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_gated_deltanet_decode_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_gated_deltanet_prefill_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_altup_predict_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_altup_correct_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_maxpool1d_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_bilstm_sequence_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_conv2d_k3s1p1_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_weighted_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); + +void shrink_thread_local_buffers(); +class BufferPool { +public: + BufferPool() = default; + ~BufferPool() = default; + + BufferPool(const BufferPool&) = delete; + BufferPool& operator=(const BufferPool&) = delete; + BufferPool(BufferPool&&) noexcept = default; + BufferPool& operator=(BufferPool&&) noexcept = default; + + char* acquire(size_t byte_size); + void release(char* ptr, size_t byte_size); + void clear(); + + size_t active_bytes() const { return active_bytes_; } + size_t pool_bytes() const { return pool_bytes_; } + size_t peak_bytes() const { return peak_bytes_; } + +private: + std::unordered_map>> free_buffers_; + size_t active_bytes_ = 0; + size_t pool_bytes_ = 0; + size_t peak_bytes_ = 0; + + size_t round_up_size(size_t size) const; +}; + +namespace ValidationUtils { + void validate_tensor_dims(const std::vector& shape, size_t required_dims, const std::string& op_name); + void validate_precision(Precision actual, Precision required, const std::string& op_name); + void validate_input_count(size_t actual, size_t required, const std::string& op_name); +} + + +class CactusGraph { +public: + CactusGraph(); + ~CactusGraph() = default; + + CactusGraph(const CactusGraph&) = delete; + CactusGraph& operator=(const CactusGraph&) = delete; + CactusGraph(CactusGraph&&) noexcept = default; + CactusGraph& operator=(CactusGraph&&) noexcept = default; + + struct DebugNodeEntry { + uint32_t layer_idx; + std::string name; + size_t node_id; + }; + + void save(const std::string& path); + static CactusGraph load(const std::string& path); + + size_t input(const std::vector& shape, Precision precision = Precision::INT8); + size_t precision_cast(size_t input, Precision target_precision); + size_t quantize_activations(size_t input); + + size_t add(size_t input1, size_t input2); + size_t add_clipped(size_t input1, size_t input2); + size_t subtract(size_t input1, size_t input2); + size_t multiply(size_t input1, size_t input2); + size_t divide(size_t input1, size_t input2); + + size_t scalar_add(size_t input, float value); + size_t scalar_subtract(size_t input, float value); + size_t scalar_multiply(size_t input, float value); + size_t scalar_divide(size_t input, float value); + size_t scalar_exp(size_t input); + size_t scalar_sqrt(size_t input); + size_t scalar_cos(size_t input); + size_t scalar_sin(size_t input); + size_t scalar_log(size_t input); + + size_t relu(size_t input); + size_t silu(size_t input); + size_t gelu(size_t input); + size_t gelu_erf(size_t input); + size_t sigmoid(size_t input); + size_t tanh(size_t input); + size_t glu(size_t input, int axis = -1); + + size_t abs(size_t input); + size_t pow(size_t input, float exponent); + size_t view(size_t input, const std::vector& new_shape); + size_t flatten(size_t input, int start_dim = 0, int end_dim = -1); + + size_t matmul(size_t input1, size_t input2, bool pretransposed_rhs = false, ComputeBackend backend = ComputeBackend::CPU); + size_t transpose(size_t input, ComputeBackend backend = ComputeBackend::CPU); + size_t transposeN(size_t input, const std::vector& permutation, ComputeBackend backend = ComputeBackend::CPU); + size_t reshape(size_t input, const std::vector& new_shape); + size_t slice(size_t input, int axis, size_t start, size_t length); + size_t index(size_t input, size_t index_value, int dim); + + size_t sum(size_t input, int axis); + size_t mean(size_t input, int axis); + size_t variance(size_t input, int axis); + size_t min(size_t input, int axis); + size_t max(size_t input, int axis); + + size_t gather(size_t embeddings, size_t indices); + size_t mmap_embeddings(const std::string& filename); + size_t mmap_weights(const std::string& filename); + void set_grouped_scales(size_t node_id, size_t group_size, size_t num_groups, void* scales_ptr); + void set_interleaved(size_t node_id, bool interleaved, size_t original_N); + + void release_weight_pages(size_t node_id); + void prefetch_weight_pages(size_t node_id); + void release_all_weight_pages(); + size_t embedding(const std::string& filename, size_t indices); + size_t embedding(size_t embedding_tensor, size_t indices); + size_t bilinear_interpolation(size_t pos_embeds, size_t dst_height, size_t dst_width, bool align_corners = true); + + size_t layernorm(size_t input, size_t weight, size_t bias, float epsilon = 1e-5f); + size_t layernorm(size_t input, size_t weight, float epsilon = 1e-5f); // No bias version + size_t groupnorm(size_t input, size_t weight, size_t bias, size_t num_groups = 32, float epsilon = 1e-5f); + size_t batchnorm(size_t input, size_t weight, size_t bias, size_t running_mean, size_t running_var, int axis = 1, float epsilon = 1e-5f); + size_t topk(size_t input, size_t k); + size_t dense_mlp_int4_fused(size_t hidden, size_t gate_weight, + size_t up_weight, size_t down_weight); + size_t moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w3_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation = Activation::SILU, + size_t per_expert_scale = 0); + size_t moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation); + size_t rms_norm(size_t input, size_t weight, float epsilon = 1e-5f); + size_t rope(size_t input, float theta, size_t position_offset = 0, ComputeBackend backend = ComputeBackend::CPU); + size_t rope_gptj(size_t input, float theta, size_t position_offset = 0, size_t rot_dim = 0, ComputeBackend backend = ComputeBackend::CPU); + size_t softmax(size_t input, int axis = -1); + size_t attention(size_t query, size_t key, size_t value, float scale, bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU); + size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, ComputeBackend backend = ComputeBackend::CPU); + size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, size_t window_size, ComputeBackend backend = ComputeBackend::CPU); + size_t attention_masked(size_t query, size_t key, size_t value, size_t mask, float scale, + bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU, + bool additive_mask = false, size_t position_offset = 0, size_t window_size = 0, + float logit_cap = 0.0f); + size_t rel_pos_bias(size_t query, size_t relative_key, float scale); + + size_t attention_int8_hybrid(size_t query, size_t key_new, size_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, + const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, + size_t window_size = 0, size_t v_head_dim = 0); + + size_t conv1d_causal(size_t input, size_t weight, size_t kernel_size, size_t dilation = 1); + size_t conv1d_k3(size_t input, size_t weight, size_t stride); + size_t conv1d_k7s3(size_t input, size_t weight, size_t bias); + size_t conv1d(size_t input, size_t weight, size_t stride); + size_t conv1d(size_t input, size_t weight, size_t bias, size_t stride); + size_t conv1d_same_depthwise_k9(size_t input, size_t weight); + size_t conv1d_same_depthwise_k9(size_t input, size_t weight, size_t bias); + size_t conv1d_pointwise(size_t input, size_t weight); + size_t conv1d_pointwise(size_t input, size_t weight, size_t bias); + size_t conv2d_k3s2p1(size_t input, size_t weight); + size_t conv2d_k3s2p1(size_t input, size_t weight, size_t bias); + size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight); + size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight, size_t bias); + size_t conv2d_pointwise_1x1(size_t input, size_t weight); + size_t conv2d_pointwise_1x1(size_t input, size_t weight, size_t bias); + + size_t lstm_cell(size_t input, size_t h_prev, size_t c_prev, size_t weight_ih, size_t weight_hh, size_t bias_ih, size_t bias_hh); + size_t gated_deltanet_decode(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, float scale = 0.0f); + size_t gated_deltanet_prefill(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, size_t chunk_size = 64, float scale = 0.0f); + size_t stft(size_t input, size_t weight, size_t stride, size_t num_fft_bins); + + size_t altup_predict(size_t coefs, const size_t* streams, size_t num_streams); + size_t altup_correct(size_t coefs, size_t innovation, const size_t* predictions, size_t num_predictions); + + size_t gaussian_topk(size_t input, float ppf); + + size_t maxpool1d(size_t input, size_t kernel_size, size_t stride); + size_t leaky_relu(size_t input, float negative_slope = 0.01f); + size_t bilstm_sequence(size_t input, size_t w_ih_fwd, size_t w_hh_fwd, size_t b_ih_fwd, size_t b_hh_fwd, + size_t w_ih_bwd, size_t w_hh_bwd, size_t b_ih_bwd, size_t b_hh_bwd); + size_t conv2d_k3s1p1(size_t input, size_t weight); + size_t conv2d_k3s1p1(size_t input, size_t weight, size_t bias); + size_t stats_pool(size_t input); + size_t weighted_stats_pool(size_t input, size_t weights); + + size_t sample(size_t logits, float temperature = 0.6f, float top_p = 0.95f, size_t top_k = 20, + const std::unordered_map& logit_bias = {}); + size_t sample_with_options(size_t logits, float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, const std::unordered_map& logit_bias = {}); + + size_t concat(size_t input1, size_t input2, int axis = 0); + size_t cat(const std::vector& inputs, int axis); + size_t scatter_topk(size_t indices, size_t values, size_t num_classes); + + void set_input(size_t node_id, const void* data, Precision precision); + void set_external_input(size_t node_id, void* data, Precision precision); + void* get_output(size_t node_id); + + void execute(const std::string& profile_file = ""); + void hard_reset(); + void soft_reset(); + void soft_reset_keep_pool(); + void set_prefill_mode(bool enabled) { prefill_mode_ = enabled; } + + void register_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); + void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); + const std::vector& get_debug_nodes() const; + void clear_debug_nodes(); + + size_t add_node(OpType op_type, const std::vector& inputs, const std::vector& output_shape, const OpParams& params = {}); + const BufferDesc& get_output_buffer(size_t node_id) const; + void allocate_buffers(); + size_t get_node_count() const; + + size_t persistent(size_t source_node); + bool is_populated(size_t persistent_node_id) const; + void invalidate_persistent(size_t persistent_node_id); + + std::vector> nodes_; + std::unordered_map node_index_map_; + +private: + static CactusGraph from_serialized(const GraphFile::SerializedGraph& serialized); + size_t next_node_id_; + std::vector> mapped_files_; + std::unordered_map weight_cache_; + std::unordered_map node_to_mapped_file_; + std::vector debug_nodes_; + BufferPool buffer_pool_; + bool prefill_mode_ = false; + + std::unordered_set persistent_node_ids_; + std::unordered_set populated_node_ids_; +}; + + +namespace GraphFile { + struct LoadedNode { + size_t node_id; + std::vector shape; + Precision precision; + size_t byte_size; + }; + + struct GraphHeader { + uint32_t magic; + uint32_t version; + uint32_t node_count; + uint32_t flags = 0; + }; + + struct NodeEntry { + uint32_t index; // serialized node index 0..n-1 + OpType op_type; + std::vector inputs; + std::vector output_shape; + Precision precision; + OpParams params; + }; + + struct SerializedGraph { + GraphHeader header; + std::vector nodes; + std::vector graph_inputs; // IDs of serialized inputs + std::vector graph_outputs; // IDs of serialized outputs + }; + + SerializedGraph load_graph(const std::string& filename); + void save_graph(const CactusGraph& graph, const std::string& filename); + + void save_node(CactusGraph& graph, size_t node_id, const std::string& filename); + + class MappedFile { + public: + MappedFile(const std::string& filename); + ~MappedFile(); + + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + MappedFile(MappedFile&& other) noexcept; + MappedFile& operator=(MappedFile&& other) noexcept; + + const std::vector& shape() const; + Precision precision() const; + size_t byte_size() const; + + size_t group_size() const { return group_size_; } + size_t num_groups() const { return num_groups_; } + const void* scales_data() const; + + bool is_interleaved() const { return is_interleaved_; } + size_t original_N() const { return original_N_; } + + void* data(); + const void* data() const; + + template + const T* typed_data() const; + + void release_pages(); + void prefetch_pages(); + + private: + int fd_; + void* mapped_data_; + size_t file_size_, data_offset_; + std::vector shape_; + Precision precision_; + size_t byte_size_; + size_t group_size_ = 0; + size_t num_groups_ = 0; + size_t scales_offset_ = 0; + size_t scales_bytes_ = 0; + uint32_t alignment_ = 32; + + bool is_interleaved_ = false; + size_t original_N_ = 0; + + void parse_header(); + void apply_madvise_hints(); + }; +} + +#endif diff --git a/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/graph_param_io.h b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/graph_param_io.h new file mode 100644 index 0000000..0362ea8 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/graph_param_io.h @@ -0,0 +1,11 @@ +#pragma once + +#include "graph.h" +#include + +namespace GraphParamIO { + +void write_op_params(std::ostream& out, OpType op_type, const OpParams& params); +void read_op_params(std::istream& in, OpType op_type, OpParams& params); + +} // namespace GraphParamIO diff --git a/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/kernel.h b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/kernel.h new file mode 100644 index 0000000..13f4a00 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/kernel.h @@ -0,0 +1,475 @@ +#ifndef KERNEL_H +#define KERNEL_H + +#include +#include + +enum class Precision; + +enum class ScalarOpType { + ADD, + SUBTRACT, + MULTIPLY, + DIVIDE, + ABS, + EXP, + POW, + SQRT, + COS, + SIN, + LOG +}; + +constexpr size_t KV_QUANT_GROUP_SIZE = 32; + +void cactus_add_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_add_f16_clipped(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_subtract_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_multiply_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_add_scaled_f16(const __fp16* base, const __fp16* src, __fp16* output, size_t num_elements, float scale); +void cactus_divide_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); + +void cactus_add_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_subtract_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_multiply_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_divide_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); + +void cactus_scalar_op_f16(const __fp16* input, __fp16* output, size_t num_elements, float scalar_value, ScalarOpType op_type); + +void cactus_gemv_int8(const int8_t* A, float A_scale, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemm_int8(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_int8(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int8_i8mm(const int8_t* A, float A_scale, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemm_int8_i8mm(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int4(const int8_t* A, float A_scale, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int4_block_range(const int8_t* A, float A_scale, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size, + size_t block_start, size_t block_end); + +void cactus_gemm_int4(const int8_t* A, const float* A_scales, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_int4(const int8_t* A, const float* A_scales, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_integer(Precision precision, + const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_f16(const __fp16* a, const __fp16* b_transposed, __fp16* c, + size_t M, size_t K, size_t N); + +void cactus_transpose_2d_f16(const __fp16* source, __fp16* destination, + size_t num_rows, size_t num_cols, size_t start_row, size_t end_row); +void cactus_transpose_f16(const __fp16* source, __fp16* destination, const size_t* shape, + const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx); + +double cactus_sum_all_f16(const __fp16* data, size_t num_elements); +void cactus_sum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +double cactus_mean_all_f16(const __fp16* data, size_t num_elements); +void cactus_mean_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +double cactus_variance_all_f16(const __fp16* data, size_t num_elements); +void cactus_variance_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +__fp16 cactus_min_all_f16(const __fp16* data, size_t num_elements); +void cactus_min_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +__fp16 cactus_max_all_f16(const __fp16* data, size_t num_elements); +void cactus_max_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +void cactus_rms_norm_f16(const __fp16* input, const __fp16* weight, __fp16* output, + size_t batch_size, size_t dims, float eps); + +void cactus_layer_norm_f16(const __fp16* input, const __fp16* weight, const __fp16* bias, + __fp16* output, size_t batch_size, size_t dims, float eps); + +void cactus_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, + size_t num_heads, size_t head_dim, size_t start_pos, float theta); + +void cactus_gpt_j_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, + size_t num_heads, size_t head_dim, size_t rot_dim, size_t start_pos, float theta); + +void cactus_softmax_f16(const __fp16* input, __fp16* output, size_t batch_size, + size_t seq_len, size_t vocab_size); + +void cactus_relu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_leaky_relu_f16(const __fp16* input, __fp16* output, size_t num_elements, float negative_slope); + +void cactus_silu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_gelu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_gelu_f16_erf(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_sigmoid_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_tanh_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_glu_f16( + const __fp16* input, + __fp16* output, + size_t outer_size, + size_t split_size, + size_t inner_size +); + +void cactus_glu_f32( + const float* input, + float* output, + size_t outer_size, + size_t split_size, + size_t inner_size +); + +void cactus_batchnorm_f16( + const __fp16* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + __fp16* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +); + +void cactus_batchnorm_f32( + const float* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + float* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +); + +void cactus_attention_f16(const __fp16* queries, const __fp16* keys, const __fp16* values, __fp16* output, + size_t batch_size, size_t seq_len, size_t kv_seq_len, size_t num_q_heads, size_t num_kv_heads, + size_t head_dim, float scale, const __fp16* mask, size_t position_offset = 0, size_t window_size = 0, + bool is_causal = true, bool mask_is_additive = false, bool mask_per_head = false, + size_t v_head_dim = 0, float logit_cap = 0.0f); + +void cactus_attention_hybrid_int8_fp16( + const __fp16* queries, + const int8_t* keys_cached, + const int8_t* values_cached, + const float* k_scales, + const float* v_scales, + const __fp16* keys_new, + const __fp16* values_new, + __fp16* output, + size_t batch_size, size_t seq_len, size_t cache_len, size_t new_len, + size_t num_q_heads, size_t num_kv_heads, size_t head_dim, + float scale, size_t position_offset = 0, bool is_causal = true, size_t window_size = 0, + size_t group_size = KV_QUANT_GROUP_SIZE, size_t v_head_dim = 0); + +void cactus_gated_deltanet_decode_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + float scale); + +void cactus_gated_deltanet_prefill_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t T, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + size_t requested_chunk_size, + float scale); + +void cactus_conv1d_causal_depthwise_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, + size_t L, + size_t C, + size_t K, + size_t dilation); + +void cactus_conv1d_f16_k3( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out, + size_t stride +); + +void cactus_conv1d_f16( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out, + size_t K, + size_t stride +); + +void cactus_stft_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, size_t L, + size_t C_in, size_t C_out, + size_t K, size_t stride, + size_t num_fft_bins +); + +void cactus_conv1d_f16_k7s3_oc8( + const __fp16* input, + const __fp16* Wpack, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out +); + +void cactus_conv1d_same_depthwise_f16_k9( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C +); + +void cactus_conv2d_f16_k3s1p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv2d_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv2d_depthwise_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C, + size_t H, + size_t W +); + +void cactus_conv2d_pointwise_f16_1x1_nchw_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv1d_pointwise_f16_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out +); + +void cactus_bilinear_interpolation_f16(const __fp16* input, __fp16* output, size_t src_height, size_t src_width, size_t embed_dim, + size_t dst_height, size_t dst_width, bool align_corners = true); + +void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); +void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); + +void cactus_sample_f32_ex(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); +void cactus_sample_f16_ex(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); + +void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* output, + const size_t* shape1, const size_t* shape2, const size_t* output_shape, + size_t ndims, int axis); +void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_shapes, + const size_t* output_shape, size_t num_inputs, size_t rank, int axis); + +void cactus_int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); +void cactus_fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); +void cactus_fp16_to_fp32(const __fp16* src, float* dst, size_t count); +void cactus_fp32_to_fp16(const float* src, __fp16* dst, size_t count); +void cactus_int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); +void cactus_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); +float cactus_fp16_max_abs(const __fp16* src, size_t count); + +void cactus_quantize_kv_fp16_to_int8( + const __fp16* src, + int8_t* dst, + float* scales, + size_t seq_len, size_t kv_heads, size_t head_dim, + size_t group_size = KV_QUANT_GROUP_SIZE); + +inline size_t kv_scales_count(size_t seq_len, size_t kv_heads, size_t head_dim, size_t group_size = KV_QUANT_GROUP_SIZE) { + size_t num_groups = (head_dim + group_size - 1) / group_size; + return seq_len * kv_heads * num_groups; +} + +void cactus_unpack_int4_to_int8(const uint8_t* packed, int8_t* unpacked, size_t unpacked_count); + +void cactus_gaussian_topk_f16( + const __fp16* input, + __fp16* output, + size_t rows, + size_t cols, + float ppf); + +void cactus_altup_predict_f16( + const __fp16* coefs, + const __fp16* const* streams, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim); + +void cactus_altup_correct_f16( + const __fp16* coefs, + const __fp16* innovation, + const __fp16* const* predictions, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim); + +void cactus_lstm_cell_f16( + const __fp16* x_input, + const __fp16* h_prev, + const __fp16* c_prev, + const __fp16* weight_ih, + const __fp16* weight_hh, + const __fp16* bias_ih, + const __fp16* bias_hh, + __fp16* h_new, + __fp16* c_new, + size_t batch_size, + size_t input_size, + size_t hidden_size +); + +void cactus_bilstm_sequence_f16( + const __fp16* input, + const __fp16* weight_ih_fwd, + const __fp16* weight_hh_fwd, + const __fp16* bias_ih_fwd, + const __fp16* bias_hh_fwd, + const __fp16* weight_ih_bwd, + const __fp16* weight_hh_bwd, + const __fp16* bias_ih_bwd, + const __fp16* bias_hh_bwd, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t input_size, + size_t hidden_size +); + +void cactus_maxpool1d_f16( + const __fp16* input, + __fp16* output, + size_t batch_size, + size_t channels, + size_t input_length, + size_t kernel_size, + size_t stride +); + +#endif diff --git a/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/kernel_utils.h b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/kernel_utils.h new file mode 100644 index 0000000..b202158 --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/kernel_utils.h @@ -0,0 +1,681 @@ +#ifndef KERNEL_UTILS_H +#define KERNEL_UTILS_H + +#include +#if defined(__APPLE__) +#include +#include +#endif +#if defined(__ANDROID__) +#include +#include +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr size_t NEON_VECTOR_SIZE = 16; +constexpr size_t STREAMING_STORE_THRESHOLD = 32768; + +inline void stream_store_f16x8(__fp16* dst, float16x8_t val) { +#if defined(__aarch64__) + float16x4_t lo = vget_low_f16(val); + float16x4_t hi = vget_high_f16(val); + __asm__ __volatile__( + "stnp %d0, %d1, [%2]" + : + : "w"(lo), "w"(hi), "r"(dst) + : "memory" + ); +#else + vst1q_f16(dst, val); +#endif +} + +inline bool cpu_has_i8mm() { +#if defined(__aarch64__) + static std::once_flag once; + static bool has = false; + + std::call_once(once, []() { +#if defined(__APPLE__) + int ret = 0; + size_t size = sizeof(ret); + if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &ret, &size, nullptr, 0) == 0) { + has = (ret == 1); + } +#elif defined(__ANDROID__) + unsigned long hwcap2 = getauxval(AT_HWCAP2); + #ifndef HWCAP2_I8MM + #define HWCAP2_I8MM (1 << 13) + #endif + has = (hwcap2 & HWCAP2_I8MM) != 0; +#endif + }); + + return has; +#else + return false; +#endif +} + +inline float32x4_t fast_exp_f32x4(float32x4_t x) { + const float32x4_t log2e = vdupq_n_f32(1.4426950408889634f); + const float32x4_t ln2 = vdupq_n_f32(0.6931471805599453f); + + const float32x4_t c0 = vdupq_n_f32(1.0f); + const float32x4_t c1 = vdupq_n_f32(0.6931471805599453f); + const float32x4_t c2 = vdupq_n_f32(0.2402265069591007f); + const float32x4_t c3 = vdupq_n_f32(0.05550410866482158f); + const float32x4_t c4 = vdupq_n_f32(0.009618129842071803f); + + x = vmaxq_f32(x, vdupq_n_f32(-87.0f)); + x = vminq_f32(x, vdupq_n_f32(87.0f)); + + float32x4_t z = vmulq_f32(x, log2e); + + int32x4_t zi = vcvtq_s32_f32(z); + float32x4_t zf = vsubq_f32(z, vcvtq_f32_s32(zi)); + + uint32x4_t neg_mask = vcltq_f32(zf, vdupq_n_f32(0.0f)); + zi = vsubq_s32(zi, vandq_s32(vreinterpretq_s32_u32(neg_mask), vdupq_n_s32(1))); + zf = vaddq_f32(zf, vreinterpretq_f32_u32(vandq_u32(neg_mask, vreinterpretq_u32_f32(vdupq_n_f32(1.0f))))); + + float32x4_t zf_ln2 = vmulq_f32(zf, ln2); + float32x4_t p = c4; + p = vfmaq_f32(c3, p, zf_ln2); + p = vfmaq_f32(c2, p, zf_ln2); + p = vfmaq_f32(c1, p, zf_ln2); + p = vfmaq_f32(c0, p, zf_ln2); + + int32x4_t exp_bits = vshlq_n_s32(vaddq_s32(zi, vdupq_n_s32(127)), 23); + float32x4_t scale = vreinterpretq_f32_s32(exp_bits); + + return vmulq_f32(p, scale); +} + +// Cephes-style 13/6 rational tanh approximation (same coefficients as Eigen). +// Constants are stored as static splatted arrays so the compiler emits a single +// pc-relative `ldr q` per load. +alignas(16) inline constexpr float kFastTanhAlpha[7][4] = { + { 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f }, + { 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f }, + { 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f }, + { 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f }, + {-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f }, + { 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f }, + {-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f }, +}; +alignas(16) inline constexpr float kFastTanhBeta[4][4] = { + { 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f }, + { 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f }, + { 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f }, + { 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f }, +}; +alignas(16) inline constexpr float kFastTanhClampHi[4] = { 9.0f, 9.0f, 9.0f, 9.0f }; +alignas(16) inline constexpr float kFastTanhClampLo[4] = {-9.0f,-9.0f,-9.0f,-9.0f }; + +inline float32x4_t fast_tanh_f32x4(float32x4_t x) { + x = vmaxq_f32(vld1q_f32(kFastTanhClampLo), vminq_f32(vld1q_f32(kFastTanhClampHi), x)); + float32x4_t x2 = vmulq_f32(x, x); + float32x4_t p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[5]), vld1q_f32(kFastTanhAlpha[6]), x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[4]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[3]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[2]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[1]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[0]), p, x2); + p = vmulq_f32(p, x); + float32x4_t q = vfmaq_f32(vld1q_f32(kFastTanhBeta[2]), vld1q_f32(kFastTanhBeta[3]), x2); + q = vfmaq_f32(vld1q_f32(kFastTanhBeta[1]), q, x2); + q = vfmaq_f32(vld1q_f32(kFastTanhBeta[0]), q, x2); + return vdivq_f32(p, q); +} + +constexpr size_t SIMD_F16_WIDTH = 8; + +inline size_t simd_align(size_t count, size_t width = SIMD_F16_WIDTH) { + return (count / width) * width; +} + +inline void f16x8_split_f32(float16x8_t v, float32x4_t& lo, float32x4_t& hi) { + lo = vcvt_f32_f16(vget_low_f16(v)); + hi = vcvt_f32_f16(vget_high_f16(v)); +} + +inline float16x8_t f32_merge_f16(float32x4_t lo, float32x4_t hi) { + return vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi)); +} + +inline float32x4_t fast_sigmoid_f32x4(float32x4_t x) { + const float32x4_t one = vdupq_n_f32(1.0f); + return vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(x)))); +} + +template +inline float16x8_t apply_f32_op_on_f16x8(float16x8_t v, F32x4Op op) { + float32x4_t lo, hi; + f16x8_split_f32(v, lo, hi); + return f32_merge_f16(op(lo), op(hi)); +} + +inline void unpack_int4_as_int8x16x2(const uint8_t* ptr, int8x16_t& high_decoded, int8x16_t& low_decoded) { + int8x16_t packed = vreinterpretq_s8_u8(vld1q_u8(ptr)); + high_decoded = vshrq_n_s8(packed, 4); + low_decoded = vshrq_n_s8(vshlq_n_s8(packed, 4), 4); +} + +namespace CactusThreading { + +#if defined(__ANDROID__) + struct CoreTopology { + std::vector performance_cores; + std::vector all_cores; + + static CoreTopology& get() { + static CoreTopology topo = detect(); + return topo; + } + + private: + static int read_sysfs_int(const char* path) { + std::ifstream f(path); + if (!f.is_open()) return -1; + int val = -1; + f >> val; + return val; + } + + static CoreTopology detect() { + CoreTopology topo; + constexpr int MAX_CPUS = 16; + std::vector> core_caps; + + for (int i = 0; i < MAX_CPUS; ++i) { + char path[128]; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpu_capacity", i); + int cap = read_sysfs_int(path); + if (cap > 0) { + core_caps.push_back({i, cap}); + topo.all_cores.push_back(i); + continue; + } + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", i); + int freq = read_sysfs_int(path); + if (freq > 0) { + core_caps.push_back({i, freq}); + topo.all_cores.push_back(i); + } + } + + if (core_caps.empty()) return topo; + + int max_cap = 0; + for (auto& [id, cap] : core_caps) { + max_cap = std::max(max_cap, cap); + } + + int threshold = static_cast(max_cap * 0.70); + for (auto& [id, cap] : core_caps) { + if (cap >= threshold) { + topo.performance_cores.push_back(id); + } + } + + return topo; + } + }; + + inline bool pin_current_thread_to_cores(const std::vector& cores) { + if (cores.empty()) return false; + cpu_set_t mask; + CPU_ZERO(&mask); + for (int core : cores) { + CPU_SET(core, &mask); + } + return sched_setaffinity(0, sizeof(mask), &mask) == 0; + } +#endif + + class ThreadPool { + private: + static constexpr size_t MAX_WORKERS = 16; + + std::vector workers; + std::deque> tasks; + + std::mutex mutex; + std::condition_variable work_available; + std::condition_variable work_done; + + bool stop{false}; + std::atomic pending_tasks{0}; + size_t num_workers_; + + void worker_thread() { + while (true) { + std::function task; + { + std::unique_lock lock(mutex); + work_available.wait(lock, [this] { + return stop || !tasks.empty(); + }); + + if (stop && tasks.empty()) { + return; + } + + task = std::move(tasks.front()); + tasks.pop_front(); + } + + task(); + + if (pending_tasks.fetch_sub(1, std::memory_order_acq_rel) == 1) { + std::lock_guard lock(mutex); + work_done.notify_one(); + } + } + } + + public: + explicit ThreadPool(size_t num_threads = std::thread::hardware_concurrency()) + : stop(false), pending_tasks(0) { + num_workers_ = std::min(num_threads, MAX_WORKERS); + if (num_workers_ == 0) num_workers_ = 1; + +#if defined(__ANDROID__) + auto& topo = CoreTopology::get(); + if (!topo.performance_cores.empty()) { + num_workers_ = std::min(num_workers_, topo.performance_cores.size()); + } +#endif + + workers.reserve(num_workers_); + for (size_t i = 0; i < num_workers_; ++i) { + workers.emplace_back([this]() { +#if defined(__ANDROID__) + auto& perf = CoreTopology::get().performance_cores; + if (!perf.empty()) { + pin_current_thread_to_cores(perf); + } +#endif + worker_thread(); + }); + } + } + + ~ThreadPool() { + { + std::lock_guard lock(mutex); + stop = true; + } + work_available.notify_all(); + for (auto& worker : workers) { + if (worker.joinable()) { + worker.join(); + } + } + } + + template + auto enqueue(F&& f) -> std::future { + using return_type = decltype(f()); + + auto task = std::make_shared>( + std::forward(f) + ); + + std::future res = task->get_future(); + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(1, std::memory_order_relaxed); + tasks.emplace_back([task](){ (*task)(); }); + } + work_available.notify_one(); + + return res; + } + + template + void enqueue_batch(size_t total_work, F task_func) { + if (total_work == 0) return; + + const size_t num_tasks = std::min(num_workers_, total_work); + const size_t per_worker = total_work / num_tasks; + const size_t remainder = total_work % num_tasks; + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(num_tasks, std::memory_order_relaxed); + + for (size_t w = 0; w < num_tasks; ++w) { + size_t start = w * per_worker + std::min(w, remainder); + size_t end = start + per_worker + (w < remainder ? 1 : 0); + tasks.emplace_back([=]() { task_func(start, end); }); + } + } + work_available.notify_all(); + } + + void wait_all() { + std::unique_lock lock(mutex); + work_done.wait(lock, [this] { + return pending_tasks.load(std::memory_order_acquire) == 0; + }); + } + + template + void enqueue_n_threads(size_t total_work, size_t num_threads, F task_func) { + if (total_work == 0 || num_threads == 0) return; + + num_threads = std::min(num_threads, std::min(num_workers_, total_work)); + const size_t per_thread = total_work / num_threads; + const size_t remainder = total_work % num_threads; + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(num_threads, std::memory_order_relaxed); + + for (size_t t = 0; t < num_threads; ++t) { + size_t start = t * per_thread + std::min(t, remainder); + size_t end = start + per_thread + (t < remainder ? 1 : 0); + tasks.emplace_back([=]() { task_func(start, end); }); + } + } + work_available.notify_all(); + } + + size_t num_workers() const { return num_workers_; } + }; + + inline ThreadPool& get_thread_pool() { + static ThreadPool pool; + return pool; + } + + struct ParallelConfig { + size_t min_work_gate; + size_t work_per_thread; + + constexpr ParallelConfig(size_t gate, size_t per_thread) + : min_work_gate(gate), work_per_thread(per_thread) {} + }; + + inline size_t get_optimal_thread_count(size_t total_work, ParallelConfig config) { + if (total_work < config.min_work_gate) return 1; + + size_t pool_size = get_thread_pool().num_workers(); + size_t num_threads = (total_work + config.work_per_thread - 1) / config.work_per_thread; + return std::min(pool_size, std::max(static_cast(1), num_threads)); + } + + struct Thresholds { + #if defined(__ANDROID__) + static constexpr ParallelConfig ATTENTION{64, 32}; + static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; + static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; + static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; + static constexpr ParallelConfig SCALAR_BASIC{30000, 15000}; + static constexpr ParallelConfig SCALAR_EXPENSIVE{10000, 5000}; + #else // Apple + static constexpr ParallelConfig ATTENTION{32, 16}; + static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; + static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; + static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; + static constexpr ParallelConfig SCALAR_BASIC{5000, 2500}; + static constexpr ParallelConfig SCALAR_EXPENSIVE{2500, 1250}; + #endif + }; + + struct GemmThreading { + #if defined(__ANDROID__) + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return 1; + return pool_size; + } + static size_t get_gemv_threads(size_t /*N_blocks*/, size_t /*pool_size*/) { + return 1; + } + #elif defined(__APPLE__) && TARGET_OS_IPHONE + static constexpr size_t GEMV_MIN_N_BLOCKS = 512; + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return std::min(pool_size, static_cast(2)); + return pool_size; + } + static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { + if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; + return std::min(pool_size, static_cast(3)); + } + #else + static constexpr size_t GEMV_MIN_N_BLOCKS = 256; + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return std::min(pool_size, static_cast(4)); + return pool_size; + } + static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { + if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; + if (N_blocks < 512) return std::min(pool_size, static_cast(2)); + return std::min(pool_size, static_cast(5)); + } + #endif + }; + + inline size_t& get_gemm_thread_override() { + static size_t override_threads = 0; + return override_threads; + } + + inline void set_gemm_threads(size_t num_threads) { + get_gemm_thread_override() = num_threads; + } + + inline void reset_gemm_threads() { + get_gemm_thread_override() = 0; + } + + class TaskHandle { + private: + std::vector> futures_; + bool auto_wait_; + + public: + TaskHandle(bool auto_wait = true) : auto_wait_(auto_wait) {} + + ~TaskHandle() { + if (auto_wait_) { + wait(); + } + } + + TaskHandle(TaskHandle&&) = default; + TaskHandle& operator=(TaskHandle&&) = default; + TaskHandle(const TaskHandle&) = delete; + TaskHandle& operator=(const TaskHandle&) = delete; + + void add_future(std::future&& f) { + futures_.push_back(std::move(f)); + } + + void wait() { + for (auto& f : futures_) { + if (f.valid()) { + f.wait(); + } + } + futures_.clear(); + } + + bool is_ready() const { + for (const auto& f : futures_) { + if (f.valid() && f.wait_for(std::chrono::seconds(0)) != std::future_status::ready) { + return false; + } + } + return true; + } + + size_t task_count() const { return futures_.size(); } + }; + + template + TaskHandle parallel_for(size_t total_work, ParallelConfig config, WorkFunc work_func, bool wait = true) { + const size_t num_threads = get_optimal_thread_count(total_work, config); + TaskHandle handle(!wait); + + if (num_threads == 1) { + if (wait) { + work_func(0, total_work); + return handle; + } + auto& pool = get_thread_pool(); + handle.add_future(pool.enqueue([work_func, total_work]() { + work_func(0, total_work); + })); + return handle; + } + + auto& pool = get_thread_pool(); + const size_t work_per_thread = total_work / num_threads; + + for (size_t t = 0; t < num_threads; ++t) { + handle.add_future(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() { + const size_t start_idx = t * work_per_thread; + const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; + work_func(start_idx, end_idx); + })); + } + + if (wait) { + handle.wait(); + } + return handle; + } + + template + void parallel_for_2d(size_t outer_size, size_t inner_size, ParallelConfig config, WorkFunc work_func) { + const size_t total_work = outer_size * inner_size; + parallel_for(total_work, config, [&](size_t start_idx, size_t end_idx) { + for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { + const size_t outer = work_idx / inner_size; + const size_t inner = work_idx % inner_size; + work_func(outer, inner); + } + }); + } + + template + ResultType parallel_reduce(size_t total_work, ParallelConfig config, + WorkFunc work_func, ResultType init_value, CombineFunc combine_func) { + const size_t num_threads = get_optimal_thread_count(total_work, config); + + if (num_threads == 1) { + return work_func(0, total_work); + } + + auto& pool = get_thread_pool(); + std::vector> futures; + std::vector partial_results(num_threads, init_value); + const size_t work_per_thread = total_work / num_threads; + + for (size_t t = 0; t < num_threads; ++t) { + futures.push_back(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() -> ResultType { + const size_t start_idx = t * work_per_thread; + const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; + return work_func(start_idx, end_idx); + })); + } + + ResultType result = init_value; + for (auto& future : futures) { + result = combine_func(result, future.get()); + } + return result; + } + + template + void parallel_gemm_tiles(size_t M, size_t total_tiles, WorkFunc work_func) { + auto& pool = get_thread_pool(); + + size_t override = get_gemm_thread_override(); + size_t num_threads = (override > 0) ? override : GemmThreading::get_num_threads(M, pool.num_workers()); + num_threads = std::min(num_threads, total_tiles); + + if (num_threads <= 1) { + work_func(0, total_tiles); + return; + } + + pool.enqueue_n_threads(total_tiles, num_threads, work_func); + pool.wait_all(); + } + +} + +template +void elementwise_op_f16(const __fp16* input, __fp16* output, size_t num_elements, + bool use_streaming, CactusThreading::ParallelConfig config, + SimdOp simd_op, ScalarOp scalar_op, size_t unroll = 4) { + CactusThreading::parallel_for(num_elements, config, + [&](size_t start, size_t end) { + const size_t n = end - start; + const size_t vec_end = start + simd_align(n); + + if (use_streaming && unroll >= 4) { + const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 4); + for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 4) { + __builtin_prefetch(&input[i + 256], 0, 0); + float16x8_t v0 = simd_op(vld1q_f16(&input[i])); + float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); + float16x8_t v2 = simd_op(vld1q_f16(&input[i + 16])); + float16x8_t v3 = simd_op(vld1q_f16(&input[i + 24])); + stream_store_f16x8(&output[i], v0); + stream_store_f16x8(&output[i + 8], v1); + stream_store_f16x8(&output[i + 16], v2); + stream_store_f16x8(&output[i + 24], v3); + } + for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { + stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } else if (use_streaming && unroll >= 2) { + const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 2); + for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 2) { + __builtin_prefetch(&input[i + 128], 0, 0); + float16x8_t v0 = simd_op(vld1q_f16(&input[i])); + float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); + stream_store_f16x8(&output[i], v0); + stream_store_f16x8(&output[i + 8], v1); + } + for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { + stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } else { + for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { + vst1q_f16(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } + for (size_t i = vec_end; i < end; ++i) { + output[i] = scalar_op(input[i]); + } + }); +} + +#endif // KERNEL_UTILS_H diff --git a/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/needle_tools.h b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/needle_tools.h new file mode 100644 index 0000000..4330cff --- /dev/null +++ b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Headers/needle_tools.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace needle { + +inline std::string to_snake_case(const std::string& name) { + std::string s; + s.reserve(name.size() * 2); + for (char c : name) { + if (std::isalnum(static_cast(c)) || c == '_') { + s += c; + } else { + if (s.empty() || s.back() != '_') s += '_'; + } + } + + std::string s2; + s2.reserve(s.size() * 2); + for (size_t i = 0; i < s.size(); i++) { + char c = s[i]; + if (i > 0 && std::isupper(static_cast(c))) { + char prev = s[i - 1]; + if (std::islower(static_cast(prev)) || std::isdigit(static_cast(prev))) { + s2 += '_'; + } + } + s2 += c; + } + + std::string s3; + s3.reserve(s2.size() * 2); + for (size_t i = 0; i < s2.size(); i++) { + s3 += s2[i]; + if (i + 1 < s2.size() && + std::isupper(static_cast(s2[i])) && + std::isupper(static_cast(s2[i + 1]))) { + if (i + 2 < s2.size() && std::islower(static_cast(s2[i + 2]))) { + s3 += '_'; + } + } + } + + std::string result; + result.reserve(s3.size()); + bool prev_underscore = false; + for (char c : s3) { + if (c == '_') { + if (!prev_underscore) result += '_'; + prev_underscore = true; + } else { + result += static_cast(std::tolower(static_cast(c))); + prev_underscore = false; + } + } + + size_t start = result.find_first_not_of('_'); + if (start == std::string::npos) return result; + size_t end = result.find_last_not_of('_'); + return result.substr(start, end - start + 1); +} + +inline void restore_tool_names(std::vector& function_calls, + const std::unordered_map& name_map) { + if (name_map.empty()) return; + for (auto& call : function_calls) { + for (const auto& [snake, orig] : name_map) { + std::string from = "\"name\":\"" + snake + "\""; + size_t pos = call.find(from); + if (pos == std::string::npos) { + from = "\"name\": \"" + snake + "\""; + pos = call.find(from); + } + if (pos != std::string::npos) { + std::string to = from.substr(0, from.size() - snake.size() - 1) + orig + "\""; + call.replace(pos, from.size(), to); + break; + } + } + } +} + +} // namespace needle diff --git a/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Info.plist b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Info.plist new file mode 100644 index 0000000..8104ada Binary files /dev/null and b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/Info.plist differ diff --git a/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/cactus b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/cactus new file mode 100755 index 0000000..cf91649 Binary files /dev/null and b/ios/cactus-ios.xcframework/ios-arm64/cactus.framework/cactus differ diff --git a/lib/audio/audio_prompt_resolver.dart b/lib/audio/audio_prompt_resolver.dart new file mode 100644 index 0000000..7de2573 --- /dev/null +++ b/lib/audio/audio_prompt_resolver.dart @@ -0,0 +1,145 @@ +// Pure prompt-routing for the flat-bundle audio path. Pulled out of +// `_sendAudioBackedPrompt` so the four-branch decision tree is a +// testable function rather than nested ifs in an async method. +// +// Decisions captured here (in order, first match wins): +// +// 1. User typed text alongside the audio → use the typed text as the +// user content; pair with the bundle's system prompt override +// (or null → caller falls back to the classroom prompt). +// 2. Bundle declares audioModes → use the mode-based SFT prompt +// (Transcribe / Answer) the user pre-selected. Per-mode override +// from `audioPromptOverrides` takes precedence — empty string is +// honoured (the QA bundle pairs audio with a blank user prompt). +// System message = bundle override OR kTrainedSystemPrompt. +// 3. Bundle has a legacy `audioUserPrompt` but no modes → use that. +// Kept for back-compat / custom installs. +// 4. Fallback → caller's `classroomFallback` user content with no +// system override (caller pairs with the classroom prompt). +// +// All decisions encoded into [AudioPromptPlan] so the caller stays +// declarative. + +import '../classroom_prompt.dart'; +import '../conversation/cards.dart'; +import '../languages.dart'; +import '../model_settings_sheet.dart'; +import 'audio_types.dart'; + +class AudioPromptPlan { + /// User content for the messages JSON. Empty string is intentional + /// (the QA bundle's Answer mode trains on a blank user message). + final String userContent; + + /// System prompt override. `null` → caller pairs with its default + /// classroom system prompt. + final String? systemPrompt; + + /// What kind of card the response should render as. + final CardType cardType; + + /// Language the response will be in. + final Language responseLang; + + /// Source language for cross-language answer cards. `null` when the + /// response language is also the source (transcribe mode, or when + /// the user typed in the same language they spoke). + final Language? sourceLang; + + /// In-progress status string for diagnostics. + final String inProgressStatus; + + const AudioPromptPlan({ + required this.userContent, + required this.systemPrompt, + required this.cardType, + required this.responseLang, + required this.sourceLang, + required this.inProgressStatus, + }); +} + +/// Resolve the prompt + card plan for one flat-bundle audio turn. +/// +/// `audioModes` is the user-mode-eligible subset for the currently +/// loaded bundle (computed by caller from `bundle.audioModes`). When +/// it is empty, the bundle doesn't declare modes — branches 3/4 apply. +AudioPromptPlan resolveAudioPrompt({ + required KnownModel? bundle, + required List audioModes, + required AudioMode currentMode, + required Language sourceLang, + required Language responseLang, + required String typedText, + required String classroomFallback, +}) { + final hasTyped = typedText.isNotEmpty; + + if (hasTyped) { + return AudioPromptPlan( + userContent: typedText, + systemPrompt: bundle?.systemPrompt, + cardType: CardType.answer, + responseLang: responseLang, + sourceLang: sourceLang != responseLang ? sourceLang : null, + inProgressStatus: 'Thinking with audio in ${responseLang.name}...', + ); + } + + if (audioModes.isNotEmpty) { + final override = bundle?.audioPromptOverrides[currentMode]; + final userContent = + override ?? _defaultUserPromptForMode(currentMode, sourceLang); + final systemPrompt = bundle?.systemPrompt ?? kTrainedSystemPrompt; + final isTranscribe = currentMode == AudioMode.transcribe; + final cardType = + isTranscribe ? CardType.voiceTranscript : CardType.answer; + final cardResponseLang = isTranscribe ? sourceLang : responseLang; + // sourceLang is only meaningful on answer cards in the mode-aware + // flat path; transcripts self-label with the spoken language. + final cardSourceLang = isTranscribe ? null : sourceLang; + final inProgress = isTranscribe + ? 'Transcribing ${sourceLang.name}...' + : 'Listening in ${responseLang.name}...'; + return AudioPromptPlan( + userContent: userContent, + systemPrompt: systemPrompt, + cardType: cardType, + responseLang: cardResponseLang, + sourceLang: cardSourceLang, + inProgressStatus: inProgress, + ); + } + + final legacyPrompt = bundle?.audioUserPrompt; + if (legacyPrompt != null) { + return AudioPromptPlan( + userContent: legacyPrompt, + systemPrompt: bundle?.systemPrompt, + cardType: CardType.answer, + responseLang: responseLang, + sourceLang: null, + inProgressStatus: 'Listening in ${responseLang.name}...', + ); + } + + return AudioPromptPlan( + userContent: classroomFallback, + systemPrompt: null, + cardType: CardType.answer, + responseLang: responseLang, + sourceLang: null, + inProgressStatus: 'Listening in ${responseLang.name}...', + ); +} + +/// Default audio user-prompt template per mode. Verbatim from the SFT +/// corpus — diverging from these strings drops on-distribution quality. +String _defaultUserPromptForMode(AudioMode mode, Language sourceLang) { + switch (mode) { + case AudioMode.transcribe: + return 'Please transcribe this ${sourceLang.name} audio.'; + case AudioMode.answer: + return 'Please answer this spoken question.'; + } +} diff --git a/lib/audio/audio_types.dart b/lib/audio/audio_types.dart new file mode 100644 index 0000000..10ab0d3 --- /dev/null +++ b/lib/audio/audio_types.dart @@ -0,0 +1,60 @@ +// Shared audio domain types. Previously private to `main.dart`, which +// meant the catalogue in `model_settings_sheet.dart` declared audio +// modes as strings ('answer' / 'transcribe') and the page parsed them +// back to an enum — a typo there silently disabled a bundle's audio. +// Lifting these to a public module lets the catalogue (and tests) +// reference the enum directly. + +import '../languages.dart'; + +/// Lifecycle of the input bar's mic mode. +/// +/// idle ──tap mic──> recording +/// recording ──tap stop──> idle (auto-transcribed) +/// recording ──silence end-pointing──> idle (auto-transcribed) +/// recording ──15s cap──> idle (auto-transcribed) +/// recording ──tap cancel──> idle (discarded, no transcribe) +enum RecPhase { idle, recording } + +/// What the user wants done with a just-transcribed voice clip. +/// Both map to trained SFT prefixes — see the audio prompt resolver. +enum VoiceIntent { translate, explain } + +/// Audio task families the user can pre-select for flat-bundle voice +/// turns (the audio_tower-direct path, not the stitched cascade). +/// Each value maps to a verbatim SFT-trained user prompt. Per-bundle +/// availability is declared in `KnownModel.audioModes`; the page picks +/// the first available mode as the default when a bundle is loaded. +enum AudioMode { transcribe, answer } + +String audioModeLabel(AudioMode m) { + switch (m) { + case AudioMode.transcribe: + return 'Transcribe'; + case AudioMode.answer: + return 'Answer'; + } +} + +/// Parses a catalogue string back to an [AudioMode]. Returns null on +/// unknown values — callers should either error out (typo in catalogue) +/// or filter them away with `whereType()`. +AudioMode? audioModeFromName(String name) { + for (final m in AudioMode.values) { + if (m.name == name) return m; + } + return null; +} + +/// In-flight transcript awaiting a user intent. Lives on the page so +/// the input bar can swap to the review UI in place of the recorder / +/// text input — no modal context-switch between hearing the transcript +/// and committing to an action. +class VoiceReviewState { + final String transcript; + final Language target; + const VoiceReviewState({required this.transcript, required this.target}); + + VoiceReviewState withTarget(Language language) => + VoiceReviewState(transcript: transcript, target: language); +} diff --git a/lib/audio_trim.dart b/lib/audio_trim.dart index 18a84a9..3b25500 100644 --- a/lib/audio_trim.dart +++ b/lib/audio_trim.dart @@ -1,21 +1,16 @@ import 'dart:io'; import 'dart:typed_data'; +import 'package:flutter/foundation.dart'; -/// Trims a 16 kHz / 16-bit / mono WAV file in-place to the time window -/// `[startMs, endMs]`, returning the same path on success. Used by the -/// recording flow to drop leading/trailing silence before the clip -/// reaches `cactusComplete` — the Gemma 4 audio_tower's FP16 encode -/// cost is linear in clip length, so 5 s of trimmed silence is roughly -/// 5 s less TTFT. +/// Trims a 16 kHz / 16-bit / mono audio file in-place to the time window +/// `[startMs, endMs]`, returning the same path on success. /// -/// Best-effort by design: any parse failure (header doesn't look like -/// our RecordConfig output, "data" chunk missing, computed slice is -/// empty) is swallowed and the original path is returned. The audio -/// is more important than the optimisation. +/// This function enforces a clean 44-byte canonical WAV header (RIFF + fmt + data) +/// regardless of whether the source was raw PCM (iOS) or a non-standard WAV. +/// This prevents "Missing fmt chunk" errors in downstream Cactus FFI calls. /// -/// Assumes the bytes-per-ms constant `32` from a 16000 Hz × 2 bytes -/// (16-bit) × 1 channel recording. If the recorder config ever -/// changes, update both ends. +/// Best-effort by design: if the file is missing or too short, it returns the +/// path untouched. String trimWavInPlace({ required String path, required int startMs, @@ -25,64 +20,105 @@ String trimWavInPlace({ try { final file = File(path); if (!file.existsSync()) return path; + final bytes = file.readAsBytesSync(); - if (bytes.length < 44) return path; - - // Scan chunks until we find "data". The record package emits the - // standard 44-byte PCM header in practice, but some encoders - // insert LIST/INFO chunks first — finding the marker is cheap and - // makes us robust to that. - int dataIdx = -1; - for (var i = 12; i < bytes.length - 8; i++) { - if (bytes[i] == 0x64 && - bytes[i + 1] == 0x61 && - bytes[i + 2] == 0x74 && - bytes[i + 3] == 0x61) { - dataIdx = i; - break; + if (bytes.length < 2) return path; // Need at least one sample + + // --- 1. Locate PCM data in the source file --- + + int pcmStart = 0; + int pcmEnd = bytes.length; + + // Check for RIFF/WAVE signature at the very start. + final hasRiff = bytes.length >= 12 && + bytes[0] == 0x52 && bytes[1] == 0x49 && + bytes[2] == 0x46 && bytes[3] == 0x46 && + bytes[8] == 0x57 && bytes[9] == 0x41 && + bytes[10] == 0x56 && bytes[11] == 0x45; + + if (hasRiff) { + debugPrint('[trim] found WAV header at $path'); + // Walk RIFF chunks to find the "data" sub-chunk. + int i = 12; + while (i < bytes.length - 8) { + final id = String.fromCharCodes(bytes.sublist(i, i + 4)); + final chunkSize = ByteData.sublistView(bytes, i + 4, i + 8) + .getUint32(0, Endian.little); + if (id == 'data') { + pcmStart = i + 8; + pcmEnd = pcmStart + chunkSize; + break; + } + i += 8 + chunkSize; + if (i.isOdd) i++; // RIFF chunks are word-aligned } + } else { + final head = bytes.length >= 4 ? String.fromCharCodes(bytes.sublist(0, 4)) : 'short'; + debugPrint('[trim] NO WAV header at $path (head=$head) — treating as raw'); + } + // If not a WAV, or 'data' chunk not found, treat as raw PCM. + // Ensure pcmEnd doesn't exceed file bounds. + if (pcmEnd > bytes.length) pcmEnd = bytes.length; + if (pcmStart >= pcmEnd) { + pcmStart = 0; + pcmEnd = bytes.length; } - if (dataIdx < 0) return path; - final dataLenView = - ByteData.sublistView(bytes, dataIdx + 4, dataIdx + 8); - final dataLen = dataLenView.getUint32(0, Endian.little); - final pcmStart = dataIdx + 8; - final pcmEnd = pcmStart + dataLen; - if (pcmEnd > bytes.length) return path; + // --- 2. Slice the PCM to the requested window --- - const bytesPerMs = 32; + const bytesPerMs = 32; // 16000 Hz × 2 bytes × 1 channel / 1000 var sliceStart = pcmStart + startMs * bytesPerMs; var sliceEnd = pcmStart + endMs * bytesPerMs; - // 16-bit samples → align to 2-byte boundaries so we never split a - // sample mid-byte. + + // Align to 2-byte sample boundaries. sliceStart -= sliceStart % 2; sliceEnd -= sliceEnd % 2; + + // Clamp to valid PCM range. if (sliceStart < pcmStart) sliceStart = pcmStart; if (sliceEnd > pcmEnd) sliceEnd = pcmEnd; if (sliceEnd <= sliceStart) return path; final newPcmLen = sliceEnd - sliceStart; - // Rebuild: original header up to and including the "data" tag, - // patched data-chunk length, sliced PCM. Then patch the RIFF - // chunk size at offset 4 (= file size - 8). Subchunk1 and other - // chunks are copied verbatim. - final out = BytesBuilder(copy: false); - out.add(bytes.sublist(0, dataIdx + 4)); - final lenBytes = Uint8List(4); - ByteData.sublistView(lenBytes).setUint32(0, newPcmLen, Endian.little); - out.add(lenBytes); - out.add(bytes.sublist(sliceStart, sliceEnd)); - - final newBytes = out.takeBytes(); - final riffSize = newBytes.length - 8; - ByteData.sublistView(newBytes, 4, 8) - .setUint32(0, riffSize, Endian.little); - - file.writeAsBytesSync(newBytes, flush: true); + // --- 3. Rebuild with a clean 44-byte canonical header --- + + const sampleRate = 16000; + const numChannels = 1; + const bitsPerSample = 16; + const byteRate = sampleRate * numChannels * (bitsPerSample ~/ 8); + const blockAlign = numChannels * (bitsPerSample ~/ 8); + final fileSize = 44 + newPcmLen; + + final out = Uint8List(fileSize); + final view = ByteData.sublistView(out); + + // RIFF header: "RIFF", size, "WAVE" + out[0] = 0x52; out[1] = 0x49; out[2] = 0x46; out[3] = 0x46; + view.setUint32(4, fileSize - 8, Endian.little); + out[8] = 0x57; out[9] = 0x41; out[10] = 0x56; out[11] = 0x45; + + // fmt sub-chunk: "fmt ", size=16, type=1 (PCM), channels, rate, ... + out[12] = 0x66; out[13] = 0x6D; out[14] = 0x74; out[15] = 0x20; + view.setUint32(16, 16, Endian.little); + view.setUint16(20, 1, Endian.little); + view.setUint16(22, numChannels, Endian.little); + view.setUint32(24, sampleRate, Endian.little); + view.setUint32(28, byteRate, Endian.little); + view.setUint16(32, blockAlign, Endian.little); + view.setUint16(34, bitsPerSample, Endian.little); + + // data sub-chunk: "data", size + out[36] = 0x64; out[37] = 0x61; out[38] = 0x74; out[39] = 0x61; + view.setUint32(40, newPcmLen, Endian.little); + + // Copy sliced PCM samples. + out.setRange(44, fileSize, bytes, sliceStart); + + file.writeAsBytesSync(out, flush: true); return path; - } catch (_) { + } catch (e) { + // On failure, return the original path untouched. return path; } } diff --git a/lib/completion/cactus_runner.dart b/lib/completion/cactus_runner.dart index a5194dc..ccc33fe 100644 --- a/lib/completion/cactus_runner.dart +++ b/lib/completion/cactus_runner.dart @@ -47,7 +47,8 @@ class _StreamingRequest { void _runStreamingWorker(_StreamingRequest req) { debugPrint( - '[worker] cactusComplete starting (addr=0x${req.modelAddr.toRadixString(16)})'); + '[worker] cactusComplete starting (addr=0x${req.modelAddr.toRadixString(16)})', + ); final model = Pointer.fromAddress(req.modelAddr); try { final result = cactusComplete( @@ -134,7 +135,9 @@ class CactusRunner { // 3. (Default) is no — natural-finish text-on-text path lets // cactus's prefix match reuse the cached system prompt. if (_kvDirty || forceReset) { - final reason = forceReset ? 'forced (audio)' : 'dirty from prior stop/error'; + final reason = forceReset + ? 'forced (audio)' + : 'dirty from prior stop/error'; debugPrint('[runner] resetting KV cache ($reason)'); cactusReset(model); _kvDirty = false; @@ -168,20 +171,22 @@ class CactusRunner { (decoded?['time_to_first_token_ms'] as num?)?.toDouble() ?? 0; final prefillTokens = (decoded?['prefill_tokens'] as num?)?.toInt() ?? 0; - final prefillTps = - (decoded?['prefill_tps'] as num?)?.toDouble() ?? 0; + final prefillTps = (decoded?['prefill_tps'] as num?)?.toDouble() ?? 0; debugPrint( - '[runner] done · decode_tps=${tps.toStringAsFixed(2)}' - ' · ttft=${ttft.toStringAsFixed(0)}ms' - ' · prefill_tokens=$prefillTokens' - ' · prefill_tps=${prefillTps.toStringAsFixed(1)}' - ' · response_chars=${cleaned?.length ?? 0}' - ' · envelope_chars=${raw.length}'); - onDone(CactusCompletion( - cleanedResponse: cleaned, - decodeTps: tps, - ttftMs: ttft, - )); + '[runner] done · decode_tps=${tps.toStringAsFixed(2)}' + ' · ttft=${ttft.toStringAsFixed(0)}ms' + ' · prefill_tokens=$prefillTokens' + ' · prefill_tps=${prefillTps.toStringAsFixed(1)}' + ' · response_chars=${cleaned?.length ?? 0}' + ' · envelope_chars=${raw.length}', + ); + onDone( + CactusCompletion( + cleanedResponse: cleaned, + decodeTps: tps, + ttftMs: ttft, + ), + ); if (!completer.isCompleted) completer.complete(); break; case 'error': diff --git a/lib/conversation/question_history.dart b/lib/conversation/question_history.dart new file mode 100644 index 0000000..fc06220 --- /dev/null +++ b/lib/conversation/question_history.dart @@ -0,0 +1,105 @@ +// Persisted history of questions the student typed. Stored in +// SharedPreferences as a length-prefixed list so the history survives +// app kills and cold launches — previously this was an in-memory list +// on the page state and was lost whenever Android reclaimed the app. + +import 'dart:convert'; + +import 'package:flutter/foundation.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +import '../preferences.dart'; + +const _maxEntries = 20; + +class QuestionHistoryEntry { + final String question; + final DateTime timestamp; + + const QuestionHistoryEntry({ + required this.question, + required this.timestamp, + }); + + Map toJson() => { + 'q': question, + 't': timestamp.millisecondsSinceEpoch, + }; + + static QuestionHistoryEntry? fromJson(Object? raw) { + if (raw is! Map) return null; + final q = raw['q']; + final t = raw['t']; + if (q is! String || t is! int) return null; + return QuestionHistoryEntry( + question: q, + timestamp: DateTime.fromMillisecondsSinceEpoch(t), + ); + } +} + +/// Newest-first list of past student questions. Mutations notify on +/// every successful change. +class QuestionHistory extends ChangeNotifier { + final List _entries = []; + bool _loaded = false; + + List get entries => + List.unmodifiable(_entries); + bool get isEmpty => _entries.isEmpty; + + Future load() async { + if (_loaded) return; + _loaded = true; + final prefs = await SharedPreferences.getInstance(); + final raw = prefs.getString(kQuestionHistoryPrefKey); + if (raw == null || raw.isEmpty) return; + try { + final list = jsonDecode(raw); + if (list is! List) return; + _entries + ..clear() + ..addAll(list + .map(QuestionHistoryEntry.fromJson) + .whereType()); + notifyListeners(); + } catch (_) { + // Corrupt blob — drop it. + } + } + + /// Insert at the top, de-dupe on identical question text, cap to + /// [_maxEntries]. Persists asynchronously; the in-memory view is + /// updated synchronously and listeners are notified immediately. + void remember(String question) { + final trimmed = question.trim(); + if (trimmed.isEmpty) return; + _entries.removeWhere((e) => e.question == trimmed); + _entries.insert( + 0, + QuestionHistoryEntry(question: trimmed, timestamp: DateTime.now()), + ); + if (_entries.length > _maxEntries) { + _entries.removeRange(_maxEntries, _entries.length); + } + notifyListeners(); + _persist(); + } + + void clear() { + if (_entries.isEmpty) return; + _entries.clear(); + notifyListeners(); + _persist(); + } + + Future _persist() async { + try { + final prefs = await SharedPreferences.getInstance(); + final raw = jsonEncode(_entries.map((e) => e.toJson()).toList()); + await prefs.setString(kQuestionHistoryPrefKey, raw); + } catch (_) { + // Best-effort; losing a single persist doesn't break the UI. + } + } +} diff --git a/lib/languages.dart b/lib/languages.dart index d2c19b4..1c3a8a9 100644 --- a/lib/languages.dart +++ b/lib/languages.dart @@ -69,21 +69,21 @@ const acholi = Language( const english = Language( code: 'eng', name: 'English', - abbreviation: 'En', + abbreviation: 'Eng', hasASRTraining: false, ); const lugbara = Language( code: 'lgg', name: 'Lugbara', - abbreviation: 'Lgb', + abbreviation: 'Lgg', hasASRTraining: false, ); const runyankole = Language( code: 'nyn', name: 'Runyankole', - abbreviation: 'Rnk', + abbreviation: 'nyn', hasASRTraining: false, ); @@ -102,16 +102,16 @@ const kinyarwanda = Language( ); const lusoga = Language( - code: 'xog', + code: 'Xog', name: 'Lusoga', - abbreviation: 'Sog', + abbreviation: 'xog', hasASRTraining: false, ); const lumasaba = Language( - code: 'myx', + code: 'Myx', name: 'Lumasaba', - abbreviation: 'Msa', + abbreviation: 'myx', hasASRTraining: false, ); diff --git a/lib/main.dart b/lib/main.dart index 7d140f6..1302ce8 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -368,6 +368,7 @@ const _maxRecordingSeconds = 15; /// below normal-volume speech; tune if testing in noisier classrooms /// shows word-onsets being clipped. const _speechThresholdDbfs = -38.0; +const _ttsMaxChars = 1000; // Safeguard against OOM on looping responses /// Sustained silence after the last detected speech before /// end-pointing fires. 1.5 s matches voice-assistant conventions — @@ -516,6 +517,7 @@ class _TranslateDemoPageState extends State { final TtsEngine _tts = TtsEngine(); final AudioPlayer _ttsPlayer = AudioPlayer(); String _ttsAccumulator = ''; + int _ttsGenerationCount = 0; StreamController? _currentTtsController; final _ttsModelManager = TtsModelManager(); @@ -542,8 +544,9 @@ class _TranslateDemoPageState extends State { /// Lives at the page level so dismissing the settings sheet doesn't /// kill the in-flight download. The sheet observes this via /// AnimatedBuilder; reopening shows current progress. - late final DownloadCoordinator _downloadCoordinator = - DownloadCoordinator(ModelManager()); // fallback; downloads pass targetManager + late final DownloadCoordinator _downloadCoordinator = DownloadCoordinator( + ModelManager(), + ); // fallback; downloads pass targetManager @override void initState() { @@ -1148,7 +1151,12 @@ class _TranslateDemoPageState extends State { if (boundary < 0) return; final chunk = _ttsAccumulator.substring(0, boundary + 1).trim(); _ttsAccumulator = _ttsAccumulator.substring(boundary + 1); + + // Safeguard: total synthesized characters per turn should not blow memory + if (_ttsGenerationCount > _ttsMaxChars) return; if (chunk.isEmpty) return; + _ttsGenerationCount += chunk.length; + for (final sub in TtsEngine.splitIntoChunks(_stripMarkdown(chunk))) { ctrl.add(sub); } @@ -1161,7 +1169,8 @@ class _TranslateDemoPageState extends State { if (ctrl == null || ctrl.isClosed) return; final remaining = _ttsAccumulator.trim(); _ttsAccumulator = ''; - if (remaining.isNotEmpty) { + if (remaining.isNotEmpty && _ttsGenerationCount <= _ttsMaxChars) { + _ttsGenerationCount += remaining.length; for (final sub in TtsEngine.splitIntoChunks(_stripMarkdown(remaining))) { ctrl.add(sub); } @@ -1174,6 +1183,7 @@ class _TranslateDemoPageState extends State { final ctrl = _currentTtsController; _currentTtsController = null; _ttsAccumulator = ''; + _ttsGenerationCount = 0; if (ctrl != null && !ctrl.isClosed) ctrl.close(); } @@ -1296,6 +1306,7 @@ class _TranslateDemoPageState extends State { if (_ttsEnabled && _tts.isLoaded) { final ctrl = StreamController(); _ttsAccumulator = ''; + _ttsGenerationCount = 0; _currentTtsController = ctrl; unawaited(_drainTtsQueue(ctrl.stream)); } @@ -1751,29 +1762,24 @@ class _TranslateDemoPageState extends State { 'rec_${DateTime.now().millisecondsSinceEpoch}.wav', ); + final recConfig = RecordConfig( + encoder: AudioEncoder.wav, + sampleRate: 16000, + numChannels: 1, + bitRate: 256000, + ); + try { - await _recorder.start( - const RecordConfig( - encoder: AudioEncoder.wav, - sampleRate: 16000, - numChannels: 1, - bitRate: 256000, - ), - path: outPath, - ); + await _recorder.start(recConfig, path: outPath); } catch (e) { _toast('Could not start recording: $e'); return; } if (!mounted) return; - // Reset the waveform so a new recording starts visually empty. for (var i = 0; i < _waveform.length; i++) { _waveform[i] = 0; } - // Clear speech-time tracking — a fresh recording starts with no - // detected speech, so end-pointing is suppressed until the first - // above-threshold sample arrives. _firstSpeechMs = null; _lastSpeechMs = null; setState(() { @@ -1803,23 +1809,16 @@ class _TranslateDemoPageState extends State { .onAmplitudeChanged(const Duration(milliseconds: 80)) .listen((amp) { if (!mounted) return; - // record's amplitude is in dBFS (negative). -60 ≈ silence, 0 ≈ peak. + // record's amplitude is in dBFS; -60 ≈ silence, 0 ≈ peak. final normalised = ((amp.current + 60) / 60).clamp(0.0, 1.0); - // Speech-time tracking for end-pointing and VAD trim. The - // recorder gives us amplitude per 80 ms window; we treat - // each window as a single time point at "now - 0 ms" — close - // enough given the 200 ms padding the trimmer adds either - // side of the speech range. final started = _recordingStartedAt; if (started != null && amp.current > _speechThresholdDbfs) { final elapsedMs = DateTime.now().difference(started).inMilliseconds; _firstSpeechMs ??= elapsedMs; _lastSpeechMs = elapsedMs; } - // End-pointing: only fires once we've heard at least one - // speech sample (so the user can take a beat before speaking - // without being auto-stopped). After that, sustained silence - // beyond _silenceTimeoutMs auto-finishes the recording. + // End-pointing only fires after first speech (so a user can + // take a beat before speaking without being auto-stopped). final lastSpeech = _lastSpeechMs; if (lastSpeech != null && started != null) { final silentMs = @@ -1835,10 +1834,9 @@ class _TranslateDemoPageState extends State { } } setState(() { - // Shift the ring buffer one slot left and write the newest - // sample onto the right end. In-place to avoid the allocation - // churn of removeAt+add at 12.5 Hz, and we kept the list as - // fixed-length so this is the only legal mutation anyway. + // Shift the ring buffer left, write newest sample on the + // right. In-place to avoid the allocation churn of + // removeAt+add at 12.5 Hz. for (var i = 0; i < _waveform.length - 1; i++) { _waveform[i] = _waveform[i + 1]; } diff --git a/lib/preferences.dart b/lib/preferences.dart new file mode 100644 index 0000000..623540d --- /dev/null +++ b/lib/preferences.dart @@ -0,0 +1,19 @@ +// SharedPreferences keys, gathered in one place so callers don't reach +// across UI files to find the right string constant. Adding a new key +// here is the only place that should ever own a string literal for +// prefs access. + +const kActiveModelSlugPrefKey = 'active_model_slug'; +const kModelUrlPrefKey = 'model_url'; +const kPreferredLanguagePrefKey = 'preferred_classroom_language'; + +/// SharedPreferences flag set to `true` immediately before we cross the +/// FFI boundary into `cactusInit`, and cleared after the call returns +/// (success or caught throw). If the process dies inside cactusInit — +/// a SIGSEGV from corrupt weights or an ABI mismatch — the flag stays +/// set across the relaunch. The cold-launch auto-load checks this flag +/// and falls back to manual-load with a retry toast instead of +/// repeating the crash. +const kModelLoadPendingPrefKey = 'model_load_pending'; + +const kQuestionHistoryPrefKey = 'question_history'; diff --git a/lib/tts/tts_engine.dart b/lib/tts/tts_engine.dart index 654f073..6164cab 100644 --- a/lib/tts/tts_engine.dart +++ b/lib/tts/tts_engine.dart @@ -17,7 +17,9 @@ class TtsEngine { _session = await OnnxRuntime().createSession( modelPath, options: OrtSessionOptions( - providers: [OrtProvider.NNAPI, OrtProvider.XNNPACK, OrtProvider.CPU], + providers: Platform.isAndroid + ? [OrtProvider.NNAPI, OrtProvider.XNNPACK, OrtProvider.CPU] + : [OrtProvider.CPU], ), ); } diff --git a/lib/util/relative_time.dart b/lib/util/relative_time.dart new file mode 100644 index 0000000..15dddcf --- /dev/null +++ b/lib/util/relative_time.dart @@ -0,0 +1,11 @@ +// Single short-relative-time formatter. Used by the top-of-card header, +// the peek-slab label, and the question-history sheet — three callsites +// previously carried near-identical copies that drifted. + +String formatRelative(DateTime t, {DateTime? now}) { + final delta = (now ?? DateTime.now()).difference(t); + if (delta.inSeconds < 60) return 'just now'; + if (delta.inMinutes < 60) return '${delta.inMinutes}m ago'; + if (delta.inHours < 24) return '${delta.inHours}h ago'; + return '${delta.inDays}d ago'; +} diff --git a/lib/widgets/jump_to_latest_pill.dart b/lib/widgets/jump_to_latest_pill.dart new file mode 100644 index 0000000..566ae77 --- /dev/null +++ b/lib/widgets/jump_to_latest_pill.dart @@ -0,0 +1,68 @@ +// Floating "↓ Jump to latest · N back" pill rendered between the card +// stack and the chip row when the user is reviewing history. Reuses +// the spec's §7.4 cue surface — "New response · tap to view" would +// slot in here as a label variant when we wire up the +// keep-focus-on-mid-review-arrivals behaviour. + +import 'package:flutter/material.dart'; + +class JumpToLatestPill extends StatelessWidget { + final int cardsBack; + final VoidCallback onTap; + + const JumpToLatestPill({ + super.key, + required this.cardsBack, + required this.onTap, + }); + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + final scheme = theme.colorScheme; + final label = cardsBack == 1 ? '1 back' : '$cardsBack back'; + return Material( + color: scheme.surfaceContainerHigh, + shape: StadiumBorder( + side: BorderSide( + color: scheme.primary.withValues(alpha: 0.5), + width: 1, + ), + ), + elevation: 4, + shadowColor: Colors.black.withValues(alpha: 0.15), + clipBehavior: Clip.antiAlias, + child: InkWell( + onTap: onTap, + customBorder: const StadiumBorder(), + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 14, vertical: 8), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Icon( + Icons.arrow_downward_rounded, + size: 16, + color: scheme.primary, + ), + const SizedBox(width: 6), + Text( + 'Jump to latest', + style: theme.textTheme.labelLarge?.copyWith( + color: scheme.onSurface, + ), + ), + const SizedBox(width: 6), + Text( + '· $label', + style: theme.textTheme.labelLarge?.copyWith( + color: scheme.onSurfaceVariant, + ), + ), + ], + ), + ), + ), + ); + } +} diff --git a/lib/widgets/question_history_sheet.dart b/lib/widgets/question_history_sheet.dart new file mode 100644 index 0000000..6f5f1f6 --- /dev/null +++ b/lib/widgets/question_history_sheet.dart @@ -0,0 +1,203 @@ +// Question history bottom sheet + the row tile + section header that +// previously lived in main.dart. Reads through [QuestionHistory] so the +// list survives an app kill (previously it was an in-memory list on the +// page state). + +import 'package:flutter/material.dart'; + +import '../conversation/question_history.dart'; +import '../util/relative_time.dart'; + +/// Show the question history sheet. Returns null when nothing was +/// picked. When the user taps an entry, the returned string is the +/// question to prefill into the input field. +Future showQuestionHistorySheet({ + required BuildContext context, + required QuestionHistory history, +}) { + final now = DateTime.now(); + return showModalBottomSheet( + context: context, + showDragHandle: true, + builder: (ctx) { + final today = history.entries + .where((entry) => _isSameDay(entry.timestamp, now)) + .toList(growable: false); + final earlier = history.entries + .where((entry) => !_isSameDay(entry.timestamp, now)) + .toList(growable: false); + return SafeArea( + child: history.isEmpty + ? const Padding( + padding: EdgeInsets.fromLTRB(20, 12, 20, 28), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Icon(Icons.history_rounded, size: 28), + SizedBox(height: 12), + Text( + 'No saved text questions yet.', + textAlign: TextAlign.center, + ), + SizedBox(height: 6), + Text( + 'When a student asks a text question, it will appear ' + 'here so you can ask it again later.', + textAlign: TextAlign.center, + ), + ], + ), + ) + : ListView( + shrinkWrap: true, + children: [ + const ListTile( + title: Text('Question history'), + subtitle: Text( + 'Tap a previous text question to ask it again.', + ), + ), + Align( + alignment: Alignment.centerRight, + child: Padding( + padding: const EdgeInsets.only(right: 16, bottom: 8), + child: TextButton.icon( + onPressed: () { + history.clear(); + Navigator.of(ctx).pop(); + }, + icon: const Icon(Icons.delete_sweep_rounded), + label: const Text('Clear history'), + ), + ), + ), + if (today.isNotEmpty) ...[ + const _HistorySectionHeader(label: 'Today'), + for (final entry in today) + _HistoryQuestionTile( + question: entry.question, + timeLabel: formatRelative(entry.timestamp), + onTap: () => + Navigator.of(ctx).pop(entry.question), + ), + ], + if (earlier.isNotEmpty) ...[ + const _HistorySectionHeader(label: 'Earlier'), + for (final entry in earlier) + _HistoryQuestionTile( + question: entry.question, + timeLabel: formatRelative(entry.timestamp), + onTap: () => + Navigator.of(ctx).pop(entry.question), + ), + ], + const SizedBox(height: 16), + ], + ), + ); + }, + ); +} + +bool _isSameDay(DateTime a, DateTime b) => + a.year == b.year && a.month == b.month && a.day == b.day; + +class _HistorySectionHeader extends StatelessWidget { + final String label; + const _HistorySectionHeader({required this.label}); + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + final scheme = theme.colorScheme; + return Padding( + padding: const EdgeInsets.fromLTRB(20, 8, 20, 8), + child: Text( + label, + style: theme.textTheme.labelLarge?.copyWith( + color: scheme.onSurfaceVariant, + ), + ), + ); + } +} + +class _HistoryQuestionTile extends StatelessWidget { + final String question; + final String timeLabel; + final VoidCallback onTap; + + const _HistoryQuestionTile({ + required this.question, + required this.timeLabel, + required this.onTap, + }); + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + final scheme = theme.colorScheme; + return Padding( + padding: const EdgeInsets.fromLTRB(16, 0, 16, 10), + child: Material( + color: scheme.surfaceContainerLow, + borderRadius: BorderRadius.circular(16), + child: InkWell( + onTap: onTap, + borderRadius: BorderRadius.circular(16), + child: Padding( + padding: const EdgeInsets.fromLTRB(14, 12, 14, 12), + child: Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Container( + width: 32, + height: 32, + decoration: BoxDecoration( + color: scheme.primaryContainer, + shape: BoxShape.circle, + ), + child: Icon( + Icons.history_rounded, + size: 16, + color: scheme.onPrimaryContainer, + ), + ), + const SizedBox(width: 12), + Expanded( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + question, + maxLines: 3, + overflow: TextOverflow.ellipsis, + style: theme.textTheme.bodyMedium?.copyWith( + color: scheme.onSurface, + ), + ), + const SizedBox(height: 8), + Text( + 'Asked $timeLabel', + style: theme.textTheme.labelMedium?.copyWith( + color: scheme.onSurfaceVariant, + ), + ), + ], + ), + ), + const SizedBox(width: 10), + Text( + 'Ask again', + style: theme.textTheme.labelLarge?.copyWith( + color: scheme.primary, + ), + ), + ], + ), + ), + ), + ), + ); + } +} diff --git a/lib/widgets/top_bar.dart b/lib/widgets/top_bar.dart new file mode 100644 index 0000000..00a82a5 --- /dev/null +++ b/lib/widgets/top_bar.dart @@ -0,0 +1,268 @@ +// Minimal top bar that replaces the default ~95 px AppBar with a +// ~48 px bar inside the SafeArea. Hosts the classroom-language pill, +// the question-history shortcut, the reset shortcut, the model-loaded +// dot, and the settings entry-point. +// +// Pulled out of main.dart so the page state isn't responsible for +// rendering five sibling private widgets inline. + +import 'package:flutter/material.dart'; + +import '../languages.dart'; + +class MinimalTopBar extends StatelessWidget { + final bool loaded; + final bool busy; + final Language classroomLanguage; + final String? quantLabel; + final VoidCallback? onLoad; + final VoidCallback? onOpenHistory; + final VoidCallback? onPickLanguage; + final VoidCallback? onResetConversation; + final VoidCallback? onSettings; + + const MinimalTopBar({ + super.key, + required this.loaded, + required this.busy, + required this.classroomLanguage, + required this.quantLabel, + required this.onLoad, + required this.onOpenHistory, + required this.onPickLanguage, + required this.onResetConversation, + required this.onSettings, + }); + + @override + Widget build(BuildContext context) { + final scheme = Theme.of(context).colorScheme; + return Padding( + padding: const EdgeInsets.fromLTRB(14, 14, 14, 10), + child: Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + crossAxisAlignment: CrossAxisAlignment.center, + children: [ + Expanded( + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + _LanguagePill( + language: classroomLanguage, + onTap: onPickLanguage, + scheme: scheme, + ), + const SizedBox(width: 6), + _HistoryButton(onTap: onOpenHistory, scheme: scheme), + const SizedBox(width: 6), + _ResetButton(onTap: onResetConversation, scheme: scheme), + ], + ), + ), + _LoadedDot( + loaded: loaded, + busy: busy, + quantLabel: quantLabel, + onTap: onLoad, + scheme: scheme, + ), + const SizedBox(width: 6), + _SettingsButton(onTap: onSettings, scheme: scheme), + ], + ), + ); + } +} + +class _HistoryButton extends StatelessWidget { + final VoidCallback? onTap; + final ColorScheme scheme; + const _HistoryButton({required this.onTap, required this.scheme}); + + @override + Widget build(BuildContext context) { + final color = onTap == null + ? scheme.onSurfaceVariant.withValues(alpha: 0.5) + : scheme.onSurfaceVariant; + return Tooltip( + message: 'Question history', + child: InkResponse( + onTap: onTap, + radius: 18, + child: Padding( + padding: const EdgeInsets.all(6), + child: Icon(Icons.history_rounded, size: 20, color: color), + ), + ), + ); + } +} + +class _LanguagePill extends StatelessWidget { + final Language language; + final VoidCallback? onTap; + final ColorScheme scheme; + + const _LanguagePill({ + required this.language, + required this.onTap, + required this.scheme, + }); + + @override + Widget build(BuildContext context) { + final fg = onTap == null + ? scheme.onSurfaceVariant.withValues(alpha: 0.5) + : scheme.onSurfaceVariant; + return Tooltip( + message: 'Classroom language: ${language.name}', + child: Material( + color: scheme.surfaceContainerLow, + borderRadius: BorderRadius.circular(999), + child: InkWell( + onTap: onTap, + borderRadius: BorderRadius.circular(999), + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 6), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Icon(Icons.language_rounded, size: 15, color: fg), + const SizedBox(width: 6), + Text( + language.abbreviation, + style: Theme.of( + context, + ).textTheme.labelLarge?.copyWith(color: fg), + ), + ], + ), + ), + ), + ), + ); + } +} + +/// 24 px circular indicator: filled primary with a white check when the +/// model is loaded, filled primary with a tiny spinner while loading, +/// and an outlined empty circle when not loaded. The not-loaded state +/// is intentionally muted — the input bar's load-gate banner is the +/// primary call-to-action; the dot is just a status reflection. +class _LoadedDot extends StatelessWidget { + final bool loaded; + final bool busy; + final String? quantLabel; + final VoidCallback? onTap; + final ColorScheme scheme; + + const _LoadedDot({ + required this.loaded, + required this.busy, + required this.quantLabel, + required this.onTap, + required this.scheme, + }); + + @override + Widget build(BuildContext context) { + final Widget content; + if (loaded) { + content = Container( + width: 24, + height: 24, + decoration: BoxDecoration( + color: scheme.primary, + shape: BoxShape.circle, + ), + child: const Icon(Icons.check_rounded, size: 14, color: Colors.white), + ); + } else if (busy) { + content = Container( + width: 24, + height: 24, + decoration: BoxDecoration( + color: scheme.primary, + shape: BoxShape.circle, + ), + child: const Padding( + padding: EdgeInsets.all(5), + child: CircularProgressIndicator( + strokeWidth: 1.8, + color: Colors.white, + ), + ), + ); + } else { + content = Container( + width: 24, + height: 24, + decoration: BoxDecoration( + shape: BoxShape.circle, + border: Border.all( + color: scheme.onSurfaceVariant.withValues(alpha: 0.4), + width: 1, + ), + ), + ); + } + final tooltip = loaded + ? 'Loaded · gemma-4-e2b · ${quantLabel ?? '…'}' + : busy + ? 'Loading model…' + : 'Tap to load model'; + return Tooltip( + message: tooltip, + child: InkResponse(onTap: onTap, radius: 18, child: content), + ); + } +} + +class _ResetButton extends StatelessWidget { + final VoidCallback? onTap; + final ColorScheme scheme; + + const _ResetButton({required this.onTap, required this.scheme}); + + @override + Widget build(BuildContext context) { + final color = onTap == null + ? scheme.onSurfaceVariant.withValues(alpha: 0.5) + : scheme.onSurfaceVariant; + return Tooltip( + message: 'Start fresh', + child: InkResponse( + onTap: onTap, + radius: 18, + child: Padding( + padding: const EdgeInsets.all(6), + child: Icon(Icons.refresh_rounded, size: 20, color: color), + ), + ), + ); + } +} + +class _SettingsButton extends StatelessWidget { + final VoidCallback? onTap; + final ColorScheme scheme; + const _SettingsButton({required this.onTap, required this.scheme}); + + @override + Widget build(BuildContext context) { + final color = onTap == null + ? scheme.onSurfaceVariant.withValues(alpha: 0.5) + : scheme.onSurfaceVariant; + return Tooltip( + message: 'Manage model', + child: InkResponse( + onTap: onTap, + radius: 22, + child: Padding( + padding: const EdgeInsets.all(6), + child: Icon(Icons.tune_rounded, size: 24, color: color), + ), + ), + ); + } +} diff --git a/lib/widgets/voice_review_bar.dart b/lib/widgets/voice_review_bar.dart new file mode 100644 index 0000000..e8e5633 --- /dev/null +++ b/lib/widgets/voice_review_bar.dart @@ -0,0 +1,169 @@ +// Inline transcript-review UI for the stitched cascade. Renders in +// place of the recorder / text input while a stitched-bundle voice +// turn is waiting on the user to pick an intent. Replaces the modal +// bottom-sheet flow with an in-place state machine: the same surface +// that held the recording waveform now holds the review. + +import 'package:flutter/material.dart'; + +import '../audio/audio_types.dart'; + +class VoiceReviewBar extends StatelessWidget { + final VoiceReviewState review; + final bool busy; + final void Function(VoiceIntent intent) onPickIntent; + final VoidCallback onPickTarget; + final VoidCallback onDiscard; + + const VoiceReviewBar({ + super.key, + required this.review, + required this.busy, + required this.onPickIntent, + required this.onPickTarget, + required this.onDiscard, + }); + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + final scheme = theme.colorScheme; + return Padding( + padding: const EdgeInsets.fromLTRB(12, 4, 12, 10), + child: Material( + color: scheme.surfaceContainerLow, + borderRadius: BorderRadius.circular(20), + child: Padding( + padding: const EdgeInsets.fromLTRB(14, 12, 14, 12), + child: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + mainAxisSize: MainAxisSize.min, + children: [ + // Transcript + discard X. Quote-mark accent on the left + // mirrors a chat bubble rhythm — user reads what was + // heard, then either commits or backs out. + Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Icon( + Icons.format_quote_rounded, + size: 18, + color: scheme.primary, + ), + const SizedBox(width: 8), + Expanded( + child: Text( + review.transcript, + style: theme.textTheme.bodyLarge, + ), + ), + const SizedBox(width: 4), + Tooltip( + message: 'Discard', + child: IconButton( + visualDensity: VisualDensity.compact, + onPressed: busy ? null : onDiscard, + icon: Icon( + Icons.close_rounded, + size: 18, + color: scheme.onSurfaceVariant, + ), + ), + ), + ], + ), + const SizedBox(height: 10), + // Target-language pill above the action buttons so the + // hierarchy reads "what language → what action". + Row( + children: [ + Text( + 'Target', + style: theme.textTheme.labelMedium?.copyWith( + color: scheme.onSurfaceVariant, + ), + ), + const SizedBox(width: 8), + Material( + color: scheme.surfaceContainerHighest, + borderRadius: BorderRadius.circular(999), + child: InkWell( + onTap: busy ? null : onPickTarget, + borderRadius: BorderRadius.circular(999), + child: Padding( + padding: const EdgeInsets.symmetric( + horizontal: 12, + vertical: 6, + ), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Icon( + Icons.language_rounded, + size: 14, + color: scheme.onSurfaceVariant, + ), + const SizedBox(width: 6), + Text( + review.target.name, + style: theme.textTheme.labelLarge?.copyWith( + color: scheme.onSurface, + ), + ), + const SizedBox(width: 2), + Icon( + Icons.expand_more_rounded, + size: 16, + color: scheme.onSurfaceVariant, + ), + ], + ), + ), + ), + ), + ], + ), + const SizedBox(height: 10), + // The pill above carries the target language, so the + // button text doesn't need to. Avoids the two-line wrap + // we saw with "Translate to Luganda" / "Explain in + // Luganda" on a phone width. + Row( + children: [ + Expanded( + child: FilledButton.icon( + onPressed: busy + ? null + : () => onPickIntent(VoiceIntent.translate), + icon: const Icon(Icons.translate_rounded, size: 18), + label: const Text('Translate'), + style: FilledButton.styleFrom( + padding: const EdgeInsets.symmetric(vertical: 12), + ), + ), + ), + const SizedBox(width: 8), + Expanded( + child: FilledButton.tonalIcon( + onPressed: busy + ? null + : () => onPickIntent(VoiceIntent.explain), + icon: const Icon( + Icons.lightbulb_outline_rounded, + size: 18, + ), + label: const Text('Explain'), + style: FilledButton.styleFrom( + padding: const EdgeInsets.symmetric(vertical: 12), + ), + ), + ), + ], + ), + ], + ), + ), + ), + ); + } +} diff --git a/linux/.gitignore b/linux/.gitignore new file mode 100644 index 0000000..d3896c9 --- /dev/null +++ b/linux/.gitignore @@ -0,0 +1 @@ +flutter/ephemeral diff --git a/linux/CMakeLists.txt b/linux/CMakeLists.txt new file mode 100644 index 0000000..9bc431d --- /dev/null +++ b/linux/CMakeLists.txt @@ -0,0 +1,128 @@ +# Project-level configuration. +cmake_minimum_required(VERSION 3.13) +project(runner LANGUAGES CXX) + +# The name of the executable created for the application. Change this to change +# the on-disk name of your application. +set(BINARY_NAME "sunflower_app") +# The unique GTK application identifier for this application. See: +# https://wiki.gnome.org/HowDoI/ChooseApplicationID +set(APPLICATION_ID "ai.sunbird.sunflower_app") + +# Explicitly opt in to modern CMake behaviors to avoid warnings with recent +# versions of CMake. +cmake_policy(SET CMP0063 NEW) + +# Load bundled libraries from the lib/ directory relative to the binary. +set(CMAKE_INSTALL_RPATH "$ORIGIN/lib") + +# Root filesystem for cross-building. +if(FLUTTER_TARGET_PLATFORM_SYSROOT) + set(CMAKE_SYSROOT ${FLUTTER_TARGET_PLATFORM_SYSROOT}) + set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT}) + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) + set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +endif() + +# Define build configuration options. +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE "Debug" CACHE + STRING "Flutter build mode" FORCE) + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS + "Debug" "Profile" "Release") +endif() + +# Compilation settings that should be applied to most targets. +# +# Be cautious about adding new options here, as plugins use this function by +# default. In most cases, you should add new options to specific targets instead +# of modifying this function. +function(APPLY_STANDARD_SETTINGS TARGET) + target_compile_features(${TARGET} PUBLIC cxx_std_14) + target_compile_options(${TARGET} PRIVATE -Wall -Werror) + target_compile_options(${TARGET} PRIVATE "$<$>:-O3>") + target_compile_definitions(${TARGET} PRIVATE "$<$>:NDEBUG>") +endfunction() + +# Flutter library and tool build rules. +set(FLUTTER_MANAGED_DIR "${CMAKE_CURRENT_SOURCE_DIR}/flutter") +add_subdirectory(${FLUTTER_MANAGED_DIR}) + +# System-level dependencies. +find_package(PkgConfig REQUIRED) +pkg_check_modules(GTK REQUIRED IMPORTED_TARGET gtk+-3.0) + +# Application build; see runner/CMakeLists.txt. +add_subdirectory("runner") + +# Run the Flutter tool portions of the build. This must not be removed. +add_dependencies(${BINARY_NAME} flutter_assemble) + +# Only the install-generated bundle's copy of the executable will launch +# correctly, since the resources must in the right relative locations. To avoid +# people trying to run the unbundled copy, put it in a subdirectory instead of +# the default top-level location. +set_target_properties(${BINARY_NAME} + PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/intermediates_do_not_run" +) + + +# Generated plugin build rules, which manage building the plugins and adding +# them to the application. +include(flutter/generated_plugins.cmake) + + +# === Installation === +# By default, "installing" just makes a relocatable bundle in the build +# directory. +set(BUILD_BUNDLE_DIR "${PROJECT_BINARY_DIR}/bundle") +if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + set(CMAKE_INSTALL_PREFIX "${BUILD_BUNDLE_DIR}" CACHE PATH "..." FORCE) +endif() + +# Start with a clean build bundle directory every time. +install(CODE " + file(REMOVE_RECURSE \"${BUILD_BUNDLE_DIR}/\") + " COMPONENT Runtime) + +set(INSTALL_BUNDLE_DATA_DIR "${CMAKE_INSTALL_PREFIX}/data") +set(INSTALL_BUNDLE_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib") + +install(TARGETS ${BINARY_NAME} RUNTIME DESTINATION "${CMAKE_INSTALL_PREFIX}" + COMPONENT Runtime) + +install(FILES "${FLUTTER_ICU_DATA_FILE}" DESTINATION "${INSTALL_BUNDLE_DATA_DIR}" + COMPONENT Runtime) + +install(FILES "${FLUTTER_LIBRARY}" DESTINATION "${INSTALL_BUNDLE_LIB_DIR}" + COMPONENT Runtime) + +foreach(bundled_library ${PLUGIN_BUNDLED_LIBRARIES}) + install(FILES "${bundled_library}" + DESTINATION "${INSTALL_BUNDLE_LIB_DIR}" + COMPONENT Runtime) +endforeach(bundled_library) + +# Copy the native assets provided by the build.dart from all packages. +set(NATIVE_ASSETS_DIR "${PROJECT_BUILD_DIR}native_assets/linux/") +install(DIRECTORY "${NATIVE_ASSETS_DIR}" + DESTINATION "${INSTALL_BUNDLE_LIB_DIR}" + COMPONENT Runtime) + +# Fully re-copy the assets directory on each build to avoid having stale files +# from a previous install. +set(FLUTTER_ASSET_DIR_NAME "flutter_assets") +install(CODE " + file(REMOVE_RECURSE \"${INSTALL_BUNDLE_DATA_DIR}/${FLUTTER_ASSET_DIR_NAME}\") + " COMPONENT Runtime) +install(DIRECTORY "${PROJECT_BUILD_DIR}/${FLUTTER_ASSET_DIR_NAME}" + DESTINATION "${INSTALL_BUNDLE_DATA_DIR}" COMPONENT Runtime) + +# Install the AOT library on non-Debug builds only. +if(NOT CMAKE_BUILD_TYPE MATCHES "Debug") + install(FILES "${AOT_LIBRARY}" DESTINATION "${INSTALL_BUNDLE_LIB_DIR}" + COMPONENT Runtime) +endif() diff --git a/linux/flutter/CMakeLists.txt b/linux/flutter/CMakeLists.txt new file mode 100644 index 0000000..d5bd016 --- /dev/null +++ b/linux/flutter/CMakeLists.txt @@ -0,0 +1,88 @@ +# This file controls Flutter-level build steps. It should not be edited. +cmake_minimum_required(VERSION 3.10) + +set(EPHEMERAL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ephemeral") + +# Configuration provided via flutter tool. +include(${EPHEMERAL_DIR}/generated_config.cmake) + +# TODO: Move the rest of this into files in ephemeral. See +# https://github.com/flutter/flutter/issues/57146. + +# Serves the same purpose as list(TRANSFORM ... PREPEND ...), +# which isn't available in 3.10. +function(list_prepend LIST_NAME PREFIX) + set(NEW_LIST "") + foreach(element ${${LIST_NAME}}) + list(APPEND NEW_LIST "${PREFIX}${element}") + endforeach(element) + set(${LIST_NAME} "${NEW_LIST}" PARENT_SCOPE) +endfunction() + +# === Flutter Library === +# System-level dependencies. +find_package(PkgConfig REQUIRED) +pkg_check_modules(GTK REQUIRED IMPORTED_TARGET gtk+-3.0) +pkg_check_modules(GLIB REQUIRED IMPORTED_TARGET glib-2.0) +pkg_check_modules(GIO REQUIRED IMPORTED_TARGET gio-2.0) + +set(FLUTTER_LIBRARY "${EPHEMERAL_DIR}/libflutter_linux_gtk.so") + +# Published to parent scope for install step. +set(FLUTTER_LIBRARY ${FLUTTER_LIBRARY} PARENT_SCOPE) +set(FLUTTER_ICU_DATA_FILE "${EPHEMERAL_DIR}/icudtl.dat" PARENT_SCOPE) +set(PROJECT_BUILD_DIR "${PROJECT_DIR}/build/" PARENT_SCOPE) +set(AOT_LIBRARY "${PROJECT_DIR}/build/lib/libapp.so" PARENT_SCOPE) + +list(APPEND FLUTTER_LIBRARY_HEADERS + "fl_basic_message_channel.h" + "fl_binary_codec.h" + "fl_binary_messenger.h" + "fl_dart_project.h" + "fl_engine.h" + "fl_json_message_codec.h" + "fl_json_method_codec.h" + "fl_message_codec.h" + "fl_method_call.h" + "fl_method_channel.h" + "fl_method_codec.h" + "fl_method_response.h" + "fl_plugin_registrar.h" + "fl_plugin_registry.h" + "fl_standard_message_codec.h" + "fl_standard_method_codec.h" + "fl_string_codec.h" + "fl_value.h" + "fl_view.h" + "flutter_linux.h" +) +list_prepend(FLUTTER_LIBRARY_HEADERS "${EPHEMERAL_DIR}/flutter_linux/") +add_library(flutter INTERFACE) +target_include_directories(flutter INTERFACE + "${EPHEMERAL_DIR}" +) +target_link_libraries(flutter INTERFACE "${FLUTTER_LIBRARY}") +target_link_libraries(flutter INTERFACE + PkgConfig::GTK + PkgConfig::GLIB + PkgConfig::GIO +) +add_dependencies(flutter flutter_assemble) + +# === Flutter tool backend === +# _phony_ is a non-existent file to force this command to run every time, +# since currently there's no way to get a full input/output list from the +# flutter tool. +add_custom_command( + OUTPUT ${FLUTTER_LIBRARY} ${FLUTTER_LIBRARY_HEADERS} + ${CMAKE_CURRENT_BINARY_DIR}/_phony_ + COMMAND ${CMAKE_COMMAND} -E env + ${FLUTTER_TOOL_ENVIRONMENT} + "${FLUTTER_ROOT}/packages/flutter_tools/bin/tool_backend.sh" + ${FLUTTER_TARGET_PLATFORM} ${CMAKE_BUILD_TYPE} + VERBATIM +) +add_custom_target(flutter_assemble DEPENDS + "${FLUTTER_LIBRARY}" + ${FLUTTER_LIBRARY_HEADERS} +) diff --git a/linux/flutter/generated_plugin_registrant.cc b/linux/flutter/generated_plugin_registrant.cc new file mode 100644 index 0000000..f53d3b2 --- /dev/null +++ b/linux/flutter/generated_plugin_registrant.cc @@ -0,0 +1,23 @@ +// +// Generated file. Do not edit. +// + +// clang-format off + +#include "generated_plugin_registrant.h" + +#include +#include +#include + +void fl_register_plugins(FlPluginRegistry* registry) { + g_autoptr(FlPluginRegistrar) audioplayers_linux_registrar = + fl_plugin_registry_get_registrar_for_plugin(registry, "AudioplayersLinuxPlugin"); + audioplayers_linux_plugin_register_with_registrar(audioplayers_linux_registrar); + g_autoptr(FlPluginRegistrar) flutter_onnxruntime_registrar = + fl_plugin_registry_get_registrar_for_plugin(registry, "FlutterOnnxruntimePlugin"); + flutter_onnxruntime_plugin_register_with_registrar(flutter_onnxruntime_registrar); + g_autoptr(FlPluginRegistrar) record_linux_registrar = + fl_plugin_registry_get_registrar_for_plugin(registry, "RecordLinuxPlugin"); + record_linux_plugin_register_with_registrar(record_linux_registrar); +} diff --git a/linux/flutter/generated_plugin_registrant.h b/linux/flutter/generated_plugin_registrant.h new file mode 100644 index 0000000..e0f0a47 --- /dev/null +++ b/linux/flutter/generated_plugin_registrant.h @@ -0,0 +1,15 @@ +// +// Generated file. Do not edit. +// + +// clang-format off + +#ifndef GENERATED_PLUGIN_REGISTRANT_ +#define GENERATED_PLUGIN_REGISTRANT_ + +#include + +// Registers Flutter plugins. +void fl_register_plugins(FlPluginRegistry* registry); + +#endif // GENERATED_PLUGIN_REGISTRANT_ diff --git a/linux/flutter/generated_plugins.cmake b/linux/flutter/generated_plugins.cmake new file mode 100644 index 0000000..c5c4696 --- /dev/null +++ b/linux/flutter/generated_plugins.cmake @@ -0,0 +1,27 @@ +# +# Generated file, do not edit. +# + +list(APPEND FLUTTER_PLUGIN_LIST + audioplayers_linux + flutter_onnxruntime + record_linux +) + +list(APPEND FLUTTER_FFI_PLUGIN_LIST + jni +) + +set(PLUGIN_BUNDLED_LIBRARIES) + +foreach(plugin ${FLUTTER_PLUGIN_LIST}) + add_subdirectory(flutter/ephemeral/.plugin_symlinks/${plugin}/linux plugins/${plugin}) + target_link_libraries(${BINARY_NAME} PRIVATE ${plugin}_plugin) + list(APPEND PLUGIN_BUNDLED_LIBRARIES $) + list(APPEND PLUGIN_BUNDLED_LIBRARIES ${${plugin}_bundled_libraries}) +endforeach(plugin) + +foreach(ffi_plugin ${FLUTTER_FFI_PLUGIN_LIST}) + add_subdirectory(flutter/ephemeral/.plugin_symlinks/${ffi_plugin}/linux plugins/${ffi_plugin}) + list(APPEND PLUGIN_BUNDLED_LIBRARIES ${${ffi_plugin}_bundled_libraries}) +endforeach(ffi_plugin) diff --git a/linux/runner/CMakeLists.txt b/linux/runner/CMakeLists.txt new file mode 100644 index 0000000..e97dabc --- /dev/null +++ b/linux/runner/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.13) +project(runner LANGUAGES CXX) + +# Define the application target. To change its name, change BINARY_NAME in the +# top-level CMakeLists.txt, not the value here, or `flutter run` will no longer +# work. +# +# Any new source files that you add to the application should be added here. +add_executable(${BINARY_NAME} + "main.cc" + "my_application.cc" + "${FLUTTER_MANAGED_DIR}/generated_plugin_registrant.cc" +) + +# Apply the standard set of build settings. This can be removed for applications +# that need different build settings. +apply_standard_settings(${BINARY_NAME}) + +# Add preprocessor definitions for the application ID. +add_definitions(-DAPPLICATION_ID="${APPLICATION_ID}") + +# Add dependency libraries. Add any application-specific dependencies here. +target_link_libraries(${BINARY_NAME} PRIVATE flutter) +target_link_libraries(${BINARY_NAME} PRIVATE PkgConfig::GTK) + +target_include_directories(${BINARY_NAME} PRIVATE "${CMAKE_SOURCE_DIR}") diff --git a/linux/runner/main.cc b/linux/runner/main.cc new file mode 100644 index 0000000..e7c5c54 --- /dev/null +++ b/linux/runner/main.cc @@ -0,0 +1,6 @@ +#include "my_application.h" + +int main(int argc, char** argv) { + g_autoptr(MyApplication) app = my_application_new(); + return g_application_run(G_APPLICATION(app), argc, argv); +} diff --git a/linux/runner/my_application.cc b/linux/runner/my_application.cc new file mode 100644 index 0000000..2a7806c --- /dev/null +++ b/linux/runner/my_application.cc @@ -0,0 +1,148 @@ +#include "my_application.h" + +#include +#ifdef GDK_WINDOWING_X11 +#include +#endif + +#include "flutter/generated_plugin_registrant.h" + +struct _MyApplication { + GtkApplication parent_instance; + char** dart_entrypoint_arguments; +}; + +G_DEFINE_TYPE(MyApplication, my_application, GTK_TYPE_APPLICATION) + +// Called when first Flutter frame received. +static void first_frame_cb(MyApplication* self, FlView* view) { + gtk_widget_show(gtk_widget_get_toplevel(GTK_WIDGET(view))); +} + +// Implements GApplication::activate. +static void my_application_activate(GApplication* application) { + MyApplication* self = MY_APPLICATION(application); + GtkWindow* window = + GTK_WINDOW(gtk_application_window_new(GTK_APPLICATION(application))); + + // Use a header bar when running in GNOME as this is the common style used + // by applications and is the setup most users will be using (e.g. Ubuntu + // desktop). + // If running on X and not using GNOME then just use a traditional title bar + // in case the window manager does more exotic layout, e.g. tiling. + // If running on Wayland assume the header bar will work (may need changing + // if future cases occur). + gboolean use_header_bar = TRUE; +#ifdef GDK_WINDOWING_X11 + GdkScreen* screen = gtk_window_get_screen(window); + if (GDK_IS_X11_SCREEN(screen)) { + const gchar* wm_name = gdk_x11_screen_get_window_manager_name(screen); + if (g_strcmp0(wm_name, "GNOME Shell") != 0) { + use_header_bar = FALSE; + } + } +#endif + if (use_header_bar) { + GtkHeaderBar* header_bar = GTK_HEADER_BAR(gtk_header_bar_new()); + gtk_widget_show(GTK_WIDGET(header_bar)); + gtk_header_bar_set_title(header_bar, "sunflower_app"); + gtk_header_bar_set_show_close_button(header_bar, TRUE); + gtk_window_set_titlebar(window, GTK_WIDGET(header_bar)); + } else { + gtk_window_set_title(window, "sunflower_app"); + } + + gtk_window_set_default_size(window, 1280, 720); + + g_autoptr(FlDartProject) project = fl_dart_project_new(); + fl_dart_project_set_dart_entrypoint_arguments( + project, self->dart_entrypoint_arguments); + + FlView* view = fl_view_new(project); + GdkRGBA background_color; + // Background defaults to black, override it here if necessary, e.g. #00000000 + // for transparent. + gdk_rgba_parse(&background_color, "#000000"); + fl_view_set_background_color(view, &background_color); + gtk_widget_show(GTK_WIDGET(view)); + gtk_container_add(GTK_CONTAINER(window), GTK_WIDGET(view)); + + // Show the window when Flutter renders. + // Requires the view to be realized so we can start rendering. + g_signal_connect_swapped(view, "first-frame", G_CALLBACK(first_frame_cb), + self); + gtk_widget_realize(GTK_WIDGET(view)); + + fl_register_plugins(FL_PLUGIN_REGISTRY(view)); + + gtk_widget_grab_focus(GTK_WIDGET(view)); +} + +// Implements GApplication::local_command_line. +static gboolean my_application_local_command_line(GApplication* application, + gchar*** arguments, + int* exit_status) { + MyApplication* self = MY_APPLICATION(application); + // Strip out the first argument as it is the binary name. + self->dart_entrypoint_arguments = g_strdupv(*arguments + 1); + + g_autoptr(GError) error = nullptr; + if (!g_application_register(application, nullptr, &error)) { + g_warning("Failed to register: %s", error->message); + *exit_status = 1; + return TRUE; + } + + g_application_activate(application); + *exit_status = 0; + + return TRUE; +} + +// Implements GApplication::startup. +static void my_application_startup(GApplication* application) { + // MyApplication* self = MY_APPLICATION(object); + + // Perform any actions required at application startup. + + G_APPLICATION_CLASS(my_application_parent_class)->startup(application); +} + +// Implements GApplication::shutdown. +static void my_application_shutdown(GApplication* application) { + // MyApplication* self = MY_APPLICATION(object); + + // Perform any actions required at application shutdown. + + G_APPLICATION_CLASS(my_application_parent_class)->shutdown(application); +} + +// Implements GObject::dispose. +static void my_application_dispose(GObject* object) { + MyApplication* self = MY_APPLICATION(object); + g_clear_pointer(&self->dart_entrypoint_arguments, g_strfreev); + G_OBJECT_CLASS(my_application_parent_class)->dispose(object); +} + +static void my_application_class_init(MyApplicationClass* klass) { + G_APPLICATION_CLASS(klass)->activate = my_application_activate; + G_APPLICATION_CLASS(klass)->local_command_line = + my_application_local_command_line; + G_APPLICATION_CLASS(klass)->startup = my_application_startup; + G_APPLICATION_CLASS(klass)->shutdown = my_application_shutdown; + G_OBJECT_CLASS(klass)->dispose = my_application_dispose; +} + +static void my_application_init(MyApplication* self) {} + +MyApplication* my_application_new() { + // Set the program name to the application ID, which helps various systems + // like GTK and desktop environments map this running application to its + // corresponding .desktop file. This ensures better integration by allowing + // the application to be recognized beyond its binary name. + g_set_prgname(APPLICATION_ID); + + return MY_APPLICATION(g_object_new(my_application_get_type(), + "application-id", APPLICATION_ID, "flags", + G_APPLICATION_NON_UNIQUE, nullptr)); +} diff --git a/linux/runner/my_application.h b/linux/runner/my_application.h new file mode 100644 index 0000000..db16367 --- /dev/null +++ b/linux/runner/my_application.h @@ -0,0 +1,21 @@ +#ifndef FLUTTER_MY_APPLICATION_H_ +#define FLUTTER_MY_APPLICATION_H_ + +#include + +G_DECLARE_FINAL_TYPE(MyApplication, + my_application, + MY, + APPLICATION, + GtkApplication) + +/** + * my_application_new: + * + * Creates a new Flutter-based application. + * + * Returns: a new #MyApplication. + */ +MyApplication* my_application_new(); + +#endif // FLUTTER_MY_APPLICATION_H_ diff --git a/macos/.gitignore b/macos/.gitignore new file mode 100644 index 0000000..746adbb --- /dev/null +++ b/macos/.gitignore @@ -0,0 +1,7 @@ +# Flutter-related +**/Flutter/ephemeral/ +**/Pods/ + +# Xcode-related +**/dgph +**/xcuserdata/ diff --git a/macos/Flutter/Flutter-Debug.xcconfig b/macos/Flutter/Flutter-Debug.xcconfig new file mode 100644 index 0000000..4b81f9b --- /dev/null +++ b/macos/Flutter/Flutter-Debug.xcconfig @@ -0,0 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig" +#include "ephemeral/Flutter-Generated.xcconfig" diff --git a/macos/Flutter/Flutter-Release.xcconfig b/macos/Flutter/Flutter-Release.xcconfig new file mode 100644 index 0000000..5caa9d1 --- /dev/null +++ b/macos/Flutter/Flutter-Release.xcconfig @@ -0,0 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig" +#include "ephemeral/Flutter-Generated.xcconfig" diff --git a/macos/Flutter/GeneratedPluginRegistrant.swift b/macos/Flutter/GeneratedPluginRegistrant.swift new file mode 100644 index 0000000..3bfd063 --- /dev/null +++ b/macos/Flutter/GeneratedPluginRegistrant.swift @@ -0,0 +1,18 @@ +// +// Generated file. Do not edit. +// + +import FlutterMacOS +import Foundation + +import audioplayers_darwin +import flutter_onnxruntime +import record_macos +import shared_preferences_foundation + +func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { + AudioplayersDarwinPlugin.register(with: registry.registrar(forPlugin: "AudioplayersDarwinPlugin")) + FlutterOnnxruntimePlugin.register(with: registry.registrar(forPlugin: "FlutterOnnxruntimePlugin")) + RecordMacOsPlugin.register(with: registry.registrar(forPlugin: "RecordMacOsPlugin")) + SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin")) +} diff --git a/macos/Podfile b/macos/Podfile new file mode 100644 index 0000000..0d12243 --- /dev/null +++ b/macos/Podfile @@ -0,0 +1,42 @@ +platform :osx, '14.0' + +# CocoaPods analytics sends network stats synchronously affecting flutter build latency. +ENV['COCOAPODS_DISABLE_STATS'] = 'true' + +project 'Runner', { + 'Debug' => :debug, + 'Profile' => :release, + 'Release' => :release, +} + +def flutter_root + generated_xcode_build_settings_path = File.expand_path(File.join('..', 'Flutter', 'ephemeral', 'Flutter-Generated.xcconfig'), __FILE__) + unless File.exist?(generated_xcode_build_settings_path) + raise "#{generated_xcode_build_settings_path} must exist. If you're running pod install manually, make sure \"flutter pub get\" is executed first" + end + + File.foreach(generated_xcode_build_settings_path) do |line| + matches = line.match(/FLUTTER_ROOT\=(.*)/) + return matches[1].strip if matches + end + raise "FLUTTER_ROOT not found in #{generated_xcode_build_settings_path}. Try deleting Flutter-Generated.xcconfig, then run \"flutter pub get\"" +end + +require File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelper'), flutter_root) + +flutter_macos_podfile_setup + +target 'Runner' do + use_frameworks! + + flutter_install_all_macos_pods File.dirname(File.realpath(__FILE__)) + target 'RunnerTests' do + inherit! :search_paths + end +end + +post_install do |installer| + installer.pods_project.targets.each do |target| + flutter_additional_macos_build_settings(target) + end +end diff --git a/macos/Podfile.lock b/macos/Podfile.lock new file mode 100644 index 0000000..3632001 --- /dev/null +++ b/macos/Podfile.lock @@ -0,0 +1,29 @@ +PODS: + - FlutterMacOS (1.0.0) + - record_macos (1.2.0): + - FlutterMacOS + - shared_preferences_foundation (0.0.1): + - Flutter + - FlutterMacOS + +DEPENDENCIES: + - FlutterMacOS (from `Flutter/ephemeral`) + - record_macos (from `Flutter/ephemeral/.symlinks/plugins/record_macos/macos`) + - shared_preferences_foundation (from `Flutter/ephemeral/.symlinks/plugins/shared_preferences_foundation/darwin`) + +EXTERNAL SOURCES: + FlutterMacOS: + :path: Flutter/ephemeral + record_macos: + :path: Flutter/ephemeral/.symlinks/plugins/record_macos/macos + shared_preferences_foundation: + :path: Flutter/ephemeral/.symlinks/plugins/shared_preferences_foundation/darwin + +SPEC CHECKSUMS: + FlutterMacOS: d0db08ddef1a9af05a5ec4b724367152bb0500b1 + record_macos: 7f227161b93c49e7e34fe681c5891c8622c8cc8b + shared_preferences_foundation: 7036424c3d8ec98dfe75ff1667cb0cd531ec82bb + +PODFILE CHECKSUM: 54d867c82ac51cbd61b565781b9fada492027009 + +COCOAPODS: 1.16.2 diff --git a/macos/Runner.xcodeproj/project.pbxproj b/macos/Runner.xcodeproj/project.pbxproj new file mode 100644 index 0000000..a36b9ea --- /dev/null +++ b/macos/Runner.xcodeproj/project.pbxproj @@ -0,0 +1,801 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 54; + objects = { + +/* Begin PBXAggregateTarget section */ + 33CC111A2044C6BA0003C045 /* Flutter Assemble */ = { + isa = PBXAggregateTarget; + buildConfigurationList = 33CC111B2044C6BA0003C045 /* Build configuration list for PBXAggregateTarget "Flutter Assemble" */; + buildPhases = ( + 33CC111E2044C6BF0003C045 /* ShellScript */, + ); + dependencies = ( + ); + name = "Flutter Assemble"; + productName = FLX; + }; +/* End PBXAggregateTarget section */ + +/* Begin PBXBuildFile section */ + 21772EFEFD81DB52DC5BCE9B /* Pods_Runner.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = B35911AE68DAEEB1F692C9FA /* Pods_Runner.framework */; }; + 331C80D8294CF71000263BE5 /* RunnerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 331C80D7294CF71000263BE5 /* RunnerTests.swift */; }; + 335BBD1B22A9A15E00E9071D /* GeneratedPluginRegistrant.swift in Sources */ = {isa = PBXBuildFile; fileRef = 335BBD1A22A9A15E00E9071D /* GeneratedPluginRegistrant.swift */; }; + 33CC10F12044A3C60003C045 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 33CC10F02044A3C60003C045 /* AppDelegate.swift */; }; + 33CC10F32044A3C60003C045 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 33CC10F22044A3C60003C045 /* Assets.xcassets */; }; + 33CC10F62044A3C60003C045 /* MainMenu.xib in Resources */ = {isa = PBXBuildFile; fileRef = 33CC10F42044A3C60003C045 /* MainMenu.xib */; }; + 33CC11132044BFA00003C045 /* MainFlutterWindow.swift in Sources */ = {isa = PBXBuildFile; fileRef = 33CC11122044BFA00003C045 /* MainFlutterWindow.swift */; }; + BAFE198A8A10983C40F66EDC /* Pods_RunnerTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 372A118AAD5C5D9155B99598 /* Pods_RunnerTests.framework */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + 331C80D9294CF71000263BE5 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 33CC10E52044A3C60003C045 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 33CC10EC2044A3C60003C045; + remoteInfo = Runner; + }; + 33CC111F2044C79F0003C045 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 33CC10E52044A3C60003C045 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 33CC111A2044C6BA0003C045; + remoteInfo = FLX; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXCopyFilesBuildPhase section */ + 33CC110E2044A8840003C045 /* Bundle Framework */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = ""; + dstSubfolderSpec = 10; + files = ( + ); + name = "Bundle Framework"; + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXCopyFilesBuildPhase section */ + +/* Begin PBXFileReference section */ + 331C80D5294CF71000263BE5 /* RunnerTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = RunnerTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + 331C80D7294CF71000263BE5 /* RunnerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RunnerTests.swift; sourceTree = ""; }; + 333000ED22D3DE5D00554162 /* Warnings.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Warnings.xcconfig; sourceTree = ""; }; + 335BBD1A22A9A15E00E9071D /* GeneratedPluginRegistrant.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GeneratedPluginRegistrant.swift; sourceTree = ""; }; + 33CC10ED2044A3C60003C045 /* sunflower_app.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = sunflower_app.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 33CC10F02044A3C60003C045 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; + 33CC10F22044A3C60003C045 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; name = Assets.xcassets; path = Runner/Assets.xcassets; sourceTree = ""; }; + 33CC10F52044A3C60003C045 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = Base; path = Base.lproj/MainMenu.xib; sourceTree = ""; }; + 33CC10F72044A3C60003C045 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; name = Info.plist; path = Runner/Info.plist; sourceTree = ""; }; + 33CC11122044BFA00003C045 /* MainFlutterWindow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MainFlutterWindow.swift; sourceTree = ""; }; + 33CEB47222A05771004F2AC0 /* Flutter-Debug.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = "Flutter-Debug.xcconfig"; sourceTree = ""; }; + 33CEB47422A05771004F2AC0 /* Flutter-Release.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = "Flutter-Release.xcconfig"; sourceTree = ""; }; + 33CEB47722A0578A004F2AC0 /* Flutter-Generated.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; name = "Flutter-Generated.xcconfig"; path = "ephemeral/Flutter-Generated.xcconfig"; sourceTree = ""; }; + 33E51913231747F40026EE4D /* DebugProfile.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = DebugProfile.entitlements; sourceTree = ""; }; + 33E51914231749380026EE4D /* Release.entitlements */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.entitlements; path = Release.entitlements; sourceTree = ""; }; + 33E5194F232828860026EE4D /* AppInfo.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = AppInfo.xcconfig; sourceTree = ""; }; + 372A118AAD5C5D9155B99598 /* Pods_RunnerTests.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_RunnerTests.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + 50781DD5A38B3D9CF1A6609C /* Pods-RunnerTests.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.debug.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.debug.xcconfig"; sourceTree = ""; }; + 599C9BB8A0102C664AC471F2 /* Pods-Runner.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.debug.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig"; sourceTree = ""; }; + 5BD36424AECF8A96FA710756 /* Pods-RunnerTests.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.profile.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.profile.xcconfig"; sourceTree = ""; }; + 7013B57A36F7A8746A2E28BA /* Pods-Runner.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.release.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig"; sourceTree = ""; }; + 7AFA3C8E1D35360C0083082E /* Release.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Release.xcconfig; sourceTree = ""; }; + 9740EEB21CF90195004384FC /* Debug.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = Debug.xcconfig; sourceTree = ""; }; + B35911AE68DAEEB1F692C9FA /* Pods_Runner.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Runner.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + CCE3AE41E818A5497D4BFE5B /* Pods-Runner.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.profile.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.profile.xcconfig"; sourceTree = ""; }; + D1569ED22927242B5D6C12A3 /* Pods-RunnerTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.release.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.release.xcconfig"; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 331C80D2294CF70F00263BE5 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + BAFE198A8A10983C40F66EDC /* Pods_RunnerTests.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 33CC10EA2044A3C60003C045 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 21772EFEFD81DB52DC5BCE9B /* Pods_Runner.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 15EFCF20B57C58844320FF1E /* Pods */ = { + isa = PBXGroup; + children = ( + 599C9BB8A0102C664AC471F2 /* Pods-Runner.debug.xcconfig */, + 7013B57A36F7A8746A2E28BA /* Pods-Runner.release.xcconfig */, + CCE3AE41E818A5497D4BFE5B /* Pods-Runner.profile.xcconfig */, + 50781DD5A38B3D9CF1A6609C /* Pods-RunnerTests.debug.xcconfig */, + D1569ED22927242B5D6C12A3 /* Pods-RunnerTests.release.xcconfig */, + 5BD36424AECF8A96FA710756 /* Pods-RunnerTests.profile.xcconfig */, + ); + name = Pods; + path = Pods; + sourceTree = ""; + }; + 331C80D6294CF71000263BE5 /* RunnerTests */ = { + isa = PBXGroup; + children = ( + 331C80D7294CF71000263BE5 /* RunnerTests.swift */, + ); + path = RunnerTests; + sourceTree = ""; + }; + 33BA886A226E78AF003329D5 /* Configs */ = { + isa = PBXGroup; + children = ( + 33E5194F232828860026EE4D /* AppInfo.xcconfig */, + 9740EEB21CF90195004384FC /* Debug.xcconfig */, + 7AFA3C8E1D35360C0083082E /* Release.xcconfig */, + 333000ED22D3DE5D00554162 /* Warnings.xcconfig */, + ); + path = Configs; + sourceTree = ""; + }; + 33CC10E42044A3C60003C045 = { + isa = PBXGroup; + children = ( + 33FAB671232836740065AC1E /* Runner */, + 33CEB47122A05771004F2AC0 /* Flutter */, + 331C80D6294CF71000263BE5 /* RunnerTests */, + 33CC10EE2044A3C60003C045 /* Products */, + D73912EC22F37F3D000D13A0 /* Frameworks */, + 15EFCF20B57C58844320FF1E /* Pods */, + ); + sourceTree = ""; + }; + 33CC10EE2044A3C60003C045 /* Products */ = { + isa = PBXGroup; + children = ( + 33CC10ED2044A3C60003C045 /* sunflower_app.app */, + 331C80D5294CF71000263BE5 /* RunnerTests.xctest */, + ); + name = Products; + sourceTree = ""; + }; + 33CC11242044D66E0003C045 /* Resources */ = { + isa = PBXGroup; + children = ( + 33CC10F22044A3C60003C045 /* Assets.xcassets */, + 33CC10F42044A3C60003C045 /* MainMenu.xib */, + 33CC10F72044A3C60003C045 /* Info.plist */, + ); + name = Resources; + path = ..; + sourceTree = ""; + }; + 33CEB47122A05771004F2AC0 /* Flutter */ = { + isa = PBXGroup; + children = ( + 335BBD1A22A9A15E00E9071D /* GeneratedPluginRegistrant.swift */, + 33CEB47222A05771004F2AC0 /* Flutter-Debug.xcconfig */, + 33CEB47422A05771004F2AC0 /* Flutter-Release.xcconfig */, + 33CEB47722A0578A004F2AC0 /* Flutter-Generated.xcconfig */, + ); + path = Flutter; + sourceTree = ""; + }; + 33FAB671232836740065AC1E /* Runner */ = { + isa = PBXGroup; + children = ( + 33CC10F02044A3C60003C045 /* AppDelegate.swift */, + 33CC11122044BFA00003C045 /* MainFlutterWindow.swift */, + 33E51913231747F40026EE4D /* DebugProfile.entitlements */, + 33E51914231749380026EE4D /* Release.entitlements */, + 33CC11242044D66E0003C045 /* Resources */, + 33BA886A226E78AF003329D5 /* Configs */, + ); + path = Runner; + sourceTree = ""; + }; + D73912EC22F37F3D000D13A0 /* Frameworks */ = { + isa = PBXGroup; + children = ( + B35911AE68DAEEB1F692C9FA /* Pods_Runner.framework */, + 372A118AAD5C5D9155B99598 /* Pods_RunnerTests.framework */, + ); + name = Frameworks; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 331C80D4294CF70F00263BE5 /* RunnerTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 331C80DE294CF71000263BE5 /* Build configuration list for PBXNativeTarget "RunnerTests" */; + buildPhases = ( + 215E4A5512F3F39BCA0701AC /* [CP] Check Pods Manifest.lock */, + 331C80D1294CF70F00263BE5 /* Sources */, + 331C80D2294CF70F00263BE5 /* Frameworks */, + 331C80D3294CF70F00263BE5 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + 331C80DA294CF71000263BE5 /* PBXTargetDependency */, + ); + name = RunnerTests; + productName = RunnerTests; + productReference = 331C80D5294CF71000263BE5 /* RunnerTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; + 33CC10EC2044A3C60003C045 /* Runner */ = { + isa = PBXNativeTarget; + buildConfigurationList = 33CC10FB2044A3C60003C045 /* Build configuration list for PBXNativeTarget "Runner" */; + buildPhases = ( + 5D04759ED41DE7A2709A86FB /* [CP] Check Pods Manifest.lock */, + 33CC10E92044A3C60003C045 /* Sources */, + 33CC10EA2044A3C60003C045 /* Frameworks */, + 33CC10EB2044A3C60003C045 /* Resources */, + 33CC110E2044A8840003C045 /* Bundle Framework */, + 3399D490228B24CF009A79C7 /* ShellScript */, + 6BF28B7F1D1D02C3CF58CF8B /* [CP] Embed Pods Frameworks */, + ); + buildRules = ( + ); + dependencies = ( + 33CC11202044C79F0003C045 /* PBXTargetDependency */, + ); + name = Runner; + productName = Runner; + productReference = 33CC10ED2044A3C60003C045 /* sunflower_app.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 33CC10E52044A3C60003C045 /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = YES; + LastSwiftUpdateCheck = 0920; + LastUpgradeCheck = 1510; + ORGANIZATIONNAME = ""; + TargetAttributes = { + 331C80D4294CF70F00263BE5 = { + CreatedOnToolsVersion = 14.0; + TestTargetID = 33CC10EC2044A3C60003C045; + }; + 33CC10EC2044A3C60003C045 = { + CreatedOnToolsVersion = 9.2; + LastSwiftMigration = 1100; + ProvisioningStyle = Automatic; + SystemCapabilities = { + com.apple.Sandbox = { + enabled = 1; + }; + }; + }; + 33CC111A2044C6BA0003C045 = { + CreatedOnToolsVersion = 9.2; + ProvisioningStyle = Manual; + }; + }; + }; + buildConfigurationList = 33CC10E82044A3C60003C045 /* Build configuration list for PBXProject "Runner" */; + compatibilityVersion = "Xcode 9.3"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 33CC10E42044A3C60003C045; + productRefGroup = 33CC10EE2044A3C60003C045 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 33CC10EC2044A3C60003C045 /* Runner */, + 331C80D4294CF70F00263BE5 /* RunnerTests */, + 33CC111A2044C6BA0003C045 /* Flutter Assemble */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 331C80D3294CF70F00263BE5 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 33CC10EB2044A3C60003C045 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 33CC10F32044A3C60003C045 /* Assets.xcassets in Resources */, + 33CC10F62044A3C60003C045 /* MainMenu.xib in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXShellScriptBuildPhase section */ + 215E4A5512F3F39BCA0701AC /* [CP] Check Pods Manifest.lock */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + "${PODS_PODFILE_DIR_PATH}/Podfile.lock", + "${PODS_ROOT}/Manifest.lock", + ); + name = "[CP] Check Pods Manifest.lock"; + outputFileListPaths = ( + ); + outputPaths = ( + "$(DERIVED_FILE_DIR)/Pods-RunnerTests-checkManifestLockResult.txt", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; + showEnvVarsInLog = 0; + }; + 3399D490228B24CF009A79C7 /* ShellScript */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "echo \"$PRODUCT_NAME.app\" > \"$PROJECT_DIR\"/Flutter/ephemeral/.app_filename && \"$FLUTTER_ROOT\"/packages/flutter_tools/bin/macos_assemble.sh embed\n"; + }; + 33CC111E2044C6BF0003C045 /* ShellScript */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + Flutter/ephemeral/FlutterInputs.xcfilelist, + ); + inputPaths = ( + Flutter/ephemeral/tripwire, + ); + outputFileListPaths = ( + Flutter/ephemeral/FlutterOutputs.xcfilelist, + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "\"$FLUTTER_ROOT\"/packages/flutter_tools/bin/macos_assemble.sh && touch Flutter/ephemeral/tripwire"; + }; + 5D04759ED41DE7A2709A86FB /* [CP] Check Pods Manifest.lock */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + "${PODS_PODFILE_DIR_PATH}/Podfile.lock", + "${PODS_ROOT}/Manifest.lock", + ); + name = "[CP] Check Pods Manifest.lock"; + outputFileListPaths = ( + ); + outputPaths = ( + "$(DERIVED_FILE_DIR)/Pods-Runner-checkManifestLockResult.txt", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; + showEnvVarsInLog = 0; + }; + 6BF28B7F1D1D02C3CF58CF8B /* [CP] Embed Pods Frameworks */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks-${CONFIGURATION}-input-files.xcfilelist", + ); + name = "[CP] Embed Pods Frameworks"; + outputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks-${CONFIGURATION}-output-files.xcfilelist", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks.sh\"\n"; + showEnvVarsInLog = 0; + }; +/* End PBXShellScriptBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 331C80D1294CF70F00263BE5 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 331C80D8294CF71000263BE5 /* RunnerTests.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 33CC10E92044A3C60003C045 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 33CC11132044BFA00003C045 /* MainFlutterWindow.swift in Sources */, + 33CC10F12044A3C60003C045 /* AppDelegate.swift in Sources */, + 335BBD1B22A9A15E00E9071D /* GeneratedPluginRegistrant.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 331C80DA294CF71000263BE5 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 33CC10EC2044A3C60003C045 /* Runner */; + targetProxy = 331C80D9294CF71000263BE5 /* PBXContainerItemProxy */; + }; + 33CC11202044C79F0003C045 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 33CC111A2044C6BA0003C045 /* Flutter Assemble */; + targetProxy = 33CC111F2044C79F0003C045 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin PBXVariantGroup section */ + 33CC10F42044A3C60003C045 /* MainMenu.xib */ = { + isa = PBXVariantGroup; + children = ( + 33CC10F52044A3C60003C045 /* Base */, + ); + name = MainMenu.xib; + path = Runner; + sourceTree = ""; + }; +/* End PBXVariantGroup section */ + +/* Begin XCBuildConfiguration section */ + 331C80DB294CF71000263BE5 /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 50781DD5A38B3D9CF1A6609C /* Pods-RunnerTests.debug.xcconfig */; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = ai.sunbird.sunflowerApp.RunnerTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 5.0; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/sunflower_app.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/sunflower_app"; + }; + name = Debug; + }; + 331C80DC294CF71000263BE5 /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = D1569ED22927242B5D6C12A3 /* Pods-RunnerTests.release.xcconfig */; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = ai.sunbird.sunflowerApp.RunnerTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 5.0; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/sunflower_app.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/sunflower_app"; + }; + name = Release; + }; + 331C80DD294CF71000263BE5 /* Profile */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 5BD36424AECF8A96FA710756 /* Pods-RunnerTests.profile.xcconfig */; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = ai.sunbird.sunflowerApp.RunnerTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 5.0; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/sunflower_app.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/sunflower_app"; + }; + name = Profile; + }; + 338D0CE9231458BD00FA5F75 /* Profile */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 7AFA3C8E1D35360C0083082E /* Release.xcconfig */; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CODE_SIGN_IDENTITY = "-"; + COPY_PHASE_STRIP = NO; + DEAD_CODE_STRIPPING = YES; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 10.15; + MTL_ENABLE_DEBUG_INFO = NO; + SDKROOT = macosx; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; + }; + name = Profile; + }; + 338D0CEA231458BD00FA5F75 /* Profile */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 33E5194F232828860026EE4D /* AppInfo.xcconfig */; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CLANG_ENABLE_MODULES = YES; + CODE_SIGN_ENTITLEMENTS = Runner/DebugProfile.entitlements; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + INFOPLIST_FILE = Runner/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + ); + PROVISIONING_PROFILE_SPECIFIER = ""; + SWIFT_VERSION = 5.0; + }; + name = Profile; + }; + 338D0CEB231458BD00FA5F75 /* Profile */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Manual; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Profile; + }; + 33CC10F92044A3C60003C045 /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 9740EEB21CF90195004384FC /* Debug.xcconfig */; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CODE_SIGN_IDENTITY = "-"; + COPY_PHASE_STRIP = NO; + DEAD_CODE_STRIPPING = YES; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 10.15; + MTL_ENABLE_DEBUG_INFO = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = macosx; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + }; + name = Debug; + }; + 33CC10FA2044A3C60003C045 /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 7AFA3C8E1D35360C0083082E /* Release.xcconfig */; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CODE_SIGN_IDENTITY = "-"; + COPY_PHASE_STRIP = NO; + DEAD_CODE_STRIPPING = YES; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 10.15; + MTL_ENABLE_DEBUG_INFO = NO; + SDKROOT = macosx; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; + }; + name = Release; + }; + 33CC10FC2044A3C60003C045 /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 33E5194F232828860026EE4D /* AppInfo.xcconfig */; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CLANG_ENABLE_MODULES = YES; + CODE_SIGN_ENTITLEMENTS = Runner/DebugProfile.entitlements; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + INFOPLIST_FILE = Runner/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + ); + PROVISIONING_PROFILE_SPECIFIER = ""; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_VERSION = 5.0; + }; + name = Debug; + }; + 33CC10FD2044A3C60003C045 /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 33E5194F232828860026EE4D /* AppInfo.xcconfig */; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CLANG_ENABLE_MODULES = YES; + CODE_SIGN_ENTITLEMENTS = Runner/Release.entitlements; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + INFOPLIST_FILE = Runner/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + ); + PROVISIONING_PROFILE_SPECIFIER = ""; + SWIFT_VERSION = 5.0; + }; + name = Release; + }; + 33CC111C2044C6BA0003C045 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Manual; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + 33CC111D2044C6BA0003C045 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 331C80DE294CF71000263BE5 /* Build configuration list for PBXNativeTarget "RunnerTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 331C80DB294CF71000263BE5 /* Debug */, + 331C80DC294CF71000263BE5 /* Release */, + 331C80DD294CF71000263BE5 /* Profile */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 33CC10E82044A3C60003C045 /* Build configuration list for PBXProject "Runner" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 33CC10F92044A3C60003C045 /* Debug */, + 33CC10FA2044A3C60003C045 /* Release */, + 338D0CE9231458BD00FA5F75 /* Profile */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 33CC10FB2044A3C60003C045 /* Build configuration list for PBXNativeTarget "Runner" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 33CC10FC2044A3C60003C045 /* Debug */, + 33CC10FD2044A3C60003C045 /* Release */, + 338D0CEA231458BD00FA5F75 /* Profile */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 33CC111B2044C6BA0003C045 /* Build configuration list for PBXAggregateTarget "Flutter Assemble" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 33CC111C2044C6BA0003C045 /* Debug */, + 33CC111D2044C6BA0003C045 /* Release */, + 338D0CEB231458BD00FA5F75 /* Profile */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 33CC10E52044A3C60003C045 /* Project object */; +} diff --git a/macos/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/macos/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 0000000..18d9810 --- /dev/null +++ b/macos/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme new file mode 100644 index 0000000..6615fb5 --- /dev/null +++ b/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/macos/Runner.xcworkspace/contents.xcworkspacedata b/macos/Runner.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..21a3cc1 --- /dev/null +++ b/macos/Runner.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,10 @@ + + + + + + + diff --git a/macos/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/macos/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 0000000..18d9810 --- /dev/null +++ b/macos/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/macos/Runner/AppDelegate.swift b/macos/Runner/AppDelegate.swift new file mode 100644 index 0000000..b3c1761 --- /dev/null +++ b/macos/Runner/AppDelegate.swift @@ -0,0 +1,13 @@ +import Cocoa +import FlutterMacOS + +@main +class AppDelegate: FlutterAppDelegate { + override func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { + return true + } + + override func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool { + return true + } +} diff --git a/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json b/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 0000000..a2ec33f --- /dev/null +++ b/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,68 @@ +{ + "images" : [ + { + "size" : "16x16", + "idiom" : "mac", + "filename" : "app_icon_16.png", + "scale" : "1x" + }, + { + "size" : "16x16", + "idiom" : "mac", + "filename" : "app_icon_32.png", + "scale" : "2x" + }, + { + "size" : "32x32", + "idiom" : "mac", + "filename" : "app_icon_32.png", + "scale" : "1x" + }, + { + "size" : "32x32", + "idiom" : "mac", + "filename" : "app_icon_64.png", + "scale" : "2x" + }, + { + "size" : "128x128", + "idiom" : "mac", + "filename" : "app_icon_128.png", + "scale" : "1x" + }, + { + "size" : "128x128", + "idiom" : "mac", + "filename" : "app_icon_256.png", + "scale" : "2x" + }, + { + "size" : "256x256", + "idiom" : "mac", + "filename" : "app_icon_256.png", + "scale" : "1x" + }, + { + "size" : "256x256", + "idiom" : "mac", + "filename" : "app_icon_512.png", + "scale" : "2x" + }, + { + "size" : "512x512", + "idiom" : "mac", + "filename" : "app_icon_512.png", + "scale" : "1x" + }, + { + "size" : "512x512", + "idiom" : "mac", + "filename" : "app_icon_1024.png", + "scale" : "2x" + } + ], + "info" : { + "version" : 1, + "author" : "xcode" + } +} diff --git a/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png new file mode 100644 index 0000000..82b6f9d Binary files /dev/null and b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png differ diff --git a/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png new file mode 100644 index 0000000..13b35eb Binary files /dev/null and b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png differ diff --git a/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png new file mode 100644 index 0000000..0a3f5fa Binary files /dev/null and b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png differ diff --git a/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png new file mode 100644 index 0000000..bdb5722 Binary files /dev/null and b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png differ diff --git a/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png new file mode 100644 index 0000000..f083318 Binary files /dev/null and b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png differ diff --git a/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png new file mode 100644 index 0000000..326c0e7 Binary files /dev/null and b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png differ diff --git a/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png new file mode 100644 index 0000000..2f1632c Binary files /dev/null and b/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png differ diff --git a/macos/Runner/Base.lproj/MainMenu.xib b/macos/Runner/Base.lproj/MainMenu.xib new file mode 100644 index 0000000..80e867a --- /dev/null +++ b/macos/Runner/Base.lproj/MainMenu.xib @@ -0,0 +1,343 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/macos/Runner/Configs/AppInfo.xcconfig b/macos/Runner/Configs/AppInfo.xcconfig new file mode 100644 index 0000000..219532f --- /dev/null +++ b/macos/Runner/Configs/AppInfo.xcconfig @@ -0,0 +1,14 @@ +// Application-level settings for the Runner target. +// +// This may be replaced with something auto-generated from metadata (e.g., pubspec.yaml) in the +// future. If not, the values below would default to using the project name when this becomes a +// 'flutter create' template. + +// The application's name. By default this is also the title of the Flutter window. +PRODUCT_NAME = sunflower_app + +// The application's bundle identifier +PRODUCT_BUNDLE_IDENTIFIER = ai.sunbird.sunflowerApp + +// The copyright displayed in application information +PRODUCT_COPYRIGHT = Copyright © 2026 ai.sunbird. All rights reserved. diff --git a/macos/Runner/Configs/Debug.xcconfig b/macos/Runner/Configs/Debug.xcconfig new file mode 100644 index 0000000..36b0fd9 --- /dev/null +++ b/macos/Runner/Configs/Debug.xcconfig @@ -0,0 +1,2 @@ +#include "../../Flutter/Flutter-Debug.xcconfig" +#include "Warnings.xcconfig" diff --git a/macos/Runner/Configs/Release.xcconfig b/macos/Runner/Configs/Release.xcconfig new file mode 100644 index 0000000..dff4f49 --- /dev/null +++ b/macos/Runner/Configs/Release.xcconfig @@ -0,0 +1,2 @@ +#include "../../Flutter/Flutter-Release.xcconfig" +#include "Warnings.xcconfig" diff --git a/macos/Runner/Configs/Warnings.xcconfig b/macos/Runner/Configs/Warnings.xcconfig new file mode 100644 index 0000000..42bcbf4 --- /dev/null +++ b/macos/Runner/Configs/Warnings.xcconfig @@ -0,0 +1,13 @@ +WARNING_CFLAGS = -Wall -Wconditional-uninitialized -Wnullable-to-nonnull-conversion -Wmissing-method-return-type -Woverlength-strings +GCC_WARN_UNDECLARED_SELECTOR = YES +CLANG_UNDEFINED_BEHAVIOR_SANITIZER_NULLABILITY = YES +CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE +CLANG_WARN__DUPLICATE_METHOD_MATCH = YES +CLANG_WARN_PRAGMA_PACK = YES +CLANG_WARN_STRICT_PROTOTYPES = YES +CLANG_WARN_COMMA = YES +GCC_WARN_STRICT_SELECTOR_MATCH = YES +CLANG_WARN_OBJC_REPEATED_USE_OF_WEAK = YES +CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES +GCC_WARN_SHADOW = YES +CLANG_WARN_UNREACHABLE_CODE = YES diff --git a/macos/Runner/DebugProfile.entitlements b/macos/Runner/DebugProfile.entitlements new file mode 100644 index 0000000..dddb8a3 --- /dev/null +++ b/macos/Runner/DebugProfile.entitlements @@ -0,0 +1,12 @@ + + + + + com.apple.security.app-sandbox + + com.apple.security.cs.allow-jit + + com.apple.security.network.server + + + diff --git a/macos/Runner/Info.plist b/macos/Runner/Info.plist new file mode 100644 index 0000000..4789daa --- /dev/null +++ b/macos/Runner/Info.plist @@ -0,0 +1,32 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIconFile + + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + $(FLUTTER_BUILD_NAME) + CFBundleVersion + $(FLUTTER_BUILD_NUMBER) + LSMinimumSystemVersion + $(MACOSX_DEPLOYMENT_TARGET) + NSHumanReadableCopyright + $(PRODUCT_COPYRIGHT) + NSMainNibFile + MainMenu + NSPrincipalClass + NSApplication + + diff --git a/macos/Runner/MainFlutterWindow.swift b/macos/Runner/MainFlutterWindow.swift new file mode 100644 index 0000000..3cc05eb --- /dev/null +++ b/macos/Runner/MainFlutterWindow.swift @@ -0,0 +1,15 @@ +import Cocoa +import FlutterMacOS + +class MainFlutterWindow: NSWindow { + override func awakeFromNib() { + let flutterViewController = FlutterViewController() + let windowFrame = self.frame + self.contentViewController = flutterViewController + self.setFrame(windowFrame, display: true) + + RegisterGeneratedPlugins(registry: flutterViewController) + + super.awakeFromNib() + } +} diff --git a/macos/Runner/Release.entitlements b/macos/Runner/Release.entitlements new file mode 100644 index 0000000..852fa1a --- /dev/null +++ b/macos/Runner/Release.entitlements @@ -0,0 +1,8 @@ + + + + + com.apple.security.app-sandbox + + + diff --git a/macos/RunnerTests/RunnerTests.swift b/macos/RunnerTests/RunnerTests.swift new file mode 100644 index 0000000..61f3bd1 --- /dev/null +++ b/macos/RunnerTests/RunnerTests.swift @@ -0,0 +1,12 @@ +import Cocoa +import FlutterMacOS +import XCTest + +class RunnerTests: XCTestCase { + + func testExample() { + // If you add code to the Runner application, consider adding tests here. + // See https://developer.apple.com/documentation/xctest for more information about using XCTest. + } + +} diff --git a/macos/cactus-macos.xcframework/Info.plist b/macos/cactus-macos.xcframework/Info.plist new file mode 100644 index 0000000..fe06ebe --- /dev/null +++ b/macos/cactus-macos.xcframework/Info.plist @@ -0,0 +1,25 @@ + + + + + AvailableLibraries + + + LibraryIdentifier + macos-arm64 + LibraryPath + cactus.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + macos + + + CFBundlePackageType + XFWK + XCFrameworkFormatVersion + 1.0 + + diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/cactus.h b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/cactus.h new file mode 100644 index 0000000..b5b6d80 --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/cactus.h @@ -0,0 +1,12 @@ +#ifndef CACTUS_H +#define CACTUS_H + +#include "graph/graph.h" +#include "kernel/kernel.h" +#include "kernel/kernel_utils.h" +#include "engine/engine.h" +#include "models/model.h" +#include "ffi/cactus_ffi.h" +#include "npu/npu.h" + +#endif // CACTUS_H \ No newline at end of file diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/cactus_cloud.h b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/cactus_cloud.h new file mode 100644 index 0000000..6911d9a --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/cactus_cloud.h @@ -0,0 +1,50 @@ +#ifndef CACTUS_CLOUD_H +#define CACTUS_CLOUD_H + +#include "cactus_utils.h" +#include +#include + +namespace cactus { +namespace ffi { + +struct CloudResponse { + std::string transcript; + std::string api_key_hash; + bool used_cloud = false; + std::string error; +}; + +struct CloudCompletionRequest { + std::vector messages; + std::vector tools; + std::string local_output; + std::vector local_function_calls; + bool has_images = false; + bool has_audio = false; + std::vector audio_pcm; + std::string cloud_key; +}; + +struct CloudCompletionResult { + bool ok = false; + bool used_cloud = false; + std::string response; + std::vector function_calls; + std::string error; +}; + +std::string cloud_base64_encode(const uint8_t* data, size_t len); +std::vector cloud_build_wav(const uint8_t* pcm, size_t pcm_bytes); +std::string resolve_cloud_api_key(const char* cloud_key_param); +CloudResponse cloud_transcribe_request(const std::string& audio_b64, + const std::string& fallback_text, + long timeout_seconds = 15L, + const char* cloud_key = nullptr); +CloudCompletionResult cloud_complete_request(const CloudCompletionRequest& request, + long timeout_ms); + +} // namespace ffi +} // namespace cactus + +#endif // CACTUS_CLOUD_H diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/cactus_ffi.h b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/cactus_ffi.h new file mode 100644 index 0000000..163e180 --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/cactus_ffi.h @@ -0,0 +1,454 @@ +#ifndef CACTUS_FFI_H +#define CACTUS_FFI_H + +#include +#include +#include + +#if __GNUC__ >= 4 + #define CACTUS_FFI_EXPORT __attribute__((visibility("default"))) + #define CACTUS_FFI_LOCAL __attribute__((visibility("hidden"))) +#else + #define CACTUS_FFI_EXPORT + #define CACTUS_FFI_LOCAL +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* cactus_model_t; +typedef void* cactus_index_t; +typedef void* cactus_stream_transcribe_t; + +typedef void (*cactus_token_callback)(const char* token, uint32_t token_id, void* user_data); + +CACTUS_FFI_EXPORT cactus_model_t cactus_init( + const char* model_path, + const char* corpus_dir, // optional: NULL if no RAG corpus + bool cache_index // false = always rebuild index, true = load cached if available +); + +CACTUS_FFI_EXPORT void cactus_destroy(cactus_model_t model); +CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model); +CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model); + +CACTUS_FFI_EXPORT int cactus_complete( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const char* tools_json, // optional + cactus_token_callback callback, // optional + void* user_data, // optional + const uint8_t* pcm_buffer, // optional: NULL when not used + size_t pcm_buffer_size // optional: 0 when not used +); + +CACTUS_FFI_EXPORT int cactus_prefill( + cactus_model_t model, + const char* messages_json, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const char* tools_json, // optional + const uint8_t* pcm_buffer, // optional: NULL when not used + size_t pcm_buffer_size // optional: 0 when not used +); + +CACTUS_FFI_EXPORT int cactus_tokenize( + cactus_model_t model, + const char* text, + uint32_t* token_buffer, + size_t token_buffer_len, + size_t* out_token_len +); + +CACTUS_FFI_EXPORT int cactus_score_window( + cactus_model_t model, + const uint32_t* tokens, + size_t token_len, + size_t start, + size_t end, + size_t context, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_transcribe( + cactus_model_t model, + const char* audio_file_path, // NULL if using pcm_buffer + const char* prompt, + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + cactus_token_callback callback, // optional + void* user_data, // optional + const uint8_t* pcm_buffer, // NULL if using audio_file_path + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_detect_language( + cactus_model_t model, + const char* audio_file_path, // NULL if using pcm_buffer + char* response_buffer, + size_t buffer_size, + const char* options_json, // optional + const uint8_t* pcm_buffer, // NULL if using audio_file_path + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT cactus_stream_transcribe_t cactus_stream_transcribe_start( + cactus_model_t model, + const char* options_json // optional +); + +CACTUS_FFI_EXPORT int cactus_stream_transcribe_process( + cactus_stream_transcribe_t stream, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_stream_transcribe_stop( + cactus_stream_transcribe_t stream, + char* response_buffer, + size_t buffer_size +); + +CACTUS_FFI_EXPORT int cactus_embed( + cactus_model_t model, + const char* text, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim, + bool normalize +); + +CACTUS_FFI_EXPORT int cactus_image_embed( + cactus_model_t model, + const char* image_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_audio_embed( + cactus_model_t model, + const char* audio_path, + float* embeddings_buffer, + size_t buffer_size, + size_t* embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_vad( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_diarize( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size +); + +CACTUS_FFI_EXPORT int cactus_embed_speaker( + cactus_model_t model, + const char* audio_file_path, + char* response_buffer, + size_t buffer_size, + const char* options_json, + const uint8_t* pcm_buffer, + size_t pcm_buffer_size, + const float* mask_weights, + size_t mask_num_frames +); + +CACTUS_FFI_EXPORT int cactus_rag_query( + cactus_model_t model, + const char* query, + char* response_buffer, + size_t buffer_size, + size_t top_k +); + +CACTUS_FFI_EXPORT cactus_index_t cactus_index_init( + const char* index_dir, + size_t embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_index_add( + cactus_index_t index, + const int* ids, + const char** documents, + const char** metadatas, // optional: can be NULL + const float** embeddings, + size_t count, + size_t embedding_dim +); + +CACTUS_FFI_EXPORT int cactus_index_delete( + cactus_index_t index, + const int* ids, + size_t ids_count +); + +CACTUS_FFI_EXPORT int cactus_index_get( + cactus_index_t index, + const int* ids, + size_t ids_count, + char** document_buffers, + size_t* document_buffer_sizes, + char** metadata_buffers, + size_t* metadata_buffer_sizes, + float** embedding_buffers, + size_t* embedding_buffer_sizes +); + +CACTUS_FFI_EXPORT int cactus_index_query( + cactus_index_t index, + const float** embeddings, + size_t embeddings_count, + size_t embedding_dim, + const char* options_json, // optional + int** id_buffers, + size_t* id_buffer_sizes, + float** score_buffers, + size_t* score_buffer_sizes +); + +CACTUS_FFI_EXPORT int cactus_index_compact(cactus_index_t index); +CACTUS_FFI_EXPORT void cactus_index_destroy(cactus_index_t index); + +CACTUS_FFI_EXPORT const char* cactus_get_last_error(void); + +// level: 0=DEBUG, 1=INFO, 2=WARN (default), 3=ERROR, 4=NONE +CACTUS_FFI_EXPORT void cactus_log_set_level(int level); + +typedef void (*cactus_log_callback_t)(int level, const char* component, const char* message, void* user_data); +CACTUS_FFI_EXPORT void cactus_log_set_callback(cactus_log_callback_t callback, void* user_data); + +CACTUS_FFI_EXPORT void cactus_set_telemetry_environment(const char* framework, const char* cache_location, const char* version); +CACTUS_FFI_EXPORT void cactus_set_app_id(const char* app_id); +CACTUS_FFI_EXPORT void cactus_telemetry_flush(void); +CACTUS_FFI_EXPORT void cactus_telemetry_shutdown(void); + +// cactus graph export +typedef void* cactus_graph_t; +typedef uint64_t cactus_node_t; + +typedef struct { + int32_t precision; + size_t rank; + size_t shape[8]; + size_t num_elements; + size_t byte_size; +} cactus_tensor_info_t; + +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_create(void); +CACTUS_FFI_EXPORT void cactus_graph_destroy(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_hard_reset(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_save(cactus_graph_t graph, const char* filename); +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_load(const char* filename); + +CACTUS_FFI_EXPORT int cactus_graph_input( + cactus_graph_t graph, const size_t* shape, size_t rank, int32_t precision, +cactus_node_t* out_node); + +CACTUS_FFI_EXPORT int cactus_graph_set_input( + cactus_graph_t graph, cactus_node_t node, const void* data, int32_t +precision); +CACTUS_FFI_EXPORT int cactus_graph_set_external_input( + cactus_graph_t graph, cactus_node_t node, void* data, int32_t precision); + +CACTUS_FFI_EXPORT int cactus_graph_precision_cast( + cactus_graph_t graph, cactus_node_t input, int32_t target_precision, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_quantize_activations( + cactus_graph_t graph, cactus_node_t input, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_add(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_add_clipped(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_subtract(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_multiply(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_divide(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_scalar_add(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_subtract(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_multiply(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_exp(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sqrt(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_cos(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sin(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_log(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_abs(cactus_graph_t graph, cactus_node_t x, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_pow(cactus_graph_t graph, cactus_node_t x, +float exponent, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_view( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_flatten( + cactus_graph_t graph, cactus_node_t x, int32_t start_dim, int32_t end_dim, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_reshape( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose( + cactus_graph_t graph, cactus_node_t x, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose_n( + cactus_graph_t graph, cactus_node_t x, const size_t* permutation, size_t rank, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_slice( + cactus_graph_t graph, cactus_node_t x, int32_t axis, size_t start, size_t length, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_index( + cactus_graph_t graph, cactus_node_t x, size_t index_value, int32_t dim, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_concat( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, +cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_cat( + cactus_graph_t graph, const cactus_node_t* nodes, size_t count, int32_t +axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_matmul( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, bool pretransposed_rhs, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gather( + cactus_graph_t graph, cactus_node_t tensor, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_tensor( + cactus_graph_t graph, cactus_node_t embedding_tensor, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_file( + cactus_graph_t graph, const char* filename, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_embeddings( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_weights( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_bilinear_interpolation( + cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_set_grouped_scales( + cactus_graph_t graph, cactus_node_t node, size_t group_size, size_t num_groups, void* scales_ptr); +CACTUS_FFI_EXPORT int cactus_graph_set_interleaved( + cactus_graph_t graph, cactus_node_t node, bool interleaved, size_t original_n); +CACTUS_FFI_EXPORT int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_prefetch_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_release_all_weight_pages(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_relu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_silu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu_erf(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sigmoid(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_tanh(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_glu(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_layernorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, float epsilon, bool has_bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_groupnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, size_t num_groups, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_batchnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t running_mean, cactus_node_t running_var, int32_t axis, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_topk(cactus_graph_t graph, cactus_node_t input, size_t k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rms_norm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope_gptj( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, size_t rot_dim, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_softmax(cactus_graph_t graph, cactus_node_t input, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, float scale, bool is_causal, size_t position_offset, size_t window_size, int32_t backend, bool use_mask, cactus_node_t mask, bool additive_mask, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rel_pos_bias( + cactus_graph_t graph, cactus_node_t query, cactus_node_t relative_key, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention_int8_hybrid( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_conv1d_causal( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t kernel_size, size_t dilation, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k7s3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_same_depthwise_k9( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_pointwise( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_depthwise_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_pointwise_1x1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_lstm_cell( + cactus_graph_t graph, cactus_node_t input, cactus_node_t h_prev, cactus_node_t c_prev, cactus_node_t weight_ih, cactus_node_t weight_hh, cactus_node_t bias_ih, cactus_node_t bias_hh, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_decode( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_prefill( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, size_t chunk_size, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_stft( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, size_t num_fft_bins, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_altup_predict( + cactus_graph_t graph, cactus_node_t coefs, const cactus_node_t* streams, size_t num_streams, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_altup_correct( + cactus_graph_t graph, cactus_node_t coefs, cactus_node_t innovation, const cactus_node_t* predictions, size_t num_predictions, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gaussian_topk( + cactus_graph_t graph, cactus_node_t input, float ppf, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_gated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_ungated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_sample( + cactus_graph_t graph, cactus_node_t logits, float temperature, float top_p, size_t top_k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scatter_topk( + cactus_graph_t graph, cactus_node_t indices, cactus_node_t values, size_t num_classes, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_persistent( + cactus_graph_t graph, cactus_node_t source_node, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_is_populated( + cactus_graph_t graph, cactus_node_t persistent_node, int32_t* out_is_populated); +CACTUS_FFI_EXPORT int cactus_graph_invalidate_persistent( + cactus_graph_t graph, cactus_node_t persistent_node); + +CACTUS_FFI_EXPORT int cactus_graph_execute(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_get_output_ptr(cactus_graph_t graph, +cactus_node_t node, void** out_ptr); +CACTUS_FFI_EXPORT int cactus_graph_get_output_info(cactus_graph_t graph, +cactus_node_t node, cactus_tensor_info_t* out_info); + +#ifdef __cplusplus +} +#endif + +#endif // CACTUS_FFI_H diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/cactus_utils.h b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/cactus_utils.h new file mode 100644 index 0000000..aad98d8 --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/cactus_utils.h @@ -0,0 +1,1869 @@ +#ifndef CACTUS_UTILS_H +#define CACTUS_UTILS_H + +#include "../engine/engine.h" +#include "../models/model.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#include +#elif defined(_WIN32) +#include +#include +#elif defined(__linux__) || defined(__ANDROID__) +#include +#endif + +inline size_t get_memory_footprint_bytes() { +#ifdef __APPLE__ + task_vm_info_data_t vm_info; + mach_msg_type_number_t count = TASK_VM_INFO_COUNT; + if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&vm_info, &count) == KERN_SUCCESS) + return vm_info.phys_footprint; + +#elif defined(_WIN32) + PROCESS_MEMORY_COUNTERS_EX pmc; + if (GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc))) + return pmc.PrivateUsage; + +#elif defined(__linux__) || defined(__ANDROID__) + std::ifstream statm("/proc/self/statm"); + if (statm.is_open()) { + size_t size, resident; + statm >> size >> resident; + return resident * sysconf(_SC_PAGESIZE); + } +#endif + return 0; +} + +inline double get_ram_usage_mb() { + return get_memory_footprint_bytes() / (1024.0 * 1024.0); +} + +struct CactusModelHandle { + std::unique_ptr model; + std::unique_ptr vad_model; + std::atomic should_stop; + std::vector processed_tokens; + struct ProcessedImage { + std::string path; + long long last_modified_timestamp = 0; + + bool operator==(const ProcessedImage& other) const { + return path == other.path && last_modified_timestamp == other.last_modified_timestamp; + } + }; + + std::vector> processed_images; + std::vector user_audio_counts; + std::mutex model_mutex; + std::string model_name; + std::unique_ptr corpus_index; + std::string corpus_dir; + size_t corpus_embedding_dim = 0; + std::vector> tool_embeddings; + std::vector tool_texts; + + CactusModelHandle() : should_stop(false) {} +}; + +extern std::string last_error_message; + +bool matches_stop_sequence(const std::vector& generated_tokens, + const std::vector>& stop_sequences); + +std::string retrieve_rag_context(CactusModelHandle* handle, const std::string& query); + +namespace cactus { +namespace audio { + +static constexpr size_t WHISPER_TARGET_FRAMES = 3000; +static constexpr int WHISPER_SAMPLE_RATE = 16000; + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_whisper_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 400; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = true; + cfg.pad_mode = "reflect"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 1e-10f; + cfg.log_mel = "log10"; + cfg.reference = 1.0f; + cfg.min_value = 1e-10f; + cfg.remove_dc_offset = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_parakeet_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 512; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = true; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 5.960464477539063e-08f; // 2^-24 guard value used by HF Parakeet. + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 1e-10f; + cfg.remove_dc_offset = false; + cfg.hann_periodic = false; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_htk_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 321; + cfg.frame_length = 320; + cfg.fft_override = 1024; + cfg.hop_length = 160; + cfg.power = 1.0f; + cfg.center = false; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 0.001f; + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 0.001f; + cfg.remove_dc_offset = false; + cfg.hann_periodic = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_gemma4_audio_spectrogram_config( + const cactus::engine::Config& model_config) { + auto cfg = get_htk_spectrogram_config(); + cfg.fft_override = model_config.audio_fft_length; + cfg.mel_floor_additive = true; + return cfg; +} + +inline cactus::engine::AudioProcessor::SpectrogramConfig get_wespeaker_spectrogram_config() { + cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; + cfg.n_fft = 512; + cfg.frame_length = 400; + cfg.hop_length = 160; + cfg.power = 2.0f; + cfg.center = false; + cfg.pad_mode = "constant"; + cfg.onesided = true; + cfg.dither = 0.0f; + cfg.mel_floor = 1.1754944e-38f; + cfg.log_mel = "log"; + cfg.reference = 1.0f; + cfg.min_value = 1.1754944e-38f; + cfg.remove_dc_offset = true; + cfg.preemphasis = 0.97f; + cfg.hann_periodic = false; + cfg.window_a0 = 0.54f; + return cfg; +} + +// Whisper v1/v2: 80 mel bins, HTK. Whisper v3: 128 mel bins, Slaney, 512-FFT, no DC removal. +inline void init_whisper_mel_filters(cactus::engine::AudioProcessor& ap, + cactus::engine::AudioProcessor::SpectrogramConfig& cfg, + size_t mel_bins) { + const size_t num_mel_filters = std::max(1, mel_bins); + const bool is_v3 = mel_bins > 80; + if (is_v3) { + cfg.fft_override = 512; + cfg.remove_dc_offset = false; + } + const size_t fft_len = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; + const size_t num_frequency_bins = fft_len / 2 + 1; + if (is_v3) { + ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, + WHISPER_SAMPLE_RATE, "slaney", "slaney"); + } else { + ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, + WHISPER_SAMPLE_RATE); + } +} + +// use_mel_floor_padding=true pads short audio with the normalized mel floor (required for v3). +inline std::vector normalize_whisper_mel(std::vector& mel, size_t n_mels, + bool use_mel_floor_padding = false) { + if (mel.empty() || n_mels == 0) return mel; + size_t n_frames = mel.size() / n_mels; + + float max_val = -std::numeric_limits::infinity(); + for (float v : mel) if (v > max_val) max_val = v; + + float min_allowed = max_val - 8.0f; + for (float& v : mel) { + if (v < min_allowed) v = min_allowed; + v = (v + 4.0f) * 0.25f; + } + + if (n_frames != WHISPER_TARGET_FRAMES) { + float pad_val = use_mel_floor_padding ? (min_allowed + 4.0f) * 0.25f : 0.0f; + std::vector fixed(n_mels * WHISPER_TARGET_FRAMES, pad_val); + size_t copy_frames = std::min(n_frames, WHISPER_TARGET_FRAMES); + for (size_t m = 0; m < n_mels; ++m) { + const float* src = &mel[m * n_frames]; + float* dst = &fixed[m * WHISPER_TARGET_FRAMES]; + std::copy(src, src + copy_frames, dst); + } + return fixed; + } + return std::move(mel); +} + +inline std::vector transpose_mel_to_frame_major(const std::vector& mel, + size_t num_mels, size_t num_frames) { + std::vector transposed(num_frames * num_mels); + for (size_t m = 0; m < num_mels; m++) { + for (size_t t = 0; t < num_frames; t++) { + transposed[t * num_mels + m] = mel[m * num_frames + t]; + } + } + return transposed; +} + +inline void apply_preemphasis(std::vector& waveform, float coefficient = 0.97f) { + if (waveform.size() < 2 || coefficient == 0.0f) { + return; + } + for (size_t i = waveform.size() - 1; i > 0; --i) { + waveform[i] -= coefficient * waveform[i - 1]; + } +} + +inline void normalize_parakeet_log_mel(std::vector& mel, size_t num_mels, float epsilon = 1e-5f) { + if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { + return; + } + const size_t num_frames = mel.size() / num_mels; + if (num_frames == 0) { + return; + } + + for (size_t m = 0; m < num_mels; ++m) { + const size_t base = m * num_frames; + float mean = 0.0f; + for (size_t t = 0; t < num_frames; ++t) { + mean += mel[base + t]; + } + mean /= static_cast(num_frames); + + float variance = 0.0f; + for (size_t t = 0; t < num_frames; ++t) { + const float d = mel[base + t] - mean; + variance += d * d; + } + const float denom = static_cast(std::max(1, num_frames - 1)); + const float inv_std = 1.0f / std::sqrt((variance / denom) + epsilon); + for (size_t t = 0; t < num_frames; ++t) { + mel[base + t] = (mel[base + t] - mean) * inv_std; + } + } +} + +inline void trim_mel_frames(std::vector& mel, size_t num_mels, size_t valid_frames) { + if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { + return; + } + size_t total_frames = mel.size() / num_mels; + if (valid_frames == 0 || valid_frames >= total_frames) { + return; + } + std::vector trimmed(num_mels * valid_frames); + for (size_t m = 0; m < num_mels; ++m) { + const float* src = &mel[m * total_frames]; + float* dst = &trimmed[m * valid_frames]; + std::copy(src, src + valid_frames, dst); + } + mel.swap(trimmed); +} + +struct AudioPreprocessResult { + std::vector features; + size_t num_frames = 0; + size_t num_soft_tokens = 0; +}; + +inline AudioPreprocessResult preprocess_audio_for_gemma4( + std::vector audio_samples, + const cactus::engine::Config& model_config +) { + AudioPreprocessResult result; + if (audio_samples.empty()) return result; + + size_t pad_amt = 320 - (audio_samples.size() % 320); + if (pad_amt < 320) + audio_samples.resize(audio_samples.size() + pad_amt, 0.0f); + + size_t mel_bins = model_config.audio_input_feat_size; + auto cfg = get_gemma4_audio_spectrogram_config(model_config); + + size_t semicausal_pad = cfg.frame_length / 2; + audio_samples.insert(audio_samples.begin(), semicausal_pad, 0.0f); + + cactus::engine::AudioProcessor ap; + size_t fft_for_mel = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; + ap.init_mel_filters(fft_for_mel / 2 + 1, mel_bins, 0.0f, 8000.0f, 16000, + nullptr, "htk"); + std::vector mel = ap.compute_spectrogram(audio_samples, cfg); + + result.num_frames = mel.size() / mel_bins; + result.features = transpose_mel_to_frame_major(mel, mel_bins, result.num_frames); + + size_t after_stage1 = (result.num_frames + 1) / 2; + result.num_soft_tokens = (after_stage1 + 1) / 2; + + return result; +} + +inline std::vector pcm_buffer_to_float_samples( + const uint8_t* pcm_buffer, size_t pcm_buffer_size +) { + const int16_t* pcm_samples = reinterpret_cast(pcm_buffer); + size_t num_samples = pcm_buffer_size / 2; + std::vector waveform_fp32(num_samples); + constexpr float inv_32768 = 1.0f / 32768.0f; + for (size_t i = 0; i < num_samples; i++) + waveform_fp32[i] = static_cast(pcm_samples[i]) * inv_32768; + return waveform_fp32; +} + +} // namespace audio +} // namespace cactus + +namespace cactus { +namespace ffi { + +inline bool env_flag_enabled(const char* key) { + const char* value = std::getenv(key); + return value && value[0] != '\0' && !(value[0] == '0' && value[1] == '\0'); +} + +inline std::string generateUUID() { +#ifdef __APPLE__ + uuid_t uuid; + uuid_generate_random(uuid); + char uuid_str[37]; + uuid_unparse_lower(uuid, uuid_str); + return std::string(uuid_str); +#else + static std::random_device rd; + static std::mt19937 gen(rd()); + static std::uniform_int_distribution<> dis(0, 15); + static std::uniform_int_distribution<> dis2(8, 11); + + std::stringstream ss; + ss << std::hex; + for (int i = 0; i < 8; i++) ss << dis(gen); + ss << "-"; + for (int i = 0; i < 4; i++) ss << dis(gen); + ss << "-4"; + for (int i = 0; i < 3; i++) ss << dis(gen); + ss << "-"; + ss << dis2(gen); + for (int i = 0; i < 3; i++) ss << dis(gen); + ss << "-"; + for (int i = 0; i < 12; i++) ss << dis(gen); + return ss.str(); +#endif +} + +struct ToolFunction { + std::string name; + std::string description; + std::unordered_map parameters; +}; + +struct InferenceOptions { + float temperature = 0.0f; + float top_p = 0.0f; + float min_p = 0.15f; + float repetition_penalty = 1.1f; + float confidence_threshold = -1.0f; + size_t top_k = 0; + size_t max_tokens = 100; + size_t tool_rag_top_k = 2; + size_t cloud_timeout_ms = 15000; + std::vector stop_sequences; + bool force_tools = false; + bool include_stop_sequences = false; + bool use_vad = true; + bool telemetry_enabled = true; + bool auto_handoff = true; + bool handoff_with_images = true; + bool enable_thinking_if_supported = false; +}; + +} // namespace ffi +} // namespace cactus + +std::vector select_relevant_tools( + CactusModelHandle* handle, + const std::string& query, + const std::vector& all_tools, + size_t top_k); + +#include "gemma_tools.h" + +namespace cactus { +namespace ffi { + +inline std::string escape_json_string(const std::string& s) { + std::ostringstream o; + for (char c : s) { + if (c == '"') o << "\\\""; + else if (c == '\n') o << "\\n"; + else if (c == '\r') o << "\\r"; + else if (c == '\t') o << "\\t"; + else if (c == '\\') o << "\\\\"; + else o << c; + } + return o.str(); +} + + +inline std::string trim_string(const std::string& s) { + size_t start = 0; + while (start < s.size() && std::isspace(static_cast(s[start]))) ++start; + size_t end = s.size(); + while (end > start && std::isspace(static_cast(s[end - 1]))) --end; + return s.substr(start, end - start); +} + +inline size_t find_matching_delimiter(const std::string& s, size_t pos, char open, char close) { + int depth = 1; + pos++; + while (pos < s.length() && depth > 0) { + if (s[pos] == open) depth++; + else if (s[pos] == close) depth--; + else if (s[pos] == '"') { + pos++; + while (pos < s.length() && s[pos] != '"') { + if (s[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + return pos; +} + +inline std::string env_or_default(const char* key, const char* fallback) { + const char* v = std::getenv(key); + if (v && v[0] != '\0') return std::string(v); + return std::string(fallback); +} + +inline std::string json_string_field(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return {}; + + size_t i = pos + pattern.size(); + while (i < json.size() && std::isspace(static_cast(json[i]))) i++; + if (i >= json.size() || json[i] != '"') return {}; + ++i; + + std::string out; + out.reserve(128); + while (i < json.size()) { + char c = json[i++]; + if (c == '"') return out; + if (c == '\\' && i < json.size()) { + char e = json[i++]; + switch (e) { + case '"': out.push_back('"'); break; + case '\\': out.push_back('\\'); break; + case '/': out.push_back('/'); break; + case 'b': out.push_back('\b'); break; + case 'f': out.push_back('\f'); break; + case 'n': out.push_back('\n'); break; + case 'r': out.push_back('\r'); break; + case 't': out.push_back('\t'); break; + default: out.push_back(e); break; + } + continue; + } + out.push_back(c); + } + return {}; +} + +inline std::string json_array_field(const std::string& json, const std::string& key) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return "[]"; + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + if (start >= json.size() || json[start] != '[') return "[]"; + + int depth = 1; + size_t end = start + 1; + while (end < json.size() && depth > 0) { + if (json[end] == '[') depth++; + else if (json[end] == ']') depth--; + end++; + } + return json.substr(start, end - start); +} + +inline std::vector split_json_array(const std::string& array_json) { + std::vector out; + if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; + + size_t i = 1; + while (i + 1 < array_json.size()) { + while (i + 1 < array_json.size() && + (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) i++; + if (i + 1 >= array_json.size() || array_json[i] != '{') break; + + size_t start = i; + int depth = 0; + bool in_str = false; + bool esc = false; + for (; i < array_json.size(); ++i) { + char c = array_json[i]; + if (in_str) { + if (esc) esc = false; + else if (c == '\\') esc = true; + else if (c == '"') in_str = false; + continue; + } + if (c == '"') { in_str = true; continue; } + if (c == '{') depth++; + if (c == '}') { + depth--; + if (depth == 0) { + out.push_back(array_json.substr(start, i - start + 1)); + i++; + break; + } + } + } + } + return out; +} + +inline std::string serialize_tools_json(const std::vector& tools) { + if (tools.empty()) return ""; + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < tools.size(); ++i) { + if (i > 0) oss << ","; + oss << "{\"type\":\"function\",\"function\":{"; + oss << "\"name\":\"" << escape_json_string(tools[i].name) << "\","; + oss << "\"description\":\"" << escape_json_string(tools[i].description) << "\""; + auto it = tools[i].parameters.find("schema"); + if (it != tools[i].parameters.end()) { + oss << ",\"parameters\":" << it->second; + } + oss << "}}"; + } + oss << "]"; + return oss.str(); +} + +namespace json_sorted { + +inline void skip_ws(const std::string& s, size_t& p) { + while (p < s.size() && std::isspace(static_cast(s[p]))) p++; +} + +inline std::string parse_string(const std::string& s, size_t& p) { + std::string r = "\""; + p++; + while (p < s.size()) { + if (s[p] == '\\') { + r += s[p++]; + if (p < s.size()) r += s[p++]; + } else if (s[p] == '"') { + r += '"'; + p++; + return r; + } else { + r += s[p++]; + } + } + return r; +} + +inline std::string parse_value(const std::string& s, size_t& p); + +inline std::string parse_object(const std::string& s, size_t& p) { + p++; + std::map entries; + skip_ws(s, p); + while (p < s.size() && s[p] != '}') { + if (s[p] == ',') { p++; skip_ws(s, p); continue; } + std::string key = parse_string(s, p); + skip_ws(s, p); + if (p < s.size() && s[p] == ':') p++; + skip_ws(s, p); + std::string val = parse_value(s, p); + entries[key] = val; + skip_ws(s, p); + } + if (p < s.size()) p++; + std::string r = "{"; + bool first = true; + for (const auto& kv : entries) { + if (!first) r += ", "; + r += kv.first + ": " + kv.second; + first = false; + } + r += "}"; + return r; +} + +inline std::string parse_array(const std::string& s, size_t& p) { + p++; + std::vector items; + skip_ws(s, p); + while (p < s.size() && s[p] != ']') { + if (s[p] == ',') { p++; skip_ws(s, p); continue; } + items.push_back(parse_value(s, p)); + skip_ws(s, p); + } + if (p < s.size()) p++; + std::string r = "["; + for (size_t i = 0; i < items.size(); i++) { + if (i > 0) r += ", "; + r += items[i]; + } + r += "]"; + return r; +} + +inline std::string parse_value(const std::string& s, size_t& p) { + skip_ws(s, p); + if (p >= s.size()) return ""; + if (s[p] == '"') return parse_string(s, p); + if (s[p] == '{') return parse_object(s, p); + if (s[p] == '[') return parse_array(s, p); + size_t start = p; + while (p < s.size() && s[p] != ',' && s[p] != '}' && s[p] != ']' && !std::isspace(static_cast(s[p]))) p++; + return s.substr(start, p - start); +} + +inline std::string reformat(const std::string& json) { + size_t p = 0; + return parse_value(json, p); +} + +} // namespace json_sorted + +inline std::string serialize_tools_for_template(const std::vector& tools) { + if (tools.empty()) return ""; + std::string result; + for (const auto& tool : tools) { + std::map func_fields; + func_fields["\"description\""] = "\"" + escape_json_string(tool.description) + "\""; + func_fields["\"name\""] = "\"" + escape_json_string(tool.name) + "\""; + auto it = tool.parameters.find("schema"); + if (it != tool.parameters.end()) { + func_fields["\"parameters\""] = json_sorted::reformat(it->second); + } + std::string func_json = "{"; + bool first = true; + for (const auto& kv : func_fields) { + if (!first) func_json += ", "; + func_json += kv.first + ": " + kv.second; + first = false; + } + func_json += "}"; + result += "\n{\"function\": " + func_json + ", \"type\": \"function\"}"; + } + return result; +} + +inline void handle_error_response(const std::string& error_message, char* response_buffer, size_t buffer_size) { + std::ostringstream json; + json << "{"; + json << "\"success\":false,"; + json << "\"error\":\"" << escape_json_string(error_message) << "\","; + json << "\"cloud_handoff\":false,"; + json << "\"response\":null,"; + json << "\"function_calls\":[],"; + json << "\"confidence\":0.0,"; + json << "\"time_to_first_token_ms\":0.0,"; + json << "\"total_time_ms\":0.0,"; + json << "\"prefill_tps\":0.0,"; + json << "\"decode_tps\":0.0,"; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; + json << "\"prefill_tokens\":0,"; + json << "\"decode_tokens\":0,"; + json << "\"total_tokens\":0"; + json << "}"; + std::string error_json = json.str(); + if (response_buffer && error_json.length() < buffer_size) { + std::strcpy(response_buffer, error_json.c_str()); + } +} + +inline std::vector parse_messages_json(const std::string& json, + std::vector& out_image_paths, + std::vector* out_audio_paths = nullptr) { + std::vector messages; + out_image_paths.clear(); + if (out_audio_paths) out_audio_paths->clear(); + + size_t pos = json.find('['); + if (pos == std::string::npos) { + throw std::runtime_error("Invalid JSON: expected array"); + } + + pos = json.find('{', pos); + while (pos != std::string::npos) { + cactus::engine::ChatMessage msg; + + size_t obj_start = pos; + int brace_count = 1; + size_t obj_end = obj_start + 1; + while (obj_end < json.length() && brace_count > 0) { + if (json[obj_end] == '{') brace_count++; + else if (json[obj_end] == '}') brace_count--; + obj_end++; + } + + size_t role_pos = json.find("\"role\"", pos); + if (role_pos == std::string::npos || role_pos >= obj_end) break; + + size_t role_start = json.find('"', role_pos + 6) + 1; + size_t role_end = json.find('"', role_start); + msg.role = json.substr(role_start, role_end - role_start); + + size_t content_pos = json.find("\"content\"", role_end); + if (content_pos != std::string::npos && content_pos < obj_end) { + size_t content_start = json.find('"', content_pos + 9) + 1; + size_t content_end = content_start; + + while (content_end < json.length()) { + content_end = json.find('"', content_end); + if (content_end == std::string::npos) break; + if (json[content_end - 1] != '\\') break; + content_end++; + } + + msg.content = json.substr(content_start, content_end - content_start); + + size_t escape_pos = 0; + while ((escape_pos = msg.content.find("\\n", escape_pos)) != std::string::npos) { + msg.content.replace(escape_pos, 2, "\n"); + escape_pos += 1; + } + escape_pos = 0; + while ((escape_pos = msg.content.find("\\\"", escape_pos)) != std::string::npos) { + msg.content.replace(escape_pos, 2, "\""); + escape_pos += 1; + } + } + + auto parse_path_array = [&](const char* key, std::vector& dest, + std::vector* out_paths) { + size_t key_pos = json.find(key, pos); + if (key_pos == std::string::npos || key_pos >= obj_end) return; + size_t array_start = json.find('[', key_pos); + if (array_start == std::string::npos || array_start >= obj_end) return; + size_t array_end = json.find(']', array_start); + if (array_end == std::string::npos || array_end >= obj_end) return; + size_t cur = array_start; + while (true) { + cur = json.find('"', cur + 1); + if (cur == std::string::npos || cur >= array_end) break; + size_t str_start = cur + 1; + size_t str_end = json.find('"', str_start); + if (str_end == std::string::npos || str_end > array_end) break; + std::string path = std::filesystem::absolute( + std::filesystem::path(json.substr(str_start, str_end - str_start))).string(); + dest.push_back(path); + if (out_paths) out_paths->push_back(path); + cur = str_end; + } + }; + + parse_path_array("\"images\"", msg.images, &out_image_paths); + parse_path_array("\"audio\"", msg.audio, out_audio_paths); + + if (msg.role == "tool") { + size_t name_pos = json.find("\"name\"", obj_start); + if (name_pos != std::string::npos && name_pos < obj_end) { + size_t name_quote = json.find('"', name_pos + 6); + if (name_quote != std::string::npos && name_quote < obj_end) { + size_t name_start = name_quote + 1; + size_t name_end = json.find('"', name_start); + if (name_end != std::string::npos && name_end < obj_end) { + msg.name = json.substr(name_start, name_end - name_start); + } + } + } + } + + size_t tool_calls_pos = json.find("\"tool_calls\"", obj_start); + if (tool_calls_pos != std::string::npos && tool_calls_pos < obj_end) { + size_t tool_calls_arr_start = json.find('[', tool_calls_pos); + if (tool_calls_arr_start != std::string::npos && tool_calls_arr_start < obj_end) { + size_t tool_calls_arr_end = find_matching_delimiter(json, tool_calls_arr_start, '[', ']'); + + size_t search_pos = tool_calls_arr_start; + while (true) { + size_t func_pos = json.find("\"function\"", search_pos); + if (func_pos == std::string::npos || func_pos >= tool_calls_arr_end) break; + + size_t func_obj_start = json.find('{', func_pos + 10); + if (func_obj_start == std::string::npos || func_obj_start >= tool_calls_arr_end) break; + + size_t func_obj_end = find_matching_delimiter(json, func_obj_start, '{', '}'); + + cactus::engine::ToolCallInfo tool_call; + + size_t fn_name_pos = json.find("\"name\"", func_obj_start); + if (fn_name_pos != std::string::npos && fn_name_pos < func_obj_end) { + size_t fn_name_quote = json.find('"', fn_name_pos + 6); + if (fn_name_quote != std::string::npos && fn_name_quote < func_obj_end) { + size_t fn_name_start = fn_name_quote + 1; + size_t fn_name_end = json.find('"', fn_name_start); + if (fn_name_end != std::string::npos && fn_name_end < func_obj_end) { + tool_call.name = json.substr(fn_name_start, fn_name_end - fn_name_start); + } + } + } + + size_t args_pos = json.find("\"arguments\"", func_obj_start); + if (args_pos != std::string::npos && args_pos < func_obj_end) { + size_t colon_pos = json.find(':', args_pos + 11); + if (colon_pos != std::string::npos && colon_pos < func_obj_end) { + size_t args_start = colon_pos + 1; + while (args_start < json.length() && std::isspace(static_cast(json[args_start]))) args_start++; + + if (args_start < func_obj_end && json[args_start] == '{') { + size_t args_end = find_matching_delimiter(json, args_start, '{', '}'); + tool_call.arguments = json.substr(args_start, args_end - args_start); + } else if (args_start < func_obj_end && json[args_start] == '"') { + size_t str_start = args_start + 1; + size_t str_end = str_start; + while (str_end < json.length() && json[str_end] != '"') { + if (json[str_end] == '\\') str_end++; + str_end++; + } + tool_call.arguments = json.substr(str_start, str_end - str_start); + } + } + } + + if (!tool_call.name.empty()) { + msg.tool_calls.push_back(tool_call); + } + search_pos = func_obj_end; + } + } + } + + messages.push_back(msg); + + pos = json.find('{', obj_end); + } + + return messages; +} + +inline std::vector parse_tools_json(const std::string& json) { + std::vector tools; + + if (json.empty()) return tools; + + size_t pos = json.find('['); + if (pos == std::string::npos) return tools; + + pos = json.find("\"function\"", pos); + while (pos != std::string::npos) { + ToolFunction tool; + size_t next_search = pos + 1; + + size_t name_pos = json.find("\"name\"", pos); + if (name_pos != std::string::npos) { + size_t name_start = json.find('"', name_pos + 6) + 1; + size_t name_end = json.find('"', name_start); + tool.name = json.substr(name_start, name_end - name_start); + } + + size_t desc_pos = json.find("\"description\"", pos); + if (desc_pos != std::string::npos) { + size_t desc_start = json.find('"', desc_pos + 13) + 1; + size_t desc_end = json.find('"', desc_start); + tool.description = json.substr(desc_start, desc_end - desc_start); + } + + size_t params_pos = json.find("\"parameters\"", pos); + if (params_pos != std::string::npos) { + size_t params_start = json.find('{', params_pos); + if (params_start != std::string::npos) { + int brace_count = 1; + size_t params_end = params_start + 1; + while (params_end < json.length() && brace_count > 0) { + if (json[params_end] == '{') brace_count++; + else if (json[params_end] == '}') brace_count--; + params_end++; + } + tool.parameters["schema"] = json.substr(params_start, params_end - params_start); + next_search = params_end; + } + } + + if (!tool.name.empty()) { + tools.push_back(tool); + } + + pos = json.find("\"function\"", next_search); + } + + return tools; +} + +inline bool try_parse_json_float(const std::string& json, const std::string& key, float& out_value) { + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return false; + + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + + size_t end = start; + while (end < json.size() && std::string(",}] \t\n\r").find(json[end]) == std::string::npos) ++end; + + try { + out_value = std::stof(json.substr(start, end - start)); + return true; + } catch (...) { + return false; + } +} + +inline std::vector parse_json_string_array_field(const std::string& json, const std::string& key) { + std::vector out; + std::string pattern = "\"" + key + "\":"; + size_t pos = json.find(pattern); + if (pos == std::string::npos) return out; + + size_t start = pos + pattern.size(); + while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; + if (start >= json.size() || json[start] != '[') return out; + + int depth = 1; + bool in_string = false; + bool escaped = false; + size_t end = start + 1; + + while (end < json.size() && depth > 0) { + char c = json[end]; + if (in_string) { + if (escaped) escaped = false; + else if (c == '\\') escaped = true; + else if (c == '"') in_string = false; + } else { + if (c == '"') in_string = true; + else if (c == '[') depth++; + else if (c == ']') depth--; + } + ++end; + } + + if (depth != 0) return out; + const std::string array_json = json.substr(start, end - start); + if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; + + size_t i = 1; + while (i + 1 < array_json.size()) { + while (i + 1 < array_json.size() && + (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) { + ++i; + } + if (i + 1 >= array_json.size() || array_json[i] == ']') break; + if (array_json[i] != '"') break; + + ++i; + std::string value; + bool escaped = false; + while (i < array_json.size()) { + char c = array_json[i++]; + if (escaped) { + switch (c) { + case '"': value.push_back('"'); break; + case '\\': value.push_back('\\'); break; + case '/': value.push_back('/'); break; + case 'b': value.push_back('\b'); break; + case 'f': value.push_back('\f'); break; + case 'n': value.push_back('\n'); break; + case 'r': value.push_back('\r'); break; + case 't': value.push_back('\t'); break; + default: value.push_back(c); break; + } + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == '"') { + out.push_back(value); + break; + } + value.push_back(c); + } + } + + return out; +} + +inline void parse_custom_vocabulary_options(const std::string& json, + std::vector& custom_vocabulary, + float& vocabulary_boost) { + custom_vocabulary.clear(); + vocabulary_boost = 5.0f; + if (json.empty()) return; + + float parsed_boost = vocabulary_boost; + if (try_parse_json_float(json, "vocabulary_boost", parsed_boost)) { + vocabulary_boost = std::clamp(parsed_boost, 0.0f, 20.0f); + } + + custom_vocabulary = parse_json_string_array_field(json, "custom_vocabulary"); +} + +inline std::unordered_map build_token_bias_map(const std::vector>& tokenized_entries, + float vocabulary_boost) { + std::unordered_map vocab_bias; + const float clamped_boost = std::clamp(vocabulary_boost, 0.0f, 20.0f); + if (clamped_boost == 0.0f) return vocab_bias; + + for (const auto& token_ids : tokenized_entries) { + for (uint32_t token_id : token_ids) { + float& entry = vocab_bias[token_id]; + if (entry < clamped_boost) { + entry = clamped_boost; + } + } + } + + return vocab_bias; +} + +inline std::unordered_map build_custom_vocabulary_bias(cactus::engine::Tokenizer* tokenizer, + const std::vector& custom_vocabulary, + float vocabulary_boost) { + if (!tokenizer || custom_vocabulary.empty()) return {}; + std::vector> tokenized_entries; + tokenized_entries.reserve(custom_vocabulary.size()); + + for (const auto& word : custom_vocabulary) { + if (word.empty()) continue; + tokenized_entries.push_back(tokenizer->encode(word)); + } + + return build_token_bias_map(tokenized_entries, vocabulary_boost); +} + +inline void apply_custom_vocabulary_options(cactus::engine::Model* model, const std::string& json) { + if (!model) return; + + std::vector custom_vocabulary; + float vocabulary_boost = 5.0f; + parse_custom_vocabulary_options(json, custom_vocabulary, vocabulary_boost); + model->set_vocab_bias(build_custom_vocabulary_bias(model->get_tokenizer(), custom_vocabulary, vocabulary_boost)); +} + +inline size_t levenshtein_ci(const std::string& a, const std::string& b) { + const size_t m = a.size(), n = b.size(); + std::vector prev(n + 1), curr(n + 1); + for (size_t j = 0; j <= n; ++j) prev[j] = j; + for (size_t i = 1; i <= m; ++i) { + curr[0] = i; + for (size_t j = 1; j <= n; ++j) { + const bool match = std::tolower(static_cast(a[i - 1])) == + std::tolower(static_cast(b[j - 1])); + curr[j] = std::min({prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + (match ? 0 : 1)}); + } + std::swap(prev, curr); + } + return prev[n]; +} + +inline std::string collapse_spaces(const std::string& s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + if (c != ' ') out += c; + } + return out; +} + +inline void apply_vocabulary_spelling_correction( + std::string& text, + const std::vector& custom_vocabulary) +{ + if (custom_vocabulary.empty() || text.empty()) return; + + struct VocabEntry { + const std::string* original; + std::string collapsed; + }; + std::vector vocab_entries; + vocab_entries.reserve(custom_vocabulary.size()); + for (const auto& v : custom_vocabulary) { + vocab_entries.push_back({&v, collapse_spaces(v)}); + } + + struct Token { std::string text; bool is_word; }; + std::vector tokens; + size_t pos = 0; + while (pos < text.size()) { + if (std::isalnum(static_cast(text[pos])) || + text[pos] == '\'' || text[pos] == '-') { + size_t start = pos; + while (pos < text.size() && (std::isalnum(static_cast(text[pos])) || + text[pos] == '\'' || text[pos] == '-')) { + ++pos; + } + tokens.push_back({text.substr(start, pos - start), true}); + } else { + size_t start = pos; + while (pos < text.size() && !std::isalnum(static_cast(text[pos])) && + text[pos] != '\'' && text[pos] != '-') { + ++pos; + } + tokens.push_back({text.substr(start, pos - start), false}); + } + } + + std::vector word_indices; + for (size_t i = 0; i < tokens.size(); ++i) { + if (tokens[i].is_word) word_indices.push_back(i); + } + + std::vector consumed(tokens.size(), false); + + auto strip_suffix = [](const std::string& word) -> std::pair { + if (word.size() >= 3 && word.substr(word.size() - 2) == "'s") { + return {word.substr(0, word.size() - 2), "'s"}; + } + if (word.size() >= 3 && word.substr(word.size() - 2) == "'t") { + return {word.substr(0, word.size() - 2), "'t"}; + } + if (word.size() >= 4 && word.back() == 's' && + word[word.size() - 2] != 's' && // avoid stripping from "boss", "class" + std::isalpha(static_cast(word[word.size() - 2]))) { + return {word.substr(0, word.size() - 1), "s"}; + } + return {word, ""}; + }; + + size_t wi = 0; + while (wi < word_indices.size()) { + size_t best_dist = std::numeric_limits::max(); + const std::string* best_match = nullptr; + size_t best_window = 0; + size_t best_first_token = 0; + size_t best_last_token = 0; + std::string best_suffix; + + for (size_t window = std::min(3, word_indices.size() - wi); window >= 1; --window) { + std::string window_collapsed; + const size_t first_tok = word_indices[wi]; + const size_t last_tok = word_indices[wi + window - 1]; + for (size_t w = 0; w < window; ++w) { + window_collapsed += tokens[word_indices[wi + w]].text; + } + + if (window == 1 && window_collapsed.size() < 3) break; + + auto [stem, suffix] = strip_suffix(window_collapsed); + const std::string* candidates[] = {&window_collapsed, &stem}; + const std::string suffixes[] = {"", suffix}; + const size_t num_candidates = suffix.empty() ? 1 : 2; + + for (size_t ci = 0; ci < num_candidates; ++ci) { + const std::string& candidate = *candidates[ci]; + if (candidate.empty()) continue; + + for (const auto& entry : vocab_entries) { + const size_t wlen = candidate.size(); + const size_t vlen = entry.collapsed.size(); + + const size_t len_diff = wlen > vlen ? wlen - vlen : vlen - wlen; + const size_t max_dist = std::max(1, std::min(wlen, vlen) / 3); + if (len_diff > max_dist) continue; + + const size_t dist = levenshtein_ci(candidate, entry.collapsed); + + // For single-edit corrections, require first char match to prevent + // false positives like "vortex" → "Cortex". + if (dist == 1 && window == 1) { + const bool first_char_match = + std::tolower(static_cast(candidate[0])) == + std::tolower(static_cast(entry.collapsed[0])); + if (!first_char_match) continue; + } + + if (dist <= max_dist && dist < best_dist) { + best_dist = dist; + best_match = entry.original; + best_window = window; + best_first_token = first_tok; + best_last_token = last_tok; + best_suffix = suffixes[ci]; + } + } + } + + if (best_dist == 0) break; + } + + // Allow dist==0 for multi-word merges where word boundaries changed. + const bool should_replace = best_match && + best_dist != std::numeric_limits::max() && + (best_dist > 0 || best_window > 1); + + if (should_replace) { + tokens[best_first_token].text = *best_match + best_suffix; + for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { + consumed[t] = true; + } + for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { + if (t > 0) consumed[t - 1] = consumed[t - 1] || !tokens[t - 1].is_word; + } + wi += best_window; + } else { + ++wi; + } + } + + std::string result; + result.reserve(text.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + if (!consumed[i]) { + result += tokens[i].text; + } + } + + text = std::move(result); +} + +inline InferenceOptions parse_inference_options_json(const std::string& json) { + InferenceOptions options; + + if (json.empty()) return options; + + size_t pos = json.find("\"temperature\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.temperature = std::stof(json.substr(pos)); + } + + pos = json.find("\"top_p\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.top_p = std::stof(json.substr(pos)); + } + + float parsed_min_p = options.min_p; + if (try_parse_json_float(json, "min_p", parsed_min_p)) { + options.min_p = std::clamp(parsed_min_p, 0.0f, 1.0f); + } + + float parsed_rep_penalty = options.repetition_penalty; + if (try_parse_json_float(json, "repetition_penalty", parsed_rep_penalty)) { + if (std::isfinite(parsed_rep_penalty) && parsed_rep_penalty > 0.0f) { + options.repetition_penalty = parsed_rep_penalty; + } + } + + pos = json.find("\"top_k\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.top_k = std::stoul(json.substr(pos)); + } + + pos = json.find("\"max_tokens\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.max_tokens = std::stoul(json.substr(pos)); + } + + pos = json.find("\"force_tools\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.force_tools = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"tool_rag_top_k\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.tool_rag_top_k = std::stoul(json.substr(pos)); + } + + pos = json.find("\"confidence_threshold\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.confidence_threshold = std::stof(json.substr(pos)); + } + + pos = json.find("\"include_stop_sequences\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.include_stop_sequences = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"use_vad\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.use_vad = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"telemetry_enabled\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.telemetry_enabled = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"auto_handoff\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.auto_handoff = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"cloud_timeout_ms\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + options.cloud_timeout_ms = std::stoul(json.substr(pos)); + } + + pos = json.find("\"handoff_with_images\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.handoff_with_images = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"enable_thinking_if_supported\""); + if (pos != std::string::npos) { + pos = json.find(':', pos) + 1; + while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; + options.enable_thinking_if_supported = (json.substr(pos, 4) == "true"); + } + + pos = json.find("\"stop_sequences\""); + if (pos != std::string::npos) { + pos = json.find('[', pos); + if (pos != std::string::npos) { + size_t end_pos = json.find(']', pos); + size_t seq_pos = json.find('"', pos); + + while (seq_pos != std::string::npos && seq_pos < end_pos) { + size_t seq_start = seq_pos + 1; + size_t seq_end = json.find('"', seq_start); + if (seq_end != std::string::npos) { + options.stop_sequences.push_back(json.substr(seq_start, seq_end - seq_start)); + } + seq_pos = json.find('"', seq_end + 1); + } + } + } + + return options; +} + +static inline std::string trim_lfm2_slice(const std::string& value, size_t begin, size_t end) { + return trim_string(value.substr(begin, end - begin)); +} + +static inline void append_lfm2_call(const std::string& entry, + std::vector& function_calls) { + if (entry.empty()) return; + + std::string trimmed_entry = trim_lfm2_slice(entry, 0, entry.size()); + if (trimmed_entry.empty()) return; + + size_t paren_pos = trimmed_entry.find('('); + if (paren_pos == std::string::npos) return; + + std::string func_name = trim_lfm2_slice(trimmed_entry, 0, paren_pos); + std::string args_str = trim_lfm2_slice(trimmed_entry, paren_pos + 1, trimmed_entry.size()); + + while (!args_str.empty() && (args_str.back() == ')' || args_str.back() == ']')) { + args_str.pop_back(); + } + args_str = trim_lfm2_slice(args_str, 0, args_str.size()); + + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":{"; + + size_t arg_pos = 0; + bool first_arg = true; + while (arg_pos < args_str.length()) { + while (arg_pos < args_str.length() && std::isspace(static_cast(args_str[arg_pos]))) { + arg_pos++; + } + + size_t eq_pos = args_str.find('=', arg_pos); + if (eq_pos == std::string::npos) break; + + std::string arg_name = args_str.substr(arg_pos, eq_pos - arg_pos); + + size_t val_start = eq_pos + 1; + size_t val_end = val_start; + bool quoted = false; + + if (val_start < args_str.length() && args_str[val_start] == '"') { + quoted = true; + val_start++; + val_end = args_str.find('"', val_start); + if (val_end == std::string::npos) break; + } else { + int depth = 0; + for (val_end = val_start; val_end < args_str.length(); val_end++) { + char c = args_str[val_end]; + if (c == '[' || c == '{') depth++; + else if (c == ']' || c == '}') depth--; + else if (c == ',' && depth == 0) break; + } + } + + std::string arg_value = args_str.substr(val_start, val_end - val_start); + + if (!quoted) { + if (arg_value == "True") arg_value = "true"; + else if (arg_value == "False") arg_value = "false"; + else if (arg_value == "None") arg_value = "null"; + } + + if (!first_arg) json_call += ","; + json_call += "\"" + arg_name + "\":"; + json_call += quoted ? ("\"" + arg_value + "\"") : arg_value; + first_arg = false; + + arg_pos = args_str.find(',', val_end); + if (arg_pos != std::string::npos) { + arg_pos++; + } else { + break; + } + } + + json_call += "}}"; + function_calls.push_back(json_call); +} + +inline void parse_function_calls_from_response(const std::string& response_text, + std::string& regular_response, + std::vector& function_calls) { + regular_response = response_text; + function_calls.clear(); + + gemma::parse_function_calls(regular_response, function_calls); + + const std::string QWEN_TOOL_START = ""; + const std::string QWEN_TOOL_END = ""; + size_t qwen_start_pos = 0; + + while ((qwen_start_pos = regular_response.find(QWEN_TOOL_START, qwen_start_pos)) != std::string::npos) { + size_t content_start = qwen_start_pos + QWEN_TOOL_START.length(); + size_t qwen_end_pos = regular_response.find(QWEN_TOOL_END, content_start); + + size_t erase_end; + std::string json_content; + + if (qwen_end_pos != std::string::npos) { + json_content = regular_response.substr(content_start, qwen_end_pos - content_start); + erase_end = qwen_end_pos + QWEN_TOOL_END.length(); + } else { + json_content = regular_response.substr(content_start); + erase_end = regular_response.length(); + } + + size_t first = json_content.find_first_not_of(" \t\n\r"); + size_t last = json_content.find_last_not_of(" \t\n\r"); + if (first != std::string::npos && last != std::string::npos) { + json_content = json_content.substr(first, last - first + 1); + } + + if (json_content.size() > 2 && json_content.find("\"name\"") != std::string::npos) { + // Unwrap array wrapper if present: [{"name":...}] -> {"name":...} + if (json_content[0] == '[') { + size_t obj_start = json_content.find('{'); + size_t obj_end = json_content.rfind('}'); + if (obj_start != std::string::npos && obj_end != std::string::npos && obj_end > obj_start) { + json_content = json_content.substr(obj_start, obj_end - obj_start + 1); + } + } + if (json_content[0] == '{') { + size_t depth = 0; + bool in_string = false; + bool escaped = false; + size_t end_pos = 0; + for (size_t c = 0; c < json_content.size(); c++) { + char ch = json_content[c]; + if (escaped) { escaped = false; continue; } + if (ch == '\\' && in_string) { escaped = true; continue; } + if (ch == '"') { in_string = !in_string; continue; } + if (!in_string) { + if (ch == '{') depth++; + else if (ch == '}' && --depth == 0) { end_pos = c + 1; break; } + } + } + if (end_pos > 0) { + function_calls.push_back(json_content.substr(0, end_pos)); + } + } + } + + regular_response.erase(qwen_start_pos, erase_end - qwen_start_pos); + } + + const std::string TOOL_CALL_START = "<|tool_call_start|>"; + const std::string TOOL_CALL_END = "<|tool_call_end|>"; + size_t lfm2_start_pos = 0; + + while ((lfm2_start_pos = regular_response.find(TOOL_CALL_START, lfm2_start_pos)) != std::string::npos) { + size_t content_start = lfm2_start_pos + TOOL_CALL_START.length(); + size_t tool_end_pos = regular_response.find(TOOL_CALL_END, content_start); + + if (tool_end_pos != std::string::npos) { + std::string tool_content = regular_response.substr(content_start, tool_end_pos - content_start); + std::string content = tool_content; + size_t trim_start = 0; + while (trim_start < content.size() && std::isspace(static_cast(content[trim_start]))) { + trim_start++; + } + + if (trim_start < content.size()) { + size_t trim_end = content.size() - 1; + while (trim_end > trim_start && std::isspace(static_cast(content[trim_end]))) { + trim_end--; + } + content = content.substr(trim_start, trim_end - trim_start + 1); + } else { + content.clear(); + } + + if (!content.empty() && content.front() == '[' && content.back() == ']') { + std::string inner = content.substr(1, content.size() - 2); + + size_t inner_first = inner.find_first_not_of(" \t\n\r"); + if (inner_first != std::string::npos && inner[inner_first] == '{') { + size_t pos = inner_first; + while (pos < inner.size()) { + if (inner[pos] == '{') { + int brace_depth = 1; + size_t obj_start = pos; + pos++; + while (pos < inner.size() && brace_depth > 0) { + if (inner[pos] == '{') brace_depth++; + else if (inner[pos] == '}') brace_depth--; + pos++; + } + if (brace_depth == 0) { + std::string json_obj = inner.substr(obj_start, pos - obj_start); + if (json_obj.find("\"name\"") != std::string::npos) { + function_calls.push_back(json_obj); + } + } + } else { + pos++; + } + } + } else { + size_t start = 0; + int paren_depth = 0; + + for (size_t i = 0; i < inner.size(); ++i) { + char c = inner[i]; + if (c == '(') { + paren_depth++; + } else if (c == ')' && paren_depth > 0) { + paren_depth--; + } else if (c == ',' && paren_depth == 0) { + append_lfm2_call(inner.substr(start, i - start), function_calls); + start = i + 1; + } + } + + if (start < inner.size()) { + append_lfm2_call(inner.substr(start), function_calls); + } + } + } else if (!content.empty()) { + append_lfm2_call(content, function_calls); + } + + regular_response.erase(lfm2_start_pos, tool_end_pos + TOOL_CALL_END.length() - lfm2_start_pos); + } else { + break; + } + } + + const char* FUNCTION_CALL_MARKER = "\"function_call\""; + size_t search_pos = 0; + const size_t text_len = regular_response.length(); + + while (search_pos < text_len) { + size_t marker_pos = regular_response.find(FUNCTION_CALL_MARKER, search_pos); + if (marker_pos == std::string::npos) break; + + size_t json_start = regular_response.find('{', marker_pos); + if (json_start == std::string::npos) break; + + int brace_count = 1; + size_t json_end = json_start + 1; + while (json_end < text_len && brace_count > 0) { + char c = regular_response[json_end]; + brace_count += (c == '{') - (c == '}'); + json_end++; + } + + if (brace_count == 0) { + function_calls.push_back(regular_response.substr(json_start, json_end - json_start)); + regular_response = regular_response.substr(0, marker_pos); + size_t last_bracket = regular_response.rfind('{'); + if(last_bracket != std::string::npos) { + regular_response = regular_response.substr(0, last_bracket); + } + } + search_pos = json_end; + } +} + +inline std::vector> find_channel_token_ranges( + const std::vector& tokens, size_t offset, + uint32_t channel_open_id, uint32_t channel_close_id) { + std::vector> ranges; + size_t pos = 0; + while (pos < tokens.size()) { + if (tokens[pos] != channel_open_id) { + pos++; + continue; + } + + size_t block_start = pos; + pos++; + while (pos < tokens.size() && tokens[pos] != channel_close_id) { + pos++; + } + if (pos < tokens.size()) { + pos++; + } + ranges.push_back({offset + block_start, pos - block_start}); + } + return ranges; +} + +inline void strip_tag_blocks(std::string& text, std::string& extracted, + const std::string& open_tag, const std::string& close_tag) { + std::string result; + size_t pos = 0; + + size_t first_close = text.find(close_tag); + size_t first_open = text.find(open_tag); + if (first_close != std::string::npos && + (first_open == std::string::npos || first_close < first_open)) { + extracted += text.substr(0, first_close); + pos = first_close + close_tag.size(); + } + + while (pos < text.size()) { + size_t open_pos = text.find(open_tag, pos); + if (open_pos == std::string::npos) { + result += text.substr(pos); + break; + } + result += text.substr(pos, open_pos - pos); + size_t content_start = open_pos + open_tag.size(); + size_t close_pos = text.find(close_tag, content_start); + if (close_pos == std::string::npos) { + if (!extracted.empty()) extracted += "\n"; + extracted += text.substr(content_start); + break; + } + if (!extracted.empty()) extracted += "\n"; + extracted += text.substr(content_start, close_pos - content_start); + pos = close_pos + close_tag.size(); + } + text = result; +} + +inline void strip_thinking_block(const std::string& input, std::string& thinking, std::string& content) { + thinking.clear(); + content = input; + + auto trim = [](std::string& s) { + size_t first = s.find_first_not_of(" \t\n\r"); + size_t last = s.find_last_not_of(" \t\n\r"); + if (first != std::string::npos && last != std::string::npos) + s = s.substr(first, last - first + 1); + else + s.clear(); + }; + + if (content.find("<|channel>") != std::string::npos || content.find("") != std::string::npos) { + strip_tag_blocks(content, thinking, "<|channel>", ""); + } else if (content.find("") != std::string::npos || content.find("") != std::string::npos) { + strip_tag_blocks(content, thinking, "", ""); + } else { + return; + } + + trim(thinking); + trim(content); +} + +struct TranscriptSegment { + float start; + float end; + std::string text; +}; + +inline std::string construct_response_json(const std::string& regular_response, + const std::vector& function_calls, + double time_to_first_token, + double total_time_ms, + double prefill_tps, + double decode_tps, + size_t prompt_tokens, + size_t completion_tokens, + float confidence = 0.0f, + bool cloud_handoff = false, + const std::string& thinking = "", + const std::vector& segments = {}) { + std::ostringstream json; + json << "{"; + json << "\"success\":true,"; + json << "\"error\":null,"; + json << "\"cloud_handoff\":" << (cloud_handoff ? "true" : "false") << ","; + json << "\"response\":\"" << escape_json_string(regular_response) << "\","; + if (!thinking.empty()) { + json << "\"thinking\":\"" << escape_json_string(thinking) << "\","; + } + json << "\"function_calls\":["; + for (size_t i = 0; i < function_calls.size(); ++i) { + if (i > 0) json << ","; + json << function_calls[i]; + } + json << "],"; + json << "\"segments\":["; + for (size_t i = 0; i < segments.size(); ++i) { + if (i > 0) json << ","; + json << "{\"start\":" << std::fixed << std::setprecision(3) << segments[i].start + << ",\"end\":" << std::fixed << std::setprecision(3) << segments[i].end + << ",\"text\":\"" << escape_json_string(segments[i].text) << "\"}"; + } + json << "],"; + json << "\"confidence\":" << std::fixed << std::setprecision(4) << confidence << ","; + json << "\"time_to_first_token_ms\":" << std::fixed << std::setprecision(2) << time_to_first_token << ","; + json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; + json << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << ","; + json << "\"decode_tps\":" << std::fixed << std::setprecision(2) << decode_tps << ","; + json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; + json << "\"prefill_tokens\":" << prompt_tokens << ","; + json << "\"decode_tokens\":" << completion_tokens << ","; + json << "\"total_tokens\":" << (prompt_tokens + completion_tokens); + json << "}"; + return json.str(); +} + +inline std::string serialize_function_calls(const std::vector& calls) { + if (calls.empty()) return "[]"; + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < calls.size(); ++i) { + if (i > 0) oss << ","; + oss << calls[i]; + } + oss << "]"; + return oss.str(); +} + +inline int validate_audio_params( + const char* component, + void* model, + char* response_buffer, size_t buffer_size, + const char* audio_file_path, + const uint8_t* pcm_buffer, size_t pcm_buffer_size) { + if (!model) { + std::string err = last_error_message.empty() ? "Model not initialized." : last_error_message; + CACTUS_LOG_ERROR(component, err); + handle_error_response(err, response_buffer, buffer_size); + return -1; + } + if (!response_buffer || buffer_size == 0) { + CACTUS_LOG_ERROR(component, "Invalid parameters: response_buffer or buffer_size"); + handle_error_response("Invalid parameters", response_buffer, buffer_size); + return -1; + } + if (!audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { + CACTUS_LOG_ERROR(component, "No audio input provided"); + handle_error_response("Either audio_file_path or pcm_buffer must be provided", response_buffer, buffer_size); + return -1; + } + if (audio_file_path && pcm_buffer && pcm_buffer_size > 0) { + CACTUS_LOG_ERROR(component, "Both audio_file_path and pcm_buffer provided"); + handle_error_response("Cannot provide both audio_file_path and pcm_buffer", response_buffer, buffer_size); + return -1; + } + if (pcm_buffer && pcm_buffer_size > 0 && (pcm_buffer_size < 2 || pcm_buffer_size % 2 != 0)) { + CACTUS_LOG_ERROR(component, "Invalid pcm_buffer_size"); + handle_error_response("pcm_buffer_size must be even and at least 2 bytes", response_buffer, buffer_size); + return -1; + } + return 0; +} + +inline std::vector pcm_to_float(const uint8_t* pcm_buffer, size_t pcm_buffer_size) { + const int16_t* samples = reinterpret_cast(pcm_buffer); + size_t n = pcm_buffer_size / 2; + std::vector out(n); + for (size_t i = 0; i < n; ++i) + out[i] = static_cast(samples[i]) / 32768.0f; + return out; +} + +} // namespace ffi +} // namespace cactus + +#ifdef __cplusplus +extern "C" { +#endif + +const char* cactus_get_last_error(); + +#ifdef __cplusplus +} +#endif + +#endif // CACTUS_UTILS_H diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/engine.h b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/engine.h new file mode 100644 index 0000000..afebfeb --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/engine.h @@ -0,0 +1,1104 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../graph/graph.h" + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc99-extensions" +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +extern "C" { + #include "../../libs/stb/stb_image.h" + #include "../../libs/stb/stb_image_resize2.h" +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + +class CactusGraph; + +namespace cactus { +namespace npu { + class NPUPrefill; +} +namespace engine { + +class Siglip2Preprocessor; + +struct Config { + uint32_t vocab_size = 151936; + uint32_t bos_token_id = 151643; + uint32_t eos_token_id = 151645; + uint32_t num_layers = 28; + uint32_t hidden_dim = 1024; + uint32_t ffn_intermediate_dim = 3072; + uint32_t attention_heads = 16; + uint32_t attention_kv_heads = 8; + uint32_t attention_head_dim = 128; + float layer_norm_eps = 1e-6f; + float rope_theta = 1000000.0f; + uint32_t num_experts = 0; + uint32_t num_shared_experts = 0; + uint32_t num_top_experts = 0; + uint32_t moe_every_n_layers = 0; + uint32_t moe_intermediate_dim = 0; + uint32_t num_dense_layers = 0; + uint32_t num_experts_per_tok = 0; + bool norm_topk_prob = false; + bool use_expert_bias = false; + float routed_scaling_factor = 1.0f; + bool tie_word_embeddings = true; + + uint32_t vision_hidden_dim = 0; + uint32_t vision_num_layers = 0; + uint32_t vision_attention_heads = 0; + uint32_t vision_image_size = 0; + uint32_t vision_patch_size = 0; + uint32_t vision_num_channels = 3; + uint32_t vision_embed_dim = 0; + uint32_t visual_tokens_per_img = 0; + bool use_pixel_shuffle = false; + uint32_t pixel_shuffle_factor = 1; + bool use_image_tokens = false; + uint32_t image_token_id = 0; + bool use_layout_tags = false; + uint32_t image_seq_len = 64; + + uint32_t global_image_size = 2048; + uint32_t max_tile_size = 512; + float rescale_factor = 0.00392156862745098f; + float image_mean = 0.5f; + float image_std = 0.5f; + + uint32_t downsample_factor = 2; + uint32_t min_tiles = 2; + uint32_t max_tiles = 10; + bool use_thumbnail = true; + uint32_t min_image_tokens = 64; + uint32_t max_image_tokens = 256; + uint32_t max_num_patches = 1024; + uint32_t tile_size = 512; + float max_pixels_tolerance = 2.0f; + bool do_image_splitting = true; + bool encoder_act_gelu = false; + bool decoder_act_gelu = false; + uint32_t num_encoder_layers = 0; + uint32_t num_decoder_layers = 0; + float partial_rotary_factor = 0.0f; + uint32_t pad_token_id = 0; + uint32_t conv_kernel_size = 0; + uint32_t subsampling_conv_kernel_size = 0; + uint32_t subsampling_conv_stride = 0; + uint32_t subsampling_conv_channels = 0; + uint32_t subsampling_factor = 0; + uint32_t num_mel_bins = 80; + std::string encoder_hidden_act = "silu"; + uint32_t linear_num_key_heads = 0; + uint32_t linear_key_head_dim = 0; + uint32_t linear_num_value_heads = 0; + uint32_t linear_value_head_dim = 0; + uint32_t linear_q_proj_dim = 0; + uint32_t linear_k_proj_dim = 0; + uint32_t linear_v_proj_dim = 0; + + uint32_t kv_lora_rank = 0; + uint32_t q_lora_rank = 0; + uint32_t qk_head_dim = 0; + uint32_t qk_nope_head_dim = 0; + uint32_t qk_rope_head_dim = 0; + uint32_t v_head_dim = 0; + uint32_t rope_interleave = 0; + bool attention_bias = false; + float rope_scaling_factor = 1.0f; + float rope_mscale_all_dim = 0.0f; + + enum class ModelType {QWEN = 0, GEMMA = 1, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7, MOONSHINE = 8, SILERO_VAD = 9, PARAKEET = 10, QWEN3P5 = 11, PARAKEET_TDT = 12, GEMMA3N = 13, YOUTU = 14, GEMMA4 = 15, PYANNOTE = 16, WESPEAKER = 17, NEEDLE = 18}; + uint32_t predictor_hidden_dim = 0; + uint32_t predictor_num_layers = 0; + uint32_t tdt_joint_dim = 0; + uint32_t tdt_num_durations = 0; + uint32_t tdt_blank_id = 0; + std::vector tdt_durations; + + ModelType model_type = ModelType::QWEN; + + enum class ModelVariant {DEFAULT = 0, VLM = 1, EXTRACT = 2, RAG = 3}; + ModelVariant model_variant = ModelVariant::DEFAULT; + + enum class Activation {GELU = 0, SILU = 1}; + Activation activation = Activation::SILU; + + enum class Backend {CPU = 0, NPU = 1}; + Backend default_backend = Backend::CPU; + + enum class Precision {INT8 = 0, FP16 = 1, FP32 = 2}; + Precision precision = Precision::FP32; + + float default_temperature = 0.6f; + float default_top_p = 0.95f; + size_t default_top_k = 20; + float default_max_tps = -1.0f; + float default_cloud_handoff_threshold = 0.0f; + size_t default_rolling_entropy_window = 10; + + std::vector layer_types; + size_t conv_L_cache = 0; + + uint32_t altup_num_inputs = 4; + uint32_t laurel_rank = 64; + uint32_t hidden_size_per_layer_input = 256; + uint32_t num_kv_shared_layers = 0; + uint32_t sliding_window = 512; + float rope_local_base_freq = 10000.0f; + float final_logit_softcapping = 0.0f; + float global_partial_rotary_factor = 1.0f; + uint32_t expert_intermediate_size = 0; + uint32_t global_head_dim = 0; + uint32_t num_global_kv_heads = 0; + bool attention_k_eq_v = false; + bool enable_moe_block = false; + std::vector activation_sparsity_ppf; + + uint32_t vision_head_dim = 64; + uint32_t vision_kv_heads = 12; + uint32_t vision_intermediate_size = 3072; + uint32_t vision_position_embedding_size = 10240; + uint32_t vision_pooling_kernel_size = 3; + uint32_t vision_default_output_length = 280; + float vision_rope_theta = 100.0f; + + uint32_t audio_hidden_dim = 0; + uint32_t audio_num_layers = 0; + uint32_t audio_num_heads = 0; + uint32_t audio_head_dim = 0; + uint32_t audio_input_feat_size = 128; + uint32_t audio_conf_conv_kernel_size = 5; + uint32_t audio_chunk_size = 12; + uint32_t audio_context_left = 13; + uint32_t audio_context_right = 0; + float audio_logit_cap = 50.0f; + float audio_residual_weight = 0.5f; + uint32_t audio_output_proj_dims = 0; + uint32_t audio_vocab_size = 128; + uint32_t audio_vocab_offset = 0; + uint32_t audio_soft_tokens = 188; + uint32_t audio_sscp_conv0_channels = 128; + uint32_t audio_sscp_conv1_channels = 32; + float audio_sscp_conv_eps = 1e-3f; + float audio_rms_norm_eps = 1e-6f; + uint32_t audio_fft_length = 1024; + uint32_t audio_token_id = 0; + bool audio_fft_overdrive = false; + uint32_t channel_open_token_id = 100; + uint32_t channel_close_token_id = 101; + + static bool is_gemma_family(ModelType t) { + return t == ModelType::GEMMA || t == ModelType::GEMMA3N || t == ModelType::GEMMA4; + } + + bool from_json(const std::string& json_path); + std::string to_json() const; +}; + + + +struct MergeRule { + std::string first; + std::string second; + std::string merged; + uint32_t priority; + + MergeRule(const std::string& f, const std::string& s, const std::string& m, uint32_t p) + : first(f), second(s), merged(m), priority(p) {} +}; + + +struct ToolCallInfo { + std::string name; + std::string arguments; +}; + +struct ChatMessage { + std::string role; + std::string content; + std::string name; + std::vector images; + std::vector audio; + size_t audio_soft_token_count = 0; + std::vector tool_calls; +}; + +inline std::string format_needle_query_text(const std::vector& messages) { + std::string system_text; + std::string user_query; + + for (const auto& msg : messages) { + if (msg.role == "system") { + if (!system_text.empty()) { + system_text += "\n"; + } + system_text += msg.content; + } else if (msg.role == "user") { + user_query = msg.content; + } + } + + if (user_query.empty() && !messages.empty()) { + user_query = messages.back().content; + } + if (system_text.empty()) { + return user_query; + } + if (user_query.empty()) { + return system_text; + } + return system_text + "\n\n" + user_query; +} + +struct ToolConstraintSpec { + std::string name; + std::vector parameter_names; + std::vector required_parameter_names; +}; + +struct TokenizerRuntimeConfig { + enum class TokenizerType { UNKNOWN, BPE, SENTENCEPIECE }; + enum class VocabFormat { UNKNOWN, ID_TAB_TOKEN, LINE_TOKEN }; + enum class Normalizer { NONE, METASPACE, BYTE_LEVEL }; + enum class Decoder { NONE, REPLACE_METASPACE, BYTE_LEVEL }; + + TokenizerType tokenizer_type = TokenizerType::UNKNOWN; + VocabFormat vocab_format = VocabFormat::UNKNOWN; + Normalizer normalizer = Normalizer::NONE; + Decoder decoder = Decoder::NONE; + bool byte_fallback = false; + bool has_chat_template = false; +}; + +TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file); +void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens); +std::vector split_with_special_tokens(const std::string& text, const std::unordered_map& special_tokens); + +inline std::string extract_json_string(const std::string& json, size_t& pos) { + std::string value; + while (pos < json.size() && json[pos] != '"') { + if (json[pos] == '\\' && pos + 1 < json.size()) { + pos++; + if (json[pos] == 'n') value += '\n'; + else if (json[pos] == 't') value += '\t'; + else if (json[pos] == 'r') value += '\r'; + else if (json[pos] == '"') value += '"'; + else if (json[pos] == '\\') value += '\\'; + else value += json[pos]; + } else { + value += json[pos]; + } + pos++; + } + if (pos < json.size()) pos++; + return value; +} + +class Tokenizer { +public: + virtual ~Tokenizer() = default; + + virtual std::vector encode(const std::string& text) const = 0; + virtual std::string decode(const std::vector& tokens) const = 0; + + virtual std::vector apply_chat_template(const std::vector& messages, bool add_generation_prompt = true) const; + virtual std::string format_chat_prompt(const std::vector& messages, bool add_generation_prompt = true, const std::string& tools_json = "", bool enable_thinking_if_supported = false) const; + + virtual uint32_t get_vocab_size() const = 0; + virtual uint32_t get_unk_token() const = 0; + virtual uint32_t get_bos_token() const = 0; + virtual uint32_t get_eos_token() const = 0; + virtual bool has_chat_template() const { return has_chat_template_; } + std::string get_default_stop_sequence() const; + + virtual bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) = 0; + + uint32_t get_image_token_id() const { return image_token_id_; } + uint32_t get_fake_token_id() const { return fake_token_id_; } + uint32_t get_global_img_token_id() const { return global_img_token_id_; } + +protected: + enum class ModelType { UNKNOWN, QWEN, QWEN3P5, GEMMA, GEMMA4, LFM2, BERT, WHISPER, PARAKEET, YOUTU, NEEDLE}; + ModelType model_type_ = ModelType::UNKNOWN; + enum class ModelVariant { DEFAULT, VLM, EXTRACT, RAG}; + ModelVariant model_variant_ = ModelVariant::DEFAULT; + bool has_chat_template_ = false; + std::string chat_template_; + + uint32_t image_token_id_ = 396; + uint32_t fake_token_id_ = 49189; + uint32_t global_img_token_id_ = 49152; + + + uint32_t vision_patch_size_ = 16; + uint32_t vision_pooling_kernel_size_ = 3; + uint32_t vision_default_output_length_ = 280; + uint32_t vision_image_size_ = 768; + TokenizerRuntimeConfig runtime_config_; + + void detect_model_type(const std::string& config_path); + void load_chat_template(const std::string& template_file); + std::string format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; + std::string format_gemma_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_gemma4_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; + std::string format_lfm2_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_lfm2_vl_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_needle_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; + std::string format_youtu_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; +}; + +class BPETokenizer : public Tokenizer { +public: + BPETokenizer(); + ~BPETokenizer(); + + bool load_vocabulary_mmap(const std::string& vocab_file, const std::string& merges_file); + bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; + + std::vector encode(const std::string& text) const override; + std::string decode(const std::vector& tokens) const override; + + uint32_t get_vocab_size() const override { return vocab_size_; } + uint32_t get_unk_token() const override { return unk_token_id_; } + uint32_t get_bos_token() const override { return bos_token_id_; } + uint32_t get_eos_token() const override { return eos_token_id_; } + +private: + std::unordered_map token_to_id_; + std::vector id_to_token_; + std::vector merge_rules_; + std::unordered_map merge_map_; + + uint32_t vocab_size_; + uint32_t unk_token_id_; + uint32_t bos_token_id_; + uint32_t eos_token_id_; + + void* vocab_mmap_ptr_; + size_t vocab_mmap_size_; + + void* merges_mmap_ptr_; + size_t merges_mmap_size_; + + std::vector apply_bpe(const std::vector& tokens) const; + std::pair find_best_merge_fast(const std::vector& tokens) const; + + std::string bytes_to_unicode(const std::string& text) const; + std::string unicode_to_bytes(const std::string& text) const; + std::vector byte_level_split(const std::string& text) const; + std::vector utf8_split(const std::string& text) const; + + void cleanup_mmap(); + +private: + mutable std::unordered_map byte_to_unicode_; + mutable std::unordered_map unicode_to_byte_; + void init_byte_mappings() const; + + std::unordered_map special_tokens_; + std::vector split_with_special_tokens(const std::string& text) const; + void load_special_tokens(const std::string& config_file); +}; + +class SPTokenizer : public Tokenizer { +public: + SPTokenizer(); + ~SPTokenizer(); + + bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; + + std::vector encode(const std::string& text) const override; + std::string decode(const std::vector& tokens) const override; + + uint32_t get_vocab_size() const override { return vocab_size_; } + uint32_t get_unk_token() const override { return unk_token_id_; } + uint32_t get_bos_token() const override { return bos_token_id_; } + uint32_t get_eos_token() const override { return eos_token_id_; } + +private: + struct TrieNode { + std::unordered_map> children; + int32_t token_id = -1; + float score = 0.0f; + }; + + std::unique_ptr trie_root_; + std::unordered_map token_to_id_; + std::vector id_to_token_; + std::vector token_scores_; + + uint32_t vocab_size_; + uint32_t unk_token_id_; + uint32_t bos_token_id_; + uint32_t eos_token_id_; + uint32_t pad_token_id_; + + bool sp_bpe_mode_ = false; + bool sp_add_dummy_prefix_ = false; + bool sp_byte_fallback_ = false; + + void* vocab_mmap_ptr_; + size_t vocab_mmap_size_; + + void build_trie(); + std::vector> tokenize_with_trie(const std::string& text) const; + std::vector tokenize_with_bpe(const std::string& text) const; + std::string preprocess_text(const std::string& text) const; + std::string postprocess_text(const std::string& text) const; + std::vector split_by_unicode_spaces(const std::string& text) const; + + void cleanup_mmap(); + + std::unordered_map special_tokens_; + std::vector split_with_special_tokens(const std::string& text) const; + void load_special_tokens(const std::string& config_file); +}; + +class ConvCache { +public: + struct CircularView { + const void* ptr1; + size_t len1; + const void* ptr2; + size_t len2; + size_t total_len; + }; + + void init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision); + CircularView get_window(size_t layer) const; + void update(CactusGraph* gb, size_t layer, const size_t latest_token); + void reset(); + + bool is_empty() const { return num_layers == 0; } + + size_t num_layers = 0; + size_t hidden_size = 0; + size_t window_size = 0; + Precision precision = Precision::FP32; + size_t element_size = 4; + +private: + struct LayerState { + std::vector data; + size_t head = 0; + size_t count = 0; + }; + + std::vector layer_states; +}; + +struct KVCache { + static constexpr size_t DEFAULT_WINDOW_SIZE = 1024; + static constexpr size_t DEFAULT_SINK_SIZE = 4; + + struct LayerCache { + std::vector keys; + std::vector values; + std::vector key_scales; + std::vector value_scales; + size_t head_dim = 0; + size_t kv_heads = 0; + }; + + std::vector layer_caches; + + size_t window_size = DEFAULT_WINDOW_SIZE; + size_t sink_size = DEFAULT_SINK_SIZE; + size_t current_seq_len = 0; + size_t total_seq_len = 0; + size_t max_seq_len = 2048; + size_t num_layers = 0; + Precision precision; + size_t element_size = 4; + + void set_window_size(size_t window, size_t sink = DEFAULT_SINK_SIZE); + size_t get_effective_seq_len() const { return current_seq_len; } + size_t get_total_seq_len() const { return total_seq_len; } + size_t get_layer_head_dim(size_t layer_idx) const { return layer_caches[layer_idx].head_dim; } + size_t get_layer_kv_heads(size_t layer_idx) const { return layer_caches[layer_idx].kv_heads; } + + void init(size_t num_layers, size_t max_seq, const std::vector& layer_dims, const std::vector& layer_kv_heads, Precision model_precision); + void reset(); + void update_from_graph(CactusGraph* gb, const std::vector& k_nodes, + const std::vector& v_nodes, size_t seq_len, + size_t num_layers); + + void update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data, + size_t num_tokens, size_t kv_heads, size_t head_dim); + + bool is_empty() const { return current_seq_len == 0; } + void* get_key_ptr(size_t layer); + void* get_value_ptr(size_t layer); + + struct CircularView { + const void* ptr1; + const void* ptr2; + size_t len1; + size_t len2; + size_t total_len; + }; + + CircularView get_key_view(size_t layer); + CircularView get_value_view(size_t layer); + + const int8_t* get_keys_int8(size_t layer) const; + const int8_t* get_values_int8(size_t layer) const; + const float* get_key_scales(size_t layer) const; + const float* get_value_scales(size_t layer) const; + + void remove_token_range(size_t start, size_t count); + void compact_to_windows(const std::vector& target_windows); +}; + +class ToolCallConstrainer { +public: + enum class State { + DONE, + + QWEN_START, + QWEN_EXPECT_OPEN_BRACE, + QWEN_EXPECT_NAME_KEY, + QWEN_EXPECT_NAME_COLON, + QWEN_EXPECT_NAME_VALUE, + QWEN_EXPECT_COMMA, + QWEN_EXPECT_ARGS_KEY, + QWEN_EXPECT_ARGS_COLON, + QWEN_IN_ARGUMENTS, + QWEN_EXPECT_CLOSE_BRACE, + QWEN_EXPECT_END, + + NEEDLE_START, + + LFM_START, + LFM_EXPECT_BRACKET, + LFM_IN_FUNC_NAME, + LFM_EXPECT_PAREN, + LFM_IN_ARGUMENTS, + LFM_EXPECT_BRACKET_CLOSE, + LFM_EXPECT_END, + + GEMMA_START, + GEMMA_EXPECT_CALL, + GEMMA_IN_FUNC_NAME, + GEMMA_EXPECT_BRACE, + GEMMA_IN_ARGUMENTS, + GEMMA_EXPECT_END + }; + + void init(Config::ModelType model_type, + const std::vector& tools, + Tokenizer* tokenizer); + + const std::unordered_map& get_bias() const { return current_bias_; } + + void update(uint32_t token_id, const std::string& decoded_text); + + void reset(); + + bool is_active() const { return active_; } + +private: + bool active_ = false; + State state_ = State::QWEN_START; + Config::ModelType model_type_ = Config::ModelType::QWEN; + Tokenizer* tokenizer_ = nullptr; + + bool is_gemma_family() const { return Config::is_gemma_family(model_type_); } + bool is_needle() const { return model_type_ == Config::ModelType::NEEDLE; } + + enum class NeedleJsonState { + FREE, + IN_NAME, + IN_ARG_KEY, + }; + + struct NeedleTrieNode { + std::unordered_map> children; + bool is_terminal = false; + }; + + std::vector tool_specs_; + std::vector function_names_; + std::string generated_text_; + int brace_depth_ = 0; + + std::string call_start_tag_; + std::string call_end_tag_; + + std::unordered_set qwen_tool_call_start_tokens_; + std::unordered_set qwen_tool_call_end_tokens_; + std::unordered_set open_brace_tokens_; + std::unordered_set close_brace_tokens_; + std::unordered_set colon_tokens_; + std::unordered_set comma_tokens_; + std::unordered_set name_key_tokens_; + std::unordered_set args_key_tokens_; + std::unordered_set quote_tokens_; + std::unordered_set backtick_tokens_; + std::unordered_set all_func_name_tokens_; + std::unordered_map> func_name_sequences_; + NeedleJsonState needle_json_state_ = NeedleJsonState::FREE; + std::string needle_buffer_; + std::string needle_constrained_buf_; + std::string needle_current_function_; + bool needle_in_arguments_ = false; + int needle_arguments_depth_ = 0; + int needle_nesting_depth_ = 0; + bool needle_in_string_value_ = false; + bool needle_prev_char_escape_ = false; + std::unique_ptr needle_name_trie_; + std::unordered_map> needle_param_tries_; + std::vector needle_token_strings_; + std::unordered_map> needle_token_index_; + + std::unordered_set tool_start_tokens_; + std::unordered_set tool_end_tokens_; + std::unordered_set bracket_open_tokens_; + std::unordered_set bracket_close_tokens_; + std::unordered_set paren_open_tokens_; + std::unordered_set paren_close_tokens_; + std::unordered_set equals_tokens_; + + std::string lfm_current_function_; + std::string lfm_args_buffer_; + std::unordered_set lfm_seen_arg_keys_; + std::unordered_map> lfm_required_params_; + std::unordered_map> lfm_all_params_; + + std::unordered_set gemma_call_start_tokens_; + std::unordered_set gemma_call_end_tokens_; + std::unordered_set gemma_response_start_tokens_; + std::unordered_set gemma_call_prefix_tokens_; + std::unordered_set escape_tokens_; + + std::unordered_map current_bias_; + + void compute_bias(); + void tokenize_grammar_elements(); + void add_tokens_for_string(const std::string& str, std::unordered_set& token_set); + void add_tokens_for_prefix_string(const std::string& prefix, std::unordered_set& token_set); + void add_tokens_containing(char needle, std::unordered_set& token_set); + void tokenize_function_names(bool quote_names); + void init_common_tokens(); + void init_needle_constraints(); + void reset_needle_constraints(); + void feed_needle_text(const std::string& text); + void feed_needle_char(char ch); + bool needle_at_arg_key_start() const; + bool needle_is_value_string_start() const; + void needle_insert_word(NeedleTrieNode* root, const std::string& word); + const NeedleTrieNode* needle_get_trie_node(const NeedleTrieNode* root, const std::string& prefix) const; + bool needle_check_token_valid(const std::string& token_text, const NeedleTrieNode* trie_node) const; +}; + +class Model { +public: + struct DebugNode { + uint32_t layer_idx; + std::string name; + size_t node_id; + }; + + Model(); + explicit Model(const Config& config); + virtual ~Model(); + + const Config& get_config() const { return config_; } + Tokenizer* get_tokenizer() const { return tokenizer_.get(); } + const std::vector& get_debug_nodes() const; + + virtual bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true); + + virtual bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, + const std::string& system_prompt = "", bool do_warmup = true); + + virtual uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f); + + virtual void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = ""); + + virtual void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, + const std::string& profile_file = ""); + + virtual uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, + float temperature = -1.0f, float top_p = -1.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f); + + virtual uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, float temperature = 0.0f, float top_p = 0.0f, + size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, + float min_p = 0.15f, float repetition_penalty = 1.1f, + float* out_token_time_start = nullptr, float* out_token_time_end = nullptr); + + std::vector get_embeddings(const std::vector& tokens, bool pooled = true, bool normalize = false, const std::string& profile_file = ""); + + virtual std::vector get_image_embeddings(const std::string& image_path); + + virtual std::vector get_audio_embeddings(const std::vector& audio_features); + + virtual void reset_cache() { kv_cache_.reset(); token_history_.clear(); } + void record_sampled_token(uint32_t token) { + if (token_history_.size() >= MAX_TOKEN_HISTORY) { + token_history_.erase(token_history_.begin(), token_history_.begin() + (MAX_TOKEN_HISTORY / 2)); + } + token_history_.push_back(token); + } + + double score_tokens_window_logprob(const std::vector& tokens, size_t start, size_t end, size_t context, size_t* tokens_scored); + + + + void set_cache_window(size_t window_size, size_t sink_size = 4) { kv_cache_.set_window_size(window_size, sink_size); } + size_t get_cache_size() const { return kv_cache_.current_seq_len; } + + bool load_npu_prefill(const std::string& model_path); + bool has_npu_prefill() const; + size_t get_prefill_chunk_size() const; + + virtual void remove_thinking_tokens(const std::vector>& ranges); + virtual void compact_kv_cache() {} + + virtual void set_tool_constraints(const std::vector& tools); + virtual void clear_tool_constraints(); + virtual void update_tool_constraints(uint32_t token_id); + + void* graph_handle_; + + void set_vocab_bias(const std::unordered_map& bias) { + vocab_bias_ = bias; + } + + void clear_vocab_bias() { + vocab_bias_.clear(); + } + + bool has_vocab_bias() const { + return !vocab_bias_.empty(); + } + + const std::unordered_map& get_vocab_bias() const { + return vocab_bias_; + } + +protected: + size_t sample_token(CactusGraph* gb, size_t logits_node_id, float temperature, float top_p, size_t top_k, + float min_p, float repetition_penalty, + const std::unordered_map* extra_bias = nullptr) const; + + static void compute_entropy(CactusGraph* gb, size_t logits_node_id, float* out_entropy); + + virtual size_t forward(const std::vector& tokens, bool use_cache = false) = 0; + + virtual size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false); + + virtual void load_weights_to_graph(CactusGraph* gb) = 0; + + virtual size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; + + virtual size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, + ComputeBackend backend) const = 0; + virtual size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, + ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; + void update_kv_cache(CactusGraph* gb, size_t seq_len); + virtual std::vector get_kv_layer_dims() const { + return std::vector(config_.num_layers, config_.attention_head_dim); + } + virtual std::vector get_kv_layer_heads() const { + return std::vector(config_.num_layers, config_.attention_kv_heads); + } + virtual void post_init() {} + virtual void post_execute_updates(CactusGraph*, size_t) {} + Config config_; + std::unique_ptr tokenizer_; + + bool initialized_; + float attention_scale_; + +protected: + KVCache kv_cache_; + std::vector cache_k_output_nodes_; + std::vector cache_v_output_nodes_; + + std::string embedding_file_path_; + size_t embedding_node_id_; + std::string model_folder_path_; + size_t output_weight_node_id_; + + mutable std::vector debug_nodes_; + + void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) const; + void clear_debug_nodes(); + + bool init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size, + const std::string& system_prompt, bool do_warmup); + bool owns_graph_; + + std::unique_ptr npu_prefill_; + void prefill_npu(const std::vector& tokens); + virtual std::vector<__fp16> get_token_embeddings(const std::vector& tokens); + + static constexpr size_t MAX_TOKEN_HISTORY = 128; + ToolCallConstrainer tool_constrainer_; + std::vector token_history_; + +private: + std::unordered_map vocab_bias_; +}; + +std::unique_ptr create_model(const std::string& model_folder); + +class Siglip2Preprocessor { +public: + struct Config { + int patch_size = 16; + int downsample_factor = 2; + int min_tiles = 2; + int max_tiles = 10; + bool use_thumbnail = true; + int min_image_tokens = 64; + int max_image_tokens = 256; + int max_num_patches = 1024; + int tile_size = 512; + float max_pixels_tolerance = 2.0f; + bool do_resize = true; + bool do_rescale = true; + bool do_normalize = true; + bool do_convert_rgb = true; + bool do_image_splitting = true; + float rescale_factor = 1.0f / 255.0f; + float image_mean[3] = {0.5f, 0.5f, 0.5f}; + float image_std[3] = {0.5f, 0.5f, 0.5f}; + }; + + struct PreprocessedImage { + std::vector pixel_values; + std::vector pixel_attention_mask; + std::vector> spatial_shapes; + std::vector pixel_values_shape; + std::vector pixel_attention_mask_shape; + std::vector spatial_shapes_shape; + int num_patches_height; + int num_patches_width; + int actual_num_patches; + int num_tiles; + int patch_dim; + int max_patches_per_tile; + + int image_rows; + int image_cols; + int image_height; + int image_width; + int tokens_per_tile; + int thumbnail_tokens; + + ~PreprocessedImage(); + }; + + struct SpatialShapeResult { + std::vector> shapes; + int grid_rows; + int grid_cols; + }; + + explicit Siglip2Preprocessor(const Config& config); + Siglip2Preprocessor(); + ~Siglip2Preprocessor(); + + PreprocessedImage preprocess_from_file(const std::string& image_path); + PreprocessedImage preprocess_from_memory(const unsigned char* img_data, int width, int height, int channels); + SpatialShapeResult compute_spatial_shapes(int height, int width); + +private: + Config config_; + + std::pair compute_pixel_limits() const; + std::vector convert_to_rgb(const unsigned char* img_data, int width, int height, int channels); + std::pair smart_resize(int height, int width); + bool is_image_too_large(int height, int width); + std::pair get_grid_layout(int height, int width); + std::pair find_closest_aspect_ratio(float aspect_ratio, int width, int height); + std::vector resize_image(const unsigned char* img_data, int src_width, int src_height, + int dst_width, int dst_height, int channels); + std::vector normalize_image(const float* img_data, int width, int height, int channels); + std::vector> convert_image_to_patches( + const std::vector& image, int width, int height, int channels, int patch_size); + PreprocessedImage pad_patches(const std::vector>& tile_patches, + const std::vector>& spatial_shapes, + int patch_dim, + int max_patches_per_tile); + int round_by_factor(int number, int factor); +}; + +class AudioProcessor { +public: + struct SpectrogramConfig { + size_t n_fft = 400; + size_t hop_length = 160; + size_t frame_length = 400; + float power = 2.0f; + bool center = true; + const char* pad_mode = "reflect"; + bool onesided = true; + float dither = 0.0f; + float mel_floor = 1e-10f; + const char* log_mel = nullptr; + float reference = 1.0f; + float min_value = 1e-10f; + bool remove_dc_offset = false; + float preemphasis = 0.0f; + bool hann_periodic = true; + float window_a0 = 0.5f; + size_t fft_override = 0; + bool mel_floor_additive = false; + }; + + AudioProcessor(); + ~AudioProcessor(); + + void init_mel_filters(size_t num_frequency_bins, size_t num_mel_filters, + float min_freq, float max_freq, size_t sampling_rate, + const char* norm = "slaney", const char* mel_scale = "slaney"); + + std::vector compute_spectrogram( + const std::vector& waveform, + const SpectrogramConfig& config); + + static std::vector compute_irfft( + const std::vector& complex_input, + size_t n, + const char* norm = "backward"); + + const std::vector& get_mel_filters() const { return mel_filters_; } + + size_t get_num_mel_filters() const { return num_mel_filters_; } + size_t get_num_frequency_bins() const { return num_frequency_bins_; } + +private: + std::vector mel_filters_; + size_t num_frequency_bins_; + size_t num_mel_filters_; +}; + +namespace index { + constexpr uint32_t MAGIC = 0x43414354; + constexpr uint32_t VERSION = 1; + + struct Document { + int id; + std::vector embedding; + std::string content; + std::string metadata; + }; + + struct QueryResult { + int doc_id; + float score; + + QueryResult(int doc_id, float score) : doc_id(doc_id), score(score) {} + }; + + struct QueryOptions { + size_t top_k = 10; + float score_threshold = -1.0f; + }; + + class Index { + public: + Index(const std::string& index_path, const std::string& data_path, size_t embedding_dim); + ~Index(); + + Index(const Index&) = delete; + Index& operator=(const Index&) = delete; + Index(Index&&) = delete; + Index& operator=(Index&&) = delete; + + void add_documents(const std::vector& documents); + void delete_documents(const std::vector& doc_ids); + std::vector get_documents(const std::vector& doc_ids); + std::vector> query(const std::vector>& embeddings, const QueryOptions& options); + void compact(); + + private: + struct IndexHeader { + uint32_t magic; + uint32_t version; + uint32_t embedding_dim; + uint32_t num_documents; + }; + + struct IndexEntry { + int32_t doc_id; + uint64_t data_offset; + uint8_t flags; // bit 0: tombstone + + const __fp16* embedding() const { + return reinterpret_cast(this + 1); + } + + static size_t size(size_t embedding_dim) { + return sizeof(IndexEntry) + embedding_dim * sizeof(__fp16); + } + }; + + struct DataHeader { + uint32_t magic; + uint32_t version; + }; + + struct DataEntry { + uint16_t content_len; + uint16_t metadata_len; + + const char* content() const { + return reinterpret_cast(this + 1); + } + + const char* metadata() const { + return content() + content_len; + } + }; + + void parse_index_header(); + void parse_data_header(); + void build_doc_id_map(); + void validate_documents(const std::vector& documents); + void validate_doc_ids(const std::vector& doc_ids); + ssize_t write_full(int fd, const void* buf, size_t count); + + std::unordered_map doc_id_map_; + + std::string index_path_, data_path_; + size_t embedding_dim_; + size_t index_entry_size_; + uint32_t num_documents_; + + int index_fd_, data_fd_; + void *mapped_index_, *mapped_data_; + size_t index_file_size_, data_file_size_; + }; +} // namespace index + +} +} diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/gemma_tools.h b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/gemma_tools.h new file mode 100644 index 0000000..f0f9fe2 --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/gemma_tools.h @@ -0,0 +1,576 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace gemma { + +inline std::string to_upper(const std::string& s) { + std::string result = s; + for (auto& c : result) c = std::toupper(c); + return result; +} + +inline std::string escape(const std::string& s) { + return "" + s + ""; +} + +inline void skip_whitespace(const std::string& json, size_t& pos) { + while (pos < json.length() && std::isspace(json[pos])) pos++; +} + +inline std::string extract_json_string(const std::string& json, size_t& pos) { + std::string value; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\' && pos + 1 < json.length()) { + pos++; + if (json[pos] == 'n') value += '\n'; + else if (json[pos] == 't') value += '\t'; + else if (json[pos] == 'r') value += '\r'; + else if (json[pos] == '"') value += '"'; + else if (json[pos] == '\\') value += '\\'; + else value += json[pos]; + } else { + value += json[pos]; + } + pos++; + } + if (pos < json.length()) pos++; + return value; +} + +std::string format_argument(const std::string& json, size_t& pos, bool escape_keys); +std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/); + +inline std::string format_argument(const std::string& json, size_t& pos, bool escape_keys = true) { + skip_whitespace(json, pos); + if (pos >= json.length()) return ""; + + char c = json[pos]; + + if (c == '"') { + pos++; + std::string value = extract_json_string(json, pos); + return escape(value); + } else if (c == '{') { + std::string result = "{"; + pos++; + bool first = true; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == '}') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + if (json[pos] != '"') break; + pos++; + std::string key = extract_json_string(json, pos); + + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == ':') pos++; + + std::string value = format_argument(json, pos, escape_keys); + + if (!first) result += ","; + first = false; + if (escape_keys) { + result += escape(key) + ":" + value; + } else { + result += key + ":" + value; + } + } + result += "}"; + return result; + } else if (c == '[') { + std::string result = "["; + pos++; + bool first = true; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == ']') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + std::string value = format_argument(json, pos, escape_keys); + + if (!first) result += ","; + first = false; + result += value; + } + result += "]"; + return result; + } else if (json.compare(pos, 4, "true") == 0) { + pos += 4; + return "true"; + } else if (json.compare(pos, 5, "false") == 0) { + pos += 5; + return "false"; + } else if (json.compare(pos, 4, "null") == 0) { + pos += 4; + return "null"; + } else { + size_t start = pos; + while (pos < json.length() && (std::isdigit(json[pos]) || json[pos] == '.' || + json[pos] == '-' || json[pos] == '+' || json[pos] == 'e' || json[pos] == 'E')) { + pos++; + } + return json.substr(start, pos - start); + } +} + +inline std::map parse_json_object_raw(const std::string& json, size_t& pos) { + std::map result; + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] != '{') return result; + pos++; + + while (pos < json.length()) { + skip_whitespace(json, pos); + if (pos >= json.length() || json[pos] == '}') { pos++; break; } + if (json[pos] == ',') { pos++; continue; } + + if (json[pos] != '"') break; + pos++; + std::string key = extract_json_string(json, pos); + + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == ':') pos++; + skip_whitespace(json, pos); + + size_t value_start = pos; + if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + pos++; + } else if (json[pos] == '{') { + int depth = 1; + pos++; + while (pos < json.length() && depth > 0) { + if (json[pos] == '{') depth++; + else if (json[pos] == '}') depth--; + else if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + } else if (json[pos] == '[') { + int depth = 1; + pos++; + while (pos < json.length() && depth > 0) { + if (json[pos] == '[') depth++; + else if (json[pos] == ']') depth--; + else if (json[pos] == '"') { + pos++; + while (pos < json.length() && json[pos] != '"') { + if (json[pos] == '\\') pos++; + pos++; + } + } + pos++; + } + } else { + while (pos < json.length() && json[pos] != ',' && json[pos] != '}') pos++; + } + result[key] = json.substr(value_start, pos - value_start); + } + return result; +} + +inline std::string get_json_string_value(const std::string& json, size_t pos) { + skip_whitespace(json, pos); + if (pos < json.length() && json[pos] == '"') { + pos++; + return extract_json_string(json, pos); + } + return ""; +} + +inline std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/) { + static const std::set standard_keys = {"description", "type", "properties", "required", "nullable"}; + + size_t pos = 0; + auto properties = parse_json_object_raw(properties_json, pos); + + std::string result; + bool first = true; + + for (const auto& [key, value_json] : properties) { + if (standard_keys.count(key)) continue; + + if (!first) result += ","; + first = false; + + size_t prop_pos = 0; + auto prop_obj = parse_json_object_raw(value_json, prop_pos); + + result += key + ":{"; + + if (prop_obj.count("description")) { + std::string desc = get_json_string_value(prop_obj["description"], 0); + result += "description:" + escape(desc); + } + + std::string type_val; + if (prop_obj.count("type")) { + type_val = get_json_string_value(prop_obj["type"], 0); + } + + if (to_upper(type_val) == "STRING") { + if (prop_obj.count("enum")) { + size_t enum_pos = 0; + std::string enum_formatted = format_argument(prop_obj["enum"], enum_pos, true); + result += ",enum:" + enum_formatted; + } + } else if (to_upper(type_val) == "OBJECT") { + if (prop_obj.count("properties")) { + std::string nested_required; + if (prop_obj.count("required")) { + nested_required = prop_obj["required"]; + } + result += ",properties:{" + format_parameters(prop_obj["properties"], nested_required) + "}"; + } + if (prop_obj.count("required")) { + std::string req_items; + size_t req_pos = 0; + skip_whitespace(prop_obj["required"], req_pos); + if (req_pos < prop_obj["required"].length() && prop_obj["required"][req_pos] == '[') { + req_pos++; + bool req_first = true; + while (req_pos < prop_obj["required"].length()) { + skip_whitespace(prop_obj["required"], req_pos); + if (prop_obj["required"][req_pos] == ']') break; + if (prop_obj["required"][req_pos] == ',') { req_pos++; continue; } + if (prop_obj["required"][req_pos] == '"') { + req_pos++; + std::string req_item = extract_json_string(prop_obj["required"], req_pos); + if (!req_first) req_items += ","; + req_first = false; + req_items += escape(req_item); + } + } + } + if (!req_items.empty()) { + result += ",required:[" + req_items + "]"; + } + } + } else if (to_upper(type_val) == "ARRAY") { + if (prop_obj.count("items")) { + result += ",items:{"; + size_t items_pos = 0; + auto items_obj = parse_json_object_raw(prop_obj["items"], items_pos); + bool items_first = true; + + for (const auto& [item_key, item_value] : items_obj) { + if (!items_first) result += ","; + items_first = false; + + if (item_key == "properties") { + std::string items_required; + if (items_obj.count("required")) { + items_required = items_obj["required"]; + } + result += "properties:{" + format_parameters(item_value, items_required) + "}"; + } else if (item_key == "required") { + result += "required:["; + size_t req_pos = 0; + skip_whitespace(item_value, req_pos); + if (req_pos < item_value.length() && item_value[req_pos] == '[') { + req_pos++; + bool req_first = true; + while (req_pos < item_value.length()) { + skip_whitespace(item_value, req_pos); + if (item_value[req_pos] == ']') break; + if (item_value[req_pos] == ',') { req_pos++; continue; } + if (item_value[req_pos] == '"') { + req_pos++; + std::string req_item = extract_json_string(item_value, req_pos); + if (!req_first) result += ","; + req_first = false; + result += escape(req_item); + } + } + } + result += "]"; + } else if (item_key == "type") { + std::string item_type = get_json_string_value(item_value, 0); + result += "type:" + escape(to_upper(item_type)); + } else { + size_t val_pos = 0; + result += item_key + ":" + format_argument(item_value, val_pos, true); + } + } + result += "}"; + } + } + + if (!type_val.empty()) { + result += ",type:" + escape(to_upper(type_val)); + } + + result += "}"; + } + + return result; +} + +inline std::string format_function_declaration(const std::string& name, + const std::string& description, + const std::string& params_json) { + std::string result = "declaration:" + name + "{"; + result += "description:" + escape(description); + + if (!params_json.empty()) { + result += ",parameters:{"; + + size_t pos = 0; + auto params = parse_json_object_raw(params_json, pos); + + if (params.count("properties")) { + std::string required_json; + if (params.count("required")) { + required_json = params["required"]; + } + result += "properties:{" + format_parameters(params["properties"], required_json) + "}"; + } + + if (params.count("required")) { + std::string req_items; + size_t req_pos = 0; + skip_whitespace(params["required"], req_pos); + if (req_pos < params["required"].length() && params["required"][req_pos] == '[') { + req_pos++; + bool first = true; + while (req_pos < params["required"].length()) { + skip_whitespace(params["required"], req_pos); + if (params["required"][req_pos] == ']') break; + if (params["required"][req_pos] == ',') { req_pos++; continue; } + if (params["required"][req_pos] == '"') { + req_pos++; + std::string item = extract_json_string(params["required"], req_pos); + if (!first) req_items += ","; + first = false; + req_items += escape(item); + } + } + } + if (!req_items.empty()) { + result += ",required:[" + req_items + "]"; + } + } + + if (params.count("type")) { + std::string type_val = get_json_string_value(params["type"], 0); + result += ",type:" + escape(to_upper(type_val)); + } + + result += "}"; + } + + result += "}"; + return result; +} + +template +inline std::string format_tools(const std::vector& tools, bool use_pipe_tags = false) { + if (tools.empty()) return ""; + + const char* decl_start = use_pipe_tags ? "<|tool>" : ""; + const char* decl_end = use_pipe_tags ? "" : ""; + + std::string result; + for (const auto& tool : tools) { + result += decl_start; + std::string params_json; + auto it = tool.parameters.find("schema"); + if (it != tool.parameters.end()) { + params_json = it->second; + } + + result += format_function_declaration(tool.name, tool.description, params_json); + result += decl_end; + } + return result; +} + + +inline size_t match_quote_tag(const std::string& s, size_t pos) { + if (s.compare(pos, 8, "") == 0) return 8; + if (s.compare(pos, 5, "<|\"|>") == 0) return 5; + return 0; +} + +inline size_t find_quote_tag(const std::string& s, size_t pos) { + size_t e = s.find("", pos); + size_t t = s.find("<|\"|>", pos); + if (e == std::string::npos) return t; + if (t == std::string::npos) return e; + return std::min(e, t); +} + +inline std::string unescape(const std::string& s) { + const std::string ESCAPE_TAG = ""; + std::string result = s; + size_t pos = 0; + while ((pos = result.find(ESCAPE_TAG, pos)) != std::string::npos) { + result.erase(pos, ESCAPE_TAG.length()); + } + return result; +} + +inline std::string args_to_json(const std::string& args_content) { + std::string result = "{"; + size_t pos = 0; + bool first = true; + + if (!args_content.empty() && args_content[0] == '{') pos = 1; + + while (pos < args_content.length()) { + while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; + if (pos >= args_content.length() || args_content[pos] == '}') break; + if (args_content[pos] == ',') { pos++; continue; } + + size_t key_start = pos; + while (pos < args_content.length() && args_content[pos] != ':') pos++; + std::string key = args_content.substr(key_start, pos - key_start); + if (pos < args_content.length()) pos++; + + std::string value; + while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; + + if (pos < args_content.length()) { + size_t qtag_len = match_quote_tag(args_content, pos); + if (qtag_len > 0) { + pos += qtag_len; + size_t val_end = find_quote_tag(args_content, pos); + if (val_end != std::string::npos) { + value = "\"" + args_content.substr(pos, val_end - pos) + "\""; + pos = val_end + match_quote_tag(args_content, val_end); + } + } else if (args_content[pos] == '{') { + int depth = 1; + size_t start = pos; + pos++; + while (pos < args_content.length() && depth > 0) { + if (args_content[pos] == '{') depth++; + else if (args_content[pos] == '}') depth--; + pos++; + } + value = args_to_json(args_content.substr(start, pos - start)); + } else if (args_content[pos] == '[') { + int depth = 1; + size_t start = pos; + pos++; + while (pos < args_content.length() && depth > 0) { + if (args_content[pos] == '[') depth++; + else if (args_content[pos] == ']') depth--; + pos++; + } + std::string arr_content = args_content.substr(start + 1, pos - start - 2); + value = "["; + size_t arr_pos = 0; + bool first_item = true; + while (arr_pos < arr_content.length()) { + while (arr_pos < arr_content.length() && (std::isspace(arr_content[arr_pos]) || arr_content[arr_pos] == ',')) arr_pos++; + if (arr_pos >= arr_content.length()) break; + + if (!first_item) value += ","; + first_item = false; + + size_t aq_len = match_quote_tag(arr_content, arr_pos); + if (aq_len > 0) { + arr_pos += aq_len; + size_t end = find_quote_tag(arr_content, arr_pos); + if (end != std::string::npos) { + value += "\"" + arr_content.substr(arr_pos, end - arr_pos) + "\""; + arr_pos = end + match_quote_tag(arr_content, end); + } + } else { + size_t end = arr_content.find_first_of(",]", arr_pos); + if (end == std::string::npos) end = arr_content.length(); + value += arr_content.substr(arr_pos, end - arr_pos); + arr_pos = end; + } + } + value += "]"; + } else { + size_t val_start = pos; + while (pos < args_content.length() && args_content[pos] != ',' && args_content[pos] != '}') { + pos++; + } + value = args_content.substr(val_start, pos - val_start); + while (!value.empty() && std::isspace(value.back())) value.pop_back(); + } + } + + if (!first) result += ","; + first = false; + result += "\"" + key + "\":" + value; + } + + result += "}"; + return result; +} + +inline void parse_function_calls(std::string& response, std::vector& function_calls) { + + const std::string CALL_START = (response.find("<|tool_call>") != std::string::npos) + ? "<|tool_call>" : ""; + const std::string CALL_END = (CALL_START == "<|tool_call>") + ? "" : ""; + size_t pos = 0; + + while ((pos = response.find(CALL_START, pos)) != std::string::npos) { + size_t content_start = pos + CALL_START.length(); + size_t call_end_pos = response.find(CALL_END, content_start); + + size_t content_end = (call_end_pos != std::string::npos) ? call_end_pos : response.length(); + std::string call_content = response.substr(content_start, content_end - content_start); + + if (call_content.compare(0, 5, "call:") == 0) { + size_t brace_pos = call_content.find('{'); + + if (brace_pos == std::string::npos) { + size_t sep_pos = call_content.find_first_of(", ", 5); + if (sep_pos != std::string::npos) { + std::string func_name = call_content.substr(5, sep_pos - 5); + size_t args_start = sep_pos + 1; + while (args_start < call_content.length() && + (call_content[args_start] == ' ' || call_content[args_start] == ',')) { + args_start++; + } + std::string args_content = "{" + call_content.substr(args_start); + if (args_content.back() != '}') args_content += "}"; + + std::string args_json = args_to_json(args_content); + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; + function_calls.push_back(json_call); + } + } else { + std::string func_name = call_content.substr(5, brace_pos - 5); + std::string args_content = call_content.substr(brace_pos); + if (args_content.back() != '}') args_content += "}"; + + std::string args_json = args_to_json(args_content); + std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; + function_calls.push_back(json_call); + } + } + + size_t erase_end = (call_end_pos != std::string::npos) ? + call_end_pos + CALL_END.length() : response.length(); + response.erase(pos, erase_end - pos); + } +} + +} // namespace gemma \ No newline at end of file diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/graph.h b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/graph.h new file mode 100644 index 0000000..b436177 --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/graph.h @@ -0,0 +1,779 @@ +#ifndef GRAPH_H +#define GRAPH_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cactus { + +enum class LogLevel { + DEBUG = 0, + INFO = 1, + WARN = 2, + ERROR = 3, + NONE = 4 +}; + +class Logger { +public: + static Logger& instance() { + static Logger logger; + return logger; + } + + void set_level(LogLevel level) { min_level_ = level; } + LogLevel get_level() const { return min_level_; } + + void set_callback(std::function cb) { + std::lock_guard lock(mutex_); + callback_ = cb; + } + + void log(LogLevel level, const std::string& component, const std::string& message) { + if (level < min_level_) return; + + std::lock_guard lock(mutex_); + + if (callback_) { + callback_(level, component, message); + } else { + std::cerr << "[" << level_string(level) << "] [" << component << "] " << message << std::endl; + } + + if (level == LogLevel::ERROR) { + last_error_ = "[" + component + "] " + message; + } + } + + const std::string& last_error() const { return last_error_; } + void clear_error() { last_error_.clear(); } + +private: + Logger() : min_level_(LogLevel::WARN) {} + + static const char* level_string(LogLevel level) { + switch (level) { + case LogLevel::DEBUG: return "DEBUG"; + case LogLevel::INFO: return "INFO"; + case LogLevel::WARN: return "WARN"; + case LogLevel::ERROR: return "ERROR"; + default: return "?"; + } + } + + LogLevel min_level_; + std::mutex mutex_; + std::string last_error_; + std::function callback_; +}; + +} // namespace cactus + +#define CACTUS_LOG(level, component, msg) \ + do { \ + if (static_cast(level) >= static_cast(cactus::Logger::instance().get_level())) { \ + std::ostringstream _cactus_log_ss; \ + _cactus_log_ss << msg; \ + cactus::Logger::instance().log(level, component, _cactus_log_ss.str()); \ + } \ + } while(0) + +#define CACTUS_LOG_DEBUG(component, msg) CACTUS_LOG(cactus::LogLevel::DEBUG, component, msg) +#define CACTUS_LOG_INFO(component, msg) CACTUS_LOG(cactus::LogLevel::INFO, component, msg) +#define CACTUS_LOG_WARN(component, msg) CACTUS_LOG(cactus::LogLevel::WARN, component, msg) +#define CACTUS_LOG_ERROR(component, msg) CACTUS_LOG(cactus::LogLevel::ERROR, component, msg) + +namespace GraphFile { + class MappedFile; + struct SerializedGraph; +} + +enum class Precision { + INT8, + FP16, + FP32, + INT4 +}; + +enum class ComputeBackend { + CPU, + NPU +}; + +enum class Activation { + SILU, + GELU, + GELU_ERF, + RELU, + SIGMOID, + TANH +}; + +enum class OpType { + INPUT, PRECISION_CAST, + ADD, ADD_CLIPPED, SUBTRACT, MULTIPLY, DIVIDE, + ABS, POW, FLATTEN, VIEW, + MATMUL, TRANSPOSE, RESHAPE, SLICE, GATHER, EMBEDDING, + BILINEAR_INTERPOLATION, + SUM, MEAN, VARIANCE, MIN, MAX, + RMS_NORM, ROPE, ROPE_GPTJ, SOFTMAX, ATTENTION, ATTENTION_INT8_HYBRID, REL_POS_BIAS, CONV1D_CAUSAL, CONV1D_K3, CONV1D_K7S3, CONV1D, CONV1D_SAME_DEPTHWISE_K9, CONV1D_POINTWISE, CONV2D_K3S2P1, CONV2D_DEPTHWISE_K3S2P1, CONV2D_POINTWISE_1X1, GLU, BATCHNORM, + SCALAR_ADD, SCALAR_SUBTRACT, SCALAR_MULTIPLY, SCALAR_DIVIDE, SCALAR_EXP, SCALAR_SQRT, SCALAR_COS, SCALAR_SIN, SCALAR_LOG, + RELU, SILU, GELU, GELU_ERF, SIGMOID, TANH, + SAMPLE, CONCAT, CAT, + SCATTER_TOPK, + TOPK, LAYERNORM, GROUPNORM, + MOE_LAYER, + INDEX, + PERSISTENT, + QUANTIZE_ACTIVATIONS, + LSTM_CELL, + GATED_DELTANET_DECODE, + GATED_DELTANET_PREFILL, + STFT, + ALTUP_PREDICT, + ALTUP_CORRECT, + GAUSSIAN_TOPK, + MAXPOOL1D, + BILSTM_SEQUENCE, + LEAKY_RELU, + CONV2D_K3S1P1, + STATS_POOL, + WEIGHTED_STATS_POOL, + DENSE_MLP_INT4_FUSED +}; + +struct PrecisionTraits { + static constexpr size_t size_of(Precision prec) { + switch (prec) { + case Precision::INT8: return 1; + case Precision::FP16: return 2; + case Precision::FP32: return 4; + case Precision::INT4: return 1; + } + return 1; + } + + static constexpr size_t packed_size_of(Precision prec, size_t count) { + switch (prec) { + case Precision::INT4: return (count + 1) / 2; + default: return count * size_of(prec); + } + } + + static size_t byte_offset_of(Precision prec, size_t element_offset) { + switch (prec) { + case Precision::INT4: + assert(element_offset % 32 == 0 && "INT4 byte offset must be group-aligned (multiple of 32)"); + return element_offset / 2; + default: return element_offset * size_of(prec); + } + } + + static constexpr bool is_integer(Precision prec) { + switch (prec) { + case Precision::INT8: return true; + case Precision::INT4: return true; + case Precision::FP16: return false; + case Precision::FP32: return false; + } + return true; + } + + static constexpr bool is_floating_point(Precision prec) { + switch (prec) { + case Precision::INT8: return false; + case Precision::INT4: return false; + case Precision::FP16: return true; + case Precision::FP32: return true; + } + return false; + } +}; + +namespace Quantization { + void int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); + void fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); + void fp16_to_fp32(const __fp16* src, float* dst, size_t count); + void fp32_to_fp16(const float* src, __fp16* dst, size_t count); + void int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); + void fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); +} + +struct TensorConfig { + Precision default_precision = Precision::INT8; + Precision compute_precision = Precision::INT8; + Precision output_precision = Precision::INT8; + bool auto_mixed_precision = false; + + static TensorConfig& global(); +}; + +struct BroadcastInfo { + std::vector output_shape; + bool needs_broadcasting; + + static BroadcastInfo compute(const std::vector& lhs, const std::vector& rhs); +}; + +class BufferPool; + +struct BufferDesc { + std::vector shape; + size_t total_size; + size_t byte_size; + std::unique_ptr data; + void* external_data; + char* pooled_data; + Precision precision; + + size_t group_size = 0; + size_t num_groups = 0; + void* scales_data = nullptr; + std::unique_ptr owned_scales; + + bool is_interleaved = false; + size_t original_N = 0; + + void* activation_scales_data = nullptr; + std::unique_ptr owned_activation_scales; + size_t num_rows_for_activation_scales = 0; + + BufferDesc(); + BufferDesc(const std::vector& s, Precision prec = Precision::INT8); + ~BufferDesc(); + + BufferDesc(BufferDesc&& other) noexcept; + BufferDesc& operator=(BufferDesc&& other) noexcept; + + BufferDesc(const BufferDesc&) = delete; + BufferDesc& operator=(const BufferDesc&) = delete; + + void* get_data(); + const void* get_data() const; + + template + T* data_as() { return static_cast(get_data()); } + + template + const T* data_as() const { return static_cast(get_data()); } + + const __fp16* scales_as_fp16() const { + return reinterpret_cast(scales_data); + } + + bool is_grouped_int8() const { + return precision == Precision::INT8 && group_size > 0; + } + + bool is_grouped_int4() const { + return precision == Precision::INT4 && group_size > 0; + } + + void set_grouped_scales(size_t gs, size_t ng, void* scales_ptr) { + group_size = gs; + num_groups = ng; + scales_data = scales_ptr; + } + + void set_interleaved(bool interleaved, size_t orig_n) { + is_interleaved = interleaved; + original_N = orig_n; + } + + bool has_activation_scales() const { + return activation_scales_data != nullptr && num_rows_for_activation_scales > 0; + } + const float* activation_scales_as_float() const { + return reinterpret_cast(activation_scales_data); + } + float* activation_scales_as_float() { + return reinterpret_cast(activation_scales_data); + } + void allocate_activation_scales(size_t num_rows) { + num_rows_for_activation_scales = num_rows; + owned_activation_scales = std::make_unique(num_rows * sizeof(float)); + activation_scales_data = owned_activation_scales.get(); + } + void set_activation_scales(void* scales_ptr, size_t num_rows) { + activation_scales_data = scales_ptr; + num_rows_for_activation_scales = num_rows; + } + + void allocate(); + void allocate_from_pool(BufferPool& pool); + void release_to_pool(BufferPool& pool); + void set_external(void* ptr); +}; + +struct OpParams { + float scalar = 0.0f; + float scale = 1.0f; + float theta = 10000.0f; + float epsilon = 1e-6f; + int axis = -1; + bool pretransposed_rhs = false; + size_t position_offset = 0; + size_t slice_start = 0; + size_t slice_length = 0; + size_t window_size = 0; + bool is_causal = true; + bool attention_mask_is_additive = false; + float logit_cap = 0.0f; + std::vector new_shape; + std::vector permutation; + Precision output_precision = Precision::INT8; + BroadcastInfo broadcast_info; + ComputeBackend backend = ComputeBackend::CPU; + + size_t dilation = 1; + size_t stride = 1; + float temperature = 1.0f; + float top_p = 1.0f; + float min_p = 0.15f; + float repetition_penalty = 1.1f; + size_t top_k = 0; + size_t random_seed = 0; + + size_t index_value = 0; + size_t num_classes = 0; + size_t num_groups = 0; + size_t dst_height = 0; + size_t dst_width = 0; + bool align_corners = true; + bool normalize_routing = false; + size_t num_experts = 0; + size_t num_experts_per_tok = 0; + bool moe_gated = true; + Activation activation = Activation::SILU; + + std::vector bias_values; + std::vector bias_indices; + + const int8_t* cached_keys_int8 = nullptr; + const int8_t* cached_values_int8 = nullptr; + const float* cached_k_scales = nullptr; + const float* cached_v_scales = nullptr; + size_t cache_seq_len = 0; + size_t num_kv_heads = 0; + size_t head_dim = 0; + size_t num_fft_bins = 0; + size_t chunk_size = 0; + size_t num_altup_inputs = 0; + size_t v_head_dim = 0; + size_t kernel_size = 0; +}; + +struct GraphNode { + size_t id; + OpType op_type; + std::vector input_ids; + BufferDesc output_buffer; + OpParams params; + + GraphNode(size_t node_id, OpType type); +}; + +using nodes_vector = std::vector>; +using node_index_map_t = std::unordered_map; + +inline const BufferDesc& get_input(const GraphNode& node, size_t idx, + const nodes_vector& nodes, + const node_index_map_t& node_index_map) { + return nodes[node_index_map.at(node.input_ids[idx])]->output_buffer; +} + +struct AxisDims { + size_t outer, axis_size, inner; + static AxisDims from_shape(const std::vector& shape, size_t axis) { + AxisDims d; + d.outer = 1; + for (size_t i = 0; i < axis; i++) d.outer *= shape[i]; + d.axis_size = shape[axis]; + d.inner = 1; + for (size_t i = axis + 1; i < shape.size(); i++) d.inner *= shape[i]; + return d; + } +}; + +template +void dispatch_binary_op(OpType op, const T* lhs, const T* rhs, T* output, size_t count); + +template +void dispatch_unary_op(OpType op, const T* input, T* output, size_t count, float param = 0.0f); + +void compute_node_optimized(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_matmul_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_transpose_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_reduce_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_reshape_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_precision_cast_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_sample_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_scatter_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_layernorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_groupnorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_persistent_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_dense_mlp_int4_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_index_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_lstm_cell_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_gated_deltanet_decode_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_gated_deltanet_prefill_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_altup_predict_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_altup_correct_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_maxpool1d_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_bilstm_sequence_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_conv2d_k3s1p1_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); +void compute_weighted_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); + +void shrink_thread_local_buffers(); +class BufferPool { +public: + BufferPool() = default; + ~BufferPool() = default; + + BufferPool(const BufferPool&) = delete; + BufferPool& operator=(const BufferPool&) = delete; + BufferPool(BufferPool&&) noexcept = default; + BufferPool& operator=(BufferPool&&) noexcept = default; + + char* acquire(size_t byte_size); + void release(char* ptr, size_t byte_size); + void clear(); + + size_t active_bytes() const { return active_bytes_; } + size_t pool_bytes() const { return pool_bytes_; } + size_t peak_bytes() const { return peak_bytes_; } + +private: + std::unordered_map>> free_buffers_; + size_t active_bytes_ = 0; + size_t pool_bytes_ = 0; + size_t peak_bytes_ = 0; + + size_t round_up_size(size_t size) const; +}; + +namespace ValidationUtils { + void validate_tensor_dims(const std::vector& shape, size_t required_dims, const std::string& op_name); + void validate_precision(Precision actual, Precision required, const std::string& op_name); + void validate_input_count(size_t actual, size_t required, const std::string& op_name); +} + + +class CactusGraph { +public: + CactusGraph(); + ~CactusGraph() = default; + + CactusGraph(const CactusGraph&) = delete; + CactusGraph& operator=(const CactusGraph&) = delete; + CactusGraph(CactusGraph&&) noexcept = default; + CactusGraph& operator=(CactusGraph&&) noexcept = default; + + struct DebugNodeEntry { + uint32_t layer_idx; + std::string name; + size_t node_id; + }; + + void save(const std::string& path); + static CactusGraph load(const std::string& path); + + size_t input(const std::vector& shape, Precision precision = Precision::INT8); + size_t precision_cast(size_t input, Precision target_precision); + size_t quantize_activations(size_t input); + + size_t add(size_t input1, size_t input2); + size_t add_clipped(size_t input1, size_t input2); + size_t subtract(size_t input1, size_t input2); + size_t multiply(size_t input1, size_t input2); + size_t divide(size_t input1, size_t input2); + + size_t scalar_add(size_t input, float value); + size_t scalar_subtract(size_t input, float value); + size_t scalar_multiply(size_t input, float value); + size_t scalar_divide(size_t input, float value); + size_t scalar_exp(size_t input); + size_t scalar_sqrt(size_t input); + size_t scalar_cos(size_t input); + size_t scalar_sin(size_t input); + size_t scalar_log(size_t input); + + size_t relu(size_t input); + size_t silu(size_t input); + size_t gelu(size_t input); + size_t gelu_erf(size_t input); + size_t sigmoid(size_t input); + size_t tanh(size_t input); + size_t glu(size_t input, int axis = -1); + + size_t abs(size_t input); + size_t pow(size_t input, float exponent); + size_t view(size_t input, const std::vector& new_shape); + size_t flatten(size_t input, int start_dim = 0, int end_dim = -1); + + size_t matmul(size_t input1, size_t input2, bool pretransposed_rhs = false, ComputeBackend backend = ComputeBackend::CPU); + size_t transpose(size_t input, ComputeBackend backend = ComputeBackend::CPU); + size_t transposeN(size_t input, const std::vector& permutation, ComputeBackend backend = ComputeBackend::CPU); + size_t reshape(size_t input, const std::vector& new_shape); + size_t slice(size_t input, int axis, size_t start, size_t length); + size_t index(size_t input, size_t index_value, int dim); + + size_t sum(size_t input, int axis); + size_t mean(size_t input, int axis); + size_t variance(size_t input, int axis); + size_t min(size_t input, int axis); + size_t max(size_t input, int axis); + + size_t gather(size_t embeddings, size_t indices); + size_t mmap_embeddings(const std::string& filename); + size_t mmap_weights(const std::string& filename); + void set_grouped_scales(size_t node_id, size_t group_size, size_t num_groups, void* scales_ptr); + void set_interleaved(size_t node_id, bool interleaved, size_t original_N); + + void release_weight_pages(size_t node_id); + void prefetch_weight_pages(size_t node_id); + void release_all_weight_pages(); + size_t embedding(const std::string& filename, size_t indices); + size_t embedding(size_t embedding_tensor, size_t indices); + size_t bilinear_interpolation(size_t pos_embeds, size_t dst_height, size_t dst_width, bool align_corners = true); + + size_t layernorm(size_t input, size_t weight, size_t bias, float epsilon = 1e-5f); + size_t layernorm(size_t input, size_t weight, float epsilon = 1e-5f); // No bias version + size_t groupnorm(size_t input, size_t weight, size_t bias, size_t num_groups = 32, float epsilon = 1e-5f); + size_t batchnorm(size_t input, size_t weight, size_t bias, size_t running_mean, size_t running_var, int axis = 1, float epsilon = 1e-5f); + size_t topk(size_t input, size_t k); + size_t dense_mlp_int4_fused(size_t hidden, size_t gate_weight, + size_t up_weight, size_t down_weight); + size_t moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w3_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation = Activation::SILU, + size_t per_expert_scale = 0); + size_t moe_layer(size_t hidden, + size_t routing_probs, + size_t topk_indices, + const std::vector& w1_weights, + const std::vector& w2_weights, + size_t num_experts, + size_t num_experts_per_tok, + bool normalize_routing, + float epsilon, + float routed_scaling_factor, + Activation activation); + size_t rms_norm(size_t input, size_t weight, float epsilon = 1e-5f); + size_t rope(size_t input, float theta, size_t position_offset = 0, ComputeBackend backend = ComputeBackend::CPU); + size_t rope_gptj(size_t input, float theta, size_t position_offset = 0, size_t rot_dim = 0, ComputeBackend backend = ComputeBackend::CPU); + size_t softmax(size_t input, int axis = -1); + size_t attention(size_t query, size_t key, size_t value, float scale, bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU); + size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, ComputeBackend backend = ComputeBackend::CPU); + size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, size_t window_size, ComputeBackend backend = ComputeBackend::CPU); + size_t attention_masked(size_t query, size_t key, size_t value, size_t mask, float scale, + bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU, + bool additive_mask = false, size_t position_offset = 0, size_t window_size = 0, + float logit_cap = 0.0f); + size_t rel_pos_bias(size_t query, size_t relative_key, float scale); + + size_t attention_int8_hybrid(size_t query, size_t key_new, size_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, + const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, + size_t window_size = 0, size_t v_head_dim = 0); + + size_t conv1d_causal(size_t input, size_t weight, size_t kernel_size, size_t dilation = 1); + size_t conv1d_k3(size_t input, size_t weight, size_t stride); + size_t conv1d_k7s3(size_t input, size_t weight, size_t bias); + size_t conv1d(size_t input, size_t weight, size_t stride); + size_t conv1d(size_t input, size_t weight, size_t bias, size_t stride); + size_t conv1d_same_depthwise_k9(size_t input, size_t weight); + size_t conv1d_same_depthwise_k9(size_t input, size_t weight, size_t bias); + size_t conv1d_pointwise(size_t input, size_t weight); + size_t conv1d_pointwise(size_t input, size_t weight, size_t bias); + size_t conv2d_k3s2p1(size_t input, size_t weight); + size_t conv2d_k3s2p1(size_t input, size_t weight, size_t bias); + size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight); + size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight, size_t bias); + size_t conv2d_pointwise_1x1(size_t input, size_t weight); + size_t conv2d_pointwise_1x1(size_t input, size_t weight, size_t bias); + + size_t lstm_cell(size_t input, size_t h_prev, size_t c_prev, size_t weight_ih, size_t weight_hh, size_t bias_ih, size_t bias_hh); + size_t gated_deltanet_decode(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, float scale = 0.0f); + size_t gated_deltanet_prefill(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, + size_t initial_state, size_t chunk_size = 64, float scale = 0.0f); + size_t stft(size_t input, size_t weight, size_t stride, size_t num_fft_bins); + + size_t altup_predict(size_t coefs, const size_t* streams, size_t num_streams); + size_t altup_correct(size_t coefs, size_t innovation, const size_t* predictions, size_t num_predictions); + + size_t gaussian_topk(size_t input, float ppf); + + size_t maxpool1d(size_t input, size_t kernel_size, size_t stride); + size_t leaky_relu(size_t input, float negative_slope = 0.01f); + size_t bilstm_sequence(size_t input, size_t w_ih_fwd, size_t w_hh_fwd, size_t b_ih_fwd, size_t b_hh_fwd, + size_t w_ih_bwd, size_t w_hh_bwd, size_t b_ih_bwd, size_t b_hh_bwd); + size_t conv2d_k3s1p1(size_t input, size_t weight); + size_t conv2d_k3s1p1(size_t input, size_t weight, size_t bias); + size_t stats_pool(size_t input); + size_t weighted_stats_pool(size_t input, size_t weights); + + size_t sample(size_t logits, float temperature = 0.6f, float top_p = 0.95f, size_t top_k = 20, + const std::unordered_map& logit_bias = {}); + size_t sample_with_options(size_t logits, float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, const std::unordered_map& logit_bias = {}); + + size_t concat(size_t input1, size_t input2, int axis = 0); + size_t cat(const std::vector& inputs, int axis); + size_t scatter_topk(size_t indices, size_t values, size_t num_classes); + + void set_input(size_t node_id, const void* data, Precision precision); + void set_external_input(size_t node_id, void* data, Precision precision); + void* get_output(size_t node_id); + + void execute(const std::string& profile_file = ""); + void hard_reset(); + void soft_reset(); + void soft_reset_keep_pool(); + void set_prefill_mode(bool enabled) { prefill_mode_ = enabled; } + + void register_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); + void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); + const std::vector& get_debug_nodes() const; + void clear_debug_nodes(); + + size_t add_node(OpType op_type, const std::vector& inputs, const std::vector& output_shape, const OpParams& params = {}); + const BufferDesc& get_output_buffer(size_t node_id) const; + void allocate_buffers(); + size_t get_node_count() const; + + size_t persistent(size_t source_node); + bool is_populated(size_t persistent_node_id) const; + void invalidate_persistent(size_t persistent_node_id); + + std::vector> nodes_; + std::unordered_map node_index_map_; + +private: + static CactusGraph from_serialized(const GraphFile::SerializedGraph& serialized); + size_t next_node_id_; + std::vector> mapped_files_; + std::unordered_map weight_cache_; + std::unordered_map node_to_mapped_file_; + std::vector debug_nodes_; + BufferPool buffer_pool_; + bool prefill_mode_ = false; + + std::unordered_set persistent_node_ids_; + std::unordered_set populated_node_ids_; +}; + + +namespace GraphFile { + struct LoadedNode { + size_t node_id; + std::vector shape; + Precision precision; + size_t byte_size; + }; + + struct GraphHeader { + uint32_t magic; + uint32_t version; + uint32_t node_count; + uint32_t flags = 0; + }; + + struct NodeEntry { + uint32_t index; // serialized node index 0..n-1 + OpType op_type; + std::vector inputs; + std::vector output_shape; + Precision precision; + OpParams params; + }; + + struct SerializedGraph { + GraphHeader header; + std::vector nodes; + std::vector graph_inputs; // IDs of serialized inputs + std::vector graph_outputs; // IDs of serialized outputs + }; + + SerializedGraph load_graph(const std::string& filename); + void save_graph(const CactusGraph& graph, const std::string& filename); + + void save_node(CactusGraph& graph, size_t node_id, const std::string& filename); + + class MappedFile { + public: + MappedFile(const std::string& filename); + ~MappedFile(); + + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + MappedFile(MappedFile&& other) noexcept; + MappedFile& operator=(MappedFile&& other) noexcept; + + const std::vector& shape() const; + Precision precision() const; + size_t byte_size() const; + + size_t group_size() const { return group_size_; } + size_t num_groups() const { return num_groups_; } + const void* scales_data() const; + + bool is_interleaved() const { return is_interleaved_; } + size_t original_N() const { return original_N_; } + + void* data(); + const void* data() const; + + template + const T* typed_data() const; + + void release_pages(); + void prefetch_pages(); + + private: + int fd_; + void* mapped_data_; + size_t file_size_, data_offset_; + std::vector shape_; + Precision precision_; + size_t byte_size_; + size_t group_size_ = 0; + size_t num_groups_ = 0; + size_t scales_offset_ = 0; + size_t scales_bytes_ = 0; + uint32_t alignment_ = 32; + + bool is_interleaved_ = false; + size_t original_N_ = 0; + + void parse_header(); + void apply_madvise_hints(); + }; +} + +#endif diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/graph_param_io.h b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/graph_param_io.h new file mode 100644 index 0000000..0362ea8 --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/graph_param_io.h @@ -0,0 +1,11 @@ +#pragma once + +#include "graph.h" +#include + +namespace GraphParamIO { + +void write_op_params(std::ostream& out, OpType op_type, const OpParams& params); +void read_op_params(std::istream& in, OpType op_type, OpParams& params); + +} // namespace GraphParamIO diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/kernel.h b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/kernel.h new file mode 100644 index 0000000..13f4a00 --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/kernel.h @@ -0,0 +1,475 @@ +#ifndef KERNEL_H +#define KERNEL_H + +#include +#include + +enum class Precision; + +enum class ScalarOpType { + ADD, + SUBTRACT, + MULTIPLY, + DIVIDE, + ABS, + EXP, + POW, + SQRT, + COS, + SIN, + LOG +}; + +constexpr size_t KV_QUANT_GROUP_SIZE = 32; + +void cactus_add_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_add_f16_clipped(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_subtract_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_multiply_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); +void cactus_add_scaled_f16(const __fp16* base, const __fp16* src, __fp16* output, size_t num_elements, float scale); +void cactus_divide_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); + +void cactus_add_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_subtract_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_multiply_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); +void cactus_divide_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, + const size_t* a_strides, const size_t* b_strides, + const size_t* output_shape, size_t ndim); + +void cactus_scalar_op_f16(const __fp16* input, __fp16* output, size_t num_elements, float scalar_value, ScalarOpType op_type); + +void cactus_gemv_int8(const int8_t* A, float A_scale, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemm_int8(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_int8(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int8_i8mm(const int8_t* A, float A_scale, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemm_int8_i8mm(const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int4(const int8_t* A, float A_scale, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size); + +void cactus_gemv_int4_block_range(const int8_t* A, float A_scale, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t K, size_t N, size_t group_size, + size_t block_start, size_t block_end); + +void cactus_gemm_int4(const int8_t* A, const float* A_scales, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_int4(const int8_t* A, const float* A_scales, + const int8_t* B_packed, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_integer(Precision precision, + const int8_t* A, const float* A_scales, + const int8_t* B, const __fp16* B_scales, + __fp16* C, size_t M, size_t K, size_t N, size_t group_size); + +void cactus_matmul_f16(const __fp16* a, const __fp16* b_transposed, __fp16* c, + size_t M, size_t K, size_t N); + +void cactus_transpose_2d_f16(const __fp16* source, __fp16* destination, + size_t num_rows, size_t num_cols, size_t start_row, size_t end_row); +void cactus_transpose_f16(const __fp16* source, __fp16* destination, const size_t* shape, + const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx); + +double cactus_sum_all_f16(const __fp16* data, size_t num_elements); +void cactus_sum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +double cactus_mean_all_f16(const __fp16* data, size_t num_elements); +void cactus_mean_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +double cactus_variance_all_f16(const __fp16* data, size_t num_elements); +void cactus_variance_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +__fp16 cactus_min_all_f16(const __fp16* data, size_t num_elements); +void cactus_min_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +__fp16 cactus_max_all_f16(const __fp16* data, size_t num_elements); +void cactus_max_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); + +void cactus_rms_norm_f16(const __fp16* input, const __fp16* weight, __fp16* output, + size_t batch_size, size_t dims, float eps); + +void cactus_layer_norm_f16(const __fp16* input, const __fp16* weight, const __fp16* bias, + __fp16* output, size_t batch_size, size_t dims, float eps); + +void cactus_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, + size_t num_heads, size_t head_dim, size_t start_pos, float theta); + +void cactus_gpt_j_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, + size_t num_heads, size_t head_dim, size_t rot_dim, size_t start_pos, float theta); + +void cactus_softmax_f16(const __fp16* input, __fp16* output, size_t batch_size, + size_t seq_len, size_t vocab_size); + +void cactus_relu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_leaky_relu_f16(const __fp16* input, __fp16* output, size_t num_elements, float negative_slope); + +void cactus_silu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_gelu_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_gelu_f16_erf(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_sigmoid_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_tanh_f16(const __fp16* input, __fp16* output, size_t num_elements); + +void cactus_glu_f16( + const __fp16* input, + __fp16* output, + size_t outer_size, + size_t split_size, + size_t inner_size +); + +void cactus_glu_f32( + const float* input, + float* output, + size_t outer_size, + size_t split_size, + size_t inner_size +); + +void cactus_batchnorm_f16( + const __fp16* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + __fp16* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +); + +void cactus_batchnorm_f32( + const float* input, + const float* weight, + const float* bias, + const float* running_mean, + const float* running_var, + float* output, + size_t outer_size, + size_t channels, + size_t inner_size, + float epsilon +); + +void cactus_attention_f16(const __fp16* queries, const __fp16* keys, const __fp16* values, __fp16* output, + size_t batch_size, size_t seq_len, size_t kv_seq_len, size_t num_q_heads, size_t num_kv_heads, + size_t head_dim, float scale, const __fp16* mask, size_t position_offset = 0, size_t window_size = 0, + bool is_causal = true, bool mask_is_additive = false, bool mask_per_head = false, + size_t v_head_dim = 0, float logit_cap = 0.0f); + +void cactus_attention_hybrid_int8_fp16( + const __fp16* queries, + const int8_t* keys_cached, + const int8_t* values_cached, + const float* k_scales, + const float* v_scales, + const __fp16* keys_new, + const __fp16* values_new, + __fp16* output, + size_t batch_size, size_t seq_len, size_t cache_len, size_t new_len, + size_t num_q_heads, size_t num_kv_heads, size_t head_dim, + float scale, size_t position_offset = 0, bool is_causal = true, size_t window_size = 0, + size_t group_size = KV_QUANT_GROUP_SIZE, size_t v_head_dim = 0); + +void cactus_gated_deltanet_decode_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + float scale); + +void cactus_gated_deltanet_prefill_f16( + const __fp16* q_data, + const __fp16* k_data, + const __fp16* v_data, + const __fp16* g_data, + const __fp16* b_data, + const __fp16* s_data, + __fp16* out, + size_t B, + size_t T, + size_t Hq, + size_t Hv, + size_t K, + size_t V, + size_t requested_chunk_size, + float scale); + +void cactus_conv1d_causal_depthwise_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, + size_t L, + size_t C, + size_t K, + size_t dilation); + +void cactus_conv1d_f16_k3( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out, + size_t stride +); + +void cactus_conv1d_f16( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out, + size_t K, + size_t stride +); + +void cactus_stft_f16( + const __fp16* input, + const __fp16* weight, + __fp16* output, + size_t N, size_t L, + size_t C_in, size_t C_out, + size_t K, size_t stride, + size_t num_fft_bins +); + +void cactus_conv1d_f16_k7s3_oc8( + const __fp16* input, + const __fp16* Wpack, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out +); + +void cactus_conv1d_same_depthwise_f16_k9( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C +); + +void cactus_conv2d_f16_k3s1p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv2d_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv2d_depthwise_f16_k3s2p1_nchw( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C, + size_t H, + size_t W +); + +void cactus_conv2d_pointwise_f16_1x1_nchw_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t C_in, + size_t H, + size_t W, + size_t C_out +); + +void cactus_conv1d_pointwise_f16_gemm( + const __fp16* input, + const __fp16* weight, + const __fp16* bias, + __fp16* output, + size_t N, + size_t L, + size_t C_in, + size_t C_out +); + +void cactus_bilinear_interpolation_f16(const __fp16* input, __fp16* output, size_t src_height, size_t src_width, size_t embed_dim, + size_t dst_height, size_t dst_width, bool align_corners = true); + +void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); +void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); + +void cactus_sample_f32_ex(const float* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); +void cactus_sample_f16_ex(const __fp16* logits, uint32_t* output, size_t vocab_size, + float temperature, float top_p, float min_p, float repetition_penalty, + size_t top_k, size_t random_seed, + const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, + size_t bias_count = 0); + +void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* output, + const size_t* shape1, const size_t* shape2, const size_t* output_shape, + size_t ndims, int axis); +void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_shapes, + const size_t* output_shape, size_t num_inputs, size_t rank, int axis); + +void cactus_int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); +void cactus_fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); +void cactus_fp16_to_fp32(const __fp16* src, float* dst, size_t count); +void cactus_fp32_to_fp16(const float* src, __fp16* dst, size_t count); +void cactus_int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); +void cactus_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); +float cactus_fp16_max_abs(const __fp16* src, size_t count); + +void cactus_quantize_kv_fp16_to_int8( + const __fp16* src, + int8_t* dst, + float* scales, + size_t seq_len, size_t kv_heads, size_t head_dim, + size_t group_size = KV_QUANT_GROUP_SIZE); + +inline size_t kv_scales_count(size_t seq_len, size_t kv_heads, size_t head_dim, size_t group_size = KV_QUANT_GROUP_SIZE) { + size_t num_groups = (head_dim + group_size - 1) / group_size; + return seq_len * kv_heads * num_groups; +} + +void cactus_unpack_int4_to_int8(const uint8_t* packed, int8_t* unpacked, size_t unpacked_count); + +void cactus_gaussian_topk_f16( + const __fp16* input, + __fp16* output, + size_t rows, + size_t cols, + float ppf); + +void cactus_altup_predict_f16( + const __fp16* coefs, + const __fp16* const* streams, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim); + +void cactus_altup_correct_f16( + const __fp16* coefs, + const __fp16* innovation, + const __fp16* const* predictions, + __fp16* output, + size_t n, + size_t seq_len, + size_t hidden_dim); + +void cactus_lstm_cell_f16( + const __fp16* x_input, + const __fp16* h_prev, + const __fp16* c_prev, + const __fp16* weight_ih, + const __fp16* weight_hh, + const __fp16* bias_ih, + const __fp16* bias_hh, + __fp16* h_new, + __fp16* c_new, + size_t batch_size, + size_t input_size, + size_t hidden_size +); + +void cactus_bilstm_sequence_f16( + const __fp16* input, + const __fp16* weight_ih_fwd, + const __fp16* weight_hh_fwd, + const __fp16* bias_ih_fwd, + const __fp16* bias_hh_fwd, + const __fp16* weight_ih_bwd, + const __fp16* weight_hh_bwd, + const __fp16* bias_ih_bwd, + const __fp16* bias_hh_bwd, + __fp16* output, + size_t batch_size, + size_t seq_len, + size_t input_size, + size_t hidden_size +); + +void cactus_maxpool1d_f16( + const __fp16* input, + __fp16* output, + size_t batch_size, + size_t channels, + size_t input_length, + size_t kernel_size, + size_t stride +); + +#endif diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/kernel_utils.h b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/kernel_utils.h new file mode 100644 index 0000000..b202158 --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/kernel_utils.h @@ -0,0 +1,681 @@ +#ifndef KERNEL_UTILS_H +#define KERNEL_UTILS_H + +#include +#if defined(__APPLE__) +#include +#include +#endif +#if defined(__ANDROID__) +#include +#include +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr size_t NEON_VECTOR_SIZE = 16; +constexpr size_t STREAMING_STORE_THRESHOLD = 32768; + +inline void stream_store_f16x8(__fp16* dst, float16x8_t val) { +#if defined(__aarch64__) + float16x4_t lo = vget_low_f16(val); + float16x4_t hi = vget_high_f16(val); + __asm__ __volatile__( + "stnp %d0, %d1, [%2]" + : + : "w"(lo), "w"(hi), "r"(dst) + : "memory" + ); +#else + vst1q_f16(dst, val); +#endif +} + +inline bool cpu_has_i8mm() { +#if defined(__aarch64__) + static std::once_flag once; + static bool has = false; + + std::call_once(once, []() { +#if defined(__APPLE__) + int ret = 0; + size_t size = sizeof(ret); + if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &ret, &size, nullptr, 0) == 0) { + has = (ret == 1); + } +#elif defined(__ANDROID__) + unsigned long hwcap2 = getauxval(AT_HWCAP2); + #ifndef HWCAP2_I8MM + #define HWCAP2_I8MM (1 << 13) + #endif + has = (hwcap2 & HWCAP2_I8MM) != 0; +#endif + }); + + return has; +#else + return false; +#endif +} + +inline float32x4_t fast_exp_f32x4(float32x4_t x) { + const float32x4_t log2e = vdupq_n_f32(1.4426950408889634f); + const float32x4_t ln2 = vdupq_n_f32(0.6931471805599453f); + + const float32x4_t c0 = vdupq_n_f32(1.0f); + const float32x4_t c1 = vdupq_n_f32(0.6931471805599453f); + const float32x4_t c2 = vdupq_n_f32(0.2402265069591007f); + const float32x4_t c3 = vdupq_n_f32(0.05550410866482158f); + const float32x4_t c4 = vdupq_n_f32(0.009618129842071803f); + + x = vmaxq_f32(x, vdupq_n_f32(-87.0f)); + x = vminq_f32(x, vdupq_n_f32(87.0f)); + + float32x4_t z = vmulq_f32(x, log2e); + + int32x4_t zi = vcvtq_s32_f32(z); + float32x4_t zf = vsubq_f32(z, vcvtq_f32_s32(zi)); + + uint32x4_t neg_mask = vcltq_f32(zf, vdupq_n_f32(0.0f)); + zi = vsubq_s32(zi, vandq_s32(vreinterpretq_s32_u32(neg_mask), vdupq_n_s32(1))); + zf = vaddq_f32(zf, vreinterpretq_f32_u32(vandq_u32(neg_mask, vreinterpretq_u32_f32(vdupq_n_f32(1.0f))))); + + float32x4_t zf_ln2 = vmulq_f32(zf, ln2); + float32x4_t p = c4; + p = vfmaq_f32(c3, p, zf_ln2); + p = vfmaq_f32(c2, p, zf_ln2); + p = vfmaq_f32(c1, p, zf_ln2); + p = vfmaq_f32(c0, p, zf_ln2); + + int32x4_t exp_bits = vshlq_n_s32(vaddq_s32(zi, vdupq_n_s32(127)), 23); + float32x4_t scale = vreinterpretq_f32_s32(exp_bits); + + return vmulq_f32(p, scale); +} + +// Cephes-style 13/6 rational tanh approximation (same coefficients as Eigen). +// Constants are stored as static splatted arrays so the compiler emits a single +// pc-relative `ldr q` per load. +alignas(16) inline constexpr float kFastTanhAlpha[7][4] = { + { 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f }, + { 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f }, + { 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f }, + { 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f }, + {-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f }, + { 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f }, + {-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f }, +}; +alignas(16) inline constexpr float kFastTanhBeta[4][4] = { + { 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f }, + { 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f }, + { 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f }, + { 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f }, +}; +alignas(16) inline constexpr float kFastTanhClampHi[4] = { 9.0f, 9.0f, 9.0f, 9.0f }; +alignas(16) inline constexpr float kFastTanhClampLo[4] = {-9.0f,-9.0f,-9.0f,-9.0f }; + +inline float32x4_t fast_tanh_f32x4(float32x4_t x) { + x = vmaxq_f32(vld1q_f32(kFastTanhClampLo), vminq_f32(vld1q_f32(kFastTanhClampHi), x)); + float32x4_t x2 = vmulq_f32(x, x); + float32x4_t p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[5]), vld1q_f32(kFastTanhAlpha[6]), x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[4]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[3]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[2]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[1]), p, x2); + p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[0]), p, x2); + p = vmulq_f32(p, x); + float32x4_t q = vfmaq_f32(vld1q_f32(kFastTanhBeta[2]), vld1q_f32(kFastTanhBeta[3]), x2); + q = vfmaq_f32(vld1q_f32(kFastTanhBeta[1]), q, x2); + q = vfmaq_f32(vld1q_f32(kFastTanhBeta[0]), q, x2); + return vdivq_f32(p, q); +} + +constexpr size_t SIMD_F16_WIDTH = 8; + +inline size_t simd_align(size_t count, size_t width = SIMD_F16_WIDTH) { + return (count / width) * width; +} + +inline void f16x8_split_f32(float16x8_t v, float32x4_t& lo, float32x4_t& hi) { + lo = vcvt_f32_f16(vget_low_f16(v)); + hi = vcvt_f32_f16(vget_high_f16(v)); +} + +inline float16x8_t f32_merge_f16(float32x4_t lo, float32x4_t hi) { + return vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi)); +} + +inline float32x4_t fast_sigmoid_f32x4(float32x4_t x) { + const float32x4_t one = vdupq_n_f32(1.0f); + return vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(x)))); +} + +template +inline float16x8_t apply_f32_op_on_f16x8(float16x8_t v, F32x4Op op) { + float32x4_t lo, hi; + f16x8_split_f32(v, lo, hi); + return f32_merge_f16(op(lo), op(hi)); +} + +inline void unpack_int4_as_int8x16x2(const uint8_t* ptr, int8x16_t& high_decoded, int8x16_t& low_decoded) { + int8x16_t packed = vreinterpretq_s8_u8(vld1q_u8(ptr)); + high_decoded = vshrq_n_s8(packed, 4); + low_decoded = vshrq_n_s8(vshlq_n_s8(packed, 4), 4); +} + +namespace CactusThreading { + +#if defined(__ANDROID__) + struct CoreTopology { + std::vector performance_cores; + std::vector all_cores; + + static CoreTopology& get() { + static CoreTopology topo = detect(); + return topo; + } + + private: + static int read_sysfs_int(const char* path) { + std::ifstream f(path); + if (!f.is_open()) return -1; + int val = -1; + f >> val; + return val; + } + + static CoreTopology detect() { + CoreTopology topo; + constexpr int MAX_CPUS = 16; + std::vector> core_caps; + + for (int i = 0; i < MAX_CPUS; ++i) { + char path[128]; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpu_capacity", i); + int cap = read_sysfs_int(path); + if (cap > 0) { + core_caps.push_back({i, cap}); + topo.all_cores.push_back(i); + continue; + } + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", i); + int freq = read_sysfs_int(path); + if (freq > 0) { + core_caps.push_back({i, freq}); + topo.all_cores.push_back(i); + } + } + + if (core_caps.empty()) return topo; + + int max_cap = 0; + for (auto& [id, cap] : core_caps) { + max_cap = std::max(max_cap, cap); + } + + int threshold = static_cast(max_cap * 0.70); + for (auto& [id, cap] : core_caps) { + if (cap >= threshold) { + topo.performance_cores.push_back(id); + } + } + + return topo; + } + }; + + inline bool pin_current_thread_to_cores(const std::vector& cores) { + if (cores.empty()) return false; + cpu_set_t mask; + CPU_ZERO(&mask); + for (int core : cores) { + CPU_SET(core, &mask); + } + return sched_setaffinity(0, sizeof(mask), &mask) == 0; + } +#endif + + class ThreadPool { + private: + static constexpr size_t MAX_WORKERS = 16; + + std::vector workers; + std::deque> tasks; + + std::mutex mutex; + std::condition_variable work_available; + std::condition_variable work_done; + + bool stop{false}; + std::atomic pending_tasks{0}; + size_t num_workers_; + + void worker_thread() { + while (true) { + std::function task; + { + std::unique_lock lock(mutex); + work_available.wait(lock, [this] { + return stop || !tasks.empty(); + }); + + if (stop && tasks.empty()) { + return; + } + + task = std::move(tasks.front()); + tasks.pop_front(); + } + + task(); + + if (pending_tasks.fetch_sub(1, std::memory_order_acq_rel) == 1) { + std::lock_guard lock(mutex); + work_done.notify_one(); + } + } + } + + public: + explicit ThreadPool(size_t num_threads = std::thread::hardware_concurrency()) + : stop(false), pending_tasks(0) { + num_workers_ = std::min(num_threads, MAX_WORKERS); + if (num_workers_ == 0) num_workers_ = 1; + +#if defined(__ANDROID__) + auto& topo = CoreTopology::get(); + if (!topo.performance_cores.empty()) { + num_workers_ = std::min(num_workers_, topo.performance_cores.size()); + } +#endif + + workers.reserve(num_workers_); + for (size_t i = 0; i < num_workers_; ++i) { + workers.emplace_back([this]() { +#if defined(__ANDROID__) + auto& perf = CoreTopology::get().performance_cores; + if (!perf.empty()) { + pin_current_thread_to_cores(perf); + } +#endif + worker_thread(); + }); + } + } + + ~ThreadPool() { + { + std::lock_guard lock(mutex); + stop = true; + } + work_available.notify_all(); + for (auto& worker : workers) { + if (worker.joinable()) { + worker.join(); + } + } + } + + template + auto enqueue(F&& f) -> std::future { + using return_type = decltype(f()); + + auto task = std::make_shared>( + std::forward(f) + ); + + std::future res = task->get_future(); + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(1, std::memory_order_relaxed); + tasks.emplace_back([task](){ (*task)(); }); + } + work_available.notify_one(); + + return res; + } + + template + void enqueue_batch(size_t total_work, F task_func) { + if (total_work == 0) return; + + const size_t num_tasks = std::min(num_workers_, total_work); + const size_t per_worker = total_work / num_tasks; + const size_t remainder = total_work % num_tasks; + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(num_tasks, std::memory_order_relaxed); + + for (size_t w = 0; w < num_tasks; ++w) { + size_t start = w * per_worker + std::min(w, remainder); + size_t end = start + per_worker + (w < remainder ? 1 : 0); + tasks.emplace_back([=]() { task_func(start, end); }); + } + } + work_available.notify_all(); + } + + void wait_all() { + std::unique_lock lock(mutex); + work_done.wait(lock, [this] { + return pending_tasks.load(std::memory_order_acquire) == 0; + }); + } + + template + void enqueue_n_threads(size_t total_work, size_t num_threads, F task_func) { + if (total_work == 0 || num_threads == 0) return; + + num_threads = std::min(num_threads, std::min(num_workers_, total_work)); + const size_t per_thread = total_work / num_threads; + const size_t remainder = total_work % num_threads; + + { + std::lock_guard lock(mutex); + pending_tasks.fetch_add(num_threads, std::memory_order_relaxed); + + for (size_t t = 0; t < num_threads; ++t) { + size_t start = t * per_thread + std::min(t, remainder); + size_t end = start + per_thread + (t < remainder ? 1 : 0); + tasks.emplace_back([=]() { task_func(start, end); }); + } + } + work_available.notify_all(); + } + + size_t num_workers() const { return num_workers_; } + }; + + inline ThreadPool& get_thread_pool() { + static ThreadPool pool; + return pool; + } + + struct ParallelConfig { + size_t min_work_gate; + size_t work_per_thread; + + constexpr ParallelConfig(size_t gate, size_t per_thread) + : min_work_gate(gate), work_per_thread(per_thread) {} + }; + + inline size_t get_optimal_thread_count(size_t total_work, ParallelConfig config) { + if (total_work < config.min_work_gate) return 1; + + size_t pool_size = get_thread_pool().num_workers(); + size_t num_threads = (total_work + config.work_per_thread - 1) / config.work_per_thread; + return std::min(pool_size, std::max(static_cast(1), num_threads)); + } + + struct Thresholds { + #if defined(__ANDROID__) + static constexpr ParallelConfig ATTENTION{64, 32}; + static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; + static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; + static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; + static constexpr ParallelConfig SCALAR_BASIC{30000, 15000}; + static constexpr ParallelConfig SCALAR_EXPENSIVE{10000, 5000}; + #else // Apple + static constexpr ParallelConfig ATTENTION{32, 16}; + static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; + static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; + static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; + static constexpr ParallelConfig SCALAR_BASIC{5000, 2500}; + static constexpr ParallelConfig SCALAR_EXPENSIVE{2500, 1250}; + #endif + }; + + struct GemmThreading { + #if defined(__ANDROID__) + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return 1; + return pool_size; + } + static size_t get_gemv_threads(size_t /*N_blocks*/, size_t /*pool_size*/) { + return 1; + } + #elif defined(__APPLE__) && TARGET_OS_IPHONE + static constexpr size_t GEMV_MIN_N_BLOCKS = 512; + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return std::min(pool_size, static_cast(2)); + return pool_size; + } + static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { + if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; + return std::min(pool_size, static_cast(3)); + } + #else + static constexpr size_t GEMV_MIN_N_BLOCKS = 256; + static size_t get_num_threads(size_t M, size_t pool_size) { + if (M <= 1) return std::min(pool_size, static_cast(4)); + return pool_size; + } + static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { + if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; + if (N_blocks < 512) return std::min(pool_size, static_cast(2)); + return std::min(pool_size, static_cast(5)); + } + #endif + }; + + inline size_t& get_gemm_thread_override() { + static size_t override_threads = 0; + return override_threads; + } + + inline void set_gemm_threads(size_t num_threads) { + get_gemm_thread_override() = num_threads; + } + + inline void reset_gemm_threads() { + get_gemm_thread_override() = 0; + } + + class TaskHandle { + private: + std::vector> futures_; + bool auto_wait_; + + public: + TaskHandle(bool auto_wait = true) : auto_wait_(auto_wait) {} + + ~TaskHandle() { + if (auto_wait_) { + wait(); + } + } + + TaskHandle(TaskHandle&&) = default; + TaskHandle& operator=(TaskHandle&&) = default; + TaskHandle(const TaskHandle&) = delete; + TaskHandle& operator=(const TaskHandle&) = delete; + + void add_future(std::future&& f) { + futures_.push_back(std::move(f)); + } + + void wait() { + for (auto& f : futures_) { + if (f.valid()) { + f.wait(); + } + } + futures_.clear(); + } + + bool is_ready() const { + for (const auto& f : futures_) { + if (f.valid() && f.wait_for(std::chrono::seconds(0)) != std::future_status::ready) { + return false; + } + } + return true; + } + + size_t task_count() const { return futures_.size(); } + }; + + template + TaskHandle parallel_for(size_t total_work, ParallelConfig config, WorkFunc work_func, bool wait = true) { + const size_t num_threads = get_optimal_thread_count(total_work, config); + TaskHandle handle(!wait); + + if (num_threads == 1) { + if (wait) { + work_func(0, total_work); + return handle; + } + auto& pool = get_thread_pool(); + handle.add_future(pool.enqueue([work_func, total_work]() { + work_func(0, total_work); + })); + return handle; + } + + auto& pool = get_thread_pool(); + const size_t work_per_thread = total_work / num_threads; + + for (size_t t = 0; t < num_threads; ++t) { + handle.add_future(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() { + const size_t start_idx = t * work_per_thread; + const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; + work_func(start_idx, end_idx); + })); + } + + if (wait) { + handle.wait(); + } + return handle; + } + + template + void parallel_for_2d(size_t outer_size, size_t inner_size, ParallelConfig config, WorkFunc work_func) { + const size_t total_work = outer_size * inner_size; + parallel_for(total_work, config, [&](size_t start_idx, size_t end_idx) { + for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { + const size_t outer = work_idx / inner_size; + const size_t inner = work_idx % inner_size; + work_func(outer, inner); + } + }); + } + + template + ResultType parallel_reduce(size_t total_work, ParallelConfig config, + WorkFunc work_func, ResultType init_value, CombineFunc combine_func) { + const size_t num_threads = get_optimal_thread_count(total_work, config); + + if (num_threads == 1) { + return work_func(0, total_work); + } + + auto& pool = get_thread_pool(); + std::vector> futures; + std::vector partial_results(num_threads, init_value); + const size_t work_per_thread = total_work / num_threads; + + for (size_t t = 0; t < num_threads; ++t) { + futures.push_back(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() -> ResultType { + const size_t start_idx = t * work_per_thread; + const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; + return work_func(start_idx, end_idx); + })); + } + + ResultType result = init_value; + for (auto& future : futures) { + result = combine_func(result, future.get()); + } + return result; + } + + template + void parallel_gemm_tiles(size_t M, size_t total_tiles, WorkFunc work_func) { + auto& pool = get_thread_pool(); + + size_t override = get_gemm_thread_override(); + size_t num_threads = (override > 0) ? override : GemmThreading::get_num_threads(M, pool.num_workers()); + num_threads = std::min(num_threads, total_tiles); + + if (num_threads <= 1) { + work_func(0, total_tiles); + return; + } + + pool.enqueue_n_threads(total_tiles, num_threads, work_func); + pool.wait_all(); + } + +} + +template +void elementwise_op_f16(const __fp16* input, __fp16* output, size_t num_elements, + bool use_streaming, CactusThreading::ParallelConfig config, + SimdOp simd_op, ScalarOp scalar_op, size_t unroll = 4) { + CactusThreading::parallel_for(num_elements, config, + [&](size_t start, size_t end) { + const size_t n = end - start; + const size_t vec_end = start + simd_align(n); + + if (use_streaming && unroll >= 4) { + const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 4); + for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 4) { + __builtin_prefetch(&input[i + 256], 0, 0); + float16x8_t v0 = simd_op(vld1q_f16(&input[i])); + float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); + float16x8_t v2 = simd_op(vld1q_f16(&input[i + 16])); + float16x8_t v3 = simd_op(vld1q_f16(&input[i + 24])); + stream_store_f16x8(&output[i], v0); + stream_store_f16x8(&output[i + 8], v1); + stream_store_f16x8(&output[i + 16], v2); + stream_store_f16x8(&output[i + 24], v3); + } + for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { + stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } else if (use_streaming && unroll >= 2) { + const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 2); + for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 2) { + __builtin_prefetch(&input[i + 128], 0, 0); + float16x8_t v0 = simd_op(vld1q_f16(&input[i])); + float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); + stream_store_f16x8(&output[i], v0); + stream_store_f16x8(&output[i + 8], v1); + } + for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { + stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } else { + for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { + vst1q_f16(&output[i], simd_op(vld1q_f16(&input[i]))); + } + } + for (size_t i = vec_end; i < end; ++i) { + output[i] = scalar_op(input[i]); + } + }); +} + +#endif // KERNEL_UTILS_H diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/needle_tools.h b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/needle_tools.h new file mode 100644 index 0000000..4330cff --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Headers/needle_tools.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace needle { + +inline std::string to_snake_case(const std::string& name) { + std::string s; + s.reserve(name.size() * 2); + for (char c : name) { + if (std::isalnum(static_cast(c)) || c == '_') { + s += c; + } else { + if (s.empty() || s.back() != '_') s += '_'; + } + } + + std::string s2; + s2.reserve(s.size() * 2); + for (size_t i = 0; i < s.size(); i++) { + char c = s[i]; + if (i > 0 && std::isupper(static_cast(c))) { + char prev = s[i - 1]; + if (std::islower(static_cast(prev)) || std::isdigit(static_cast(prev))) { + s2 += '_'; + } + } + s2 += c; + } + + std::string s3; + s3.reserve(s2.size() * 2); + for (size_t i = 0; i < s2.size(); i++) { + s3 += s2[i]; + if (i + 1 < s2.size() && + std::isupper(static_cast(s2[i])) && + std::isupper(static_cast(s2[i + 1]))) { + if (i + 2 < s2.size() && std::islower(static_cast(s2[i + 2]))) { + s3 += '_'; + } + } + } + + std::string result; + result.reserve(s3.size()); + bool prev_underscore = false; + for (char c : s3) { + if (c == '_') { + if (!prev_underscore) result += '_'; + prev_underscore = true; + } else { + result += static_cast(std::tolower(static_cast(c))); + prev_underscore = false; + } + } + + size_t start = result.find_first_not_of('_'); + if (start == std::string::npos) return result; + size_t end = result.find_last_not_of('_'); + return result.substr(start, end - start + 1); +} + +inline void restore_tool_names(std::vector& function_calls, + const std::unordered_map& name_map) { + if (name_map.empty()) return; + for (auto& call : function_calls) { + for (const auto& [snake, orig] : name_map) { + std::string from = "\"name\":\"" + snake + "\""; + size_t pos = call.find(from); + if (pos == std::string::npos) { + from = "\"name\": \"" + snake + "\""; + pos = call.find(from); + } + if (pos != std::string::npos) { + std::string to = from.substr(0, from.size() - snake.size() - 1) + orig + "\""; + call.replace(pos, from.size(), to); + break; + } + } + } +} + +} // namespace needle diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Resources/Info.plist b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Resources/Info.plist new file mode 100644 index 0000000..93d77f9 --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Resources/Info.plist @@ -0,0 +1,48 @@ + + + + + BuildMachineOSBuild + 25D2128 + CFBundleDevelopmentRegion + English + CFBundleExecutable + cactus + CFBundleIdentifier + com.cactuscompute.cactus + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + cactus + CFBundlePackageType + FMWK + CFBundleShortVersionString + 1.0.0 + CFBundleSupportedPlatforms + + MacOSX + + CFBundleVersion + 1.0.0 + DTCompiler + com.apple.compilers.llvm.clang.1_0 + DTPlatformBuild + 25F70 + DTPlatformName + macosx + DTPlatformVersion + 26.5 + DTSDKBuild + 25F70 + DTSDKName + macosx26.5 + DTXcode + 2650 + DTXcodeBuild + 17F42 + LSMinimumSystemVersion + 13.0 + MinimumOSVersion + 12.0 + + diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/A/Resources/Info.plist b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/A/Resources/Info.plist new file mode 100644 index 0000000..93d77f9 --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/A/Resources/Info.plist @@ -0,0 +1,48 @@ + + + + + BuildMachineOSBuild + 25D2128 + CFBundleDevelopmentRegion + English + CFBundleExecutable + cactus + CFBundleIdentifier + com.cactuscompute.cactus + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + cactus + CFBundlePackageType + FMWK + CFBundleShortVersionString + 1.0.0 + CFBundleSupportedPlatforms + + MacOSX + + CFBundleVersion + 1.0.0 + DTCompiler + com.apple.compilers.llvm.clang.1_0 + DTPlatformBuild + 25F70 + DTPlatformName + macosx + DTPlatformVersion + 26.5 + DTSDKBuild + 25F70 + DTSDKName + macosx26.5 + DTXcode + 2650 + DTXcodeBuild + 17F42 + LSMinimumSystemVersion + 13.0 + MinimumOSVersion + 12.0 + + diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/A/_CodeSignature/CodeResources b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/A/_CodeSignature/CodeResources new file mode 100644 index 0000000..1509aa7 --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/A/_CodeSignature/CodeResources @@ -0,0 +1,128 @@ + + + + + files + + Resources/Info.plist + + 9QxDBnhZ0vLRn1uYrizq051dLW4= + + + files2 + + Resources/Info.plist + + hash2 + + u48mqQipQ7T8DKDuELQF3Zzn3Y/Vtl3kDfEYvVWo6ig= + + + + rules + + ^Resources/ + + ^Resources/.*\.lproj/ + + optional + + weight + 1000 + + ^Resources/.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Resources/Base\.lproj/ + + weight + 1010 + + ^version.plist$ + + + rules2 + + .*\.dSYM($|/) + + weight + 11 + + ^(.*/)?\.DS_Store$ + + omit + + weight + 2000 + + ^(Frameworks|SharedFrameworks|PlugIns|Plug-ins|XPCServices|Helpers|MacOS|Library/(Automator|Spotlight|LoginItems))/ + + nested + + weight + 10 + + ^.* + + ^Info\.plist$ + + omit + + weight + 20 + + ^PkgInfo$ + + omit + + weight + 20 + + ^Resources/ + + weight + 20 + + ^Resources/.*\.lproj/ + + optional + + weight + 1000 + + ^Resources/.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Resources/Base\.lproj/ + + weight + 1010 + + ^[^/]+$ + + nested + + weight + 10 + + ^embedded\.provisionprofile$ + + weight + 20 + + ^version\.plist$ + + weight + 20 + + + + diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/A/cactus b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/A/cactus new file mode 100755 index 0000000..cc4b7b7 Binary files /dev/null and b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/A/cactus differ diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/Current/Resources/Info.plist b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/Current/Resources/Info.plist new file mode 100644 index 0000000..93d77f9 --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/Current/Resources/Info.plist @@ -0,0 +1,48 @@ + + + + + BuildMachineOSBuild + 25D2128 + CFBundleDevelopmentRegion + English + CFBundleExecutable + cactus + CFBundleIdentifier + com.cactuscompute.cactus + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + cactus + CFBundlePackageType + FMWK + CFBundleShortVersionString + 1.0.0 + CFBundleSupportedPlatforms + + MacOSX + + CFBundleVersion + 1.0.0 + DTCompiler + com.apple.compilers.llvm.clang.1_0 + DTPlatformBuild + 25F70 + DTPlatformName + macosx + DTPlatformVersion + 26.5 + DTSDKBuild + 25F70 + DTSDKName + macosx26.5 + DTXcode + 2650 + DTXcodeBuild + 17F42 + LSMinimumSystemVersion + 13.0 + MinimumOSVersion + 12.0 + + diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/Current/_CodeSignature/CodeResources b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/Current/_CodeSignature/CodeResources new file mode 100644 index 0000000..1509aa7 --- /dev/null +++ b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/Current/_CodeSignature/CodeResources @@ -0,0 +1,128 @@ + + + + + files + + Resources/Info.plist + + 9QxDBnhZ0vLRn1uYrizq051dLW4= + + + files2 + + Resources/Info.plist + + hash2 + + u48mqQipQ7T8DKDuELQF3Zzn3Y/Vtl3kDfEYvVWo6ig= + + + + rules + + ^Resources/ + + ^Resources/.*\.lproj/ + + optional + + weight + 1000 + + ^Resources/.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Resources/Base\.lproj/ + + weight + 1010 + + ^version.plist$ + + + rules2 + + .*\.dSYM($|/) + + weight + 11 + + ^(.*/)?\.DS_Store$ + + omit + + weight + 2000 + + ^(Frameworks|SharedFrameworks|PlugIns|Plug-ins|XPCServices|Helpers|MacOS|Library/(Automator|Spotlight|LoginItems))/ + + nested + + weight + 10 + + ^.* + + ^Info\.plist$ + + omit + + weight + 20 + + ^PkgInfo$ + + omit + + weight + 20 + + ^Resources/ + + weight + 20 + + ^Resources/.*\.lproj/ + + optional + + weight + 1000 + + ^Resources/.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Resources/Base\.lproj/ + + weight + 1010 + + ^[^/]+$ + + nested + + weight + 10 + + ^embedded\.provisionprofile$ + + weight + 20 + + ^version\.plist$ + + weight + 20 + + + + diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/Current/cactus b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/Current/cactus new file mode 100755 index 0000000..cc4b7b7 Binary files /dev/null and b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/Versions/Current/cactus differ diff --git a/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/cactus b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/cactus new file mode 100755 index 0000000..cc4b7b7 Binary files /dev/null and b/macos/cactus-macos.xcframework/macos-arm64/cactus.framework/cactus differ diff --git a/web/favicon.png b/web/favicon.png new file mode 100644 index 0000000..8aaa46a Binary files /dev/null and b/web/favicon.png differ diff --git a/web/icons/Icon-192.png b/web/icons/Icon-192.png new file mode 100644 index 0000000..b749bfe Binary files /dev/null and b/web/icons/Icon-192.png differ diff --git a/web/icons/Icon-512.png b/web/icons/Icon-512.png new file mode 100644 index 0000000..88cfd48 Binary files /dev/null and b/web/icons/Icon-512.png differ diff --git a/web/icons/Icon-maskable-192.png b/web/icons/Icon-maskable-192.png new file mode 100644 index 0000000..eb9b4d7 Binary files /dev/null and b/web/icons/Icon-maskable-192.png differ diff --git a/web/icons/Icon-maskable-512.png b/web/icons/Icon-maskable-512.png new file mode 100644 index 0000000..d69c566 Binary files /dev/null and b/web/icons/Icon-maskable-512.png differ diff --git a/web/index.html b/web/index.html new file mode 100644 index 0000000..5e57212 --- /dev/null +++ b/web/index.html @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + sunflower_app + + + + + + + diff --git a/web/manifest.json b/web/manifest.json new file mode 100644 index 0000000..d0d1dc7 --- /dev/null +++ b/web/manifest.json @@ -0,0 +1,35 @@ +{ + "name": "sunflower_app", + "short_name": "sunflower_app", + "start_url": ".", + "display": "standalone", + "background_color": "#0175C2", + "theme_color": "#0175C2", + "description": "A new Flutter project.", + "orientation": "portrait-primary", + "prefer_related_applications": false, + "icons": [ + { + "src": "icons/Icon-192.png", + "sizes": "192x192", + "type": "image/png" + }, + { + "src": "icons/Icon-512.png", + "sizes": "512x512", + "type": "image/png" + }, + { + "src": "icons/Icon-maskable-192.png", + "sizes": "192x192", + "type": "image/png", + "purpose": "maskable" + }, + { + "src": "icons/Icon-maskable-512.png", + "sizes": "512x512", + "type": "image/png", + "purpose": "maskable" + } + ] +} diff --git a/windows/.gitignore b/windows/.gitignore new file mode 100644 index 0000000..d492d0d --- /dev/null +++ b/windows/.gitignore @@ -0,0 +1,17 @@ +flutter/ephemeral/ + +# Visual Studio user-specific files. +*.suo +*.user +*.userosscache +*.sln.docstates + +# Visual Studio build-related files. +x64/ +x86/ + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ diff --git a/windows/CMakeLists.txt b/windows/CMakeLists.txt new file mode 100644 index 0000000..1a20e04 --- /dev/null +++ b/windows/CMakeLists.txt @@ -0,0 +1,108 @@ +# Project-level configuration. +cmake_minimum_required(VERSION 3.14) +project(sunflower_app LANGUAGES CXX) + +# The name of the executable created for the application. Change this to change +# the on-disk name of your application. +set(BINARY_NAME "sunflower_app") + +# Explicitly opt in to modern CMake behaviors to avoid warnings with recent +# versions of CMake. +cmake_policy(VERSION 3.14...3.25) + +# Define build configuration option. +get_property(IS_MULTICONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) +if(IS_MULTICONFIG) + set(CMAKE_CONFIGURATION_TYPES "Debug;Profile;Release" + CACHE STRING "" FORCE) +else() + if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE "Debug" CACHE + STRING "Flutter build mode" FORCE) + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS + "Debug" "Profile" "Release") + endif() +endif() +# Define settings for the Profile build mode. +set(CMAKE_EXE_LINKER_FLAGS_PROFILE "${CMAKE_EXE_LINKER_FLAGS_RELEASE}") +set(CMAKE_SHARED_LINKER_FLAGS_PROFILE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE}") +set(CMAKE_C_FLAGS_PROFILE "${CMAKE_C_FLAGS_RELEASE}") +set(CMAKE_CXX_FLAGS_PROFILE "${CMAKE_CXX_FLAGS_RELEASE}") + +# Use Unicode for all projects. +add_definitions(-DUNICODE -D_UNICODE) + +# Compilation settings that should be applied to most targets. +# +# Be cautious about adding new options here, as plugins use this function by +# default. In most cases, you should add new options to specific targets instead +# of modifying this function. +function(APPLY_STANDARD_SETTINGS TARGET) + target_compile_features(${TARGET} PUBLIC cxx_std_17) + target_compile_options(${TARGET} PRIVATE /W4 /WX /wd"4100") + target_compile_options(${TARGET} PRIVATE /EHsc) + target_compile_definitions(${TARGET} PRIVATE "_HAS_EXCEPTIONS=0") + target_compile_definitions(${TARGET} PRIVATE "$<$:_DEBUG>") +endfunction() + +# Flutter library and tool build rules. +set(FLUTTER_MANAGED_DIR "${CMAKE_CURRENT_SOURCE_DIR}/flutter") +add_subdirectory(${FLUTTER_MANAGED_DIR}) + +# Application build; see runner/CMakeLists.txt. +add_subdirectory("runner") + + +# Generated plugin build rules, which manage building the plugins and adding +# them to the application. +include(flutter/generated_plugins.cmake) + + +# === Installation === +# Support files are copied into place next to the executable, so that it can +# run in place. This is done instead of making a separate bundle (as on Linux) +# so that building and running from within Visual Studio will work. +set(BUILD_BUNDLE_DIR "$") +# Make the "install" step default, as it's required to run. +set(CMAKE_VS_INCLUDE_INSTALL_TO_DEFAULT_BUILD 1) +if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + set(CMAKE_INSTALL_PREFIX "${BUILD_BUNDLE_DIR}" CACHE PATH "..." FORCE) +endif() + +set(INSTALL_BUNDLE_DATA_DIR "${CMAKE_INSTALL_PREFIX}/data") +set(INSTALL_BUNDLE_LIB_DIR "${CMAKE_INSTALL_PREFIX}") + +install(TARGETS ${BINARY_NAME} RUNTIME DESTINATION "${CMAKE_INSTALL_PREFIX}" + COMPONENT Runtime) + +install(FILES "${FLUTTER_ICU_DATA_FILE}" DESTINATION "${INSTALL_BUNDLE_DATA_DIR}" + COMPONENT Runtime) + +install(FILES "${FLUTTER_LIBRARY}" DESTINATION "${INSTALL_BUNDLE_LIB_DIR}" + COMPONENT Runtime) + +if(PLUGIN_BUNDLED_LIBRARIES) + install(FILES "${PLUGIN_BUNDLED_LIBRARIES}" + DESTINATION "${INSTALL_BUNDLE_LIB_DIR}" + COMPONENT Runtime) +endif() + +# Copy the native assets provided by the build.dart from all packages. +set(NATIVE_ASSETS_DIR "${PROJECT_BUILD_DIR}native_assets/windows/") +install(DIRECTORY "${NATIVE_ASSETS_DIR}" + DESTINATION "${INSTALL_BUNDLE_LIB_DIR}" + COMPONENT Runtime) + +# Fully re-copy the assets directory on each build to avoid having stale files +# from a previous install. +set(FLUTTER_ASSET_DIR_NAME "flutter_assets") +install(CODE " + file(REMOVE_RECURSE \"${INSTALL_BUNDLE_DATA_DIR}/${FLUTTER_ASSET_DIR_NAME}\") + " COMPONENT Runtime) +install(DIRECTORY "${PROJECT_BUILD_DIR}/${FLUTTER_ASSET_DIR_NAME}" + DESTINATION "${INSTALL_BUNDLE_DATA_DIR}" COMPONENT Runtime) + +# Install the AOT library on non-Debug builds only. +install(FILES "${AOT_LIBRARY}" DESTINATION "${INSTALL_BUNDLE_DATA_DIR}" + CONFIGURATIONS Profile;Release + COMPONENT Runtime) diff --git a/windows/flutter/CMakeLists.txt b/windows/flutter/CMakeLists.txt new file mode 100644 index 0000000..903f489 --- /dev/null +++ b/windows/flutter/CMakeLists.txt @@ -0,0 +1,109 @@ +# This file controls Flutter-level build steps. It should not be edited. +cmake_minimum_required(VERSION 3.14) + +set(EPHEMERAL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ephemeral") + +# Configuration provided via flutter tool. +include(${EPHEMERAL_DIR}/generated_config.cmake) + +# TODO: Move the rest of this into files in ephemeral. See +# https://github.com/flutter/flutter/issues/57146. +set(WRAPPER_ROOT "${EPHEMERAL_DIR}/cpp_client_wrapper") + +# Set fallback configurations for older versions of the flutter tool. +if (NOT DEFINED FLUTTER_TARGET_PLATFORM) + set(FLUTTER_TARGET_PLATFORM "windows-x64") +endif() + +# === Flutter Library === +set(FLUTTER_LIBRARY "${EPHEMERAL_DIR}/flutter_windows.dll") + +# Published to parent scope for install step. +set(FLUTTER_LIBRARY ${FLUTTER_LIBRARY} PARENT_SCOPE) +set(FLUTTER_ICU_DATA_FILE "${EPHEMERAL_DIR}/icudtl.dat" PARENT_SCOPE) +set(PROJECT_BUILD_DIR "${PROJECT_DIR}/build/" PARENT_SCOPE) +set(AOT_LIBRARY "${PROJECT_DIR}/build/windows/app.so" PARENT_SCOPE) + +list(APPEND FLUTTER_LIBRARY_HEADERS + "flutter_export.h" + "flutter_windows.h" + "flutter_messenger.h" + "flutter_plugin_registrar.h" + "flutter_texture_registrar.h" +) +list(TRANSFORM FLUTTER_LIBRARY_HEADERS PREPEND "${EPHEMERAL_DIR}/") +add_library(flutter INTERFACE) +target_include_directories(flutter INTERFACE + "${EPHEMERAL_DIR}" +) +target_link_libraries(flutter INTERFACE "${FLUTTER_LIBRARY}.lib") +add_dependencies(flutter flutter_assemble) + +# === Wrapper === +list(APPEND CPP_WRAPPER_SOURCES_CORE + "core_implementations.cc" + "standard_codec.cc" +) +list(TRANSFORM CPP_WRAPPER_SOURCES_CORE PREPEND "${WRAPPER_ROOT}/") +list(APPEND CPP_WRAPPER_SOURCES_PLUGIN + "plugin_registrar.cc" +) +list(TRANSFORM CPP_WRAPPER_SOURCES_PLUGIN PREPEND "${WRAPPER_ROOT}/") +list(APPEND CPP_WRAPPER_SOURCES_APP + "flutter_engine.cc" + "flutter_view_controller.cc" +) +list(TRANSFORM CPP_WRAPPER_SOURCES_APP PREPEND "${WRAPPER_ROOT}/") + +# Wrapper sources needed for a plugin. +add_library(flutter_wrapper_plugin STATIC + ${CPP_WRAPPER_SOURCES_CORE} + ${CPP_WRAPPER_SOURCES_PLUGIN} +) +apply_standard_settings(flutter_wrapper_plugin) +set_target_properties(flutter_wrapper_plugin PROPERTIES + POSITION_INDEPENDENT_CODE ON) +set_target_properties(flutter_wrapper_plugin PROPERTIES + CXX_VISIBILITY_PRESET hidden) +target_link_libraries(flutter_wrapper_plugin PUBLIC flutter) +target_include_directories(flutter_wrapper_plugin PUBLIC + "${WRAPPER_ROOT}/include" +) +add_dependencies(flutter_wrapper_plugin flutter_assemble) + +# Wrapper sources needed for the runner. +add_library(flutter_wrapper_app STATIC + ${CPP_WRAPPER_SOURCES_CORE} + ${CPP_WRAPPER_SOURCES_APP} +) +apply_standard_settings(flutter_wrapper_app) +target_link_libraries(flutter_wrapper_app PUBLIC flutter) +target_include_directories(flutter_wrapper_app PUBLIC + "${WRAPPER_ROOT}/include" +) +add_dependencies(flutter_wrapper_app flutter_assemble) + +# === Flutter tool backend === +# _phony_ is a non-existent file to force this command to run every time, +# since currently there's no way to get a full input/output list from the +# flutter tool. +set(PHONY_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/_phony_") +set_source_files_properties("${PHONY_OUTPUT}" PROPERTIES SYMBOLIC TRUE) +add_custom_command( + OUTPUT ${FLUTTER_LIBRARY} ${FLUTTER_LIBRARY_HEADERS} + ${CPP_WRAPPER_SOURCES_CORE} ${CPP_WRAPPER_SOURCES_PLUGIN} + ${CPP_WRAPPER_SOURCES_APP} + ${PHONY_OUTPUT} + COMMAND ${CMAKE_COMMAND} -E env + ${FLUTTER_TOOL_ENVIRONMENT} + "${FLUTTER_ROOT}/packages/flutter_tools/bin/tool_backend.bat" + ${FLUTTER_TARGET_PLATFORM} $ + VERBATIM +) +add_custom_target(flutter_assemble DEPENDS + "${FLUTTER_LIBRARY}" + ${FLUTTER_LIBRARY_HEADERS} + ${CPP_WRAPPER_SOURCES_CORE} + ${CPP_WRAPPER_SOURCES_PLUGIN} + ${CPP_WRAPPER_SOURCES_APP} +) diff --git a/windows/flutter/generated_plugin_registrant.cc b/windows/flutter/generated_plugin_registrant.cc new file mode 100644 index 0000000..6d1ca7e --- /dev/null +++ b/windows/flutter/generated_plugin_registrant.cc @@ -0,0 +1,20 @@ +// +// Generated file. Do not edit. +// + +// clang-format off + +#include "generated_plugin_registrant.h" + +#include +#include +#include + +void RegisterPlugins(flutter::PluginRegistry* registry) { + AudioplayersWindowsPluginRegisterWithRegistrar( + registry->GetRegistrarForPlugin("AudioplayersWindowsPlugin")); + FlutterOnnxruntimePluginRegisterWithRegistrar( + registry->GetRegistrarForPlugin("FlutterOnnxruntimePlugin")); + RecordWindowsPluginCApiRegisterWithRegistrar( + registry->GetRegistrarForPlugin("RecordWindowsPluginCApi")); +} diff --git a/windows/flutter/generated_plugin_registrant.h b/windows/flutter/generated_plugin_registrant.h new file mode 100644 index 0000000..dc139d8 --- /dev/null +++ b/windows/flutter/generated_plugin_registrant.h @@ -0,0 +1,15 @@ +// +// Generated file. Do not edit. +// + +// clang-format off + +#ifndef GENERATED_PLUGIN_REGISTRANT_ +#define GENERATED_PLUGIN_REGISTRANT_ + +#include + +// Registers Flutter plugins. +void RegisterPlugins(flutter::PluginRegistry* registry); + +#endif // GENERATED_PLUGIN_REGISTRANT_ diff --git a/windows/flutter/generated_plugins.cmake b/windows/flutter/generated_plugins.cmake new file mode 100644 index 0000000..6df67d4 --- /dev/null +++ b/windows/flutter/generated_plugins.cmake @@ -0,0 +1,27 @@ +# +# Generated file, do not edit. +# + +list(APPEND FLUTTER_PLUGIN_LIST + audioplayers_windows + flutter_onnxruntime + record_windows +) + +list(APPEND FLUTTER_FFI_PLUGIN_LIST + jni +) + +set(PLUGIN_BUNDLED_LIBRARIES) + +foreach(plugin ${FLUTTER_PLUGIN_LIST}) + add_subdirectory(flutter/ephemeral/.plugin_symlinks/${plugin}/windows plugins/${plugin}) + target_link_libraries(${BINARY_NAME} PRIVATE ${plugin}_plugin) + list(APPEND PLUGIN_BUNDLED_LIBRARIES $) + list(APPEND PLUGIN_BUNDLED_LIBRARIES ${${plugin}_bundled_libraries}) +endforeach(plugin) + +foreach(ffi_plugin ${FLUTTER_FFI_PLUGIN_LIST}) + add_subdirectory(flutter/ephemeral/.plugin_symlinks/${ffi_plugin}/windows plugins/${ffi_plugin}) + list(APPEND PLUGIN_BUNDLED_LIBRARIES ${${ffi_plugin}_bundled_libraries}) +endforeach(ffi_plugin) diff --git a/windows/runner/CMakeLists.txt b/windows/runner/CMakeLists.txt new file mode 100644 index 0000000..394917c --- /dev/null +++ b/windows/runner/CMakeLists.txt @@ -0,0 +1,40 @@ +cmake_minimum_required(VERSION 3.14) +project(runner LANGUAGES CXX) + +# Define the application target. To change its name, change BINARY_NAME in the +# top-level CMakeLists.txt, not the value here, or `flutter run` will no longer +# work. +# +# Any new source files that you add to the application should be added here. +add_executable(${BINARY_NAME} WIN32 + "flutter_window.cpp" + "main.cpp" + "utils.cpp" + "win32_window.cpp" + "${FLUTTER_MANAGED_DIR}/generated_plugin_registrant.cc" + "Runner.rc" + "runner.exe.manifest" +) + +# Apply the standard set of build settings. This can be removed for applications +# that need different build settings. +apply_standard_settings(${BINARY_NAME}) + +# Add preprocessor definitions for the build version. +target_compile_definitions(${BINARY_NAME} PRIVATE "FLUTTER_VERSION=\"${FLUTTER_VERSION}\"") +target_compile_definitions(${BINARY_NAME} PRIVATE "FLUTTER_VERSION_MAJOR=${FLUTTER_VERSION_MAJOR}") +target_compile_definitions(${BINARY_NAME} PRIVATE "FLUTTER_VERSION_MINOR=${FLUTTER_VERSION_MINOR}") +target_compile_definitions(${BINARY_NAME} PRIVATE "FLUTTER_VERSION_PATCH=${FLUTTER_VERSION_PATCH}") +target_compile_definitions(${BINARY_NAME} PRIVATE "FLUTTER_VERSION_BUILD=${FLUTTER_VERSION_BUILD}") + +# Disable Windows macros that collide with C++ standard library functions. +target_compile_definitions(${BINARY_NAME} PRIVATE "NOMINMAX") + +# Add dependency libraries and include directories. Add any application-specific +# dependencies here. +target_link_libraries(${BINARY_NAME} PRIVATE flutter flutter_wrapper_app) +target_link_libraries(${BINARY_NAME} PRIVATE "dwmapi.lib") +target_include_directories(${BINARY_NAME} PRIVATE "${CMAKE_SOURCE_DIR}") + +# Run the Flutter tool portions of the build. This must not be removed. +add_dependencies(${BINARY_NAME} flutter_assemble) diff --git a/windows/runner/Runner.rc b/windows/runner/Runner.rc new file mode 100644 index 0000000..8a9db75 --- /dev/null +++ b/windows/runner/Runner.rc @@ -0,0 +1,121 @@ +// Microsoft Visual C++ generated resource script. +// +#pragma code_page(65001) +#include "resource.h" + +#define APSTUDIO_READONLY_SYMBOLS +///////////////////////////////////////////////////////////////////////////// +// +// Generated from the TEXTINCLUDE 2 resource. +// +#include "winres.h" + +///////////////////////////////////////////////////////////////////////////// +#undef APSTUDIO_READONLY_SYMBOLS + +///////////////////////////////////////////////////////////////////////////// +// English (United States) resources + +#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) +LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US + +#ifdef APSTUDIO_INVOKED +///////////////////////////////////////////////////////////////////////////// +// +// TEXTINCLUDE +// + +1 TEXTINCLUDE +BEGIN + "resource.h\0" +END + +2 TEXTINCLUDE +BEGIN + "#include ""winres.h""\r\n" + "\0" +END + +3 TEXTINCLUDE +BEGIN + "\r\n" + "\0" +END + +#endif // APSTUDIO_INVOKED + + +///////////////////////////////////////////////////////////////////////////// +// +// Icon +// + +// Icon with lowest ID value placed first to ensure application icon +// remains consistent on all systems. +IDI_APP_ICON ICON "resources\\app_icon.ico" + + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// + +#if defined(FLUTTER_VERSION_MAJOR) && defined(FLUTTER_VERSION_MINOR) && defined(FLUTTER_VERSION_PATCH) && defined(FLUTTER_VERSION_BUILD) +#define VERSION_AS_NUMBER FLUTTER_VERSION_MAJOR,FLUTTER_VERSION_MINOR,FLUTTER_VERSION_PATCH,FLUTTER_VERSION_BUILD +#else +#define VERSION_AS_NUMBER 1,0,0,0 +#endif + +#if defined(FLUTTER_VERSION) +#define VERSION_AS_STRING FLUTTER_VERSION +#else +#define VERSION_AS_STRING "1.0.0" +#endif + +VS_VERSION_INFO VERSIONINFO + FILEVERSION VERSION_AS_NUMBER + PRODUCTVERSION VERSION_AS_NUMBER + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +#ifdef _DEBUG + FILEFLAGS VS_FF_DEBUG +#else + FILEFLAGS 0x0L +#endif + FILEOS VOS__WINDOWS32 + FILETYPE VFT_APP + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904e4" + BEGIN + VALUE "CompanyName", "ai.sunbird" "\0" + VALUE "FileDescription", "sunflower_app" "\0" + VALUE "FileVersion", VERSION_AS_STRING "\0" + VALUE "InternalName", "sunflower_app" "\0" + VALUE "LegalCopyright", "Copyright (C) 2026 ai.sunbird. All rights reserved." "\0" + VALUE "OriginalFilename", "sunflower_app.exe" "\0" + VALUE "ProductName", "sunflower_app" "\0" + VALUE "ProductVersion", VERSION_AS_STRING "\0" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1252 + END +END + +#endif // English (United States) resources +///////////////////////////////////////////////////////////////////////////// + + + +#ifndef APSTUDIO_INVOKED +///////////////////////////////////////////////////////////////////////////// +// +// Generated from the TEXTINCLUDE 3 resource. +// + + +///////////////////////////////////////////////////////////////////////////// +#endif // not APSTUDIO_INVOKED diff --git a/windows/runner/flutter_window.cpp b/windows/runner/flutter_window.cpp new file mode 100644 index 0000000..955ee30 --- /dev/null +++ b/windows/runner/flutter_window.cpp @@ -0,0 +1,71 @@ +#include "flutter_window.h" + +#include + +#include "flutter/generated_plugin_registrant.h" + +FlutterWindow::FlutterWindow(const flutter::DartProject& project) + : project_(project) {} + +FlutterWindow::~FlutterWindow() {} + +bool FlutterWindow::OnCreate() { + if (!Win32Window::OnCreate()) { + return false; + } + + RECT frame = GetClientArea(); + + // The size here must match the window dimensions to avoid unnecessary surface + // creation / destruction in the startup path. + flutter_controller_ = std::make_unique( + frame.right - frame.left, frame.bottom - frame.top, project_); + // Ensure that basic setup of the controller was successful. + if (!flutter_controller_->engine() || !flutter_controller_->view()) { + return false; + } + RegisterPlugins(flutter_controller_->engine()); + SetChildContent(flutter_controller_->view()->GetNativeWindow()); + + flutter_controller_->engine()->SetNextFrameCallback([&]() { + this->Show(); + }); + + // Flutter can complete the first frame before the "show window" callback is + // registered. The following call ensures a frame is pending to ensure the + // window is shown. It is a no-op if the first frame hasn't completed yet. + flutter_controller_->ForceRedraw(); + + return true; +} + +void FlutterWindow::OnDestroy() { + if (flutter_controller_) { + flutter_controller_ = nullptr; + } + + Win32Window::OnDestroy(); +} + +LRESULT +FlutterWindow::MessageHandler(HWND hwnd, UINT const message, + WPARAM const wparam, + LPARAM const lparam) noexcept { + // Give Flutter, including plugins, an opportunity to handle window messages. + if (flutter_controller_) { + std::optional result = + flutter_controller_->HandleTopLevelWindowProc(hwnd, message, wparam, + lparam); + if (result) { + return *result; + } + } + + switch (message) { + case WM_FONTCHANGE: + flutter_controller_->engine()->ReloadSystemFonts(); + break; + } + + return Win32Window::MessageHandler(hwnd, message, wparam, lparam); +} diff --git a/windows/runner/flutter_window.h b/windows/runner/flutter_window.h new file mode 100644 index 0000000..6da0652 --- /dev/null +++ b/windows/runner/flutter_window.h @@ -0,0 +1,33 @@ +#ifndef RUNNER_FLUTTER_WINDOW_H_ +#define RUNNER_FLUTTER_WINDOW_H_ + +#include +#include + +#include + +#include "win32_window.h" + +// A window that does nothing but host a Flutter view. +class FlutterWindow : public Win32Window { + public: + // Creates a new FlutterWindow hosting a Flutter view running |project|. + explicit FlutterWindow(const flutter::DartProject& project); + virtual ~FlutterWindow(); + + protected: + // Win32Window: + bool OnCreate() override; + void OnDestroy() override; + LRESULT MessageHandler(HWND window, UINT const message, WPARAM const wparam, + LPARAM const lparam) noexcept override; + + private: + // The project to run. + flutter::DartProject project_; + + // The Flutter instance hosted by this window. + std::unique_ptr flutter_controller_; +}; + +#endif // RUNNER_FLUTTER_WINDOW_H_ diff --git a/windows/runner/main.cpp b/windows/runner/main.cpp new file mode 100644 index 0000000..671b4a9 --- /dev/null +++ b/windows/runner/main.cpp @@ -0,0 +1,43 @@ +#include +#include +#include + +#include "flutter_window.h" +#include "utils.h" + +int APIENTRY wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev, + _In_ wchar_t *command_line, _In_ int show_command) { + // Attach to console when present (e.g., 'flutter run') or create a + // new console when running with a debugger. + if (!::AttachConsole(ATTACH_PARENT_PROCESS) && ::IsDebuggerPresent()) { + CreateAndAttachConsole(); + } + + // Initialize COM, so that it is available for use in the library and/or + // plugins. + ::CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED); + + flutter::DartProject project(L"data"); + + std::vector command_line_arguments = + GetCommandLineArguments(); + + project.set_dart_entrypoint_arguments(std::move(command_line_arguments)); + + FlutterWindow window(project); + Win32Window::Point origin(10, 10); + Win32Window::Size size(1280, 720); + if (!window.Create(L"sunflower_app", origin, size)) { + return EXIT_FAILURE; + } + window.SetQuitOnClose(true); + + ::MSG msg; + while (::GetMessage(&msg, nullptr, 0, 0)) { + ::TranslateMessage(&msg); + ::DispatchMessage(&msg); + } + + ::CoUninitialize(); + return EXIT_SUCCESS; +} diff --git a/windows/runner/resource.h b/windows/runner/resource.h new file mode 100644 index 0000000..66a65d1 --- /dev/null +++ b/windows/runner/resource.h @@ -0,0 +1,16 @@ +//{{NO_DEPENDENCIES}} +// Microsoft Visual C++ generated include file. +// Used by Runner.rc +// +#define IDI_APP_ICON 101 + +// Next default values for new objects +// +#ifdef APSTUDIO_INVOKED +#ifndef APSTUDIO_READONLY_SYMBOLS +#define _APS_NEXT_RESOURCE_VALUE 102 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 +#endif +#endif diff --git a/windows/runner/resources/app_icon.ico b/windows/runner/resources/app_icon.ico new file mode 100644 index 0000000..c04e20c Binary files /dev/null and b/windows/runner/resources/app_icon.ico differ diff --git a/windows/runner/runner.exe.manifest b/windows/runner/runner.exe.manifest new file mode 100644 index 0000000..153653e --- /dev/null +++ b/windows/runner/runner.exe.manifest @@ -0,0 +1,14 @@ + + + + + PerMonitorV2 + + + + + + + + + diff --git a/windows/runner/utils.cpp b/windows/runner/utils.cpp new file mode 100644 index 0000000..3a0b465 --- /dev/null +++ b/windows/runner/utils.cpp @@ -0,0 +1,65 @@ +#include "utils.h" + +#include +#include +#include +#include + +#include + +void CreateAndAttachConsole() { + if (::AllocConsole()) { + FILE *unused; + if (freopen_s(&unused, "CONOUT$", "w", stdout)) { + _dup2(_fileno(stdout), 1); + } + if (freopen_s(&unused, "CONOUT$", "w", stderr)) { + _dup2(_fileno(stdout), 2); + } + std::ios::sync_with_stdio(); + FlutterDesktopResyncOutputStreams(); + } +} + +std::vector GetCommandLineArguments() { + // Convert the UTF-16 command line arguments to UTF-8 for the Engine to use. + int argc; + wchar_t** argv = ::CommandLineToArgvW(::GetCommandLineW(), &argc); + if (argv == nullptr) { + return std::vector(); + } + + std::vector command_line_arguments; + + // Skip the first argument as it's the binary name. + for (int i = 1; i < argc; i++) { + command_line_arguments.push_back(Utf8FromUtf16(argv[i])); + } + + ::LocalFree(argv); + + return command_line_arguments; +} + +std::string Utf8FromUtf16(const wchar_t* utf16_string) { + if (utf16_string == nullptr) { + return std::string(); + } + unsigned int target_length = ::WideCharToMultiByte( + CP_UTF8, WC_ERR_INVALID_CHARS, utf16_string, + -1, nullptr, 0, nullptr, nullptr) + -1; // remove the trailing null character + int input_length = (int)wcslen(utf16_string); + std::string utf8_string; + if (target_length == 0 || target_length > utf8_string.max_size()) { + return utf8_string; + } + utf8_string.resize(target_length); + int converted_length = ::WideCharToMultiByte( + CP_UTF8, WC_ERR_INVALID_CHARS, utf16_string, + input_length, utf8_string.data(), target_length, nullptr, nullptr); + if (converted_length == 0) { + return std::string(); + } + return utf8_string; +} diff --git a/windows/runner/utils.h b/windows/runner/utils.h new file mode 100644 index 0000000..3879d54 --- /dev/null +++ b/windows/runner/utils.h @@ -0,0 +1,19 @@ +#ifndef RUNNER_UTILS_H_ +#define RUNNER_UTILS_H_ + +#include +#include + +// Creates a console for the process, and redirects stdout and stderr to +// it for both the runner and the Flutter library. +void CreateAndAttachConsole(); + +// Takes a null-terminated wchar_t* encoded in UTF-16 and returns a std::string +// encoded in UTF-8. Returns an empty std::string on failure. +std::string Utf8FromUtf16(const wchar_t* utf16_string); + +// Gets the command line arguments passed in as a std::vector, +// encoded in UTF-8. Returns an empty std::vector on failure. +std::vector GetCommandLineArguments(); + +#endif // RUNNER_UTILS_H_ diff --git a/windows/runner/win32_window.cpp b/windows/runner/win32_window.cpp new file mode 100644 index 0000000..60608d0 --- /dev/null +++ b/windows/runner/win32_window.cpp @@ -0,0 +1,288 @@ +#include "win32_window.h" + +#include +#include + +#include "resource.h" + +namespace { + +/// Window attribute that enables dark mode window decorations. +/// +/// Redefined in case the developer's machine has a Windows SDK older than +/// version 10.0.22000.0. +/// See: https://docs.microsoft.com/windows/win32/api/dwmapi/ne-dwmapi-dwmwindowattribute +#ifndef DWMWA_USE_IMMERSIVE_DARK_MODE +#define DWMWA_USE_IMMERSIVE_DARK_MODE 20 +#endif + +constexpr const wchar_t kWindowClassName[] = L"FLUTTER_RUNNER_WIN32_WINDOW"; + +/// Registry key for app theme preference. +/// +/// A value of 0 indicates apps should use dark mode. A non-zero or missing +/// value indicates apps should use light mode. +constexpr const wchar_t kGetPreferredBrightnessRegKey[] = + L"Software\\Microsoft\\Windows\\CurrentVersion\\Themes\\Personalize"; +constexpr const wchar_t kGetPreferredBrightnessRegValue[] = L"AppsUseLightTheme"; + +// The number of Win32Window objects that currently exist. +static int g_active_window_count = 0; + +using EnableNonClientDpiScaling = BOOL __stdcall(HWND hwnd); + +// Scale helper to convert logical scaler values to physical using passed in +// scale factor +int Scale(int source, double scale_factor) { + return static_cast(source * scale_factor); +} + +// Dynamically loads the |EnableNonClientDpiScaling| from the User32 module. +// This API is only needed for PerMonitor V1 awareness mode. +void EnableFullDpiSupportIfAvailable(HWND hwnd) { + HMODULE user32_module = LoadLibraryA("User32.dll"); + if (!user32_module) { + return; + } + auto enable_non_client_dpi_scaling = + reinterpret_cast( + GetProcAddress(user32_module, "EnableNonClientDpiScaling")); + if (enable_non_client_dpi_scaling != nullptr) { + enable_non_client_dpi_scaling(hwnd); + } + FreeLibrary(user32_module); +} + +} // namespace + +// Manages the Win32Window's window class registration. +class WindowClassRegistrar { + public: + ~WindowClassRegistrar() = default; + + // Returns the singleton registrar instance. + static WindowClassRegistrar* GetInstance() { + if (!instance_) { + instance_ = new WindowClassRegistrar(); + } + return instance_; + } + + // Returns the name of the window class, registering the class if it hasn't + // previously been registered. + const wchar_t* GetWindowClass(); + + // Unregisters the window class. Should only be called if there are no + // instances of the window. + void UnregisterWindowClass(); + + private: + WindowClassRegistrar() = default; + + static WindowClassRegistrar* instance_; + + bool class_registered_ = false; +}; + +WindowClassRegistrar* WindowClassRegistrar::instance_ = nullptr; + +const wchar_t* WindowClassRegistrar::GetWindowClass() { + if (!class_registered_) { + WNDCLASS window_class{}; + window_class.hCursor = LoadCursor(nullptr, IDC_ARROW); + window_class.lpszClassName = kWindowClassName; + window_class.style = CS_HREDRAW | CS_VREDRAW; + window_class.cbClsExtra = 0; + window_class.cbWndExtra = 0; + window_class.hInstance = GetModuleHandle(nullptr); + window_class.hIcon = + LoadIcon(window_class.hInstance, MAKEINTRESOURCE(IDI_APP_ICON)); + window_class.hbrBackground = 0; + window_class.lpszMenuName = nullptr; + window_class.lpfnWndProc = Win32Window::WndProc; + RegisterClass(&window_class); + class_registered_ = true; + } + return kWindowClassName; +} + +void WindowClassRegistrar::UnregisterWindowClass() { + UnregisterClass(kWindowClassName, nullptr); + class_registered_ = false; +} + +Win32Window::Win32Window() { + ++g_active_window_count; +} + +Win32Window::~Win32Window() { + --g_active_window_count; + Destroy(); +} + +bool Win32Window::Create(const std::wstring& title, + const Point& origin, + const Size& size) { + Destroy(); + + const wchar_t* window_class = + WindowClassRegistrar::GetInstance()->GetWindowClass(); + + const POINT target_point = {static_cast(origin.x), + static_cast(origin.y)}; + HMONITOR monitor = MonitorFromPoint(target_point, MONITOR_DEFAULTTONEAREST); + UINT dpi = FlutterDesktopGetDpiForMonitor(monitor); + double scale_factor = dpi / 96.0; + + HWND window = CreateWindow( + window_class, title.c_str(), WS_OVERLAPPEDWINDOW, + Scale(origin.x, scale_factor), Scale(origin.y, scale_factor), + Scale(size.width, scale_factor), Scale(size.height, scale_factor), + nullptr, nullptr, GetModuleHandle(nullptr), this); + + if (!window) { + return false; + } + + UpdateTheme(window); + + return OnCreate(); +} + +bool Win32Window::Show() { + return ShowWindow(window_handle_, SW_SHOWNORMAL); +} + +// static +LRESULT CALLBACK Win32Window::WndProc(HWND const window, + UINT const message, + WPARAM const wparam, + LPARAM const lparam) noexcept { + if (message == WM_NCCREATE) { + auto window_struct = reinterpret_cast(lparam); + SetWindowLongPtr(window, GWLP_USERDATA, + reinterpret_cast(window_struct->lpCreateParams)); + + auto that = static_cast(window_struct->lpCreateParams); + EnableFullDpiSupportIfAvailable(window); + that->window_handle_ = window; + } else if (Win32Window* that = GetThisFromHandle(window)) { + return that->MessageHandler(window, message, wparam, lparam); + } + + return DefWindowProc(window, message, wparam, lparam); +} + +LRESULT +Win32Window::MessageHandler(HWND hwnd, + UINT const message, + WPARAM const wparam, + LPARAM const lparam) noexcept { + switch (message) { + case WM_DESTROY: + window_handle_ = nullptr; + Destroy(); + if (quit_on_close_) { + PostQuitMessage(0); + } + return 0; + + case WM_DPICHANGED: { + auto newRectSize = reinterpret_cast(lparam); + LONG newWidth = newRectSize->right - newRectSize->left; + LONG newHeight = newRectSize->bottom - newRectSize->top; + + SetWindowPos(hwnd, nullptr, newRectSize->left, newRectSize->top, newWidth, + newHeight, SWP_NOZORDER | SWP_NOACTIVATE); + + return 0; + } + case WM_SIZE: { + RECT rect = GetClientArea(); + if (child_content_ != nullptr) { + // Size and position the child window. + MoveWindow(child_content_, rect.left, rect.top, rect.right - rect.left, + rect.bottom - rect.top, TRUE); + } + return 0; + } + + case WM_ACTIVATE: + if (child_content_ != nullptr) { + SetFocus(child_content_); + } + return 0; + + case WM_DWMCOLORIZATIONCOLORCHANGED: + UpdateTheme(hwnd); + return 0; + } + + return DefWindowProc(window_handle_, message, wparam, lparam); +} + +void Win32Window::Destroy() { + OnDestroy(); + + if (window_handle_) { + DestroyWindow(window_handle_); + window_handle_ = nullptr; + } + if (g_active_window_count == 0) { + WindowClassRegistrar::GetInstance()->UnregisterWindowClass(); + } +} + +Win32Window* Win32Window::GetThisFromHandle(HWND const window) noexcept { + return reinterpret_cast( + GetWindowLongPtr(window, GWLP_USERDATA)); +} + +void Win32Window::SetChildContent(HWND content) { + child_content_ = content; + SetParent(content, window_handle_); + RECT frame = GetClientArea(); + + MoveWindow(content, frame.left, frame.top, frame.right - frame.left, + frame.bottom - frame.top, true); + + SetFocus(child_content_); +} + +RECT Win32Window::GetClientArea() { + RECT frame; + GetClientRect(window_handle_, &frame); + return frame; +} + +HWND Win32Window::GetHandle() { + return window_handle_; +} + +void Win32Window::SetQuitOnClose(bool quit_on_close) { + quit_on_close_ = quit_on_close; +} + +bool Win32Window::OnCreate() { + // No-op; provided for subclasses. + return true; +} + +void Win32Window::OnDestroy() { + // No-op; provided for subclasses. +} + +void Win32Window::UpdateTheme(HWND const window) { + DWORD light_mode; + DWORD light_mode_size = sizeof(light_mode); + LSTATUS result = RegGetValue(HKEY_CURRENT_USER, kGetPreferredBrightnessRegKey, + kGetPreferredBrightnessRegValue, + RRF_RT_REG_DWORD, nullptr, &light_mode, + &light_mode_size); + + if (result == ERROR_SUCCESS) { + BOOL enable_dark_mode = light_mode == 0; + DwmSetWindowAttribute(window, DWMWA_USE_IMMERSIVE_DARK_MODE, + &enable_dark_mode, sizeof(enable_dark_mode)); + } +} diff --git a/windows/runner/win32_window.h b/windows/runner/win32_window.h new file mode 100644 index 0000000..e901dde --- /dev/null +++ b/windows/runner/win32_window.h @@ -0,0 +1,102 @@ +#ifndef RUNNER_WIN32_WINDOW_H_ +#define RUNNER_WIN32_WINDOW_H_ + +#include + +#include +#include +#include + +// A class abstraction for a high DPI-aware Win32 Window. Intended to be +// inherited from by classes that wish to specialize with custom +// rendering and input handling +class Win32Window { + public: + struct Point { + unsigned int x; + unsigned int y; + Point(unsigned int x, unsigned int y) : x(x), y(y) {} + }; + + struct Size { + unsigned int width; + unsigned int height; + Size(unsigned int width, unsigned int height) + : width(width), height(height) {} + }; + + Win32Window(); + virtual ~Win32Window(); + + // Creates a win32 window with |title| that is positioned and sized using + // |origin| and |size|. New windows are created on the default monitor. Window + // sizes are specified to the OS in physical pixels, hence to ensure a + // consistent size this function will scale the inputted width and height as + // as appropriate for the default monitor. The window is invisible until + // |Show| is called. Returns true if the window was created successfully. + bool Create(const std::wstring& title, const Point& origin, const Size& size); + + // Show the current window. Returns true if the window was successfully shown. + bool Show(); + + // Release OS resources associated with window. + void Destroy(); + + // Inserts |content| into the window tree. + void SetChildContent(HWND content); + + // Returns the backing Window handle to enable clients to set icon and other + // window properties. Returns nullptr if the window has been destroyed. + HWND GetHandle(); + + // If true, closing this window will quit the application. + void SetQuitOnClose(bool quit_on_close); + + // Return a RECT representing the bounds of the current client area. + RECT GetClientArea(); + + protected: + // Processes and route salient window messages for mouse handling, + // size change and DPI. Delegates handling of these to member overloads that + // inheriting classes can handle. + virtual LRESULT MessageHandler(HWND window, + UINT const message, + WPARAM const wparam, + LPARAM const lparam) noexcept; + + // Called when CreateAndShow is called, allowing subclass window-related + // setup. Subclasses should return false if setup fails. + virtual bool OnCreate(); + + // Called when Destroy is called. + virtual void OnDestroy(); + + private: + friend class WindowClassRegistrar; + + // OS callback called by message pump. Handles the WM_NCCREATE message which + // is passed when the non-client area is being created and enables automatic + // non-client DPI scaling so that the non-client area automatically + // responds to changes in DPI. All other messages are handled by + // MessageHandler. + static LRESULT CALLBACK WndProc(HWND const window, + UINT const message, + WPARAM const wparam, + LPARAM const lparam) noexcept; + + // Retrieves a class instance pointer for |window| + static Win32Window* GetThisFromHandle(HWND const window) noexcept; + + // Update the window frame's theme to match the system theme. + static void UpdateTheme(HWND const window); + + bool quit_on_close_ = false; + + // window handle for top level window. + HWND window_handle_ = nullptr; + + // window handle for hosted content. + HWND child_content_ = nullptr; +}; + +#endif // RUNNER_WIN32_WINDOW_H_