From 6522eca4d006c15bbc9ba6db5d946704c1e95769 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 17:45:33 -0400 Subject: [PATCH 01/18] add recon and decisions docs for local mode --- .gitignore | 6 +++ DECISIONS.md | 105 +++++++++++++++++++++++++++++++++++++++++++++++++++ RECON.md | 102 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 213 insertions(+) create mode 100644 DECISIONS.md create mode 100644 RECON.md diff --git a/.gitignore b/.gitignore index 832e80a1a..9ed64dba7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,9 @@ build/ releases/ .claude/ coding-plans/ + +# local model weights must never land in the repo +models/ +*.safetensors +*.gguf +DerivedData/ diff --git a/DECISIONS.md b/DECISIONS.md new file mode 100644 index 000000000..0405ee27a --- /dev/null +++ b/DECISIONS.md @@ -0,0 +1,105 @@ +# DECISIONS — Local Mode + +Each entry: the call, and why. Deviations from the original plan are marked **[deviation]**. + +## Runtime & model + +- **MLX via `ml-explore/mlx-swift-lm` pinned `.upToNextMajor(from: "3.31.3")`** plus + `huggingface/swift-huggingface` (≥0.9.0) and `huggingface/swift-transformers` (≥1.3.0). + The LLM libraries moved out of `mlx-swift-examples` in April 2026; this is the maintained + home. Let mlx-swift-lm own the mlx-swift version (it pins `.upToNextMinor(from: 0.31.4)`). +- **Model: `mlx-community/Llama-3.2-3B-Instruct-4bit` (~1.8 GB)** via + `LLMRegistry.llama3_2_3B_4bit`. It's a registry constant (path of least resistance), no + thinking-token surprises (Qwen3-family templates emit them), fits the ≤2.5 GB budget, and + the name reads well in a demo. One model, not a menu. +- **Load path: the `#huggingFaceLoadModelContainer` macro** — the README-canonical route. It + pulls swift-syntax (slower clean builds); the alternative is ~30 lines of hand-written + Downloader/TokenizerLoader adapters. Sugar wins: less custom plumbing in the diff, and clean + builds are a one-time tax. +- **Model storage: `~/Library/Application Support/Clicky/models/huggingface`**, pinned with + `HubCache(location: .fixed(directory:))`. Application Support, not Caches — macOS purges + Caches under disk pressure and silently re-downloading 1.8 GB is a bad surprise. Never in + the repo; `.gitignore` extended. + +## Architecture + +- **`BuddyChatProvider` protocol + `CloudChatProvider` + `LocalChatProvider`**, mirroring the + existing `BuddyTranscriptionProvider` pattern (Farza's own seam, completed for chat). + `CloudChatProvider` wraps the untouched `ClaudeAPI` — cloud behavior stays byte-for-byte. + Routing branches in `CompanionManager` at the request site, because `ClaudeAPI`'s proxy URL + is `private let` on a single lazy instance and the picker pipeline is plain Strings. +- **"Local" is a third `modelOptionButton` with sentinel modelID `local`**, stored in the same + `selectedClaudeModel` UserDefaults key. Hardening: the stored value is validated on load and + unknown values fall back to Sonnet, so a stale `local` can never reach the Anthropic API + (which would 404). +- **The local engine lives in `LocalChatProvider`** (owns ModelContainer + download state), + exposed to SwiftUI through `@Published` state on CompanionManager like everything else. MVVM + graph unchanged. + +## Behavior contract (Tier 1 honesty) + +- **No screenshot in Local Mode** — capture is skipped entirely, not captured-and-dropped. + `ClaudeAPI`-style empty-images requests already degrade cleanly; pointing needs a capture to + map pixels→points, so it no-ops by construction as well as by prompt. +- **`[POINT:...]` disabled in Local Mode.** A 3B model guessing screen coordinates it has never + seen is a clown show. The local system prompt bans the tag; the end-anchored parse regex + would choke on malformed emissions anyway and TTS would read the tag aloud. +- **Short local system prompt** (~10 lines): keeps the clicky voice rules (lowercase, 1-2 + sentences, TTS-friendly), drops all screen/pointing/multi-monitor material — 4.3k chars of + vision instructions wasted on a text-only 3B model otherwise. +- **[deviation] Analytics content events are gated in Local Mode** — promoted from Tier 2 to + Tier 1 after recon found PostHog uploads the *full transcript and full response text* with no + opt-out. "Your screen stays on your Mac" while words go to a US analytics cloud is a + self-own. Local Mode sends a single content-free `local_mode_selected` event; transcript and + response events don't fire. +- **[deviation] Local Mode gets its own spoken error line** instead of the global "I'm all out + of credits, please DM Farza" fallback — that line is a lie when the actual failure is local + inference. Kept Farza-casual. +- **[deviation] The "screen stays on your Mac" notice lives in the menu-bar panel**, under the + picker, shown whenever Local is selected — not a once-per-session overlay toast. Recon: the + overlay fades out aggressively (transient mode) and has no stable text surface; the panel is + where the user makes the choice, so the disclosure sits at the decision point. + +## Voice loop (offline) + +- **TTS: `AVSpeechSynthesizer` client implementing the same surface as ElevenLabsTTSClient** + (`speakText` returns at playback start, `isPlaying` true until didFinish, synchronous + `stopPlayback`) — the transient-cursor hide loop polls `isPlaying` every 200ms and hangs + forever if it never flips. Robotic next to ElevenLabs; it's the *offline* voice and the + tradeoff is the point. +- **STT: Apple Speech provider, switched at runtime.** The factory resolves once at init today; + the refactor re-resolves the provider at session start (the one seam where it's used) based + on the selected mode. On-device recognition is verified via `supportsOnDeviceRecognition`, + not assumed — if unsupported, the UI says so rather than silently sending audio to Apple. + First switch to Local will trigger the speech-recognition permission prompt (usage string + already shipped in Info.plist). + +## Streaming & UX + +- **[deviation] No streamed-text rendering in Tier 1.** The brief assumed the overlay renders + tokens; recon shows the app discards them (no-op `onTextChunk`, dead + `CompanionResponseOverlay`). Clicky's medium is voice. Local Mode matches the existing + pipeline (spinner → spoken reply) and the latency badge (Tier 2) makes the speed difference + visible. Reviving 217 lines of untested dead overlay code that resizes an NSPanel per token + at 40 tok/s is a drive-by refactor with demo risk — declined. +- **First-token latency is measured in the provider layer** (wrap the first `onTextChunk` + invocation) — no changes inside `ClaudeAPI`. +- **max_tokens parity: local generation capped at 1024** to match the cloud path + (`ClaudeAPI.swift:144`); KV growth bounded via `GenerateParameters`. + +## Out of scope (explicit) + +- Local vision (Tier 3 gate: only after demo video is recordable; cut if TTFT > ~6s). +- Auto-routing local/cloud (Tier 3, manual picker must be solid first). +- Fixing the worker-URL placeholders, the unauthenticated worker, the dead + `ElementLocationDetector`, the camera entitlement, or any known warning — not our diff. +- Streaming TTS, sentence-chunked TTS — escape hatch only if full-response TTS feels bad. + +## Process + +- **Terminal `xcodebuild` is used for compile verification only while this machine has zero + TCC grants for clicky** (fresh clone, never run). Farza's rule exists because terminal + builds invalidate *granted* permissions; there are none yet. The moment the app runs from + Xcode and permissions are granted, terminal builds stop. +- Cloud side is blocked on a deployed worker URL (placeholders in repo). Local-first per the + escape hatch; side-by-side demo shots happen once the worker exists. diff --git a/RECON.md b/RECON.md new file mode 100644 index 000000000..90748c6d6 --- /dev/null +++ b/RECON.md @@ -0,0 +1,102 @@ +# RECON — what the code actually does (verified 2026-06-12) + +Call path, push-to-talk release → response, with the things the docs get wrong. + +## The pipeline + +``` +ctrl+option release + └─ GlobalPushToTalkShortcutMonitor (CGEvent tap) → .released + └─ CompanionManager.handleShortcutTransition (~:528) + └─ BuddyDictationManager.stopPushToTalk → AssemblyAI ForceEndpoint + └─ onFinalTranscriptReady → submitDraftText + └─ CompanionManager.sendTranscriptToClaudeWithScreenshot (:586-726) + ├─ voiceState = .processing (spinner) + ├─ captureAllScreensAsJPEG() ← mandatory; failure aborts everything + ├─ ClaudeAPI.analyzeImageStreaming → worker /chat → SSE + │ └─ onTextChunk: { _ in } ← tokens are DISCARDED + ├─ parsePointingCoordinates (end-anchored [POINT:...] regex, :784) + ├─ ElevenLabsTTSClient.speakText (full MP3 download, then play) + │ └─ voiceState = .responding only after play() starts + └─ voiceState = .idle +``` + +## Where the brief / docs and reality diverge + +- **No streamed text ever renders.** `onTextChunk` is a deliberate no-op + (CompanionManager.swift:618, comment: "No streaming text display — spinner stays until TTS + plays"). `CompanionResponseOverlay.swift` (the styled streaming bubble, 217 lines) is dead + code — zero instantiations. Clicky's response medium is **voice**, full stop. "Stream local + tokens into the existing overlay" therefore means: match the voice pipeline, and make latency + visible some other way (badge). +- **Analytics uploads conversation content.** `ClickyAnalytics.trackUserMessageSent` / + `trackAIResponseReceived` send the full transcript and full AI response to PostHog US cloud + (ClickyAnalytics.swift:84-96), no opt-out. A Local Mode that leaves this on is privacy + theater — gating it is part of the feature, not polish. +- **Both worker URLs are placeholders** (`CompanionManager.swift:73`, + `AssemblyAIStreamingTranscriptionProvider.swift:22`). Fresh clone = cloud chat, TTS, and + streaming STT are all dead until you deploy the worker. Local Mode is the only path that can + work out of the box. +- **AssemblyAI does not fall back to Apple Speech on network failure.** The fallback chain in + the factory is config-time only — and partially dead, since AssemblyAI's `isConfigured` is + hardcoded `true`. Mid-session network loss surfaces an error; nothing reroutes. +- **Every pipeline error speaks "I'm all out of credits. Please DM Farza…"** + (CompanionManager.swift:761-766, via NSSpeechSynthesizer) — wrong and confusing for a local + inference failure. +- ElevenLabs client claims streaming playback in its header comment; it actually buffers the + entire MP3 (`session.data(for:)`) before playing. +- `ElementLocationDetector.swift` (direct api.anthropic.com + raw key) is never instantiated — + latent dead code, ignore it. + +## The seams we exploit + +- **Model picker is a plain String pipeline, zero enums.** `modelOptionButton(label:modelID:)` + is generic (CompanionPanelView.swift:625-642); `selectedModel` is `@Published String` + persisted under UserDefaults `selectedClaudeModel` (CompanionManager.swift:111-117). Adding + "Local" is one call-site line. The real work is routing: `ClaudeAPI`'s URL is `private let`, + single lazy instance — so we branch at the request site, exactly where a provider protocol + belongs. +- **`BuddyTranscriptionProvider` is the architectural precedent** — protocol + factory, + resolved from Info.plist key `VoiceTranscriptionProvider` (currently `assemblyai`). Caveat: + resolved **once** in `BuddyDictationManager.init` into a `let`; runtime switching needs a + small refactor at the single seam where the provider is touched (`startRecognitionSession`, + BuddyDictationManager.swift:514). +- **Apple Speech provider already exists** (AppleSpeechTranscriptionProvider.swift) and + `NSSpeechRecognitionUsageDescription` is already in Info.plist. It's only guaranteed + on-device when `supportsOnDeviceRecognition` is true — we check, not assume. +- **TTS client contract** (what an AVSpeechSynthesizer replacement must honor, from + ElevenLabsTTSClient.swift + call sites): `speakText` returns when playback *starts*; + `isPlaying` is polled every 200ms by the transient-cursor hide loop + (CompanionManager.swift:732-756) and must reliably flip false on finish; `stopPlayback()` + must silence synchronously (called on every key press). +- **Screenshot skip degrades cleanly**: `ClaudeAPI` builds content blocks per image, so + `images: []` is already a valid text-only request. Pointing then silently no-ops (the + pixel→point mapping needs a capture) — which is what we want in Local Mode anyway. +- **System prompt** is `companionVoiceResponseSystemPrompt` (CompanionManager.swift:544-577), + 4.3k chars, mostly pointing protocol + multi-screen rules — a 3B model gets a short variant. +- **pbxproj is objectVersion 77** (filesystem-synchronized groups): new Swift files are picked + up automatically; only SPM package references need project edits. +- App Sandbox **off**, Hardened Runtime **on**, `network.client` entitlement present. MLX needs + no JIT entitlements; model downloads to Application Support work unrestricted. + +## MLX Swift, June 2026 state (web-verified) + +- `MLXLLM`/`MLXLMCommon` **moved out of mlx-swift-examples** into + `github.com/ml-explore/mlx-swift-lm` — latest tag **3.31.3** (2026-04-15); 3.x is a breaking + release (Downloader/Tokenizer protocols, `HubApi` removed). Companions: + `huggingface/swift-huggingface` ≥0.9.0 (HubClient), `huggingface/swift-transformers` ≥1.3.0. +- Blessed load path: `#huggingFaceLoadModelContainer(configuration:progressHandler:)` macro → + `ModelContainer`; streaming via `generate(input:parameters:)` → `AsyncStream` + (`.chunk` text, `.info` carries `tokensPerSecond` — free benchmark instrumentation). +- `LLMRegistry.llama3_2_3B_4bit` (mlx-community/Llama-3.2-3B-Instruct-4bit, ~1.8 GB) is a + registry constant. Qwen2.5-3B is not (loads via explicit config, but why fight it). +- Download cache defaults to `~/.cache/huggingface/hub`; pin to Application Support via + `HubCache(location: .fixed(directory:))`. +- Memory: `MLX.GPU.set(cacheLimit:)` is deprecated → `MLX.Memory.cacheLimit`. 3B-4bit is + comfortable on this 16 GB M1 Pro; bound KV growth via `GenerateParameters` anyway. +- SwiftPM CLI cannot compile MLX's Metal shaders — full Xcode required (installed: 26.5). + +## Dev machine + +M1 Pro / 16 GB / macOS 26.6 / Xcode 26.5 + Metal toolchain. Expect ~30-60 tok/s decode for +3B-4bit (estimate — the README table gets measured numbers only). From 96cb09002c53d9b3971e7cd44e3ac3e048577adf Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 17:48:58 -0400 Subject: [PATCH 02/18] add chat provider protocol and route responses through it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the BuddyTranscriptionProvider pattern so chat backends are interchangeable. CloudChatProvider wraps the untouched ClaudeAPI — no behavior change for Sonnet/Opus. --- leanring-buddy/BuddyChatProvider.swift | 108 +++++++++++++++++++++++++ leanring-buddy/CompanionManager.swift | 25 ++++-- 2 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 leanring-buddy/BuddyChatProvider.swift diff --git a/leanring-buddy/BuddyChatProvider.swift b/leanring-buddy/BuddyChatProvider.swift new file mode 100644 index 000000000..fa4dda8fa --- /dev/null +++ b/leanring-buddy/BuddyChatProvider.swift @@ -0,0 +1,108 @@ +// +// BuddyChatProvider.swift +// leanring-buddy +// +// Shared protocol surface for chat response backends — the same pattern +// BuddyTranscriptionProvider uses for voice transcription. The cloud +// provider wraps the existing ClaudeAPI (via the Cloudflare Worker proxy); +// the local provider runs an on-device model via MLX. +// + +import Foundation + +/// The result of one fully-streamed chat response. +struct BuddyChatResponse { + /// The complete response text after the stream finished. + let text: String + /// Seconds from sending the request to the first streamed text chunk + /// arriving. This is the number the latency badge shows, because it's + /// the part of the wait the user actually feels. + let firstTokenLatencySeconds: TimeInterval? + /// Seconds from sending the request to the end of the stream. + let totalDurationSeconds: TimeInterval +} + +@MainActor +protocol BuddyChatProvider: AnyObject { + /// Short lowercase name for logs and the latency badge ("cloud", "local"). + var displayName: String { get } + + /// Sends one chat turn and streams the response back. `onTextChunk` + /// receives the accumulated text so far (not the delta) on the main + /// actor each time new text arrives — the same ergonomics ClaudeAPI + /// already uses, so call sites don't change shape. + func generateStreamingResponse( + images: [(data: Data, label: String)], + systemPrompt: String, + conversationHistory: [(userTranscript: String, assistantResponse: String)], + userPrompt: String, + onTextChunk: @escaping @MainActor @Sendable (String) -> Void + ) async throws -> BuddyChatResponse +} + +/// Cloud chat provider — a thin wrapper around the existing ClaudeAPI so the +/// Sonnet/Opus request path behaves exactly as it did before the provider +/// seam existed. The only addition is first-token timing, measured out here +/// so ClaudeAPI itself stays untouched. +@MainActor +final class CloudChatProvider: BuddyChatProvider { + let displayName = "cloud" + + private let claudeAPI: ClaudeAPI + + /// The Claude model ID sent with each request (e.g. "claude-sonnet-4-6"). + var model: String { + get { claudeAPI.model } + set { claudeAPI.model = newValue } + } + + init(claudeAPI: ClaudeAPI) { + self.claudeAPI = claudeAPI + } + + func generateStreamingResponse( + images: [(data: Data, label: String)], + systemPrompt: String, + conversationHistory: [(userTranscript: String, assistantResponse: String)], + userPrompt: String, + onTextChunk: @escaping @MainActor @Sendable (String) -> Void + ) async throws -> BuddyChatResponse { + let requestStartDate = Date() + let firstChunkRecorder = BuddyFirstChunkRecorder() + + // ClaudeAPI's history tuple labels predate the provider seam — + // map ours onto them rather than touching the working client. + let historyForClaudeAPI = conversationHistory.map { exchange in + (userPlaceholder: exchange.userTranscript, assistantResponse: exchange.assistantResponse) + } + + let (fullResponseText, totalDurationSeconds) = try await claudeAPI.analyzeImageStreaming( + images: images, + systemPrompt: systemPrompt, + conversationHistory: historyForClaudeAPI, + userPrompt: userPrompt, + onTextChunk: { accumulatedText in + if firstChunkRecorder.firstChunkDate == nil { + firstChunkRecorder.firstChunkDate = Date() + } + onTextChunk(accumulatedText) + } + ) + + let firstTokenLatencySeconds = firstChunkRecorder.firstChunkDate + .map { firstChunkDate in firstChunkDate.timeIntervalSince(requestStartDate) } + + return BuddyChatResponse( + text: fullResponseText, + firstTokenLatencySeconds: firstTokenLatencySeconds, + totalDurationSeconds: totalDurationSeconds + ) + } +} + +/// Tiny main-actor box so a streaming closure can record when the first +/// chunk arrived without tripping concurrency checks. +@MainActor +final class BuddyFirstChunkRecorder { + var firstChunkDate: Date? +} diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index 0234cf19f..cfc50f024 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -76,6 +76,15 @@ final class CompanionManager: ObservableObject { return ClaudeAPI(proxyURL: "\(Self.workerBaseURL)/chat", model: selectedModel) }() + /// Cloud chat provider wrapping ClaudeAPI. Voice responses route through + /// the provider seam so cloud and local backends are interchangeable. + private lazy var cloudChatProvider = CloudChatProvider(claudeAPI: claudeAPI) + + /// The chat provider matching the current model selection. + private var activeChatProvider: any BuddyChatProvider { + cloudChatProvider + } + private lazy var elevenLabsTTSClient: ElevenLabsTTSClient = { return ElevenLabsTTSClient(proxyURL: "\(Self.workerBaseURL)/tts") }() @@ -605,20 +614,16 @@ final class CompanionManager: ObservableObject { return (data: capture.imageData, label: capture.label + dimensionInfo) } - // Pass conversation history so Claude remembers prior exchanges - let historyForAPI = conversationHistory.map { entry in - (userPlaceholder: entry.userTranscript, assistantResponse: entry.assistantResponse) - } - - let (fullResponseText, _) = try await claudeAPI.analyzeImageStreaming( + let chatResponse = try await activeChatProvider.generateStreamingResponse( images: labeledImages, systemPrompt: Self.companionVoiceResponseSystemPrompt, - conversationHistory: historyForAPI, + conversationHistory: conversationHistory, userPrompt: transcript, onTextChunk: { _ in // No streaming text display — spinner stays until TTS plays } ) + let fullResponseText = chatResponse.text guard !Task.isCancelled else { return } @@ -982,12 +987,16 @@ final class CompanionManager: ObservableObject { let dimensionInfo = " (image dimensions: \(cursorScreenCapture.screenshotWidthInPixels)x\(cursorScreenCapture.screenshotHeightInPixels) pixels)" let labeledImages = [(data: cursorScreenCapture.imageData, label: cursorScreenCapture.label + dimensionInfo)] - let (fullResponseText, _) = try await claudeAPI.analyzeImageStreaming( + // Always the cloud provider — the demo needs vision + pointing, + // which the local model deliberately doesn't do. + let demoChatResponse = try await cloudChatProvider.generateStreamingResponse( images: labeledImages, systemPrompt: Self.onboardingDemoSystemPrompt, + conversationHistory: [], userPrompt: "look around my screen and find something interesting to point at", onTextChunk: { _ in } ) + let fullResponseText = demoChatResponse.text let parseResult = Self.parsePointingCoordinates(from: fullResponseText) From 4505b9b35667c3108431c742394cee41e4eaad37 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 18:00:51 -0400 Subject: [PATCH 03/18] add mlx local inference engine and on-device tts client LocalChatProvider runs Llama-3.2-3B-Instruct-4bit via mlx-swift-lm, downloads once to Application Support, then loads from disk offline. AppleSpeechSynthesisClient mirrors the ElevenLabs client contract so the voice loop can run with networking off. --- leanring-buddy.xcodeproj/project.pbxproj | 57 ++++ .../AppleSpeechSynthesisClient.swift | 73 +++++ leanring-buddy/BuddyChatProvider.swift | 6 +- leanring-buddy/BuddyTextToSpeechClient.swift | 28 ++ leanring-buddy/LocalChatProvider.swift | 253 ++++++++++++++++++ 5 files changed, 416 insertions(+), 1 deletion(-) create mode 100644 leanring-buddy/AppleSpeechSynthesisClient.swift create mode 100644 leanring-buddy/BuddyTextToSpeechClient.swift create mode 100644 leanring-buddy/LocalChatProvider.swift diff --git a/leanring-buddy.xcodeproj/project.pbxproj b/leanring-buddy.xcodeproj/project.pbxproj index 75e572618..8cb0909dc 100644 --- a/leanring-buddy.xcodeproj/project.pbxproj +++ b/leanring-buddy.xcodeproj/project.pbxproj @@ -121,6 +121,11 @@ packageProductDependencies = ( AA00BB022F6500020039DA55 /* Sparkle */, AA00BB052F6500050039DA55 /* PostHog */, + CC00AA042F6600040039DA55 /* MLXLLM */, + CC00AA052F6600050039DA55 /* MLXLMCommon */, + CC00AA062F6600060039DA55 /* MLXHuggingFace */, + CC00AA072F6600070039DA55 /* HuggingFace */, + CC00AA082F6600080039DA55 /* Tokenizers */, ); productName = "leanring-buddy"; productReference = 28F22CBF2F56440300A0FC59 /* Clicky.app */; @@ -207,6 +212,9 @@ packageReferences = ( AA00BB012F6500010039DA55 /* XCRemoteSwiftPackageReference "Sparkle" */, AA00BB042F6500040039DA55 /* XCRemoteSwiftPackageReference "posthog-ios" */, + CC00AA012F6600010039DA55 /* XCRemoteSwiftPackageReference "mlx-swift-lm" */, + CC00AA022F6600020039DA55 /* XCRemoteSwiftPackageReference "swift-huggingface" */, + CC00AA032F6600030039DA55 /* XCRemoteSwiftPackageReference "swift-transformers" */, ); preferredProjectObjectVersion = 77; productRefGroup = 28F22CC02F56440300A0FC59 /* Products */; @@ -616,6 +624,30 @@ minimumVersion = 3.0.0; }; }; + CC00AA012F6600010039DA55 /* XCRemoteSwiftPackageReference "mlx-swift-lm" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/ml-explore/mlx-swift-lm"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 3.31.3; + }; + }; + CC00AA022F6600020039DA55 /* XCRemoteSwiftPackageReference "swift-huggingface" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/huggingface/swift-huggingface"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 0.9.0; + }; + }; + CC00AA032F6600030039DA55 /* XCRemoteSwiftPackageReference "swift-transformers" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/huggingface/swift-transformers"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 1.3.0; + }; + }; /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ @@ -629,6 +661,31 @@ package = AA00BB042F6500040039DA55 /* XCRemoteSwiftPackageReference "posthog-ios" */; productName = PostHog; }; + CC00AA042F6600040039DA55 /* MLXLLM */ = { + isa = XCSwiftPackageProductDependency; + package = CC00AA012F6600010039DA55 /* XCRemoteSwiftPackageReference "mlx-swift-lm" */; + productName = MLXLLM; + }; + CC00AA052F6600050039DA55 /* MLXLMCommon */ = { + isa = XCSwiftPackageProductDependency; + package = CC00AA012F6600010039DA55 /* XCRemoteSwiftPackageReference "mlx-swift-lm" */; + productName = MLXLMCommon; + }; + CC00AA062F6600060039DA55 /* MLXHuggingFace */ = { + isa = XCSwiftPackageProductDependency; + package = CC00AA012F6600010039DA55 /* XCRemoteSwiftPackageReference "mlx-swift-lm" */; + productName = MLXHuggingFace; + }; + CC00AA072F6600070039DA55 /* HuggingFace */ = { + isa = XCSwiftPackageProductDependency; + package = CC00AA022F6600020039DA55 /* XCRemoteSwiftPackageReference "swift-huggingface" */; + productName = HuggingFace; + }; + CC00AA082F6600080039DA55 /* Tokenizers */ = { + isa = XCSwiftPackageProductDependency; + package = CC00AA032F6600030039DA55 /* XCRemoteSwiftPackageReference "swift-transformers" */; + productName = Tokenizers; + }; /* End XCSwiftPackageProductDependency section */ }; rootObject = 28F22CB72F56440300A0FC59 /* Project object */; diff --git a/leanring-buddy/AppleSpeechSynthesisClient.swift b/leanring-buddy/AppleSpeechSynthesisClient.swift new file mode 100644 index 000000000..98a4d01b8 --- /dev/null +++ b/leanring-buddy/AppleSpeechSynthesisClient.swift @@ -0,0 +1,73 @@ +// +// AppleSpeechSynthesisClient.swift +// leanring-buddy +// +// On-device text-to-speech for Local Mode, backed by AVSpeechSynthesizer. +// More robotic than ElevenLabs — that's the tradeoff for a voice that +// works with wifi off and never sends a word anywhere. +// + +import AVFoundation +import Foundation + +@MainActor +final class AppleSpeechSynthesisClient: NSObject, BuddyTextToSpeechClient { + /// Must stay strongly referenced for the whole utterance — if the + /// synthesizer deallocates, speech stops and didFinish never fires. + private let speechSynthesizer = AVSpeechSynthesizer() + + /// True from speech start until the utterance finishes or is stopped. + /// Flipped false by the delegate callbacks below — if it never flips, + /// CompanionManager's transient-hide loop would wait forever. + private(set) var isPlaying = false + + override init() { + super.init() + speechSynthesizer.delegate = self + } + + /// Speaks `text` with the best installed US English voice. Returns as + /// soon as speech is queued — the same "returns when playback starts" + /// contract as the ElevenLabs client. + func speakText(_ text: String) async throws { + try Task.checkCancellation() + + let utterance = AVSpeechUtterance(string: text) + utterance.voice = Self.bestAvailableEnglishVoice() + + isPlaying = true + speechSynthesizer.speak(utterance) + } + + func stopPlayback() { + speechSynthesizer.stopSpeaking(at: .immediate) + isPlaying = false + } + + /// Highest-quality installed en-US voice. Premium and enhanced voices + /// only exist if the user downloaded them in System Settings → Spoken + /// Content — the compact default is the floor, not the goal. + private static func bestAvailableEnglishVoice() -> AVSpeechSynthesisVoice? { + let englishVoices = AVSpeechSynthesisVoice.speechVoices() + .filter { voice in voice.language == "en-US" } + let premiumVoice = englishVoices.first { voice in voice.quality == .premium } + let enhancedVoice = englishVoices.first { voice in voice.quality == .enhanced } + return premiumVoice ?? enhancedVoice ?? AVSpeechSynthesisVoice(language: "en-US") + } +} + +extension AppleSpeechSynthesisClient: AVSpeechSynthesizerDelegate { + // Delegate callbacks arrive on an arbitrary queue — hop back to the + // main actor before touching isPlaying. + nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { + Task { @MainActor in + self.isPlaying = false + } + } + + nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance) { + Task { @MainActor in + self.isPlaying = false + } + } +} diff --git a/leanring-buddy/BuddyChatProvider.swift b/leanring-buddy/BuddyChatProvider.swift index fa4dda8fa..0eaa76023 100644 --- a/leanring-buddy/BuddyChatProvider.swift +++ b/leanring-buddy/BuddyChatProvider.swift @@ -20,6 +20,9 @@ struct BuddyChatResponse { let firstTokenLatencySeconds: TimeInterval? /// Seconds from sending the request to the end of the stream. let totalDurationSeconds: TimeInterval + /// Decode speed reported by the engine, if it reports one. Local + /// inference fills this in; the cloud path has no equivalent signal. + let tokensPerSecond: Double? } @MainActor @@ -95,7 +98,8 @@ final class CloudChatProvider: BuddyChatProvider { return BuddyChatResponse( text: fullResponseText, firstTokenLatencySeconds: firstTokenLatencySeconds, - totalDurationSeconds: totalDurationSeconds + totalDurationSeconds: totalDurationSeconds, + tokensPerSecond: nil ) } } diff --git a/leanring-buddy/BuddyTextToSpeechClient.swift b/leanring-buddy/BuddyTextToSpeechClient.swift new file mode 100644 index 000000000..aa79f115a --- /dev/null +++ b/leanring-buddy/BuddyTextToSpeechClient.swift @@ -0,0 +1,28 @@ +// +// BuddyTextToSpeechClient.swift +// leanring-buddy +// +// Shared protocol surface for text-to-speech backends — the same pattern +// as BuddyChatProvider and BuddyTranscriptionProvider. ElevenLabs is the +// cloud voice; AppleSpeechSynthesisClient is the on-device voice used by +// Local Mode so the whole answer loop works offline. +// + +import Foundation + +/// The contract CompanionManager relies on (it was ElevenLabsTTSClient's +/// implicit contract before Local Mode existed): +/// - `speakText` returns as soon as playback starts, not when it ends. +/// - `isPlaying` stays true until playback finishes or is stopped — the +/// transient-cursor hide loop polls it every 200ms and waits on it. +/// - `stopPlayback` silences audio synchronously. +@MainActor +protocol BuddyTextToSpeechClient: AnyObject { + func speakText(_ text: String) async throws + var isPlaying: Bool { get } + func stopPlayback() +} + +// ElevenLabsTTSClient already satisfies the contract exactly — the protocol +// was written around its existing behavior. +extension ElevenLabsTTSClient: BuddyTextToSpeechClient {} diff --git a/leanring-buddy/LocalChatProvider.swift b/leanring-buddy/LocalChatProvider.swift new file mode 100644 index 000000000..cbaef9e00 --- /dev/null +++ b/leanring-buddy/LocalChatProvider.swift @@ -0,0 +1,253 @@ +// +// LocalChatProvider.swift +// leanring-buddy +// +// On-device chat provider for Local Mode, backed by MLX. Runs +// Llama-3.2-3B-Instruct (4-bit, ~1.8 GB) entirely on the Mac's GPU — +// no network, no screenshots, nothing leaves the machine. +// +// The model downloads once from Hugging Face into Application Support +// with visible progress, then loads straight from disk on every later +// launch (so Local Mode works with wifi off after the first download). +// + +import Combine +import Foundation +import HuggingFace +import MLXHuggingFace +import MLXLLM +import MLXLMCommon +import Tokenizers + +@MainActor +final class LocalChatProvider: BuddyChatProvider, ObservableObject { + /// Sentinel stored in the same UserDefaults slot as the Claude model IDs. + /// CompanionManager routes to this provider whenever selectedModel + /// equals this value — the string never reaches the Anthropic API. + static let modelPickerID = "local" + + /// The one supported local model. A registry constant, 4-bit quantized, + /// small enough for a 16 GB machine with plenty of room to breathe. + private static let modelConfiguration = LLMRegistry.llama3_2_3B_4bit + + let displayName = "local" + + /// Where the local model lives on the panel UI's terms. The panel shows + /// a progress bar during .downloading and an error + retry on .failed. + enum ModelReadiness: Equatable { + case notDownloaded + case downloading(progressFraction: Double) + case loading + case ready + case failed(errorDescription: String) + } + + @Published private(set) var modelReadiness: ModelReadiness = .notDownloaded + + /// The loaded model, kept in memory for the life of the app so responses + /// after the first don't pay the load cost again. + private var loadedModelContainer: ModelContainer? + + /// In-flight load, shared so a double-tap on the picker doesn't kick off + /// two downloads. + private var modelLoadTask: Task? + + /// UserDefaults key holding the on-disk directory of the downloaded model. + /// When this directory exists we load straight from disk — no network, + /// which is what makes Local Mode work offline across relaunches. + private static let downloadedModelDirectoryDefaultsKey = "localModelDownloadedDirectory" + + /// Models download under Application Support, not Caches — macOS purges + /// Caches under disk pressure and silently re-downloading 1.8 GB is a + /// bad surprise. + private static func modelCacheDirectory() throws -> URL { + let applicationSupportDirectory = try FileManager.default.url( + for: .applicationSupportDirectory, + in: .userDomainMask, + appropriateFor: nil, + create: true + ) + return applicationSupportDirectory + .appendingPathComponent("Clicky", isDirectory: true) + .appendingPathComponent("models", isDirectory: true) + .appendingPathComponent("huggingface", isDirectory: true) + } + + init() { + // If the model was downloaded on an earlier launch, report it as + // ready-to-load rather than not-downloaded so the picker doesn't + // threaten the user with another 1.8 GB download. + if Self.previouslyDownloadedModelDirectory() != nil { + modelReadiness = .loading + } + } + + /// The model directory from a previous successful download, if it still + /// exists on disk. + private static func previouslyDownloadedModelDirectory() -> URL? { + guard let storedPath = UserDefaults.standard.string(forKey: downloadedModelDirectoryDefaultsKey) else { + return nil + } + let storedDirectoryURL = URL(fileURLWithPath: storedPath, isDirectory: true) + guard FileManager.default.fileExists(atPath: storedDirectoryURL.path) else { + return nil + } + return storedDirectoryURL + } + + /// Kicks off (or joins) the download + load. Safe to call repeatedly — + /// the picker calls this every time "Local" is selected. + @discardableResult + func loadModelIfNeeded() -> Task { + if let modelLoadTask { + return modelLoadTask + } + + let newModelLoadTask = Task { + do { + let modelContainer = try await loadModelContainer() + loadedModelContainer = modelContainer + modelReadiness = .ready + return modelContainer + } catch { + // Clear the task so a retry actually retries instead of + // re-awaiting the failed task. + modelLoadTask = nil + modelReadiness = .failed(errorDescription: error.localizedDescription) + throw error + } + } + modelLoadTask = newModelLoadTask + return newModelLoadTask + } + + private func loadModelContainer() async throws -> ModelContainer { + // Fast path: the model is already on disk from a previous launch. + // Loads with zero network — this is what keeps Local Mode working + // when the machine is offline. + if let downloadedModelDirectory = Self.previouslyDownloadedModelDirectory() { + modelReadiness = .loading + print("🏠 Local model: loading from disk at \(downloadedModelDirectory.path)") + return try await LLMModelFactory.shared.loadContainer( + from: downloadedModelDirectory, + using: #huggingFaceTokenizerLoader() + ) + } + + // First time: download from Hugging Face with visible progress. + modelReadiness = .downloading(progressFraction: 0) + let hubCacheDirectory = try Self.modelCacheDirectory() + let hubClient = HubClient(cache: HubCache(cacheDirectory: hubCacheDirectory)) + print("🏠 Local model: downloading \(Self.modelConfiguration.name) to \(hubCacheDirectory.path)") + + let modelContainer = try await LLMModelFactory.shared.loadContainer( + from: #hubDownloader(hubClient), + using: #huggingFaceTokenizerLoader(), + configuration: Self.modelConfiguration, + progressHandler: { downloadProgress in + let progressFraction = downloadProgress.fractionCompleted + Task { @MainActor [weak self] in + // The handler also fires during post-download loading; + // don't regress the state once loading has started. + guard let self, case .downloading = self.modelReadiness else { return } + self.modelReadiness = .downloading(progressFraction: progressFraction) + } + } + ) + + modelReadiness = .loading + + // Remember where the weights landed so future launches load from + // disk directly (and offline). + let modelDirectory = try await modelContainer.modelDirectory + UserDefaults.standard.set(modelDirectory.path, forKey: Self.downloadedModelDirectoryDefaultsKey) + print("🏠 Local model: downloaded to \(modelDirectory.path)") + + return modelContainer + } + + // MARK: - Generation + + /// Capped to match the cloud path's max_tokens (1024) and a bounded KV + /// window so memory stays flat over a long session. + private static let localGenerateParameters = GenerateParameters( + maxTokens: 1024, + maxKVSize: 4096, + temperature: 0.7 + ) + + func generateStreamingResponse( + images: [(data: Data, label: String)], + systemPrompt: String, + conversationHistory: [(userTranscript: String, assistantResponse: String)], + userPrompt: String, + onTextChunk: @escaping @MainActor @Sendable (String) -> Void + ) async throws -> BuddyChatResponse { + // `images` is intentionally ignored: in Local Mode the screenshot is + // never captured, so there is nothing to attach by construction. + let modelContainer: ModelContainer + if let loadedModelContainer { + modelContainer = loadedModelContainer + } else { + modelContainer = try await loadModelIfNeeded().value + } + + let requestStartDate = Date() + + var chatMessages: [Chat.Message] = [.system(systemPrompt)] + for exchange in conversationHistory { + chatMessages.append(.user(exchange.userTranscript)) + chatMessages.append(.assistant(exchange.assistantResponse)) + } + chatMessages.append(.user(userPrompt)) + + let languageModelInput = try await modelContainer.prepare(input: UserInput(chat: chatMessages)) + let generationStream = try await modelContainer.generate( + input: languageModelInput, + parameters: Self.localGenerateParameters + ) + + var accumulatedResponseText = "" + var firstChunkDate: Date? + var reportedTokensPerSecond: Double? + + for await generation in generationStream { + // The user pressed the hotkey again — stop consuming; dropping + // the stream tears down the underlying generation. + if Task.isCancelled { break } + + switch generation { + case .chunk(let textChunk): + if firstChunkDate == nil { + firstChunkDate = Date() + } + accumulatedResponseText += textChunk + onTextChunk(accumulatedResponseText) + case .info(let completionInfo): + reportedTokensPerSecond = completionInfo.tokensPerSecond + case .toolCall: + // The local prompt defines no tools; nothing to do. + break + } + } + + try Task.checkCancellation() + + let totalDurationSeconds = Date().timeIntervalSince(requestStartDate) + let firstTokenLatencySeconds = firstChunkDate + .map { firstChunkDate in firstChunkDate.timeIntervalSince(requestStartDate) } + + if let reportedTokensPerSecond { + let firstTokenDescription = firstTokenLatencySeconds + .map { String(format: "%.2fs", $0) } ?? "n/a" + print("🏠 Local generation: first token \(firstTokenDescription), \(String(format: "%.1f", reportedTokensPerSecond)) tok/s, \(String(format: "%.1fs", totalDurationSeconds)) total") + } + + return BuddyChatResponse( + text: accumulatedResponseText.trimmingCharacters(in: .whitespacesAndNewlines), + firstTokenLatencySeconds: firstTokenLatencySeconds, + totalDurationSeconds: totalDurationSeconds, + tokensPerSecond: reportedTokensPerSecond + ) + } +} From f7d72f368cd5522f94340a2cd1ae9807a782fbf6 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 18:00:51 -0400 Subject: [PATCH 04/18] add local inference benchmark harness Same model, cache dir, and generation parameters as the app, so the README numbers are measured on the actual pipeline, not estimated. --- bench/LocalModeBench/Package.swift | 28 +++++ .../Sources/LocalModeBench/Benchmark.swift | 114 ++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 bench/LocalModeBench/Package.swift create mode 100644 bench/LocalModeBench/Sources/LocalModeBench/Benchmark.swift diff --git a/bench/LocalModeBench/Package.swift b/bench/LocalModeBench/Package.swift new file mode 100644 index 000000000..58a6c184f --- /dev/null +++ b/bench/LocalModeBench/Package.swift @@ -0,0 +1,28 @@ +// swift-tools-version: 6.1 +// Tiny harness that measures Local Mode inference on the actual dev +// machine — same model, same cache directory, same generation parameters +// as LocalChatProvider. The README's benchmark numbers come from here and +// from the in-app latency badge, never from guesses. +import PackageDescription + +let package = Package( + name: "LocalModeBench", + platforms: [.macOS(.v14)], + dependencies: [ + .package(url: "https://github.com/ml-explore/mlx-swift-lm", .upToNextMajor(from: "3.31.3")), + .package(url: "https://github.com/huggingface/swift-huggingface", from: "0.9.0"), + .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.0"), + ], + targets: [ + .executableTarget( + name: "LocalModeBench", + dependencies: [ + .product(name: "MLXLLM", package: "mlx-swift-lm"), + .product(name: "MLXLMCommon", package: "mlx-swift-lm"), + .product(name: "MLXHuggingFace", package: "mlx-swift-lm"), + .product(name: "HuggingFace", package: "swift-huggingface"), + .product(name: "Tokenizers", package: "swift-transformers"), + ] + ) + ] +) diff --git a/bench/LocalModeBench/Sources/LocalModeBench/Benchmark.swift b/bench/LocalModeBench/Sources/LocalModeBench/Benchmark.swift new file mode 100644 index 000000000..a71ac4ae1 --- /dev/null +++ b/bench/LocalModeBench/Sources/LocalModeBench/Benchmark.swift @@ -0,0 +1,114 @@ +// +// Benchmark.swift +// LocalModeBench +// +// Measures first-token latency and decode speed for the exact model and +// parameters Clicky's Local Mode uses. Run it on the machine you're +// quoting numbers for. Build with Xcode (SwiftPM CLI can't compile MLX's +// Metal shaders): +// +// xcodebuild -scheme LocalModeBench -destination 'platform=macOS' \ +// -derivedDataPath /tmp/bench-dd -skipMacroValidation build +// /tmp/bench-dd/Build/Products/Debug/LocalModeBench +// + +import Foundation +import HuggingFace +import MLXHuggingFace +import MLXLLM +import MLXLMCommon +import Tokenizers + +@main +struct LocalModeBench { + // Keep in sync with CompanionManager's local mode system prompt — the + // prompt length affects prefill time, which dominates first-token latency. + static let localVoiceSystemPrompt = """ + you're clicky, a friendly always-on companion that lives in the user's menu bar. the user just spoke to you via push-to-talk. you're running fully on their mac right now — no cloud, no screen access — so answer from what they said and what you remember of this conversation. your reply will be spoken aloud via text-to-speech, so write the way you'd actually talk. + + rules: + - default to one or two sentences. be direct and dense. if the user asks you to go deeper, give a longer explanation. + - all lowercase, casual, warm. no emojis. + - write for the ear, not the eye. short sentences. no lists, bullet points, markdown, or formatting — just natural speech. + - don't use abbreviations or symbols that sound weird read aloud. write "for example" not "e.g.", spell out small numbers. + - you can't see the screen in local mode. if the user asks about something on their screen, say so plainly and ask them to read or describe it — or suggest flipping to a cloud model for screen questions. + - never say "simply" or "just". + - never output coordinate tags like [POINT:...] — you can't point at the screen in local mode. + """ + + static let voiceStyleQuestions = [ + "what does git rebase actually do?", + "what's the difference between a thread and a process?", + "give me a quick dinner idea with eggs and rice", + ] + + static func main() async throws { + // Same directory LocalChatProvider uses, so running the bench also + // pre-warms the app's model cache (and vice versa). + let modelCacheDirectory = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Library/Application Support/Clicky/models/huggingface", isDirectory: true) + let hubClient = HubClient(cache: HubCache(cacheDirectory: modelCacheDirectory)) + + print("loading mlx-community/Llama-3.2-3B-Instruct-4bit (downloads ~1.8 GB on first run)...") + let loadStartDate = Date() + let modelContainer = try await LLMModelFactory.shared.loadContainer( + from: #hubDownloader(hubClient), + using: #huggingFaceTokenizerLoader(), + configuration: LLMRegistry.llama3_2_3B_4bit, + progressHandler: { downloadProgress in + print(String(format: "download: %3.0f%%", downloadProgress.fractionCompleted * 100)) + } + ) + print(String(format: "model ready in %.1fs", Date().timeIntervalSince(loadStartDate))) + + // Same parameters as LocalChatProvider.localGenerateParameters. + let generateParameters = GenerateParameters( + maxTokens: 1024, + maxKVSize: 4096, + temperature: 0.7 + ) + + for (questionIndex, question) in voiceStyleQuestions.enumerated() { + let requestStartDate = Date() + var firstTokenLatencySeconds: TimeInterval? + var reportedTokensPerSecond: Double = 0 + var responseText = "" + + let languageModelInput = try await modelContainer.prepare( + input: UserInput(chat: [ + .system(localVoiceSystemPrompt), + .user(question), + ]) + ) + let generationStream = try await modelContainer.generate( + input: languageModelInput, + parameters: generateParameters + ) + + for await generation in generationStream { + switch generation { + case .chunk(let textChunk): + if firstTokenLatencySeconds == nil { + firstTokenLatencySeconds = Date().timeIntervalSince(requestStartDate) + } + responseText += textChunk + case .info(let completionInfo): + reportedTokensPerSecond = completionInfo.tokensPerSecond + case .toolCall: + break + } + } + + let totalDurationSeconds = Date().timeIntervalSince(requestStartDate) + print(String( + format: "q%d: first token %.2fs | %.1f tok/s | total %.1fs | %d chars", + questionIndex + 1, + firstTokenLatencySeconds ?? -1, + reportedTokensPerSecond, + totalDurationSeconds, + responseText.count + )) + print(" \"\(responseText.prefix(120))\(responseText.count > 120 ? "…" : "")\"") + } + } +} From edb4e681b19d9bfbd3575118bf3e9d5f0ad2c0f2 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 18:06:17 -0400 Subject: [PATCH 05/18] wire local mode end to end Local in the picker downloads the model with visible progress, then routes voice responses through on-device inference. The contract: no screenshot captured, no pointing, no conversation content in analytics, Apple Speech for STT and TTS so the loop works offline. Cloud requests fail fast with a nudge toward Local when offline instead of hanging two minutes on waitsForConnectivity. --- leanring-buddy/BuddyDictationManager.swift | 15 +- leanring-buddy/ClickyAnalytics.swift | 9 + leanring-buddy/CompanionManager.swift | 244 ++++++++++++++++++--- leanring-buddy/CompanionPanelView.swift | 56 +++++ 4 files changed, 290 insertions(+), 34 deletions(-) diff --git a/leanring-buddy/BuddyDictationManager.swift b/leanring-buddy/BuddyDictationManager.swift index 5bca26779..f3077c8c0 100644 --- a/leanring-buddy/BuddyDictationManager.swift +++ b/leanring-buddy/BuddyDictationManager.swift @@ -262,7 +262,7 @@ final class BuddyDictationManager: NSObject, ObservableObject { return AVCaptureDevice.authorizationStatus(for: .audio) == .notDetermined } - private let transcriptionProvider: any BuddyTranscriptionProvider + private var transcriptionProvider: any BuddyTranscriptionProvider private let audioEngine = AVAudioEngine() private var activeTranscriptionSession: (any BuddyStreamingTranscriptionSession)? private var activeStartSource: BuddyDictationStartSource? @@ -291,6 +291,19 @@ final class BuddyDictationManager: NSObject, ObservableObject { self.contextualKeyterms = contextualKeyterms } + /// Local Mode prefers Apple's on-device speech recognition so the whole + /// push-to-talk loop works offline. The provider is only consumed when a + /// session starts, so swapping here affects the next dictation session — + /// an in-flight session keeps the provider it started with. + func setPrefersOnDeviceTranscription(_ prefersOnDeviceTranscription: Bool) { + let newTranscriptionProvider: any BuddyTranscriptionProvider = prefersOnDeviceTranscription + ? AppleSpeechTranscriptionProvider() + : BuddyTranscriptionProviderFactory.makeDefaultProvider() + transcriptionProvider = newTranscriptionProvider + transcriptionProviderDisplayName = newTranscriptionProvider.displayName + print("🎙️ Transcription: switched to \(newTranscriptionProvider.displayName)") + } + func startPersistentDictationFromMicrophoneButton( currentDraftText: String, updateDraftText: @escaping (String) -> Void, diff --git a/leanring-buddy/ClickyAnalytics.swift b/leanring-buddy/ClickyAnalytics.swift index 29e26138e..854dcc41b 100644 --- a/leanring-buddy/ClickyAnalytics.swift +++ b/leanring-buddy/ClickyAnalytics.swift @@ -103,6 +103,15 @@ enum ClickyAnalytics { ]) } + // MARK: - Local Mode + + /// User switched the model picker to Local. Deliberately content-free — + /// in Local Mode the transcript and response events are not sent at all, + /// because a privacy mode that uploads conversations is a self-own. + static func trackLocalModeSelected() { + PostHogSDK.shared.capture("local_mode_selected") + } + // MARK: - Errors /// An error occurred during the AI response pipeline. diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index cfc50f024..edfe9c28a 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -10,6 +10,7 @@ import AVFoundation import Combine import Foundation +import Network import PostHog import ScreenCaptureKit import SwiftUI @@ -73,22 +74,56 @@ final class CompanionManager: ObservableObject { private static let workerBaseURL = "https://your-worker-name.your-subdomain.workers.dev" private lazy var claudeAPI: ClaudeAPI = { - return ClaudeAPI(proxyURL: "\(Self.workerBaseURL)/chat", model: selectedModel) + // Never hand the local sentinel to the cloud client — if the app + // launches with Local already selected, the cloud client still needs + // a real Claude model for the onboarding demo path. + let cloudModel = selectedModel == LocalChatProvider.modelPickerID + ? "claude-sonnet-4-6" + : selectedModel + return ClaudeAPI(proxyURL: "\(Self.workerBaseURL)/chat", model: cloudModel) }() /// Cloud chat provider wrapping ClaudeAPI. Voice responses route through /// the provider seam so cloud and local backends are interchangeable. private lazy var cloudChatProvider = CloudChatProvider(claudeAPI: claudeAPI) + /// On-device chat provider for Local Mode (MLX, Llama-3.2-3B-Instruct). + /// The panel reads its modelReadiness for the download progress UI. + let localChatProvider = LocalChatProvider() + + /// True when the picker is set to Local — responses run on-device, the + /// screenshot is never captured, and conversation content never leaves + /// the Mac (including analytics). + var isLocalModeActive: Bool { + selectedModel == LocalChatProvider.modelPickerID + } + /// The chat provider matching the current model selection. private var activeChatProvider: any BuddyChatProvider { - cloudChatProvider + isLocalModeActive ? localChatProvider : cloudChatProvider } private lazy var elevenLabsTTSClient: ElevenLabsTTSClient = { return ElevenLabsTTSClient(proxyURL: "\(Self.workerBaseURL)/tts") }() + /// On-device voice for Local Mode. More robotic than ElevenLabs — it's + /// the offline voice, and that tradeoff is the point. + private lazy var appleSpeechSynthesisClient = AppleSpeechSynthesisClient() + + /// The voice matching the current model selection. ElevenLabs needs the + /// network; Apple's synthesizer keeps the whole loop on the machine. + private var activeTextToSpeechClient: any BuddyTextToSpeechClient { + isLocalModeActive ? appleSpeechSynthesisClient : elevenLabsTTSClient + } + + /// Stops whichever voice is talking. Stopping both is cheap and safe, + /// and means a mode switch mid-playback can't leave audio running. + private func stopAllSpeechPlayback() { + elevenLabsTTSClient.stopPlayback() + appleSpeechSynthesisClient.stopPlayback() + } + /// Conversation history so Claude remembers prior exchanges within a session. /// Each entry is the user's transcript and Claude's response. private var conversationHistory: [(userTranscript: String, assistantResponse: String)] = [] @@ -100,6 +135,14 @@ final class CompanionManager: ObservableObject { private var shortcutTransitionCancellable: AnyCancellable? private var voiceStateCancellable: AnyCancellable? private var audioPowerCancellable: AnyCancellable? + private var localModelStateCancellable: AnyCancellable? + + /// Live network reachability, used to fail cloud requests fast when the + /// machine is offline (the URLSession is configured to wait for + /// connectivity, which would otherwise spin for two minutes) and to + /// suggest Local Mode in the panel. + private let networkPathMonitor = NWPathMonitor() + @Published private(set) var isNetworkAvailable = true private var accessibilityCheckTimer: Timer? private var pendingKeyboardShortcutStartTask: Task? /// Scheduled hide for transient cursor mode — cancelled if the user @@ -116,13 +159,44 @@ final class CompanionManager: ObservableObject { /// Used by the panel to show accurate status text ("Active" vs "Ready"). @Published private(set) var isOverlayVisible: Bool = false - /// The Claude model used for voice responses. Persisted to UserDefaults. - @Published var selectedModel: String = UserDefaults.standard.string(forKey: "selectedClaudeModel") ?? "claude-sonnet-4-6" + /// The model used for voice responses — a Claude model ID or the Local + /// sentinel. Persisted to UserDefaults. + @Published var selectedModel: String = CompanionManager.validatedStoredModelSelection() + + /// Model IDs the picker can produce. Anything else found in UserDefaults + /// (older builds, manual edits) falls back to the default, so an unknown + /// string can never reach the Anthropic API and 404. + private static let knownModelSelectionIDs = [ + "claude-sonnet-4-6", + "claude-opus-4-6", + LocalChatProvider.modelPickerID, + ] + + private static func validatedStoredModelSelection() -> String { + guard let storedModelID = UserDefaults.standard.string(forKey: "selectedClaudeModel"), + knownModelSelectionIDs.contains(storedModelID) else { + return "claude-sonnet-4-6" + } + return storedModelID + } func setSelectedModel(_ model: String) { selectedModel = model UserDefaults.standard.set(model, forKey: "selectedClaudeModel") - claudeAPI.model = model + + if model == LocalChatProvider.modelPickerID { + // Content-free event — in Local Mode the transcript/response + // analytics events don't fire at all. + ClickyAnalytics.trackLocalModeSelected() + // Start the download/load now so the first push-to-talk isn't + // stuck behind 1.8 GB of weights. + localChatProvider.loadModelIfNeeded() + // Prefer on-device transcription so the whole loop works offline. + buddyDictationManager.setPrefersOnDeviceTranscription(true) + } else { + claudeAPI.model = model + buddyDictationManager.setPrefersOnDeviceTranscription(false) + } } /// User preference for whether the Clicky cursor should be shown. @@ -192,6 +266,29 @@ final class CompanionManager: ObservableObject { // well before the onboarding demo fires at ~40s into the video. _ = claudeAPI + // Re-publish local model state changes (download progress, readiness) + // so the panel re-renders — nested ObservableObjects don't bubble + // their changes up automatically. + localModelStateCancellable = localChatProvider.objectWillChange + .receive(on: DispatchQueue.main) + .sink { [weak self] _ in + self?.objectWillChange.send() + } + + networkPathMonitor.pathUpdateHandler = { [weak self] networkPath in + Task { @MainActor [weak self] in + self?.isNetworkAvailable = networkPath.status == .satisfied + } + } + networkPathMonitor.start(queue: DispatchQueue.global(qos: .utility)) + + // If the app launched with Local already selected, warm the model + // and the on-device transcription preference right away. + if isLocalModeActive { + localChatProvider.loadModelIfNeeded() + buddyDictationManager.setPrefersOnDeviceTranscription(true) + } + // If the user already completed onboarding AND all permissions are // still granted, show the cursor overlay immediately. If permissions // were revoked (e.g. signing change), don't show the cursor — the @@ -502,7 +599,7 @@ final class CompanionManager: ObservableObject { // Cancel any in-progress response and TTS from a previous utterance currentResponseTask?.cancel() - elevenLabsTTSClient.stopPlayback() + stopAllSpeechPlayback() clearDetectedElementLocation() // Dismiss the onboarding prompt if it's showing @@ -527,10 +624,15 @@ final class CompanionManager: ObservableObject { // Partial transcripts are hidden (waveform-only UI) }, submitDraftText: { [weak self] finalTranscript in - self?.lastTranscript = finalTranscript + guard let self else { return } + self.lastTranscript = finalTranscript print("🗣️ Companion received transcript: \(finalTranscript)") - ClickyAnalytics.trackUserMessageSent(transcript: finalTranscript) - self?.sendTranscriptToClaudeWithScreenshot(transcript: finalTranscript) + // Local Mode privacy contract: the transcript never + // leaves the machine, analytics included. + if !self.isLocalModeActive { + ClickyAnalytics.trackUserMessageSent(transcript: finalTranscript) + } + self.sendTranscriptToClaudeWithScreenshot(transcript: finalTranscript) } ) } @@ -585,6 +687,23 @@ final class CompanionManager: ObservableObject { - element is on screen 2 (not where cursor is): "that's over on your other monitor — see the terminal window? [POINT:400,300:terminal:screen2]" """ + /// Trimmed system prompt for Local Mode. The local model never sees the + /// screen and can't point, so the 4.3k chars of vision and coordinate + /// instructions above would only waste a small model's attention (and + /// prefill time — prompt length is most of first-token latency). + private static let localModeVoiceResponseSystemPrompt = """ + you're clicky, a friendly always-on companion that lives in the user's menu bar. the user just spoke to you via push-to-talk. you're running fully on their mac right now — no cloud, no screen access — so answer from what they said and what you remember of this conversation. your reply will be spoken aloud via text-to-speech, so write the way you'd actually talk. + + rules: + - default to one or two sentences. be direct and dense. if the user asks you to go deeper, give a longer explanation. + - all lowercase, casual, warm. no emojis. + - write for the ear, not the eye. short sentences. no lists, bullet points, markdown, or formatting — just natural speech. + - don't use abbreviations or symbols that sound weird read aloud. write "for example" not "e.g.", spell out small numbers. + - you can't see the screen in local mode. if the user asks about something on their screen, say so plainly and ask them to read or describe it — or suggest flipping to a cloud model for screen questions. + - never say "simply" or "just". + - never output coordinate tags like [POINT:...] — you can't point at the screen in local mode. + """ + // MARK: - AI Response Pipeline /// Captures a screenshot, sends it along with the transcript to Claude, @@ -594,29 +713,70 @@ final class CompanionManager: ObservableObject { /// the buddy to fly to that element on screen. private func sendTranscriptToClaudeWithScreenshot(transcript: String) { currentResponseTask?.cancel() - elevenLabsTTSClient.stopPlayback() + stopAllSpeechPlayback() currentResponseTask = Task { // Stay in processing (spinner) state — no streaming text displayed voiceState = .processing + // Snapshot the mode once so a picker change mid-response can't + // mix cloud and local behavior within one exchange. + let isLocalModeRequest = isLocalModeActive + do { - // Capture all connected screens so the AI has full context - let screenCaptures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG() + // Local Mode isn't usable until the model is on disk and in + // memory — say so instead of leaving the user staring at a + // spinner while 1.8 GB downloads. + if isLocalModeRequest && localChatProvider.modelReadiness != .ready { + localChatProvider.loadModelIfNeeded() + try await appleSpeechSynthesisClient.speakText( + "still warming up my local brain. check the menu bar for progress and try again in a moment." + ) + voiceState = .responding + if !Task.isCancelled { + voiceState = .idle + scheduleTransientHideIfNeeded() + } + return + } - guard !Task.isCancelled else { return } + // Cloud requests hang for two minutes when offline because the + // session waits for connectivity — fail fast and point at the + // mode that still works. + if !isLocalModeRequest && !isNetworkAvailable { + speakOfflineCloudNudge() + if !Task.isCancelled { + voiceState = .idle + scheduleTransientHideIfNeeded() + } + return + } - // Build image labels with the actual screenshot pixel dimensions - // so Claude's coordinate space matches the image it sees. We - // scale from screenshot pixels to display points ourselves. - let labeledImages = screenCaptures.map { capture in - let dimensionInfo = " (image dimensions: \(capture.screenshotWidthInPixels)x\(capture.screenshotHeightInPixels) pixels)" - return (data: capture.imageData, label: capture.label + dimensionInfo) + // Local Mode contract: the screenshot is never captured — not + // captured-and-dropped. Cloud mode captures all screens like + // it always has. + var screenCaptures: [CompanionScreenCapture] = [] + var labeledImages: [(data: Data, label: String)] = [] + if !isLocalModeRequest { + // Capture all connected screens so the AI has full context + screenCaptures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG() + + guard !Task.isCancelled else { return } + + // Build image labels with the actual screenshot pixel dimensions + // so Claude's coordinate space matches the image it sees. We + // scale from screenshot pixels to display points ourselves. + labeledImages = screenCaptures.map { capture in + let dimensionInfo = " (image dimensions: \(capture.screenshotWidthInPixels)x\(capture.screenshotHeightInPixels) pixels)" + return (data: capture.imageData, label: capture.label + dimensionInfo) + } } let chatResponse = try await activeChatProvider.generateStreamingResponse( images: labeledImages, - systemPrompt: Self.companionVoiceResponseSystemPrompt, + systemPrompt: isLocalModeRequest + ? Self.localModeVoiceResponseSystemPrompt + : Self.companionVoiceResponseSystemPrompt, conversationHistory: conversationHistory, userPrompt: transcript, onTextChunk: { _ in @@ -700,19 +860,23 @@ final class CompanionManager: ObservableObject { print("🧠 Conversation history: \(conversationHistory.count) exchanges") - ClickyAnalytics.trackAIResponseReceived(response: spokenText) + // Local Mode privacy contract: the response text never leaves + // the machine, analytics included. + if !isLocalModeRequest { + ClickyAnalytics.trackAIResponseReceived(response: spokenText) + } // Play the response via TTS. Keep the spinner (processing state) // until the audio actually starts playing, then switch to responding. if !spokenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { do { - try await elevenLabsTTSClient.speakText(spokenText) - // speakText returns after player.play() — audio is now playing + try await activeTextToSpeechClient.speakText(spokenText) + // speakText returns once audio playback has started voiceState = .responding } catch { ClickyAnalytics.trackTTSError(error: error.localizedDescription) - print("⚠️ ElevenLabs TTS error: \(error)") - speakCreditsErrorFallback() + print("⚠️ TTS error: \(error)") + speakErrorFallback(isLocalModeRequest: isLocalModeRequest) } } } catch is CancellationError { @@ -720,7 +884,7 @@ final class CompanionManager: ObservableObject { } catch { ClickyAnalytics.trackResponseError(error: error.localizedDescription) print("⚠️ Companion response error: \(error)") - speakCreditsErrorFallback() + speakErrorFallback(isLocalModeRequest: isLocalModeRequest) } if !Task.isCancelled { @@ -739,8 +903,9 @@ final class CompanionManager: ObservableObject { transientHideTask?.cancel() transientHideTask = Task { - // Wait for TTS audio to finish playing - while elevenLabsTTSClient.isPlaying { + // Wait for TTS audio to finish playing. Both voices are checked + // so a mode switch mid-wait can't fade the overlay mid-sentence. + while elevenLabsTTSClient.isPlaying || appleSpeechSynthesisClient.isPlaying { try? await Task.sleep(nanoseconds: 200_000_000) guard !Task.isCancelled else { return } } @@ -760,11 +925,24 @@ final class CompanionManager: ObservableObject { } } - /// Speaks a hardcoded error message using macOS system TTS when API - /// credits run out. Uses NSSpeechSynthesizer so it works even when - /// ElevenLabs is down. - private func speakCreditsErrorFallback() { - let utterance = "I'm all out of credits. Please DM Farza and tell him to bring me back to life." + /// Speaks a hardcoded error message using macOS system TTS (works even + /// when every network service is down). The cloud line blames credits — + /// the usual cloud failure. Local Mode gets an honest local line, + /// because credits have nothing to do with on-device inference failing. + private func speakErrorFallback(isLocalModeRequest: Bool) { + let utterance = isLocalModeRequest + ? "hmm, my local brain hiccuped. give it another try." + : "I'm all out of credits. Please DM Farza and tell him to bring me back to life." + let synthesizer = NSSpeechSynthesizer() + synthesizer.startSpeaking(utterance) + voiceState = .responding + } + + /// Spoken when a cloud request is attempted with no network. Uses the + /// system synthesizer for the same reason as the error fallback — it + /// can't depend on the thing that's missing. + private func speakOfflineCloudNudge() { + let utterance = "looks like the internet is out. flip me to local in the menu bar and i can keep helping." let synthesizer = NSSpeechSynthesizer() synthesizer.startSpeaking(utterance) voiceState = .responding diff --git a/leanring-buddy/CompanionPanelView.swift b/leanring-buddy/CompanionPanelView.swift index 76789b4c6..a2c0b9829 100644 --- a/leanring-buddy/CompanionPanelView.swift +++ b/leanring-buddy/CompanionPanelView.swift @@ -31,6 +31,9 @@ struct CompanionPanelView: View { modelPickerRow .padding(.horizontal, 16) + + localModelStatusRow + .padding(.horizontal, 16) } if !companionManager.allPermissionsGranted { @@ -609,6 +612,7 @@ struct CompanionPanelView: View { HStack(spacing: 0) { modelOptionButton(label: "Sonnet", modelID: "claude-sonnet-4-6") modelOptionButton(label: "Opus", modelID: "claude-opus-4-6") + modelOptionButton(label: "Local", modelID: LocalChatProvider.modelPickerID) } .background( RoundedRectangle(cornerRadius: 6, style: .continuous) @@ -622,6 +626,58 @@ struct CompanionPanelView: View { .padding(.vertical, 4) } + /// One-line status under the picker while Local is selected: download + /// progress, load state, the privacy line once ready, or an error with + /// a retry. The panel is the stable surface for this — the cursor + /// overlay fades in and out on its own schedule. When the network is + /// down and a cloud model is selected, nudges toward Local instead. + @ViewBuilder + private var localModelStatusRow: some View { + if companionManager.selectedModel == LocalChatProvider.modelPickerID { + VStack(alignment: .leading, spacing: 6) { + switch companionManager.localChatProvider.modelReadiness { + case .notDownloaded: + EmptyView() + case .downloading(let progressFraction): + ProgressView(value: progressFraction) + .progressViewStyle(.linear) + .tint(DS.Colors.accent) + Text("downloading the local model — \(Int(progressFraction * 100))%") + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textTertiary) + case .loading: + Text("waking up the local model…") + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textTertiary) + case .ready: + Text("Local Mode — your screen stays on your Mac.") + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textTertiary) + case .failed(let errorDescription): + Text("model download hiccuped: \(errorDescription)") + .font(.system(size: 11)) + .foregroundColor(Color(red: 0.9, green: 0.4, blue: 0.4)) + .fixedSize(horizontal: false, vertical: true) + Button(action: { + companionManager.localChatProvider.loadModelIfNeeded() + }) { + Text("Try again") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + } + .buttonStyle(.plain) + .pointerCursor() + } + } + .padding(.top, 8) + } else if !companionManager.isNetworkAvailable { + Text("no internet — flip to Local to keep talking") + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textTertiary) + .padding(.top, 8) + } + } + private func modelOptionButton(label: String, modelID: String) -> some View { let isSelected = companionManager.selectedModel == modelID return Button(action: { From 338db339d7e9e29ebfd157aa526a6ee6d008497e Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 18:07:45 -0400 Subject: [PATCH 06/18] show provider and first-token latency badge during responses Turns the local-vs-cloud speed difference into something visible: a small chip near the cursor while the answer plays, with decode speed included when the local engine reports it. --- leanring-buddy/CompanionManager.swift | 20 ++++++++++++++++++++ leanring-buddy/OverlayWindow.swift | 24 ++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index edfe9c28a..13475fb69 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -43,6 +43,11 @@ final class CompanionManager: ObservableObject { /// BlueCursorView uses this instead of a random pointer phrase. @Published var detectedElementBubbleText: String? + /// Provider + first-token latency for the most recent response, shown + /// as a small badge near the cursor while the answer plays ("local · + /// 0.6s · 58 tok/s"). Cleared when the next push-to-talk begins. + @Published private(set) var lastResponseLatencyDescription: String? + // MARK: - Onboarding Video State (shared across all screen overlays) @Published var onboardingVideoPlayer: AVPlayer? @@ -601,6 +606,7 @@ final class CompanionManager: ObservableObject { currentResponseTask?.cancel() stopAllSpeechPlayback() clearDetectedElementLocation() + lastResponseLatencyDescription = nil // Dismiss the onboarding prompt if it's showing if showOnboardingPrompt { @@ -785,6 +791,20 @@ final class CompanionManager: ObservableObject { ) let fullResponseText = chatResponse.text + // Feed the latency badge: provider + first-token time, plus + // decode speed when the engine reports one (local only). + if let firstTokenLatencySeconds = chatResponse.firstTokenLatencySeconds { + var latencyDescription = String( + format: "%@ · %.1fs", + isLocalModeRequest ? "local" : "cloud", + firstTokenLatencySeconds + ) + if let tokensPerSecond = chatResponse.tokensPerSecond { + latencyDescription += String(format: " · %.0f tok/s", tokensPerSecond) + } + lastResponseLatencyDescription = latencyDescription + } + guard !Task.isCancelled else { return } // Parse the [POINT:...] tag from Claude's response diff --git a/leanring-buddy/OverlayWindow.swift b/leanring-buddy/OverlayWindow.swift index 884ebcbfb..1a139db36 100644 --- a/leanring-buddy/OverlayWindow.swift +++ b/leanring-buddy/OverlayWindow.swift @@ -294,6 +294,30 @@ struct BlueCursorView: View { } } + // Latency badge — provider + first-token time, shown while the + // answer is being spoken. The local-vs-cloud speed difference as + // something you can see, not just feel. + if isCursorOnThisScreen && companionManager.voiceState == .responding, + let latencyDescription = companionManager.lastResponseLatencyDescription { + Text(latencyDescription) + .font(.system(size: 9, weight: .medium).monospacedDigit()) + .foregroundColor(DS.Colors.textSecondary) + .padding(.horizontal, 6) + .padding(.vertical, 3) + .background( + RoundedRectangle(cornerRadius: 5, style: .continuous) + .fill(DS.Colors.surface1.opacity(0.92)) + ) + .overlay( + RoundedRectangle(cornerRadius: 5, style: .continuous) + .stroke(DS.Colors.borderSubtle, lineWidth: 0.5) + ) + .fixedSize() + .position(x: cursorPosition.x + 16, y: cursorPosition.y - 20) + .animation(.spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) + .animation(.easeIn(duration: 0.25), value: companionManager.voiceState) + } + // Blue triangle cursor — shown when idle or while TTS is playing (responding). // All three states (triangle, waveform, spinner) stay in the view tree // permanently and cross-fade via opacity so SwiftUI doesn't remove/re-insert From 467b61e3e5d4426b4a0da039940124b9af00df09 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 18:12:25 -0400 Subject: [PATCH 07/18] add demo script, remix readme, and update agent docs Benchmark table holds measured numbers from bench/LocalModeBench on the dev machine (M1 Pro 16GB): 0.6-0.7s first token, 54-60 tok/s, 3.0s model load from disk. Cloud column deliberately absent until a worker exists to measure against. --- AGENTS.md | 20 ++++++--- DECISIONS.md | 28 +++++++++++++ DEMO_SCRIPT.md | 59 ++++++++++++++++++++++++++ README-REMIX.md | 109 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 211 insertions(+), 5 deletions(-) create mode 100644 DEMO_SCRIPT.md create mode 100644 README-REMIX.md diff --git a/AGENTS.md b/AGENTS.md index 6946d4419..9a675e4ff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,9 +14,9 @@ All API keys live on a Cloudflare Worker proxy — nothing sensitive ships in th - **App Type**: Menu bar-only (`LSUIElement=true`), no dock icon or main window - **Framework**: SwiftUI (macOS native) with AppKit bridging for menu bar panel and cursor overlay - **Pattern**: MVVM with `@StateObject` / `@Published` state management -- **AI Chat**: Claude (Sonnet 4.6 default, Opus 4.6 optional) via Cloudflare Worker proxy with SSE streaming +- **AI Chat**: Pluggable chat-provider layer (`BuddyChatProvider`) — Claude (Sonnet 4.6 default, Opus 4.6 optional) via Cloudflare Worker proxy with SSE streaming, or **Local Mode**: Llama-3.2-3B-Instruct-4bit on-device via MLX (`mlx-swift-lm`) - **Speech-to-Text**: AssemblyAI real-time streaming (`u3-rt-pro` model) via websocket, with OpenAI and Apple Speech as fallbacks -- **Text-to-Speech**: ElevenLabs (`eleven_flash_v2_5` model) via Cloudflare Worker proxy +- **Text-to-Speech**: Pluggable TTS layer (`BuddyTextToSpeechClient`) — ElevenLabs (`eleven_flash_v2_5` model) via Cloudflare Worker proxy, or `AVSpeechSynthesizer` on-device in Local Mode - **Screen Capture**: ScreenCaptureKit (macOS 14.2+), multi-monitor support - **Voice Input**: Push-to-talk via `AVAudioEngine` + pluggable transcription-provider layer. System-wide keyboard shortcut via listen-only CGEvent tap. - **Element Pointing**: Claude embeds `[POINT:x,y:label:screenN]` tags in responses. The overlay parses these, maps coordinates to the correct monitor, and animates the blue cursor along a bezier arc to the target. @@ -48,15 +48,17 @@ Worker vars: `ELEVENLABS_VOICE_ID` **Transient Cursor Mode**: When "Show Clicky" is off, pressing the hotkey fades in the cursor overlay for the duration of the interaction (recording → response → TTS → optional pointing), then fades it out automatically after 1 second of inactivity. +**Local Mode**: A third model-picker option runs the whole answer loop on-device: Apple Speech for transcription, Llama-3.2-3B-Instruct-4bit via MLX for the response, `AVSpeechSynthesizer` for the voice. The behavioral contract is strict — the screenshot is *never captured* (not captured-and-dropped), `[POINT:...]` pointing is disabled via a trimmed local system prompt, and the transcript/response analytics events do not fire (a content-free `local_mode_selected` event is the only signal). The model downloads once (~1.8 GB) to `~/Library/Application Support/Clicky/models/huggingface` with progress in the panel, then loads from disk on later launches so Local Mode works fully offline. Cloud requests fail fast with a spoken nudge toward Local when the network is down (the URLSession otherwise waits 120s for connectivity). + ## Key Files | File | Lines | Purpose | |------|-------|---------| | `leanring_buddyApp.swift` | ~89 | Menu bar app entry point. Uses `@NSApplicationDelegateAdaptor` with `CompanionAppDelegate` which creates `MenuBarPanelManager` and starts `CompanionManager`. No main window — the app lives entirely in the status bar. | -| `CompanionManager.swift` | ~1026 | Central state machine. Owns dictation, shortcut monitoring, screen capture, Claude API, ElevenLabs TTS, and overlay management. Tracks voice state (idle/listening/processing/responding), conversation history, model selection, and cursor visibility. Coordinates the full push-to-talk → screenshot → Claude → TTS → pointing pipeline. | +| `CompanionManager.swift` | ~1233 | Central state machine. Owns dictation, shortcut monitoring, screen capture, the chat providers (cloud + local), both TTS clients, and overlay management. Tracks voice state (idle/listening/processing/responding), conversation history, model selection, network reachability, and cursor visibility. Coordinates the full push-to-talk → (screenshot) → AI → TTS → pointing pipeline, with the Local Mode branches (no capture, trimmed prompt, gated analytics). | | `MenuBarPanelManager.swift` | ~243 | NSStatusItem + custom NSPanel lifecycle. Creates the menu bar icon, manages the floating companion panel (show/hide/position), installs click-outside-to-dismiss monitor. | -| `CompanionPanelView.swift` | ~761 | SwiftUI panel content for the menu bar dropdown. Shows companion status, push-to-talk instructions, model picker (Sonnet/Opus), permissions UI, DM feedback button, and quit button. Dark aesthetic using `DS` design system. | -| `OverlayWindow.swift` | ~881 | Full-screen transparent overlay hosting the blue cursor, response text, waveform, and spinner. Handles cursor animation, element pointing with bezier arcs, multi-monitor coordinate mapping, and fade-out transitions. | +| `CompanionPanelView.swift` | ~817 | SwiftUI panel content for the menu bar dropdown. Shows companion status, push-to-talk instructions, model picker (Sonnet/Opus/Local) with the local model download-progress row and privacy notice, permissions UI, DM feedback button, and quit button. Dark aesthetic using `DS` design system. | +| `OverlayWindow.swift` | ~905 | Full-screen transparent overlay hosting the blue cursor, waveform, spinner, and the latency badge (provider + first-token time, shown while a response plays). Handles cursor animation, element pointing with bezier arcs, multi-monitor coordinate mapping, and fade-out transitions. | | `CompanionResponseOverlay.swift` | ~217 | SwiftUI view for the response text bubble and waveform displayed next to the cursor in the overlay. | | `CompanionScreenCaptureUtility.swift` | ~132 | Multi-monitor screenshot capture using ScreenCaptureKit. Returns labeled image data for each connected display. | | `BuddyDictationManager.swift` | ~866 | Push-to-talk voice pipeline. Handles microphone capture via `AVAudioEngine`, provider-aware permission checks, keyboard/button dictation sessions, transcript finalization, shortcut parsing, contextual keyterms, and live audio-level reporting for waveform feedback. | @@ -67,6 +69,10 @@ Worker vars: `ELEVENLABS_VOICE_ID` | `BuddyAudioConversionSupport.swift` | ~108 | Audio conversion helpers. Converts live mic buffers to PCM16 mono audio and builds WAV payloads for upload-based providers. | | `GlobalPushToTalkShortcutMonitor.swift` | ~132 | System-wide push-to-talk monitor. Owns the listen-only `CGEvent` tap and publishes press/release transitions. | | `ClaudeAPI.swift` | ~291 | Claude vision API client with streaming (SSE) and non-streaming modes. TLS warmup optimization, image MIME detection, conversation history support. | +| `BuddyChatProvider.swift` | ~112 | Protocol surface for chat backends, mirroring the transcription-provider pattern. `CloudChatProvider` wraps `ClaudeAPI` unchanged and measures first-token latency for the badge. | +| `LocalChatProvider.swift` | ~253 | On-device chat provider for Local Mode. Runs Llama-3.2-3B-Instruct-4bit via MLX (`mlx-swift-lm`); downloads once to Application Support with progress, then loads from disk (offline) on later launches. Publishes `modelReadiness` for the panel UI. | +| `BuddyTextToSpeechClient.swift` | ~28 | Protocol surface for TTS backends (`speakText` returns at playback start, `isPlaying`, synchronous `stopPlayback`). ElevenLabs and Apple Speech both conform. | +| `AppleSpeechSynthesisClient.swift` | ~73 | On-device TTS for Local Mode via `AVSpeechSynthesizer`. Picks the best installed en-US voice (premium > enhanced > compact) and honors the ElevenLabs client's timing contract. | | `OpenAIAPI.swift` | ~142 | OpenAI GPT vision API client. | | `ElevenLabsTTSClient.swift` | ~81 | ElevenLabs TTS client. Sends text to the Worker proxy, plays back audio via `AVAudioPlayer`. Exposes `isPlaying` for transient cursor scheduling. | | `ElementLocationDetector.swift` | ~335 | Detects UI element locations in screenshots for cursor pointing. | @@ -88,6 +94,10 @@ open leanring-buddy.xcodeproj # deprecated onChange warning in OverlayWindow.swift. Do NOT attempt to fix these. ``` +Local Mode needs Xcode 16.3+ (mlx-swift-lm is a Swift 6.1-tools package and uses Swift +macros — Xcode will ask once to "Trust & Enable" the `MLXHuggingFaceMacros` plugin; say yes). +MLX's Metal shaders can't be compiled by the SwiftPM CLI, so builds must go through Xcode. + **Do NOT run `xcodebuild` from the terminal** — it invalidates TCC (Transparency, Consent, and Control) permissions and the app will need to re-request screen recording, accessibility, etc. ## Cloudflare Worker diff --git a/DECISIONS.md b/DECISIONS.md index 0405ee27a..c2b197bbc 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -95,6 +95,34 @@ Each entry: the call, and why. Deviations from the original plan are marked **[d `ElementLocationDetector`, the camera entitlement, or any known warning — not our diff. - Streaming TTS, sentence-chunked TTS — escape hatch only if full-response TTS feels bad. +## Decisions made during implementation + +- **[deviation] Offline fast-fail for cloud requests.** ClaudeAPI's URLSession sets + `waitsForConnectivity`, so with wifi off a cloud request spins for up to 120 seconds — + unusable and undemoable. CompanionManager now checks an `NWPathMonitor` before cloud + requests and speaks the "flip me to local" nudge immediately. This changes cloud behavior + only in the no-network case (120s hang → instant honest answer); `ClaudeAPI` itself is + untouched. +- **[deviation] Benchmark harness committed at `bench/LocalModeBench`.** "Real measurements + only" needs a reproducible source: same model, cache directory, and generation parameters + as the app. The README table comes from its output on this machine. +- **Model-not-ready voice line.** Push-to-talk in Local Mode before the download/load + finishes speaks "still warming up my local brain…" instead of holding the spinner hostage + to a 1.8 GB download. +- **Latency badge lives in the cursor overlay**, above the triangle, only during + `.responding` — the menu-bar panel is auto-dismissed when push-to-talk starts, so it can't + host a badge the demo can see. Muted chip (surface fill, 9pt), not a second blue bubble. +- **Offline relaunch path is load-from-directory, not hub-cache-hit.** After the first + download the resolved model directory is persisted; later launches call the + `loadContainer(from: directory)` overload with zero network involvement, rather than + trusting the hub client's cache lookup to behave offline. +- **Stray `[POINT:...]` tags from the local model are stripped, not rendered.** The existing + end-anchored parse runs in both modes; in Local Mode the coordinate is unusable by + construction (no screen capture to map pixels with), so the tag is removed from the spoken + text and nothing flies. +- **CLI builds use `-skipMacroValidation`** (mlx-swift-lm ships Swift macros); in the Xcode + GUI this is the one-time "Trust & Enable" prompt. + ## Process - **Terminal `xcodebuild` is used for compile verification only while this machine has zero diff --git a/DEMO_SCRIPT.md b/DEMO_SCRIPT.md new file mode 100644 index 000000000..7e066fc0e --- /dev/null +++ b/DEMO_SCRIPT.md @@ -0,0 +1,59 @@ +# DEMO SCRIPT — Clicky Local Mode, 60 seconds + +One continuous screen recording plus a phone shot of the wifi toggle if you want drama. +Practice the run twice; the beats are tight. + +## Prep (do all of this before recording) + +1. **Build & run from Xcode** (never `xcodebuild` from terminal once permissions are granted): + open `leanring-buddy.xcodeproj`, Signing & Capabilities → select your personal team, + Cmd+R. Xcode will ask to **Trust & Enable** the `MLXHuggingFaceMacros` plugin — say yes. +2. Complete onboarding once (grant all four permissions, email, Start). +3. **Cloud side**: the worker URL must be set for the cloud beats — + `CompanionManager.swift` `workerBaseURL` and the AssemblyAI token URL in + `AssemblyAIStreamingTranscriptionProvider.swift`. No worker? Record shots 3–5 only; + the local story carries the video. +4. **While online**: open the panel, click **Local**. Watch the progress bar; wait for + *"Local Mode — your screen stays on your Mac."* (~1.8 GB first time; instant if the + bench already ran). First push-to-talk in Local will prompt for Speech Recognition — grant it. +5. Quit and relaunch once. Pick Local again — it loads from disk, no network. This is the + path the demo relies on. +6. Optional but worth it: System Settings → Accessibility → Spoken Content → System Voice → + Manage Voices → download an **Enhanced** en-US voice. The local voice picks the best + installed automatically. +7. Flip back to **Sonnet** before recording starts. + +## Shots + +**1. (0–10s) Cloud baseline.** +Screen with something real (code, a chart). Hold ctrl+option: *"what am i looking at here?"* +Clicky answers in the ElevenLabs voice, points at the thing. Latency badge reads +`cloud · ~2s`. No narration needed — this is just "clicky works as always." + +**2. (10–20s) Kill the wifi. On camera.** +Menu bar → wifi off. Hold ctrl+option, ask anything. Clicky answers **instantly, out loud**: +*"looks like the internet is out. flip me to local in the menu bar and i can keep helping."* +(No spinner-of-death — the app fails fast and tells you what to do.) + +**3. (20–40s) Flip to Local. The money shot.** +Open the panel — picker now shows **Sonnet / Opus / Local**. Click Local. The line under it: +*"Local Mode — your screen stays on your Mac."* Close panel. Hold ctrl+option: +*"explain what a mutex is."* Spinner for under a second, then the answer — spoken, wifi still +off. Badge reads `local · 0.6s · 58 tok/s`. Let the badge sit on screen a beat. + +**4. (40–55s) The privacy line.** +Ask a follow-up (*"give me an example in swift"*) to show it remembers context. Over it, +one spoken or caption line: **"wifi's off. the screenshot was never taken. nothing left +this mac — not even analytics."** + +**5. (55–60s) Card.** +> **Clicky Local Mode** — built on MLX. +> 0.6s to first token · 58 tok/s · $0 · works on a plane. +> Weekend remix by Shrey Patel / Coconut Labs. + +## If a beat goes sideways + +- Local answer feels slow on the first try → that's the cold first generation (~1.5s first + token); ask a second question, it drops to ~0.6s. Record the second. +- Apple voice sounds flat → it's the offline voice, own it; the README explains the tradeoff. +- Pointing fires in shot 1 but flies somewhere dumb → re-record shot 1 with a simpler screen. diff --git a/README-REMIX.md b/README-REMIX.md new file mode 100644 index 000000000..ec1250433 --- /dev/null +++ b/README-REMIX.md @@ -0,0 +1,109 @@ +# Clicky Local Mode + +Flip the model picker to **Local** and clicky answers on your Mac. First token in 0.6 +seconds. Costs nothing per question. Works on a plane. And your screen never leaves the +machine — in Local Mode the screenshot is not captured at all. + +This is a weekend remix of the open-source clicky by Shrey Patel ([Coconut Labs](https://coconutlabs.org)). +I build inference systems — local inference on Apple Silicon and fairness scheduling for +shared inference servers — so this is the feature I couldn't not build. + +## Why this feature + +**Privacy.** An assistant that sees everything you see has to earn trust structurally, not +with a policy page. Local Mode is the structural answer: no screenshot, no upload, nothing +to leak. While mapping the code I also found the analytics layer ships full transcripts and +full AI responses to PostHog — so in Local Mode those events don't fire either. A privacy +mode that phones home is a self-own. + +**Cost.** The kid in Lagos doesn't have an API budget. A 3B model on the GPU he already +owns makes the buddy free at the margin. + +**Latency.** No network round trip, no proxy hop. 0.6s to first token is a felt difference +in a cursor companion. + +**The platform play.** Apple Intelligence is coming for this category. A clicky that's +natively excellent on Apple Silicon — Metal, MLX, on-device speech both directions — is +the counter-move. + +## What got built + +- **`BuddyChatProvider`** — a protocol seam for chat backends, mirroring the + `BuddyTranscriptionProvider` pattern already in the codebase. The cloud path wraps the + existing `ClaudeAPI` untouched; Sonnet/Opus behave exactly as before. +- **`LocalChatProvider`** — Llama-3.2-3B-Instruct (4-bit, ~1.8 GB) via + [mlx-swift-lm](https://github.com/ml-explore/mlx-swift-lm). Downloads once with a progress + bar in the panel, then loads from disk in 3 seconds on every later launch — fully offline. +- **Offline voice loop** — Local Mode switches STT to Apple Speech (on-device) and TTS to + `AVSpeechSynthesizer`. Push-to-talk → transcription → answer → spoken reply, wifi off. +- **A strict behavioral contract** — no screenshot captured (not captured-and-dropped), + cursor pointing disabled (a 3B model guessing screen coordinates is a clown show), a + trimmed local system prompt, and no conversation content in analytics. +- **Latency badge** — provider + first-token time next to the cursor while the answer + plays (`local · 0.6s · 58 tok/s`). The benchmark as UI. +- **Offline fast-fail** — cloud requests used to hang up to 120s with no network + (`waitsForConnectivity`). Now clicky says it's offline and points you at Local, instantly. + +``` +push-to-talk ──► speech-to-text ──► BuddyChatProvider ──► text-to-speech + │ │ │ + cloud: AssemblyAI ClaudeAPI (worker) ElevenLabs + local: Apple Speech MLX · Llama-3.2-3B AVSpeech + no screenshot · no pointing + no content analytics +``` + +## Measured numbers + +M1 Pro, 16 GB, macOS 26.6. Measured with `bench/LocalModeBench` — same model, same cache +directory, same generation parameters as the app. Run it yourself; never trust a README. + +| metric | local mode | +|---|---| +| first token | **0.6–0.7 s** | +| decode speed | **54–60 tok/s** | +| model load from disk | 3.0 s | +| one-time download | 1.8 GB (83 s for me) | +| cost per question | $0 | +| network needed | none | + +Cloud numbers aren't in this table because this clone ships placeholder worker URLs and I +won't print numbers I didn't measure. The in-app latency badge shows both live — +typical Claude time-to-first-token through the worker is on the order of 1.5–2.5s, and the +demo video shows the side-by-side. + +```bash +cd bench/LocalModeBench +xcodebuild -scheme LocalModeBench -destination 'platform=macOS' \ + -derivedDataPath /tmp/bench-dd -skipMacroValidation build +/tmp/bench-dd/Build/Products/Debug/LocalModeBench +``` + +## The honest tradeoffs + +- The Apple voice is robotic next to ElevenLabs. It's the *offline* voice — the tradeoff is + the point, and you can soften it by downloading an Enhanced voice in System Settings. +- A 3B model is great at quick explanations and conversation, and it is not Opus. That's + why it's a picker option, not a replacement. +- No local vision yet. Local Mode says so plainly when you ask about the screen, instead of + hallucinating coordinates. See below. + +## What I'd build next + +1. **An auto-routing brain.** Short question, no screen needed → local. Screen question or + deep reasoning → cloud. The picker disappears; clicky just feels instant and cheap. +2. **Fully-local screen understanding.** A 2B-class VLM via MLX answering single-screenshot + questions on-device — true private screen help, the half of clicky Local Mode can't do yet. +3. **Fleet-scale inference fairness.** When thousands of clicky buddies share hosted + inference, one chatty user starves the rest — scheduling and fairness on the inference + server is the moat nobody sees coming. This is literally my research lane + ([KVWarden](https://kvwarden.org): tenant-fair scheduling for shared LLM servers). + +## Running it + +Xcode 16.3+ (Swift macros — Trust & Enable the `MLXHuggingFaceMacros` plugin when asked). +Open `leanring-buddy.xcodeproj`, set your signing team, Cmd+R. Pick **Local** in the panel, +let the model download once, then turn your wifi off and talk to it. + +MIT, like the original. Thanks to Farza for open-sourcing a codebase fun enough to spend a +weekend inside. From b689861e7cfb7a86e48d6724d5c7e51b00f43a15 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 18:12:42 -0400 Subject: [PATCH 08/18] pin resolved package versions for local mode deps --- bench/LocalModeBench/Package.resolved | 141 ++++++++++++++++++ .../xcshareddata/swiftpm/Package.resolved | 137 ++++++++++++++++- 2 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 bench/LocalModeBench/Package.resolved diff --git a/bench/LocalModeBench/Package.resolved b/bench/LocalModeBench/Package.resolved new file mode 100644 index 000000000..ae1286924 --- /dev/null +++ b/bench/LocalModeBench/Package.resolved @@ -0,0 +1,141 @@ +{ + "originHash" : "91d93bfc755da5aaac28b35f28612d3b59ff1711a5af2c8f2c90dcb772047b6d", + "pins" : [ + { + "identity" : "eventsource", + "kind" : "remoteSourceControl", + "location" : "https://github.com/mattt/EventSource.git", + "state" : { + "revision" : "a3a85a85214caf642abaa96ae664e4c772a59f6e", + "version" : "1.4.1" + } + }, + { + "identity" : "mlx-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ml-explore/mlx-swift", + "state" : { + "revision" : "dc43e62d7055353c7f99fa071a4e71d29dfddc44", + "version" : "0.31.4" + } + }, + { + "identity" : "mlx-swift-lm", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ml-explore/mlx-swift-lm", + "state" : { + "revision" : "1c05248bb0899e2a7a4962b84d319cf12f4e12aa", + "version" : "3.31.3" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "a9a5efd40eaf558a2bcd48d64b1d1646be686008", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "b601256eab081c0f92f059e12818ac1d4f178ff7", + "version" : "1.3.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "1b6b2e274e85105bfa155183145a1dcfd63331f1", + "version" : "4.5.0" + } + }, + { + "identity" : "swift-huggingface", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-huggingface", + "state" : { + "revision" : "b721959445b617d0bf03910b2b4aced345fd93bf", + "version" : "0.9.0" + } + }, + { + "identity" : "swift-jinja", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-jinja.git", + "state" : { + "revision" : "0b67ecb79139f6addef8699eff3622808aa6c7dc", + "version" : "2.3.6" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "77b84ac2cd2ac9e4ac67d19f045fd5b434f56967", + "version" : "2.101.0" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax.git", + "state" : { + "revision" : "0687f71944021d616d34d922343dcef086855920", + "version" : "600.0.1" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "666fba77fc401d8e63aa37435b1257fc88c40518", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-transformers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-transformers", + "state" : { + "revision" : "2fa33e1f5e7131a7fc64c28e6d161dcec0d24820", + "version" : "1.3.3" + } + }, + { + "identity" : "yyjson", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ibireme/yyjson.git", + "state" : { + "revision" : "8b4a38dc994a110abaec8a400615567bd996105f", + "version" : "0.12.0" + } + } + ], + "version" : 3 +} diff --git a/leanring-buddy.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/leanring-buddy.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index d88adb216..b62f4ea05 100644 --- a/leanring-buddy.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/leanring-buddy.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,6 +1,33 @@ { - "originHash" : "3c6fb67fefedcfcd00708e24ca8088151f21dccfc0ade32ea80c406646277e89", + "originHash" : "6aff27d8175449f7f20a9c686cdf2735f3363e69249e4f5a5bf546642b6aaaf7", "pins" : [ + { + "identity" : "eventsource", + "kind" : "remoteSourceControl", + "location" : "https://github.com/mattt/EventSource.git", + "state" : { + "revision" : "a3a85a85214caf642abaa96ae664e4c772a59f6e", + "version" : "1.4.1" + } + }, + { + "identity" : "mlx-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ml-explore/mlx-swift", + "state" : { + "revision" : "dc43e62d7055353c7f99fa071a4e71d29dfddc44", + "version" : "0.31.4" + } + }, + { + "identity" : "mlx-swift-lm", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ml-explore/mlx-swift-lm", + "state" : { + "revision" : "1c05248bb0899e2a7a4962b84d319cf12f4e12aa", + "version" : "3.31.3" + } + }, { "identity" : "plcrashreporter", "kind" : "remoteSourceControl", @@ -27,6 +54,114 @@ "revision" : "21d8df80440b1ca3b65fa82e40782f1e5a9e6ba2", "version" : "2.9.0" } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "a9a5efd40eaf558a2bcd48d64b1d1646be686008", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "b601256eab081c0f92f059e12818ac1d4f178ff7", + "version" : "1.3.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "1b6b2e274e85105bfa155183145a1dcfd63331f1", + "version" : "4.5.0" + } + }, + { + "identity" : "swift-huggingface", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-huggingface", + "state" : { + "revision" : "b721959445b617d0bf03910b2b4aced345fd93bf", + "version" : "0.9.0" + } + }, + { + "identity" : "swift-jinja", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-jinja.git", + "state" : { + "revision" : "0b67ecb79139f6addef8699eff3622808aa6c7dc", + "version" : "2.3.6" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "77b84ac2cd2ac9e4ac67d19f045fd5b434f56967", + "version" : "2.101.0" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax.git", + "state" : { + "revision" : "0687f71944021d616d34d922343dcef086855920", + "version" : "600.0.1" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "666fba77fc401d8e63aa37435b1257fc88c40518", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-transformers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-transformers", + "state" : { + "revision" : "2fa33e1f5e7131a7fc64c28e6d161dcec0d24820", + "version" : "1.3.3" + } + }, + { + "identity" : "yyjson", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ibireme/yyjson.git", + "state" : { + "revision" : "8b4a38dc994a110abaec8a400615567bd996105f", + "version" : "0.12.0" + } } ], "version" : 3 From ed2ddefdfffe199552c39b4284b63a725178b5b9 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 18:35:57 -0400 Subject: [PATCH 09/18] let the bench take a one-off question for demos A custom question prints its full answer; the default sweep keeps short previews. --- .../Sources/LocalModeBench/Benchmark.swift | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/bench/LocalModeBench/Sources/LocalModeBench/Benchmark.swift b/bench/LocalModeBench/Sources/LocalModeBench/Benchmark.swift index a71ac4ae1..c65cfdbbc 100644 --- a/bench/LocalModeBench/Sources/LocalModeBench/Benchmark.swift +++ b/bench/LocalModeBench/Sources/LocalModeBench/Benchmark.swift @@ -36,12 +36,19 @@ struct LocalModeBench { - never output coordinate tags like [POINT:...] — you can't point at the screen in local mode. """ - static let voiceStyleQuestions = [ + static let defaultVoiceStyleQuestions = [ "what does git rebase actually do?", "what's the difference between a thread and a process?", "give me a quick dinner idea with eggs and rice", ] + /// Pass a question as the first argument to run it instead of the + /// default set — handy for demos and one-off latency checks. + static var voiceStyleQuestions: [String] { + let customQuestion = CommandLine.arguments.dropFirst().joined(separator: " ") + return customQuestion.isEmpty ? defaultVoiceStyleQuestions : [customQuestion] + } + static func main() async throws { // Same directory LocalChatProvider uses, so running the bench also // pre-warms the app's model cache (and vice versa). @@ -108,7 +115,13 @@ struct LocalModeBench { totalDurationSeconds, responseText.count )) - print(" \"\(responseText.prefix(120))\(responseText.count > 120 ? "…" : "")\"") + // Custom one-off questions get the full answer (demo use); + // the default sweep keeps a short preview so the table stays readable. + if voiceStyleQuestions.count == 1 { + print(" \"\(responseText)\"") + } else { + print(" \"\(responseText.prefix(120))\(responseText.count > 120 ? "…" : "")\"") + } } } } From cce853d0819aed05fd0e9024a48572a32829250e Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 18:59:55 -0400 Subject: [PATCH 10/18] design offline autonomous takeover mode (spike) Local VLM (Qwen2.5-VL-3B via MLXVLM) drives a guarded computer-use loop. Dry-run default, kill switch, offline-only enforcement, confirm gate before irreversible actions. Input monitoring stays on-device. --- TAKEOVER.md | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 TAKEOVER.md diff --git a/TAKEOVER.md b/TAKEOVER.md new file mode 100644 index 000000000..e21a94243 --- /dev/null +++ b/TAKEOVER.md @@ -0,0 +1,86 @@ +# TAKEOVER — offline autonomous mode (spike) + +Status: **spike, in progress.** This is the "Jarvis" feature — clicky takes over the cursor +and keyboard and does a task for you. The hard constraint, from the user: **offline only.** + +## The contradiction, resolved + +"Autonomous + offline" can't run on the Local Mode text model. A computer-use loop has to +*see* the screen to drive it, and Llama-3.2-3B is text-only — which is exactly why Local +Mode disables screenshots and pointing. So takeover swaps in a local **vision** model: + +- `MLXVLM` ships in the same `mlx-swift-lm` package already wired into the app. +- `VLMRegistry.qwen2_5VL3BInstruct4Bit` and `.qwen3VL4BInstruct4Bit` are registry constants + — small enough for a 16 GB M1 Pro, 4-bit. +- Screenshot → `UserInput.Image.ciImage(CIImage)` → same `ModelContainer.prepare/generate` + path as the text model. + +**The honest reality:** small local VLMs are unreliable at precise UI coordinate grounding. +A 3B model will misclick. Frontier cloud models nail "click that exact button"; a 3B on +your Mac won't, consistently. So the rails below aren't bureaucracy — they're what makes a +misgrounding click harmless instead of destructive. + +## Loop + +``` +"hey clicky, take over — " + └─ precheck: in Local Mode? network actually off? VLM loaded? (else refuse, say why) + └─ loop, capped at N steps and M seconds: + screenshot ─► downscale ─► local VLM ─► one JSON action + ▲ (cursor screen only) │ + │ ▼ + │ guardrail check ─► execute (or dry-run) ─► narrate via TTS + └──────────────── re-screenshot ◄─────────────────────┘ + exit on: action=="done" · step/time cap · stuck-loop detect · KILL SWITCH +``` + +Action schema (minimal — what a 3B can reliably emit): +`{"action":"click|type|scroll|key|done|give_up", ...args, "why":"..."}` + +## Guardrails (non-negotiable, priority order) + +1. **Offline-only enforcement.** Refuses to start unless the picker is on Local *and* + `NWPathMonitor` reports no network. The whole point is privacy; takeover that could phone + home is off the table. +2. **Kill switch.** A global key (ESC held, or a dedicated combo) aborts mid-loop instantly, + via a listen-only CGEvent tap that the synthesizer can't trigger itself. +3. **Dry-run by default.** Clicky moves the cursor to where it *would* act and narrates the + step — but does not click or type. Safe to demo on anything. Real actions require an + explicit **arm** toggle. +4. **Confirm gate before irreversible actions** (even when armed): typing into a field, + pressing return/enter, any key combo, or clicking a control whose label matches + send/delete/buy/confirm/submit/pay. Clicky asks first. +5. **Hard caps.** Step count and wall-clock ceiling; stuck-loop detection (same + action/screenshot repeating) bails with a spoken "i'm stuck on this one." + +## Input monitoring (the "watch what I do" piece) + +Listen-only observation of your clicks + keystrokes so clicky has context for takeover +("you were just in VS Code on line 40"). Strict shape: + +- **Opt-in toggle**, off by default. **Offline-only.** On-device **bounded ring buffer** + (last ~N events), **never transmitted, never persisted to disk.** +- Uses a passive `CGEventTapOptions.listenOnly` tap — likely needs the separate macOS + **Input Monitoring** permission (`kTCCServiceListenEvent`), distinct from Accessibility. + (Recon confirming.) Usage string added to Info.plist. +- This is you watching your own machine for your own assistant — kept local and gated so it + stays that and only that. + +## What this is and isn't + +- It's a constrained, rail-guarded, offline computer-use agent on a small local VLM. A real + "oh damn" demo beat on a *safe, bounded task* ("open hacker news, scroll to the top + comment"). +- It is not a reliable hands-off operator. ~3–8 s per step on M1 Pro, and it will misground. + Dry-run is the default for a reason. + +## Build order + +1. `ComputerUseActionExecutor` — CGEvent synthesis (move/click/type/scroll/key) with a + dry-run flag and the kill-switch tap. Lowest-level, testable in isolation. +2. `LocalVisionAgent` — VLM load (reuses LocalChatProvider's download/cache machinery), + the JSON-action prompt, the loop, stuck detection, caps. +3. `TakeoverController` on CompanionManager — offline/local enforcement, arm/dry-run state, + confirm gates, TTS narration, wires to the existing screenshot utility + voice. +4. Input-monitor observer + its permission + Info.plist string + opt-in toggle in the panel. +5. Trigger phrase ("take over") detected in the transcript, routed to TakeoverController. From d19c16b475d21838228d8be830daa322b13bf5d6 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 19:24:59 -0400 Subject: [PATCH 11/18] add offline takeover engine: vision agent + CGEvent executor + guarded loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qwen2.5-VL-3B (MLXVLM) emits one JSON action per screenshot; the controller guards and executes via CoreGraphics input synthesis. Rails: offline+Accessibility precondition, ESC kill switch, step + wall-clock caps, dry-run by default, danger gate behind a second opt-in. 'take over — ' in a transcript routes here. --- leanring-buddy.xcodeproj/project.pbxproj | 6 + leanring-buddy/CompanionManager.swift | 67 ++++ .../ComputerUseActionExecutor.swift | 242 +++++++++++++++ leanring-buddy/TakeoverAction.swift | 156 ++++++++++ leanring-buddy/TakeoverController.swift | 293 ++++++++++++++++++ leanring-buddy/TakeoverVisionAgent.swift | 171 ++++++++++ 6 files changed, 935 insertions(+) create mode 100644 leanring-buddy/ComputerUseActionExecutor.swift create mode 100644 leanring-buddy/TakeoverAction.swift create mode 100644 leanring-buddy/TakeoverController.swift create mode 100644 leanring-buddy/TakeoverVisionAgent.swift diff --git a/leanring-buddy.xcodeproj/project.pbxproj b/leanring-buddy.xcodeproj/project.pbxproj index 8cb0909dc..ae9a44ce4 100644 --- a/leanring-buddy.xcodeproj/project.pbxproj +++ b/leanring-buddy.xcodeproj/project.pbxproj @@ -126,6 +126,7 @@ CC00AA062F6600060039DA55 /* MLXHuggingFace */, CC00AA072F6600070039DA55 /* HuggingFace */, CC00AA082F6600080039DA55 /* Tokenizers */, + CC00AA092F6600090039DA55 /* MLXVLM */, ); productName = "leanring-buddy"; productReference = 28F22CBF2F56440300A0FC59 /* Clicky.app */; @@ -686,6 +687,11 @@ package = CC00AA032F6600030039DA55 /* XCRemoteSwiftPackageReference "swift-transformers" */; productName = Tokenizers; }; + CC00AA092F6600090039DA55 /* MLXVLM */ = { + isa = XCSwiftPackageProductDependency; + package = CC00AA012F6600010039DA55 /* XCRemoteSwiftPackageReference "mlx-swift-lm" */; + productName = MLXVLM; + }; /* End XCSwiftPackageProductDependency section */ }; rootObject = 28F22CB72F56440300A0FC59 /* Project object */; diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index 13475fb69..1094decbd 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -96,6 +96,58 @@ final class CompanionManager: ObservableObject { /// The panel reads its modelReadiness for the download progress UI. let localChatProvider = LocalChatProvider() + // MARK: - Takeover (offline autonomous mode) + + /// The local vision model that drives takeover. Separate from the text + /// chat model — takeover needs to see the screen. + let takeoverVisionAgent = TakeoverVisionAgent() + private let takeoverActionExecutor = ComputerUseActionExecutor() + + /// The offline autonomous loop. Dry-run by default; offline + Accessibility + /// enforced before it will run. + lazy var takeoverController: TakeoverController = { + TakeoverController( + visionAgent: takeoverVisionAgent, + executor: takeoverActionExecutor, + captureCursorScreen: { + let captures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG() + return captures.first(where: { $0.isCursorScreen }) ?? captures.first + }, + narrate: { [weak self] line in + // Offline voice only — takeover never touches the cloud TTS. + try? await self?.appleSpeechSynthesisClient.speakText(line) + }, + isNetworkAvailable: { [weak self] in self?.isNetworkAvailable ?? true } + ) + }() + + private var takeoverStateCancellable: AnyCancellable? + private var takeoverModelCancellable: AnyCancellable? + + /// Phrases that hand control to takeover mode. The rest of the utterance + /// becomes the task. + private static let takeoverTriggerPhrases = ["take over", "takeover"] + + /// If the transcript is a takeover command, routes it and returns true. + /// Offline + Accessibility are enforced inside TakeoverController.start. + private func routeTranscriptToTakeoverIfRequested(_ transcript: String) -> Bool { + let lowered = transcript.lowercased() + guard let phrase = Self.takeoverTriggerPhrases.first(where: { lowered.contains($0) }) else { + return false + } + // The task is whatever follows the trigger phrase. + var task = transcript + if let range = lowered.range(of: phrase) { + task = String(transcript[range.upperBound...]) + } + task = task.trimmingCharacters(in: CharacterSet(charactersIn: " ,.—-:")).trimmingCharacters(in: .whitespacesAndNewlines) + guard !task.isEmpty else { return false } + + takeoverVisionAgent.loadModelIfNeeded() + takeoverController.start(task: task) + return true + } + /// True when the picker is set to Local — responses run on-device, the /// screenshot is never captured, and conversation content never leaves /// the Mac (including analytics). @@ -280,6 +332,15 @@ final class CompanionManager: ObservableObject { self?.objectWillChange.send() } + // Re-publish takeover state (run progress, model download) so the panel + // re-renders — nested ObservableObjects don't bubble up automatically. + takeoverStateCancellable = takeoverController.objectWillChange + .receive(on: DispatchQueue.main) + .sink { [weak self] _ in self?.objectWillChange.send() } + takeoverModelCancellable = takeoverVisionAgent.objectWillChange + .receive(on: DispatchQueue.main) + .sink { [weak self] _ in self?.objectWillChange.send() } + networkPathMonitor.pathUpdateHandler = { [weak self] networkPath in Task { @MainActor [weak self] in self?.isNetworkAvailable = networkPath.status == .satisfied @@ -633,6 +694,12 @@ final class CompanionManager: ObservableObject { guard let self else { return } self.lastTranscript = finalTranscript print("🗣️ Companion received transcript: \(finalTranscript)") + // "take over — " hands off to the offline autonomous + // loop instead of a normal voice response. + if self.routeTranscriptToTakeoverIfRequested(finalTranscript) { + self.voiceState = .idle + return + } // Local Mode privacy contract: the transcript never // leaves the machine, analytics included. if !self.isLocalModeActive { diff --git a/leanring-buddy/ComputerUseActionExecutor.swift b/leanring-buddy/ComputerUseActionExecutor.swift new file mode 100644 index 000000000..71e05f79b --- /dev/null +++ b/leanring-buddy/ComputerUseActionExecutor.swift @@ -0,0 +1,242 @@ +// +// ComputerUseActionExecutor.swift +// leanring-buddy +// +// Low-level macOS input synthesis for takeover mode — moves the cursor, +// clicks, types, scrolls, and presses key combos via CoreGraphics Quartz +// Event Services. Needs only the Accessibility grant the app already holds +// (App Sandbox is off). Every synthesized event is stamped with a sentinel +// so the kill-switch tap can tell the app's own events from the user's. +// + +import AppKit +import Carbon.HIToolbox +import CoreGraphics +import Foundation + +@MainActor +final class ComputerUseActionExecutor { + /// Stamped into every synthesized event (kCGEventSourceUserData), so the + /// kill-switch tap ignores our own events. "CLKY". + private static let eventSentinel: Int64 = 0x434C_4B59 + + /// Synthesizing input needs the Accessibility TCC grant (distinct from the + /// Screen Recording grant the app already uses for screenshots). + var isAccessibilityTrusted: Bool { AXIsProcessTrusted() } + + // MARK: - Coordinate conversion + + /// Converts an AppKit global point (bottom-left origin, Y-up — what the + /// rest of the app and the pointing pipeline use) to the Quartz global + /// point CGEvent expects (top-left origin, Y-down). The flip is ALWAYS + /// against the primary display height, even for points on a secondary + /// monitor — this is the #1 takeover bug if you get it wrong. + private func quartzPoint(fromAppKitGlobal appKitPoint: CGPoint) -> CGPoint { + let primaryDisplayHeight = NSScreen.screens.first?.frame.height ?? appKitPoint.y + return CGPoint(x: appKitPoint.x, y: primaryDisplayHeight - appKitPoint.y) + } + + private func eventSource() -> CGEventSource? { + CGEventSource(stateID: .combinedSessionState) + } + + private func stamp(_ event: CGEvent) { + event.setIntegerValueField(.eventSourceUserData, value: Self.eventSentinel) + } + + // MARK: - Cursor + + /// Teleports the visible cursor without posting a move event. Used in + /// dry-run to show where clicky *would* act — harmless (no click, no app + /// notification). + func moveCursorVisible(toAppKitGlobal appKitPoint: CGPoint) { + CGWarpMouseCursorPosition(quartzPoint(fromAppKitGlobal: appKitPoint)) + } + + // MARK: - Click + + func click(atAppKitGlobal appKitPoint: CGPoint) { + let point = quartzPoint(fromAppKitGlobal: appKitPoint) + // Warp first so hover state is consistent before the click lands. + CGWarpMouseCursorPosition(point) + guard let down = CGEvent(mouseEventSource: eventSource(), mouseType: .leftMouseDown, + mouseCursorPosition: point, mouseButton: .left), + let up = CGEvent(mouseEventSource: eventSource(), mouseType: .leftMouseUp, + mouseCursorPosition: point, mouseButton: .left) else { return } + stamp(down); stamp(up) + down.post(tap: .cghidEventTap) + up.post(tap: .cghidEventTap) + } + + // MARK: - Type + + /// Types arbitrary text via Unicode injection (emoji/non-ASCII safe). + /// Used only for literal text entry — shortcuts go through `pressKey` so + /// they honor the keyboard layout. + func typeText(_ text: String) { + for character in text { + let utf16Scalars = Array(String(character).utf16) + guard let down = CGEvent(keyboardEventSource: eventSource(), virtualKey: 0, keyDown: true), + let up = CGEvent(keyboardEventSource: eventSource(), virtualKey: 0, keyDown: false) else { continue } + utf16Scalars.withUnsafeBufferPointer { buffer in + down.keyboardSetUnicodeString(stringLength: utf16Scalars.count, unicodeString: buffer.baseAddress) + up.keyboardSetUnicodeString(stringLength: utf16Scalars.count, unicodeString: buffer.baseAddress) + } + stamp(down); stamp(up) + down.post(tap: .cghidEventTap) + up.post(tap: .cghidEventTap) + } + } + + // MARK: - Key combo + + /// Presses a combo like "return", "esc", "cmd+s", "cmd+shift+t". + /// Returns false if the combo's main key isn't recognized. + @discardableResult + func pressKey(combo: String) -> Bool { + var flags: CGEventFlags = [] + var mainKeyCode: CGKeyCode? + + for rawToken in combo.lowercased().split(separator: "+") { + let token = rawToken.trimmingCharacters(in: .whitespaces) + switch token { + case "cmd", "command": flags.insert(.maskCommand) + case "ctrl", "control": flags.insert(.maskControl) + case "alt", "option", "opt": flags.insert(.maskAlternate) + case "shift": flags.insert(.maskShift) + default: mainKeyCode = Self.virtualKeyCode(for: token) + } + } + + guard let keyCode = mainKeyCode, + let down = CGEvent(keyboardEventSource: eventSource(), virtualKey: keyCode, keyDown: true), + let up = CGEvent(keyboardEventSource: eventSource(), virtualKey: keyCode, keyDown: false) else { + return false + } + down.flags = flags; up.flags = flags + stamp(down); stamp(up) + down.post(tap: .cghidEventTap) + up.post(tap: .cghidEventTap) + return true + } + + // MARK: - Scroll + + func scroll(atAppKitGlobal appKitPoint: CGPoint, direction: TakeoverScrollDirection, amount: Int) { + CGWarpMouseCursorPosition(quartzPoint(fromAppKitGlobal: appKitPoint)) + // wheel1 = vertical (positive scrolls up), wheel2 = horizontal. + let lineDelta = Int32(amount * 3) + var wheel1: Int32 = 0 + var wheel2: Int32 = 0 + switch direction { + case .up: wheel1 = lineDelta + case .down: wheel1 = -lineDelta + case .left: wheel2 = lineDelta + case .right: wheel2 = -lineDelta + } + guard let event = CGEvent(scrollWheelEvent2Source: eventSource(), units: .line, wheelCount: 2, + wheel1: wheel1, wheel2: wheel2, wheel3: 0) else { return } + stamp(event) + event.post(tap: .cghidEventTap) + } + + // MARK: - Kill switch + + private var killSwitchTap: CFMachPort? + private var killSwitchRunLoopSource: CFRunLoopSource? + private var onKillSwitch: (() -> Void)? + + /// Starts a global listen-only tap that fires `onKill` when the user + /// presses ESC. Ignores the app's own synthesized events via the + /// sentinel, so driving the cursor never trips it. + func startKillSwitchMonitor(onKill: @escaping () -> Void) { + stopKillSwitchMonitor() + self.onKillSwitch = onKill + + let eventMask = (1 << CGEventType.keyDown.rawValue) + let callback: CGEventTapCallBack = { _, type, event, userInfo in + guard let userInfo else { return Unmanaged.passUnretained(event) } + let executor = Unmanaged.fromOpaque(userInfo).takeUnretainedValue() + executor.handleKillSwitchEvent(type: type, event: event) + return Unmanaged.passUnretained(event) + } + + guard let tap = CGEvent.tapCreate( + tap: .cgSessionEventTap, + place: .headInsertEventTap, + options: .listenOnly, + eventsOfInterest: CGEventMask(eventMask), + callback: callback, + userInfo: Unmanaged.passUnretained(self).toOpaque() + ) else { + print("⚠️ Takeover: failed to create kill-switch tap") + return + } + self.killSwitchTap = tap + let runLoopSource = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0) + self.killSwitchRunLoopSource = runLoopSource + CFRunLoopAddSource(CFRunLoopGetMain(), runLoopSource, .commonModes) + CGEvent.tapEnable(tap: tap, enable: true) + } + + func stopKillSwitchMonitor() { + if let killSwitchTap { CGEvent.tapEnable(tap: killSwitchTap, enable: false) } + if let killSwitchRunLoopSource { + CFRunLoopRemoveSource(CFRunLoopGetMain(), killSwitchRunLoopSource, .commonModes) + } + killSwitchTap = nil + killSwitchRunLoopSource = nil + onKillSwitch = nil + } + + private nonisolated func handleKillSwitchEvent(type: CGEventType, event: CGEvent) { + // The tap can be disabled by the system; re-enable it so the kill + // switch can't silently die mid-takeover. + if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput { + if let killSwitchTap = MainActor.assumeIsolated({ self.killSwitchTap }) { + CGEvent.tapEnable(tap: killSwitchTap, enable: true) + } + return + } + // Ignore our own synthesized events. + if event.getIntegerValueField(.eventSourceUserData) == Self.eventSentinel { return } + let keyCode = UInt16(event.getIntegerValueField(.keyboardEventKeycode)) + if type == .keyDown && keyCode == UInt16(kVK_Escape) { + Task { @MainActor in self.onKillSwitch?() } + } + } + + // MARK: - Keycodes + + private static func virtualKeyCode(for token: String) -> CGKeyCode? { + switch token { + case "return", "enter": return CGKeyCode(kVK_Return) + case "esc", "escape": return CGKeyCode(kVK_Escape) + case "tab": return CGKeyCode(kVK_Tab) + case "space": return CGKeyCode(kVK_Space) + case "delete", "backspace": return CGKeyCode(kVK_Delete) + default: + if token.count == 1, let scalar = token.unicodeScalars.first { + return ansiKeyCode(for: Character(scalar)) + } + return nil + } + } + + /// ANSI virtual keycodes for a-z and 0-9 (kVK_ANSI_* values are not + /// sequential, so they're mapped explicitly). + private static func ansiKeyCode(for character: Character) -> CGKeyCode? { + let map: [Character: Int] = [ + "a": kVK_ANSI_A, "b": kVK_ANSI_B, "c": kVK_ANSI_C, "d": kVK_ANSI_D, "e": kVK_ANSI_E, + "f": kVK_ANSI_F, "g": kVK_ANSI_G, "h": kVK_ANSI_H, "i": kVK_ANSI_I, "j": kVK_ANSI_J, + "k": kVK_ANSI_K, "l": kVK_ANSI_L, "m": kVK_ANSI_M, "n": kVK_ANSI_N, "o": kVK_ANSI_O, + "p": kVK_ANSI_P, "q": kVK_ANSI_Q, "r": kVK_ANSI_R, "s": kVK_ANSI_S, "t": kVK_ANSI_T, + "u": kVK_ANSI_U, "v": kVK_ANSI_V, "w": kVK_ANSI_W, "x": kVK_ANSI_X, "y": kVK_ANSI_Y, + "z": kVK_ANSI_Z, + "0": kVK_ANSI_0, "1": kVK_ANSI_1, "2": kVK_ANSI_2, "3": kVK_ANSI_3, "4": kVK_ANSI_4, + "5": kVK_ANSI_5, "6": kVK_ANSI_6, "7": kVK_ANSI_7, "8": kVK_ANSI_8, "9": kVK_ANSI_9, + ] + guard let code = map[character] else { return nil } + return CGKeyCode(code) + } +} diff --git a/leanring-buddy/TakeoverAction.swift b/leanring-buddy/TakeoverAction.swift new file mode 100644 index 000000000..f8327b2e2 --- /dev/null +++ b/leanring-buddy/TakeoverAction.swift @@ -0,0 +1,156 @@ +// +// TakeoverAction.swift +// leanring-buddy +// +// The action vocabulary for offline takeover mode. The local vision model +// emits exactly one of these as JSON per turn; the controller guards it and +// the executor performs it. Deliberately tiny — seven verbs a 3B model can +// emit reliably. +// + +import Foundation + +enum TakeoverScrollDirection: String { + case up, down, left, right +} + +enum TakeoverAction { + /// Left-click at a pixel in the screenshot's coordinate space. + case click(x: Int, y: Int, targetLabel: String, why: String) + /// Type literal text. Never presses Return — submission is a separate `key`. + case type(text: String, why: String) + /// A key or combo: "return", "esc", "cmd+s", "cmd+shift+t". + case key(combo: String, why: String) + /// Scroll at a point. + case scroll(x: Int, y: Int, direction: TakeoverScrollDirection, amount: Int, why: String) + /// Task complete — the loop exits cleanly. + case done(summary: String) + /// The model can't proceed — the loop exits and speaks the reason. + case giveUp(reason: String) + + /// One-clause rationale, used for TTS narration and the audit trace. + var narration: String { + switch self { + case .click(_, _, let targetLabel, let why): return why.isEmpty ? "clicking \(targetLabel)" : why + case .type(_, let why): return why.isEmpty ? "typing" : why + case .key(let combo, let why): return why.isEmpty ? "pressing \(combo)" : why + case .scroll(_, _, let direction, _, let why): return why.isEmpty ? "scrolling \(direction.rawValue)" : why + case .done(let summary): return summary + case .giveUp(let reason): return reason + } + } + + /// Short label for the audit trace ("click Save (480,300)"). + var traceLabel: String { + switch self { + case .click(let x, let y, let targetLabel, _): return "click \(targetLabel) (\(x),\(y))" + case .type(let text, _): return "type \"\(text.prefix(24))\"" + case .key(let combo, _): return "key \(combo)" + case .scroll(_, _, let direction, let amount, _): return "scroll \(direction.rawValue) \(amount)" + case .done(let summary): return "done: \(summary.prefix(40))" + case .giveUp(let reason): return "give_up: \(reason.prefix(40))" + } + } + + /// True if two actions are "the same move" for stuck-detection: same verb, + /// coordinates within ~10px, same text/combo. + static func areEquivalent(_ lhs: TakeoverAction, _ rhs: TakeoverAction) -> Bool { + switch (lhs, rhs) { + case let (.click(ax, ay, _, _), .click(bx, by, _, _)): + return abs(ax - bx) <= 10 && abs(ay - by) <= 10 + case let (.type(at, _), .type(bt, _)): + return at == bt + case let (.key(ac, _), .key(bc, _)): + return ac.lowercased() == bc.lowercased() + case let (.scroll(_, _, ad, _, _), .scroll(_, _, bd, _, _)): + return ad == bd + default: + return false + } + } +} + +/// Parses the single JSON action object out of a local model's raw output. +/// Tolerant of ```json fences and leading/trailing prose; rejects multi-action +/// output and out-of-vocab verbs. +enum TakeoverActionParser { + enum ParseError: LocalizedError { + case noJSONObject + case unknownAction(String) + case missingField(String) + var errorDescription: String? { + switch self { + case .noJSONObject: return "no json action found in model output" + case .unknownAction(let action): return "unknown action \"\(action)\"" + case .missingField(let field): return "action missing field \"\(field)\"" + } + } + } + + static func parse(_ rawModelOutput: String) throws -> TakeoverAction { + guard let jsonObjectString = firstJSONObject(in: rawModelOutput), + let jsonData = jsonObjectString.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: jsonData) as? [String: Any] else { + throw ParseError.noJSONObject + } + + guard let actionName = object["action"] as? String else { + throw ParseError.missingField("action") + } + + func stringField(_ name: String) -> String { (object[name] as? String) ?? "" } + func intField(_ name: String) -> Int? { + if let intValue = object[name] as? Int { return intValue } + if let doubleValue = object[name] as? Double { return Int(doubleValue) } + if let stringValue = object[name] as? String { return Int(stringValue) } + return nil + } + + switch actionName.lowercased() { + case "click": + guard let x = intField("x"), let y = intField("y") else { throw ParseError.missingField("x/y") } + return .click(x: x, y: y, targetLabel: stringField("target_label"), why: stringField("why")) + case "type": + return .type(text: stringField("text"), why: stringField("why")) + case "key": + let combo = stringField("combo") + guard !combo.isEmpty else { throw ParseError.missingField("combo") } + return .key(combo: combo, why: stringField("why")) + case "scroll": + guard let x = intField("x"), let y = intField("y") else { throw ParseError.missingField("x/y") } + let direction = TakeoverScrollDirection(rawValue: stringField("dir").lowercased()) ?? .down + let amount = intField("amount") ?? 3 + return .scroll(x: x, y: y, direction: direction, amount: max(1, min(amount, 5)), why: stringField("why")) + case "done": + return .done(summary: stringField("summary")) + case "give_up", "giveup": + return .giveUp(reason: stringField("reason")) + default: + throw ParseError.unknownAction(actionName) + } + } + + /// Returns the first balanced `{...}` substring, ignoring code fences. + private static func firstJSONObject(in text: String) -> String? { + let characters = Array(text) + guard let openIndex = characters.firstIndex(of: "{") else { return nil } + var depth = 0 + var insideString = false + var escaped = false + var endIndex: Int? + for index in openIndex.. local VLM -> one action -> +// guardrails -> execute (or dry-run) -> narrate -> repeat. Everything here +// is built so a misgrounding small model is harmless, not destructive. +// +// Rails, in priority order: +// 1. Offline + local + Accessibility precondition (refuses to start otherwise). +// 2. Kill switch (ESC) aborts instantly, mid-action. +// 3. Hard caps: step count + wall-clock. +// 4. Dry-run by default; real actions need an explicit arm. +// 5. Danger gate: destructive actions need the extra "allow risky" opt-in. +// + +import Combine +import Foundation + +@MainActor +final class TakeoverController: ObservableObject { + enum RunState: Equatable { + case idle + case running(step: Int, lastAction: String) + case finished(summary: String) + case stopped(reason: String) + } + + /// Dry-run moves the cursor to where it would act and narrates, but never + /// clicks/types. The safe default — flip `isArmed` on to perform real input. + @Published var isArmed = false + /// Even when armed, destructive actions (submit/return/combos, danger-word + /// clicks) are skipped unless this is also on. Two deliberate switches + /// stand between a 3B model and anything irreversible. + @Published var allowRiskyActions = false + @Published private(set) var runState: RunState = .idle + + private static let stepCap = 25 + private static let wallClockCapSeconds: TimeInterval = 120 + private static let dangerWordPattern = + #"(?i)\b(send|delete|remove|buy|purchase|pay|confirm|submit|post|publish|discard|trash|erase|format|reset|sign\s?out|log\s?out)\b"# + + private let visionAgent: TakeoverVisionAgent + private let executor: ComputerUseActionExecutor + private let captureCursorScreen: () async throws -> CompanionScreenCapture? + private let narrate: (String) async -> Void + private let isNetworkAvailable: () -> Bool + + private var runTask: Task? + private var killRequested = false + + init( + visionAgent: TakeoverVisionAgent, + executor: ComputerUseActionExecutor, + captureCursorScreen: @escaping () async throws -> CompanionScreenCapture?, + narrate: @escaping (String) async -> Void, + isNetworkAvailable: @escaping () -> Bool + ) { + self.visionAgent = visionAgent + self.executor = executor + self.captureCursorScreen = captureCursorScreen + self.narrate = narrate + self.isNetworkAvailable = isNetworkAvailable + } + + // MARK: - Preconditions + + enum StartRefusal: LocalizedError { + case networkUp + case accessibilityMissing + case alreadyRunning + var errorDescription: String? { + switch self { + case .networkUp: return "takeover is offline-only — turn off wifi first." + case .accessibilityMissing: return "i need accessibility permission to control the cursor — grant it in system settings." + case .alreadyRunning: return "already running a takeover." + } + } + } + + var isRunning: Bool { + if case .running = runState { return true } + return false + } + + // MARK: - Run + + func start(task: String) { + guard !isRunning else { return } + + // Rail 1: offline + Accessibility, refuse loudly otherwise. + if isNetworkAvailable() { + failToStart(.networkUp); return + } + if !executor.isAccessibilityTrusted { + failToStart(.accessibilityMissing); return + } + + killRequested = false + runState = .running(step: 0, lastAction: "looking at your screen") + + // Rail 2: kill switch live for the whole run. + executor.startKillSwitchMonitor { [weak self] in + self?.requestKill() + } + + runTask = Task { await runLoop(task: task) } + } + + func requestKill() { + killRequested = true + runTask?.cancel() + } + + private func failToStart(_ refusal: StartRefusal) { + runState = .stopped(reason: refusal.localizedDescription) + Task { await narrate(refusal.localizedDescription) } + } + + private func runLoop(task: String) async { + defer { + executor.stopKillSwitchMonitor() + // Rail 4: arming never persists past a run. + isArmed = false + allowRiskyActions = false + } + + let startDate = Date() + var historyTrace: [String] = [] + var recentActions: [TakeoverAction] = [] + + for step in 0..= Self.wallClockCapSeconds { + await narrate("i've spent my time budget on this — stopping here.") + finish(.stopped(reason: "time cap")); return + } + + let capture: CompanionScreenCapture? + do { + capture = try await captureCursorScreen() + } catch { + finish(.stopped(reason: "couldn't capture the screen")); return + } + guard let capture else { finish(.stopped(reason: "no screen to look at")); return } + + let action: TakeoverAction + do { + action = try await visionAgent.decideNextAction( + task: task, + historyTrace: historyTrace, + screenshotJPEG: capture.imageData, + screenshotWidthInPixels: capture.screenshotWidthInPixels, + screenshotHeightInPixels: capture.screenshotHeightInPixels + ) + } catch is CancellationError { + finish(.stopped(reason: "stopped.")); return + } catch { + // Misgrounding / parse failure: one honest retry, then stop — + // never click blindly. + await narrate("i'm not reading the screen clearly — stopping so i don't click the wrong thing.") + finish(.stopped(reason: "couldn't ground the screen")); return + } + + switch action { + case .done(let summary): + await narrate(summary.isEmpty ? "all done." : summary) + finish(.finished(summary: summary)); return + case .giveUp(let reason): + await narrate(reason.isEmpty ? "i can't do this one." : reason) + finish(.stopped(reason: reason)); return + default: + break + } + + // Rail (stuck): same move three times running -> bail. + if recentActions.count >= 2, + recentActions.suffix(2).allSatisfy({ TakeoverAction.areEquivalent($0, action) }) { + await narrate("i'm going in circles here — stopping.") + finish(.stopped(reason: "stuck in a loop")); return + } + + runState = .running(step: step + 1, lastAction: action.traceLabel) + await narrate(action.narration) + + await performGuarded(action: action, on: capture) + + historyTrace.append(action.traceLabel) + recentActions.append(action) + if killRequested || Task.isCancelled { finish(.stopped(reason: "stopped.")); return } + + // Let the screen settle before the next screenshot. + try? await Task.sleep(nanoseconds: 600_000_000) + } + + await narrate("i've taken as many steps as i'm allowed for this one.") + finish(.stopped(reason: "step cap")) + } + + // MARK: - Guarded execution + + private func performGuarded(action: TakeoverAction, on capture: CompanionScreenCapture) async { + // Rail 4: dry-run only moves the cursor to the target and narrates. + let dryRun = !isArmed + + switch action { + case .click(let x, let y, let targetLabel, let why): + guard let appKitPoint = appKitGlobalPoint(forPixelX: x, pixelY: y, on: capture) else { return } + if dryRun { + executor.moveCursorVisible(toAppKitGlobal: appKitPoint) + return + } + // Rail 5: danger-word clicks need the risky opt-in. + if isDangerous(targetLabel + " " + why) && !allowRiskyActions { + await narrate("that looks like it might do something irreversible, so i'll leave that click to you.") + executor.moveCursorVisible(toAppKitGlobal: appKitPoint) + return + } + executor.click(atAppKitGlobal: appKitPoint) + + case .type(let text, _): + if dryRun { return } + // Rail 5: all text entry needs the risky opt-in. + if !allowRiskyActions { + await narrate("i'd type \"\(text.prefix(40))\" here — flip on allow-risky if you want me to.") + return + } + executor.typeText(text) + + case .key(let combo, _): + if dryRun { return } + // Rail 5: submission (return) and any combo need the risky opt-in. + let lowered = combo.lowercased() + let isSubmitOrCombo = lowered == "return" || lowered == "enter" || lowered.contains("+") + if isSubmitOrCombo && !allowRiskyActions { + await narrate("pressing \(combo) could submit something — i'll leave that to you.") + return + } + executor.pressKey(combo: combo) + + case .scroll(let x, let y, let direction, let amount, _): + guard let appKitPoint = appKitGlobalPoint(forPixelX: x, pixelY: y, on: capture) else { return } + if dryRun { + executor.moveCursorVisible(toAppKitGlobal: appKitPoint) + return + } + executor.scroll(atAppKitGlobal: appKitPoint, direction: direction, amount: amount) + + case .done, .giveUp: + break + } + } + + private func isDangerous(_ text: String) -> Bool { + text.range(of: Self.dangerWordPattern, options: .regularExpression) != nil + } + + /// Maps a screenshot pixel to an AppKit global point — the same scale the + /// pointing pipeline uses (px -> display points -> bottom-left global). + /// The executor flips this to Quartz top-left at the post boundary. + private func appKitGlobalPoint(forPixelX pixelX: Int, pixelY: Int, on capture: CompanionScreenCapture) -> CGPoint? { + let screenshotWidth = CGFloat(capture.screenshotWidthInPixels) + let screenshotHeight = CGFloat(capture.screenshotHeightInPixels) + guard screenshotWidth > 0, screenshotHeight > 0 else { return nil } + + let displayWidth = CGFloat(capture.displayWidthInPoints) + let displayHeight = CGFloat(capture.displayHeightInPoints) + let displayFrame = capture.displayFrame + + let clampedX = max(0, min(CGFloat(pixelX), screenshotWidth)) + let clampedY = max(0, min(CGFloat(pixelY), screenshotHeight)) + + let displayLocalX = clampedX * (displayWidth / screenshotWidth) + let displayLocalY = clampedY * (displayHeight / screenshotHeight) + let appKitY = displayHeight - displayLocalY + + return CGPoint(x: displayLocalX + displayFrame.origin.x, + y: appKitY + displayFrame.origin.y) + } + + private func finish(_ state: RunState) { + runState = state + runTask = nil + } +} diff --git a/leanring-buddy/TakeoverVisionAgent.swift b/leanring-buddy/TakeoverVisionAgent.swift new file mode 100644 index 000000000..5a0ae11d7 --- /dev/null +++ b/leanring-buddy/TakeoverVisionAgent.swift @@ -0,0 +1,171 @@ +// +// TakeoverVisionAgent.swift +// leanring-buddy +// +// The brain for offline takeover: a local vision-language model (Qwen2.5-VL-3B +// via MLX) that looks at one screenshot plus the task and recent history and +// emits exactly one JSON action. No native tool-use API — the controller +// parses, guards, and executes what this returns. +// +// Honest reality: small local VLMs misground on dense/high-res screens. This +// is why takeover defaults to dry-run and is scoped to simple, low-density UIs. +// + +import Combine +import CoreImage +import Foundation +import HuggingFace +import MLX +import MLXHuggingFace +import MLXLMCommon +import MLXVLM +import Tokenizers + +@MainActor +final class TakeoverVisionAgent: ObservableObject { + /// Swap to "mlx-community/Holo1-3B-4bit" (a Qwen2.5-VL-3B UI-grounding + /// finetune) for better click accuracy — it would need a Click(x,y) + /// parser instead of the JSON one. Qwen2.5-VL-3B is the proven registry + /// default and emits the JSON schema fine. + private static let modelConfiguration = VLMRegistry.qwen2_5VL3BInstruct4Bit + + enum Readiness: Equatable { + case notLoaded + case downloading(progressFraction: Double) + case loading + case ready + case failed(errorDescription: String) + } + + @Published private(set) var readiness: Readiness = .notLoaded + + private var loadedModelContainer: ModelContainer? + private var modelLoadTask: Task? + + private static func modelCacheDirectory() throws -> URL { + let applicationSupportDirectory = try FileManager.default.url( + for: .applicationSupportDirectory, in: .userDomainMask, appropriateFor: nil, create: true) + return applicationSupportDirectory + .appendingPathComponent("Clicky", isDirectory: true) + .appendingPathComponent("models", isDirectory: true) + .appendingPathComponent("huggingface", isDirectory: true) + } + + @discardableResult + func loadModelIfNeeded() -> Task { + if let modelLoadTask { return modelLoadTask } + let newTask = Task { + do { + let container = try await loadModelContainer() + loadedModelContainer = container + readiness = .ready + return container + } catch { + modelLoadTask = nil + readiness = .failed(errorDescription: error.localizedDescription) + throw error + } + } + modelLoadTask = newTask + return newTask + } + + private func loadModelContainer() async throws -> ModelContainer { + // Bound MLX's buffer cache before a long multi-step loop — unbounded + // KV/cache growth is the main per-step OOM risk on 16 GB. + MLX.Memory.cacheLimit = 32 * 1024 * 1024 + + readiness = .downloading(progressFraction: 0) + let hubClient = HubClient(cache: HubCache(cacheDirectory: try Self.modelCacheDirectory())) + print("👁️ Takeover VLM: loading \(Self.modelConfiguration.name)") + + let container = try await VLMModelFactory.shared.loadContainer( + from: #hubDownloader(hubClient), + using: #huggingFaceTokenizerLoader(), + configuration: Self.modelConfiguration, + progressHandler: { downloadProgress in + let fraction = downloadProgress.fractionCompleted + Task { @MainActor [weak self] in + guard let self, case .downloading = self.readiness else { return } + self.readiness = .downloading(progressFraction: fraction) + } + } + ) + readiness = .loading + return container + } + + // MARK: - Prompt + + private static let systemPrompt = """ + you control a mac by looking at one screenshot and choosing ONE action. + output EXACTLY ONE json object and NOTHING else — no prose, no markdown, no second action. + all coordinates are integer pixels in the screenshot, origin top-left (0,0 = top-left corner). + + allowed actions (pick one): + {"action":"click","x":N,"y":N,"target_label":"short name of the thing","why":"one short clause"} + {"action":"type","text":"literal text","why":"..."} + {"action":"key","combo":"return | esc | cmd+s | ...","why":"..."} + {"action":"scroll","x":N,"y":N,"dir":"up|down|left|right","amount":1-5,"why":"..."} + {"action":"done","summary":"what you accomplished"} + {"action":"give_up","reason":"why you cannot continue"} + + rules: + - one action per turn. the screen will update and you'll get a new screenshot. + - typing NEVER presses enter; use a key action with "return" to submit. + - click the CENTER of the target. if you're unsure where something is, give_up instead of guessing. + - when the goal is clearly already done on screen, output done. + - keep target_label and why to a few words. + """ + + /// Asks the model for the next action given the current screen. + func decideNextAction( + task: String, + historyTrace: [String], + screenshotJPEG: Data, + screenshotWidthInPixels: Int, + screenshotHeightInPixels: Int + ) async throws -> TakeoverAction { + let container: ModelContainer + if let loadedModelContainer { + container = loadedModelContainer + } else { + container = try await loadModelIfNeeded().value + } + + guard let screenshotImage = CIImage(data: screenshotJPEG) else { + throw NSError(domain: "TakeoverVisionAgent", code: -1, + userInfo: [NSLocalizedDescriptionKey: "couldn't read the screenshot"]) + } + + // Text BEFORE the image improves click accuracy (Anthropic computer-use tip). + var userText = "TASK: \(task)\n" + if !historyTrace.isEmpty { + userText += "SO FAR:\n" + historyTrace.suffix(8).enumerated() + .map { " step\($0.offset + 1): \($0.element)" }.joined(separator: "\n") + "\n" + } + userText += "The screenshot below is \(screenshotWidthInPixels)x\(screenshotHeightInPixels) pixels. Choose one action." + + let userInput = UserInput(chat: [ + .system(Self.systemPrompt), + .user(userText, images: [.ciImage(screenshotImage)]), + ]) + + let languageModelInput = try await container.prepare(input: userInput) + // Tight token budget — the model only needs to emit one short JSON object. + let generationStream = try await container.generate( + input: languageModelInput, + parameters: GenerateParameters(maxTokens: 200, temperature: 0.2) + ) + + var rawOutput = "" + for await generation in generationStream { + if Task.isCancelled { break } + if case .chunk(let textChunk) = generation { rawOutput += textChunk } + } + try Task.checkCancellation() + + print("👁️ Takeover VLM raw: \(rawOutput.prefix(200))") + return try TakeoverActionParser.parse(rawOutput) + } +} From b796e213ee6a07eb4973bffcbf44da95d8228b53 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 19:26:05 -0400 Subject: [PATCH 12/18] add takeover controls to the panel Arm/dry-run toggle, the risky-actions opt-in (only shown when armed), live step status, and a stop button. Says offline-only and that esc stops it. --- leanring-buddy/CompanionPanelView.swift | 84 +++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/leanring-buddy/CompanionPanelView.swift b/leanring-buddy/CompanionPanelView.swift index a2c0b9829..030ddc92a 100644 --- a/leanring-buddy/CompanionPanelView.swift +++ b/leanring-buddy/CompanionPanelView.swift @@ -34,6 +34,9 @@ struct CompanionPanelView: View { localModelStatusRow .padding(.horizontal, 16) + + takeoverSection + .padding(.horizontal, 16) } if !companionManager.allPermissionsGranted { @@ -678,6 +681,87 @@ struct CompanionPanelView: View { } } + // MARK: - Takeover (offline autonomous) + + /// Experimental offline takeover controls: arm/dry-run, the risky opt-in, + /// and live run status. Says plainly that it's offline-only and how to stop. + @ViewBuilder + private var takeoverSection: some View { + VStack(alignment: .leading, spacing: 6) { + Divider() + .background(DS.Colors.borderSubtle) + .padding(.vertical, 4) + + HStack { + Text("Takeover") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + Text("offline · experimental") + .font(.system(size: 9, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + Spacer() + Text(takeoverStatusText) + .font(.system(size: 10, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + } + + Text("say \"take over\" then a task. dry-run moves the cursor only. press esc to stop.") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + .fixedSize(horizontal: false, vertical: true) + + Toggle(isOn: Binding( + get: { companionManager.takeoverController.isArmed }, + set: { companionManager.takeoverController.isArmed = $0 } + )) { + Text("Arm (perform real clicks)") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + } + .toggleStyle(.switch) + .tint(DS.Colors.accent) + .scaleEffect(0.8, anchor: .leading) + + if companionManager.takeoverController.isArmed { + Toggle(isOn: Binding( + get: { companionManager.takeoverController.allowRiskyActions }, + set: { companionManager.takeoverController.allowRiskyActions = $0 } + )) { + Text("Allow typing + submits") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(Color(red: 0.9, green: 0.55, blue: 0.35)) + } + .toggleStyle(.switch) + .tint(Color(red: 0.9, green: 0.55, blue: 0.35)) + .scaleEffect(0.8, anchor: .leading) + } + + if companionManager.takeoverController.isRunning { + Button(action: { companionManager.takeoverController.requestKill() }) { + Text("Stop") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(.white) + .padding(.horizontal, 12) + .padding(.vertical, 5) + .background(RoundedRectangle(cornerRadius: 6, style: .continuous) + .fill(Color(red: 0.85, green: 0.3, blue: 0.3))) + } + .buttonStyle(.plain) + .pointerCursor() + } + } + .padding(.top, 8) + } + + private var takeoverStatusText: String { + switch companionManager.takeoverController.runState { + case .idle: return companionManager.takeoverController.isArmed ? "armed" : "dry-run" + case .running(let step, _): return "step \(step)…" + case .finished: return "done" + case .stopped: return "stopped" + } + } + private func modelOptionButton(label: String, modelID: String) -> some View { let isSelected = companionManager.selectedModel == modelID return Button(action: { From 0c013ae0b33babbe37285b186759c8258dd070c2 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 19:33:34 -0400 Subject: [PATCH 13/18] harden action parser with regex salvage; add grounding smoke test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Smoke test verified Qwen2.5-VL-3B grounds a button at (768,309) vs true center (800,308) on a synthetic UI — but emits malformed JSON ('x:768,309'). Strict JSON rejects it; the regex fallback recovers the right coordinate so a correct grounding in shaky syntax still executes. --- .../Package.resolved | 141 ++++++++++++++++++ .../TakeoverGroundingSmokeTest/Package.swift | 28 ++++ .../TakeoverGroundingSmokeTest/main.swift | 96 ++++++++++++ leanring-buddy/TakeoverAction.swift | 48 +++++- 4 files changed, 312 insertions(+), 1 deletion(-) create mode 100644 bench/TakeoverGroundingSmokeTest/Package.resolved create mode 100644 bench/TakeoverGroundingSmokeTest/Package.swift create mode 100644 bench/TakeoverGroundingSmokeTest/Sources/TakeoverGroundingSmokeTest/main.swift diff --git a/bench/TakeoverGroundingSmokeTest/Package.resolved b/bench/TakeoverGroundingSmokeTest/Package.resolved new file mode 100644 index 000000000..b13c0c85d --- /dev/null +++ b/bench/TakeoverGroundingSmokeTest/Package.resolved @@ -0,0 +1,141 @@ +{ + "originHash" : "a9ec0d39012ddfacddafd43e06365541f8b1d43b587e852f425c292444ccb74a", + "pins" : [ + { + "identity" : "eventsource", + "kind" : "remoteSourceControl", + "location" : "https://github.com/mattt/EventSource.git", + "state" : { + "revision" : "a3a85a85214caf642abaa96ae664e4c772a59f6e", + "version" : "1.4.1" + } + }, + { + "identity" : "mlx-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ml-explore/mlx-swift", + "state" : { + "revision" : "dc43e62d7055353c7f99fa071a4e71d29dfddc44", + "version" : "0.31.4" + } + }, + { + "identity" : "mlx-swift-lm", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ml-explore/mlx-swift-lm", + "state" : { + "revision" : "1c05248bb0899e2a7a4962b84d319cf12f4e12aa", + "version" : "3.31.3" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "a9a5efd40eaf558a2bcd48d64b1d1646be686008", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "b601256eab081c0f92f059e12818ac1d4f178ff7", + "version" : "1.3.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "1b6b2e274e85105bfa155183145a1dcfd63331f1", + "version" : "4.5.0" + } + }, + { + "identity" : "swift-huggingface", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-huggingface", + "state" : { + "revision" : "b721959445b617d0bf03910b2b4aced345fd93bf", + "version" : "0.9.0" + } + }, + { + "identity" : "swift-jinja", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-jinja.git", + "state" : { + "revision" : "0b67ecb79139f6addef8699eff3622808aa6c7dc", + "version" : "2.3.6" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "77b84ac2cd2ac9e4ac67d19f045fd5b434f56967", + "version" : "2.101.0" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax.git", + "state" : { + "revision" : "0687f71944021d616d34d922343dcef086855920", + "version" : "600.0.1" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "7502b711c92a17741fa625d722b0ccbd595d8ed1", + "version" : "1.7.2" + } + }, + { + "identity" : "swift-transformers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-transformers", + "state" : { + "revision" : "2fa33e1f5e7131a7fc64c28e6d161dcec0d24820", + "version" : "1.3.3" + } + }, + { + "identity" : "yyjson", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ibireme/yyjson.git", + "state" : { + "revision" : "8b4a38dc994a110abaec8a400615567bd996105f", + "version" : "0.12.0" + } + } + ], + "version" : 3 +} diff --git a/bench/TakeoverGroundingSmokeTest/Package.swift b/bench/TakeoverGroundingSmokeTest/Package.swift new file mode 100644 index 000000000..36f0ceb16 --- /dev/null +++ b/bench/TakeoverGroundingSmokeTest/Package.swift @@ -0,0 +1,28 @@ +// swift-tools-version: 6.1 +// Smoke test for the takeover brain: load the local VLM, hand it a real +// screenshot + a task, and check it emits a parseable JSON action with a +// coordinate. Proves the offline grounding path works before wiring trust in +// the in-app loop. Build with Xcode (MLX Metal shaders). +import PackageDescription + +let package = Package( + name: "TakeoverGroundingSmokeTest", + platforms: [.macOS(.v14)], + dependencies: [ + .package(url: "https://github.com/ml-explore/mlx-swift-lm", .upToNextMajor(from: "3.31.3")), + .package(url: "https://github.com/huggingface/swift-huggingface", from: "0.9.0"), + .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.0"), + ], + targets: [ + .executableTarget( + name: "TakeoverGroundingSmokeTest", + dependencies: [ + .product(name: "MLXVLM", package: "mlx-swift-lm"), + .product(name: "MLXLMCommon", package: "mlx-swift-lm"), + .product(name: "MLXHuggingFace", package: "mlx-swift-lm"), + .product(name: "HuggingFace", package: "swift-huggingface"), + .product(name: "Tokenizers", package: "swift-transformers"), + ] + ) + ] +) diff --git a/bench/TakeoverGroundingSmokeTest/Sources/TakeoverGroundingSmokeTest/main.swift b/bench/TakeoverGroundingSmokeTest/Sources/TakeoverGroundingSmokeTest/main.swift new file mode 100644 index 000000000..f4840c191 --- /dev/null +++ b/bench/TakeoverGroundingSmokeTest/Sources/TakeoverGroundingSmokeTest/main.swift @@ -0,0 +1,96 @@ +// +// main.swift +// TakeoverGroundingSmokeTest +// +// Loads Qwen2.5-VL-3B-Instruct-4bit (the takeover brain), feeds it a +// screenshot path + a task, and prints the raw output and the parsed action. +// Usage: +// TakeoverGroundingSmokeTest "" +// + +import CoreImage +import Foundation +import HuggingFace +import MLX +import MLXHuggingFace +import MLXLMCommon +import MLXVLM +import Tokenizers + +let systemPrompt = """ +you control a mac by looking at one screenshot and choosing ONE action. +output EXACTLY ONE json object and NOTHING else — no prose, no markdown, no second action. +all coordinates are integer pixels in the screenshot, origin top-left (0,0 = top-left corner). + +allowed actions (pick one): +{"action":"click","x":N,"y":N,"target_label":"short name of the thing","why":"one short clause"} +{"action":"type","text":"literal text","why":"..."} +{"action":"key","combo":"return | esc | cmd+s | ...","why":"..."} +{"action":"scroll","x":N,"y":N,"dir":"up|down|left|right","amount":1-5,"why":"..."} +{"action":"done","summary":"what you accomplished"} +{"action":"give_up","reason":"why you cannot continue"} + +rules: +- one action per turn. +- click the CENTER of the target. if unsure, give_up instead of guessing. +- keep target_label and why to a few words. +""" + +let arguments = CommandLine.arguments +guard arguments.count >= 3 else { + print("usage: TakeoverGroundingSmokeTest \"\"") + exit(2) +} +let screenshotPath = arguments[1] +let task = arguments[2] + +guard let screenshotImage = CIImage(contentsOf: URL(fileURLWithPath: screenshotPath)) else { + print("could not read image at \(screenshotPath)") + exit(2) +} +let widthInPixels = Int(screenshotImage.extent.width) +let heightInPixels = Int(screenshotImage.extent.height) + +MLX.Memory.cacheLimit = 32 * 1024 * 1024 + +let modelCacheDirectory = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Library/Application Support/Clicky/models/huggingface", isDirectory: true) +let hubClient = HubClient(cache: HubCache(cacheDirectory: modelCacheDirectory)) + +print("loading Qwen2.5-VL-3B-Instruct-4bit (downloads ~3 GB on first run)...") +let loadStart = Date() +let modelContainer = try await VLMModelFactory.shared.loadContainer( + from: #hubDownloader(hubClient), + using: #huggingFaceTokenizerLoader(), + configuration: VLMRegistry.qwen2_5VL3BInstruct4Bit, + progressHandler: { progress in + print(String(format: "download: %3.0f%%", progress.fractionCompleted * 100)) + } +) +print(String(format: "model ready in %.1fs", Date().timeIntervalSince(loadStart))) + +let userText = "TASK: \(task)\nThe screenshot below is \(widthInPixels)x\(heightInPixels) pixels. Choose one action." +let userInput = UserInput(chat: [ + .system(systemPrompt), + .user(userText, images: [.ciImage(screenshotImage)]), +]) + +let requestStart = Date() +let languageModelInput = try await modelContainer.prepare(input: userInput) +let stream = try await modelContainer.generate( + input: languageModelInput, + parameters: GenerateParameters(maxTokens: 200, temperature: 0.2) +) + +var rawOutput = "" +var firstTokenLatency: TimeInterval? +for await generation in stream { + if case .chunk(let chunk) = generation { + if firstTokenLatency == nil { firstTokenLatency = Date().timeIntervalSince(requestStart) } + rawOutput += chunk + } +} + +print(String(format: "first token %.2fs | total %.1fs", firstTokenLatency ?? -1, Date().timeIntervalSince(requestStart))) +print("RAW: \(rawOutput)") +print("PARSEABLE-JSON: \(rawOutput.contains("{") && rawOutput.contains("}"))") diff --git a/leanring-buddy/TakeoverAction.swift b/leanring-buddy/TakeoverAction.swift index f8327b2e2..7732a4cb0 100644 --- a/leanring-buddy/TakeoverAction.swift +++ b/leanring-buddy/TakeoverAction.swift @@ -91,7 +91,11 @@ enum TakeoverActionParser { guard let jsonObjectString = firstJSONObject(in: rawModelOutput), let jsonData = jsonObjectString.data(using: .utf8), let object = try? JSONSerialization.jsonObject(with: jsonData) as? [String: Any] else { - throw ParseError.noJSONObject + // Small VLMs often emit *almost* valid JSON — e.g. a doubled + // coordinate `"x":768,309`. The grounding is usually right even when + // the syntax is shaky, so salvage the fields by regex before giving + // up and safe-failing. + return try parseByRegex(rawModelOutput) } guard let actionName = object["action"] as? String else { @@ -130,6 +134,48 @@ enum TakeoverActionParser { } } + /// Field-by-field salvage for almost-valid JSON. Pulls the first integer + /// after each numeric key (so `"x":768,309` yields 768) and the first + /// quoted value after each string key. + private static func parseByRegex(_ text: String) throws -> TakeoverAction { + func quoted(_ key: String) -> String? { + firstCapture(in: text, pattern: "\"\(key)\"\\s*:\\s*\"([^\"]*)\"") + } + func integer(_ key: String) -> Int? { + firstCapture(in: text, pattern: "\"\(key)\"\\s*:\\s*(-?\\d+)").flatMap(Int.init) + } + guard let actionName = quoted("action") else { throw ParseError.noJSONObject } + + switch actionName.lowercased() { + case "click": + guard let x = integer("x"), let y = integer("y") else { throw ParseError.missingField("x/y") } + return .click(x: x, y: y, targetLabel: quoted("target_label") ?? "", why: quoted("why") ?? "") + case "type": + return .type(text: quoted("text") ?? "", why: quoted("why") ?? "") + case "key": + guard let combo = quoted("combo"), !combo.isEmpty else { throw ParseError.missingField("combo") } + return .key(combo: combo, why: quoted("why") ?? "") + case "scroll": + guard let x = integer("x"), let y = integer("y") else { throw ParseError.missingField("x/y") } + let direction = TakeoverScrollDirection(rawValue: (quoted("dir") ?? "down").lowercased()) ?? .down + return .scroll(x: x, y: y, direction: direction, amount: max(1, min(integer("amount") ?? 3, 5)), why: quoted("why") ?? "") + case "done": + return .done(summary: quoted("summary") ?? "") + case "give_up", "giveup": + return .giveUp(reason: quoted("reason") ?? "") + default: + throw ParseError.unknownAction(actionName) + } + } + + private static func firstCapture(in text: String, pattern: String) -> String? { + guard let regex = try? NSRegularExpression(pattern: pattern), + let match = regex.firstMatch(in: text, range: NSRange(text.startIndex..., in: text)), + match.numberOfRanges >= 2, + let range = Range(match.range(at: 1), in: text) else { return nil } + return String(text[range]) + } + /// Returns the first balanced `{...}` substring, ignoring code fences. private static func firstJSONObject(in text: String) -> String? { let characters = Array(text) From 1f24686156f75a09e842c597f6918a70f58f04eb Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 19:35:53 -0400 Subject: [PATCH 14/18] tolerate unquoted keys in action salvage; record measured grounding 2/3 synthetic-UI targets grounded correctly; the top-bar edge target missed, matching the ScreenSpot-Pro failure mode. Numbers + Holo1 upgrade path in TAKEOVER.md. --- TAKEOVER.md | 23 +++++++++++++++++++++++ leanring-buddy/TakeoverAction.swift | 6 ++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/TAKEOVER.md b/TAKEOVER.md index e21a94243..53ac3b73e 100644 --- a/TAKEOVER.md +++ b/TAKEOVER.md @@ -66,6 +66,29 @@ Listen-only observation of your clicks + keystrokes so clicky has context for ta - This is you watching your own machine for your own assistant — kept local and gated so it stays that and only that. +## Measured grounding (synthetic UI, Qwen2.5-VL-3B-4bit, M1 Pro) + +Smoke test (`bench/TakeoverGroundingSmokeTest`) on a generated 1280x800 screen +with a search box, a blue Search button, and a top-bar Settings label: + +| task | true target (top-left px) | model output | verdict | +|---|---|---|---| +| click the search button | (800, 308) | (768, 309) | hit (inside button) | +| click the search box | (540, 308) | (600, 308) | hit (inside field) | +| open settings | (1175, 42) | (1077, 453) | miss (y way off) | + +Two of three. The miss is the top-bar edge target — exactly the +ScreenSpot-Pro failure mode the research flagged (generic Qwen2.5-VL-3B ~24% on +dense/edge targets). First token ~6-9s per step. Verdict: central, well-labeled +targets land; edge/small/dense targets miss. Scope demos accordingly; dry-run +default is vindicated. The model also emits malformed JSON often (doubled +coords, unquoted keys) — the parser's regex salvage recovers the coordinate +when the grounding itself is right. + +Upgrade path: `mlx-community/Holo1-3B-4bit` (a UI-grounding finetune, ~3 GB, +emits `Click(x,y)` natively) benchmarks far better on edge/dense targets — a +one-line model swap plus a `Click(x,y)` parser. + ## What this is and isn't - It's a constrained, rail-guarded, offline computer-use agent on a small local VLM. A real diff --git a/leanring-buddy/TakeoverAction.swift b/leanring-buddy/TakeoverAction.swift index 7732a4cb0..08ea9aa2f 100644 --- a/leanring-buddy/TakeoverAction.swift +++ b/leanring-buddy/TakeoverAction.swift @@ -138,11 +138,13 @@ enum TakeoverActionParser { /// after each numeric key (so `"x":768,309` yields 768) and the first /// quoted value after each string key. private static func parseByRegex(_ text: String) throws -> TakeoverAction { + // Small VLMs drop quotes off keys too ("y":308 vs y:308), so the key + // quotes are optional in these patterns. func quoted(_ key: String) -> String? { - firstCapture(in: text, pattern: "\"\(key)\"\\s*:\\s*\"([^\"]*)\"") + firstCapture(in: text, pattern: "\"?\(key)\"?\\s*:\\s*\"([^\"]*)\"") } func integer(_ key: String) -> Int? { - firstCapture(in: text, pattern: "\"\(key)\"\\s*:\\s*(-?\\d+)").flatMap(Int.init) + firstCapture(in: text, pattern: "\"?\(key)\"?\\s*:\\s*(-?\\d+)").flatMap(Int.init) } guard let actionName = quoted("action") else { throw ParseError.noJSONObject } From d562e7e7fccd95084d25923de8637d55c4a63c17 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Fri, 12 Jun 2026 23:12:50 -0400 Subject: [PATCH 15/18] mark executor + loop + kill switch as unproven until run in the app --- TAKEOVER.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/TAKEOVER.md b/TAKEOVER.md index 53ac3b73e..e3058a9de 100644 --- a/TAKEOVER.md +++ b/TAKEOVER.md @@ -1,7 +1,13 @@ # TAKEOVER — offline autonomous mode (spike) -Status: **spike, in progress.** This is the "Jarvis" feature — clicky takes over the cursor -and keyboard and does a task for you. The hard constraint, from the user: **offline only.** +Status: **spike — partially verified.** This is the "Jarvis" feature — clicky takes over the +cursor and keyboard and does a task for you. The hard constraint, from the user: **offline only.** + +**Honest verification state:** the whole thing *compiles*; the VLM brain *grounds in isolation* +(smoke test, see below). What has NOT executed even once: the CGEvent executor (no synthesized +click/keystroke has actually been posted), the full capture→decide→execute→re-capture loop, and +the kill switch — none are testable headlessly, all need a real GUI session with the Accessibility +grant. Treat executor + loop + kill switch as **unproven** until run in the app. ## The contradiction, resolved From 54e9a6900f08cc6fe6ba9e73aeb410de5083c054 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Sat, 13 Jun 2026 21:56:31 -0400 Subject: [PATCH 16/18] add cloud guided-tour mode (e.g. Logic Pro beats) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clicky screenshots the screen, asks Sonnet for the next control + narration, flies the overlay cursor to it, speaks, and advances — hands-off, pure pointing (no clicks, no Accessibility). Triggered by the panel button or 'teach me a beat'. Worker base URL is now overridable via UserDefaults for local wrangler dev; ATS allows localhost. --- leanring-buddy/CompanionManager.swift | 161 +++++++++++++++++++++++- leanring-buddy/CompanionPanelView.swift | 56 +++++++++ leanring-buddy/Info.plist | 7 ++ 3 files changed, 222 insertions(+), 2 deletions(-) diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index 1094decbd..6750edc76 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -75,8 +75,13 @@ final class CompanionManager: ObservableObject { // streamingResponseText, so no separate response overlay manager is needed. /// Base URL for the Cloudflare Worker proxy. All API requests route - /// through this so keys never ship in the app binary. - private static let workerBaseURL = "https://your-worker-name.your-subdomain.workers.dev" + /// through this so keys never ship in the app binary. Overridable via the + /// `workerBaseURLOverride` default so a local `wrangler dev` worker + /// (http://127.0.0.1:8787) can be used without editing source. + private static var workerBaseURL: String { + UserDefaults.standard.string(forKey: "workerBaseURLOverride") + ?? "https://your-worker-name.your-subdomain.workers.dev" + } private lazy var claudeAPI: ClaudeAPI = { // Never hand the local sentinel to the cloud client — if the app @@ -189,6 +194,11 @@ final class CompanionManager: ObservableObject { /// speaks again so a new response can begin immediately. private var currentResponseTask: Task? + /// True while the cloud guided tour (e.g. Logic Pro beat-making) is running — + /// Clicky autonomously points its cursor at the next control each step. + @Published private(set) var isGuidedTourRunning = false + private var guidedTourTask: Task? + private var shortcutTransitionCancellable: AnyCancellable? private var voiceStateCancellable: AnyCancellable? private var audioPowerCancellable: AnyCancellable? @@ -694,6 +704,14 @@ final class CompanionManager: ObservableObject { guard let self else { return } self.lastTranscript = finalTranscript print("🗣️ Companion received transcript: \(finalTranscript)") + // "teach me a beat" / "guided tour" starts the cloud + // guided tour (Clicky points through the steps). + let lowered = finalTranscript.lowercased() + if lowered.contains("guided tour") || lowered.contains("teach me") || lowered.contains("make a beat") { + self.startGuidedLogicTour() + self.voiceState = .idle + return + } // "take over — " hands off to the offline autonomous // loop instead of a normal voice response. if self.routeTranscriptToTakeoverIfRequested(finalTranscript) { @@ -1035,6 +1053,145 @@ final class CompanionManager: ObservableObject { voiceState = .responding } + // MARK: - Guided Tour (cloud pointing, e.g. Logic Pro beats) + + private static let guidedLogicTourSystemPrompt = """ + you're clicky, guiding the user through making a basic boom bap beat in logic pro. you can see their screen. each turn, look at the screenshot, decide the SINGLE next thing they should do, say it in one or two spoken sentences (all lowercase, casual, written for the ear), and point at the exact control with a [POINT:x,y:label] tag. + + the steps, in order — figure out which one they're on from what's on screen: + 1. create a software instrument track (the plus button or the track menu) + 2. open drum machine designer, or pick a drum kit from the library + 3. open the step sequencer (or the piano roll) + 4. program the kick on beats one and three + 5. program the snare on beats two and four + 6. program hi-hats on every eighth note + 7. press play to hear the beat + + point at exactly one control per turn. the screenshot is labeled with its pixel dimensions — use that coordinate space, origin top-left. if the beat is clearly already playing or finished, say so briefly and end with [POINT:none]. + """ + + /// Starts the autonomous cloud guided tour. Clicky screenshots the screen, + /// asks Sonnet for the next control + narration, flies the overlay cursor to + /// it, speaks, and advances — hands-off. Cloud-only (needs the worker); + /// pure pointing, so no real clicks and no Accessibility needed. + func startGuidedLogicTour() { + guard !isGuidedTourRunning else { return } + guard isNetworkAvailable else { + Task { try? await appleSpeechSynthesisClient.speakText( + "the guided tour needs the internet for the precise pointing. turn wifi back on and try again.") } + return + } + // Force a known-good cloud model for the tour regardless of the picker. + cloudChatProvider.model = "claude-sonnet-4-6" + isGuidedTourRunning = true + // Make sure the overlay is visible so the flying cursor can be seen. + if !isOverlayVisible { + overlayWindowManager.hasShownOverlayBefore = true + overlayWindowManager.showOverlay(onScreens: NSScreen.screens, companionManager: self) + isOverlayVisible = true + } + guidedTourTask = Task { await runGuidedLogicTour() } + } + + func stopGuidedTour() { + guidedTourTask?.cancel() + guidedTourTask = nil + isGuidedTourRunning = false + clearDetectedElementLocation() + voiceState = .idle + } + + private func runGuidedLogicTour() async { + defer { + isGuidedTourRunning = false + voiceState = .idle + } + let maxSteps = 9 + await narrateTour("okay, let's make a beat. follow my cursor.") + + for _ in 0.. display point -> AppKit global transform the voice pipeline uses, + /// kept separate so the working voice path is untouched. + private func flyOverlayCursor(toParseResult parseResult: PointingParseResult, screenCaptures: [CompanionScreenCapture]) { + guard let pointCoordinate = parseResult.coordinate else { return } + let targetScreenCapture: CompanionScreenCapture? = { + if let screenNumber = parseResult.screenNumber, + screenNumber >= 1 && screenNumber <= screenCaptures.count { + return screenCaptures[screenNumber - 1] + } + return screenCaptures.first(where: { $0.isCursorScreen }) ?? screenCaptures.first + }() + guard let capture = targetScreenCapture else { return } + + let screenshotWidth = CGFloat(capture.screenshotWidthInPixels) + let screenshotHeight = CGFloat(capture.screenshotHeightInPixels) + guard screenshotWidth > 0, screenshotHeight > 0 else { return } + let displayWidth = CGFloat(capture.displayWidthInPoints) + let displayHeight = CGFloat(capture.displayHeightInPoints) + let displayFrame = capture.displayFrame + + let clampedX = max(0, min(pointCoordinate.x, screenshotWidth)) + let clampedY = max(0, min(pointCoordinate.y, screenshotHeight)) + let displayLocalX = clampedX * (displayWidth / screenshotWidth) + let displayLocalY = clampedY * (displayHeight / screenshotHeight) + let appKitY = displayHeight - displayLocalY + + detectedElementScreenLocation = CGPoint( + x: displayLocalX + displayFrame.origin.x, + y: appKitY + displayFrame.origin.y) + detectedElementDisplayFrame = displayFrame + detectedElementBubbleText = parseResult.elementLabel + } + // MARK: - Point Tag Parsing /// Result of parsing a [POINT:...] tag from Claude's response. diff --git a/leanring-buddy/CompanionPanelView.swift b/leanring-buddy/CompanionPanelView.swift index 030ddc92a..7534665a5 100644 --- a/leanring-buddy/CompanionPanelView.swift +++ b/leanring-buddy/CompanionPanelView.swift @@ -37,6 +37,9 @@ struct CompanionPanelView: View { takeoverSection .padding(.horizontal, 16) + + guidedTourSection + .padding(.horizontal, 16) } if !companionManager.allPermissionsGranted { @@ -762,6 +765,59 @@ struct CompanionPanelView: View { } } + // MARK: - Guided Tour (cloud pointing) + + /// One-click start for the cloud guided tour (e.g. Logic Pro beats): + /// Clicky's cursor flies to each control and narrates. Hands-off after the + /// click — record with the OS recorder (Cmd-Shift-5) for the demo video. + @ViewBuilder + private var guidedTourSection: some View { + VStack(alignment: .leading, spacing: 6) { + Divider() + .background(DS.Colors.borderSubtle) + .padding(.vertical, 4) + + HStack { + Text("Guided tour") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + Text("cloud · points only") + .font(.system(size: 9, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + Spacer() + if companionManager.isGuidedTourRunning { + Button(action: { companionManager.stopGuidedTour() }) { + Text("Stop") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(.white) + .padding(.horizontal, 12).padding(.vertical, 5) + .background(RoundedRectangle(cornerRadius: 6, style: .continuous) + .fill(Color(red: 0.85, green: 0.3, blue: 0.3))) + } + .buttonStyle(.plain) + .pointerCursor() + } else { + Button(action: { companionManager.startGuidedLogicTour() }) { + Text("Make a Logic beat") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(.white) + .padding(.horizontal, 12).padding(.vertical, 5) + .background(RoundedRectangle(cornerRadius: 6, style: .continuous) + .fill(DS.Colors.accent)) + } + .buttonStyle(.plain) + .pointerCursor() + } + } + + Text("open logic pro, then click — clicky points through the beat. or say \"teach me a beat\".") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + .fixedSize(horizontal: false, vertical: true) + } + .padding(.top, 8) + } + private func modelOptionButton(label: String, modelID: String) -> some View { let isSelected = companionManager.selectedModel == modelID return Button(action: { diff --git a/leanring-buddy/Info.plist b/leanring-buddy/Info.plist index e3d2b455a..906c8ca34 100644 --- a/leanring-buddy/Info.plist +++ b/leanring-buddy/Info.plist @@ -16,5 +16,12 @@ Clicky needs screen recording access to see your screen and help you. NSSpeechRecognitionUsageDescription Clicky uses speech recognition to transcribe your voice when you talk to it + + NSAppTransportSecurity + + NSAllowsLocalNetworking + + From c15f5806013ada2e5ca4119dbaea819ccc4f1a81 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Mon, 15 Jun 2026 13:28:06 -0400 Subject: [PATCH 17/18] add permission-free local-mode demo window + ignore wrangler cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ClickyLocalDemo runs the real LocalChatProvider path (same model, params, offline TTS) in a window with typed input — a way to show Local Mode without mic/screen permissions. --- .gitignore | 1 + bench/ClickyLocalDemo/Package.resolved | 141 ++++++++++ bench/ClickyLocalDemo/Package.swift | 29 ++ .../Sources/ClickyLocalDemo/main.swift | 262 ++++++++++++++++++ 4 files changed, 433 insertions(+) create mode 100644 bench/ClickyLocalDemo/Package.resolved create mode 100644 bench/ClickyLocalDemo/Package.swift create mode 100644 bench/ClickyLocalDemo/Sources/ClickyLocalDemo/main.swift diff --git a/.gitignore b/.gitignore index 9ed64dba7..60c8c7cb1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ worker/node_modules/ worker/.dev.vars +worker/.wrangler/ .DS_Store *.xcuserstate build/ diff --git a/bench/ClickyLocalDemo/Package.resolved b/bench/ClickyLocalDemo/Package.resolved new file mode 100644 index 000000000..89f5360bb --- /dev/null +++ b/bench/ClickyLocalDemo/Package.resolved @@ -0,0 +1,141 @@ +{ + "originHash" : "9326ba538fa9015ce3f823afe4c85d44b7da9e7d62e231b88ff5d84ba76a71a0", + "pins" : [ + { + "identity" : "eventsource", + "kind" : "remoteSourceControl", + "location" : "https://github.com/mattt/EventSource.git", + "state" : { + "revision" : "a3a85a85214caf642abaa96ae664e4c772a59f6e", + "version" : "1.4.1" + } + }, + { + "identity" : "mlx-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ml-explore/mlx-swift", + "state" : { + "revision" : "dc43e62d7055353c7f99fa071a4e71d29dfddc44", + "version" : "0.31.4" + } + }, + { + "identity" : "mlx-swift-lm", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ml-explore/mlx-swift-lm", + "state" : { + "revision" : "1c05248bb0899e2a7a4962b84d319cf12f4e12aa", + "version" : "3.31.3" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "a9a5efd40eaf558a2bcd48d64b1d1646be686008", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "b601256eab081c0f92f059e12818ac1d4f178ff7", + "version" : "1.3.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "a0cb0954ecb21e4e31b0070e6ed5674e8556685a", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "1b6b2e274e85105bfa155183145a1dcfd63331f1", + "version" : "4.5.0" + } + }, + { + "identity" : "swift-huggingface", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-huggingface", + "state" : { + "revision" : "b721959445b617d0bf03910b2b4aced345fd93bf", + "version" : "0.9.0" + } + }, + { + "identity" : "swift-jinja", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-jinja.git", + "state" : { + "revision" : "0b67ecb79139f6addef8699eff3622808aa6c7dc", + "version" : "2.3.6" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "77b84ac2cd2ac9e4ac67d19f045fd5b434f56967", + "version" : "2.101.0" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax.git", + "state" : { + "revision" : "0687f71944021d616d34d922343dcef086855920", + "version" : "600.0.1" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "7502b711c92a17741fa625d722b0ccbd595d8ed1", + "version" : "1.7.2" + } + }, + { + "identity" : "swift-transformers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-transformers", + "state" : { + "revision" : "2fa33e1f5e7131a7fc64c28e6d161dcec0d24820", + "version" : "1.3.3" + } + }, + { + "identity" : "yyjson", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ibireme/yyjson.git", + "state" : { + "revision" : "8b4a38dc994a110abaec8a400615567bd996105f", + "version" : "0.12.0" + } + } + ], + "version" : 3 +} diff --git a/bench/ClickyLocalDemo/Package.swift b/bench/ClickyLocalDemo/Package.swift new file mode 100644 index 000000000..f24418db3 --- /dev/null +++ b/bench/ClickyLocalDemo/Package.swift @@ -0,0 +1,29 @@ +// swift-tools-version: 6.1 +// A runnable window app that demonstrates Clicky's Local Mode feature without +// needing mic / accessibility / screen-recording grants: type a question, the +// on-device model answers (streamed text + latency badge), and it speaks the +// reply via AVSpeechSynthesizer — the same model, params, and offline TTS the +// real app uses in Local Mode. Build with Xcode (MLX Metal shaders). +import PackageDescription + +let package = Package( + name: "ClickyLocalDemo", + platforms: [.macOS(.v14)], + dependencies: [ + .package(url: "https://github.com/ml-explore/mlx-swift-lm", .upToNextMajor(from: "3.31.3")), + .package(url: "https://github.com/huggingface/swift-huggingface", from: "0.9.0"), + .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.0"), + ], + targets: [ + .executableTarget( + name: "ClickyLocalDemo", + dependencies: [ + .product(name: "MLXLLM", package: "mlx-swift-lm"), + .product(name: "MLXLMCommon", package: "mlx-swift-lm"), + .product(name: "MLXHuggingFace", package: "mlx-swift-lm"), + .product(name: "HuggingFace", package: "swift-huggingface"), + .product(name: "Tokenizers", package: "swift-transformers"), + ] + ) + ] +) diff --git a/bench/ClickyLocalDemo/Sources/ClickyLocalDemo/main.swift b/bench/ClickyLocalDemo/Sources/ClickyLocalDemo/main.swift new file mode 100644 index 000000000..4b1a7c0eb --- /dev/null +++ b/bench/ClickyLocalDemo/Sources/ClickyLocalDemo/main.swift @@ -0,0 +1,262 @@ +// +// ClickyLocalDemo +// +// A watch-only window app for Clicky's Local Mode. Mirrors the real app's +// LocalChatProvider (same model: Llama-3.2-3B-Instruct-4bit, same cache dir, +// same generation params, same local system prompt) and AppleSpeechSynthesis +// client — but takes typed input instead of push-to-talk, so it needs none of +// the mic / accessibility / screen-recording permissions the menu-bar app does. +// +// It auto-runs a question on launch: model loads from disk, the answer streams +// into the bubble with a "local · 0.6s · 57 tok/s" badge, and the reply is +// spoken aloud. Wifi can be off the whole time. +// + +import AppKit +import AVFoundation +import Foundation +import HuggingFace +import MLX +import MLXHuggingFace +import MLXLLM +import MLXLMCommon +import SwiftUI +import Tokenizers + +// MARK: - Local system prompt (verbatim from the app's Local Mode) + +let localSystemPrompt = """ +you're clicky, a friendly always-on companion that lives in the user's menu bar. the user just spoke to you via push-to-talk. you're running fully on their mac right now — no cloud, no screen access — so answer from what they said and what you remember of this conversation. your reply will be spoken aloud via text-to-speech, so write the way you'd actually talk. + +rules: +- default to one or two sentences. be direct and dense. if the user asks you to go deeper, give a longer explanation. +- all lowercase, casual, warm. no emojis. +- write for the ear, not the eye. short sentences. no lists, bullet points, markdown, or formatting — just natural speech. +- don't use abbreviations or symbols that sound weird read aloud. write "for example" not "e.g.", spell out small numbers. +- you can't see the screen in local mode. if the user asks about something on their screen, say so plainly and ask them to read or describe it — or suggest flipping to a cloud model for screen questions. +- never say "simply" or "just". +- never output coordinate tags like [POINT:...] — you can't point at the screen in local mode. +""" + +// MARK: - On-device engine (mirrors LocalChatProvider) + +@MainActor +final class DemoLocalEngine: ObservableObject { + enum Phase: Equatable { + case idle, loading(String), answering, spoken + } + @Published var phase: Phase = .idle + @Published var answer: String = "" + @Published var latencyBadge: String = "" + @Published var isSpeaking = false + + private var modelContainer: ModelContainer? + private let speechSynthesizer = AVSpeechSynthesizer() + private let speechDelegate = SpeechDelegate() + + init() { + speechSynthesizer.delegate = speechDelegate + speechDelegate.onFinish = { [weak self] in self?.isSpeaking = false } + } + + private func cacheDirectory() throws -> URL { + let appSupport = try FileManager.default.url( + for: .applicationSupportDirectory, in: .userDomainMask, appropriateFor: nil, create: true) + return appSupport.appendingPathComponent("Clicky/models/huggingface", isDirectory: true) + } + + func ask(_ question: String) { + Task { await run(question) } + } + + private func run(_ question: String) async { + do { + if modelContainer == nil { + phase = .loading("loading the on-device model…") + MLX.Memory.cacheLimit = 32 * 1024 * 1024 + let hubClient = HubClient(cache: HubCache(cacheDirectory: try cacheDirectory())) + modelContainer = try await LLMModelFactory.shared.loadContainer( + from: #hubDownloader(hubClient), + using: #huggingFaceTokenizerLoader(), + configuration: LLMRegistry.llama3_2_3B_4bit, + progressHandler: { progress in + let pct = Int(progress.fractionCompleted * 100) + Task { @MainActor [weak self] in self?.phase = .loading("downloading model… \(pct)%") } + } + ) + } + guard let modelContainer else { return } + + phase = .answering + answer = "" + latencyBadge = "" + + let input = try await modelContainer.prepare(input: UserInput(chat: [ + .system(localSystemPrompt), + .user(question), + ])) + let requestStart = Date() + var firstTokenAt: TimeInterval? + var tokensPerSecond: Double? + + let stream = try await modelContainer.generate( + input: input, + parameters: GenerateParameters(maxTokens: 512, maxKVSize: 4096, temperature: 0.7)) + + for await generation in stream { + switch generation { + case .chunk(let text): + if firstTokenAt == nil { firstTokenAt = Date().timeIntervalSince(requestStart) } + answer += text + case .info(let info): + tokensPerSecond = info.tokensPerSecond + case .toolCall: + break + } + } + + answer = answer.trimmingCharacters(in: .whitespacesAndNewlines) + if let firstTokenAt { + var badge = String(format: "local · %.1fs", firstTokenAt) + if let tokensPerSecond { badge += String(format: " · %.0f tok/s", tokensPerSecond) } + latencyBadge = badge + FileHandle.standardError.write(Data("[demo] \(badge)\n".utf8)) + } + FileHandle.standardError.write(Data("[demo] answer: \(answer)\n".utf8)) + FileHandle.standardError.write(Data("[demo] speaking…\n".utf8)) + + speak(answer) + phase = .spoken + } catch { + answer = "demo error: \(error.localizedDescription)" + phase = .idle + } + } + + private func speak(_ text: String) { + let utterance = AVSpeechUtterance(string: text) + let englishVoices = AVSpeechSynthesisVoice.speechVoices().filter { $0.language == "en-US" } + utterance.voice = englishVoices.first(where: { $0.quality == .premium }) + ?? englishVoices.first(where: { $0.quality == .enhanced }) + ?? AVSpeechSynthesisVoice(language: "en-US") + isSpeaking = true + speechSynthesizer.speak(utterance) + } +} + +final class SpeechDelegate: NSObject, AVSpeechSynthesizerDelegate { + var onFinish: (() -> Void)? + func speechSynthesizer(_ s: AVSpeechSynthesizer, didFinish u: AVSpeechUtterance) { + Task { @MainActor in self.onFinish?() } + } +} + +// MARK: - UI (Clicky dark aesthetic) + +struct DemoView: View { + @StateObject private var engine = DemoLocalEngine() + @State private var question = "i'm making a boom bap beat in logic pro at 90 bpm — give me a kick, snare, and hi-hat pattern i can program on the step sequencer." + + private let surface = Color(red: 0.09, green: 0.10, blue: 0.095) + private let cursorBlue = Color(red: 0.30, green: 0.55, blue: 1.0) + + var body: some View { + VStack(alignment: .leading, spacing: 16) { + HStack(spacing: 8) { + Circle().fill(Color.green).frame(width: 8, height: 8) + Text("Clicky").font(.system(size: 15, weight: .semibold)).foregroundColor(.white) + Text("Local Mode").font(.system(size: 11, weight: .medium)).foregroundColor(.gray) + Spacer() + // The picker, with Local lit (as it looks in the real panel) + HStack(spacing: 0) { + pill("Sonnet", on: false); pill("Opus", on: false); pill("Local", on: true) + } + .background(RoundedRectangle(cornerRadius: 6).fill(Color.white.opacity(0.06))) + } + + Text("your screen stays on your mac · works offline · $0") + .font(.system(size: 11)).foregroundColor(.gray) + + HStack(spacing: 8) { + TextField("ask anything…", text: $question) + .textFieldStyle(.plain) + .font(.system(size: 13)).foregroundColor(.white) + .padding(.horizontal, 12).padding(.vertical, 9) + .background(RoundedRectangle(cornerRadius: 8).fill(Color.white.opacity(0.07))) + Button(action: { engine.ask(question) }) { + Text("Ask").font(.system(size: 13, weight: .semibold)).foregroundColor(.white) + .padding(.horizontal, 16).padding(.vertical, 9) + .background(RoundedRectangle(cornerRadius: 8).fill(cursorBlue)) + }.buttonStyle(.plain) + } + + statusLine + + ScrollView { + Text(engine.answer.isEmpty ? " " : engine.answer) + .font(.system(size: 14)).foregroundColor(.white).lineSpacing(4) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(14) + .background(RoundedRectangle(cornerRadius: 10).fill(Color.white.opacity(0.05))) + } + .frame(maxHeight: .infinity) + + if !engine.latencyBadge.isEmpty { + HStack(spacing: 6) { + Text(engine.latencyBadge) + .font(.system(size: 11, weight: .medium).monospacedDigit()) + .foregroundColor(.white.opacity(0.85)) + .padding(.horizontal, 8).padding(.vertical, 4) + .background(RoundedRectangle(cornerRadius: 6).fill(cursorBlue.opacity(0.25))) + if engine.isSpeaking { + Image(systemName: "speaker.wave.2.fill").foregroundColor(cursorBlue) + Text("speaking…").font(.system(size: 11)).foregroundColor(.gray) + } + } + } + } + .padding(22) + .frame(minWidth: 520, minHeight: 460) + .background(surface) + .onAppear { + // Watch-only: kick off the demo question shortly after the window shows. + DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { engine.ask(question) } + } + } + + @ViewBuilder private var statusLine: some View { + switch engine.phase { + case .idle: Text(" ").font(.system(size: 11)) + case .loading(let message): label(message, dim: true) + case .answering: label("thinking on-device…", dim: true) + case .spoken: label("answered locally — wifi never touched", dim: true) + } + } + + private func label(_ text: String, dim: Bool) -> some View { + Text(text).font(.system(size: 11)).foregroundColor(dim ? .gray : .white) + } + + private func pill(_ title: String, on: Bool) -> some View { + Text(title).font(.system(size: 11, weight: .medium)) + .foregroundColor(on ? .white : .gray) + .padding(.horizontal, 10).padding(.vertical, 5) + .background(RoundedRectangle(cornerRadius: 5).fill(on ? cursorBlue.opacity(0.5) : .clear)) + } +} + +// MARK: - AppKit bootstrap (window without a full .app bundle) + +let app = NSApplication.shared +app.setActivationPolicy(.regular) + +let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 560, height: 500), + styleMask: [.titled, .closable, .miniaturizable, .resizable], + backing: .buffered, defer: false) +window.title = "Clicky — Local Mode (demo)" +window.center() +window.contentView = NSHostingView(rootView: DemoView()) +window.makeKeyAndOrderFront(nil) +app.activate(ignoringOtherApps: true) +app.run() From 548584439a0308ff6e2a7a98b662877bf84e6827 Mon Sep 17 00:00:00 2001 From: Shrey Patel Date: Mon, 15 Jun 2026 13:48:28 -0400 Subject: [PATCH 18/18] polish: address pre-PR review Blockers: restore the user's cloud model after a guided tour (was silently downgrading Opus to Sonnet for the session); refuse takeover if the ESC kill switch can't be installed instead of running blind. Fixes: route fallback voices through the retained Apple Speech client (local NSSpeechSynthesizer was deallocating mid-sentence); gate bare destructive keys (delete/backspace) behind the risky opt-in; tighten guided-tour/takeover triggers so they don't hijack ordinary cloud queries; guard empty TTS utterances; don't treat repeated scrolls as stuck. Cleanup: share the model-cache dir helper, use DS color tokens in the new panel UI, document the takeover + guided-tour files. --- AGENTS.md | 9 +++++ .../Sources/ClickyLocalDemo/main.swift | 2 +- .../AppleSpeechSynthesisClient.swift | 4 ++ leanring-buddy/ClickyModelCache.swift | 26 +++++++++++++ leanring-buddy/CompanionManager.swift | 38 +++++++++++++------ leanring-buddy/CompanionPanelView.swift | 8 ++-- .../ComputerUseActionExecutor.swift | 10 +++-- leanring-buddy/LocalChatProvider.swift | 18 +-------- leanring-buddy/OverlayWindow.swift | 2 +- leanring-buddy/TakeoverAction.swift | 6 ++- leanring-buddy/TakeoverController.swift | 30 ++++++++++----- leanring-buddy/TakeoverVisionAgent.swift | 11 +----- 12 files changed, 105 insertions(+), 59 deletions(-) create mode 100644 leanring-buddy/ClickyModelCache.swift diff --git a/AGENTS.md b/AGENTS.md index 9a675e4ff..db95a05ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,6 +48,10 @@ Worker vars: `ELEVENLABS_VOICE_ID` **Transient Cursor Mode**: When "Show Clicky" is off, pressing the hotkey fades in the cursor overlay for the duration of the interaction (recording → response → TTS → optional pointing), then fades it out automatically after 1 second of inactivity. +**Guided tour (experimental, cloud)**: A panel button ("Make a Logic beat") or the phrase "teach me a beat" starts an autonomous cloud loop — Clicky screenshots the screen, asks Sonnet for the next control + narration, flies the *overlay* cursor to it (no real clicks, no Accessibility), narrates via Apple Speech, and advances. `CompanionManager.startGuidedLogicTour` + `guidedTourSection` in the panel. Reliable because Sonnet grounding is accurate; pure pointing so it can't damage anything. + +**Takeover (experimental, offline)**: The phrase "take over — " (offline only) hands the cursor + keyboard to a guarded local-VLM loop. See `TAKEOVER.md` — the brain grounds in isolation but small VLMs misclick on dense UIs; dry-run is the default and the executor/loop are unproven in a live GUI. + **Local Mode**: A third model-picker option runs the whole answer loop on-device: Apple Speech for transcription, Llama-3.2-3B-Instruct-4bit via MLX for the response, `AVSpeechSynthesizer` for the voice. The behavioral contract is strict — the screenshot is *never captured* (not captured-and-dropped), `[POINT:...]` pointing is disabled via a trimmed local system prompt, and the transcript/response analytics events do not fire (a content-free `local_mode_selected` event is the only signal). The model downloads once (~1.8 GB) to `~/Library/Application Support/Clicky/models/huggingface` with progress in the panel, then loads from disk on later launches so Local Mode works fully offline. Cloud requests fail fast with a spoken nudge toward Local when the network is down (the URLSession otherwise waits 120s for connectivity). ## Key Files @@ -73,6 +77,11 @@ Worker vars: `ELEVENLABS_VOICE_ID` | `LocalChatProvider.swift` | ~253 | On-device chat provider for Local Mode. Runs Llama-3.2-3B-Instruct-4bit via MLX (`mlx-swift-lm`); downloads once to Application Support with progress, then loads from disk (offline) on later launches. Publishes `modelReadiness` for the panel UI. | | `BuddyTextToSpeechClient.swift` | ~28 | Protocol surface for TTS backends (`speakText` returns at playback start, `isPlaying`, synchronous `stopPlayback`). ElevenLabs and Apple Speech both conform. | | `AppleSpeechSynthesisClient.swift` | ~73 | On-device TTS for Local Mode via `AVSpeechSynthesizer`. Picks the best installed en-US voice (premium > enhanced > compact) and honors the ElevenLabs client's timing contract. | +| `ClickyModelCache.swift` | ~30 | Single source of truth for the on-device model directory (Application Support, not Caches). Shared by the local chat provider and takeover vision agent. | +| `TakeoverController.swift` | ~300 | **Experimental.** Offline autonomous "takeover" loop (e.g. drive an app). Screenshot → local VLM → one guarded action → execute → repeat. Rails: dry-run default, ESC kill switch, offline+Accessibility gate, danger-action opt-in, step/time caps. See `TAKEOVER.md` for the honest verification state. | +| `TakeoverVisionAgent.swift` | ~165 | **Experimental.** The takeover brain — Qwen2.5-VL-3B (MLXVLM) emitting one JSON action per screenshot. | +| `TakeoverAction.swift` | ~205 | **Experimental.** The 6-verb takeover action vocabulary + a tolerant JSON/regex parser for shaky small-VLM output. | +| `ComputerUseActionExecutor.swift` | ~245 | **Experimental.** CoreGraphics input synthesis (move/click/type/scroll/key) + the ESC kill-switch tap. Needs only the existing Accessibility grant. | | `OpenAIAPI.swift` | ~142 | OpenAI GPT vision API client. | | `ElevenLabsTTSClient.swift` | ~81 | ElevenLabs TTS client. Sends text to the Worker proxy, plays back audio via `AVAudioPlayer`. Exposes `isPlaying` for transient cursor scheduling. | | `ElementLocationDetector.swift` | ~335 | Detects UI element locations in screenshots for cursor pointing. | diff --git a/bench/ClickyLocalDemo/Sources/ClickyLocalDemo/main.swift b/bench/ClickyLocalDemo/Sources/ClickyLocalDemo/main.swift index 4b1a7c0eb..8b66c67d0 100644 --- a/bench/ClickyLocalDemo/Sources/ClickyLocalDemo/main.swift +++ b/bench/ClickyLocalDemo/Sources/ClickyLocalDemo/main.swift @@ -146,7 +146,7 @@ final class DemoLocalEngine: ObservableObject { final class SpeechDelegate: NSObject, AVSpeechSynthesizerDelegate { var onFinish: (() -> Void)? - func speechSynthesizer(_ s: AVSpeechSynthesizer, didFinish u: AVSpeechUtterance) { + func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { Task { @MainActor in self.onFinish?() } } } diff --git a/leanring-buddy/AppleSpeechSynthesisClient.swift b/leanring-buddy/AppleSpeechSynthesisClient.swift index 98a4d01b8..0dcaee1b1 100644 --- a/leanring-buddy/AppleSpeechSynthesisClient.swift +++ b/leanring-buddy/AppleSpeechSynthesisClient.swift @@ -32,6 +32,10 @@ final class AppleSpeechSynthesisClient: NSObject, BuddyTextToSpeechClient { func speakText(_ text: String) async throws { try Task.checkCancellation() + // Nothing to say — return without arming isPlaying, or the empty + // utterance never fires didFinish and the transient-hide loop hangs. + guard !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return } + let utterance = AVSpeechUtterance(string: text) utterance.voice = Self.bestAvailableEnglishVoice() diff --git a/leanring-buddy/ClickyModelCache.swift b/leanring-buddy/ClickyModelCache.swift new file mode 100644 index 000000000..b0b2d267f --- /dev/null +++ b/leanring-buddy/ClickyModelCache.swift @@ -0,0 +1,26 @@ +// +// ClickyModelCache.swift +// leanring-buddy +// +// Single source of truth for where on-device models live. Application +// Support (not Caches) so macOS doesn't purge multi-GB weights under disk +// pressure. Shared by the local chat provider and the takeover vision agent. +// + +import Foundation + +enum ClickyModelCache { + /// `~/Library/Application Support/Clicky/models/huggingface`, created if needed. + static func huggingFaceCacheDirectory() throws -> URL { + let applicationSupportDirectory = try FileManager.default.url( + for: .applicationSupportDirectory, + in: .userDomainMask, + appropriateFor: nil, + create: true + ) + return applicationSupportDirectory + .appendingPathComponent("Clicky", isDirectory: true) + .appendingPathComponent("models", isDirectory: true) + .appendingPathComponent("huggingface", isDirectory: true) + } +} diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index 6750edc76..1b4cdac64 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -136,6 +136,9 @@ final class CompanionManager: ObservableObject { /// If the transcript is a takeover command, routes it and returns true. /// Offline + Accessibility are enforced inside TakeoverController.start. private func routeTranscriptToTakeoverIfRequested(_ transcript: String) -> Bool { + // Takeover is offline-only, so only intercept the trigger when offline. + // Online, "take over the narration" falls through to the normal cloud path. + guard !isNetworkAvailable else { return false } let lowered = transcript.lowercased() guard let phrase = Self.takeoverTriggerPhrases.first(where: { lowered.contains($0) }) else { return false @@ -198,6 +201,9 @@ final class CompanionManager: ObservableObject { /// Clicky autonomously points its cursor at the next control each step. @Published private(set) var isGuidedTourRunning = false private var guidedTourTask: Task? + /// The user's cloud model before a guided tour forced Sonnet, restored when + /// the tour ends so an Opus user isn't silently downgraded for the session. + private var cloudModelBeforeGuidedTour: String? private var shortcutTransitionCancellable: AnyCancellable? private var voiceStateCancellable: AnyCancellable? @@ -704,10 +710,12 @@ final class CompanionManager: ObservableObject { guard let self else { return } self.lastTranscript = finalTranscript print("🗣️ Companion received transcript: \(finalTranscript)") - // "teach me a beat" / "guided tour" starts the cloud - // guided tour (Clicky points through the steps). + // Specific guided-tour commands only — broad phrases like + // "teach me" would hijack ordinary cloud questions + // ("teach me about closures"). let lowered = finalTranscript.lowercased() - if lowered.contains("guided tour") || lowered.contains("teach me") || lowered.contains("make a beat") { + let guidedTourTriggers = ["guided tour", "teach me a beat", "make me a beat", "walk me through a beat"] + if guidedTourTriggers.contains(where: { lowered.contains($0) }) { self.startGuidedLogicTour() self.voiceState = .idle return @@ -1038,19 +1046,20 @@ final class CompanionManager: ObservableObject { let utterance = isLocalModeRequest ? "hmm, my local brain hiccuped. give it another try." : "I'm all out of credits. Please DM Farza and tell him to bring me back to life." - let synthesizer = NSSpeechSynthesizer() - synthesizer.startSpeaking(utterance) + // Route through the owned Apple Speech client, which retains its + // synthesizer for the utterance's lifetime. A local NSSpeechSynthesizer + // would deallocate on return and cut the speech off — and this is the + // one moment that line is the only feedback the user gets. voiceState = .responding + Task { try? await appleSpeechSynthesisClient.speakText(utterance) } } - /// Spoken when a cloud request is attempted with no network. Uses the - /// system synthesizer for the same reason as the error fallback — it - /// can't depend on the thing that's missing. + /// Spoken when a cloud request is attempted with no network. On-device + /// voice so it can't depend on the thing that's missing. private func speakOfflineCloudNudge() { let utterance = "looks like the internet is out. flip me to local in the menu bar and i can keep helping." - let synthesizer = NSSpeechSynthesizer() - synthesizer.startSpeaking(utterance) voiceState = .responding + Task { try? await appleSpeechSynthesisClient.speakText(utterance) } } // MARK: - Guided Tour (cloud pointing, e.g. Logic Pro beats) @@ -1081,7 +1090,9 @@ final class CompanionManager: ObservableObject { "the guided tour needs the internet for the precise pointing. turn wifi back on and try again.") } return } - // Force a known-good cloud model for the tour regardless of the picker. + // Force a known-good cloud model for the tour regardless of the picker, + // remembering the user's choice so it's restored when the tour ends. + cloudModelBeforeGuidedTour = cloudChatProvider.model cloudChatProvider.model = "claude-sonnet-4-6" isGuidedTourRunning = true // Make sure the overlay is visible so the flying cursor can be seen. @@ -1105,6 +1116,11 @@ final class CompanionManager: ObservableObject { defer { isGuidedTourRunning = false voiceState = .idle + // Restore the user's cloud model on every exit path (done, cancel, error). + if let previousModel = cloudModelBeforeGuidedTour { + cloudChatProvider.model = previousModel + cloudModelBeforeGuidedTour = nil + } } let maxSteps = 9 await narrateTour("okay, let's make a beat. follow my cursor.") diff --git a/leanring-buddy/CompanionPanelView.swift b/leanring-buddy/CompanionPanelView.swift index 7534665a5..91856933e 100644 --- a/leanring-buddy/CompanionPanelView.swift +++ b/leanring-buddy/CompanionPanelView.swift @@ -662,7 +662,7 @@ struct CompanionPanelView: View { case .failed(let errorDescription): Text("model download hiccuped: \(errorDescription)") .font(.system(size: 11)) - .foregroundColor(Color(red: 0.9, green: 0.4, blue: 0.4)) + .foregroundColor(DS.Colors.destructiveText) .fixedSize(horizontal: false, vertical: true) Button(action: { companionManager.localChatProvider.loadModelIfNeeded() @@ -732,10 +732,10 @@ struct CompanionPanelView: View { )) { Text("Allow typing + submits") .font(.system(size: 11, weight: .medium)) - .foregroundColor(Color(red: 0.9, green: 0.55, blue: 0.35)) + .foregroundColor(DS.Colors.warning) } .toggleStyle(.switch) - .tint(Color(red: 0.9, green: 0.55, blue: 0.35)) + .tint(DS.Colors.warning) .scaleEffect(0.8, anchor: .leading) } @@ -792,7 +792,7 @@ struct CompanionPanelView: View { .foregroundColor(.white) .padding(.horizontal, 12).padding(.vertical, 5) .background(RoundedRectangle(cornerRadius: 6, style: .continuous) - .fill(Color(red: 0.85, green: 0.3, blue: 0.3))) + .fill(DS.Colors.destructive)) } .buttonStyle(.plain) .pointerCursor() diff --git a/leanring-buddy/ComputerUseActionExecutor.swift b/leanring-buddy/ComputerUseActionExecutor.swift index 71e05f79b..05f67d7c2 100644 --- a/leanring-buddy/ComputerUseActionExecutor.swift +++ b/leanring-buddy/ComputerUseActionExecutor.swift @@ -148,8 +148,11 @@ final class ComputerUseActionExecutor { /// Starts a global listen-only tap that fires `onKill` when the user /// presses ESC. Ignores the app's own synthesized events via the - /// sentinel, so driving the cursor never trips it. - func startKillSwitchMonitor(onKill: @escaping () -> Void) { + /// sentinel, so driving the cursor never trips it. Returns false if the + /// tap can't be created — the caller must refuse to run without a kill + /// switch rather than drive input blind. + @discardableResult + func startKillSwitchMonitor(onKill: @escaping () -> Void) -> Bool { stopKillSwitchMonitor() self.onKillSwitch = onKill @@ -170,13 +173,14 @@ final class ComputerUseActionExecutor { userInfo: Unmanaged.passUnretained(self).toOpaque() ) else { print("⚠️ Takeover: failed to create kill-switch tap") - return + return false } self.killSwitchTap = tap let runLoopSource = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0) self.killSwitchRunLoopSource = runLoopSource CFRunLoopAddSource(CFRunLoopGetMain(), runLoopSource, .commonModes) CGEvent.tapEnable(tap: tap, enable: true) + return true } func stopKillSwitchMonitor() { diff --git a/leanring-buddy/LocalChatProvider.swift b/leanring-buddy/LocalChatProvider.swift index cbaef9e00..23bfab8a4 100644 --- a/leanring-buddy/LocalChatProvider.swift +++ b/leanring-buddy/LocalChatProvider.swift @@ -57,22 +57,6 @@ final class LocalChatProvider: BuddyChatProvider, ObservableObject { /// which is what makes Local Mode work offline across relaunches. private static let downloadedModelDirectoryDefaultsKey = "localModelDownloadedDirectory" - /// Models download under Application Support, not Caches — macOS purges - /// Caches under disk pressure and silently re-downloading 1.8 GB is a - /// bad surprise. - private static func modelCacheDirectory() throws -> URL { - let applicationSupportDirectory = try FileManager.default.url( - for: .applicationSupportDirectory, - in: .userDomainMask, - appropriateFor: nil, - create: true - ) - return applicationSupportDirectory - .appendingPathComponent("Clicky", isDirectory: true) - .appendingPathComponent("models", isDirectory: true) - .appendingPathComponent("huggingface", isDirectory: true) - } - init() { // If the model was downloaded on an earlier launch, report it as // ready-to-load rather than not-downloaded so the picker doesn't @@ -136,7 +120,7 @@ final class LocalChatProvider: BuddyChatProvider, ObservableObject { // First time: download from Hugging Face with visible progress. modelReadiness = .downloading(progressFraction: 0) - let hubCacheDirectory = try Self.modelCacheDirectory() + let hubCacheDirectory = try ClickyModelCache.huggingFaceCacheDirectory() let hubClient = HubClient(cache: HubCache(cacheDirectory: hubCacheDirectory)) print("🏠 Local model: downloading \(Self.modelConfiguration.name) to \(hubCacheDirectory.path)") diff --git a/leanring-buddy/OverlayWindow.swift b/leanring-buddy/OverlayWindow.swift index 1a139db36..23de1f76a 100644 --- a/leanring-buddy/OverlayWindow.swift +++ b/leanring-buddy/OverlayWindow.swift @@ -295,7 +295,7 @@ struct BlueCursorView: View { } // Latency badge — provider + first-token time, shown while the - // answer is being spoken. The local-vs-cloud speed difference as + // answer is being spoken. Makes the local-vs-cloud speed difference // something you can see, not just feel. if isCursorOnThisScreen && companionManager.voiceState == .responding, let latencyDescription = companionManager.lastResponseLatencyDescription { diff --git a/leanring-buddy/TakeoverAction.swift b/leanring-buddy/TakeoverAction.swift index 08ea9aa2f..888fb0c81 100644 --- a/leanring-buddy/TakeoverAction.swift +++ b/leanring-buddy/TakeoverAction.swift @@ -62,8 +62,10 @@ enum TakeoverAction { return at == bt case let (.key(ac, _), .key(bc, _)): return ac.lowercased() == bc.lowercased() - case let (.scroll(_, _, ad, _, _), .scroll(_, _, bd, _, _)): - return ad == bd + case (.scroll, .scroll): + // Repeated scrolls in the same direction are legitimate progress + // (scrolling down a long page), not a stuck loop. + return false default: return false } diff --git a/leanring-buddy/TakeoverController.swift b/leanring-buddy/TakeoverController.swift index a7536191f..cb9055185 100644 --- a/leanring-buddy/TakeoverController.swift +++ b/leanring-buddy/TakeoverController.swift @@ -38,7 +38,7 @@ final class TakeoverController: ObservableObject { private static let stepCap = 25 private static let wallClockCapSeconds: TimeInterval = 120 private static let dangerWordPattern = - #"(?i)\b(send|delete|remove|buy|purchase|pay|confirm|submit|post|publish|discard|trash|erase|format|reset|sign\s?out|log\s?out)\b"# + #"(?i)\b(send|delete|backspace|remove|buy|purchase|pay|confirm|submit|post|publish|discard|trash|erase|format|reset|sign\s?out|log\s?out)\b"# private let visionAgent: TakeoverVisionAgent private let executor: ComputerUseActionExecutor @@ -96,14 +96,21 @@ final class TakeoverController: ObservableObject { failToStart(.accessibilityMissing); return } - killRequested = false - runState = .running(step: 0, lastAction: "looking at your screen") - - // Rail 2: kill switch live for the whole run. - executor.startKillSwitchMonitor { [weak self] in + // Rail 2: kill switch must be live before any input is driven. If the + // tap can't be installed, refuse to run rather than drive blind. + let killSwitchInstalled = executor.startKillSwitchMonitor { [weak self] in self?.requestKill() } + guard killSwitchInstalled else { + isArmed = false + allowRiskyActions = false + runState = .stopped(reason: "couldn't install the emergency stop") + Task { await narrate("i couldn't set up the emergency stop, so i won't take over. try again.") } + return + } + killRequested = false + runState = .running(step: 0, lastAction: "looking at your screen") runTask = Task { await runLoop(task: task) } } @@ -235,13 +242,16 @@ final class TakeoverController: ObservableObject { } executor.typeText(text) - case .key(let combo, _): + case .key(let combo, let why): if dryRun { return } - // Rail 5: submission (return) and any combo need the risky opt-in. + // Rail 5: submission (return), any combo, AND destructive bare keys + // (delete/backspace and anything matching the danger words) need the + // risky opt-in — a bare "delete" must not evade the gate. let lowered = combo.lowercased() let isSubmitOrCombo = lowered == "return" || lowered == "enter" || lowered.contains("+") - if isSubmitOrCombo && !allowRiskyActions { - await narrate("pressing \(combo) could submit something — i'll leave that to you.") + let isDestructiveKey = isDangerous(combo + " " + why) + if (isSubmitOrCombo || isDestructiveKey) && !allowRiskyActions { + await narrate("pressing \(combo) could change something — i'll leave that to you.") return } executor.pressKey(combo: combo) diff --git a/leanring-buddy/TakeoverVisionAgent.swift b/leanring-buddy/TakeoverVisionAgent.swift index 5a0ae11d7..914f512a0 100644 --- a/leanring-buddy/TakeoverVisionAgent.swift +++ b/leanring-buddy/TakeoverVisionAgent.swift @@ -42,15 +42,6 @@ final class TakeoverVisionAgent: ObservableObject { private var loadedModelContainer: ModelContainer? private var modelLoadTask: Task? - private static func modelCacheDirectory() throws -> URL { - let applicationSupportDirectory = try FileManager.default.url( - for: .applicationSupportDirectory, in: .userDomainMask, appropriateFor: nil, create: true) - return applicationSupportDirectory - .appendingPathComponent("Clicky", isDirectory: true) - .appendingPathComponent("models", isDirectory: true) - .appendingPathComponent("huggingface", isDirectory: true) - } - @discardableResult func loadModelIfNeeded() -> Task { if let modelLoadTask { return modelLoadTask } @@ -76,7 +67,7 @@ final class TakeoverVisionAgent: ObservableObject { MLX.Memory.cacheLimit = 32 * 1024 * 1024 readiness = .downloading(progressFraction: 0) - let hubClient = HubClient(cache: HubCache(cacheDirectory: try Self.modelCacheDirectory())) + let hubClient = HubClient(cache: HubCache(cacheDirectory: try ClickyModelCache.huggingFaceCacheDirectory())) print("👁️ Takeover VLM: loading \(Self.modelConfiguration.name)") let container = try await VLMModelFactory.shared.loadContainer(