Add Composio tool-calling as opt-in toggle

.gitignore

@@ -6,3 +6,4 @@ build/
 releases/
 .claude/
 coding-plans/
+worker/.wrangler/

AGENTS.md

@@ -29,12 +29,14 @@ The app never calls external APIs directly. All requests go through a Cloudflare
 
 | Route | Upstream | Purpose |
 |-------|----------|---------|
-| `POST /chat` | `api.anthropic.com/v1/messages` | Claude vision + streaming chat |
+| `POST /chat` | `api.anthropic.com/v1/messages` | Claude vision + streaming chat (no tools) |
+| `POST /chat-tools` | Anthropic + Composio | Tool-augmented chat, agentic loop server-side, non-streaming JSON response. See "Composio Tool-Calling" below. |
 | `POST /tts` | `api.elevenlabs.io/v1/text-to-speech/{voiceId}` | ElevenLabs TTS audio |
 | `POST /transcribe-token` | `streaming.assemblyai.com/v3/token` | Fetches a short-lived (480s) AssemblyAI websocket token |
 
-Worker secrets: `ANTHROPIC_API_KEY`, `ASSEMBLYAI_API_KEY`, `ELEVENLABS_API_KEY`
+Worker secrets: `ANTHROPIC_API_KEY`, `ASSEMBLYAI_API_KEY`, `ELEVENLABS_API_KEY`, `COMPOSIO_API_KEY` (optional, only required for `/chat-tools`)
 Worker vars: `ELEVENLABS_VOICE_ID`
+Worker compatibility: `nodejs_compat` flag enabled — required for `@composio/core`'s Node-style imports to bundle on Workers.
 
 ### Key Architecture Decisions
 
@@ -48,6 +50,8 @@ Worker vars: `ELEVENLABS_VOICE_ID`
 
 **Transient Cursor Mode**: When "Show Clicky" is off, pressing the hotkey fades in the cursor overlay for the duration of the interaction (recording → response → TTS → optional pointing), then fades it out automatically after 1 second of inactivity.
 
+**Composio Tool-Calling (opt-in)**: When the user flips **Tools (Composio)** on in the panel, voice requests route from `claudeAPI.analyzeImageStreaming` (POST `/chat`, SSE) to `claudeAPI.analyzeImageWithTools` (POST `/chat-tools`, single non-streaming JSON). The Worker runs the Claude ↔ Composio agentic loop server-side (capped at 5 iterations), executes tools against the user's connected accounts, and returns `{ text, tool_calls, stop_reason }`. The pointing flow (`[POINT:x,y:label:screenN]`) is only parsed in the streaming path — tools mode is for acting, not pointing. The user is identified to Composio by a stable per-install UUID stored in UserDefaults under `clickyComposioUserId`, sent on every `/chat-tools` request as the `x-clicky-user-id` header. Off by default: opt-in because it requires `composio link <toolkit> --user-id <clicky-uuid>` setup out-of-band.
+
 ## Key Files
 
 | File | Lines | Purpose |
@@ -74,7 +78,8 @@ Worker vars: `ELEVENLABS_VOICE_ID`
 | `ClickyAnalytics.swift` | ~121 | PostHog analytics integration for usage tracking. |
 | `WindowPositionManager.swift` | ~262 | Window placement logic, Screen Recording permission flow, and accessibility permission helpers. |
 | `AppBundleConfiguration.swift` | ~28 | Runtime configuration reader for keys stored in the app bundle Info.plist. |
-| `worker/src/index.ts` | ~142 | Cloudflare Worker proxy. Three routes: `/chat` (Claude), `/tts` (ElevenLabs), `/transcribe-token` (AssemblyAI temp token). |
+| `worker/src/index.ts` | ~310 | Cloudflare Worker proxy. Four routes: `/chat` (Claude streaming passthrough), `/chat-tools` (Claude + Composio agentic loop, non-streaming JSON), `/tts` (ElevenLabs), `/transcribe-token` (AssemblyAI temp token). Uses `@anthropic-ai/sdk`, `@composio/core`, `@composio/anthropic`. Requires `nodejs_compat` Workers flag. |
+| `worker/test-tools.sh` | ~30 | Curl-based smoke test for the `/chat-tools` endpoint against `wrangler dev`. Useful for verifying the Composio loop without rebuilding the Swift app. |
 
 ## Build & Run
 

README.md

@@ -48,6 +48,7 @@ If you want to do it yourself, here's the deal.
 - Node.js 18+ (for the Cloudflare Worker)
 - A [Cloudflare](https://cloudflare.com) account (free tier works)
 - API keys for: [Anthropic](https://console.anthropic.com), [AssemblyAI](https://www.assemblyai.com), [ElevenLabs](https://elevenlabs.io)
+- Optional, for tool-calling: [Composio](https://app.composio.dev) API key
 
 ### 1. Set up the Cloudflare Worker
 
@@ -64,6 +65,11 @@ Now add your secrets. Wrangler will prompt you to paste each one:
 npx wrangler secret put ANTHROPIC_API_KEY
 npx wrangler secret put ASSEMBLYAI_API_KEY
 npx wrangler secret put ELEVENLABS_API_KEY
+
+# Optional — only required if you want the Composio tool-calling toggle
+# in the panel ("Tools (Composio)") to do anything. See the Tools section
+# below.
+npx wrangler secret put COMPOSIO_API_KEY
 ```
 
 For the ElevenLabs voice ID, open `wrangler.toml` and set it there (it's not sensitive):
@@ -97,6 +103,7 @@ ANTHROPIC_API_KEY=sk-ant-...
 ASSEMBLYAI_API_KEY=...
 ELEVENLABS_API_KEY=...
 ELEVENLABS_VOICE_ID=...
+COMPOSIO_API_KEY=...   # optional, only for the Tools toggle
 ```
 
 Then update the proxy URLs in the Swift code to point to `http://localhost:8787` instead of the deployed Worker URL while developing. Grep for `clicky-proxy` to find them all.
@@ -155,6 +162,49 @@ worker/                  # Cloudflare Worker proxy
 CLAUDE.md                # Full architecture doc (agents read this)
 ```
 
+## Tool-calling with Composio (optional)
+
+Clicky can do more than see, hear, and point — it can also _act_. Flip the
+**Tools (Composio)** toggle in the panel and Claude can execute actions
+inside your connected apps (Slack, Gmail, Linear, Notion, GitHub, Google
+Calendar, and ~1000 others) on your behalf via [Composio](https://composio.dev).
+
+Example: hold push-to-talk on a stack trace and say _"send this to #eng on
+Slack"_. Clicky takes a screenshot, Claude reads it, calls
+`SLACK_SEND_MESSAGE` against your connected Slack workspace, and replies
+"Done."
+
+### Setup
+
+1. Get an API key from [app.composio.dev](https://app.composio.dev).
+2. Set it on the Worker (or in `.dev.vars` for local development):
+
+   ```bash
+   cd worker
+   npx wrangler secret put COMPOSIO_API_KEY
+   ```
+
+3. Connect the apps you want Clicky to use against the same user-id Clicky
+   sends from your Mac. Clicky generates a stable id on first launch and
+   stores it under the UserDefaults key `clickyComposioUserId` — easiest
+   path is to read it once from the macOS defaults and pass it to Composio:
+
+   ```bash
+   defaults read so.clicky.leanring-buddy clickyComposioUserId
+   # → clicky-<some-uuid>
+
+   composio link slack --user-id "clicky-<some-uuid>"
+   composio link gmail --user-id "clicky-<some-uuid>"
+   # ...etc
+   ```
+
+4. Flip the **Tools (Composio)** toggle in the panel. Voice requests now
+   route through `/chat-tools` on the Worker, which runs Claude ↔ Composio
+   tool calls server-side and returns the final response.
+
+When the toggle is **off**, Clicky behaves exactly like before — streaming
+chat through `/chat`, no tools, no Composio dependency at runtime.
+
 ## Contributing
 
 PRs welcome. If you're using Claude Code, it already knows the codebase — just tell it what you want to build and point it at `CLAUDE.md`.

leanring-buddy/ClaudeAPI.swift

@@ -211,6 +211,123 @@ class ClaudeAPI {
         return (text: accumulatedResponseText, duration: duration)
     }
 
+    /// Tool-augmented vision request. Sends the same image+prompt payload as
+    /// `analyzeImageStreaming` to a tool-enabled endpoint (e.g. `/chat-tools` on
+    /// the Worker) which runs the Composio agentic loop server-side and
+    /// returns a single non-streaming JSON response of the shape:
+    ///   { "text": "<final assistant text>",
+    ///     "tool_calls": [{ "name": "...", "input": {...}, "result": {...} }, ...],
+    ///     "stop_reason": "end_turn" | "tool_use" | "max_tokens" | ... }
+    ///
+    /// `clickyUserId` identifies the Composio session owner. Different users
+    /// of the same Worker have isolated tool connections (Slack, Gmail, etc.)
+    /// keyed off this id, so it must be stable per Mac.
+    func analyzeImageWithTools(
+        toolsProxyURL: String,
+        clickyUserId: String,
+        images: [(data: Data, label: String)],
+        systemPrompt: String,
+        conversationHistory: [(userPlaceholder: String, assistantResponse: String)] = [],
+        userPrompt: String
+    ) async throws -> (text: String, toolCallNames: [String], duration: TimeInterval) {
+        let toolsRequestStartTime = Date()
+
+        guard let toolsEndpointURL = URL(string: toolsProxyURL) else {
+            throw NSError(
+                domain: "ClaudeAPI",
+                code: -1,
+                userInfo: [NSLocalizedDescriptionKey: "Invalid tools proxy URL: \(toolsProxyURL)"]
+            )
+        }
+
+        var toolsRequest = URLRequest(url: toolsEndpointURL)
+        toolsRequest.httpMethod = "POST"
+        toolsRequest.timeoutInterval = 180
+        toolsRequest.setValue("application/json", forHTTPHeaderField: "Content-Type")
+        // The Worker uses this header to namespace the user's Composio session
+        // (their connected Slack/Gmail/etc. accounts).
+        toolsRequest.setValue(clickyUserId, forHTTPHeaderField: "x-clicky-user-id")
+
+        var requestMessages: [[String: Any]] = []
+
+        for (userPlaceholder, assistantResponse) in conversationHistory {
+            requestMessages.append(["role": "user", "content": userPlaceholder])
+            requestMessages.append(["role": "assistant", "content": assistantResponse])
+        }
+
+        var currentUserContentBlocks: [[String: Any]] = []
+        for image in images {
+            currentUserContentBlocks.append([
+                "type": "image",
+                "source": [
+                    "type": "base64",
+                    "media_type": detectImageMediaType(for: image.data),
+                    "data": image.data.base64EncodedString()
+                ]
+            ])
+            currentUserContentBlocks.append([
+                "type": "text",
+                "text": image.label
+            ])
+        }
+        currentUserContentBlocks.append([
+            "type": "text",
+            "text": userPrompt
+        ])
+        requestMessages.append(["role": "user", "content": currentUserContentBlocks])
+
+        // Note: no `stream: true` — the Worker runs the tool loop and replies
+        // with a single JSON payload, even though `messages` is shaped like a
+        // standard Anthropic Messages request.
+        let toolsRequestBody: [String: Any] = [
+            "model": model,
+            "max_tokens": 1024,
+            "system": systemPrompt,
+            "messages": requestMessages
+        ]
+
+        let toolsRequestBodyData = try JSONSerialization.data(withJSONObject: toolsRequestBody)
+        toolsRequest.httpBody = toolsRequestBodyData
+        let toolsRequestPayloadMB = Double(toolsRequestBodyData.count) / 1_048_576.0
+        print("🌐 Claude tools request: \(String(format: "%.1f", toolsRequestPayloadMB))MB, \(images.count) image(s), userId=\(clickyUserId)")
+
+        let (toolsResponseData, toolsResponse) = try await session.data(for: toolsRequest)
+
+        guard let toolsHTTPResponse = toolsResponse as? HTTPURLResponse,
+              (200...299).contains(toolsHTTPResponse.statusCode) else {
+            let toolsErrorBody = String(data: toolsResponseData, encoding: .utf8) ?? "Unknown error"
+            throw NSError(
+                domain: "ClaudeAPI",
+                code: (toolsResponse as? HTTPURLResponse)?.statusCode ?? -1,
+                userInfo: [NSLocalizedDescriptionKey: "Tools API Error: \(toolsErrorBody)"]
+            )
+        }
+
+        guard let toolsResponseJSON = try JSONSerialization.jsonObject(with: toolsResponseData) as? [String: Any],
+              let finalAssistantText = toolsResponseJSON["text"] as? String else {
+            throw NSError(
+                domain: "ClaudeAPI",
+                code: -1,
+                userInfo: [NSLocalizedDescriptionKey: "Invalid tools response format"]
+            )
+        }
+
+        // Pull tool call names so we can log/announce what the agent did.
+        // Result payloads are intentionally not surfaced to the UI — they can
+        // be huge JSON blobs and Claude's `text` already summarizes them.
+        var executedToolCallNames: [String] = []
+        if let toolCallEntries = toolsResponseJSON["tool_calls"] as? [[String: Any]] {
+            for toolCallEntry in toolCallEntries {
+                if let toolCallName = toolCallEntry["name"] as? String {
+                    executedToolCallNames.append(toolCallName)
+                }
+            }
+        }
+
+        let toolsRequestDuration = Date().timeIntervalSince(toolsRequestStartTime)
+        return (text: finalAssistantText, toolCallNames: executedToolCallNames, duration: toolsRequestDuration)
+    }
+
     /// Non-streaming fallback for validation requests where we don't need progressive display.
     func analyzeImage(
         images: [(data: Data, label: String)],

leanring-buddy/CompanionManager.swift

@@ -76,10 +76,30 @@ final class CompanionManager: ObservableObject {
         return ClaudeAPI(proxyURL: "\(Self.workerBaseURL)/chat", model: selectedModel)
     }()
 
+    /// URL of the tool-augmented chat endpoint on the Worker. Used when
+    /// `isToolsEnabled` is on so Claude can call Composio-backed tools
+    /// (Slack, Gmail, etc.) inside the user's connected accounts.
+    private static let toolsProxyURL = "\(workerBaseURL)/chat-tools"
+
     private lazy var elevenLabsTTSClient: ElevenLabsTTSClient = {
         return ElevenLabsTTSClient(proxyURL: "\(Self.workerBaseURL)/tts")
     }()
 
+    /// Stable per-install identifier used to namespace the user's Composio
+    /// session on the Worker. Generated once on first launch and persisted
+    /// to UserDefaults so the same Mac maps to the same set of connected
+    /// Composio apps (Slack, Gmail, etc.) across restarts.
+    private static let clickyComposioUserId: String = {
+        let userDefaultsKey = "clickyComposioUserId"
+        if let existingClickyComposioUserId = UserDefaults.standard.string(forKey: userDefaultsKey),
+           !existingClickyComposioUserId.isEmpty {
+            return existingClickyComposioUserId
+        }
+        let newClickyComposioUserId = "clicky-\(UUID().uuidString.lowercased())"
+        UserDefaults.standard.set(newClickyComposioUserId, forKey: userDefaultsKey)
+        return newClickyComposioUserId
+    }()
+
     /// Conversation history so Claude remembers prior exchanges within a session.
     /// Each entry is the user's transcript and Claude's response.
     private var conversationHistory: [(userTranscript: String, assistantResponse: String)] = []
@@ -139,6 +159,18 @@ final class CompanionManager: ObservableObject {
         }
     }
 
+    /// User preference for whether Composio tool-calling is enabled. When on,
+    /// voice requests route to `/chat-tools` on the Worker which can execute
+    /// connected-app actions (Slack, Gmail, Linear, etc.) on the user's
+    /// behalf instead of just chatting and pointing. Off by default — opt-in
+    /// because it requires `composio link <toolkit>` setup out-of-band.
+    @Published var isToolsEnabled: Bool = UserDefaults.standard.bool(forKey: "isToolsEnabled")
+
+    func setToolsEnabled(_ enabled: Bool) {
+        isToolsEnabled = enabled
+        UserDefaults.standard.set(enabled, forKey: "isToolsEnabled")
+    }
+
     /// Whether the user has completed onboarding at least once. Persisted
     /// to UserDefaults so the Start button only appears on first launch.
     var hasCompletedOnboarding: Bool {
@@ -610,15 +642,37 @@ final class CompanionManager: ObservableObject {
                     (userPlaceholder: entry.userTranscript, assistantResponse: entry.assistantResponse)
                 }
 
-                let (fullResponseText, _) = try await claudeAPI.analyzeImageStreaming(
-                    images: labeledImages,
-                    systemPrompt: Self.companionVoiceResponseSystemPrompt,
-                    conversationHistory: historyForAPI,
-                    userPrompt: transcript,
-                    onTextChunk: { _ in
-                        // No streaming text display — spinner stays until TTS plays
+                // When the Composio tools toggle is on, route through /chat-tools
+                // so Claude can execute connected-app actions server-side. The
+                // tools path is non-streaming (the Worker runs the agentic loop
+                // and returns a single final response) and skips the [POINT:...]
+                // pointing flow because tools mode is about acting, not pointing.
+                let fullResponseText: String
+                if isToolsEnabled {
+                    let toolsResult = try await claudeAPI.analyzeImageWithTools(
+                        toolsProxyURL: Self.toolsProxyURL,
+                        clickyUserId: Self.clickyComposioUserId,
+                        images: labeledImages,
+                        systemPrompt: Self.companionVoiceResponseSystemPrompt,
+                        conversationHistory: historyForAPI,
+                        userPrompt: transcript
+                    )
+                    fullResponseText = toolsResult.text
+                    if !toolsResult.toolCallNames.isEmpty {
+                        print("🛠️ Composio tools executed: \(toolsResult.toolCallNames.joined(separator: ", "))")
                     }
-                )
+                } else {
+                    let streamingResult = try await claudeAPI.analyzeImageStreaming(
+                        images: labeledImages,
+                        systemPrompt: Self.companionVoiceResponseSystemPrompt,
+                        conversationHistory: historyForAPI,
+                        userPrompt: transcript,
+                        onTextChunk: { _ in
+                            // No streaming text display — spinner stays until TTS plays
+                        }
+                    )
+                    fullResponseText = streamingResult.text
+                }
 
                 guard !Task.isCancelled else { return }
 

leanring-buddy/CompanionPanelView.swift

@@ -58,6 +58,14 @@ struct CompanionPanelView: View {
             //         .padding(.horizontal, 16)
             // }
 
+            if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted {
+                Spacer()
+                    .frame(height: 16)
+
+                toolsToggleRow
+                    .padding(.horizontal, 16)
+            }
+
             if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted {
                 Spacer()
                     .frame(height: 16)
@@ -574,6 +582,39 @@ struct CompanionPanelView: View {
         .padding(.vertical, 4)
     }
 
+    // MARK: - Composio Tools Toggle
+
+    /// Toggles whether voice requests can call Composio-backed tools (Slack,
+    /// Gmail, Linear, etc.) on the user's connected accounts. Off by default —
+    /// requires `composio link <toolkit>` setup against the Clicky user-id
+    /// before any tool calls will succeed.
+    private var toolsToggleRow: some View {
+        HStack {
+            HStack(spacing: 8) {
+                Image(systemName: "wrench.and.screwdriver")
+                    .font(.system(size: 12, weight: .medium))
+                    .foregroundColor(DS.Colors.textTertiary)
+                    .frame(width: 16)
+
+                Text("Tools (Composio)")
+                    .font(.system(size: 13, weight: .medium))
+                    .foregroundColor(DS.Colors.textSecondary)
+            }
+
+            Spacer()
+
+            Toggle("", isOn: Binding(
+                get: { companionManager.isToolsEnabled },
+                set: { companionManager.setToolsEnabled($0) }
+            ))
+            .toggleStyle(.switch)
+            .labelsHidden()
+            .tint(DS.Colors.accent)
+            .scaleEffect(0.8)
+        }
+        .padding(.vertical, 4)
+    }
+
     private var speechToTextProviderRow: some View {
         HStack {
             HStack(spacing: 8) {