From 27797d473e6dcc4910eb6948992e170b3844d589 Mon Sep 17 00:00:00 2001 From: octavi42 Date: Mon, 25 May 2026 11:32:27 +0300 Subject: [PATCH 01/14] Add evolving teaching skills with local storage and E2E harness. Clicky now learns from successful tutoring sessions, injects matched skills into prompts, and includes a mock-worker E2E test for the read/write loop. --- FUTURE_IMPLEMENTATION.md | 153 +++++++++++++ IMPLEMENTATION_AND_E2E_PLAN.md | 165 ++++++++++++++ leanring-buddy/ClaudeAPI.swift | 47 ++++ leanring-buddy/ClickyAnalytics.swift | 31 +++ leanring-buddy/ClickyE2EConfiguration.swift | 43 ++++ leanring-buddy/CompanionManager.swift | 213 ++++++++++++++++++- leanring-buddy/CompanionPanelView.swift | 87 ++++++++ leanring-buddy/SkillCurator.swift | 38 ++++ leanring-buddy/SkillMatcher.swift | 95 +++++++++ leanring-buddy/SkillSynthesizer.swift | 104 +++++++++ leanring-buddy/SkillTriggerEvaluator.swift | 84 ++++++++ leanring-buddy/TeachingPromptBuilder.swift | 41 ++++ leanring-buddy/TeachingSkill.swift | 186 ++++++++++++++++ leanring-buddy/TeachingSkillStore.swift | 89 ++++++++ leanring-buddy/leanring_buddyApp.swift | 3 + leanring-buddyTests/TeachingSkillTests.swift | 102 +++++++++ tests/e2e/README.md | 80 +++++++ tests/e2e/mock-worker.mjs | 95 +++++++++ tests/e2e/teaching-skills.sh | 70 ++++++ 19 files changed, 1723 insertions(+), 3 deletions(-) create mode 100644 FUTURE_IMPLEMENTATION.md create mode 100644 IMPLEMENTATION_AND_E2E_PLAN.md create mode 100644 leanring-buddy/ClickyE2EConfiguration.swift create mode 100644 leanring-buddy/SkillCurator.swift create mode 100644 leanring-buddy/SkillMatcher.swift create mode 100644 leanring-buddy/SkillSynthesizer.swift create mode 100644 leanring-buddy/SkillTriggerEvaluator.swift create mode 100644 leanring-buddy/TeachingPromptBuilder.swift create mode 100644 leanring-buddy/TeachingSkill.swift create mode 100644 leanring-buddy/TeachingSkillStore.swift create mode 100644 leanring-buddyTests/TeachingSkillTests.swift create mode 100644 tests/e2e/README.md create mode 100644 tests/e2e/mock-worker.mjs create mode 100755 tests/e2e/teaching-skills.sh diff --git a/FUTURE_IMPLEMENTATION.md b/FUTURE_IMPLEMENTATION.md new file mode 100644 index 000000000..41f6e07f8 --- /dev/null +++ b/FUTURE_IMPLEMENTATION.md @@ -0,0 +1,153 @@ +# Future Implementation — Clicky Fork + +Planned features captured from product brainstorming. Not implemented yet. + +**Implementation order & E2E test plan:** see [`IMPLEMENTATION_AND_E2E_PLAN.md`](./IMPLEMENTATION_AND_E2E_PLAN.md) — implement Evolving Teaching Skills first, then run automated user-perspective E2E tests. + +--- + +## 1. Evolving Teaching Skills (Auto-Learning) + +**Problem:** Clicky starts from zero every session. It does not accumulate knowledge about how to teach specific apps or workflows. + +**Concept:** After successful tutoring interactions, Clicky writes reusable teaching skills locally — inspired by Hermes Agent's learning loop, adapted for screen-native teaching (not CLI automation). + +### What a teaching skill is + +A folder under `~/.clicky/skills//SKILL.md` using the [agentskills.io](https://agentskills.io) format: + +- App or workflow name (e.g. `teach-final-cut-color`, `teach-xcode-source-control`) +- Step order and common user mistakes +- UI vocabulary (button labels, menu paths, shortcuts) +- Pointing heuristics (where to point first, what to avoid) +- Completion signals ("user said got it", step confirmed, correct UI state visible) + +### Learning loop + +1. User completes a tutoring session (push-to-talk + screen + pointing). +2. Clicky evaluates whether the session is worth persisting (multi-step help, user confirmed success, or repeated topic). +3. Clicky writes or updates a `SKILL.md` with what worked. +4. Next session: relevant skills are loaded into the system prompt when matching apps or topics appear on screen. + +### Curator (maintenance) + +Prevent skill bloat and stale UI knowledge: + +- Track usage: viewed, used, patched +- Auto-transition: `active → stale → archived` after inactivity (e.g. 30 / 90 days) +- Optional LLM review pass to merge duplicates or patch drift when app UI changes +- User can pin important skills so they are never archived + +### Triggers (when to create/update a skill) + +- Multi-step guidance completed successfully +- User explicitly confirms ("got it", "thanks that worked") +- Same topic asked 2+ times in one week +- NOT: generic Q&A unrelated to screen UI + +### Implementation notes + +| Layer | Approach | +|-------|----------| +| Storage | `~/.clicky/skills/` (local only, privacy-first) | +| Format | `SKILL.md` with YAML frontmatter (`name`, `description`, optional `paths` for app bundles) | +| Injection | Extend `companionVoiceResponseSystemPrompt` with matched skills at request time | +| Worker | Optional `/skill-synthesize` route for post-session skill drafting (keeps API keys off-device) | +| Swift | New `TeachingSkillStore`, `SkillCurator`, hook in `CompanionManager` after successful responses | +| Phase 2 | GEPA-style optimization from traces (Hermes self-evolution pattern) — research/demo only | + +### Success criteria + +- Repeat questions on the same app get faster, more accurate pointing +- Skills stay small and relevant (Curator prevents catalog pollution) +- Demo: "Clicky remembered how to teach me Xcode commits from last week" + +--- + +## 2. Niche Discovery (Onboarding + Suggestions) + +**Problem:** Many users do not know what Clicky is for or what to ask. Blank-slate AI assistants fail the "first 60 seconds" test. + +**Concept:** Help users discover Clicky through niche-specific, concrete examples — not generic "I'm an AI helper" messaging. + +### Core UX + +1. **Onboarding:** Ask what the user does (content creator, developer, student, designer, etc.) or infer from screen. +2. **Suggestion cards:** Show 3–5 actionable example prompts in the menu bar panel. +3. **Spoken intro (optional):** Clicky briefly explains what it can do *for their niche* via TTS. +4. **Live suggestions:** When screen context matches (e.g. Final Cut, OBS, Premiere), surface timely prompts. + +### Example — Content creators + +- "Show me where to add captions in this timeline" +- "Walk me through export settings for YouTube" +- "How do I color grade this clip?" +- "Where is the transition menu?" + +Each suggestion should imply: **voice + screen + pointing**, not plain chat. + +### Niche packs (optional, pairs with teaching skills) + +``` +~/.clicky/niches/ + content-creator/ + examples.json # suggested prompts + intro-script.txt # optional spoken onboarding + developer/ + examples.json + student/ + examples.json +``` + +Niche selection can seed which teaching skills to prioritize and which apps to watch for. + +### Detection strategies + +| Method | Effort | Notes | +|--------|--------|-------| +| User picks niche in onboarding | Low | Store in UserDefaults (`selectedUserNiche`) | +| Frontmost app bundle ID | Medium | Map `com.apple.FinalCut` → content creator suggestions | +| Screenshot + Claude one-shot classify | Medium | Fallback when app mapping unknown | +| Proactive idle suggestions | Higher | Tutor mode synergy — suggest when user pauses in a known app | + +### Implementation notes + +| Layer | Approach | +|-------|----------| +| Swift | `NicheDiscoveryManager`, extend `CompanionPanelView` with suggestion UI | +| Persistence | `UserDefaults` for niche; optional `~/.clicky/profile.md` for richer profile | +| Prompts | Niche-specific example strings + optional niche clause in system prompt | +| Screen | Reuse `CompanionScreenCaptureUtility` + frontmost app from `NSWorkspace` | +| Analytics | Track which suggestions get tapped / spoken (PostHog via existing `ClickyAnalytics`) | + +### Success criteria + +- New users ask a useful question within first session (measurable via analytics) +- Reduced "what do I use this for?" support confusion +- Niche feels personal: content creators see creator examples, not developer ones + +--- + +## Suggested build order + +1. **Niche discovery (light)** — onboarding picker + static suggestion cards per niche +2. **App-aware suggestions** — detect frontmost app, swap examples dynamically +3. **Teaching skills (write)** — post-session skill creation for successful multi-step help +4. **Teaching skills (read + Curator)** — load skills into prompts, maintain library over time +5. **Combine** — niche picks starting suggestions; skills deepen tutoring in that niche over time + +--- + +## Out of scope (for this fork) + +- Full Hermes Agent port (Telegram, VPS, 20+ tools) +- Cloud skill sync / marketplace (local-first unless explicitly added later) +- Replacing push-to-talk with Realtime API (separate future idea) + +--- + +## References + +- Hermes Agent skills + Curator: https://github.com/NousResearch/hermes-agent +- Agent Skills standard: https://agentskills.io +- Related fork ideas: Skilly (teaching skills), moonmidas/clicky (Tutor mode) diff --git a/IMPLEMENTATION_AND_E2E_PLAN.md b/IMPLEMENTATION_AND_E2E_PLAN.md new file mode 100644 index 000000000..25e5dca6f --- /dev/null +++ b/IMPLEMENTATION_AND_E2E_PLAN.md @@ -0,0 +1,165 @@ +# Evolving Teaching Skills — Implementation & E2E Plan + +Captured from product/eng discussion. **Order of work: implement the feature first, then run full automated E2E tests.** + +Related: [`FUTURE_IMPLEMENTATION.md`](./FUTURE_IMPLEMENTATION.md) (feature spec) + +--- + +## Goal + +Clicky learns from successful tutoring sessions and reuses that knowledge in future sessions via local `SKILL.md` files under `~/.clicky/skills/`. + +**Demo success:** *"I taught Clicky how to walk me through Xcode commits yesterday — today it remembers and points better on the first try."* + +--- + +## Phase 1 — Foundation (implement first) + +1. **`TeachingSkillStore.swift`** + - Create/read/update skills at `~/.clicky/skills//SKILL.md` + - Parse YAML frontmatter: `name`, `description`, `bundleIds`, `status`, `lastUsed`, `usageCount` + - Index skills on app launch + +2. **Session trace capture** + - Extend `CompanionManager` beyond in-RAM `conversationHistory` (~10 turns) + - Persist lightweight session log: transcript, response, frontmost app bundle ID, pointing used or not + - Hook after each successful exchange in `sendTranscriptToClaudeWithScreenshot` + +3. **Skill matching at request time** + - Frontmost app via `NSWorkspace.shared.frontmostApplication` + - Match by bundle ID + keyword overlap with user transcript + - Load top 1–3 skills into system prompt (small token budget) + +4. **Prompt injection** + - Replace static `companionVoiceResponseSystemPrompt` with a builder: + - base Clicky prompt + - + matched teaching skills + - Pass composed prompt to `claudeAPI.analyzeImageStreaming` + +--- + +## Phase 2 — Write loop (evolving) + +5. **Trigger evaluator** + - After session / N exchanges, decide create-or-update: + - multi-step help (2+ exchanges with pointing) + - user confirms ("got it", "thanks", "that worked") + - same topic repeated within 7 days + - Skip generic off-screen Q&A + +6. **Skill synthesizer** + - Option A: local Claude call via existing `/chat` worker with synthesis prompt + - Option B: new worker route `POST /skill-synthesize` + - Input: session trace → Output: `SKILL.md` (steps, UI labels, pointing tips, mistakes) + +7. **Create vs update** + - Merge into existing similar skill instead of duplicating + - Bump `lastUsed`, increment `usageCount` + +--- + +## Phase 3 — Curator + +8. **`SkillCurator.swift`** + - Track: viewed, used, patched, archived + - Auto-archive: 30d inactive → stale, 90d → archived + - User pin (never archive) + - Optional LLM pass to merge duplicates / patch stale UI labels + +--- + +## Phase 4 — UX (recommended) + +9. **Minimal UI in `CompanionPanelView`** + - List skills, pin/delete, toggle "learn from sessions" + +10. **Analytics** + - Track: skill matched, created, used (via `ClickyAnalytics` / PostHog) + +--- + +## Files to touch + +| File | Role | +|------|------| +| `CompanionManager.swift` | trace capture, triggers, prompt builder, post-session hook | +| **New** `TeachingSkillStore.swift` | filesystem + parsing | +| **New** `SkillCurator.swift` | lifecycle + dedup | +| **New** `SkillMatcher.swift` | app/topic matching | +| `CompanionPanelView.swift` | skills UI | +| `worker/src/index.ts` | optional `/skill-synthesize` | + +--- + +## E2E automation — full user-perspective stack + +Pure XCUITest alone is not enough for Clicky (mic, screen capture, global shortcuts, pointing overlay, LLM). Use this macOS automation stack instead. + +| Layer | Tool | Purpose | +|-------|------|---------| +| Permissions | [`@guidepup/setup`](https://github.com/guidepup/setup) or [`jacobsalmela/tccutil`](https://github.com/jacobsalmela/tccutil) | Pre-grant Screen Recording, Mic, Accessibility in TCC.db | +| Push-to-talk | [`NaryaAI/voice-testing-tools`](https://github.com/NaryaAI/voice-testing-tools) `simulate-keypress.swift` | Hold/release `ctrl+option` (Clicky's shortcut) via HID events | +| Voice input | Same repo `tts.mjs` + **BlackHole 2ch** | Deterministic TTS → virtual mic → real STT path | +| Screen + UI | [**Peekaboo**](https://github.com/steipete/Peekaboo) `.peekaboo.json` or [**axcli**](https://github.com/andelf/axcli) | Launch app, screenshots, verify overlay/cursor, AX tree | +| Deterministic AI | Mock worker with recorded Claude fixtures | Repeatable pass/fail (real API is flaky) | + +### Prerequisites + +1. **Stable code signing** — ad-hoc builds reset TCC on every rebuild; use dev/Developer ID cert so permissions persist. +2. **Self-hosted Mac test machine** — one-time TCC pre-seed via `@guidepup/setup`; fully unattended cloud CI without setup is unreliable. +3. **BlackHole as mic input** during tests to exercise real transcription. +4. **Mock worker** for E2E assertions (record/replay fixtures). + +### Example E2E flow + +```bash +# One-time setup +npx @guidepup/setup --ci +brew install blackhole-2ch sox switchaudio-osx + +# Launch Clicky + fixture app +open /path/to/Clicky.app +open -a TextEdit + +# Session 1: ask for help +node tools/tts.mjs "how do I save this document?" & +swift tools/simulate-keypress.swift down:ctrl down:option wait:3000 up:option up:ctrl + +# Confirm success (triggers skill write) +node tools/tts.mjs "got it thanks that worked" & +swift tools/simulate-keypress.swift down:ctrl down:option wait:2000 up:option up:ctrl + +# Assert skill file exists +test -f ~/.clicky/skills/teach-textedit-save/SKILL.md + +# Session 2: repeat question, verify skill loaded + pointing +peekaboo run tests/e2e/teaching-skills.peekaboo.json --json +``` + +### Clicky test hooks to add (small, E2E-friendly) + +- Launch flag `-CLICKY_E2E=1`: skip onboarding, point at test worker URL, faster timeouts +- Optional `--inject-transcript` fallback if STT flakes (still runs response + skill loop) +- Expose for assertions: `lastSystemPrompt`, `lastSkillMatched`, overlay accessibility identifiers + +--- + +## Execution order + +1. Implement Phases 1–2 (read + write loop) — minimum viable evolving skills +2. Add test hooks + mock worker fixtures +3. Scaffold `tests/e2e/` (voice-testing-tools + Peekaboo scripts) +4. One-time Mac test machine setup (TCC, BlackHole, signing) +5. Run full E2E regression +6. Phase 3–4 (Curator + UI) as follow-up + +--- + +## References + +- Hermes Agent skills + Curator: https://github.com/NousResearch/hermes-agent +- Agent Skills standard: https://agentskills.io +- Voice E2E tools: https://github.com/NaryaAI/voice-testing-tools +- macOS GUI automation: https://github.com/steipete/Peekaboo +- TCC automation: https://github.com/guidepup/setup diff --git a/leanring-buddy/ClaudeAPI.swift b/leanring-buddy/ClaudeAPI.swift index 0c7070b56..4f5f528ee 100644 --- a/leanring-buddy/ClaudeAPI.swift +++ b/leanring-buddy/ClaudeAPI.swift @@ -288,4 +288,51 @@ class ClaudeAPI { let duration = Date().timeIntervalSince(startTime) return (text: text, duration: duration) } + + /// Sends a text-only request to Claude. Used for skill synthesis and other + /// non-vision tasks that still route through the worker proxy. + func sendTextMessage( + systemPrompt: String, + userPrompt: String, + maxTokens: Int = 1024 + ) async throws -> (text: String, duration: TimeInterval) { + let startTime = Date() + + var request = makeAPIRequest() + let body: [String: Any] = [ + "model": model, + "max_tokens": maxTokens, + "system": systemPrompt, + "messages": [ + ["role": "user", "content": userPrompt] + ] + ] + + request.httpBody = try JSONSerialization.data(withJSONObject: body) + let (data, response) = try await session.data(for: request) + + guard let httpResponse = response as? HTTPURLResponse, + (200...299).contains(httpResponse.statusCode) else { + let responseString = String(data: data, encoding: .utf8) ?? "Unknown error" + throw NSError( + domain: "ClaudeAPI", + code: (response as? HTTPURLResponse)?.statusCode ?? -1, + userInfo: [NSLocalizedDescriptionKey: "API Error: \(responseString)"] + ) + } + + let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] + guard let content = json?["content"] as? [[String: Any]], + let textBlock = content.first(where: { ($0["type"] as? String) == "text" }), + let text = textBlock["text"] as? String else { + throw NSError( + domain: "ClaudeAPI", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "Invalid response format"] + ) + } + + let duration = Date().timeIntervalSince(startTime) + return (text: text, duration: duration) + } } diff --git a/leanring-buddy/ClickyAnalytics.swift b/leanring-buddy/ClickyAnalytics.swift index 29e26138e..8f6bf4151 100644 --- a/leanring-buddy/ClickyAnalytics.swift +++ b/leanring-buddy/ClickyAnalytics.swift @@ -118,4 +118,35 @@ enum ClickyAnalytics { "error": error ]) } + + // MARK: - Teaching Skills + + static func trackTeachingSkillsMatched(skillIDs: [String], bundleID: String?) { + PostHogSDK.shared.capture("teaching_skills_matched", properties: [ + "skill_ids": skillIDs, + "bundle_id": bundleID ?? "unknown", + "count": skillIDs.count + ]) + } + + static func trackTeachingSkillWriteTriggered(reason: String, topic: String) { + PostHogSDK.shared.capture("teaching_skill_write_triggered", properties: [ + "reason": reason, + "topic": topic + ]) + } + + static func trackTeachingSkillSaved(skillID: String, reason: String, updatedExisting: Bool) { + PostHogSDK.shared.capture("teaching_skill_saved", properties: [ + "skill_id": skillID, + "reason": reason, + "updated_existing": updatedExisting + ]) + } + + static func trackTeachingSkillDeleted(skillID: String) { + PostHogSDK.shared.capture("teaching_skill_deleted", properties: [ + "skill_id": skillID + ]) + } } diff --git a/leanring-buddy/ClickyE2EConfiguration.swift b/leanring-buddy/ClickyE2EConfiguration.swift new file mode 100644 index 000000000..694345111 --- /dev/null +++ b/leanring-buddy/ClickyE2EConfiguration.swift @@ -0,0 +1,43 @@ +// +// ClickyE2EConfiguration.swift +// leanring-buddy +// +// Launch flags used by automated end-to-end tests. +// + +import Foundation + +enum ClickyE2EConfiguration { + static var isEnabled: Bool { + ProcessInfo.processInfo.arguments.contains("-CLICKY_E2E=1") + } + + static var workerBaseURL: String? { + argumentValue(for: "-CLICKY_WORKER_URL=") + } + + static var injectTranscript: String? { + argumentValue(for: "-CLICKY_INJECT_TRANSCRIPT=") + } + + static var injectTranscript2: String? { + argumentValue(for: "-CLICKY_INJECT_TRANSCRIPT_2=") + } + + static func applyLaunchOverrides() { + guard isEnabled else { return } + + UserDefaults.standard.set(true, forKey: "hasCompletedOnboarding") + UserDefaults.standard.set(true, forKey: "hasSubmittedEmail") + UserDefaults.standard.set(true, forKey: "isClickyCursorEnabled") + } + + private static func argumentValue(for prefix: String) -> String? { + ProcessInfo.processInfo.arguments + .first(where: { $0.hasPrefix(prefix) }) + .map { String($0.dropFirst(prefix.count)) } + .flatMap { value in + value.isEmpty ? nil : value + } + } +} diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index 0234cf19f..db2781d1a 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -8,6 +8,7 @@ // import AVFoundation +import AppKit import Combine import Foundation import PostHog @@ -70,7 +71,26 @@ final class CompanionManager: ObservableObject { /// Base URL for the Cloudflare Worker proxy. All API requests route /// through this so keys never ship in the app binary. - private static let workerBaseURL = "https://your-worker-name.your-subdomain.workers.dev" + private static var workerBaseURL: String { + ClickyE2EConfiguration.workerBaseURL ?? "https://your-worker-name.your-subdomain.workers.dev" + } + + private let teachingSkillStore = TeachingSkillStore() + private var sessionTrace: [SessionTraceEntry] = [] + private var skillWriteTask: Task? + + /// Skills currently on disk, exposed for the panel UI. + @Published private(set) var teachingSkills: [TeachingSkill] = [] + + /// When disabled, Clicky still reads skills but will not create new ones. + @Published var isLearningFromSessionsEnabled: Bool = UserDefaults.standard.object(forKey: "isLearningFromSessionsEnabled") == nil + ? true + : UserDefaults.standard.bool(forKey: "isLearningFromSessionsEnabled") + + /// Exposed for automated E2E assertions. + @Published private(set) var lastSystemPrompt: String? + @Published private(set) var lastMatchedSkillNames: [String] = [] + @Published private(set) var lastSkillWriteTrigger: String? private lazy var claudeAPI: ClaudeAPI = { return ClaudeAPI(proxyURL: "\(Self.workerBaseURL)/chat", model: selectedModel) @@ -110,6 +130,159 @@ final class CompanionManager: ObservableObject { /// The Claude model used for voice responses. Persisted to UserDefaults. @Published var selectedModel: String = UserDefaults.standard.string(forKey: "selectedClaudeModel") ?? "claude-sonnet-4-6" + func setLearningFromSessionsEnabled(_ enabled: Bool) { + isLearningFromSessionsEnabled = enabled + UserDefaults.standard.set(enabled, forKey: "isLearningFromSessionsEnabled") + } + + func refreshTeachingSkills() { + teachingSkillStore.loadSkills() + teachingSkills = teachingSkillStore.skills + } + + func deleteTeachingSkill(id: String) { + do { + try teachingSkillStore.deleteSkill(id: id) + teachingSkills = teachingSkillStore.skills + ClickyAnalytics.trackTeachingSkillDeleted(skillID: id) + } catch { + print("⚠️ Failed to delete teaching skill \(id): \(error)") + } + } + + func setTeachingSkillPinned(id: String, pinned: Bool) { + do { + try teachingSkillStore.setPinned(id: id, pinned: pinned) + teachingSkills = teachingSkillStore.skills + } catch { + print("⚠️ Failed to pin teaching skill \(id): \(error)") + } + } + + /// Allows E2E tests to bypass microphone/STT and exercise the response + skill loop directly. + func injectTranscriptForE2E(_ transcript: String) async { + await withCheckedContinuation { (continuation: CheckedContinuation) in + sendTranscriptToClaudeWithScreenshot(transcript: transcript) { + continuation.resume() + } + } + } + + func runE2EInjectSequenceIfNeeded() { + guard ClickyE2EConfiguration.isEnabled, + let firstTranscript = ClickyE2EConfiguration.injectTranscript else { + return + } + + Task { + try? await Task.sleep(nanoseconds: 2_000_000_000) + await injectTranscriptForE2E(firstTranscript) + + if let secondTranscript = ClickyE2EConfiguration.injectTranscript2 { + try? await Task.sleep(nanoseconds: 1_000_000_000) + await injectTranscriptForE2E(secondTranscript) + } + } + } + + private func frontmostApplicationBundleId() -> String? { + NSWorkspace.shared.frontmostApplication?.bundleIdentifier + } + + private func bootstrapTeachingSkills() { + teachingSkillStore.loadSkills() + SkillCurator.curate(store: teachingSkillStore) + teachingSkills = teachingSkillStore.skills + } + + private func matchedSkills(for transcript: String) -> [TeachingSkill] { + let matches = SkillMatcher.matchSkills( + from: teachingSkillStore.skills, + bundleId: frontmostApplicationBundleId(), + transcript: transcript + ) + return matches.map(\.skill) + } + + private func recordSessionExchange( + transcript: String, + spokenResponse: String, + pointed: Bool + ) { + sessionTrace.append( + SessionTraceEntry( + timestamp: Date(), + userTranscript: transcript, + assistantResponse: spokenResponse, + bundleId: frontmostApplicationBundleId(), + pointed: pointed + ) + ) + + if sessionTrace.count > 20 { + sessionTrace.removeFirst(sessionTrace.count - 20) + } + } + + private func maybeWriteTeachingSkill(after transcript: String) { + guard isLearningFromSessionsEnabled else { return } + guard SkillTriggerEvaluator.isScreenTeachingSession(sessionTrace) else { return } + guard let trigger = SkillTriggerEvaluator.shouldWriteSkill( + sessionTrace: sessionTrace, + latestTranscript: transcript + ) else { + return + } + + lastSkillWriteTrigger = trigger.reason.rawValue + ClickyAnalytics.trackTeachingSkillWriteTriggered(reason: trigger.reason.rawValue, topic: trigger.topic) + + let traceSnapshot = sessionTrace + let bundleId = frontmostApplicationBundleId() + skillWriteTask?.cancel() + skillWriteTask = Task { + do { + let existingSkill = SkillMatcher.findSimilarSkill( + in: teachingSkillStore.skills, + bundleId: bundleId, + topic: trigger.topic + ) + + let synthesized = try await SkillSynthesizer.synthesizeSkillContent( + sessionTrace: traceSnapshot, + trigger: trigger, + existingSkill: existingSkill, + claudeAPI: claudeAPI + ) + + guard !Task.isCancelled else { return } + + let skill = SkillSynthesizer.buildSkill( + id: existingSkill?.id, + name: synthesized.name, + description: synthesized.description, + body: synthesized.body, + bundleId: bundleId, + existingSkill: existingSkill + ) + + _ = try teachingSkillStore.saveSkill(skill) + SkillCurator.curate(store: teachingSkillStore) + teachingSkills = teachingSkillStore.skills + sessionTrace.removeAll() + + ClickyAnalytics.trackTeachingSkillSaved( + skillID: skill.id, + reason: trigger.reason.rawValue, + updatedExisting: existingSkill != nil + ) + print("📚 Saved teaching skill: \(skill.id)") + } catch { + print("⚠️ Failed to synthesize teaching skill: \(error)") + } + } + } + func setSelectedModel(_ model: String) { selectedModel = model UserDefaults.standard.set(model, forKey: "selectedClaudeModel") @@ -173,6 +346,7 @@ final class CompanionManager: ObservableObject { } func start() { + bootstrapTeachingSkills() refreshAllPermissions() print("🔑 Clicky start — accessibility: \(hasAccessibilityPermission), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission), onboarded: \(hasCompletedOnboarding)") startPermissionPolling() @@ -295,6 +469,8 @@ final class CompanionManager: ObservableObject { currentResponseTask?.cancel() currentResponseTask = nil + skillWriteTask?.cancel() + skillWriteTask = nil shortcutTransitionCancellable?.cancel() voiceStateCancellable?.cancel() audioPowerCancellable?.cancel() @@ -583,11 +759,15 @@ final class CompanionManager: ObservableObject { /// the spinner/processing state until TTS audio begins playing. /// Claude's response may include a [POINT:x,y:label] tag which triggers /// the buddy to fly to that element on screen. - private func sendTranscriptToClaudeWithScreenshot(transcript: String) { + private func sendTranscriptToClaudeWithScreenshot( + transcript: String, + onComplete: (@MainActor () -> Void)? = nil + ) { currentResponseTask?.cancel() elevenLabsTTSClient.stopPlayback() currentResponseTask = Task { + defer { onComplete?() } // Stay in processing (spinner) state — no streaming text displayed voiceState = .processing @@ -610,9 +790,29 @@ final class CompanionManager: ObservableObject { (userPlaceholder: entry.userTranscript, assistantResponse: entry.assistantResponse) } + let matchedTeachingSkills = matchedSkills(for: transcript) + lastMatchedSkillNames = matchedTeachingSkills.map(\.name) + let systemPrompt = TeachingPromptBuilder.buildVoiceResponsePrompt( + basePrompt: Self.companionVoiceResponseSystemPrompt, + matchedSkills: matchedTeachingSkills + ) + lastSystemPrompt = systemPrompt + + for skill in matchedTeachingSkills { + _ = try? teachingSkillStore.markUsed(skill) + } + teachingSkills = teachingSkillStore.skills + + if !matchedTeachingSkills.isEmpty { + ClickyAnalytics.trackTeachingSkillsMatched( + skillIDs: matchedTeachingSkills.map(\.id), + bundleID: frontmostApplicationBundleId() + ) + } + let (fullResponseText, _) = try await claudeAPI.analyzeImageStreaming( images: labeledImages, - systemPrompt: Self.companionVoiceResponseSystemPrompt, + systemPrompt: systemPrompt, conversationHistory: historyForAPI, userPrompt: transcript, onTextChunk: { _ in @@ -695,6 +895,13 @@ final class CompanionManager: ObservableObject { print("🧠 Conversation history: \(conversationHistory.count) exchanges") + recordSessionExchange( + transcript: transcript, + spokenResponse: spokenText, + pointed: hasPointCoordinate + ) + maybeWriteTeachingSkill(after: transcript) + ClickyAnalytics.trackAIResponseReceived(response: spokenText) // Play the response via TTS. Keep the spinner (processing state) diff --git a/leanring-buddy/CompanionPanelView.swift b/leanring-buddy/CompanionPanelView.swift index 76789b4c6..a9824f9da 100644 --- a/leanring-buddy/CompanionPanelView.swift +++ b/leanring-buddy/CompanionPanelView.swift @@ -25,6 +25,14 @@ struct CompanionPanelView: View { .padding(.top, 16) .padding(.horizontal, 16) + if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { + Spacer() + .frame(height: 12) + + teachingSkillsSection + .padding(.horizontal, 16) + } + if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { Spacer() .frame(height: 12) @@ -596,6 +604,85 @@ struct CompanionPanelView: View { .padding(.vertical, 4) } + // MARK: - Teaching Skills + + private var teachingSkillsSection: some View { + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Teaching Skills") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + + Spacer() + + Toggle("", isOn: Binding( + get: { companionManager.isLearningFromSessionsEnabled }, + set: { companionManager.setLearningFromSessionsEnabled($0) } + )) + .toggleStyle(.switch) + .labelsHidden() + .tint(DS.Colors.accent) + .scaleEffect(0.8) + } + + Text(companionManager.isLearningFromSessionsEnabled + ? "Clicky learns from successful tutoring sessions." + : "Learning paused — saved skills still apply.") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + + if companionManager.teachingSkills.isEmpty { + Text("No skills yet. Teach Clicky something on screen and confirm it worked.") + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textTertiary) + .fixedSize(horizontal: false, vertical: true) + } else { + ForEach(companionManager.teachingSkills.prefix(4)) { skill in + teachingSkillRow(skill) + } + } + } + .padding(.vertical, 4) + } + + private func teachingSkillRow(_ skill: TeachingSkill) -> some View { + HStack(spacing: 8) { + VStack(alignment: .leading, spacing: 2) { + Text(skill.name) + .font(.system(size: 12, weight: .semibold)) + .foregroundColor(DS.Colors.textSecondary) + .lineLimit(1) + + Text("\(skill.usageCount) uses • \(skill.status.rawValue)") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + } + + Spacer() + + Button(action: { + companionManager.setTeachingSkillPinned(id: skill.id, pinned: !skill.isPinned) + }) { + Image(systemName: skill.isPinned ? "pin.fill" : "pin") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(skill.isPinned ? DS.Colors.accent : DS.Colors.textTertiary) + } + .buttonStyle(.plain) + .pointerCursor() + + Button(action: { + companionManager.deleteTeachingSkill(id: skill.id) + }) { + Image(systemName: "trash") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + } + .buttonStyle(.plain) + .pointerCursor() + } + .padding(.vertical, 4) + } + // MARK: - Model Picker private var modelPickerRow: some View { diff --git a/leanring-buddy/SkillCurator.swift b/leanring-buddy/SkillCurator.swift new file mode 100644 index 000000000..e2a1649cf --- /dev/null +++ b/leanring-buddy/SkillCurator.swift @@ -0,0 +1,38 @@ +// +// SkillCurator.swift +// leanring-buddy +// +// Maintains the teaching skill library over time. +// + +import Foundation + +enum SkillCurator { + private static let staleAfterDays = 30 + private static let archiveAfterDays = 90 + + static func curate(store: TeachingSkillStore, now: Date = Date()) { + let calendar = Calendar.current + + for skill in store.skills { + guard !skill.isPinned else { continue } + guard let lastUsed = skill.lastUsed else { continue } + + let daysSinceUse = calendar.dateComponents([.day], from: lastUsed, to: now).day ?? 0 + var updated = skill + var skillChanged = false + + if daysSinceUse >= archiveAfterDays, updated.status != .archived { + updated.status = .archived + skillChanged = true + } else if daysSinceUse >= staleAfterDays, updated.status == .active { + updated.status = .stale + skillChanged = true + } + + if skillChanged { + try? store.saveSkill(updated) + } + } + } +} diff --git a/leanring-buddy/SkillMatcher.swift b/leanring-buddy/SkillMatcher.swift new file mode 100644 index 000000000..10d33c4a2 --- /dev/null +++ b/leanring-buddy/SkillMatcher.swift @@ -0,0 +1,95 @@ +// +// SkillMatcher.swift +// leanring-buddy +// +// Matches teaching skills to the current app and user transcript. +// + +import Foundation + +struct SkillMatch: Equatable { + let skill: TeachingSkill + let score: Int +} + +enum SkillMatcher { + static func matchSkills( + from skills: [TeachingSkill], + bundleId: String?, + transcript: String, + limit: Int = 3 + ) -> [SkillMatch] { + let queryTokens = tokenize(transcript) + let eligibleSkills = skills.filter { skill in + skill.status != .archived || skill.isPinned + } + + let scored = eligibleSkills.compactMap { skill -> SkillMatch? in + var score = 0 + + if let bundleId, skill.bundleIds.contains(bundleId) { + score += 12 + } + + let haystackTokens = Set( + tokenize(skill.name) + + tokenize(skill.description) + + tokenize(skill.body) + ) + score += queryTokens.filter { haystackTokens.contains($0) }.count + + if skill.status == .active { + score += 1 + } + score += min(skill.usageCount, 5) + + return score > 0 ? SkillMatch(skill: skill, score: score) : nil + } + + return Array( + scored + .sorted { lhs, rhs in + if lhs.score != rhs.score { return lhs.score > rhs.score } + return (lhs.skill.lastUsed ?? .distantPast) > (rhs.skill.lastUsed ?? .distantPast) + } + .prefix(limit) + ) + } + + static func findSimilarSkill( + in skills: [TeachingSkill], + bundleId: String?, + topic: String + ) -> TeachingSkill? { + let topicTokens = Set(tokenize(topic)) + guard !topicTokens.isEmpty else { return nil } + + let candidates = skills.filter { skill in + guard let bundleId else { return true } + return skill.bundleIds.isEmpty || skill.bundleIds.contains(bundleId) + } + + return candidates.max { lhs, rhs in + overlapScore(lhs, topicTokens: topicTokens) < overlapScore(rhs, topicTokens: topicTokens) + } + .flatMap { candidate in + overlapScore(candidate, topicTokens: topicTokens) >= 2 ? candidate : nil + } + } + + private static func overlapScore(_ skill: TeachingSkill, topicTokens: Set) -> Int { + let skillTokens = Set( + tokenize(skill.name) + + tokenize(skill.description) + + tokenize(skill.body) + ) + return topicTokens.filter { skillTokens.contains($0) }.count + } + + static func tokenize(_ text: String) -> [String] { + text + .lowercased() + .components(separatedBy: CharacterSet.alphanumerics.inverted) + .filter { $0.count >= 3 } + } +} diff --git a/leanring-buddy/SkillSynthesizer.swift b/leanring-buddy/SkillSynthesizer.swift new file mode 100644 index 000000000..bd4d9947b --- /dev/null +++ b/leanring-buddy/SkillSynthesizer.swift @@ -0,0 +1,104 @@ +// +// SkillSynthesizer.swift +// leanring-buddy +// +// Drafts or updates teaching skills from a completed tutoring session trace. +// + +import Foundation + +enum SkillSynthesizer { + private static let synthesisSystemPrompt = """ + you write teaching skills for clicky, a screen-native voice tutor that points at ui elements. + + given a tutoring session transcript, produce a markdown teaching skill body only. do not include yaml frontmatter. + + include: + - the app/workflow being taught + - ordered steps + - exact ui labels, menu paths, and shortcuts when known + - pointing heuristics (what to point at first, what to avoid) + - common mistakes the user made or might make + - completion signals ("user said got it", visible ui state) + + keep it concise, practical, and specific to macos ui teaching. all lowercase. + """ + + static func synthesizeSkillContent( + sessionTrace: [SessionTraceEntry], + trigger: SkillWriteTrigger, + existingSkill: TeachingSkill?, + claudeAPI: ClaudeAPI + ) async throws -> (name: String, description: String, body: String) { + let sessionSummary = renderSessionSummary(sessionTrace, trigger: trigger, existingSkill: existingSkill) + let userPrompt = existingSkill == nil + ? "create a new teaching skill from this session:\n\n\(sessionSummary)" + : "update this teaching skill using the new session. preserve useful existing guidance and improve it:\n\nexisting skill:\n\(existingSkill?.body ?? "")\n\nnew session:\n\(sessionSummary)" + + let response = try await claudeAPI.sendTextMessage( + systemPrompt: synthesisSystemPrompt, + userPrompt: userPrompt, + maxTokens: 1200 + ) + + let cleanedBody = response.text.trimmingCharacters(in: .whitespacesAndNewlines) + let fallbackName = "teach-\(trigger.topic.replacingOccurrences(of: " ", with: "-"))" + let name = existingSkill?.name ?? fallbackName + let description = existingSkill?.description ?? "Teach the user how to \(trigger.topic)" + return (name: name, description: description, body: cleanedBody) + } + + static func buildSkill( + id: String?, + name: String, + description: String, + body: String, + bundleId: String?, + existingSkill: TeachingSkill? + ) -> TeachingSkill { + let resolvedID = id ?? existingSkill?.id ?? TeachingSkill.slug(from: name) + let bundleIds: [String] + if let existingSkill, !existingSkill.bundleIds.isEmpty { + bundleIds = existingSkill.bundleIds + } else if let bundleId { + bundleIds = [bundleId] + } else { + bundleIds = [] + } + + return TeachingSkill( + id: resolvedID, + name: name, + description: description, + bundleIds: bundleIds, + status: .active, + lastUsed: Date(), + usageCount: existingSkill?.usageCount ?? 0, + isPinned: existingSkill?.isPinned ?? false, + body: body + ) + } + + private static func renderSessionSummary( + _ sessionTrace: [SessionTraceEntry], + trigger: SkillWriteTrigger, + existingSkill: TeachingSkill? + ) -> String { + var lines: [String] = [] + lines.append("trigger: \(trigger.reason.rawValue)") + lines.append("topic: \(trigger.topic)") + if let existingSkill { + lines.append("existing skill id: \(existingSkill.id)") + } + + for (index, entry) in sessionTrace.enumerated() { + lines.append("exchange \(index + 1):") + lines.append("user: \(entry.userTranscript)") + lines.append("assistant: \(entry.assistantResponse)") + lines.append("bundle id: \(entry.bundleId ?? "unknown")") + lines.append("pointed: \(entry.pointed ? "yes" : "no")") + } + + return lines.joined(separator: "\n") + } +} diff --git a/leanring-buddy/SkillTriggerEvaluator.swift b/leanring-buddy/SkillTriggerEvaluator.swift new file mode 100644 index 000000000..1b434b2db --- /dev/null +++ b/leanring-buddy/SkillTriggerEvaluator.swift @@ -0,0 +1,84 @@ +// +// SkillTriggerEvaluator.swift +// leanring-buddy +// +// Decides when a tutoring session should create or update a teaching skill. +// + +import Foundation + +struct SkillWriteTrigger: Equatable { + enum Reason: String, Equatable { + case userConfirmed + case multiStepPointing + case repeatedTopic + } + + let reason: Reason + let topic: String +} + +enum SkillTriggerEvaluator { + private static let confirmationPhrases = [ + "got it", + "that worked", + "thanks that worked", + "thank you that worked", + "perfect", + "helpful", + "makes sense now", + "that helps" + ] + + static func shouldWriteSkill( + sessionTrace: [SessionTraceEntry], + latestTranscript: String + ) -> SkillWriteTrigger? { + guard !sessionTrace.isEmpty else { return nil } + + let normalizedTranscript = latestTranscript.lowercased() + let pointedExchanges = sessionTrace.filter(\.pointed) + let topic = deriveTopic(from: sessionTrace) + + if confirmationPhrases.contains(where: { normalizedTranscript.contains($0) }) { + return SkillWriteTrigger(reason: .userConfirmed, topic: topic) + } + + if pointedExchanges.count >= 2 { + return SkillWriteTrigger(reason: .multiStepPointing, topic: topic) + } + + if hasRepeatedTopic(in: sessionTrace) { + return SkillWriteTrigger(reason: .repeatedTopic, topic: topic) + } + + return nil + } + + static func deriveTopic(from sessionTrace: [SessionTraceEntry]) -> String { + let combined = sessionTrace + .map(\.userTranscript) + .joined(separator: " ") + let tokens = SkillMatcher.tokenize(combined) + return tokens.prefix(6).joined(separator: " ") + } + + private static func hasRepeatedTopic(in sessionTrace: [SessionTraceEntry]) -> Bool { + guard sessionTrace.count >= 2 else { return false } + let tokenSets = sessionTrace.map { Set(SkillMatcher.tokenize($0.userTranscript)) } + for index in tokenSets.indices { + for otherIndex in tokenSets.indices where otherIndex > index { + let overlap = tokenSets[index].intersection(tokenSets[otherIndex]) + if overlap.count >= 2 { + return true + } + } + } + return false + } + + static func isScreenTeachingSession(_ sessionTrace: [SessionTraceEntry]) -> Bool { + sessionTrace.contains(where: \.pointed) || + sessionTrace.contains(where: { $0.bundleId != nil }) + } +} diff --git a/leanring-buddy/TeachingPromptBuilder.swift b/leanring-buddy/TeachingPromptBuilder.swift new file mode 100644 index 000000000..4cfcf8dfb --- /dev/null +++ b/leanring-buddy/TeachingPromptBuilder.swift @@ -0,0 +1,41 @@ +// +// TeachingPromptBuilder.swift +// leanring-buddy +// +// Composes the voice response system prompt with matched teaching skills. +// + +import Foundation + +enum TeachingPromptBuilder { + static func buildVoiceResponsePrompt( + basePrompt: String, + matchedSkills: [TeachingSkill], + maxSkillCharacters: Int = 3500 + ) -> String { + guard !matchedSkills.isEmpty else { return basePrompt } + + var usedCharacters = 0 + var renderedSkills: [String] = [] + + for skill in matchedSkills { + let rendered = skill.renderedMarkdown() + if usedCharacters + rendered.count > maxSkillCharacters, !renderedSkills.isEmpty { + break + } + renderedSkills.append(rendered) + usedCharacters += rendered.count + } + + guard !renderedSkills.isEmpty else { return basePrompt } + + return """ + \(basePrompt) + + teaching skills: + the following local teaching notes were learned from earlier successful tutoring sessions on this mac. reuse them when they clearly apply. prefer the saved ui labels, menu paths, shortcuts, and pointing order over guessing. + + \(renderedSkills.joined(separator: "\n\n")) + """ + } +} diff --git a/leanring-buddy/TeachingSkill.swift b/leanring-buddy/TeachingSkill.swift new file mode 100644 index 000000000..54caddbdd --- /dev/null +++ b/leanring-buddy/TeachingSkill.swift @@ -0,0 +1,186 @@ +// +// TeachingSkill.swift +// leanring-buddy +// +// Model and parsing for local teaching skills stored as SKILL.md files. +// + +import Foundation + +enum TeachingSkillStatus: String, Codable, CaseIterable { + case active + case stale + case archived +} + +struct TeachingSkill: Identifiable, Equatable { + let id: String + var name: String + var description: String + var bundleIds: [String] + var status: TeachingSkillStatus + var lastUsed: Date? + var usageCount: Int + var isPinned: Bool + var body: String + + var folderURL: URL { + TeachingSkillStore.skillsRootURL.appendingPathComponent(id, isDirectory: true) + } + + var fileURL: URL { + folderURL.appendingPathComponent("SKILL.md") + } + + func renderedMarkdown(maxBodyCharacters: Int = 2500) -> String { + var content = body.trimmingCharacters(in: .whitespacesAndNewlines) + if content.count > maxBodyCharacters { + let endIndex = content.index(content.startIndex, offsetBy: maxBodyCharacters) + content = String(content[.. String { + let lastUsedValue = lastUsed.map { TeachingSkill.dateFormatter.string(from: $0) } ?? "" + let bundleLines = bundleIds.map { " - \($0)" }.joined(separator: "\n") + return """ + --- + name: \(name) + description: \(TeachingSkill.yamlEscape(description)) + bundleIds: + \(bundleLines.isEmpty ? " []" : bundleLines) + status: \(status.rawValue) + lastUsed: \(lastUsedValue) + usageCount: \(usageCount) + pinned: \(isPinned) + --- + + \(body.trimmingCharacters(in: .whitespacesAndNewlines)) + + """ + } + + static func slug(from name: String) -> String { + let lowered = name.lowercased() + let allowed = lowered.map { character -> Character in + if character.isLetter || character.isNumber { return character } + if character == " " || character == "-" || character == "_" { return "-" } + return "-" + } + let collapsed = String(allowed) + .replacingOccurrences(of: "-+", with: "-", options: .regularExpression) + .trimmingCharacters(in: CharacterSet(charactersIn: "-")) + return collapsed.isEmpty ? "teaching-skill" : collapsed + } + + static func parse(id: String, markdown: String) -> TeachingSkill? { + let trimmed = markdown.trimmingCharacters(in: .whitespacesAndNewlines) + guard trimmed.hasPrefix("---") else { return nil } + + let components = trimmed.components(separatedBy: "---") + guard components.count >= 3 else { return nil } + + let frontmatter = components[1] + let body = components.dropFirst(2).joined(separator: "---").trimmingCharacters(in: .whitespacesAndNewlines) + let metadata = parseFrontmatter(frontmatter) + + let name = metadata["name"] ?? id + let description = metadata["description"] ?? "" + let bundleIds = metadata["bundleIds"]? + .split(separator: ",") + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } ?? [] + let status = TeachingSkillStatus(rawValue: metadata["status"] ?? "active") ?? .active + let lastUsed = metadata["lastUsed"].flatMap { dateFormatter.date(from: $0) } + let usageCount = Int(metadata["usageCount"] ?? "0") ?? 0 + let isPinned = (metadata["pinned"] ?? "false").lowercased() == "true" + + return TeachingSkill( + id: id, + name: name, + description: description, + bundleIds: bundleIds, + status: status, + lastUsed: lastUsed, + usageCount: usageCount, + isPinned: isPinned, + body: body + ) + } + + private static func parseFrontmatter(_ frontmatter: String) -> [String: String] { + var result: [String: String] = [:] + var currentListKey: String? + var listValues: [String] = [] + + func flushList() { + guard let key = currentListKey else { return } + result[key] = listValues.joined(separator: ",") + currentListKey = nil + listValues = [] + } + + for line in frontmatter.split(separator: "\n", omittingEmptySubsequences: false) { + let trimmedLine = line.trimmingCharacters(in: .whitespaces) + if trimmedLine.hasPrefix("- ") { + listValues.append(String(trimmedLine.dropFirst(2)).trimmingCharacters(in: .whitespaces)) + continue + } + + flushList() + + guard let separatorIndex = trimmedLine.firstIndex(of: ":") else { continue } + let key = String(trimmedLine[.. String { + if value.contains(":") || value.contains("\"") || value.contains("\n") { + return "\"\(value.replacingOccurrences(of: "\"", with: "\\\""))\"" + } + return value + } + + private static func unyamlEscape(_ value: String) -> String { + if value.hasPrefix("\""), value.hasSuffix("\""), value.count >= 2 { + let inner = value.dropFirst().dropLast() + return inner.replacingOccurrences(of: "\\\"", with: "\"") + } + return value + } + + static let dateFormatter: DateFormatter = { + let formatter = DateFormatter() + formatter.calendar = Calendar(identifier: .gregorian) + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.timeZone = TimeZone(secondsFromGMT: 0) + formatter.dateFormat = "yyyy-MM-dd" + return formatter + }() +} + +struct SessionTraceEntry: Equatable { + let timestamp: Date + let userTranscript: String + let assistantResponse: String + let bundleId: String? + let pointed: Bool +} diff --git a/leanring-buddy/TeachingSkillStore.swift b/leanring-buddy/TeachingSkillStore.swift new file mode 100644 index 000000000..7df911100 --- /dev/null +++ b/leanring-buddy/TeachingSkillStore.swift @@ -0,0 +1,89 @@ +// +// TeachingSkillStore.swift +// leanring-buddy +// +// Local filesystem storage for teaching skills under ~/.clicky/skills/. +// + +import Foundation + +final class TeachingSkillStore { + static let skillsRootURL: URL = { + let home = FileManager.default.homeDirectoryForCurrentUser + return home.appendingPathComponent(".clicky/skills", isDirectory: true) + }() + + private(set) var skills: [TeachingSkill] = [] + + func loadSkills() { + let fileManager = FileManager.default + try? fileManager.createDirectory(at: Self.skillsRootURL, withIntermediateDirectories: true) + + guard let folders = try? fileManager.contentsOfDirectory( + at: Self.skillsRootURL, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + ) else { + skills = [] + return + } + + var loaded: [TeachingSkill] = [] + for folder in folders where (try? folder.resourceValues(forKeys: [.isDirectoryKey]).isDirectory) == true { + let skillFile = folder.appendingPathComponent("SKILL.md") + guard let markdown = try? String(contentsOf: skillFile, encoding: .utf8), + let skill = TeachingSkill.parse(id: folder.lastPathComponent, markdown: markdown) else { + continue + } + loaded.append(skill) + } + + skills = loaded.sorted { lhs, rhs in + if lhs.usageCount != rhs.usageCount { return lhs.usageCount > rhs.usageCount } + return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending + } + } + + @discardableResult + func saveSkill(_ skill: TeachingSkill) throws -> TeachingSkill { + let fileManager = FileManager.default + try fileManager.createDirectory(at: skill.folderURL, withIntermediateDirectories: true) + try skill.serialize().write(to: skill.fileURL, atomically: true, encoding: .utf8) + if let index = skills.firstIndex(where: { $0.id == skill.id }) { + skills[index] = skill + } else { + skills.append(skill) + } + skills.sort { lhs, rhs in + if lhs.usageCount != rhs.usageCount { return lhs.usageCount > rhs.usageCount } + return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending + } + return skill + } + + func deleteSkill(id: String) throws { + let folder = Self.skillsRootURL.appendingPathComponent(id, isDirectory: true) + try FileManager.default.removeItem(at: folder) + skills.removeAll { $0.id == id } + } + + func skill(withID id: String) -> TeachingSkill? { + skills.first { $0.id == id } + } + + func markUsed(_ skill: TeachingSkill) throws -> TeachingSkill { + var updated = skill + updated.lastUsed = Date() + updated.usageCount += 1 + if updated.status == .stale { + updated.status = .active + } + return try saveSkill(updated) + } + + func setPinned(id: String, pinned: Bool) throws { + guard var skill = skill(withID: id) else { return } + skill.isPinned = pinned + _ = try saveSkill(skill) + } +} diff --git a/leanring-buddy/leanring_buddyApp.swift b/leanring-buddy/leanring_buddyApp.swift index b004a8960..362e5bf61 100644 --- a/leanring-buddy/leanring_buddyApp.swift +++ b/leanring-buddy/leanring_buddyApp.swift @@ -41,9 +41,12 @@ final class CompanionAppDelegate: NSObject, NSApplicationDelegate { ClickyAnalytics.configure() ClickyAnalytics.trackAppOpened() + ClickyE2EConfiguration.applyLaunchOverrides() menuBarPanelManager = MenuBarPanelManager(companionManager: companionManager) companionManager.start() + + companionManager.runE2EInjectSequenceIfNeeded() // Auto-open the panel if the user still needs to do something: // either they haven't onboarded yet, or permissions were revoked. if !companionManager.hasCompletedOnboarding || !companionManager.allPermissionsGranted { diff --git a/leanring-buddyTests/TeachingSkillTests.swift b/leanring-buddyTests/TeachingSkillTests.swift new file mode 100644 index 000000000..071db30e2 --- /dev/null +++ b/leanring-buddyTests/TeachingSkillTests.swift @@ -0,0 +1,102 @@ +// +// TeachingSkillTests.swift +// leanring-buddyTests +// + +import Foundation +import Testing +@testable import leanring_buddy + +struct TeachingSkillTests { + @Test func parsesSkillFrontmatterAndBody() throws { + let markdown = """ + --- + name: teach-xcode-source-control + description: Walk through committing in Xcode + bundleIds: + - com.apple.dt.Xcode + status: active + lastUsed: 2026-05-24 + usageCount: 3 + pinned: true + --- + + step one: open source control. + step two: click commit. + """ + + let skill = try #require(TeachingSkill.parse(id: "teach-xcode-source-control", markdown: markdown)) + #expect(skill.name == "teach-xcode-source-control") + #expect(skill.description == "Walk through committing in Xcode") + #expect(skill.bundleIds == ["com.apple.dt.Xcode"]) + #expect(skill.usageCount == 3) + #expect(skill.isPinned) + #expect(skill.body.contains("step one")) + } + + @Test func matchesSkillsByBundleAndKeywords() { + let skill = TeachingSkill( + id: "teach-textedit-save", + name: "teach-textedit-save", + description: "Save a document in TextEdit", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 2, + isPinned: false, + body: "click file then save or use command s" + ) + + let matches = SkillMatcher.matchSkills( + from: [skill], + bundleId: "com.apple.TextEdit", + transcript: "how do I save this document?" + ) + + #expect(matches.count == 1) + #expect(matches.first?.skill.id == "teach-textedit-save") + } + + @Test func triggerFiresOnUserConfirmation() { + let trace = [ + SessionTraceEntry( + timestamp: Date(), + userTranscript: "how do I save this?", + assistantResponse: "click file then save", + bundleId: "com.apple.TextEdit", + pointed: true + ) + ] + + let trigger = SkillTriggerEvaluator.shouldWriteSkill( + sessionTrace: trace, + latestTranscript: "got it thanks that worked" + ) + + #expect(trigger?.reason == .userConfirmed) + } + + @Test func promptBuilderInjectsMatchedSkills() { + let skill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Save a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: nil, + usageCount: 0, + isPinned: false, + body: "use file > save" + ) + + let prompt = TeachingPromptBuilder.buildVoiceResponsePrompt( + basePrompt: "base prompt", + matchedSkills: [skill] + ) + + #expect(prompt.contains("base prompt")) + #expect(prompt.contains("teaching skills:")) + #expect(prompt.contains("Save in TextEdit")) + #expect(prompt.contains("use file > save")) + } +} diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 000000000..3b2ef52fc --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,80 @@ +# Evolving Teaching Skills — E2E Automation + +Clicky needs more than XCUITest because the real flow uses push-to-talk, microphone, screen capture, global shortcuts, and LLM responses. + +## Stack + +| Layer | Tool | Purpose | +|-------|------|---------| +| Permissions | `@guidepup/setup` or `tccutil` | Pre-grant Screen Recording, Mic, Accessibility | +| Push-to-talk | `NaryaAI/voice-testing-tools` | Simulate `ctrl+option` via HID events | +| Voice input | `tts.mjs` + BlackHole 2ch | Deterministic TTS into virtual mic | +| Screen/UI | Peekaboo or `axcli` | Launch apps, verify overlay/cursor | +| AI | Mock worker fixtures | Repeatable pass/fail without live Claude | + +## One-time setup + +```bash +npx @guidepup/setup --ci +brew install blackhole-2ch sox switchaudio-osx +``` + +Use stable code signing so TCC permissions survive rebuilds. + +## Fast path (inject transcript) + +For skill-loop testing without mic/STT: + +```bash +open /path/to/Clicky.app --args \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL=http://127.0.0.1:8787 \ + -CLICKY_INJECT_TRANSCRIPT="how do I save this document?" +``` + +Then confirm success: + +```bash +open -a Clicky --args \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL=http://127.0.0.1:8787 \ + -CLICKY_INJECT_TRANSCRIPT="got it thanks that worked" +``` + +Assert: + +```bash +test -f ~/.clicky/skills/teach-textedit-save/SKILL.md +``` + +Or run the bundled script: + +```bash +chmod +x tests/e2e/teaching-skills.sh +CLICKY_APP=/path/to/Clicky.app CLICKY_WORKER_URL=http://127.0.0.1:8787 ./tests/e2e/teaching-skills.sh +``` + +## Full user-perspective path + +1. Launch Clicky + fixture app (`TextEdit`, `Xcode`, etc.) +2. Play TTS into BlackHole while holding push-to-talk with `simulate-keypress.swift` +3. Confirm with a second spoken phrase +4. Use Peekaboo to verify overlay/pointing and read `lastMatchedSkillNames` if exposed via debug hooks +5. Assert skill file exists and a repeat question loads the skill into `lastSystemPrompt` + +## E2E launch flags + +- `-CLICKY_E2E=1` — skip onboarding defaults, faster test startup +- `-CLICKY_WORKER_URL=` — point API calls at mock worker +- `-CLICKY_INJECT_TRANSCRIPT=` — bypass mic/STT and run response + skill loop directly + +## Mock worker + +Record Claude `/chat` fixtures once, then replay them in CI. Keep vision and synthesis responses separate so skill writes stay deterministic. + +## What to assert + +1. Skill file created under `~/.clicky/skills/` +2. Second session includes skill content in composed system prompt +3. Pointing tag present when skill includes UI guidance +4. Curator archives stale skills after inactivity (unit-tested separately) diff --git a/tests/e2e/mock-worker.mjs b/tests/e2e/mock-worker.mjs new file mode 100644 index 000000000..e0825f37b --- /dev/null +++ b/tests/e2e/mock-worker.mjs @@ -0,0 +1,95 @@ +#!/usr/bin/env node +import http from "node:http"; + +const PORT = Number(process.env.MOCK_WORKER_PORT ?? 8787); + +function jsonResponse(res, status, body) { + res.writeHead(status, { "content-type": "application/json" }); + res.end(JSON.stringify(body)); +} + +function streamVoiceResponse(res) { + const text = + "click file then save, or use command s. [POINT:120,40:file menu]"; + const chunks = text.match(/.{1,12}/g) ?? [text]; + + res.writeHead(200, { + "content-type": "text/event-stream", + "cache-control": "no-cache", + }); + + for (const chunk of chunks) { + const payload = { + type: "content_block_delta", + delta: { type: "text_delta", text: chunk }, + }; + res.write(`data: ${JSON.stringify(payload)}\n\n`); + } + res.write("data: [DONE]\n\n"); + res.end(); +} + +function synthesisResponse(res) { + jsonResponse(res, 200, { + content: [ + { + type: "text", + text: [ + "workflow: save a document in textedit", + "step one: click file in the menu bar", + "step two: choose save or press command s", + "pointing: start at the file menu near the top left", + "common mistake: looking for a floppy disk icon instead of file > save", + "completion: user says got it or thanks that worked", + ].join("\n"), + }, + ], + }); +} + +const server = http.createServer(async (req, res) => { + if (req.method !== "POST") { + res.writeHead(405); + res.end("Method not allowed"); + return; + } + + const body = await new Promise((resolve, reject) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + req.on("error", reject); + }); + + if (req.url === "/tts") { + res.writeHead(200, { "content-type": "audio/mpeg" }); + res.end(Buffer.from([0xff, 0xfb, 0x90, 0x00])); + return; + } + + if (req.url === "/chat") { + let parsed = {}; + try { + parsed = JSON.parse(body); + } catch { + res.writeHead(400); + res.end("invalid json"); + return; + } + + if (parsed.stream === true) { + streamVoiceResponse(res); + return; + } + + synthesisResponse(res); + return; + } + + res.writeHead(404); + res.end("Not found"); +}); + +server.listen(PORT, "127.0.0.1", () => { + console.log(`mock worker listening on http://127.0.0.1:${PORT}`); +}); diff --git a/tests/e2e/teaching-skills.sh b/tests/e2e/teaching-skills.sh new file mode 100755 index 000000000..c4045ec06 --- /dev/null +++ b/tests/e2e/teaching-skills.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" +CLICKY_APP="${CLICKY_APP:-$ROOT_DIR/build/E2E/Clicky.app}" +WORKER_URL="${CLICKY_WORKER_URL:-http://127.0.0.1:8787}" +SKILLS_DIR="$HOME/.clicky/skills" +MOCK_WORKER_PID="" +CLICKY_PID="" + +cleanup() { + if [[ -n "$MOCK_WORKER_PID" ]]; then + kill "$MOCK_WORKER_PID" 2>/dev/null || true + fi + if [[ -n "$CLICKY_PID" ]]; then + kill "$CLICKY_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT + +echo "Building Clicky for E2E..." +mkdir -p "$ROOT_DIR/build/E2E" +xcodebuild \ + -project "$ROOT_DIR/leanring-buddy.xcodeproj" \ + -scheme leanring-buddy \ + -destination 'platform=macOS' \ + -derivedDataPath "$ROOT_DIR/build/E2E/DerivedData" \ + CODE_SIGN_IDENTITY="-" \ + CODE_SIGNING_ALLOWED=NO \ + build >/tmp/clicky-e2e-build.log 2>&1 + +BUILT_APP="$ROOT_DIR/build/E2E/DerivedData/Build/Products/Debug/Clicky.app" +rm -rf "$CLICKY_APP" +ditto "$BUILT_APP" "$CLICKY_APP" + +echo "Starting mock worker on $WORKER_URL..." +node "$ROOT_DIR/tests/e2e/mock-worker.mjs" >/tmp/clicky-e2e-worker.log 2>&1 & +MOCK_WORKER_PID=$! +sleep 1 + +echo "Resetting prior teaching skills..." +rm -rf "$SKILLS_DIR" +mkdir -p "$SKILLS_DIR" + +echo "Launching Clicky in E2E mode..." +"$CLICKY_APP/Contents/MacOS/Clicky" \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL="$WORKER_URL" \ + -CLICKY_INJECT_TRANSCRIPT="how do I save this document?" \ + -CLICKY_INJECT_TRANSCRIPT_2="got it thanks that worked" >/tmp/clicky-e2e-app.log 2>&1 & +CLICKY_PID=$! + +echo "Waiting for skill write..." +for _ in $(seq 1 30); do + if compgen -G "$SKILLS_DIR/*/SKILL.md" >/dev/null; then + SKILL_FILE="$(ls "$SKILLS_DIR"/*/SKILL.md | head -1)" + echo "PASS: teaching skill written to $SKILL_FILE" + echo "--- skill preview ---" + head -20 "$SKILL_FILE" + exit 0 + fi + sleep 1 +done + +echo "FAIL: no teaching skill written within 30s" +echo "--- app log ---" +tail -40 /tmp/clicky-e2e-app.log || true +echo "--- worker log ---" +tail -20 /tmp/clicky-e2e-worker.log || true +exit 1 From 9fde7779570196d271602f45d8535f82403ed1e9 Mon Sep 17 00:00:00 2001 From: octavi42 Date: Mon, 25 May 2026 12:27:28 +0300 Subject: [PATCH 02/14] Polish teaching skills and add automated E2E CI. Derive skill names from the primary question, persist cross-session topic history, verify read-path injection via Phase B relaunch, and run the full harness locally and on macos-14 GitHub Actions. --- .github/workflows/e2e-teaching-skills.yml | 59 ++++++ AGENTS.md | 1 + leanring-buddy/ClickyE2EConfiguration.swift | 17 ++ leanring-buddy/CompanionManager.swift | 45 ++++- leanring-buddy/SkillMatcher.swift | 11 + leanring-buddy/SkillSynthesizer.swift | 21 +- leanring-buddy/SkillTriggerEvaluator.swift | 71 ++++++- leanring-buddy/TeachingSkill.swift | 98 +++++++++ .../TeachingTopicHistoryStore.swift | 100 +++++++++ leanring-buddyTests/TeachingSkillTests.swift | 129 +++++++++++- tests/e2e/README.md | 189 +++++++++++++----- tests/e2e/run-all.sh | 9 + tests/e2e/teaching-skills.sh | 75 ++++++- 13 files changed, 744 insertions(+), 81 deletions(-) create mode 100644 .github/workflows/e2e-teaching-skills.yml create mode 100644 leanring-buddy/TeachingTopicHistoryStore.swift create mode 100755 tests/e2e/run-all.sh diff --git a/.github/workflows/e2e-teaching-skills.yml b/.github/workflows/e2e-teaching-skills.yml new file mode 100644 index 000000000..d502976d6 --- /dev/null +++ b/.github/workflows/e2e-teaching-skills.yml @@ -0,0 +1,59 @@ +name: E2E Teaching Skills + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + teaching-skills-e2e: + name: Teaching skills E2E (macOS) + runs-on: macos-14 + timeout-minutes: 30 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + # leanring-buddyTests is not wired into the shared Xcode scheme yet. + # Build-for-testing is best-effort; failures here do not block E2E. + - name: Build unit tests (optional) + continue-on-error: true + run: | + set -o pipefail + xcodebuild \ + -project leanring-buddy.xcodeproj \ + -scheme leanring-buddy \ + -destination 'platform=macOS' \ + -derivedDataPath build/E2E/DerivedData \ + CODE_SIGN_IDENTITY="-" \ + CODE_SIGNING_ALLOWED=NO \ + build-for-testing 2>&1 | tee /tmp/clicky-unit-build.log + echo "Unit test build-for-testing succeeded." + + - name: Run teaching-skills E2E + run: | + chmod +x tests/e2e/run-all.sh tests/e2e/teaching-skills.sh + ./tests/e2e/run-all.sh + + - name: Upload E2E logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: teaching-skills-e2e-logs + if-no-files-found: ignore + path: | + /tmp/clicky-e2e-app.log + /tmp/clicky-e2e-app-read.log + /tmp/clicky-e2e-worker.log + /tmp/clicky-e2e-build.log + /tmp/clicky-unit-build.log + ~/.clicky/e2e-last-system-prompt.txt diff --git a/AGENTS.md b/AGENTS.md index 6946d4419..cd9961463 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -74,6 +74,7 @@ Worker vars: `ELEVENLABS_VOICE_ID` | `ClickyAnalytics.swift` | ~121 | PostHog analytics integration for usage tracking. | | `WindowPositionManager.swift` | ~262 | Window placement logic, Screen Recording permission flow, and accessibility permission helpers. | | `AppBundleConfiguration.swift` | ~28 | Runtime configuration reader for keys stored in the app bundle Info.plist. | +| `TeachingTopicHistoryStore.swift` | ~95 | Persists teaching topic tokens, bundle IDs, and timestamps to `~/.clicky/topic-history.json` for cross-session repeat detection. | | `worker/src/index.ts` | ~142 | Cloudflare Worker proxy. Three routes: `/chat` (Claude), `/tts` (ElevenLabs), `/transcribe-token` (AssemblyAI temp token). | ## Build & Run diff --git a/leanring-buddy/ClickyE2EConfiguration.swift b/leanring-buddy/ClickyE2EConfiguration.swift index 694345111..4efc45f66 100644 --- a/leanring-buddy/ClickyE2EConfiguration.swift +++ b/leanring-buddy/ClickyE2EConfiguration.swift @@ -24,6 +24,15 @@ enum ClickyE2EConfiguration { argumentValue(for: "-CLICKY_INJECT_TRANSCRIPT_2=") } + static var injectTranscript3: String? { + argumentValue(for: "-CLICKY_INJECT_TRANSCRIPT_3=") + } + + static var lastSystemPromptFileURL: URL { + FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent(".clicky/e2e-last-system-prompt.txt") + } + static func applyLaunchOverrides() { guard isEnabled else { return } @@ -32,6 +41,14 @@ enum ClickyE2EConfiguration { UserDefaults.standard.set(true, forKey: "isClickyCursorEnabled") } + static func writeLastSystemPromptForE2E(_ systemPrompt: String) { + guard isEnabled else { return } + + let directory = lastSystemPromptFileURL.deletingLastPathComponent() + try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try? systemPrompt.write(to: lastSystemPromptFileURL, atomically: true, encoding: .utf8) + } + private static func argumentValue(for prefix: String) -> String? { ProcessInfo.processInfo.arguments .first(where: { $0.hasPrefix(prefix) }) diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index db2781d1a..e4e0a9a11 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -76,6 +76,7 @@ final class CompanionManager: ObservableObject { } private let teachingSkillStore = TeachingSkillStore() + private let topicHistoryStore = TeachingTopicHistoryStore() private var sessionTrace: [SessionTraceEntry] = [] private var skillWriteTask: Task? @@ -169,18 +170,20 @@ final class CompanionManager: ObservableObject { } func runE2EInjectSequenceIfNeeded() { - guard ClickyE2EConfiguration.isEnabled, - let firstTranscript = ClickyE2EConfiguration.injectTranscript else { - return - } + guard ClickyE2EConfiguration.isEnabled else { return } Task { try? await Task.sleep(nanoseconds: 2_000_000_000) - await injectTranscriptForE2E(firstTranscript) - if let secondTranscript = ClickyE2EConfiguration.injectTranscript2 { - try? await Task.sleep(nanoseconds: 1_000_000_000) - await injectTranscriptForE2E(secondTranscript) + if let firstTranscript = ClickyE2EConfiguration.injectTranscript { + await injectTranscriptForE2E(firstTranscript) + + if let secondTranscript = ClickyE2EConfiguration.injectTranscript2 { + try? await Task.sleep(nanoseconds: 1_000_000_000) + await injectTranscriptForE2E(secondTranscript) + } + } else if let thirdTranscript = ClickyE2EConfiguration.injectTranscript3 { + await injectTranscriptForE2E(thirdTranscript) } } } @@ -191,6 +194,7 @@ final class CompanionManager: ObservableObject { private func bootstrapTeachingSkills() { teachingSkillStore.loadSkills() + topicHistoryStore.load() SkillCurator.curate(store: teachingSkillStore) teachingSkills = teachingSkillStore.skills } @@ -222,6 +226,14 @@ final class CompanionManager: ObservableObject { if sessionTrace.count > 20 { sessionTrace.removeFirst(sessionTrace.count - 20) } + + if !SkillTriggerEvaluator.isConfirmationTranscript(transcript) { + let topic = SkillTriggerEvaluator.deriveTopic(fromQuestion: transcript) + topicHistoryStore.recordTopic( + topic: topic, + bundleId: frontmostApplicationBundleId() + ) + } } private func maybeWriteTeachingSkill(after transcript: String) { @@ -229,7 +241,8 @@ final class CompanionManager: ObservableObject { guard SkillTriggerEvaluator.isScreenTeachingSession(sessionTrace) else { return } guard let trigger = SkillTriggerEvaluator.shouldWriteSkill( sessionTrace: sessionTrace, - latestTranscript: transcript + latestTranscript: transcript, + topicHistory: topicHistoryStore.entries ) else { return } @@ -257,8 +270,14 @@ final class CompanionManager: ObservableObject { guard !Task.isCancelled else { return } + let metadata = SkillSynthesizer.buildSkillMetadata( + sessionTrace: traceSnapshot, + trigger: trigger, + bundleId: bundleId + ) + let skill = SkillSynthesizer.buildSkill( - id: existingSkill?.id, + id: existingSkill?.id ?? metadata.id, name: synthesized.name, description: synthesized.description, body: synthesized.body, @@ -269,6 +288,11 @@ final class CompanionManager: ObservableObject { _ = try teachingSkillStore.saveSkill(skill) SkillCurator.curate(store: teachingSkillStore) teachingSkills = teachingSkillStore.skills + topicHistoryStore.recordTopic( + topic: trigger.topic, + bundleId: bundleId, + skillId: skill.id + ) sessionTrace.removeAll() ClickyAnalytics.trackTeachingSkillSaved( @@ -797,6 +821,7 @@ final class CompanionManager: ObservableObject { matchedSkills: matchedTeachingSkills ) lastSystemPrompt = systemPrompt + ClickyE2EConfiguration.writeLastSystemPromptForE2E(systemPrompt) for skill in matchedTeachingSkills { _ = try? teachingSkillStore.markUsed(skill) diff --git a/leanring-buddy/SkillMatcher.swift b/leanring-buddy/SkillMatcher.swift index 10d33c4a2..917e6971a 100644 --- a/leanring-buddy/SkillMatcher.swift +++ b/leanring-buddy/SkillMatcher.swift @@ -13,6 +13,17 @@ struct SkillMatch: Equatable { } enum SkillMatcher { + static let topicStopwords: Set = [ + "how", "do", "does", "did", "can", "could", "would", "should", "what", "where", "when", "why", "which", + "the", "this", "that", "these", "those", "a", "an", "my", "your", "me", "you", "and", "or", "but", + "is", "are", "was", "were", "be", "been", "being", "to", "in", "on", "at", "for", "of", "with", "from", + "please", "help", "show", "tell", "explain", "use", "using", "got", "thanks", "thank", "worked", "perfect" + ] + + static func meaningfulTokens(_ text: String) -> [String] { + tokenize(text).filter { !topicStopwords.contains($0) } + } + static func matchSkills( from skills: [TeachingSkill], bundleId: String?, diff --git a/leanring-buddy/SkillSynthesizer.swift b/leanring-buddy/SkillSynthesizer.swift index bd4d9947b..4473b7756 100644 --- a/leanring-buddy/SkillSynthesizer.swift +++ b/leanring-buddy/SkillSynthesizer.swift @@ -42,12 +42,24 @@ enum SkillSynthesizer { ) let cleanedBody = response.text.trimmingCharacters(in: .whitespacesAndNewlines) - let fallbackName = "teach-\(trigger.topic.replacingOccurrences(of: " ", with: "-"))" - let name = existingSkill?.name ?? fallbackName - let description = existingSkill?.description ?? "Teach the user how to \(trigger.topic)" + let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: sessionTrace) ?? trigger.topic + let bundleId = sessionTrace.compactMap(\.bundleId).first + let metadata = TeachingSkill.buildMetadata(primaryQuestion: primaryQuestion, bundleId: bundleId) + + let name = existingSkill?.name ?? metadata.name + let description = existingSkill?.description ?? metadata.description return (name: name, description: description, body: cleanedBody) } + static func buildSkillMetadata( + sessionTrace: [SessionTraceEntry], + trigger: SkillWriteTrigger, + bundleId: String? + ) -> TeachingSkill.Metadata { + let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: sessionTrace) ?? trigger.topic + return TeachingSkill.buildMetadata(primaryQuestion: primaryQuestion, bundleId: bundleId) + } + static func buildSkill( id: String?, name: String, @@ -87,6 +99,9 @@ enum SkillSynthesizer { var lines: [String] = [] lines.append("trigger: \(trigger.reason.rawValue)") lines.append("topic: \(trigger.topic)") + if let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: sessionTrace) { + lines.append("primary question: \(primaryQuestion)") + } if let existingSkill { lines.append("existing skill id: \(existingSkill.id)") } diff --git a/leanring-buddy/SkillTriggerEvaluator.swift b/leanring-buddy/SkillTriggerEvaluator.swift index 1b434b2db..1f34c30b0 100644 --- a/leanring-buddy/SkillTriggerEvaluator.swift +++ b/leanring-buddy/SkillTriggerEvaluator.swift @@ -32,13 +32,15 @@ enum SkillTriggerEvaluator { static func shouldWriteSkill( sessionTrace: [SessionTraceEntry], - latestTranscript: String + latestTranscript: String, + topicHistory: [TeachingTopicHistoryEntry] = [] ) -> SkillWriteTrigger? { guard !sessionTrace.isEmpty else { return nil } let normalizedTranscript = latestTranscript.lowercased() let pointedExchanges = sessionTrace.filter(\.pointed) let topic = deriveTopic(from: sessionTrace) + let bundleId = sessionTrace.compactMap(\.bundleId).last if confirmationPhrases.contains(where: { normalizedTranscript.contains($0) }) { return SkillWriteTrigger(reason: .userConfirmed, topic: topic) @@ -52,20 +54,46 @@ enum SkillTriggerEvaluator { return SkillWriteTrigger(reason: .repeatedTopic, topic: topic) } + if hasCrossSessionRepeatedTopic( + topic: topic, + bundleId: bundleId, + topicHistory: topicHistory + ) { + return SkillWriteTrigger(reason: .repeatedTopic, topic: topic) + } + return nil } - static func deriveTopic(from sessionTrace: [SessionTraceEntry]) -> String { - let combined = sessionTrace + static func primaryTeachingQuestion(from sessionTrace: [SessionTraceEntry]) -> String? { + sessionTrace .map(\.userTranscript) - .joined(separator: " ") - let tokens = SkillMatcher.tokenize(combined) + .first { !isConfirmationTranscript($0) } + } + + static func isConfirmationTranscript(_ transcript: String) -> Bool { + let normalized = transcript + .lowercased() + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !normalized.isEmpty else { return true } + return confirmationPhrases.contains { normalized.contains($0) } + } + + static func deriveTopic(from sessionTrace: [SessionTraceEntry]) -> String { + guard let primaryQuestion = primaryTeachingQuestion(from: sessionTrace) else { return "" } + return deriveTopic(fromQuestion: primaryQuestion) + } + + static func deriveTopic(fromQuestion question: String) -> String { + let tokens = SkillMatcher.meaningfulTokens(question) return tokens.prefix(6).joined(separator: " ") } private static func hasRepeatedTopic(in sessionTrace: [SessionTraceEntry]) -> Bool { - guard sessionTrace.count >= 2 else { return false } - let tokenSets = sessionTrace.map { Set(SkillMatcher.tokenize($0.userTranscript)) } + let teachingEntries = sessionTrace.filter { !isConfirmationTranscript($0.userTranscript) } + guard teachingEntries.count >= 2 else { return false } + + let tokenSets = teachingEntries.map { Set(SkillMatcher.meaningfulTokens($0.userTranscript)) } for index in tokenSets.indices { for otherIndex in tokenSets.indices where otherIndex > index { let overlap = tokenSets[index].intersection(tokenSets[otherIndex]) @@ -77,6 +105,35 @@ enum SkillTriggerEvaluator { return false } + private static func hasCrossSessionRepeatedTopic( + topic: String, + bundleId: String?, + topicHistory: [TeachingTopicHistoryEntry] + ) -> Bool { + let topicTokens = Set(SkillMatcher.meaningfulTokens(topic)) + guard topicTokens.count >= 1 else { return false } + + let cutoff = Calendar.current.date(byAdding: .day, value: -7, to: Date()) ?? .distantPast + let matchingEntries = topicHistory.filter { entry in + entry.timestamp >= cutoff && + bundleIdsMatch(entry.bundleId, bundleId) && + entry.topicTokens.filter { topicTokens.contains($0) }.count >= 2 + } + + return matchingEntries.count >= 2 + } + + private static func bundleIdsMatch(_ lhs: String?, _ rhs: String?) -> Bool { + switch (lhs, rhs) { + case (nil, nil): + return true + case (let left?, let right?): + return left == right + case (nil, _), (_, nil): + return true + } + } + static func isScreenTeachingSession(_ sessionTrace: [SessionTraceEntry]) -> Bool { sessionTrace.contains(where: \.pointed) || sessionTrace.contains(where: { $0.bundleId != nil }) diff --git a/leanring-buddy/TeachingSkill.swift b/leanring-buddy/TeachingSkill.swift index 54caddbdd..222da8dfd 100644 --- a/leanring-buddy/TeachingSkill.swift +++ b/leanring-buddy/TeachingSkill.swift @@ -24,6 +24,23 @@ struct TeachingSkill: Identifiable, Equatable { var isPinned: Bool var body: String + struct Metadata: Equatable { + let id: String + let name: String + let description: String + } + + private static let knownAppNames: [String: String] = [ + "com.apple.TextEdit": "TextEdit", + "com.apple.dt.Xcode": "Xcode", + "com.apple.finder": "Finder", + "com.apple.Safari": "Safari", + "com.apple.mail": "Mail", + "com.apple.Notes": "Notes", + "com.apple.systempreferences": "System Settings", + "com.apple.Preview": "Preview" + ] + var folderURL: URL { TeachingSkillStore.skillsRootURL.appendingPathComponent(id, isDirectory: true) } @@ -79,6 +96,87 @@ struct TeachingSkill: Identifiable, Equatable { return collapsed.isEmpty ? "teaching-skill" : collapsed } + static func displayName(forBundleId bundleId: String?) -> String? { + guard let bundleId else { return nil } + if let knownName = knownAppNames[bundleId] { + return knownName + } + guard let lastComponent = bundleId.split(separator: ".").last else { return nil } + return String(lastComponent) + } + + static func buildMetadata(primaryQuestion: String, bundleId: String?) -> Metadata { + let actionTokens = SkillMatcher.meaningfulTokens(primaryQuestion) + let primaryActionToken = actionTokens.first ?? "task" + let appName = displayName(forBundleId: bundleId) + let appSlug = appName.map { slug(from: $0) } + + let skillID: String + let skillName: String + if let appName, let appSlug { + skillID = "teach-\(appSlug)-\(slug(from: primaryActionToken))" + skillName = "\(capitalizeWord(primaryActionToken)) in \(appName)" + } else { + let actionPhrase = actionTokens.prefix(3).joined(separator: " ") + skillID = "teach-\(slug(from: actionPhrase.isEmpty ? primaryActionToken : actionPhrase))" + skillName = capitalizeWords(actionPhrase.isEmpty ? primaryActionToken : actionPhrase) + } + + let skillDescription = descriptionFromQuestion(primaryQuestion) + return Metadata(id: skillID, name: skillName, description: skillDescription) + } + + private static func descriptionFromQuestion(_ question: String) -> String { + var cleanedQuestion = question.trimmingCharacters(in: .whitespacesAndNewlines) + if cleanedQuestion.hasSuffix("?") { + cleanedQuestion.removeLast() + } + + let loweredQuestion = cleanedQuestion.lowercased() + let questionPrefixes = [ + "how do i ", + "how can i ", + "how to ", + "what is the way to ", + "what is ", + "where do i ", + "where is ", + "where can i " + ] + + for prefix in questionPrefixes where loweredQuestion.hasPrefix(prefix) { + cleanedQuestion = String(cleanedQuestion.dropFirst(prefix.count)) + break + } + + let meaningfulTokens = SkillMatcher.meaningfulTokens(cleanedQuestion) + if !meaningfulTokens.isEmpty { + return "Walk the user through \(meaningfulTokens.joined(separator: " "))" + } + + cleanedQuestion = cleanedQuestion + .trimmingCharacters(in: .whitespacesAndNewlines) + .replacingOccurrences(of: " ", with: " ") + + if cleanedQuestion.isEmpty { + return "Walk the user through this task" + } + + return "Walk the user through \(cleanedQuestion)" + } + + private static func capitalizeWord(_ word: String) -> String { + guard let firstCharacter = word.first else { return word } + return String(firstCharacter).uppercased() + word.dropFirst() + } + + private static func capitalizeWords(_ phrase: String) -> String { + phrase + .split(separator: " ") + .map { capitalizeWord(String($0)) } + .joined(separator: " ") + } + static func parse(id: String, markdown: String) -> TeachingSkill? { let trimmed = markdown.trimmingCharacters(in: .whitespacesAndNewlines) guard trimmed.hasPrefix("---") else { return nil } diff --git a/leanring-buddy/TeachingTopicHistoryStore.swift b/leanring-buddy/TeachingTopicHistoryStore.swift new file mode 100644 index 000000000..e708d73a7 --- /dev/null +++ b/leanring-buddy/TeachingTopicHistoryStore.swift @@ -0,0 +1,100 @@ +// +// TeachingTopicHistoryStore.swift +// leanring-buddy +// +// Persists lightweight teaching topic history under ~/.clicky/topic-history.json. +// + +import Foundation + +struct TeachingTopicHistoryEntry: Codable, Equatable { + let topicTokens: [String] + let bundleId: String? + let timestamp: Date + var skillId: String? +} + +final class TeachingTopicHistoryStore { + static let historyFileURL: URL = { + FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent(".clicky/topic-history.json") + }() + + private(set) var entries: [TeachingTopicHistoryEntry] = [] + private let maxEntryCount = 200 + private let historyFileURL: URL + + init(historyFileURL: URL = TeachingTopicHistoryStore.historyFileURL) { + self.historyFileURL = historyFileURL + } + + func load() { + guard FileManager.default.fileExists(atPath: historyFileURL.path), + let data = try? Data(contentsOf: historyFileURL), + let decoded = try? JSONDecoder().decode([TeachingTopicHistoryEntry].self, from: data) else { + entries = [] + return + } + entries = decoded + } + + func save() { + let directory = historyFileURL.deletingLastPathComponent() + try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + guard let data = try? JSONEncoder().encode(entries) else { return } + try? data.write(to: historyFileURL, options: .atomic) + } + + func recordTopic(topic: String, bundleId: String?, skillId: String? = nil) { + let topicTokens = SkillMatcher.meaningfulTokens(topic) + guard !topicTokens.isEmpty else { return } + + entries.append( + TeachingTopicHistoryEntry( + topicTokens: topicTokens, + bundleId: bundleId, + timestamp: Date(), + skillId: skillId + ) + ) + + if entries.count > maxEntryCount { + entries.removeFirst(entries.count - maxEntryCount) + } + save() + } + + func hasRepeatedTopic( + topic: String, + bundleId: String?, + withinDays: Int = 7, + now: Date = Date() + ) -> Bool { + let topicTokens = Set(SkillMatcher.meaningfulTokens(topic)) + guard topicTokens.count >= 1 else { return false } + + let cutoff = Calendar.current.date(byAdding: .day, value: -withinDays, to: now) ?? .distantPast + let matchingEntries = entries.filter { entry in + entry.timestamp >= cutoff && + bundleIdsMatch(entry.bundleId, bundleId) && + tokenOverlapCount(Set(entry.topicTokens), topicTokens) >= 2 + } + + return matchingEntries.count >= 2 + } + + private func bundleIdsMatch(_ lhs: String?, _ rhs: String?) -> Bool { + switch (lhs, rhs) { + case (nil, nil): + return true + case (let left?, let right?): + return left == right + case (nil, _), (_, nil): + return true + } + } + + private func tokenOverlapCount(_ lhs: Set, _ rhs: Set) -> Int { + lhs.intersection(rhs).count + } +} diff --git a/leanring-buddyTests/TeachingSkillTests.swift b/leanring-buddyTests/TeachingSkillTests.swift index 071db30e2..c8037c581 100644 --- a/leanring-buddyTests/TeachingSkillTests.swift +++ b/leanring-buddyTests/TeachingSkillTests.swift @@ -37,8 +37,8 @@ struct TeachingSkillTests { @Test func matchesSkillsByBundleAndKeywords() { let skill = TeachingSkill( id: "teach-textedit-save", - name: "teach-textedit-save", - description: "Save a document in TextEdit", + name: "Save in TextEdit", + description: "Walk the user through saving a document", bundleIds: ["com.apple.TextEdit"], status: .active, lastUsed: Date(), @@ -76,11 +76,105 @@ struct TeachingSkillTests { #expect(trigger?.reason == .userConfirmed) } + @Test func topicIgnoresConfirmationPhrase() { + let trace = [ + SessionTraceEntry( + timestamp: Date(), + userTranscript: "how do I save this document?", + assistantResponse: "click file then save", + bundleId: "com.apple.TextEdit", + pointed: true + ), + SessionTraceEntry( + timestamp: Date(), + userTranscript: "got it thanks that worked", + assistantResponse: "great", + bundleId: "com.apple.TextEdit", + pointed: false + ) + ] + + let topic = SkillTriggerEvaluator.deriveTopic(from: trace) + #expect(topic == "save document") + #expect(SkillTriggerEvaluator.primaryTeachingQuestion(from: trace) == "how do I save this document?") + } + + @Test func slugAndNameAreCleanForSaveQuestionWithConfirmation() { + let trace = [ + SessionTraceEntry( + timestamp: Date(), + userTranscript: "how do I save this document?", + assistantResponse: "click file then save", + bundleId: "com.apple.TextEdit", + pointed: true + ), + SessionTraceEntry( + timestamp: Date(), + userTranscript: "got it thanks that worked", + assistantResponse: "great", + bundleId: "com.apple.TextEdit", + pointed: false + ) + ] + + let trigger = SkillTriggerEvaluator.shouldWriteSkill( + sessionTrace: trace, + latestTranscript: "got it thanks that worked" + ) + let metadata = SkillSynthesizer.buildSkillMetadata( + sessionTrace: trace, + trigger: try #require(trigger), + bundleId: "com.apple.TextEdit" + ) + + #expect(metadata.id == "teach-textedit-save") + #expect(metadata.name == "Save in TextEdit") + #expect(metadata.description == "Walk the user through save document") + #expect(!metadata.id.contains("got")) + #expect(!metadata.id.contains("thanks")) + #expect(!metadata.id.contains("worked")) + } + + @Test func crossSessionRepeatDetectionWithin7Days() throws { + let tempHistoryURL = FileManager.default.temporaryDirectory + .appendingPathComponent("clicky-topic-history-test-\(UUID().uuidString).json") + defer { try? FileManager.default.removeItem(at: tempHistoryURL) } + + let topicHistoryStore = TeachingTopicHistoryStore(historyFileURL: tempHistoryURL) + topicHistoryStore.load() + + let bundleId = "com.apple.TextEdit" + let topic = "save document" + + topicHistoryStore.recordTopic(topic: topic, bundleId: bundleId) + + let trace = [ + SessionTraceEntry( + timestamp: Date(), + userTranscript: "how do I save this document?", + assistantResponse: "click file then save", + bundleId: bundleId, + pointed: true + ) + ] + + topicHistoryStore.recordTopic(topic: topic, bundleId: bundleId) + + let trigger = SkillTriggerEvaluator.shouldWriteSkill( + sessionTrace: trace, + latestTranscript: "how do I save this document?", + topicHistory: topicHistoryStore.entries + ) + + #expect(trigger?.reason == .repeatedTopic) + #expect(trigger?.topic == "save document") + } + @Test func promptBuilderInjectsMatchedSkills() { let skill = TeachingSkill( id: "teach-textedit-save", name: "Save in TextEdit", - description: "Save a document", + description: "Walk the user through saving a document", bundleIds: ["com.apple.TextEdit"], status: .active, lastUsed: nil, @@ -99,4 +193,33 @@ struct TeachingSkillTests { #expect(prompt.contains("Save in TextEdit")) #expect(prompt.contains("use file > save")) } + + @Test func matchedSkillAppearsInComposedPrompt() { + let skill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: nil, + usageCount: 0, + isPinned: false, + body: "click file then save or use command s" + ) + + let matches = SkillMatcher.matchSkills( + from: [skill], + bundleId: "com.apple.TextEdit", + transcript: "how do I save this document?" + ) + let matchedSkills = matches.map(\.skill) + let prompt = TeachingPromptBuilder.buildVoiceResponsePrompt( + basePrompt: "companion voice prompt", + matchedSkills: matchedSkills + ) + + #expect(matchedSkills.map(\.name) == ["Save in TextEdit"]) + #expect(prompt.contains("Save in TextEdit")) + #expect(prompt.contains("click file then save or use command s")) + } } diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 3b2ef52fc..c381e64f8 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -1,80 +1,173 @@ # Evolving Teaching Skills — E2E Automation -Clicky needs more than XCUITest because the real flow uses push-to-talk, microphone, screen capture, global shortcuts, and LLM responses. +Headless E2E for the teaching-skills write path (Phase A) and read path (Phase B). Uses a mock Cloudflare Worker — no live Claude or ElevenLabs keys required. -## Stack +## One-command run -| Layer | Tool | Purpose | -|-------|------|---------| -| Permissions | `@guidepup/setup` or `tccutil` | Pre-grant Screen Recording, Mic, Accessibility | -| Push-to-talk | `NaryaAI/voice-testing-tools` | Simulate `ctrl+option` via HID events | -| Voice input | `tts.mjs` + BlackHole 2ch | Deterministic TTS into virtual mic | -| Screen/UI | Peekaboo or `axcli` | Launch apps, verify overlay/cursor | -| AI | Mock worker fixtures | Repeatable pass/fail without live Claude | +From the repo root: + +```bash +chmod +x tests/e2e/run-all.sh tests/e2e/teaching-skills.sh +./tests/e2e/run-all.sh +echo exit:$? +``` -## One-time setup +Or invoke the test script directly: ```bash -npx @guidepup/setup --ci -brew install blackhole-2ch sox switchaudio-osx +./tests/e2e/teaching-skills.sh ``` -Use stable code signing so TCC permissions survive rebuilds. +Optional environment overrides: -## Fast path (inject transcript) +| Variable | Default | Purpose | +|----------|---------|---------| +| `CLICKY_APP` | `build/E2E/Clicky.app` | Built app bundle path | +| `CLICKY_WORKER_URL` | `http://127.0.0.1:8787` | Mock worker base URL | -For skill-loop testing without mic/STT: +### Expected PASS output -```bash -open /path/to/Clicky.app --args \ - -CLICKY_E2E=1 \ - -CLICKY_WORKER_URL=http://127.0.0.1:8787 \ - -CLICKY_INJECT_TRANSCRIPT="how do I save this document?" +``` +Building Clicky for E2E... +Starting mock worker on http://127.0.0.1:8787... +Resetting prior teaching skills... +Phase A: launch Clicky and teach a save workflow... +PASS: teaching skill written to ~/.clicky/skills/teach--save/SKILL.md +--- skill preview --- +... +PASS: skill slug is clean (teach--save) +Phase B: relaunch Clicky and verify saved skill is injected into prompt... +PASS: saved skill content found in composed system prompt +--- prompt preview --- +teaching skills: +... + +E2E PASS: Phase A (write) + Phase B (read-path) succeeded ``` -Then confirm success: +Exit code `0` on success, non-zero on any failure. -```bash -open -a Clicky --args \ - -CLICKY_E2E=1 \ - -CLICKY_WORKER_URL=http://127.0.0.1:8787 \ - -CLICKY_INJECT_TRANSCRIPT="got it thanks that worked" -``` +## What the test verifies -Assert: +### Phase A — Write path -```bash -test -f ~/.clicky/skills/teach-textedit-save/SKILL.md -``` +1. Builds Clicky unsigned (`CODE_SIGN_IDENTITY="-"`, `CODE_SIGNING_ALLOWED=NO`) +2. Starts `tests/e2e/mock-worker.mjs` on port 8787 +3. Clears `~/.clicky/skills/` and `~/.clicky/e2e-last-system-prompt.txt` +4. Launches Clicky with injected transcripts (question + confirmation) +5. Asserts within 30s: + - `~/.clicky/skills/*/SKILL.md` exists + - Skill slug contains `save` + - Slug does **not** contain `got`, `thanks`, or `worked` -Or run the bundled script: +### Phase B — Read path -```bash -chmod +x tests/e2e/teaching-skills.sh -CLICKY_APP=/path/to/Clicky.app CLICKY_WORKER_URL=http://127.0.0.1:8787 ./tests/e2e/teaching-skills.sh +1. Kills Clicky from Phase A (skills remain on disk) +2. Relaunches with `-CLICKY_INJECT_TRANSCRIPT_3="how do I save this document?"` +3. Asserts within 30s: + - `~/.clicky/e2e-last-system-prompt.txt` exists + - File contains `teaching skills:` + - File contains `save` (saved skill injected into system prompt) + +## CI + +GitHub Actions workflow: [`.github/workflows/e2e-teaching-skills.yml`](../../.github/workflows/e2e-teaching-skills.yml) + +- Triggers on `push` and `pull_request` to `main` +- Runner: `macos-14` +- Runs `./tests/e2e/run-all.sh` +- Uploads `/tmp/clicky-e2e-*.log` and `~/.clicky/e2e-last-system-prompt.txt` as artifacts on failure + +Optional CI badge (after the workflow has run on GitHub): + +```markdown +![E2E Teaching Skills](https://github.com///actions/workflows/e2e-teaching-skills.yml/badge.svg) ``` -## Full user-perspective path +### Unit tests in CI + +The workflow attempts `xcodebuild build-for-testing` before E2E. The shared Xcode scheme currently does not include `leanring-buddyTests`, so this step is best-effort (`continue-on-error: true`) and does not block E2E. Unit tests live in `leanring-buddyTests/TeachingSkillTests.swift` and can be run from Xcode until the scheme is wired. + +## Troubleshooting + +Log files (overwritten each run): + +| Log | Contents | +|-----|----------| +| `/tmp/clicky-e2e-build.log` | Xcode build output | +| `/tmp/clicky-e2e-worker.log` | Mock worker stdout | +| `/tmp/clicky-e2e-app.log` | Clicky stdout during Phase A | +| `/tmp/clicky-e2e-app-read.log` | Clicky stdout during Phase B | + +Debug artifacts under `~/.clicky/`: + +| Path | Purpose | +|------|---------| +| `~/.clicky/skills//SKILL.md` | Saved teaching skill | +| `~/.clicky/e2e-last-system-prompt.txt` | Last composed system prompt (E2E mode) | -1. Launch Clicky + fixture app (`TextEdit`, `Xcode`, etc.) -2. Play TTS into BlackHole while holding push-to-talk with `simulate-keypress.swift` -3. Confirm with a second spoken phrase -4. Use Peekaboo to verify overlay/pointing and read `lastMatchedSkillNames` if exposed via debug hooks -5. Assert skill file exists and a repeat question loads the skill into `lastSystemPrompt` +Common failures: + +- **Build fails** — check `/tmp/clicky-e2e-build.log`; requires Xcode and macOS 14.2+ SDK +- **No skill written in 30s** — check `/tmp/clicky-e2e-app.log` and worker log; mock worker must be reachable at `127.0.0.1:8787` +- **Read path fails** — skill from Phase A must still exist under `~/.clicky/skills/`; check `/tmp/clicky-e2e-app-read.log` ## E2E launch flags +Defined in `leanring-buddy/ClickyE2EConfiguration.swift`: + - `-CLICKY_E2E=1` — skip onboarding defaults, faster test startup - `-CLICKY_WORKER_URL=` — point API calls at mock worker -- `-CLICKY_INJECT_TRANSCRIPT=` — bypass mic/STT and run response + skill loop directly +- `-CLICKY_INJECT_TRANSCRIPT=` — first injected transcript (write path) +- `-CLICKY_INJECT_TRANSCRIPT_2=` — second injected transcript (confirmation) +- `-CLICKY_INJECT_TRANSCRIPT_3=` — read-path-only launch (skills already on disk) + +Writes `~/.clicky/e2e-last-system-prompt.txt` on each injected response for headless assertions. ## Mock worker -Record Claude `/chat` fixtures once, then replay them in CI. Keep vision and synthesis responses separate so skill writes stay deterministic. +`tests/e2e/mock-worker.mjs` serves deterministic `/chat` and `/tts` responses on `http://127.0.0.1:8787`. Streaming responses include a `[POINT:...]` tag; non-streaming synthesis responses produce skill body content. + +```bash +node tests/e2e/mock-worker.mjs +``` + +## Manual inject (without full E2E script) + +Write path only: + +```bash +open build/E2E/Clicky.app --args \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL=http://127.0.0.1:8787 \ + -CLICKY_INJECT_TRANSCRIPT="how do I save this document?" \ + -CLICKY_INJECT_TRANSCRIPT_2="got it thanks that worked" +``` + +Read path only (after a skill exists): + +```bash +open build/E2E/Clicky.app --args \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL=http://127.0.0.1:8787 \ + -CLICKY_INJECT_TRANSCRIPT_3="how do I save this document?" + +grep "teaching skills:" ~/.clicky/e2e-last-system-prompt.txt +``` + +## Full user-perspective path (future) + +Not covered by the current headless E2E script: + +| Layer | Tool | Purpose | +|-------|------|---------| +| Permissions | `@guidepup/setup` or `tccutil` | Pre-grant Screen Recording, Mic, Accessibility | +| Push-to-talk | `NaryaAI/voice-testing-tools` | Simulate `ctrl+option` via HID events | +| Voice input | `tts.mjs` + BlackHole 2ch | Deterministic TTS into virtual mic | +| Screen/UI | Peekaboo or `axcli` | Launch apps, verify overlay/cursor | + +TCC permission seeding in CI is documented as a follow-up — it can be flaky on ephemeral runners. -## What to assert +## Related unit tests -1. Skill file created under `~/.clicky/skills/` -2. Second session includes skill content in composed system prompt -3. Pointing tag present when skill includes UI guidance -4. Curator archives stale skills after inactivity (unit-tested separately) +`leanring-buddyTests/TeachingSkillTests.swift` covers topic extraction, slug generation, cross-session repeat detection, and prompt injection logic without launching the app. diff --git a/tests/e2e/run-all.sh b/tests/e2e/run-all.sh new file mode 100755 index 000000000..b0d557a99 --- /dev/null +++ b/tests/e2e/run-all.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +echo "Running Clicky teaching-skills E2E..." +echo "" + +exec "$SCRIPT_DIR/teaching-skills.sh" "$@" diff --git a/tests/e2e/teaching-skills.sh b/tests/e2e/teaching-skills.sh index c4045ec06..abcdf1317 100755 --- a/tests/e2e/teaching-skills.sh +++ b/tests/e2e/teaching-skills.sh @@ -5,6 +5,7 @@ ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" CLICKY_APP="${CLICKY_APP:-$ROOT_DIR/build/E2E/Clicky.app}" WORKER_URL="${CLICKY_WORKER_URL:-http://127.0.0.1:8787}" SKILLS_DIR="$HOME/.clicky/skills" +PROMPT_FILE="$HOME/.clicky/e2e-last-system-prompt.txt" MOCK_WORKER_PID="" CLICKY_PID="" @@ -41,8 +42,9 @@ sleep 1 echo "Resetting prior teaching skills..." rm -rf "$SKILLS_DIR" mkdir -p "$SKILLS_DIR" +rm -f "$PROMPT_FILE" -echo "Launching Clicky in E2E mode..." +echo "Phase A: launch Clicky and teach a save workflow..." "$CLICKY_APP/Contents/MacOS/Clicky" \ -CLICKY_E2E=1 \ -CLICKY_WORKER_URL="$WORKER_URL" \ @@ -50,21 +52,74 @@ echo "Launching Clicky in E2E mode..." -CLICKY_INJECT_TRANSCRIPT_2="got it thanks that worked" >/tmp/clicky-e2e-app.log 2>&1 & CLICKY_PID=$! -echo "Waiting for skill write..." +SKILL_FILE="" for _ in $(seq 1 30); do if compgen -G "$SKILLS_DIR/*/SKILL.md" >/dev/null; then SKILL_FILE="$(ls "$SKILLS_DIR"/*/SKILL.md | head -1)" - echo "PASS: teaching skill written to $SKILL_FILE" - echo "--- skill preview ---" - head -20 "$SKILL_FILE" - exit 0 + break fi sleep 1 done -echo "FAIL: no teaching skill written within 30s" +if [[ -z "$SKILL_FILE" ]]; then + echo "FAIL: no teaching skill written within 30s" + echo "--- app log ---" + tail -40 /tmp/clicky-e2e-app.log || true + echo "--- worker log ---" + tail -20 /tmp/clicky-e2e-worker.log || true + exit 1 +fi + +SKILL_ID="$(basename "$(dirname "$SKILL_FILE")")" +echo "PASS: teaching skill written to $SKILL_FILE" +echo "--- skill preview ---" +head -20 "$SKILL_FILE" + +if [[ "$SKILL_ID" != *save* ]]; then + echo "FAIL: skill slug '$SKILL_ID' does not contain 'save'" + exit 1 +fi + +if [[ "$SKILL_ID" == *got* ]] || [[ "$SKILL_ID" == *thanks* ]] || [[ "$SKILL_ID" == *worked* ]]; then + echo "FAIL: skill slug '$SKILL_ID' contains confirmation phrase tokens" + exit 1 +fi + +echo "PASS: skill slug is clean ($SKILL_ID)" + +kill "$CLICKY_PID" 2>/dev/null || true +CLICKY_PID="" +sleep 1 +rm -f "$PROMPT_FILE" + +echo "Phase B: relaunch Clicky and verify saved skill is injected into prompt..." +"$CLICKY_APP/Contents/MacOS/Clicky" \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL="$WORKER_URL" \ + -CLICKY_INJECT_TRANSCRIPT_3="how do I save this document?" >/tmp/clicky-e2e-app-read.log 2>&1 & +CLICKY_PID=$! + +for _ in $(seq 1 30); do + if [[ -f "$PROMPT_FILE" ]] && grep -q "teaching skills:" "$PROMPT_FILE"; then + if grep -qi "save" "$PROMPT_FILE"; then + echo "PASS: saved skill content found in composed system prompt" + echo "--- prompt preview ---" + grep -A 8 "teaching skills:" "$PROMPT_FILE" | head -12 + echo "" + echo "E2E PASS: Phase A (write) + Phase B (read-path) succeeded" + exit 0 + fi + fi + sleep 1 +done + +echo "FAIL: composed system prompt did not include saved skill content within 30s" echo "--- app log ---" -tail -40 /tmp/clicky-e2e-app.log || true -echo "--- worker log ---" -tail -20 /tmp/clicky-e2e-worker.log || true +tail -40 /tmp/clicky-e2e-app-read.log || true +if [[ -f "$PROMPT_FILE" ]]; then + echo "--- prompt file ---" + head -40 "$PROMPT_FILE" || true +else + echo "prompt file missing: $PROMPT_FILE" +fi exit 1 From a89fd195f8a5f820ee0776d54d4d6616ab0d0e97 Mon Sep 17 00:00:00 2001 From: octavi42 Date: Mon, 25 May 2026 12:54:13 +0300 Subject: [PATCH 03/14] Scope teaching-skills CI to feature branch only. E2E workflow runs teaching-skills.sh without niche-discovery tests. --- .github/workflows/e2e-teaching-skills.yml | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/e2e-teaching-skills.yml b/.github/workflows/e2e-teaching-skills.yml index d502976d6..cb010c540 100644 --- a/.github/workflows/e2e-teaching-skills.yml +++ b/.github/workflows/e2e-teaching-skills.yml @@ -4,9 +4,11 @@ on: push: branches: - main + - feature/teaching-skills pull_request: branches: - main + - feature/teaching-skills jobs: teaching-skills-e2e: @@ -23,22 +25,6 @@ jobs: with: node-version: "20" - # leanring-buddyTests is not wired into the shared Xcode scheme yet. - # Build-for-testing is best-effort; failures here do not block E2E. - - name: Build unit tests (optional) - continue-on-error: true - run: | - set -o pipefail - xcodebuild \ - -project leanring-buddy.xcodeproj \ - -scheme leanring-buddy \ - -destination 'platform=macOS' \ - -derivedDataPath build/E2E/DerivedData \ - CODE_SIGN_IDENTITY="-" \ - CODE_SIGNING_ALLOWED=NO \ - build-for-testing 2>&1 | tee /tmp/clicky-unit-build.log - echo "Unit test build-for-testing succeeded." - - name: Run teaching-skills E2E run: | chmod +x tests/e2e/run-all.sh tests/e2e/teaching-skills.sh @@ -55,5 +41,4 @@ jobs: /tmp/clicky-e2e-app-read.log /tmp/clicky-e2e-worker.log /tmp/clicky-e2e-build.log - /tmp/clicky-unit-build.log ~/.clicky/e2e-last-system-prompt.txt From c7df3f808bec96bff246b1b11fa35d8269498a50 Mon Sep 17 00:00:00 2001 From: octavi42 Date: Wed, 27 May 2026 16:53:13 +0300 Subject: [PATCH 04/14] Tighten teaching skill writes to confirmation-only with patch updates. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hermes-style loop: stable (app, task) identity, target-app resolution from session text, patch-first synthesis, and E2E Phase C — so refinements update one skill instead of spawning duplicates mid-tutorial. --- leanring-buddy/CompanionManager.swift | 55 +++-- leanring-buddy/SkillMatcher.swift | 43 +++- leanring-buddy/SkillSynthesizer.swift | 81 +++++-- leanring-buddy/SkillTargetAppResolver.swift | 36 ++++ leanring-buddy/SkillTriggerEvaluator.swift | 84 ++------ leanring-buddy/TeachingSkill.swift | 60 +++++- leanring-buddy/leanring_buddyApp.swift | 6 +- leanring-buddyTests/TeachingSkillTests.swift | 210 ++++++++++++++++++- tests/e2e/mock-worker.mjs | 40 +++- tests/e2e/teaching-skills.sh | 145 +++++++------ 10 files changed, 564 insertions(+), 196 deletions(-) create mode 100644 leanring-buddy/SkillTargetAppResolver.swift diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index e4e0a9a11..1a581cabb 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -176,13 +176,23 @@ final class CompanionManager: ObservableObject { try? await Task.sleep(nanoseconds: 2_000_000_000) if let firstTranscript = ClickyE2EConfiguration.injectTranscript { + print("🧪 E2E inject 1: \(firstTranscript)") await injectTranscriptForE2E(firstTranscript) if let secondTranscript = ClickyE2EConfiguration.injectTranscript2 { - try? await Task.sleep(nanoseconds: 1_000_000_000) + try? await Task.sleep(nanoseconds: 2_000_000_000) + print("🧪 E2E inject 2: \(secondTranscript)") await injectTranscriptForE2E(secondTranscript) } + + if let thirdTranscript = ClickyE2EConfiguration.injectTranscript3 { + try? await Task.sleep(nanoseconds: 2_000_000_000) + print("🧪 E2E inject 3: \(thirdTranscript)") + await injectTranscriptForE2E(thirdTranscript) + try? await Task.sleep(nanoseconds: 3_000_000_000) + } } else if let thirdTranscript = ClickyE2EConfiguration.injectTranscript3 { + print("🧪 E2E inject read-path: \(thirdTranscript)") await injectTranscriptForE2E(thirdTranscript) } } @@ -200,9 +210,10 @@ final class CompanionManager: ObservableObject { } private func matchedSkills(for transcript: String) -> [TeachingSkill] { + let bundleId = TeachingSkill.detectBundleId(in: transcript) ?? frontmostApplicationBundleId() let matches = SkillMatcher.matchSkills( from: teachingSkillStore.skills, - bundleId: frontmostApplicationBundleId(), + bundleId: bundleId, transcript: transcript ) return matches.map(\.skill) @@ -251,20 +262,25 @@ final class CompanionManager: ObservableObject { ClickyAnalytics.trackTeachingSkillWriteTriggered(reason: trigger.reason.rawValue, topic: trigger.topic) let traceSnapshot = sessionTrace - let bundleId = frontmostApplicationBundleId() + let targetBundleId = SkillTargetAppResolver.resolveTargetBundleId( + from: traceSnapshot, + frontmostBundleId: frontmostApplicationBundleId() + ) + let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: traceSnapshot) ?? trigger.topic skillWriteTask?.cancel() skillWriteTask = Task { do { - let existingSkill = SkillMatcher.findSimilarSkill( + let existingSkill = SkillMatcher.findSkillForUpdate( in: teachingSkillStore.skills, - bundleId: bundleId, - topic: trigger.topic + targetBundleId: targetBundleId, + primaryQuestion: primaryQuestion ) let synthesized = try await SkillSynthesizer.synthesizeSkillContent( sessionTrace: traceSnapshot, trigger: trigger, existingSkill: existingSkill, + targetBundleId: targetBundleId, claudeAPI: claudeAPI ) @@ -273,7 +289,7 @@ final class CompanionManager: ObservableObject { let metadata = SkillSynthesizer.buildSkillMetadata( sessionTrace: traceSnapshot, trigger: trigger, - bundleId: bundleId + targetBundleId: targetBundleId ) let skill = SkillSynthesizer.buildSkill( @@ -281,7 +297,9 @@ final class CompanionManager: ObservableObject { name: synthesized.name, description: synthesized.description, body: synthesized.body, - bundleId: bundleId, + targetBundleId: targetBundleId, + taskSlug: metadata.taskSlug, + primaryQuestion: primaryQuestion, existingSkill: existingSkill ) @@ -290,7 +308,7 @@ final class CompanionManager: ObservableObject { teachingSkills = teachingSkillStore.skills topicHistoryStore.recordTopic( topic: trigger.topic, - bundleId: bundleId, + bundleId: targetBundleId, skillId: skill.id ) sessionTrace.removeAll() @@ -932,14 +950,17 @@ final class CompanionManager: ObservableObject { // Play the response via TTS. Keep the spinner (processing state) // until the audio actually starts playing, then switch to responding. if !spokenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { - do { - try await elevenLabsTTSClient.speakText(spokenText) - // speakText returns after player.play() — audio is now playing - voiceState = .responding - } catch { - ClickyAnalytics.trackTTSError(error: error.localizedDescription) - print("⚠️ ElevenLabs TTS error: \(error)") - speakCreditsErrorFallback() + if ClickyE2EConfiguration.isEnabled { + voiceState = .idle + } else { + do { + try await elevenLabsTTSClient.speakText(spokenText) + voiceState = .responding + } catch { + ClickyAnalytics.trackTTSError(error: error.localizedDescription) + print("⚠️ ElevenLabs TTS error: \(error)") + speakCreditsErrorFallback() + } } } } catch is CancellationError { diff --git a/leanring-buddy/SkillMatcher.swift b/leanring-buddy/SkillMatcher.swift index 917e6971a..3197e6ba5 100644 --- a/leanring-buddy/SkillMatcher.swift +++ b/leanring-buddy/SkillMatcher.swift @@ -72,12 +72,49 @@ enum SkillMatcher { bundleId: String?, topic: String ) -> TeachingSkill? { - let topicTokens = Set(tokenize(topic)) + findSkillForUpdate( + in: skills, + targetBundleId: bundleId, + primaryQuestion: topic + ) + } + + static func findSkillForUpdate( + in skills: [TeachingSkill], + targetBundleId: String?, + primaryQuestion: String + ) -> TeachingSkill? { + let resolvedTargetBundleId = TeachingSkill.detectBundleId(in: primaryQuestion) ?? targetBundleId + let expectedSkillId = TeachingSkill.stableSkillId( + bundleId: resolvedTargetBundleId, + primaryQuestion: primaryQuestion + ) + let expectedTaskSlug = TeachingSkill.taskSlug(from: primaryQuestion) + + if let exactMatch = skills.first(where: { $0.id == expectedSkillId }) { + return exactMatch + } + + if let resolvedTargetBundleId { + let bundleMatches = skills.filter { skill in + skill.bundleIds.contains(resolvedTargetBundleId) && + (skill.taskSlug == expectedTaskSlug || skill.id.hasSuffix("-\(expectedTaskSlug)")) + } + + if let bestBundleMatch = bundleMatches.max(by: { lhs, rhs in + if lhs.usageCount != rhs.usageCount { return lhs.usageCount < rhs.usageCount } + return (lhs.lastUsed ?? .distantPast) < (rhs.lastUsed ?? .distantPast) + }) { + return bestBundleMatch + } + } + + let topicTokens = Set(meaningfulTokens(primaryQuestion)) guard !topicTokens.isEmpty else { return nil } let candidates = skills.filter { skill in - guard let bundleId else { return true } - return skill.bundleIds.isEmpty || skill.bundleIds.contains(bundleId) + guard let resolvedTargetBundleId else { return true } + return skill.bundleIds.isEmpty || skill.bundleIds.contains(resolvedTargetBundleId) } return candidates.max { lhs, rhs in diff --git a/leanring-buddy/SkillSynthesizer.swift b/leanring-buddy/SkillSynthesizer.swift index 4473b7756..fd01d71eb 100644 --- a/leanring-buddy/SkillSynthesizer.swift +++ b/leanring-buddy/SkillSynthesizer.swift @@ -19,33 +19,69 @@ enum SkillSynthesizer { - exact ui labels, menu paths, and shortcuts when known - pointing heuristics (what to point at first, what to avoid) - common mistakes the user made or might make + - user preferences and constraints from the session (e.g. keyboard only, no menu) - completion signals ("user said got it", visible ui state) keep it concise, practical, and specific to macos ui teaching. all lowercase. """ + private static let patchSystemPrompt = """ + you update an existing teaching skill for clicky, a screen-native voice tutor that points at ui elements. + + given the existing skill body and a new tutoring session, produce an updated markdown body only. do not include yaml frontmatter. + + patch-first rules: + - preserve guidance that still works + - merge new constraints and refinements from the latest session + - do not duplicate steps or create a parallel workflow + - honor user preferences such as "keyboard only" or "avoid the menu" + - keep ordered steps, ui labels, shortcuts, pointing heuristics, and completion signals + + keep it concise, practical, and specific to macos ui teaching. all lowercase. + """ + static func synthesizeSkillContent( sessionTrace: [SessionTraceEntry], trigger: SkillWriteTrigger, existingSkill: TeachingSkill?, + targetBundleId: String?, claudeAPI: ClaudeAPI ) async throws -> (name: String, description: String, body: String) { - let sessionSummary = renderSessionSummary(sessionTrace, trigger: trigger, existingSkill: existingSkill) - let userPrompt = existingSkill == nil - ? "create a new teaching skill from this session:\n\n\(sessionSummary)" - : "update this teaching skill using the new session. preserve useful existing guidance and improve it:\n\nexisting skill:\n\(existingSkill?.body ?? "")\n\nnew session:\n\(sessionSummary)" + let sessionSummary = renderSessionSummary( + sessionTrace, + trigger: trigger, + existingSkill: existingSkill, + targetBundleId: targetBundleId + ) + let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: sessionTrace) ?? trigger.topic + let metadata = TeachingSkill.buildMetadata(primaryQuestion: primaryQuestion, bundleId: targetBundleId) + + let userPrompt: String + let systemPrompt: String + if let existingSkill { + systemPrompt = patchSystemPrompt + userPrompt = """ + patch this existing teaching skill using the new session. preserve useful existing guidance and merge refinements: + + existing skill id: \(existingSkill.id) + existing skill body: + \(existingSkill.body) + + new session: + \(sessionSummary) + """ + } else { + systemPrompt = synthesisSystemPrompt + userPrompt = "create a new teaching skill from this session:\n\n\(sessionSummary)" + } let response = try await claudeAPI.sendTextMessage( - systemPrompt: synthesisSystemPrompt, + systemPrompt: systemPrompt, userPrompt: userPrompt, maxTokens: 1200 ) let cleanedBody = response.text.trimmingCharacters(in: .whitespacesAndNewlines) - let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: sessionTrace) ?? trigger.topic - let bundleId = sessionTrace.compactMap(\.bundleId).first - let metadata = TeachingSkill.buildMetadata(primaryQuestion: primaryQuestion, bundleId: bundleId) - let name = existingSkill?.name ?? metadata.name let description = existingSkill?.description ?? metadata.description return (name: name, description: description, body: cleanedBody) @@ -54,10 +90,10 @@ enum SkillSynthesizer { static func buildSkillMetadata( sessionTrace: [SessionTraceEntry], trigger: SkillWriteTrigger, - bundleId: String? + targetBundleId: String? ) -> TeachingSkill.Metadata { let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: sessionTrace) ?? trigger.topic - return TeachingSkill.buildMetadata(primaryQuestion: primaryQuestion, bundleId: bundleId) + return TeachingSkill.buildMetadata(primaryQuestion: primaryQuestion, bundleId: targetBundleId) } static func buildSkill( @@ -65,15 +101,21 @@ enum SkillSynthesizer { name: String, description: String, body: String, - bundleId: String?, + targetBundleId: String?, + taskSlug: String?, + primaryQuestion: String, existingSkill: TeachingSkill? ) -> TeachingSkill { - let resolvedID = id ?? existingSkill?.id ?? TeachingSkill.slug(from: name) + let resolvedID = id + ?? existingSkill?.id + ?? TeachingSkill.stableSkillId(bundleId: targetBundleId, primaryQuestion: primaryQuestion) + let resolvedTaskSlug = taskSlug ?? existingSkill?.taskSlug ?? TeachingSkill.taskSlug(from: primaryQuestion) + let bundleIds: [String] - if let existingSkill, !existingSkill.bundleIds.isEmpty { + if let targetBundleId { + bundleIds = [targetBundleId] + } else if let existingSkill, !existingSkill.bundleIds.isEmpty { bundleIds = existingSkill.bundleIds - } else if let bundleId { - bundleIds = [bundleId] } else { bundleIds = [] } @@ -87,6 +129,7 @@ enum SkillSynthesizer { lastUsed: Date(), usageCount: existingSkill?.usageCount ?? 0, isPinned: existingSkill?.isPinned ?? false, + taskSlug: resolvedTaskSlug, body: body ) } @@ -94,11 +137,13 @@ enum SkillSynthesizer { private static func renderSessionSummary( _ sessionTrace: [SessionTraceEntry], trigger: SkillWriteTrigger, - existingSkill: TeachingSkill? + existingSkill: TeachingSkill?, + targetBundleId: String? ) -> String { var lines: [String] = [] lines.append("trigger: \(trigger.reason.rawValue)") lines.append("topic: \(trigger.topic)") + lines.append("target bundle id: \(targetBundleId ?? "unknown")") if let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: sessionTrace) { lines.append("primary question: \(primaryQuestion)") } @@ -110,7 +155,7 @@ enum SkillSynthesizer { lines.append("exchange \(index + 1):") lines.append("user: \(entry.userTranscript)") lines.append("assistant: \(entry.assistantResponse)") - lines.append("bundle id: \(entry.bundleId ?? "unknown")") + lines.append("frontmost bundle id: \(entry.bundleId ?? "unknown")") lines.append("pointed: \(entry.pointed ? "yes" : "no")") } diff --git a/leanring-buddy/SkillTargetAppResolver.swift b/leanring-buddy/SkillTargetAppResolver.swift new file mode 100644 index 000000000..18d0d5ff5 --- /dev/null +++ b/leanring-buddy/SkillTargetAppResolver.swift @@ -0,0 +1,36 @@ +// +// SkillTargetAppResolver.swift +// leanring-buddy +// +// Resolves which app a tutoring session is teaching, which may differ from the +// frontmost app (e.g. terminal focused while TextEdit is visible on screen). +// + +import Foundation + +enum SkillTargetAppResolver { + static func resolveTargetBundleId( + from sessionTrace: [SessionTraceEntry], + frontmostBundleId: String?, + existingSkill: TeachingSkill? = nil + ) -> String? { + if let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: sessionTrace), + let bundleIdFromPrimaryQuestion = TeachingSkill.detectBundleId(in: primaryQuestion) { + return bundleIdFromPrimaryQuestion + } + + for entry in sessionTrace { + for text in [entry.userTranscript, entry.assistantResponse] { + if let bundleId = TeachingSkill.detectBundleId(in: text) { + return bundleId + } + } + } + + if let existingSkill, let existingBundleId = existingSkill.bundleIds.first { + return existingBundleId + } + + return frontmostBundleId + } +} diff --git a/leanring-buddy/SkillTriggerEvaluator.swift b/leanring-buddy/SkillTriggerEvaluator.swift index 1f34c30b0..5ada47617 100644 --- a/leanring-buddy/SkillTriggerEvaluator.swift +++ b/leanring-buddy/SkillTriggerEvaluator.swift @@ -10,8 +10,6 @@ import Foundation struct SkillWriteTrigger: Equatable { enum Reason: String, Equatable { case userConfirmed - case multiStepPointing - case repeatedTopic } let reason: Reason @@ -30,6 +28,7 @@ enum SkillTriggerEvaluator { "that helps" ] + /// Hermes-style write gate: persist skills only after explicit user confirmation. static func shouldWriteSkill( sessionTrace: [SessionTraceEntry], latestTranscript: String, @@ -37,32 +36,16 @@ enum SkillTriggerEvaluator { ) -> SkillWriteTrigger? { guard !sessionTrace.isEmpty else { return nil } - let normalizedTranscript = latestTranscript.lowercased() - let pointedExchanges = sessionTrace.filter(\.pointed) - let topic = deriveTopic(from: sessionTrace) - let bundleId = sessionTrace.compactMap(\.bundleId).last - - if confirmationPhrases.contains(where: { normalizedTranscript.contains($0) }) { - return SkillWriteTrigger(reason: .userConfirmed, topic: topic) - } - - if pointedExchanges.count >= 2 { - return SkillWriteTrigger(reason: .multiStepPointing, topic: topic) - } - - if hasRepeatedTopic(in: sessionTrace) { - return SkillWriteTrigger(reason: .repeatedTopic, topic: topic) - } + let normalizedTranscript = latestTranscript + .lowercased() + .trimmingCharacters(in: .whitespacesAndNewlines) - if hasCrossSessionRepeatedTopic( - topic: topic, - bundleId: bundleId, - topicHistory: topicHistory - ) { - return SkillWriteTrigger(reason: .repeatedTopic, topic: topic) + guard confirmationPhrases.contains(where: { normalizedTranscript.contains($0) }) else { + return nil } - return nil + let topic = deriveTopic(from: sessionTrace) + return SkillWriteTrigger(reason: .userConfirmed, topic: topic) } static func primaryTeachingQuestion(from sessionTrace: [SessionTraceEntry]) -> String? { @@ -89,53 +72,12 @@ enum SkillTriggerEvaluator { return tokens.prefix(6).joined(separator: " ") } - private static func hasRepeatedTopic(in sessionTrace: [SessionTraceEntry]) -> Bool { - let teachingEntries = sessionTrace.filter { !isConfirmationTranscript($0.userTranscript) } - guard teachingEntries.count >= 2 else { return false } - - let tokenSets = teachingEntries.map { Set(SkillMatcher.meaningfulTokens($0.userTranscript)) } - for index in tokenSets.indices { - for otherIndex in tokenSets.indices where otherIndex > index { - let overlap = tokenSets[index].intersection(tokenSets[otherIndex]) - if overlap.count >= 2 { - return true - } - } - } - return false - } - - private static func hasCrossSessionRepeatedTopic( - topic: String, - bundleId: String?, - topicHistory: [TeachingTopicHistoryEntry] - ) -> Bool { - let topicTokens = Set(SkillMatcher.meaningfulTokens(topic)) - guard topicTokens.count >= 1 else { return false } - - let cutoff = Calendar.current.date(byAdding: .day, value: -7, to: Date()) ?? .distantPast - let matchingEntries = topicHistory.filter { entry in - entry.timestamp >= cutoff && - bundleIdsMatch(entry.bundleId, bundleId) && - entry.topicTokens.filter { topicTokens.contains($0) }.count >= 2 - } - - return matchingEntries.count >= 2 - } - - private static func bundleIdsMatch(_ lhs: String?, _ rhs: String?) -> Bool { - switch (lhs, rhs) { - case (nil, nil): - return true - case (let left?, let right?): - return left == right - case (nil, _), (_, nil): - return true - } - } - static func isScreenTeachingSession(_ sessionTrace: [SessionTraceEntry]) -> Bool { sessionTrace.contains(where: \.pointed) || - sessionTrace.contains(where: { $0.bundleId != nil }) + sessionTrace.contains(where: { $0.bundleId != nil }) || + sessionTrace.contains { entry in + TeachingSkill.detectBundleId(in: entry.userTranscript) != nil || + TeachingSkill.detectBundleId(in: entry.assistantResponse) != nil + } } } diff --git a/leanring-buddy/TeachingSkill.swift b/leanring-buddy/TeachingSkill.swift index 222da8dfd..9a08a3372 100644 --- a/leanring-buddy/TeachingSkill.swift +++ b/leanring-buddy/TeachingSkill.swift @@ -22,12 +22,14 @@ struct TeachingSkill: Identifiable, Equatable { var lastUsed: Date? var usageCount: Int var isPinned: Bool + var taskSlug: String? var body: String struct Metadata: Equatable { let id: String let name: String let description: String + let taskSlug: String } private static let knownAppNames: [String: String] = [ @@ -76,6 +78,7 @@ struct TeachingSkill: Identifiable, Equatable { lastUsed: \(lastUsedValue) usageCount: \(usageCount) pinned: \(isPinned) + taskSlug: \(taskSlug ?? "") --- \(body.trimmingCharacters(in: .whitespacesAndNewlines)) @@ -105,25 +108,67 @@ struct TeachingSkill: Identifiable, Equatable { return String(lastComponent) } + static func detectBundleId(in text: String) -> String? { + let loweredText = text.lowercased() + + for (bundleId, displayName) in knownAppNames { + let loweredDisplayName = displayName.lowercased() + if loweredText.contains(loweredDisplayName) { + return bundleId + } + + let compactDisplayName = loweredDisplayName.replacingOccurrences(of: " ", with: "") + if compactDisplayName.count >= 4, loweredText.contains(compactDisplayName) { + return bundleId + } + } + + if loweredText.contains("text edit") || loweredText.contains("textedit") { + return "com.apple.TextEdit" + } + + return nil + } + + static func taskSlug(from primaryQuestion: String) -> String { + let actionTokens = SkillMatcher.meaningfulTokens(primaryQuestion) + let primaryActionToken = actionTokens.first ?? "task" + return slug(from: primaryActionToken) + } + + static func stableSkillId(bundleId: String?, primaryQuestion: String) -> String { + let resolvedTaskSlug = taskSlug(from: primaryQuestion) + let appName = displayName(forBundleId: bundleId) + + if let appName { + return "teach-\(slug(from: appName))-\(resolvedTaskSlug)" + } + + let actionPhrase = SkillMatcher.meaningfulTokens(primaryQuestion).prefix(3).joined(separator: " ") + if actionPhrase.isEmpty { + return "teach-\(resolvedTaskSlug)" + } + + return "teach-\(slug(from: actionPhrase))" + } + static func buildMetadata(primaryQuestion: String, bundleId: String?) -> Metadata { let actionTokens = SkillMatcher.meaningfulTokens(primaryQuestion) let primaryActionToken = actionTokens.first ?? "task" + let resolvedTaskSlug = taskSlug(from: primaryQuestion) let appName = displayName(forBundleId: bundleId) - let appSlug = appName.map { slug(from: $0) } + let skillID = stableSkillId(bundleId: bundleId, primaryQuestion: primaryQuestion) - let skillID: String let skillName: String - if let appName, let appSlug { - skillID = "teach-\(appSlug)-\(slug(from: primaryActionToken))" + if let appName { skillName = "\(capitalizeWord(primaryActionToken)) in \(appName)" } else { let actionPhrase = actionTokens.prefix(3).joined(separator: " ") - skillID = "teach-\(slug(from: actionPhrase.isEmpty ? primaryActionToken : actionPhrase))" skillName = capitalizeWords(actionPhrase.isEmpty ? primaryActionToken : actionPhrase) } let skillDescription = descriptionFromQuestion(primaryQuestion) - return Metadata(id: skillID, name: skillName, description: skillDescription) + return Metadata(id: skillID, name: skillName, description: skillDescription, taskSlug: resolvedTaskSlug) } private static func descriptionFromQuestion(_ question: String) -> String { @@ -198,6 +243,8 @@ struct TeachingSkill: Identifiable, Equatable { let lastUsed = metadata["lastUsed"].flatMap { dateFormatter.date(from: $0) } let usageCount = Int(metadata["usageCount"] ?? "0") ?? 0 let isPinned = (metadata["pinned"] ?? "false").lowercased() == "true" + let taskSlug = metadata["taskSlug"]?.trimmingCharacters(in: .whitespacesAndNewlines) + let resolvedTaskSlug = (taskSlug?.isEmpty == false) ? taskSlug : nil return TeachingSkill( id: id, @@ -208,6 +255,7 @@ struct TeachingSkill: Identifiable, Equatable { lastUsed: lastUsed, usageCount: usageCount, isPinned: isPinned, + taskSlug: resolvedTaskSlug, body: body ) } diff --git a/leanring-buddy/leanring_buddyApp.swift b/leanring-buddy/leanring_buddyApp.swift index 362e5bf61..c0df95a6e 100644 --- a/leanring-buddy/leanring_buddyApp.swift +++ b/leanring-buddy/leanring_buddyApp.swift @@ -49,7 +49,11 @@ final class CompanionAppDelegate: NSObject, NSApplicationDelegate { companionManager.runE2EInjectSequenceIfNeeded() // Auto-open the panel if the user still needs to do something: // either they haven't onboarded yet, or permissions were revoked. - if !companionManager.hasCompletedOnboarding || !companionManager.allPermissionsGranted { + // Skip during headless E2E — the ad-hoc E2E build has a different code + // signature than the Xcode Debug app, so TCC checks (accessibility, etc.) + // will look "denied" even when the dev build is already approved. + if !ClickyE2EConfiguration.isEnabled, + !companionManager.hasCompletedOnboarding || !companionManager.allPermissionsGranted { menuBarPanelManager?.showPanelOnLaunch() } registerAsLoginItemIfNeeded() diff --git a/leanring-buddyTests/TeachingSkillTests.swift b/leanring-buddyTests/TeachingSkillTests.swift index c8037c581..85c4aaf37 100644 --- a/leanring-buddyTests/TeachingSkillTests.swift +++ b/leanring-buddyTests/TeachingSkillTests.swift @@ -19,6 +19,7 @@ struct TeachingSkillTests { lastUsed: 2026-05-24 usageCount: 3 pinned: true + taskSlug: commit --- step one: open source control. @@ -31,6 +32,7 @@ struct TeachingSkillTests { #expect(skill.bundleIds == ["com.apple.dt.Xcode"]) #expect(skill.usageCount == 3) #expect(skill.isPinned) + #expect(skill.taskSlug == "commit") #expect(skill.body.contains("step one")) } @@ -44,6 +46,7 @@ struct TeachingSkillTests { lastUsed: Date(), usageCount: 2, isPinned: false, + taskSlug: "save", body: "click file then save or use command s" ) @@ -76,6 +79,32 @@ struct TeachingSkillTests { #expect(trigger?.reason == .userConfirmed) } + @Test func doesNotWriteBeforeUserConfirmation() { + let trace = [ + SessionTraceEntry( + timestamp: Date(), + userTranscript: "how do I save this document?", + assistantResponse: "click file then save", + bundleId: "com.apple.TextEdit", + pointed: true + ), + SessionTraceEntry( + timestamp: Date(), + userTranscript: "where is the save button?", + assistantResponse: "pointing at file menu", + bundleId: "com.apple.TextEdit", + pointed: true + ) + ] + + let trigger = SkillTriggerEvaluator.shouldWriteSkill( + sessionTrace: trace, + latestTranscript: "where is the save button?" + ) + + #expect(trigger == nil) + } + @Test func topicIgnoresConfirmationPhrase() { let trace = [ SessionTraceEntry( @@ -99,7 +128,7 @@ struct TeachingSkillTests { #expect(SkillTriggerEvaluator.primaryTeachingQuestion(from: trace) == "how do I save this document?") } - @Test func slugAndNameAreCleanForSaveQuestionWithConfirmation() { + @Test func slugAndNameAreCleanForSaveQuestionWithConfirmation() throws { let trace = [ SessionTraceEntry( timestamp: Date(), @@ -124,18 +153,19 @@ struct TeachingSkillTests { let metadata = SkillSynthesizer.buildSkillMetadata( sessionTrace: trace, trigger: try #require(trigger), - bundleId: "com.apple.TextEdit" + targetBundleId: "com.apple.TextEdit" ) #expect(metadata.id == "teach-textedit-save") #expect(metadata.name == "Save in TextEdit") #expect(metadata.description == "Walk the user through save document") + #expect(metadata.taskSlug == "save") #expect(!metadata.id.contains("got")) #expect(!metadata.id.contains("thanks")) #expect(!metadata.id.contains("worked")) } - @Test func crossSessionRepeatDetectionWithin7Days() throws { + @Test func crossSessionRepeatDoesNotAutoWriteWithoutConfirmation() throws { let tempHistoryURL = FileManager.default.temporaryDirectory .appendingPathComponent("clicky-topic-history-test-\(UUID().uuidString).json") defer { try? FileManager.default.removeItem(at: tempHistoryURL) } @@ -166,8 +196,72 @@ struct TeachingSkillTests { topicHistory: topicHistoryStore.entries ) - #expect(trigger?.reason == .repeatedTopic) - #expect(trigger?.topic == "save document") + #expect(trigger == nil) + } + + @Test func resolvesTargetAppFromMentionedAppNotFrontmostBundle() { + let trace = [ + SessionTraceEntry( + timestamp: Date(), + userTranscript: "how do I save this document in TextEdit?", + assistantResponse: "use command s", + bundleId: "com.mitchellh.ghostty", + pointed: true + ) + ] + + let targetBundleId = SkillTargetAppResolver.resolveTargetBundleId( + from: trace, + frontmostBundleId: "com.mitchellh.ghostty" + ) + + #expect(targetBundleId == "com.apple.TextEdit") + } + + @Test func findSkillForUpdateUsesStableIdentity() { + let existingSkill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 2, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s" + ) + + let matchedSkill = SkillMatcher.findSkillForUpdate( + in: [existingSkill], + targetBundleId: "com.apple.TextEdit", + primaryQuestion: "how do I save this document?" + ) + + #expect(matchedSkill?.id == "teach-textedit-save") + } + + @Test func findSkillForUpdateMatchesRefinementToExistingSkill() { + let existingSkill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 2, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s" + ) + + let matchedSkill = SkillMatcher.findSkillForUpdate( + in: [existingSkill], + targetBundleId: "com.mitchellh.ghostty", + primaryQuestion: "how do I save this document in TextEdit?" + ) + + #expect(matchedSkill?.id == "teach-textedit-save") } @Test func promptBuilderInjectsMatchedSkills() { @@ -180,6 +274,7 @@ struct TeachingSkillTests { lastUsed: nil, usageCount: 0, isPinned: false, + taskSlug: "save", body: "use file > save" ) @@ -204,6 +299,7 @@ struct TeachingSkillTests { lastUsed: nil, usageCount: 0, isPinned: false, + taskSlug: "save", body: "click file then save or use command s" ) @@ -222,4 +318,108 @@ struct TeachingSkillTests { #expect(prompt.contains("Save in TextEdit")) #expect(prompt.contains("click file then save or use command s")) } + + @Test func detectsDuplicateSkillsWithOverlappingContent() { + let saveSkill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 3, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s shortcut" + ) + let duplicateSaveSkill = TeachingSkill( + id: "teach-textedit-save-document", + name: "Save document in TextEdit", + description: "Help save the current document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date().addingTimeInterval(-86400), + usageCount: 1, + isPinned: false, + taskSlug: "save", + body: "open file menu then choose save for the document" + ) + + let duplicatePairs = SkillMatcher.findDuplicateSkillPairs( + in: [saveSkill, duplicateSaveSkill], + minimumOverlapScore: 3 + ) + + #expect(duplicatePairs.count == 1) + #expect(duplicatePairs.first?.primarySkill.id == "teach-textedit-save") + #expect(duplicatePairs.first?.duplicateSkill.id == "teach-textedit-save-document") + } + + @Test func ignoresPinnedSkillsForDuplicateDetection() { + let pinnedSkill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 3, + isPinned: true, + taskSlug: "save", + body: "click file then save or use command s shortcut" + ) + let overlappingSkill = TeachingSkill( + id: "teach-textedit-save-copy", + name: "Save document in TextEdit", + description: "Help save the current document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 1, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s shortcut" + ) + + let duplicatePairs = SkillMatcher.findDuplicateSkillPairs( + in: [pinnedSkill, overlappingSkill], + minimumOverlapScore: 3 + ) + + #expect(duplicatePairs.isEmpty) + } + + @Test func ignoresSkillsWithDifferentBundleIdsForDuplicateDetection() { + let textEditSkill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 3, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s shortcut" + ) + let xcodeSkill = TeachingSkill( + id: "teach-xcode-save", + name: "Save in Xcode", + description: "Walk the user through saving a file", + bundleIds: ["com.apple.dt.Xcode"], + status: .active, + lastUsed: Date(), + usageCount: 1, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s shortcut" + ) + + let duplicatePairs = SkillMatcher.findDuplicateSkillPairs( + in: [textEditSkill, xcodeSkill], + minimumOverlapScore: 3 + ) + + #expect(duplicatePairs.isEmpty) + } } diff --git a/tests/e2e/mock-worker.mjs b/tests/e2e/mock-worker.mjs index e0825f37b..bebb0073e 100644 --- a/tests/e2e/mock-worker.mjs +++ b/tests/e2e/mock-worker.mjs @@ -29,19 +29,39 @@ function streamVoiceResponse(res) { res.end(); } -function synthesisResponse(res) { +function synthesisResponse(res, parsed) { + const messages = parsed.messages ?? []; + const lastMessage = messages.at(-1) ?? {}; + const content = lastMessage.content; + const promptText = typeof content === "string" + ? content + : Array.isArray(content) + ? content.map((part) => part.text ?? "").join("\n") + : ""; + const isPatch = promptText.includes("patch this existing teaching skill"); + + const text = isPatch + ? [ + "workflow: save a document in textedit", + "preference: keyboard shortcut only, avoid the file menu", + "step one: press command s to save", + "pointing: do not point at the file menu unless the user asks for menu steps", + "completion: user says got it or thanks that worked", + ].join("\n") + : [ + "workflow: save a document in textedit", + "step one: click file in the menu bar", + "step two: choose save or press command s", + "pointing: start at the file menu near the top left", + "common mistake: looking for a floppy disk icon instead of file > save", + "completion: user says got it or thanks that worked", + ].join("\n"); + jsonResponse(res, 200, { content: [ { type: "text", - text: [ - "workflow: save a document in textedit", - "step one: click file in the menu bar", - "step two: choose save or press command s", - "pointing: start at the file menu near the top left", - "common mistake: looking for a floppy disk icon instead of file > save", - "completion: user says got it or thanks that worked", - ].join("\n"), + text, }, ], }); @@ -82,7 +102,7 @@ const server = http.createServer(async (req, res) => { return; } - synthesisResponse(res); + synthesisResponse(res, parsed); return; } diff --git a/tests/e2e/teaching-skills.sh b/tests/e2e/teaching-skills.sh index abcdf1317..1c80c4d50 100755 --- a/tests/e2e/teaching-skills.sh +++ b/tests/e2e/teaching-skills.sh @@ -1,59 +1,29 @@ #!/usr/bin/env bash set -euo pipefail -ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" -CLICKY_APP="${CLICKY_APP:-$ROOT_DIR/build/E2E/Clicky.app}" -WORKER_URL="${CLICKY_WORKER_URL:-http://127.0.0.1:8787}" -SKILLS_DIR="$HOME/.clicky/skills" -PROMPT_FILE="$HOME/.clicky/e2e-last-system-prompt.txt" -MOCK_WORKER_PID="" -CLICKY_PID="" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/common.sh +source "$SCRIPT_DIR/lib/common.sh" -cleanup() { - if [[ -n "$MOCK_WORKER_PID" ]]; then - kill "$MOCK_WORKER_PID" 2>/dev/null || true - fi - if [[ -n "$CLICKY_PID" ]]; then - kill "$CLICKY_PID" 2>/dev/null || true - fi -} -trap cleanup EXIT - -echo "Building Clicky for E2E..." -mkdir -p "$ROOT_DIR/build/E2E" -xcodebuild \ - -project "$ROOT_DIR/leanring-buddy.xcodeproj" \ - -scheme leanring-buddy \ - -destination 'platform=macOS' \ - -derivedDataPath "$ROOT_DIR/build/E2E/DerivedData" \ - CODE_SIGN_IDENTITY="-" \ - CODE_SIGNING_ALLOWED=NO \ - build >/tmp/clicky-e2e-build.log 2>&1 - -BUILT_APP="$ROOT_DIR/build/E2E/DerivedData/Build/Products/Debug/Clicky.app" -rm -rf "$CLICKY_APP" -ditto "$BUILT_APP" "$CLICKY_APP" - -echo "Starting mock worker on $WORKER_URL..." -node "$ROOT_DIR/tests/e2e/mock-worker.mjs" >/tmp/clicky-e2e-worker.log 2>&1 & -MOCK_WORKER_PID=$! -sleep 1 +trap e2e_cleanup EXIT + +ensure_clicky_built +start_mock_worker echo "Resetting prior teaching skills..." rm -rf "$SKILLS_DIR" mkdir -p "$SKILLS_DIR" -rm -f "$PROMPT_FILE" +rm -f "$E2E_PROMPT_FILE" echo "Phase A: launch Clicky and teach a save workflow..." -"$CLICKY_APP/Contents/MacOS/Clicky" \ +launch_clicky /tmp/clicky-e2e-app.log \ -CLICKY_E2E=1 \ -CLICKY_WORKER_URL="$WORKER_URL" \ - -CLICKY_INJECT_TRANSCRIPT="how do I save this document?" \ - -CLICKY_INJECT_TRANSCRIPT_2="got it thanks that worked" >/tmp/clicky-e2e-app.log 2>&1 & -CLICKY_PID=$! + -CLICKY_INJECT_TRANSCRIPT="how do I save this document in TextEdit?" \ + -CLICKY_INJECT_TRANSCRIPT_2="got it thanks that worked" SKILL_FILE="" -for _ in $(seq 1 30); do +for _ in $(seq 1 90); do if compgen -G "$SKILLS_DIR/*/SKILL.md" >/dev/null; then SKILL_FILE="$(ls "$SKILLS_DIR"/*/SKILL.md | head -1)" break @@ -62,7 +32,7 @@ for _ in $(seq 1 30); do done if [[ -z "$SKILL_FILE" ]]; then - echo "FAIL: no teaching skill written within 30s" + echo "FAIL: no teaching skill written within 90s" echo "--- app log ---" tail -40 /tmp/clicky-e2e-app.log || true echo "--- worker log ---" @@ -75,8 +45,8 @@ echo "PASS: teaching skill written to $SKILL_FILE" echo "--- skill preview ---" head -20 "$SKILL_FILE" -if [[ "$SKILL_ID" != *save* ]]; then - echo "FAIL: skill slug '$SKILL_ID' does not contain 'save'" +if [[ "$SKILL_ID" != "teach-textedit-save" ]]; then + echo "FAIL: expected skill id teach-textedit-save, got '$SKILL_ID'" exit 1 fi @@ -90,36 +60,81 @@ echo "PASS: skill slug is clean ($SKILL_ID)" kill "$CLICKY_PID" 2>/dev/null || true CLICKY_PID="" sleep 1 -rm -f "$PROMPT_FILE" +rm -f "$E2E_PROMPT_FILE" echo "Phase B: relaunch Clicky and verify saved skill is injected into prompt..." -"$CLICKY_APP/Contents/MacOS/Clicky" \ +launch_clicky /tmp/clicky-e2e-app-read.log \ -CLICKY_E2E=1 \ -CLICKY_WORKER_URL="$WORKER_URL" \ - -CLICKY_INJECT_TRANSCRIPT_3="how do I save this document?" >/tmp/clicky-e2e-app-read.log 2>&1 & -CLICKY_PID=$! + -CLICKY_INJECT_TRANSCRIPT_3="how do I save this document in TextEdit?" -for _ in $(seq 1 30); do - if [[ -f "$PROMPT_FILE" ]] && grep -q "teaching skills:" "$PROMPT_FILE"; then - if grep -qi "save" "$PROMPT_FILE"; then +for _ in $(seq 1 90); do + if [[ -f "$E2E_PROMPT_FILE" ]] && grep -q "teaching skills:" "$E2E_PROMPT_FILE"; then + if grep -qi "save" "$E2E_PROMPT_FILE"; then echo "PASS: saved skill content found in composed system prompt" echo "--- prompt preview ---" - grep -A 8 "teaching skills:" "$PROMPT_FILE" | head -12 - echo "" - echo "E2E PASS: Phase A (write) + Phase B (read-path) succeeded" - exit 0 + grep -A 8 "teaching skills:" "$E2E_PROMPT_FILE" | head -12 + break fi fi sleep 1 done -echo "FAIL: composed system prompt did not include saved skill content within 30s" -echo "--- app log ---" -tail -40 /tmp/clicky-e2e-app-read.log || true -if [[ -f "$PROMPT_FILE" ]]; then - echo "--- prompt file ---" - head -40 "$PROMPT_FILE" || true -else - echo "prompt file missing: $PROMPT_FILE" +if [[ ! -f "$E2E_PROMPT_FILE" ]] || ! grep -q "teaching skills:" "$E2E_PROMPT_FILE"; then + echo "FAIL: composed system prompt did not include saved skill content within 90s" + echo "--- app log ---" + tail -40 /tmp/clicky-e2e-app-read.log || true + if [[ -f "$E2E_PROMPT_FILE" ]]; then + echo "--- prompt file ---" + head -40 "$E2E_PROMPT_FILE" || true + else + echo "prompt file missing: $E2E_PROMPT_FILE" + fi + exit 1 +fi + +kill "$CLICKY_PID" 2>/dev/null || true +CLICKY_PID="" +sleep 1 + +SKILL_COUNT_BEFORE_PATCH="$(find "$SKILLS_DIR" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" + +echo "Phase C: patch existing skill after refinement + confirmation..." +launch_clicky /tmp/clicky-e2e-app-patch.log \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL="$WORKER_URL" \ + -CLICKY_INJECT_TRANSCRIPT="how do I save this document in TextEdit?" \ + -CLICKY_INJECT_TRANSCRIPT_2="only use the keyboard shortcut not the file menu" \ + -CLICKY_INJECT_TRANSCRIPT_3="got it thanks that worked" + +for _ in $(seq 1 90); do + if [[ -f "$SKILL_FILE" ]] && grep -qi "keyboard shortcut only\|avoid the file menu" "$SKILL_FILE"; then + break + fi + sleep 1 +done + +SKILL_COUNT_AFTER_PATCH="$(find "$SKILLS_DIR" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" + +if [[ "$SKILL_COUNT_AFTER_PATCH" != "$SKILL_COUNT_BEFORE_PATCH" ]]; then + echo "FAIL: expected $SKILL_COUNT_BEFORE_PATCH skill folder(s) after patch, found $SKILL_COUNT_AFTER_PATCH" + ls -la "$SKILLS_DIR" || true + exit 1 +fi + +if [[ "$(basename "$(dirname "$SKILL_FILE")")" != "teach-textedit-save" ]]; then + echo "FAIL: expected patched skill id teach-textedit-save, got $(basename "$(dirname "$SKILL_FILE")")" + exit 1 fi -exit 1 + +if ! grep -qi "keyboard shortcut only\|avoid the file menu" "$SKILL_FILE"; then + echo "FAIL: patched skill body did not include keyboard-only refinement" + echo "--- skill body ---" + tail -20 "$SKILL_FILE" || true + exit 1 +fi + +echo "PASS: existing skill patched in place ($SKILL_ID)" +echo "" +echo "E2E PASS: Phase A (write) + Phase B (read-path) + Phase C (patch) succeeded" +exit 0 From 534c11271855bbe202203acb34ed56493cd55c08 Mon Sep 17 00:00:00 2001 From: octavi42 Date: Wed, 27 May 2026 17:52:36 +0300 Subject: [PATCH 05/14] Align push-to-talk and worker proxy with main fork. Input Monitoring gates the CGEvent tap instead of Accessibility, and ClickyWorkerBaseURL routes Claude, TTS, and AssemblyAI through the Cloudflare Worker so manual testing matches main. --- leanring-buddy/AppBundleConfiguration.swift | 12 ++ ...mblyAIStreamingTranscriptionProvider.swift | 4 +- leanring-buddy/CompanionManager.swift | 31 +++-- leanring-buddy/CompanionPanelView.swift | 106 +++++++++++++++++- .../GlobalPushToTalkShortcutMonitor.swift | 47 +++++++- leanring-buddy/Info.plist | 4 + leanring-buddy/WindowPositionManager.swift | 36 ++++++ 7 files changed, 221 insertions(+), 19 deletions(-) diff --git a/leanring-buddy/AppBundleConfiguration.swift b/leanring-buddy/AppBundleConfiguration.swift index 6fd233757..c7671af04 100644 --- a/leanring-buddy/AppBundleConfiguration.swift +++ b/leanring-buddy/AppBundleConfiguration.swift @@ -8,6 +8,18 @@ import Foundation enum AppBundleConfiguration { + private static let placeholderWorkerBaseURL = "https://your-worker-name.your-subdomain.workers.dev" + + /// Cloudflare Worker base URL for Claude, TTS, and AssemblyAI token routes. + /// Set `ClickyWorkerBaseURL` in Info.plist (no trailing slash). + static var workerBaseURL: String { + stringValue(forKey: "ClickyWorkerBaseURL") ?? placeholderWorkerBaseURL + } + + static var isWorkerBaseURLConfigured: Bool { + workerBaseURL != placeholderWorkerBaseURL + } + static func stringValue(forKey key: String) -> String? { if let value = Bundle.main.object(forInfoDictionaryKey: key) as? String { let trimmedValue = value.trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/leanring-buddy/AssemblyAIStreamingTranscriptionProvider.swift b/leanring-buddy/AssemblyAIStreamingTranscriptionProvider.swift index d21286b65..bab81a1a2 100644 --- a/leanring-buddy/AssemblyAIStreamingTranscriptionProvider.swift +++ b/leanring-buddy/AssemblyAIStreamingTranscriptionProvider.swift @@ -19,7 +19,9 @@ struct AssemblyAIStreamingTranscriptionProviderError: LocalizedError { final class AssemblyAIStreamingTranscriptionProvider: BuddyTranscriptionProvider { /// URL for the Cloudflare Worker endpoint that returns a short-lived /// AssemblyAI streaming token. The real API key never leaves the server. - private static let tokenProxyURL = "https://your-worker-name.your-subdomain.workers.dev/transcribe-token" + private static var tokenProxyURL: String { + "\(AppBundleConfiguration.workerBaseURL)/transcribe-token" + } let displayName = "AssemblyAI" let requiresSpeechRecognitionPermission = false diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index 1a581cabb..cbec4416b 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -28,6 +28,7 @@ final class CompanionManager: ObservableObject { @Published private(set) var lastTranscript: String? @Published private(set) var currentAudioPowerLevel: CGFloat = 0 @Published private(set) var hasAccessibilityPermission = false + @Published private(set) var hasInputMonitoringPermission = false @Published private(set) var hasScreenRecordingPermission = false @Published private(set) var hasMicrophonePermission = false @Published private(set) var hasScreenContentPermission = false @@ -72,7 +73,12 @@ final class CompanionManager: ObservableObject { /// Base URL for the Cloudflare Worker proxy. All API requests route /// through this so keys never ship in the app binary. private static var workerBaseURL: String { - ClickyE2EConfiguration.workerBaseURL ?? "https://your-worker-name.your-subdomain.workers.dev" + ClickyE2EConfiguration.workerBaseURL ?? AppBundleConfiguration.workerBaseURL + } + + /// True when the global CGEvent tap is active and can detect Control+Option. + var isPushToTalkHotkeyActive: Bool { + hasInputMonitoringPermission && globalPushToTalkShortcutMonitor.isEventTapInstalled } private let teachingSkillStore = TeachingSkillStore() @@ -118,10 +124,13 @@ final class CompanionManager: ObservableObject { /// speaks again before the delay elapses. private var transientHideTask: Task? - /// True when all three required permissions (accessibility, screen recording, - /// microphone) are granted. Used by the panel to show a single "all good" state. + /// True when all required permissions are granted. Used by the panel to show a single "all good" state. var allPermissionsGranted: Bool { - hasAccessibilityPermission && hasScreenRecordingPermission && hasMicrophonePermission && hasScreenContentPermission + hasAccessibilityPermission + && hasInputMonitoringPermission + && hasScreenRecordingPermission + && hasMicrophonePermission + && hasScreenContentPermission } /// Whether the blue cursor overlay is currently visible on screen. @@ -390,7 +399,7 @@ final class CompanionManager: ObservableObject { func start() { bootstrapTeachingSkills() refreshAllPermissions() - print("🔑 Clicky start — accessibility: \(hasAccessibilityPermission), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission), onboarded: \(hasCompletedOnboarding)") + print("🔑 Clicky start — accessibility: \(hasAccessibilityPermission), inputMonitoring: \(hasInputMonitoringPermission), pushToTalkActive: \(isPushToTalkHotkeyActive), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission), onboarded: \(hasCompletedOnboarding)") startPermissionPolling() bindVoiceStateObservation() bindAudioPowerLevel() @@ -522,6 +531,7 @@ final class CompanionManager: ObservableObject { func refreshAllPermissions() { let previouslyHadAccessibility = hasAccessibilityPermission + let previouslyHadInputMonitoring = hasInputMonitoringPermission let previouslyHadScreenRecording = hasScreenRecordingPermission let previouslyHadMicrophone = hasMicrophonePermission let previouslyHadAll = allPermissionsGranted @@ -529,7 +539,10 @@ final class CompanionManager: ObservableObject { let currentlyHasAccessibility = WindowPositionManager.hasAccessibilityPermission() hasAccessibilityPermission = currentlyHasAccessibility - if currentlyHasAccessibility { + let currentlyHasInputMonitoring = WindowPositionManager.hasInputMonitoringPermission() + hasInputMonitoringPermission = currentlyHasInputMonitoring + + if currentlyHasInputMonitoring { globalPushToTalkShortcutMonitor.start() } else { globalPushToTalkShortcutMonitor.stop() @@ -542,15 +555,19 @@ final class CompanionManager: ObservableObject { // Debug: log permission state on changes if previouslyHadAccessibility != hasAccessibilityPermission + || previouslyHadInputMonitoring != hasInputMonitoringPermission || previouslyHadScreenRecording != hasScreenRecordingPermission || previouslyHadMicrophone != hasMicrophonePermission { - print("🔑 Permissions — accessibility: \(hasAccessibilityPermission), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission)") + print("🔑 Permissions — accessibility: \(hasAccessibilityPermission), inputMonitoring: \(hasInputMonitoringPermission), pushToTalkActive: \(isPushToTalkHotkeyActive), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission)") } // Track individual permission grants as they happen if !previouslyHadAccessibility && hasAccessibilityPermission { ClickyAnalytics.trackPermissionGranted(permission: "accessibility") } + if !previouslyHadInputMonitoring && hasInputMonitoringPermission { + ClickyAnalytics.trackPermissionGranted(permission: "input_monitoring") + } if !previouslyHadScreenRecording && hasScreenRecordingPermission { ClickyAnalytics.trackPermissionGranted(permission: "screen_recording") } diff --git a/leanring-buddy/CompanionPanelView.swift b/leanring-buddy/CompanionPanelView.swift index a9824f9da..fdd80bbd7 100644 --- a/leanring-buddy/CompanionPanelView.swift +++ b/leanring-buddy/CompanionPanelView.swift @@ -135,10 +135,31 @@ struct CompanionPanelView: View { @ViewBuilder private var permissionsCopySection: some View { if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { - Text("Hold Control+Option to talk.") - .font(.system(size: 12, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) - .frame(maxWidth: .infinity, alignment: .leading) + VStack(alignment: .leading, spacing: 6) { + Text("Hold ^ Control + ⌥ Option to talk.") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + + Text("Use Control (not ⌘ Command). Speak while holding, then release.") + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textTertiary) + .fixedSize(horizontal: false, vertical: true) + + if !companionManager.isPushToTalkHotkeyActive { + Text("Push-to-talk is inactive. Enable Input Monitoring for Clicky, then quit and reopen the app.") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.warning) + .fixedSize(horizontal: false, vertical: true) + } + + if !AppBundleConfiguration.isWorkerBaseURLConfigured { + Text("API proxy is not configured. Set ClickyWorkerBaseURL in Info.plist to your Cloudflare Worker URL.") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.warning) + .fixedSize(horizontal: false, vertical: true) + } + } + .frame(maxWidth: .infinity, alignment: .leading) } else if companionManager.allPermissionsGranted && !companionManager.hasSubmittedEmail { VStack(alignment: .leading, spacing: 4) { Text("Drop your email to get started.") @@ -161,7 +182,7 @@ struct CompanionPanelView: View { .font(.system(size: 12, weight: .bold)) .foregroundColor(DS.Colors.textSecondary) - Text("Some permissions were revoked. Grant all four below to keep using Clicky.") + Text("Some permissions were revoked. Grant all five below to keep using Clicky.") .font(.system(size: 11)) .foregroundColor(DS.Colors.textTertiary) .fixedSize(horizontal: false, vertical: true) @@ -262,6 +283,8 @@ struct CompanionPanelView: View { accessibilityPermissionRow + inputMonitoringPermissionRow + screenRecordingPermissionRow if companionManager.hasScreenRecordingPermission { @@ -341,6 +364,79 @@ struct CompanionPanelView: View { .padding(.vertical, 6) } + private var inputMonitoringPermissionRow: some View { + let isGranted = companionManager.hasInputMonitoringPermission + return HStack { + HStack(spacing: 8) { + Image(systemName: "keyboard") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(isGranted ? DS.Colors.textTertiary : DS.Colors.warning) + .frame(width: 16) + + VStack(alignment: .leading, spacing: 1) { + Text("Input Monitoring") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + + Text(isGranted + ? "Required for Control+Option push-to-talk" + : "Enable this for the global hotkey to work") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + } + } + + Spacer() + + if isGranted { + HStack(spacing: 4) { + Circle() + .fill(DS.Colors.success) + .frame(width: 6, height: 6) + Text("Granted") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.success) + } + } else { + HStack(spacing: 6) { + Button(action: { + WindowPositionManager.requestInputMonitoringPermission() + }) { + Text("Grant") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(DS.Colors.textOnAccent) + .padding(.horizontal, 10) + .padding(.vertical, 4) + .background( + Capsule() + .fill(DS.Colors.accent) + ) + } + .buttonStyle(.plain) + .pointerCursor() + + Button(action: { + WindowPositionManager.revealAppInFinder() + WindowPositionManager.openInputMonitoringSettings() + }) { + Text("Find App") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(DS.Colors.textSecondary) + .padding(.horizontal, 10) + .padding(.vertical, 4) + .background( + Capsule() + .stroke(DS.Colors.borderSubtle, lineWidth: 0.8) + ) + } + .buttonStyle(.plain) + .pointerCursor() + } + } + } + .padding(.vertical, 6) + } + private var screenRecordingPermissionRow: some View { let isGranted = companionManager.hasScreenRecordingPermission return HStack { diff --git a/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift b/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift index 8020269b4..50b87808d 100644 --- a/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift +++ b/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift @@ -11,12 +11,21 @@ import AppKit import Combine import CoreGraphics import Foundation +import os final class GlobalPushToTalkShortcutMonitor: ObservableObject { + private static let logger = Logger( + subsystem: Bundle.main.bundleIdentifier ?? "com.yourcompany.leanring-buddy", + category: "PushToTalk" + ) + let shortcutTransitionPublisher = PassthroughSubject() private var globalEventTap: CFMachPort? private var globalEventTapRunLoopSource: CFRunLoopSource? + /// True once a CGEvent tap is installed and enabled for this process. + @Published private(set) var isEventTapInstalled = false + /// Mutated exclusively from the CGEvent tap callback, which runs on /// `CFRunLoopGetMain()` and therefore always executes on the main thread. /// Published so the overlay can hide immediately on key release without @@ -28,11 +37,21 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { } func start() { - // If the event tap is already running, don't restart it. - // Restarting resets isShortcutCurrentlyPressed, which would kill - // the waveform overlay mid-press when the permission poller calls - // refreshAllPermissions → start() every few seconds. - guard globalEventTap == nil else { return } + guard WindowPositionManager.hasInputMonitoringPermission() else { + Self.logger.error("Input Monitoring permission missing — push-to-talk hotkey inactive") + stop() + return + } + + if let globalEventTap, !CGEvent.tapIsEnabled(tap: globalEventTap) { + Self.logger.notice("Recreating disabled CGEvent tap") + stop() + } + + guard globalEventTap == nil else { + isEventTapInstalled = true + return + } let monitoredEventTypes: [CGEventType] = [.flagsChanged, .keyDown, .keyUp] let eventMask = monitoredEventTypes.reduce(CGEventMask(0)) { currentMask, eventType in @@ -62,7 +81,14 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { callback: eventTapCallback, userInfo: Unmanaged.passUnretained(self).toOpaque() ) else { - print("⚠️ Global push-to-talk: couldn't create CGEvent tap") + if !CGPreflightListenEventAccess() { + Self.logger.error("Input Monitoring permission required for push-to-talk") + print("⚠️ Global push-to-talk: Input Monitoring permission is required (System Settings → Privacy & Security → Input Monitoring)") + } else { + Self.logger.error("CGEvent tapCreate returned nil despite Input Monitoring being granted") + print("⚠️ Global push-to-talk: couldn't create CGEvent tap — quit and reopen Clicky") + } + isEventTapInstalled = false return } @@ -72,7 +98,9 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { 0 ) else { CFMachPortInvalidate(globalEventTap) + Self.logger.error("Couldn't create CGEvent tap run loop source") print("⚠️ Global push-to-talk: couldn't create event tap run loop source") + isEventTapInstalled = false return } @@ -81,10 +109,13 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { CFRunLoopAddSource(CFRunLoopGetMain(), globalEventTapRunLoopSource, .commonModes) CGEvent.tapEnable(tap: globalEventTap, enable: true) + isEventTapInstalled = CGEvent.tapIsEnabled(tap: globalEventTap) + Self.logger.notice("CGEvent tap installed (enabled=\(self.isEventTapInstalled, privacy: .public))") } func stop() { isShortcutCurrentlyPressed = false + isEventTapInstalled = false if let globalEventTapRunLoopSource { CFRunLoopRemoveSource(CFRunLoopGetMain(), globalEventTapRunLoopSource, .commonModes) @@ -102,8 +133,10 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { event: CGEvent ) -> Unmanaged? { if eventType == .tapDisabledByTimeout || eventType == .tapDisabledByUserInput { + Self.logger.notice("CGEvent tap disabled by system — re-enabling") if let globalEventTap { CGEvent.tapEnable(tap: globalEventTap, enable: true) + isEventTapInstalled = CGEvent.tapIsEnabled(tap: globalEventTap) } return Unmanaged.passUnretained(event) } @@ -120,9 +153,11 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { case .none: break case .pressed: + Self.logger.notice("Push-to-talk shortcut pressed") isShortcutCurrentlyPressed = true shortcutTransitionPublisher.send(.pressed) case .released: + Self.logger.notice("Push-to-talk shortcut released") isShortcutCurrentlyPressed = false shortcutTransitionPublisher.send(.released) } diff --git a/leanring-buddy/Info.plist b/leanring-buddy/Info.plist index e3d2b455a..578241346 100644 --- a/leanring-buddy/Info.plist +++ b/leanring-buddy/Info.plist @@ -2,6 +2,8 @@ + ClickyWorkerBaseURL + https://clicky-proxy.c-z-tavi.workers.dev LSUIElement SUFeedURL @@ -10,6 +12,8 @@ /l3d2rw5ZZFRU3AadP/w2Zf8FHfhA6bKv16BQOV5OSk= VoiceTranscriptionProvider assemblyai + NSAccessibilityUsageDescription + Clicky needs Accessibility access to help position windows and point at on-screen elements. NSMicrophoneUsageDescription Clicky uses your microphone so you can talk to it NSScreenCaptureUsageDescription diff --git a/leanring-buddy/WindowPositionManager.swift b/leanring-buddy/WindowPositionManager.swift index 8f6edf650..4af437ca8 100644 --- a/leanring-buddy/WindowPositionManager.swift +++ b/leanring-buddy/WindowPositionManager.swift @@ -8,6 +8,7 @@ import AppKit import ApplicationServices +import CoreGraphics import ScreenCaptureKit enum PermissionRequestPresentationDestination: Equatable { @@ -19,6 +20,7 @@ enum PermissionRequestPresentationDestination: Equatable { @MainActor class WindowPositionManager { private static var hasAttemptedAccessibilitySystemPromptDuringCurrentLaunch = false + private static var hasAttemptedInputMonitoringSystemPromptDuringCurrentLaunch = false private static var hasAttemptedScreenRecordingSystemPromptDuringCurrentLaunch = false private static let hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey = "com.learningbuddy.hasPreviouslyConfirmedScreenRecordingPermission" @@ -73,6 +75,40 @@ class WindowPositionManager { NSWorkspace.shared.activateFileViewerSelecting([appURL]) } + // MARK: - Input Monitoring Permission + + /// Returns true if the app can install the global CGEvent tap used for push-to-talk. + static func hasInputMonitoringPermission() -> Bool { + CGPreflightListenEventAccess() + } + + /// Presents the system Input Monitoring prompt once, then opens System Settings. + @discardableResult + static func requestInputMonitoringPermission() -> PermissionRequestPresentationDestination { + let presentationDestination = permissionRequestPresentationDestination( + hasPermissionNow: hasInputMonitoringPermission(), + hasAttemptedSystemPrompt: hasAttemptedInputMonitoringSystemPromptDuringCurrentLaunch + ) + + switch presentationDestination { + case .alreadyGranted: + return .alreadyGranted + case .systemPrompt: + hasAttemptedInputMonitoringSystemPromptDuringCurrentLaunch = true + _ = CGRequestListenEventAccess() + case .systemSettings: + openInputMonitoringSettings() + } + + return presentationDestination + } + + /// Opens System Settings to the Input Monitoring pane. + static func openInputMonitoringSettings() { + guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent") else { return } + NSWorkspace.shared.open(url) + } + // MARK: - Screen Recording Permission /// Returns true if Screen Recording permission is granted. From 82569c0cdb09f47be3b2ac1393c475d0aa04695a Mon Sep 17 00:00:00 2001 From: octavi42 Date: Fri, 29 May 2026 17:29:09 +0300 Subject: [PATCH 06/14] Add teaching skills library UI, curator v2, and E2E harness polish. Ship the full skills library view, LLM curator passes, shared E2E helpers, skills-library regression script, and teaching-only launch hooks without niche or vault dependencies. --- .github/workflows/e2e-teaching-skills.yml | 6 +- AGENTS.md | 20 +- leanring-buddy.xcodeproj/project.pbxproj | 8 +- .../xcschemes/leanring-buddy.xcscheme | 91 ++++++ leanring-buddy/ClickyAnalytics.swift | 29 ++ leanring-buddy/ClickyE2EConfiguration.swift | 65 ++++- leanring-buddy/CompanionManager.swift | 51 ++++ leanring-buddy/CompanionPanelView.swift | 25 ++ leanring-buddy/SkillCurator.swift | 138 +++++++++ leanring-buddy/SkillMatcher.swift | 74 +++++ leanring-buddy/TeachingSkillStore.swift | 12 + .../TeachingSkillsLibraryView.swift | 264 ++++++++++++++++++ leanring-buddy/leanring_buddyApp.swift | 1 + tests/e2e/lib/common.sh | 155 ++++++++++ tests/e2e/run-all.sh | 40 ++- tests/e2e/skills-library.sh | 97 +++++++ 16 files changed, 1062 insertions(+), 14 deletions(-) create mode 100644 leanring-buddy.xcodeproj/xcshareddata/xcschemes/leanring-buddy.xcscheme create mode 100644 leanring-buddy/TeachingSkillsLibraryView.swift create mode 100755 tests/e2e/lib/common.sh create mode 100755 tests/e2e/skills-library.sh diff --git a/.github/workflows/e2e-teaching-skills.yml b/.github/workflows/e2e-teaching-skills.yml index cb010c540..4b64a3b26 100644 --- a/.github/workflows/e2e-teaching-skills.yml +++ b/.github/workflows/e2e-teaching-skills.yml @@ -5,15 +5,17 @@ on: branches: - main - feature/teaching-skills + - feature/teaching-skills-upstream pull_request: branches: - main - feature/teaching-skills + - feature/teaching-skills-upstream jobs: teaching-skills-e2e: name: Teaching skills E2E (macOS) - runs-on: macos-14 + runs-on: macos-15 timeout-minutes: 30 steps: @@ -27,7 +29,7 @@ jobs: - name: Run teaching-skills E2E run: | - chmod +x tests/e2e/run-all.sh tests/e2e/teaching-skills.sh + chmod +x tests/e2e/run-all.sh tests/e2e/teaching-skills.sh tests/e2e/skills-library.sh tests/e2e/lib/common.sh ./tests/e2e/run-all.sh - name: Upload E2E logs on failure diff --git a/AGENTS.md b/AGENTS.md index cd9961463..5800ab181 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,9 +53,9 @@ Worker vars: `ELEVENLABS_VOICE_ID` | File | Lines | Purpose | |------|-------|---------| | `leanring_buddyApp.swift` | ~89 | Menu bar app entry point. Uses `@NSApplicationDelegateAdaptor` with `CompanionAppDelegate` which creates `MenuBarPanelManager` and starts `CompanionManager`. No main window — the app lives entirely in the status bar. | -| `CompanionManager.swift` | ~1026 | Central state machine. Owns dictation, shortcut monitoring, screen capture, Claude API, ElevenLabs TTS, and overlay management. Tracks voice state (idle/listening/processing/responding), conversation history, model selection, and cursor visibility. Coordinates the full push-to-talk → screenshot → Claude → TTS → pointing pipeline. | +| `CompanionManager.swift` | ~1070 | Central state machine. Owns dictation, shortcut monitoring, screen capture, Claude API, ElevenLabs TTS, overlay management, teaching skills, and niche discovery. Tracks voice state (idle/listening/processing/responding), conversation history, model selection, and cursor visibility. Coordinates the full push-to-talk → screenshot → Claude → TTS → pointing pipeline. | | `MenuBarPanelManager.swift` | ~243 | NSStatusItem + custom NSPanel lifecycle. Creates the menu bar icon, manages the floating companion panel (show/hide/position), installs click-outside-to-dismiss monitor. | -| `CompanionPanelView.swift` | ~761 | SwiftUI panel content for the menu bar dropdown. Shows companion status, push-to-talk instructions, model picker (Sonnet/Opus), permissions UI, DM feedback button, and quit button. Dark aesthetic using `DS` design system. | +| `CompanionPanelView.swift` | ~850 | SwiftUI panel content for the menu bar dropdown. Shows companion status, push-to-talk instructions, niche onboarding/suggestions, teaching skills UI, model picker (Sonnet/Opus), permissions UI, DM feedback button, and quit button. Dark aesthetic using `DS` design system. | | `OverlayWindow.swift` | ~881 | Full-screen transparent overlay hosting the blue cursor, response text, waveform, and spinner. Handles cursor animation, element pointing with bezier arcs, multi-monitor coordinate mapping, and fade-out transitions. | | `CompanionResponseOverlay.swift` | ~217 | SwiftUI view for the response text bubble and waveform displayed next to the cursor in the overlay. | | `CompanionScreenCaptureUtility.swift` | ~132 | Multi-monitor screenshot capture using ScreenCaptureKit. Returns labeled image data for each connected display. | @@ -71,10 +71,24 @@ Worker vars: `ELEVENLABS_VOICE_ID` | `ElevenLabsTTSClient.swift` | ~81 | ElevenLabs TTS client. Sends text to the Worker proxy, plays back audio via `AVAudioPlayer`. Exposes `isPlaying` for transient cursor scheduling. | | `ElementLocationDetector.swift` | ~335 | Detects UI element locations in screenshots for cursor pointing. | | `DesignSystem.swift` | ~880 | Design system tokens — colors, corner radii, shared styles. All UI references `DS.Colors`, `DS.CornerRadius`, etc. | -| `ClickyAnalytics.swift` | ~121 | PostHog analytics integration for usage tracking. | +| `ClickyAnalytics.swift` | ~175 | PostHog analytics integration for usage tracking. | | `WindowPositionManager.swift` | ~262 | Window placement logic, Screen Recording permission flow, and accessibility permission helpers. | | `AppBundleConfiguration.swift` | ~28 | Runtime configuration reader for keys stored in the app bundle Info.plist. | | `TeachingTopicHistoryStore.swift` | ~95 | Persists teaching topic tokens, bundle IDs, and timestamps to `~/.clicky/topic-history.json` for cross-session repeat detection. | +| `TeachingSkillStore.swift` | ~100 | Create/read/update/delete teaching skills at `~/.clicky/skills//SKILL.md`. | +| `TeachingSkill.swift` | ~280 | Model and YAML frontmatter parsing for teaching skills. | +| `SkillMatcher.swift` | ~170 | App/topic matching and duplicate skill pair detection. | +| `SkillCurator.swift` | ~175 | Time-based stale/archive lifecycle plus throttled LLM merge/patch passes. | +| `SkillSynthesizer.swift` | ~120 | Post-session skill drafting via Claude API. | +| `SkillTriggerEvaluator.swift` | ~120 | Decides when to create or update a teaching skill from session traces. | +| `TeachingPromptBuilder.swift` | ~35 | Composes voice response system prompt with matched teaching skills. | +| `TeachingSkillsLibraryView.swift` | ~260 | Full skills library UI with status filters, detail view, pin/delete/restore. | +| `ClickyE2EConfiguration.swift` | ~75 | Launch flags and E2E debug artifact paths for automated tests. | +| `NicheDiscoveryManager.swift` | ~175 | Niche onboarding, static/app-aware suggestion cards, local override loading. | +| `UserNiche.swift` | ~35 | User niche enum (general, content creator, developer, student, designer). | +| `Resources/niche-examples.json` | ~80 | Bundled niche suggestion packs and app-specific prompt maps. | +| `leanring-buddyTests/TeachingSkillTests.swift` | ~320 | Unit tests for skill parsing, matching, triggers, prompt injection, duplicate detection. | +| `tests/e2e/full-stack/` | — | Scaffold for full user-perspective E2E (BlackHole, PTT simulation, Peekaboo). | | `worker/src/index.ts` | ~142 | Cloudflare Worker proxy. Three routes: `/chat` (Claude), `/tts` (ElevenLabs), `/transcribe-token` (AssemblyAI temp token). | ## Build & Run diff --git a/leanring-buddy.xcodeproj/project.pbxproj b/leanring-buddy.xcodeproj/project.pbxproj index 75e572618..56a30bac5 100644 --- a/leanring-buddy.xcodeproj/project.pbxproj +++ b/leanring-buddy.xcodeproj/project.pbxproj @@ -416,6 +416,7 @@ ENABLE_HARDENED_RUNTIME = YES; ENABLE_OUTGOING_NETWORK_CONNECTIONS = YES; ENABLE_PREVIEWS = YES; + ENABLE_TESTABILITY = YES; ENABLE_USER_SELECTED_FILES = readonly; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_FILE = "leanring-buddy/Info.plist"; @@ -429,6 +430,7 @@ ); MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = "com.yourcompany.leanring-buddy"; + PRODUCT_MODULE_NAME = leanring_buddy; PRODUCT_NAME = Clicky; REGISTER_APP_GROUPS = YES; STRING_CATALOG_GENERATE_SYMBOLS = YES; @@ -454,6 +456,7 @@ ENABLE_HARDENED_RUNTIME = YES; ENABLE_OUTGOING_NETWORK_CONNECTIONS = YES; ENABLE_PREVIEWS = YES; + ENABLE_TESTABILITY = YES; ENABLE_USER_SELECTED_FILES = readonly; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_FILE = "leanring-buddy/Info.plist"; @@ -467,6 +470,7 @@ ); MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = "com.yourcompany.leanring-buddy"; + PRODUCT_MODULE_NAME = leanring_buddy; PRODUCT_NAME = Clicky; REGISTER_APP_GROUPS = YES; STRING_CATALOG_GENERATE_SYMBOLS = YES; @@ -495,7 +499,7 @@ SWIFT_EMIT_LOC_STRINGS = NO; SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; SWIFT_VERSION = 5.0; - TEST_HOST = "$(BUILT_PRODUCTS_DIR)/leanring-buddy.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/leanring-buddy"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/Clicky.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Clicky"; }; name = Debug; }; @@ -516,7 +520,7 @@ SWIFT_EMIT_LOC_STRINGS = NO; SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; SWIFT_VERSION = 5.0; - TEST_HOST = "$(BUILT_PRODUCTS_DIR)/leanring-buddy.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/leanring-buddy"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/Clicky.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Clicky"; }; name = Release; }; diff --git a/leanring-buddy.xcodeproj/xcshareddata/xcschemes/leanring-buddy.xcscheme b/leanring-buddy.xcodeproj/xcshareddata/xcschemes/leanring-buddy.xcscheme new file mode 100644 index 000000000..1b6cd72c9 --- /dev/null +++ b/leanring-buddy.xcodeproj/xcshareddata/xcschemes/leanring-buddy.xcscheme @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/leanring-buddy/ClickyAnalytics.swift b/leanring-buddy/ClickyAnalytics.swift index 8f6bf4151..84ac27da7 100644 --- a/leanring-buddy/ClickyAnalytics.swift +++ b/leanring-buddy/ClickyAnalytics.swift @@ -149,4 +149,33 @@ enum ClickyAnalytics { "skill_id": skillID ]) } + + static func trackTeachingSkillMerged(primarySkillID: String, duplicateSkillID: String) { + PostHogSDK.shared.capture("teaching_skill_merged", properties: [ + "primary_skill_id": primarySkillID, + "duplicate_skill_id": duplicateSkillID + ]) + } + + static func trackTeachingSkillPatched(skillID: String) { + PostHogSDK.shared.capture("teaching_skill_patched", properties: [ + "skill_id": skillID + ]) + } + + // MARK: - Niche Discovery + + static func trackNicheSelected(niche: String) { + PostHogSDK.shared.capture("niche_selected", properties: [ + "niche": niche + ]) + } + + static func trackNicheSuggestionTapped(suggestion: String, niche: String, bundleID: String?) { + PostHogSDK.shared.capture("niche_suggestion_tapped", properties: [ + "suggestion": suggestion, + "niche": niche, + "bundle_id": bundleID ?? "unknown" + ]) + } } diff --git a/leanring-buddy/ClickyE2EConfiguration.swift b/leanring-buddy/ClickyE2EConfiguration.swift index 4efc45f66..31126da1d 100644 --- a/leanring-buddy/ClickyE2EConfiguration.swift +++ b/leanring-buddy/ClickyE2EConfiguration.swift @@ -28,9 +28,29 @@ enum ClickyE2EConfiguration { argumentValue(for: "-CLICKY_INJECT_TRANSCRIPT_3=") } - static var lastSystemPromptFileURL: URL { + static var restoreSkillID: String? { + argumentValue(for: "-CLICKY_E2E_RESTORE_SKILL=") + } + + static var clickyDirectoryURL: URL { FileManager.default.homeDirectoryForCurrentUser - .appendingPathComponent(".clicky/e2e-last-system-prompt.txt") + .appendingPathComponent(".clicky", isDirectory: true) + } + + static var lastSystemPromptFileURL: URL { + clickyDirectoryURL.appendingPathComponent("e2e-last-system-prompt.txt") + } + + static var lastMatchedSkillIDFileURL: URL { + clickyDirectoryURL.appendingPathComponent("e2e-last-matched-skill-id.txt") + } + + static var skillsCountFileURL: URL { + clickyDirectoryURL.appendingPathComponent("e2e-skills-count.txt") + } + + static var skillLibraryStateFileURL: URL { + clickyDirectoryURL.appendingPathComponent("e2e-skill-library-state.txt") } static func applyLaunchOverrides() { @@ -43,10 +63,45 @@ enum ClickyE2EConfiguration { static func writeLastSystemPromptForE2E(_ systemPrompt: String) { guard isEnabled else { return } + writeText(systemPrompt, to: lastSystemPromptFileURL) + } + + static func writeLastMatchedSkillIDForE2E(_ skillID: String?) { + guard isEnabled else { return } + + if let skillID { + writeText(skillID, to: lastMatchedSkillIDFileURL) + } else { + try? FileManager.default.removeItem(at: lastMatchedSkillIDFileURL) + } + } + + static func writeSkillsCountForE2E(_ skillsCount: Int) { + guard isEnabled else { return } + writeText(String(skillsCount), to: skillsCountFileURL) + } + + static func writeSkillLibraryStateForE2E(_ skills: [TeachingSkill]) { + guard isEnabled else { return } + + let libraryEntries = skills.map { skill in + [ + "id": skill.id, + "status": skill.status.rawValue, + "pinned": skill.isPinned + ] as [String: Any] + } + + guard let jsonData = try? JSONSerialization.data(withJSONObject: libraryEntries, options: [.prettyPrinted]), + let jsonString = String(data: jsonData, encoding: .utf8) else { + return + } + writeText(jsonString, to: skillLibraryStateFileURL) + } - let directory = lastSystemPromptFileURL.deletingLastPathComponent() - try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - try? systemPrompt.write(to: lastSystemPromptFileURL, atomically: true, encoding: .utf8) + private static func writeText(_ text: String, to fileURL: URL) { + try? FileManager.default.createDirectory(at: clickyDirectoryURL, withIntermediateDirectories: true) + try? text.write(to: fileURL, atomically: true, encoding: .utf8) } private static func argumentValue(for prefix: String) -> String? { diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index cbec4416b..225fc50cf 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -85,6 +85,7 @@ final class CompanionManager: ObservableObject { private let topicHistoryStore = TeachingTopicHistoryStore() private var sessionTrace: [SessionTraceEntry] = [] private var skillWriteTask: Task? + private var curatorLLMTask: Task? /// Skills currently on disk, exposed for the panel UI. @Published private(set) var teachingSkills: [TeachingSkill] = [] @@ -169,6 +170,20 @@ final class CompanionManager: ObservableObject { } } + func restoreTeachingSkill(id: String) { + do { + try teachingSkillStore.restoreSkill(id: id) + teachingSkills = teachingSkillStore.skills + writeE2EArtifactsIfNeeded() + } catch { + print("⚠️ Failed to restore teaching skill \(id): \(error)") + } + } + + func teachingSkills(withStatus status: TeachingSkillStatus?) -> [TeachingSkill] { + teachingSkillStore.skills(withStatus: status) + } + /// Allows E2E tests to bypass microphone/STT and exercise the response + skill loop directly. func injectTranscriptForE2E(_ transcript: String) async { await withCheckedContinuation { (continuation: CheckedContinuation) in @@ -207,6 +222,27 @@ final class CompanionManager: ObservableObject { } } + func runE2EBootstrapActionsIfNeeded() { + guard ClickyE2EConfiguration.isEnabled else { return } + + Task { + try? await Task.sleep(nanoseconds: 500_000_000) + + if let restoreSkillID = ClickyE2EConfiguration.restoreSkillID { + restoreTeachingSkill(id: restoreSkillID) + } + + writeE2EArtifactsIfNeeded() + } + } + + func writeE2EArtifactsIfNeeded() { + guard ClickyE2EConfiguration.isEnabled else { return } + + ClickyE2EConfiguration.writeSkillsCountForE2E(teachingSkillStore.skills.count) + ClickyE2EConfiguration.writeSkillLibraryStateForE2E(teachingSkillStore.skills) + } + private func frontmostApplicationBundleId() -> String? { NSWorkspace.shared.frontmostApplication?.bundleIdentifier } @@ -216,6 +252,16 @@ final class CompanionManager: ObservableObject { topicHistoryStore.load() SkillCurator.curate(store: teachingSkillStore) teachingSkills = teachingSkillStore.skills + runCuratorLLMPassesIfNeeded() + writeE2EArtifactsIfNeeded() + } + + private func runCuratorLLMPassesIfNeeded() { + curatorLLMTask?.cancel() + curatorLLMTask = Task { + await SkillCurator.curateWithLLMPasses(store: teachingSkillStore, claudeAPI: claudeAPI) + teachingSkills = teachingSkillStore.skills + } } private func matchedSkills(for transcript: String) -> [TeachingSkill] { @@ -315,6 +361,7 @@ final class CompanionManager: ObservableObject { _ = try teachingSkillStore.saveSkill(skill) SkillCurator.curate(store: teachingSkillStore) teachingSkills = teachingSkillStore.skills + runCuratorLLMPassesIfNeeded() topicHistoryStore.recordTopic( topic: trigger.topic, bundleId: targetBundleId, @@ -327,6 +374,7 @@ final class CompanionManager: ObservableObject { reason: trigger.reason.rawValue, updatedExisting: existingSkill != nil ) + writeE2EArtifactsIfNeeded() print("📚 Saved teaching skill: \(skill.id)") } catch { print("⚠️ Failed to synthesize teaching skill: \(error)") @@ -868,6 +916,9 @@ final class CompanionManager: ObservableObject { skillIDs: matchedTeachingSkills.map(\.id), bundleID: frontmostApplicationBundleId() ) + ClickyE2EConfiguration.writeLastMatchedSkillIDForE2E(matchedTeachingSkills.first?.id) + } else { + ClickyE2EConfiguration.writeLastMatchedSkillIDForE2E(nil) } let (fullResponseText, _) = try await claudeAPI.analyzeImageStreaming( diff --git a/leanring-buddy/CompanionPanelView.swift b/leanring-buddy/CompanionPanelView.swift index fdd80bbd7..09e7e7223 100644 --- a/leanring-buddy/CompanionPanelView.swift +++ b/leanring-buddy/CompanionPanelView.swift @@ -13,8 +13,21 @@ import SwiftUI struct CompanionPanelView: View { @ObservedObject var companionManager: CompanionManager @State private var emailInput: String = "" + @State private var isShowingTeachingSkillsLibrary = false var body: some View { + if isShowingTeachingSkillsLibrary { + TeachingSkillsLibraryView(companionManager: companionManager) { + isShowingTeachingSkillsLibrary = false + } + .frame(width: 320) + .background(panelBackground) + } else { + mainPanelContent + } + } + + private var mainPanelContent: some View { VStack(alignment: .leading, spacing: 0) { panelHeader Divider() @@ -719,6 +732,7 @@ struct CompanionPanelView: View { .labelsHidden() .tint(DS.Colors.accent) .scaleEffect(0.8) + .accessibilityIdentifier("clicky.panel.teaching-skills.learn-toggle") } Text(companionManager.isLearningFromSessionsEnabled @@ -736,6 +750,17 @@ struct CompanionPanelView: View { ForEach(companionManager.teachingSkills.prefix(4)) { skill in teachingSkillRow(skill) } + + Button(action: { + isShowingTeachingSkillsLibrary = true + }) { + Text("View all") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.accent) + } + .buttonStyle(.plain) + .pointerCursor() + .accessibilityIdentifier("clicky.panel.teaching-skills.view-all") } } .padding(.vertical, 4) diff --git a/leanring-buddy/SkillCurator.swift b/leanring-buddy/SkillCurator.swift index e2a1649cf..11a2f1161 100644 --- a/leanring-buddy/SkillCurator.swift +++ b/leanring-buddy/SkillCurator.swift @@ -10,8 +10,17 @@ import Foundation enum SkillCurator { private static let staleAfterDays = 30 private static let archiveAfterDays = 90 + private static let maxMergePassesPerLaunch = 1 + private static let maxPatchPassesPerLaunch = 1 + + private static var mergePassesUsedThisLaunch = 0 + private static var patchPassesUsedThisLaunch = 0 static func curate(store: TeachingSkillStore, now: Date = Date()) { + curateTimeBasedLifecycle(store: store, now: now) + } + + static func curateTimeBasedLifecycle(store: TeachingSkillStore, now: Date = Date()) { let calendar = Calendar.current for skill in store.skills { @@ -35,4 +44,133 @@ enum SkillCurator { } } } + + static func curateWithLLMPasses(store: TeachingSkillStore, claudeAPI: ClaudeAPI) async { + await mergeOneDuplicatePairIfNeeded(store: store, claudeAPI: claudeAPI) + await patchOneStaleSkillIfNeeded(store: store, claudeAPI: claudeAPI) + } + + private static func mergeOneDuplicatePairIfNeeded( + store: TeachingSkillStore, + claudeAPI: ClaudeAPI + ) async { + guard mergePassesUsedThisLaunch < maxMergePassesPerLaunch else { return } + + let duplicatePairs = SkillMatcher.findDuplicateSkillPairs(in: store.skills) + guard let duplicatePair = duplicatePairs.first else { return } + + do { + let mergedSkill = try await mergeDuplicateSkills( + primarySkill: duplicatePair.primarySkill, + duplicateSkill: duplicatePair.duplicateSkill, + claudeAPI: claudeAPI + ) + _ = try store.saveSkill(mergedSkill) + try store.deleteSkill(id: duplicatePair.duplicateSkill.id) + mergePassesUsedThisLaunch += 1 + ClickyAnalytics.trackTeachingSkillMerged( + primarySkillID: duplicatePair.primarySkill.id, + duplicateSkillID: duplicatePair.duplicateSkill.id + ) + print("📚 Curator merged duplicate skills: \(duplicatePair.duplicateSkill.id) → \(duplicatePair.primarySkill.id)") + } catch { + print("⚠️ Curator failed to merge duplicate skills: \(error)") + } + } + + private static func patchOneStaleSkillIfNeeded( + store: TeachingSkillStore, + claudeAPI: ClaudeAPI + ) async { + guard patchPassesUsedThisLaunch < maxPatchPassesPerLaunch else { return } + + guard let staleSkill = store.skills.first(where: { $0.status == .stale && !$0.isPinned }) else { + return + } + + do { + let patchedSkill = try await patchStaleSkill(staleSkill, claudeAPI: claudeAPI) + _ = try store.saveSkill(patchedSkill) + patchPassesUsedThisLaunch += 1 + ClickyAnalytics.trackTeachingSkillPatched(skillID: staleSkill.id) + print("📚 Curator patched stale skill: \(staleSkill.id)") + } catch { + print("⚠️ Curator failed to patch stale skill \(staleSkill.id): \(error)") + } + } + + private static let mergeSystemPrompt = """ + you merge duplicate teaching skills for clicky, a screen-native voice tutor. + + given two similar skills, produce a single merged markdown body only. do not include yaml frontmatter. + + preserve the best steps, ui labels, pointing tips, and completion signals from both skills. + remove redundancy. keep it concise and practical. all lowercase. + """ + + private static let patchSystemPrompt = """ + you refresh stale teaching skills for clicky when macos app ui may have changed. + + given an existing teaching skill, produce an updated markdown body only. do not include yaml frontmatter. + + keep the same workflow intent but update ui labels, menu paths, and pointing heuristics for current macos ui. + note any labels that may have changed. all lowercase. + """ + + private static func mergeDuplicateSkills( + primarySkill: TeachingSkill, + duplicateSkill: TeachingSkill, + claudeAPI: ClaudeAPI + ) async throws -> TeachingSkill { + let userPrompt = """ + merge these duplicate teaching skills into one improved skill body. + + primary skill (\(primarySkill.name)): + \(primarySkill.body) + + duplicate skill (\(duplicateSkill.name)): + \(duplicateSkill.body) + """ + + let response = try await claudeAPI.sendTextMessage( + systemPrompt: mergeSystemPrompt, + userPrompt: userPrompt, + maxTokens: 1200 + ) + + var mergedSkill = primarySkill + mergedSkill.body = response.text.trimmingCharacters(in: .whitespacesAndNewlines) + mergedSkill.usageCount = primarySkill.usageCount + duplicateSkill.usageCount + mergedSkill.lastUsed = Date() + mergedSkill.status = .active + return mergedSkill + } + + private static func patchStaleSkill( + _ staleSkill: TeachingSkill, + claudeAPI: ClaudeAPI + ) async throws -> TeachingSkill { + let userPrompt = """ + refresh this stale teaching skill for current macos ui: + + skill name: \(staleSkill.name) + description: \(staleSkill.description) + bundle ids: \(staleSkill.bundleIds.joined(separator: ", ")) + + existing body: + \(staleSkill.body) + """ + + let response = try await claudeAPI.sendTextMessage( + systemPrompt: patchSystemPrompt, + userPrompt: userPrompt, + maxTokens: 1200 + ) + + var patchedSkill = staleSkill + patchedSkill.body = response.text.trimmingCharacters(in: .whitespacesAndNewlines) + patchedSkill.status = .active + patchedSkill.lastUsed = Date() + return patchedSkill + } } diff --git a/leanring-buddy/SkillMatcher.swift b/leanring-buddy/SkillMatcher.swift index 3197e6ba5..e5013a851 100644 --- a/leanring-buddy/SkillMatcher.swift +++ b/leanring-buddy/SkillMatcher.swift @@ -12,6 +12,12 @@ struct SkillMatch: Equatable { let score: Int } +struct DuplicateSkillPair: Equatable { + let primarySkill: TeachingSkill + let duplicateSkill: TeachingSkill + let overlapScore: Int +} + enum SkillMatcher { static let topicStopwords: Set = [ "how", "do", "does", "did", "can", "could", "would", "should", "what", "where", "when", "why", "which", @@ -125,6 +131,74 @@ enum SkillMatcher { } } + static func tokenOverlapScore(between leftSkill: TeachingSkill, and rightSkill: TeachingSkill) -> Int { + let leftTokens = Set( + tokenize(leftSkill.name) + + tokenize(leftSkill.description) + + tokenize(leftSkill.body) + ) + let rightTokens = Set( + tokenize(rightSkill.name) + + tokenize(rightSkill.description) + + tokenize(rightSkill.body) + ) + return leftTokens.filter { rightTokens.contains($0) }.count + } + + static func findDuplicateSkillPairs( + in skills: [TeachingSkill], + minimumOverlapScore: Int = 3 + ) -> [DuplicateSkillPair] { + let eligibleSkills = skills.filter { skill in + !skill.isPinned && skill.status != .archived + } + + var duplicatePairs: [DuplicateSkillPair] = [] + + for leftIndex in 0..= minimumOverlapScore else { continue } + + let primarySkill: TeachingSkill + let duplicateSkill: TeachingSkill + if leftSkill.usageCount != rightSkill.usageCount { + if leftSkill.usageCount > rightSkill.usageCount { + primarySkill = leftSkill + duplicateSkill = rightSkill + } else { + primarySkill = rightSkill + duplicateSkill = leftSkill + } + } else if (leftSkill.lastUsed ?? .distantPast) >= (rightSkill.lastUsed ?? .distantPast) { + primarySkill = leftSkill + duplicateSkill = rightSkill + } else { + primarySkill = rightSkill + duplicateSkill = leftSkill + } + + duplicatePairs.append( + DuplicateSkillPair( + primarySkill: primarySkill, + duplicateSkill: duplicateSkill, + overlapScore: overlapScore + ) + ) + } + } + + return duplicatePairs + } + private static func overlapScore(_ skill: TeachingSkill, topicTokens: Set) -> Int { let skillTokens = Set( tokenize(skill.name) + diff --git a/leanring-buddy/TeachingSkillStore.swift b/leanring-buddy/TeachingSkillStore.swift index 7df911100..67e5c811e 100644 --- a/leanring-buddy/TeachingSkillStore.swift +++ b/leanring-buddy/TeachingSkillStore.swift @@ -86,4 +86,16 @@ final class TeachingSkillStore { skill.isPinned = pinned _ = try saveSkill(skill) } + + func restoreSkill(id: String) throws { + guard var skill = skill(withID: id) else { return } + skill.status = .active + skill.lastUsed = Date() + _ = try saveSkill(skill) + } + + func skills(withStatus status: TeachingSkillStatus?) -> [TeachingSkill] { + guard let status else { return skills } + return skills.filter { $0.status == status } + } } diff --git a/leanring-buddy/TeachingSkillsLibraryView.swift b/leanring-buddy/TeachingSkillsLibraryView.swift new file mode 100644 index 000000000..d3dcb1cf3 --- /dev/null +++ b/leanring-buddy/TeachingSkillsLibraryView.swift @@ -0,0 +1,264 @@ +// +// TeachingSkillsLibraryView.swift +// leanring-buddy +// +// Full teaching skills library with status filters, detail view, and lifecycle actions. +// + +import SwiftUI + +enum TeachingSkillsLibraryFilter: String, CaseIterable, Identifiable { + case all + case active + case stale + case archived + + var id: String { rawValue } + + var label: String { + switch self { + case .all: return "All" + case .active: return "Active" + case .stale: return "Stale" + case .archived: return "Archived" + } + } + + var status: TeachingSkillStatus? { + switch self { + case .all: return nil + case .active: return .active + case .stale: return .stale + case .archived: return .archived + } + } +} + +struct TeachingSkillsLibraryView: View { + @ObservedObject var companionManager: CompanionManager + let onBack: () -> Void + + @State private var selectedFilter: TeachingSkillsLibraryFilter = .all + @State private var selectedSkill: TeachingSkill? + + private var filteredSkills: [TeachingSkill] { + companionManager.teachingSkills(withStatus: selectedFilter.status) + } + + var body: some View { + VStack(alignment: .leading, spacing: 0) { + libraryHeader + + Divider() + .background(DS.Colors.borderSubtle) + .padding(.horizontal, 16) + + if let selectedSkill { + skillDetailView(selectedSkill) + } else { + filterPicker + .padding(.horizontal, 16) + .padding(.top, 12) + + skillsList + } + } + } + + private var libraryHeader: some View { + HStack(spacing: 8) { + Button(action: { + if selectedSkill != nil { + selectedSkill = nil + } else { + onBack() + } + }) { + Image(systemName: "chevron.left") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(DS.Colors.textTertiary) + .frame(width: 24, height: 24) + .background( + Circle() + .fill(Color.white.opacity(0.08)) + ) + } + .buttonStyle(.plain) + .pointerCursor() + + Text(selectedSkill?.name ?? "Teaching Skills") + .font(.system(size: 14, weight: .semibold)) + .foregroundColor(DS.Colors.textSecondary) + .lineLimit(1) + + Spacer() + } + .padding(.horizontal, 16) + .padding(.vertical, 14) + } + + private var filterPicker: some View { + HStack(spacing: 4) { + ForEach(TeachingSkillsLibraryFilter.allCases) { filter in + Button(action: { + selectedFilter = filter + }) { + Text(filter.label) + .font(.system(size: 11, weight: .medium)) + .foregroundColor(selectedFilter == filter ? DS.Colors.textOnAccent : DS.Colors.textTertiary) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background( + RoundedRectangle(cornerRadius: DS.CornerRadius.small, style: .continuous) + .fill(selectedFilter == filter ? DS.Colors.accent : Color.white.opacity(0.06)) + ) + } + .buttonStyle(.plain) + .pointerCursor() + .accessibilityIdentifier("clicky.panel.skills-library.filter.\(filter.rawValue)") + } + } + } + + private var skillsList: some View { + ScrollView { + if filteredSkills.isEmpty { + Text("No \(selectedFilter == .all ? "" : selectedFilter.label.lowercased() + " ")skills.") + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textTertiary) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 16) + .padding(.top, 12) + } else { + VStack(spacing: 0) { + ForEach(filteredSkills) { skill in + skillRow(skill) + Divider() + .background(DS.Colors.borderSubtle.opacity(0.5)) + .padding(.leading, 16) + } + } + .padding(.top, 8) + } + } + .frame(maxHeight: 280) + } + + private func skillRow(_ skill: TeachingSkill) -> some View { + HStack(spacing: 8) { + Button(action: { + selectedSkill = skill + }) { + VStack(alignment: .leading, spacing: 2) { + HStack(spacing: 4) { + Text(skill.name) + .font(.system(size: 12, weight: .semibold)) + .foregroundColor(DS.Colors.textSecondary) + .lineLimit(1) + + if skill.isPinned { + Image(systemName: "pin.fill") + .font(.system(size: 9)) + .foregroundColor(DS.Colors.accent) + } + } + + Text("\(skill.usageCount) uses • \(skill.status.rawValue)") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + } + .frame(maxWidth: .infinity, alignment: .leading) + } + .buttonStyle(.plain) + .pointerCursor() + + if skill.status == .archived || skill.status == .stale { + Button(action: { + companionManager.restoreTeachingSkill(id: skill.id) + }) { + Image(systemName: "arrow.uturn.backward") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + } + .buttonStyle(.plain) + .pointerCursor() + } + + Button(action: { + companionManager.setTeachingSkillPinned(id: skill.id, pinned: !skill.isPinned) + }) { + Image(systemName: skill.isPinned ? "pin.fill" : "pin") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(skill.isPinned ? DS.Colors.accent : DS.Colors.textTertiary) + } + .buttonStyle(.plain) + .pointerCursor() + + Button(action: { + if selectedSkill?.id == skill.id { + selectedSkill = nil + } + companionManager.deleteTeachingSkill(id: skill.id) + }) { + Image(systemName: "trash") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + } + .buttonStyle(.plain) + .pointerCursor() + } + .padding(.horizontal, 16) + .padding(.vertical, 8) + .accessibilityIdentifier("clicky.panel.skills-library.row.\(skill.id)") + } + + private func skillDetailView(_ skill: TeachingSkill) -> some View { + ScrollView { + VStack(alignment: .leading, spacing: 12) { + VStack(alignment: .leading, spacing: 4) { + Text(skill.description) + .font(.system(size: 12)) + .foregroundColor(DS.Colors.textSecondary) + + Text("\(skill.usageCount) uses • \(skill.status.rawValue)") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + + if !skill.bundleIds.isEmpty { + Text(skill.bundleIds.joined(separator: ", ")) + .font(.system(size: 10, design: .monospaced)) + .foregroundColor(DS.Colors.textTertiary) + } + } + + Divider() + .background(DS.Colors.borderSubtle) + + skillMarkdownBody(skill.body) + } + .padding(.horizontal, 16) + .padding(.vertical, 12) + } + .frame(maxHeight: 300) + } + + @ViewBuilder + private func skillMarkdownBody(_ body: String) -> some View { + if let attributedBody = try? AttributedString( + markdown: body, + options: AttributedString.MarkdownParsingOptions(interpretedSyntax: .inlineOnlyPreservingWhitespace) + ) { + Text(attributedBody) + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textSecondary) + .frame(maxWidth: .infinity, alignment: .leading) + .textSelection(.enabled) + } else { + Text(body) + .font(.system(size: 11, design: .monospaced)) + .foregroundColor(DS.Colors.textSecondary) + .frame(maxWidth: .infinity, alignment: .leading) + .textSelection(.enabled) + } + } +} diff --git a/leanring-buddy/leanring_buddyApp.swift b/leanring-buddy/leanring_buddyApp.swift index c0df95a6e..dff03cc06 100644 --- a/leanring-buddy/leanring_buddyApp.swift +++ b/leanring-buddy/leanring_buddyApp.swift @@ -47,6 +47,7 @@ final class CompanionAppDelegate: NSObject, NSApplicationDelegate { companionManager.start() companionManager.runE2EInjectSequenceIfNeeded() + companionManager.runE2EBootstrapActionsIfNeeded() // Auto-open the panel if the user still needs to do something: // either they haven't onboarded yet, or permissions were revoked. // Skip during headless E2E — the ad-hoc E2E build has a different code diff --git a/tests/e2e/lib/common.sh b/tests/e2e/lib/common.sh new file mode 100755 index 000000000..920c97aac --- /dev/null +++ b/tests/e2e/lib/common.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# Shared helpers for Clicky headless E2E scripts. + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +CLICKY_APP="${CLICKY_APP:-$ROOT_DIR/build/E2E/Clicky.app}" +WORKER_URL="${CLICKY_WORKER_URL:-http://127.0.0.1:8787}" +SKILLS_DIR="$HOME/.clicky/skills" +CLICKY_DIR="$HOME/.clicky" + +E2E_PROMPT_FILE="$CLICKY_DIR/e2e-last-system-prompt.txt" +E2E_MATCHED_SKILL_FILE="$CLICKY_DIR/e2e-last-matched-skill-id.txt" +E2E_SUGGESTIONS_FILE="$CLICKY_DIR/e2e-last-suggestions.txt" +E2E_SELECTED_NICHE_FILE="$CLICKY_DIR/e2e-selected-niche.txt" +E2E_SKILLS_COUNT_FILE="$CLICKY_DIR/e2e-skills-count.txt" +E2E_LIBRARY_STATE_FILE="$CLICKY_DIR/e2e-skill-library-state.txt" + +MOCK_WORKER_PID="" +CLICKY_PID="" + +e2e_cleanup() { + if [[ -n "$MOCK_WORKER_PID" ]]; then + kill "$MOCK_WORKER_PID" 2>/dev/null || true + MOCK_WORKER_PID="" + fi + if [[ -n "$CLICKY_PID" ]]; then + kill "$CLICKY_PID" 2>/dev/null || true + CLICKY_PID="" + fi +} + +ensure_clicky_built() { + if [[ "${SKIP_E2E_BUILD:-}" == "1" ]] && [[ -d "$CLICKY_APP" ]]; then + echo "Using existing Clicky build at $CLICKY_APP" + return 0 + fi + + echo "Building Clicky for E2E..." + mkdir -p "$ROOT_DIR/build/E2E" + xcodebuild \ + -project "$ROOT_DIR/leanring-buddy.xcodeproj" \ + -scheme leanring-buddy \ + -destination 'platform=macOS' \ + -derivedDataPath "$ROOT_DIR/build/E2E/DerivedData" \ + CODE_SIGN_IDENTITY="-" \ + CODE_SIGNING_ALLOWED=NO \ + build >/tmp/clicky-e2e-build.log 2>&1 + + local built_app="$ROOT_DIR/build/E2E/DerivedData/Build/Products/Debug/Clicky.app" + rm -rf "$CLICKY_APP" + ditto "$built_app" "$CLICKY_APP" + echo "Built Clicky at $CLICKY_APP" +} + +worker_port_from_url() { + local url="$1" + local port="8787" + + if [[ "$url" =~ :([0-9]+)(/|$|\?) ]]; then + port="${BASH_REMATCH[1]}" + fi + + echo "$port" +} + +start_mock_worker() { + local worker_port + worker_port="$(worker_port_from_url "$WORKER_URL")" + echo "Starting mock worker on $WORKER_URL (port $worker_port)..." + MOCK_WORKER_PORT="$worker_port" node "$ROOT_DIR/tests/e2e/mock-worker.mjs" >/tmp/clicky-e2e-worker.log 2>&1 & + MOCK_WORKER_PID=$! + sleep 1 +} + +launch_clicky() { + local log_file="${1:-/tmp/clicky-e2e-app.log}" + shift || true + + "$CLICKY_APP/Contents/MacOS/Clicky" "$@" >"$log_file" 2>&1 & + CLICKY_PID=$! +} + +wait_for_file() { + local file_path="$1" + local timeout_seconds="${2:-30}" + local description="${3:-$file_path}" + + for _ in $(seq 1 "$timeout_seconds"); do + if [[ -f "$file_path" ]]; then + return 0 + fi + sleep 1 + done + + echo "FAIL: timed out waiting for $description ($file_path)" + return 1 +} + +wait_for_file_content() { + local file_path="$1" + local pattern="$2" + local timeout_seconds="${3:-30}" + local description="${4:-$file_path}" + + for _ in $(seq 1 "$timeout_seconds"); do + if [[ -f "$file_path" ]] && grep -q "$pattern" "$file_path"; then + return 0 + fi + sleep 1 + done + + echo "FAIL: timed out waiting for '$pattern' in $description" + return 1 +} + +reset_e2e_artifacts() { + rm -f \ + "$E2E_PROMPT_FILE" \ + "$E2E_MATCHED_SKILL_FILE" \ + "$E2E_SUGGESTIONS_FILE" \ + "$E2E_SELECTED_NICHE_FILE" \ + "$E2E_SKILLS_COUNT_FILE" \ + "$E2E_LIBRARY_STATE_FILE" +} + +seed_teaching_skill() { + local skill_id="$1" + local skill_status="$2" + local skill_pinned="${3:-false}" + local skill_name="$4" + local skill_body="$5" + + mkdir -p "$SKILLS_DIR/$skill_id" + cat >"$SKILLS_DIR/$skill_id/SKILL.md" </dev/null || true + echo "--- worker log ---" + tail -20 /tmp/clicky-e2e-worker.log 2>/dev/null || true +} diff --git a/tests/e2e/run-all.sh b/tests/e2e/run-all.sh index b0d557a99..c50642854 100755 --- a/tests/e2e/run-all.sh +++ b/tests/e2e/run-all.sh @@ -2,8 +2,44 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/common.sh +source "$SCRIPT_DIR/lib/common.sh" -echo "Running Clicky teaching-skills E2E..." +FULL_STACK=false +for arg in "$@"; do + case "$arg" in + --full-stack) + FULL_STACK=true + ;; + esac +done + +ensure_clicky_built +export SKIP_E2E_BUILD=1 + +HEADLESS_SCRIPTS=( + teaching-skills.sh + niche-discovery.sh + skills-library.sh + full-user-journey.sh +) + +echo "=== Clicky E2E run-all ===" echo "" -exec "$SCRIPT_DIR/teaching-skills.sh" "$@" +for headless_script in "${HEADLESS_SCRIPTS[@]}"; do + echo "----------------------------------------" + echo "Running $headless_script" + echo "----------------------------------------" + bash "$SCRIPT_DIR/$headless_script" + echo "" +done + +if [[ "$FULL_STACK" == "true" ]]; then + echo "----------------------------------------" + echo "Running full-stack automated runner" + echo "----------------------------------------" + bash "$SCRIPT_DIR/full-stack/run-automated.sh" +fi + +echo "=== E2E run-all PASS ===" diff --git a/tests/e2e/skills-library.sh b/tests/e2e/skills-library.sh new file mode 100755 index 000000000..9f62477f8 --- /dev/null +++ b/tests/e2e/skills-library.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/common.sh +source "$SCRIPT_DIR/lib/common.sh" + +ACTIVE_SKILL_ID="teach-e2e-active-test" +ARCHIVED_SKILL_ID="teach-e2e-archived-test" + +trap e2e_cleanup EXIT + +ensure_clicky_built +start_mock_worker + +echo "Seeding teaching skills for library E2E..." +rm -rf "$SKILLS_DIR" +mkdir -p "$SKILLS_DIR" +reset_e2e_artifacts + +seed_teaching_skill "$ACTIVE_SKILL_ID" "active" "false" "E2E Active Skill" "active skill body for e2e library test" +seed_teaching_skill "$ARCHIVED_SKILL_ID" "archived" "false" "E2E Archived Skill" "archived skill body for e2e library test" + +echo "Phase A: write skill library snapshot..." +launch_clicky /tmp/clicky-e2e-library-a.log \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL="$WORKER_URL" + +if ! wait_for_file "$E2E_LIBRARY_STATE_FILE" 30 "skill library state artifact"; then + print_failure_logs /tmp/clicky-e2e-library-a.log + exit 1 +fi + +if ! grep -q "\"id\" : \"$ACTIVE_SKILL_ID\"" "$E2E_LIBRARY_STATE_FILE"; then + echo "FAIL: library state missing active skill $ACTIVE_SKILL_ID" + cat "$E2E_LIBRARY_STATE_FILE" + exit 1 +fi + +if ! grep -q "\"id\" : \"$ARCHIVED_SKILL_ID\"" "$E2E_LIBRARY_STATE_FILE"; then + echo "FAIL: library state missing archived skill $ARCHIVED_SKILL_ID" + cat "$E2E_LIBRARY_STATE_FILE" + exit 1 +fi + +if ! grep -q "\"status\" : \"active\"" "$E2E_LIBRARY_STATE_FILE"; then + echo "FAIL: library state missing active status" + exit 1 +fi + +if ! grep -q "\"status\" : \"archived\"" "$E2E_LIBRARY_STATE_FILE"; then + echo "FAIL: library state missing archived status" + exit 1 +fi + +echo "PASS: library snapshot contains both seeded skills with correct statuses" +echo "--- library state preview ---" +head -20 "$E2E_LIBRARY_STATE_FILE" + +kill "$CLICKY_PID" 2>/dev/null || true +CLICKY_PID="" +sleep 1 + +echo "" +echo "Phase B: restore archived skill via E2E hook..." +rm -f "$E2E_LIBRARY_STATE_FILE" + +launch_clicky /tmp/clicky-e2e-library-b.log \ + -CLICKY_E2E=1 \ + -CLICKY_E2E_RESTORE_SKILL="$ARCHIVED_SKILL_ID" \ + -CLICKY_WORKER_URL="$WORKER_URL" + +if ! wait_for_file "$E2E_LIBRARY_STATE_FILE" 30 "updated library state artifact"; then + print_failure_logs /tmp/clicky-e2e-library-b.log + exit 1 +fi + +if ! python3 - <<'PY' "$E2E_LIBRARY_STATE_FILE" "$ARCHIVED_SKILL_ID" +import json, sys +path, skill_id = sys.argv[1], sys.argv[2] +entries = json.load(open(path)) +match = next((entry for entry in entries if entry.get("id") == skill_id), None) +if not match: + raise SystemExit(f"missing skill {skill_id}") +if match.get("status") != "active": + raise SystemExit(f"expected active after restore, got {match.get('status')}") +print("ok") +PY +then + echo "FAIL: archived skill was not restored to active" + cat "$E2E_LIBRARY_STATE_FILE" + exit 1 +fi + +echo "PASS: archived skill restored to active in library snapshot" +echo "" +echo "E2E PASS: skills-library (A + B) succeeded" From 4612168f07edbd5dac4b55fa355837edca49f80f Mon Sep 17 00:00:00 2001 From: octavi42 Date: Fri, 29 May 2026 17:32:26 +0300 Subject: [PATCH 07/14] Limit teaching-skills E2E run-all to teaching-only scripts. Drop niche and full-stack scripts from the upstream PR branch so CI matches the scoped feature set. --- tests/e2e/run-all.sh | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/e2e/run-all.sh b/tests/e2e/run-all.sh index c50642854..2ea9e6cff 100755 --- a/tests/e2e/run-all.sh +++ b/tests/e2e/run-all.sh @@ -5,23 +5,12 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" # shellcheck source=lib/common.sh source "$SCRIPT_DIR/lib/common.sh" -FULL_STACK=false -for arg in "$@"; do - case "$arg" in - --full-stack) - FULL_STACK=true - ;; - esac -done - ensure_clicky_built export SKIP_E2E_BUILD=1 HEADLESS_SCRIPTS=( teaching-skills.sh - niche-discovery.sh skills-library.sh - full-user-journey.sh ) echo "=== Clicky E2E run-all ===" @@ -35,11 +24,4 @@ for headless_script in "${HEADLESS_SCRIPTS[@]}"; do echo "" done -if [[ "$FULL_STACK" == "true" ]]; then - echo "----------------------------------------" - echo "Running full-stack automated runner" - echo "----------------------------------------" - bash "$SCRIPT_DIR/full-stack/run-automated.sh" -fi - echo "=== E2E run-all PASS ===" From 789a8cb9a1a3ac743b7600fdde43daf82207923b Mon Sep 17 00:00:00 2001 From: Cristea-Zablau Octavian <53470992+octavi42@users.noreply.github.com> Date: Mon, 8 Jun 2026 12:52:31 +0300 Subject: [PATCH 08/14] Add runtime data-home isolation for parallel app instances (#6) * Add runtime data-home isolation for parallel app instances. Introduce ClickyPaths/ClickyDefaults/ClickyLaunchArguments so the app's data home (~/.clicky), UserDefaults suite, and global push-to-talk can be redirected per instance via -CLICKY_HOME / -CLICKY_DEFAULTS_SUITE / -CLICKY_DISABLE_GLOBAL_PTT (or the CLICKY_HOME env var). This lets multiple worktree builds run side by side without clobbering each other's skills, preferences, or fighting over the global hotkey. Routes skill and topic stores through ClickyPaths, threads CLICKY_HOME through the E2E harness, and adds unit coverage plus a @MainActor fix so the test target compiles. * Make input-monitoring preflight nonisolated to fix Xcode 16.4 build. GlobalPushToTalkShortcutMonitor.start() runs in a nonisolated context and calls WindowPositionManager.hasInputMonitoringPermission(), which Xcode 16.4 (used by CI) rejects as a cross-actor call. The helper only reads live TCC state via a pure C preflight, so marking it nonisolated is safe and unblocks the CI build on both Xcode 16.4 and 17. --- leanring-buddy/ClickyE2EConfiguration.swift | 16 +-- leanring-buddy/ClickyPaths.swift | 60 +++++++++++ leanring-buddy/CompanionManager.swift | 28 +++--- .../GlobalPushToTalkShortcutMonitor.swift | 5 + leanring-buddy/TeachingSkillStore.swift | 7 +- .../TeachingTopicHistoryStore.swift | 7 +- leanring-buddy/WindowPositionManager.swift | 11 ++- leanring-buddyTests/ClickyPathsTests.swift | 99 +++++++++++++++++++ leanring-buddyTests/leanring_buddyTests.swift | 1 + tests/e2e/lib/common.sh | 7 +- 10 files changed, 201 insertions(+), 40 deletions(-) create mode 100644 leanring-buddy/ClickyPaths.swift create mode 100644 leanring-buddyTests/ClickyPathsTests.swift diff --git a/leanring-buddy/ClickyE2EConfiguration.swift b/leanring-buddy/ClickyE2EConfiguration.swift index 31126da1d..25332957d 100644 --- a/leanring-buddy/ClickyE2EConfiguration.swift +++ b/leanring-buddy/ClickyE2EConfiguration.swift @@ -33,8 +33,7 @@ enum ClickyE2EConfiguration { } static var clickyDirectoryURL: URL { - FileManager.default.homeDirectoryForCurrentUser - .appendingPathComponent(".clicky", isDirectory: true) + ClickyPaths.home } static var lastSystemPromptFileURL: URL { @@ -56,9 +55,9 @@ enum ClickyE2EConfiguration { static func applyLaunchOverrides() { guard isEnabled else { return } - UserDefaults.standard.set(true, forKey: "hasCompletedOnboarding") - UserDefaults.standard.set(true, forKey: "hasSubmittedEmail") - UserDefaults.standard.set(true, forKey: "isClickyCursorEnabled") + ClickyDefaults.shared.set(true, forKey: "hasCompletedOnboarding") + ClickyDefaults.shared.set(true, forKey: "hasSubmittedEmail") + ClickyDefaults.shared.set(true, forKey: "isClickyCursorEnabled") } static func writeLastSystemPromptForE2E(_ systemPrompt: String) { @@ -105,11 +104,6 @@ enum ClickyE2EConfiguration { } private static func argumentValue(for prefix: String) -> String? { - ProcessInfo.processInfo.arguments - .first(where: { $0.hasPrefix(prefix) }) - .map { String($0.dropFirst(prefix.count)) } - .flatMap { value in - value.isEmpty ? nil : value - } + ClickyLaunchArguments.value(forPrefix: prefix) } } diff --git a/leanring-buddy/ClickyPaths.swift b/leanring-buddy/ClickyPaths.swift new file mode 100644 index 000000000..b1cbd3baa --- /dev/null +++ b/leanring-buddy/ClickyPaths.swift @@ -0,0 +1,60 @@ +// +// ClickyPaths.swift +// leanring-buddy +// +// Centralizes Clicky data home, launch arguments, and isolated UserDefaults. +// Enables parallel worktree testing via CLICKY_HOME / -CLICKY_HOME=. +// + +import Foundation + +enum ClickyLaunchArguments { + static func value(forPrefix prefix: String) -> String? { + ProcessInfo.processInfo.arguments + .first { $0.hasPrefix(prefix) } + .map { String($0.dropFirst(prefix.count)) } + .flatMap { $0.isEmpty ? nil : $0 } + } + + static func isPresent(_ flag: String) -> Bool { + ProcessInfo.processInfo.arguments.contains(flag) + } +} + +enum ClickyPaths { + /// Test-only override. Production never sets this. + static var overrideHomeForTesting: URL? + + static var home: URL { + if let overrideHomeForTesting { + return overrideHomeForTesting + } + if let argumentPath = ClickyLaunchArguments.value(forPrefix: "-CLICKY_HOME=") { + return URL(fileURLWithPath: argumentPath, isDirectory: true) + } + if let environmentPath = ProcessInfo.processInfo.environment["CLICKY_HOME"], + !environmentPath.isEmpty { + return URL(fileURLWithPath: environmentPath, isDirectory: true) + } + return FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent(".clicky", isDirectory: true) + } + + static var skills: URL { + home.appendingPathComponent("skills", isDirectory: true) + } + + static var topicHistory: URL { + home.appendingPathComponent("topic-history.json") + } +} + +enum ClickyDefaults { + static let shared: UserDefaults = { + if let suiteName = ClickyLaunchArguments.value(forPrefix: "-CLICKY_DEFAULTS_SUITE="), + let suiteDefaults = UserDefaults(suiteName: suiteName) { + return suiteDefaults + } + return .standard + }() +} diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index 225fc50cf..ef4bc2cfd 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -91,9 +91,9 @@ final class CompanionManager: ObservableObject { @Published private(set) var teachingSkills: [TeachingSkill] = [] /// When disabled, Clicky still reads skills but will not create new ones. - @Published var isLearningFromSessionsEnabled: Bool = UserDefaults.standard.object(forKey: "isLearningFromSessionsEnabled") == nil + @Published var isLearningFromSessionsEnabled: Bool = ClickyDefaults.shared.object(forKey: "isLearningFromSessionsEnabled") == nil ? true - : UserDefaults.standard.bool(forKey: "isLearningFromSessionsEnabled") + : ClickyDefaults.shared.bool(forKey: "isLearningFromSessionsEnabled") /// Exposed for automated E2E assertions. @Published private(set) var lastSystemPrompt: String? @@ -139,11 +139,11 @@ final class CompanionManager: ObservableObject { @Published private(set) var isOverlayVisible: Bool = false /// The Claude model used for voice responses. Persisted to UserDefaults. - @Published var selectedModel: String = UserDefaults.standard.string(forKey: "selectedClaudeModel") ?? "claude-sonnet-4-6" + @Published var selectedModel: String = ClickyDefaults.shared.string(forKey: "selectedClaudeModel") ?? "claude-sonnet-4-6" func setLearningFromSessionsEnabled(_ enabled: Bool) { isLearningFromSessionsEnabled = enabled - UserDefaults.standard.set(enabled, forKey: "isLearningFromSessionsEnabled") + ClickyDefaults.shared.set(enabled, forKey: "isLearningFromSessionsEnabled") } func refreshTeachingSkills() { @@ -384,20 +384,20 @@ final class CompanionManager: ObservableObject { func setSelectedModel(_ model: String) { selectedModel = model - UserDefaults.standard.set(model, forKey: "selectedClaudeModel") + ClickyDefaults.shared.set(model, forKey: "selectedClaudeModel") claudeAPI.model = model } /// User preference for whether the Clicky cursor should be shown. /// When toggled off, the overlay is hidden and push-to-talk is disabled. /// Persisted to UserDefaults so the choice survives app restarts. - @Published var isClickyCursorEnabled: Bool = UserDefaults.standard.object(forKey: "isClickyCursorEnabled") == nil + @Published var isClickyCursorEnabled: Bool = ClickyDefaults.shared.object(forKey: "isClickyCursorEnabled") == nil ? true - : UserDefaults.standard.bool(forKey: "isClickyCursorEnabled") + : ClickyDefaults.shared.bool(forKey: "isClickyCursorEnabled") func setClickyCursorEnabled(_ enabled: Bool) { isClickyCursorEnabled = enabled - UserDefaults.standard.set(enabled, forKey: "isClickyCursorEnabled") + ClickyDefaults.shared.set(enabled, forKey: "isClickyCursorEnabled") transientHideTask?.cancel() transientHideTask = nil @@ -414,12 +414,12 @@ final class CompanionManager: ObservableObject { /// Whether the user has completed onboarding at least once. Persisted /// to UserDefaults so the Start button only appears on first launch. var hasCompletedOnboarding: Bool { - get { UserDefaults.standard.bool(forKey: "hasCompletedOnboarding") } - set { UserDefaults.standard.set(newValue, forKey: "hasCompletedOnboarding") } + get { ClickyDefaults.shared.bool(forKey: "hasCompletedOnboarding") } + set { ClickyDefaults.shared.set(newValue, forKey: "hasCompletedOnboarding") } } /// Whether the user has submitted their email during onboarding. - @Published var hasSubmittedEmail: Bool = UserDefaults.standard.bool(forKey: "hasSubmittedEmail") + @Published var hasSubmittedEmail: Bool = ClickyDefaults.shared.bool(forKey: "hasSubmittedEmail") /// Submits the user's email to FormSpark and identifies them in PostHog. func submitEmail(_ email: String) { @@ -427,7 +427,7 @@ final class CompanionManager: ObservableObject { guard !trimmedEmail.isEmpty else { return } hasSubmittedEmail = true - UserDefaults.standard.set(true, forKey: "hasSubmittedEmail") + ClickyDefaults.shared.set(true, forKey: "hasSubmittedEmail") // Identify user in PostHog PostHogSDK.shared.identify(trimmedEmail, userProperties: [ @@ -625,7 +625,7 @@ final class CompanionManager: ObservableObject { // Screen content permission is persisted — once the user has approved the // SCShareableContent picker, we don't need to re-check it. if !hasScreenContentPermission { - hasScreenContentPermission = UserDefaults.standard.bool(forKey: "hasScreenContentPermission") + hasScreenContentPermission = ClickyDefaults.shared.bool(forKey: "hasScreenContentPermission") } if !previouslyHadAll && allPermissionsGranted { @@ -661,7 +661,7 @@ final class CompanionManager: ObservableObject { isRequestingScreenContent = false guard didCapture else { return } hasScreenContentPermission = true - UserDefaults.standard.set(true, forKey: "hasScreenContentPermission") + ClickyDefaults.shared.set(true, forKey: "hasScreenContentPermission") ClickyAnalytics.trackPermissionGranted(permission: "screen_content") // If onboarding was already completed, show the cursor overlay now diff --git a/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift b/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift index 50b87808d..02b9a72eb 100644 --- a/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift +++ b/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift @@ -37,6 +37,11 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { } func start() { + if ClickyLaunchArguments.isPresent("-CLICKY_DISABLE_GLOBAL_PTT=1") { + Self.logger.notice("Global push-to-talk disabled via launch flag") + return + } + guard WindowPositionManager.hasInputMonitoringPermission() else { Self.logger.error("Input Monitoring permission missing — push-to-talk hotkey inactive") stop() diff --git a/leanring-buddy/TeachingSkillStore.swift b/leanring-buddy/TeachingSkillStore.swift index 67e5c811e..86a1488d4 100644 --- a/leanring-buddy/TeachingSkillStore.swift +++ b/leanring-buddy/TeachingSkillStore.swift @@ -8,10 +8,9 @@ import Foundation final class TeachingSkillStore { - static let skillsRootURL: URL = { - let home = FileManager.default.homeDirectoryForCurrentUser - return home.appendingPathComponent(".clicky/skills", isDirectory: true) - }() + static var skillsRootURL: URL { + ClickyPaths.skills + } private(set) var skills: [TeachingSkill] = [] diff --git a/leanring-buddy/TeachingTopicHistoryStore.swift b/leanring-buddy/TeachingTopicHistoryStore.swift index e708d73a7..bc5ce2d94 100644 --- a/leanring-buddy/TeachingTopicHistoryStore.swift +++ b/leanring-buddy/TeachingTopicHistoryStore.swift @@ -15,10 +15,9 @@ struct TeachingTopicHistoryEntry: Codable, Equatable { } final class TeachingTopicHistoryStore { - static let historyFileURL: URL = { - FileManager.default.homeDirectoryForCurrentUser - .appendingPathComponent(".clicky/topic-history.json") - }() + static var historyFileURL: URL { + ClickyPaths.topicHistory + } private(set) var entries: [TeachingTopicHistoryEntry] = [] private let maxEntryCount = 200 diff --git a/leanring-buddy/WindowPositionManager.swift b/leanring-buddy/WindowPositionManager.swift index 4af437ca8..f5517f6d0 100644 --- a/leanring-buddy/WindowPositionManager.swift +++ b/leanring-buddy/WindowPositionManager.swift @@ -78,7 +78,10 @@ class WindowPositionManager { // MARK: - Input Monitoring Permission /// Returns true if the app can install the global CGEvent tap used for push-to-talk. - static func hasInputMonitoringPermission() -> Bool { + /// `nonisolated` because it only reads the live TCC state via a pure C preflight call + /// (no main-actor state), so the nonisolated push-to-talk monitor can call it directly. + /// Without this, Xcode 16.4 rejects the call from `GlobalPushToTalkShortcutMonitor.start()`. + nonisolated static func hasInputMonitoringPermission() -> Bool { CGPreflightListenEventAccess() } @@ -115,7 +118,7 @@ class WindowPositionManager { static func hasScreenRecordingPermission() -> Bool { let hasScreenRecordingPermissionNow = CGPreflightScreenCaptureAccess() if hasScreenRecordingPermissionNow { - UserDefaults.standard.set(true, forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) + ClickyDefaults.shared.set(true, forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) } return hasScreenRecordingPermissionNow } @@ -127,7 +130,7 @@ class WindowPositionManager { static func shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch() -> Bool { shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch( hasScreenRecordingPermissionNow: hasScreenRecordingPermission(), - hasPreviouslyConfirmedScreenRecordingPermission: UserDefaults.standard.bool(forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) + hasPreviouslyConfirmedScreenRecordingPermission: ClickyDefaults.shared.bool(forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) ) } @@ -139,7 +142,7 @@ class WindowPositionManager { } static func clearPreviouslyConfirmedScreenRecordingPermission() { - UserDefaults.standard.removeObject(forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) + ClickyDefaults.shared.removeObject(forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) } /// Prompts the system dialog for Screen Recording permission. diff --git a/leanring-buddyTests/ClickyPathsTests.swift b/leanring-buddyTests/ClickyPathsTests.swift new file mode 100644 index 000000000..d9fb5196d --- /dev/null +++ b/leanring-buddyTests/ClickyPathsTests.swift @@ -0,0 +1,99 @@ +// +// ClickyPathsTests.swift +// leanring-buddyTests +// + +import Foundation +import Testing +@testable import leanring_buddy + +// Serialized because these tests mutate the shared `ClickyPaths.overrideHomeForTesting` +// global. Without this, Swift Testing's default parallel execution lets them clobber +// each other's override and produces flaky failures. +@Suite(.serialized) +struct ClickyPathsTests { + @Test func defaultHomeUsesClickyDirectoryInUserHome() { + ClickyPaths.overrideHomeForTesting = nil + + let expectedHome = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent(".clicky", isDirectory: true) + + #expect(ClickyPaths.home == expectedHome) + } + + @Test func overrideHomeRedirectsSkillsAndTopicHistory() throws { + let temporaryHome = FileManager.default.temporaryDirectory + .appendingPathComponent("clicky-test-\(UUID().uuidString)", isDirectory: true) + defer { + ClickyPaths.overrideHomeForTesting = nil + try? FileManager.default.removeItem(at: temporaryHome) + } + + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + ClickyPaths.overrideHomeForTesting = temporaryHome + + #expect(ClickyPaths.home == temporaryHome) + #expect(ClickyPaths.skills == temporaryHome.appendingPathComponent("skills", isDirectory: true)) + #expect(ClickyPaths.topicHistory == temporaryHome.appendingPathComponent("topic-history.json")) + } + + @Test func teachingSkillStoreUsesClickyPathsSkillsDirectory() throws { + let temporaryHome = FileManager.default.temporaryDirectory + .appendingPathComponent("clicky-test-\(UUID().uuidString)", isDirectory: true) + defer { + ClickyPaths.overrideHomeForTesting = nil + try? FileManager.default.removeItem(at: temporaryHome) + } + + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + ClickyPaths.overrideHomeForTesting = temporaryHome + + #expect(TeachingSkillStore.skillsRootURL == ClickyPaths.skills) + } + + @Test func teachingTopicHistoryStoreUsesClickyPathsTopicHistoryFile() throws { + let temporaryHome = FileManager.default.temporaryDirectory + .appendingPathComponent("clicky-test-\(UUID().uuidString)", isDirectory: true) + defer { + ClickyPaths.overrideHomeForTesting = nil + try? FileManager.default.removeItem(at: temporaryHome) + } + + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + ClickyPaths.overrideHomeForTesting = temporaryHome + + #expect(TeachingTopicHistoryStore.historyFileURL == ClickyPaths.topicHistory) + } + + @Test func teachingSkillStoreWritesToIsolatedHome() throws { + let temporaryHome = FileManager.default.temporaryDirectory + .appendingPathComponent("clicky-test-\(UUID().uuidString)", isDirectory: true) + defer { + ClickyPaths.overrideHomeForTesting = nil + try? FileManager.default.removeItem(at: temporaryHome) + } + + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + ClickyPaths.overrideHomeForTesting = temporaryHome + + let skill = TeachingSkill( + id: "teach-textedit-save", + name: "teach-textedit-save", + description: "Save in TextEdit", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: nil, + usageCount: 0, + isPinned: false, + taskSlug: nil, + body: "step one: press command-s." + ) + + let store = TeachingSkillStore() + _ = try store.saveSkill(skill) + + let skillFile = temporaryHome + .appendingPathComponent("skills/teach-textedit-save/SKILL.md") + #expect(FileManager.default.fileExists(atPath: skillFile.path)) + } +} diff --git a/leanring-buddyTests/leanring_buddyTests.swift b/leanring-buddyTests/leanring_buddyTests.swift index 188fe7ae0..73ff6f0b7 100644 --- a/leanring-buddyTests/leanring_buddyTests.swift +++ b/leanring-buddyTests/leanring_buddyTests.swift @@ -8,6 +8,7 @@ import Testing @testable import leanring_buddy +@MainActor struct leanring_buddyTests { @Test func firstPermissionRequestUsesSystemPromptOnly() async throws { diff --git a/tests/e2e/lib/common.sh b/tests/e2e/lib/common.sh index 920c97aac..ce9329d19 100755 --- a/tests/e2e/lib/common.sh +++ b/tests/e2e/lib/common.sh @@ -4,8 +4,9 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" CLICKY_APP="${CLICKY_APP:-$ROOT_DIR/build/E2E/Clicky.app}" WORKER_URL="${CLICKY_WORKER_URL:-http://127.0.0.1:8787}" -SKILLS_DIR="$HOME/.clicky/skills" -CLICKY_DIR="$HOME/.clicky" +CLICKY_DIR="${CLICKY_HOME:-$HOME/.clicky}" +SKILLS_DIR="$CLICKY_DIR/skills" +export CLICKY_HOME="$CLICKY_DIR" E2E_PROMPT_FILE="$CLICKY_DIR/e2e-last-system-prompt.txt" E2E_MATCHED_SKILL_FILE="$CLICKY_DIR/e2e-last-matched-skill-id.txt" @@ -75,7 +76,7 @@ launch_clicky() { local log_file="${1:-/tmp/clicky-e2e-app.log}" shift || true - "$CLICKY_APP/Contents/MacOS/Clicky" "$@" >"$log_file" 2>&1 & + "$CLICKY_APP/Contents/MacOS/Clicky" -CLICKY_HOME="$CLICKY_DIR" "$@" >"$log_file" 2>&1 & CLICKY_PID=$! } From f5f316de3313de0cb0ce5d6da7fc4472a35493ac Mon Sep 17 00:00:00 2001 From: Cristea-Zablau Octavian <53470992+octavi42@users.noreply.github.com> Date: Mon, 8 Jun 2026 13:10:42 +0300 Subject: [PATCH 09/14] Replace ScreenCaptureKit screenshots with Core Graphics capture. (#7) ScreenCaptureKit routes through screencaptureui on macOS Tahoe and shows floating screenshot thumbnails during voice sessions. Switch companion capture to CGWindowListCreateImage (via a runtime dlsym wrapper), suppress and dismiss leftover thumbnail UI for the app lifetime, and harden permission refresh so dev builds reflect live TCC state after granting in System Settings. Also fixes a CGWindowID crash when window numbers are out of UInt32 range. --- leanring-buddy/CompanionManager.swift | 171 ++++++++++++++---- leanring-buddy/CompanionPanelView.swift | 38 ++-- .../CompanionScreenCaptureUtility.swift | 115 +++++------- .../CoreGraphicsScreenCaptureUtility.swift | 159 ++++++++++++++++ ...ScreenshotFloatingThumbnailDismisser.swift | 58 ++++++ ...reenshotFloatingThumbnailSuppression.swift | 117 ++++++++++++ leanring-buddy/MenuBarPanelManager.swift | 2 + leanring-buddy/WindowPositionManager.swift | 42 ++++- leanring-buddy/leanring_buddyApp.swift | 10 +- 9 files changed, 589 insertions(+), 123 deletions(-) create mode 100644 leanring-buddy/CoreGraphicsScreenCaptureUtility.swift create mode 100644 leanring-buddy/MacOSScreenshotFloatingThumbnailDismisser.swift create mode 100644 leanring-buddy/MacOSScreenshotFloatingThumbnailSuppression.swift diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index ef4bc2cfd..9ed4806ae 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -12,7 +12,6 @@ import AppKit import Combine import Foundation import PostHog -import ScreenCaptureKit import SwiftUI enum CompanionVoiceState { @@ -120,11 +119,19 @@ final class CompanionManager: ObservableObject { private var voiceStateCancellable: AnyCancellable? private var audioPowerCancellable: AnyCancellable? private var accessibilityCheckTimer: Timer? + private var workspaceActivationObserver: NSObjectProtocol? + private var permissionRefreshBurstTask: Task? + private var hasLoggedPermissionDiagnostics = false private var pendingKeyboardShortcutStartTask: Task? /// Scheduled hide for transient cursor mode — cancelled if the user /// speaks again before the delay elapses. private var transientHideTask: Task? + /// Path to the Clicky.app bundle for this run. Shown when TCC must target this build. + var runningApplicationBundlePath: String { + Bundle.main.bundlePath + } + /// True when all required permissions are granted. Used by the panel to show a single "all good" state. var allPermissionsGranted: Bool { hasAccessibilityPermission @@ -449,6 +456,7 @@ final class CompanionManager: ObservableObject { refreshAllPermissions() print("🔑 Clicky start — accessibility: \(hasAccessibilityPermission), inputMonitoring: \(hasInputMonitoringPermission), pushToTalkActive: \(isPushToTalkHotkeyActive), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission), onboarded: \(hasCompletedOnboarding)") startPermissionPolling() + startWorkspaceActivationObservation() bindVoiceStateObservation() bindAudioPowerLevel() bindShortcutTransitions() @@ -575,15 +583,43 @@ final class CompanionManager: ObservableObject { audioPowerCancellable?.cancel() accessibilityCheckTimer?.invalidate() accessibilityCheckTimer = nil + permissionRefreshBurstTask?.cancel() + permissionRefreshBurstTask = nil + if let workspaceActivationObserver { + NSWorkspace.shared.notificationCenter.removeObserver(workspaceActivationObserver) + self.workspaceActivationObserver = nil + } + } + + func requestAccessibilityPermissionFromPanel() { + _ = WindowPositionManager.requestAccessibilityPermission() + schedulePermissionRefreshBurst() + } + + func requestInputMonitoringPermissionFromPanel() { + _ = WindowPositionManager.requestInputMonitoringPermission() + schedulePermissionRefreshBurst() + } + + func requestScreenRecordingPermissionFromPanel() { + _ = WindowPositionManager.requestScreenRecordingPermission() + schedulePermissionRefreshBurst() + } + + func schedulePermissionRefreshBurstAfterReturningFromSettings() { + schedulePermissionRefreshBurst() } func refreshAllPermissions() { + clearStalePermissionUserDefaultsIfLiveTCCDenied() + let previouslyHadAccessibility = hasAccessibilityPermission let previouslyHadInputMonitoring = hasInputMonitoringPermission let previouslyHadScreenRecording = hasScreenRecordingPermission let previouslyHadMicrophone = hasMicrophonePermission let previouslyHadAll = allPermissionsGranted + WindowPositionManager.refreshAccessibilityTrustCache() let currentlyHasAccessibility = WindowPositionManager.hasAccessibilityPermission() hasAccessibilityPermission = currentlyHasAccessibility @@ -596,7 +632,8 @@ final class CompanionManager: ObservableObject { globalPushToTalkShortcutMonitor.stop() } - hasScreenRecordingPermission = WindowPositionManager.hasScreenRecordingPermission() + hasScreenRecordingPermission = WindowPositionManager + .shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch() let micAuthStatus = AVCaptureDevice.authorizationStatus(for: .audio) hasMicrophonePermission = micAuthStatus == .authorized @@ -622,62 +659,85 @@ final class CompanionManager: ObservableObject { if !previouslyHadMicrophone && hasMicrophonePermission { ClickyAnalytics.trackPermissionGranted(permission: "microphone") } - // Screen content permission is persisted — once the user has approved the - // SCShareableContent picker, we don't need to re-check it. - if !hasScreenContentPermission { - hasScreenContentPermission = ClickyDefaults.shared.bool(forKey: "hasScreenContentPermission") - } + refreshScreenContentPermissionFromCaptureTest() if !previouslyHadAll && allPermissionsGranted { ClickyAnalytics.trackAllPermissionsGranted() } + + if !allPermissionsGranted && !hasLoggedPermissionDiagnostics { + hasLoggedPermissionDiagnostics = true + WindowPositionManager.logPermissionDiagnosticsSnapshot() + } } - /// Triggers the macOS screen content picker by performing a dummy - /// screenshot capture. Once the user approves, we persist the grant - /// so they're never asked again during onboarding. + /// Screen content uses the same macOS screen-recording permission as capture. + /// This avoids ScreenCaptureKit, which on Tahoe triggers screencaptureui popups. @Published private(set) var isRequestingScreenContent = false func requestScreenContentPermission() { guard !isRequestingScreenContent else { return } isRequestingScreenContent = true + + if !hasScreenRecordingPermission { + _ = WindowPositionManager.requestScreenRecordingPermission() + } + Task { - do { - let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) - guard let display = content.displays.first else { - await MainActor.run { isRequestingScreenContent = false } - return - } - let filter = SCContentFilter(display: display, excludingWindows: []) - let config = SCStreamConfiguration() - config.width = 320 - config.height = 240 - let image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config) - // Verify the capture actually returned real content — a 0x0 or - // fully-empty image means the user denied the prompt. - let didCapture = image.width > 0 && image.height > 0 - print("🔑 Screen content capture result — width: \(image.width), height: \(image.height), didCapture: \(didCapture)") - await MainActor.run { - isRequestingScreenContent = false - guard didCapture else { return } - hasScreenContentPermission = true - ClickyDefaults.shared.set(true, forKey: "hasScreenContentPermission") + try? await Task.sleep(nanoseconds: 500_000_000) + await MainActor.run { + self.isRequestingScreenContent = false + self.refreshScreenContentPermissionFromCaptureTest() + + if self.hasScreenContentPermission { ClickyAnalytics.trackPermissionGranted(permission: "screen_content") + } - // If onboarding was already completed, show the cursor overlay now - if hasCompletedOnboarding && allPermissionsGranted && !isOverlayVisible && isClickyCursorEnabled { - overlayWindowManager.hasShownOverlayBefore = true - overlayWindowManager.showOverlay(onScreens: NSScreen.screens, companionManager: self) - isOverlayVisible = true - } + if self.hasCompletedOnboarding + && self.allPermissionsGranted + && !self.isOverlayVisible + && self.isClickyCursorEnabled { + self.overlayWindowManager.hasShownOverlayBefore = true + self.overlayWindowManager.showOverlay(onScreens: NSScreen.screens, companionManager: self) + self.isOverlayVisible = true } - } catch { - print("⚠️ Screen content permission request failed: \(error)") - await MainActor.run { isRequestingScreenContent = false } } } } + private func refreshScreenContentPermissionFromCaptureTest() { + guard WindowPositionManager.isScreenCapturePreflightGranted() else { + hasScreenContentPermission = false + return + } + + if ClickyDefaults.shared.bool(forKey: "hasScreenContentPermission") { + hasScreenContentPermission = true + return + } + + guard hasScreenRecordingPermission else { + hasScreenContentPermission = false + return + } + guard CompanionScreenCaptureUtility.verifyScreenCaptureAccess() else { + hasScreenContentPermission = false + return + } + + hasScreenContentPermission = true + ClickyDefaults.shared.set(true, forKey: "hasScreenContentPermission") + } + + /// Old builds can leave screen-permission prefs set while this binary has no TCC access. + private func clearStalePermissionUserDefaultsIfLiveTCCDenied() { + guard !WindowPositionManager.isScreenCapturePreflightGranted() else { return } + + WindowPositionManager.clearPreviouslyConfirmedScreenRecordingPermission() + WindowPositionManager.clearCachedScreenContentPermission() + hasScreenContentPermission = false + } + // MARK: - Private /// Triggers the system microphone prompt if the user has never been asked. @@ -695,11 +755,42 @@ final class CompanionManager: ObservableObject { /// user grants them in System Settings. Screen Recording is the exception — /// macOS requires an app restart for that one to take effect. private func startPermissionPolling() { - accessibilityCheckTimer = Timer.scheduledTimer(withTimeInterval: 1.5, repeats: true) { [weak self] _ in + let permissionPollingTimer = Timer(timeInterval: 1.5, repeats: true) { [weak self] _ in Task { @MainActor [weak self] in self?.refreshAllPermissions() } } + // .common keeps polling alive while the user is in System Settings. + RunLoop.main.add(permissionPollingTimer, forMode: .common) + accessibilityCheckTimer = permissionPollingTimer + } + + private func startWorkspaceActivationObservation() { + workspaceActivationObserver = NSWorkspace.shared.notificationCenter.addObserver( + forName: NSWorkspace.didActivateApplicationNotification, + object: nil, + queue: .main + ) { [weak self] notification in + guard let self else { return } + guard let activatedApplication = notification.userInfo?[NSWorkspace.applicationUserInfoKey] as? NSRunningApplication, + activatedApplication.bundleIdentifier == Bundle.main.bundleIdentifier else { + return + } + self.refreshAllPermissions() + } + } + + /// Polls several times after a Grant tap or return from System Settings. + private func schedulePermissionRefreshBurst() { + permissionRefreshBurstTask?.cancel() + permissionRefreshBurstTask = Task { @MainActor [weak self] in + guard let self else { return } + for _ in 0..<12 { + if Task.isCancelled { return } + refreshAllPermissions() + try? await Task.sleep(nanoseconds: 1_000_000_000) + } + } } private func bindAudioPowerLevel() { diff --git a/leanring-buddy/CompanionPanelView.swift b/leanring-buddy/CompanionPanelView.swift index 09e7e7223..910e57f75 100644 --- a/leanring-buddy/CompanionPanelView.swift +++ b/leanring-buddy/CompanionPanelView.swift @@ -199,6 +199,16 @@ struct CompanionPanelView: View { .font(.system(size: 11)) .foregroundColor(DS.Colors.textTertiary) .fixedSize(horizontal: false, vertical: true) + + Text("Enable THIS build: tap Find App, drag Clicky.app into the list, then quit and reopen Clicky.") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.warning) + .fixedSize(horizontal: false, vertical: true) + + Text(companionManager.runningApplicationBundlePath) + .font(.system(size: 9, design: .monospaced)) + .foregroundColor(DS.Colors.textTertiary) + .fixedSize(horizontal: false, vertical: true) } .frame(maxWidth: .infinity, alignment: .leading) } else { @@ -316,9 +326,17 @@ struct CompanionPanelView: View { .foregroundColor(isGranted ? DS.Colors.textTertiary : DS.Colors.warning) .frame(width: 16) - Text("Accessibility") - .font(.system(size: 13, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) + VStack(alignment: .leading, spacing: 1) { + Text("Accessibility") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + + if !isGranted { + Text("Use Find App, enable this build, then quit and reopen") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + } + } } Spacer() @@ -337,7 +355,7 @@ struct CompanionPanelView: View { Button(action: { // Triggers the system accessibility prompt (AXIsProcessTrustedWithOptions) // on first attempt, then opens System Settings on subsequent attempts. - WindowPositionManager.requestAccessibilityPermission() + companionManager.requestAccessibilityPermissionFromPanel() }) { Text("Grant") .font(.system(size: 11, weight: .semibold)) @@ -356,8 +374,7 @@ struct CompanionPanelView: View { // Reveals the app in Finder so the user can drag it into // the Accessibility list if it doesn't appear automatically // (common with unsigned dev builds). - WindowPositionManager.revealAppInFinder() - WindowPositionManager.openAccessibilitySettings() + WindowPositionManager.prepareAccessibilityReGrantFromFinder() }) { Text("Find App") .font(.system(size: 11, weight: .semibold)) @@ -393,7 +410,7 @@ struct CompanionPanelView: View { Text(isGranted ? "Required for Control+Option push-to-talk" - : "Enable this for the global hotkey to work") + : "Use Find App, enable this build, then quit and reopen") .font(.system(size: 10)) .foregroundColor(DS.Colors.textTertiary) } @@ -413,7 +430,7 @@ struct CompanionPanelView: View { } else { HStack(spacing: 6) { Button(action: { - WindowPositionManager.requestInputMonitoringPermission() + companionManager.requestInputMonitoringPermissionFromPanel() }) { Text("Grant") .font(.system(size: 11, weight: .semibold)) @@ -429,8 +446,7 @@ struct CompanionPanelView: View { .pointerCursor() Button(action: { - WindowPositionManager.revealAppInFinder() - WindowPositionManager.openInputMonitoringSettings() + WindowPositionManager.prepareInputMonitoringReGrantFromFinder() }) { Text("Find App") .font(.system(size: 11, weight: .semibold)) @@ -488,7 +504,7 @@ struct CompanionPanelView: View { // Triggers the native macOS screen recording prompt on first // attempt (auto-adds app to the list), then opens System Settings // on subsequent attempts. - WindowPositionManager.requestScreenRecordingPermission() + companionManager.requestScreenRecordingPermissionFromPanel() }) { Text("Grant") .font(.system(size: 11, weight: .semibold)) diff --git a/leanring-buddy/CompanionScreenCaptureUtility.swift b/leanring-buddy/CompanionScreenCaptureUtility.swift index 797841783..dbd5c98c4 100644 --- a/leanring-buddy/CompanionScreenCaptureUtility.swift +++ b/leanring-buddy/CompanionScreenCaptureUtility.swift @@ -8,7 +8,6 @@ // import AppKit -import ScreenCaptureKit struct CompanionScreenCapture { let imageData: Data @@ -23,110 +22,90 @@ struct CompanionScreenCapture { @MainActor enum CompanionScreenCaptureUtility { + private static let maxScreenshotDimensionInPixels = 1280 /// Captures all connected displays as JPEG data, labeling each with /// whether the user's cursor is on that screen. This gives the AI /// full context across multiple monitors. static func captureAllScreensAsJPEG() async throws -> [CompanionScreenCapture] { - let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) - - guard !content.displays.isEmpty else { - throw NSError(domain: "CompanionScreenCapture", code: -1, - userInfo: [NSLocalizedDescriptionKey: "No display available for capture"]) + guard WindowPositionManager.hasScreenRecordingPermission() else { + throw NSError( + domain: "CompanionScreenCapture", + code: -4, + userInfo: [NSLocalizedDescriptionKey: "Screen recording permission is required"] + ) } - let mouseLocation = NSEvent.mouseLocation - - // Exclude all windows belonging to this app so the AI sees - // only the user's content, not our overlays or panels. - let ownBundleIdentifier = Bundle.main.bundleIdentifier - let ownAppWindows = content.windows.filter { window in - window.owningApplication?.bundleIdentifier == ownBundleIdentifier - } + MacOSScreenshotFloatingThumbnailDismisser.dismissIfNeeded() - // Build a lookup from display ID to NSScreen so we can use AppKit-coordinate - // frames instead of CG-coordinate frames. NSEvent.mouseLocation and NSScreen.frame - // both use AppKit coordinates (bottom-left origin), while SCDisplay.frame uses - // Core Graphics coordinates (top-left origin). On multi-display setups, the Y - // origins differ for secondary displays, which breaks cursor-contains checks - // and downstream coordinate conversions. - var nsScreenByDisplayID: [CGDirectDisplayID: NSScreen] = [:] - for screen in NSScreen.screens { - if let screenNumber = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID { - nsScreenByDisplayID[screenNumber] = screen - } - } - - // Sort displays so the cursor screen is always first - let sortedDisplays = content.displays.sorted { displayA, displayB in - let frameA = nsScreenByDisplayID[displayA.displayID]?.frame ?? displayA.frame - let frameB = nsScreenByDisplayID[displayB.displayID]?.frame ?? displayB.frame - let aContainsCursor = frameA.contains(mouseLocation) - let bContainsCursor = frameB.contains(mouseLocation) + let mouseLocation = NSEvent.mouseLocation + let sortedScreens = NSScreen.screens.sorted { screenA, screenB in + let aContainsCursor = screenA.frame.contains(mouseLocation) + let bContainsCursor = screenB.frame.contains(mouseLocation) if aContainsCursor != bContainsCursor { return aContainsCursor } return false } + guard !sortedScreens.isEmpty else { + throw NSError( + domain: "CompanionScreenCapture", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "No display available for capture"] + ) + } + var capturedScreens: [CompanionScreenCapture] = [] - for (displayIndex, display) in sortedDisplays.enumerated() { - // Use NSScreen.frame (AppKit coordinates, bottom-left origin) so - // displayFrame is in the same coordinate system as NSEvent.mouseLocation - // and the overlay window's screenFrame in BlueCursorView. - let displayFrame = nsScreenByDisplayID[display.displayID]?.frame - ?? CGRect(x: display.frame.origin.x, y: display.frame.origin.y, - width: CGFloat(display.width), height: CGFloat(display.height)) + for (displayIndex, screen) in sortedScreens.enumerated() { + let displayFrame = screen.frame let isCursorScreen = displayFrame.contains(mouseLocation) + let belowWindowID = CoreGraphicsScreenCaptureUtility.topmostOwnedWindowID(on: screen) - let filter = SCContentFilter(display: display, excludingWindows: ownAppWindows) - - let configuration = SCStreamConfiguration() - let maxDimension = 1280 - let aspectRatio = CGFloat(display.width) / CGFloat(display.height) - if display.width >= display.height { - configuration.width = maxDimension - configuration.height = Int(CGFloat(maxDimension) / aspectRatio) - } else { - configuration.height = maxDimension - configuration.width = Int(CGFloat(maxDimension) * aspectRatio) - } - - let cgImage = try await SCScreenshotManager.captureImage( - contentFilter: filter, - configuration: configuration - ) - - guard let jpegData = NSBitmapImageRep(cgImage: cgImage) - .representation(using: .jpeg, properties: [.compressionFactor: 0.8]) else { + guard let capture = CoreGraphicsScreenCaptureUtility.captureScreenAsJPEG( + screen: screen, + belowWindowID: belowWindowID, + maxDimension: maxScreenshotDimensionInPixels + ) else { continue } let screenLabel: String - if sortedDisplays.count == 1 { + if sortedScreens.count == 1 { screenLabel = "user's screen (cursor is here)" } else if isCursorScreen { - screenLabel = "screen \(displayIndex + 1) of \(sortedDisplays.count) — cursor is on this screen (primary focus)" + screenLabel = "screen \(displayIndex + 1) of \(sortedScreens.count) — cursor is on this screen (primary focus)" } else { - screenLabel = "screen \(displayIndex + 1) of \(sortedDisplays.count) — secondary screen" + screenLabel = "screen \(displayIndex + 1) of \(sortedScreens.count) — secondary screen" } capturedScreens.append(CompanionScreenCapture( - imageData: jpegData, + imageData: capture.imageData, label: screenLabel, isCursorScreen: isCursorScreen, displayWidthInPoints: Int(displayFrame.width), displayHeightInPoints: Int(displayFrame.height), displayFrame: displayFrame, - screenshotWidthInPixels: configuration.width, - screenshotHeightInPixels: configuration.height + screenshotWidthInPixels: capture.screenshotWidthInPixels, + screenshotHeightInPixels: capture.screenshotHeightInPixels )) } + MacOSScreenshotFloatingThumbnailDismisser.dismissIfNeeded() + MacOSScreenshotFloatingThumbnailDismisser.cleanupTemporaryScreenshotArtifacts() + guard !capturedScreens.isEmpty else { - throw NSError(domain: "CompanionScreenCapture", code: -2, - userInfo: [NSLocalizedDescriptionKey: "Failed to capture any screen"]) + throw NSError( + domain: "CompanionScreenCapture", + code: -2, + userInfo: [NSLocalizedDescriptionKey: "Failed to capture any screen"] + ) } return capturedScreens } + + /// Verifies screen capture works without invoking ScreenCaptureKit. + static func verifyScreenCaptureAccess(maxDimension: Int = 64) -> Bool { + CoreGraphicsScreenCaptureUtility.canCaptureAnyScreen(maxDimension: maxDimension) + } } diff --git a/leanring-buddy/CoreGraphicsScreenCaptureUtility.swift b/leanring-buddy/CoreGraphicsScreenCaptureUtility.swift new file mode 100644 index 000000000..17f20b054 --- /dev/null +++ b/leanring-buddy/CoreGraphicsScreenCaptureUtility.swift @@ -0,0 +1,159 @@ +// +// CoreGraphicsScreenCaptureUtility.swift +// leanring-buddy +// +// Captures the screen with CGWindowListCreateImage instead of ScreenCaptureKit. +// On macOS Tahoe, ScreenCaptureKit screenshot/stream APIs route through +// screencaptureui and show the floating screenshot thumbnail popup. +// + +import AppKit +import CoreGraphics +import Darwin + +struct CoreGraphicsScreenJPEGCapture { + let imageData: Data + let screenshotWidthInPixels: Int + let screenshotHeightInPixels: Int +} + +enum CoreGraphicsScreenCaptureUtility { + static func captureScreenAsJPEG( + screen: NSScreen, + belowWindowID: CGWindowID?, + maxDimension: Int + ) -> CoreGraphicsScreenJPEGCapture? { + let quartzCaptureFrame = screen.quartzCaptureFrame + let captureOptions: CGWindowListOption = belowWindowID == nil + ? .optionOnScreenOnly + : .optionOnScreenBelowWindow + let referenceWindowID = belowWindowID ?? kCGNullWindowID + + guard let capturedImage = LegacyCoreGraphicsWindowCapture.createImage( + in: quartzCaptureFrame, + listOption: captureOptions, + windowID: referenceWindowID, + imageOptions: [.bestResolution, .boundsIgnoreFraming] + ) else { + return nil + } + + guard let resizedImage = resizeImage(capturedImage, maxDimension: maxDimension), + let jpegData = NSBitmapImageRep(cgImage: resizedImage) + .representation(using: .jpeg, properties: [.compressionFactor: 0.8]) else { + return nil + } + + return CoreGraphicsScreenJPEGCapture( + imageData: jpegData, + screenshotWidthInPixels: resizedImage.width, + screenshotHeightInPixels: resizedImage.height + ) + } + + static func topmostOwnedWindowID(on screen: NSScreen) -> CGWindowID? { + let visibleOwnedWindows = NSApp.windows.filter { window in + window.isVisible + && window.alphaValue > 0 + && screen.frame.intersects(window.frame) + } + + guard let topmostOwnedWindow = visibleOwnedWindows.max(by: { $0.windowNumber < $1.windowNumber }) else { + return nil + } + + // NSWindow.windowNumber is an Int that can be negative or exceed UInt32 on + // recent macOS (e.g. status bar windows), so convert safely instead of + // force-trapping with the non-failable CGWindowID initializer. + guard topmostOwnedWindow.windowNumber > 0, + let topmostOwnedWindowID = CGWindowID(exactly: topmostOwnedWindow.windowNumber) else { + return nil + } + + return topmostOwnedWindowID + } + + static func canCaptureAnyScreen(maxDimension: Int = 64) -> Bool { + guard let screen = NSScreen.main ?? NSScreen.screens.first else { return false } + return captureScreenAsJPEG( + screen: screen, + belowWindowID: topmostOwnedWindowID(on: screen), + maxDimension: maxDimension + ) != nil + } + + private static func resizeImage(_ image: CGImage, maxDimension: Int) -> CGImage? { + let sourceWidth = image.width + let sourceHeight = image.height + guard sourceWidth > 0, sourceHeight > 0 else { return nil } + + let aspectRatio = CGFloat(sourceWidth) / CGFloat(sourceHeight) + let targetWidth: Int + let targetHeight: Int + + if sourceWidth >= sourceHeight { + targetWidth = maxDimension + targetHeight = max(1, Int(CGFloat(maxDimension) / aspectRatio)) + } else { + targetHeight = maxDimension + targetWidth = max(1, Int(CGFloat(maxDimension) * aspectRatio)) + } + + guard let context = CGContext( + data: nil, + width: targetWidth, + height: targetHeight, + bitsPerComponent: 8, + bytesPerRow: 0, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) else { + return nil + } + + context.interpolationQuality = .high + context.draw(image, in: CGRect(x: 0, y: 0, width: targetWidth, height: targetHeight)) + return context.makeImage() + } +} + +/// Runtime wrapper around CGWindowListCreateImage. The symbol is marked +/// unavailable in the macOS 15 SDK, but it still works and does not route +/// through Tahoe's screencaptureui floating thumbnail. +private enum LegacyCoreGraphicsWindowCapture { + private typealias CGWindowListCreateImageFunction = @convention(c) ( + CGRect, + CGWindowListOption, + CGWindowID, + CGWindowImageOption + ) -> Unmanaged? + + private static let createImageFunction: CGWindowListCreateImageFunction? = { + guard let symbol = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "CGWindowListCreateImage") else { + return nil + } + return unsafeBitCast(symbol, to: CGWindowListCreateImageFunction.self) + }() + + static func createImage( + in screenBounds: CGRect, + listOption: CGWindowListOption, + windowID: CGWindowID, + imageOptions: CGWindowImageOption + ) -> CGImage? { + createImageFunction?(screenBounds, listOption, windowID, imageOptions)?.takeRetainedValue() + } +} + +private extension NSScreen { + var quartzCaptureFrame: CGRect { + let screenFrame = frame + guard let primaryScreen = NSScreen.screens.first else { return screenFrame } + return CGRect( + x: screenFrame.origin.x, + y: primaryScreen.frame.height - screenFrame.origin.y - screenFrame.height, + width: screenFrame.width, + height: screenFrame.height + ) + } +} diff --git a/leanring-buddy/MacOSScreenshotFloatingThumbnailDismisser.swift b/leanring-buddy/MacOSScreenshotFloatingThumbnailDismisser.swift new file mode 100644 index 000000000..99647fe02 --- /dev/null +++ b/leanring-buddy/MacOSScreenshotFloatingThumbnailDismisser.swift @@ -0,0 +1,58 @@ +// +// MacOSScreenshotFloatingThumbnailDismisser.swift +// leanring-buddy +// +// macOS Tahoe keeps screencaptureui alive and shows floating screenshot +// thumbnails when ScreenCaptureKit takes captures. Clicky now captures via +// Core Graphics, but we still tear down leftover thumbnail UI if it appears. +// + +import Foundation + +enum MacOSScreenshotFloatingThumbnailDismisser { + private static let screencaptureUIProcessNames = [ + "screencaptureui", + ] + + static func dismissIfNeeded() { + for processName in screencaptureUIProcessNames { + terminateProcess(named: processName) + } + } + + static func cleanupTemporaryScreenshotArtifacts() { + let temporaryItemsDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent("TemporaryItems", isDirectory: true) + + guard let temporaryItemURLs = try? FileManager.default.contentsOfDirectory( + at: temporaryItemsDirectory, + includingPropertiesForKeys: nil + ) else { + return + } + + for temporaryItemURL in temporaryItemURLs where temporaryItemURL.lastPathComponent + .localizedCaseInsensitiveContains("screencaptureui") { + try? FileManager.default.removeItem(at: temporaryItemURL) + } + + scheduleFollowUpDismissal() + } + + private static func scheduleFollowUpDismissal() { + Task { + for _ in 0..<3 { + try? await Task.sleep(nanoseconds: 150_000_000) + dismissIfNeeded() + } + } + } + + private static func terminateProcess(named processName: String) { + let terminateProcess = Process() + terminateProcess.executableURL = URL(fileURLWithPath: "/usr/bin/killall") + terminateProcess.arguments = ["-9", processName] + try? terminateProcess.run() + terminateProcess.waitUntilExit() + } +} diff --git a/leanring-buddy/MacOSScreenshotFloatingThumbnailSuppression.swift b/leanring-buddy/MacOSScreenshotFloatingThumbnailSuppression.swift new file mode 100644 index 000000000..6d3079844 --- /dev/null +++ b/leanring-buddy/MacOSScreenshotFloatingThumbnailSuppression.swift @@ -0,0 +1,117 @@ +// +// MacOSScreenshotFloatingThumbnailSuppression.swift +// leanring-buddy +// +// Keeps macOS from showing screenshot floating thumbnails while Clicky is +// running. Restores the user's prior preference when the app quits. +// + +import CoreFoundation +import Foundation + +enum MacOSScreenshotFloatingThumbnailSuppression { + private static let screencapturePreferenceDomain = "com.apple.screencapture" as CFString + private static let screencaptureUIPreferenceDomain = "com.apple.screencaptureui" as CFString + private static let showThumbnailPreferenceKey = "show-thumbnail" as CFString + private static let thumbnailExpirationPreferenceKey = "thumbnailExpiration" as CFString + private static var savedShowThumbnailPreference: CFPropertyList? + private static var savedThumbnailExpirationPreference: CFPropertyList? + private static var isSuppressingForAppLifetime = false + + static func enableForAppLifetime() { + guard !isSuppressingForAppLifetime else { return } + + savedShowThumbnailPreference = CFPreferencesCopyAppValue( + showThumbnailPreferenceKey, + screencapturePreferenceDomain + ) + savedThumbnailExpirationPreference = CFPreferencesCopyAppValue( + thumbnailExpirationPreferenceKey, + screencaptureUIPreferenceDomain + ) + + CFPreferencesSetValue( + showThumbnailPreferenceKey, + kCFBooleanFalse, + screencapturePreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + CFPreferencesSetValue( + thumbnailExpirationPreferenceKey, + 0.001 as CFPropertyList, + screencaptureUIPreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + + CFPreferencesSynchronize( + screencapturePreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + CFPreferencesSynchronize( + screencaptureUIPreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + + isSuppressingForAppLifetime = true + MacOSScreenshotFloatingThumbnailDismisser.dismissIfNeeded() + } + + static func restoreAfterAppTermination() { + guard isSuppressingForAppLifetime else { return } + + if let savedShowThumbnailPreference { + CFPreferencesSetValue( + showThumbnailPreferenceKey, + savedShowThumbnailPreference, + screencapturePreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + } else { + CFPreferencesSetValue( + showThumbnailPreferenceKey, + nil, + screencapturePreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + } + + if let savedThumbnailExpirationPreference { + CFPreferencesSetValue( + thumbnailExpirationPreferenceKey, + savedThumbnailExpirationPreference, + screencaptureUIPreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + } else { + CFPreferencesSetValue( + thumbnailExpirationPreferenceKey, + nil, + screencaptureUIPreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + } + + CFPreferencesSynchronize( + screencapturePreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + CFPreferencesSynchronize( + screencaptureUIPreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + + savedShowThumbnailPreference = nil + savedThumbnailExpirationPreference = nil + isSuppressingForAppLifetime = false + } +} diff --git a/leanring-buddy/MenuBarPanelManager.swift b/leanring-buddy/MenuBarPanelManager.swift index e5eb98de7..3a56d5fed 100644 --- a/leanring-buddy/MenuBarPanelManager.swift +++ b/leanring-buddy/MenuBarPanelManager.swift @@ -131,6 +131,8 @@ final class MenuBarPanelManager: NSObject { createPanel() } + companionManager.refreshAllPermissions() + positionPanelBelowStatusItem() panel?.makeKeyAndOrderFront(nil) diff --git a/leanring-buddy/WindowPositionManager.swift b/leanring-buddy/WindowPositionManager.swift index f5517f6d0..d283cb527 100644 --- a/leanring-buddy/WindowPositionManager.swift +++ b/leanring-buddy/WindowPositionManager.swift @@ -9,8 +9,6 @@ import AppKit import ApplicationServices import CoreGraphics -import ScreenCaptureKit - enum PermissionRequestPresentationDestination: Equatable { case alreadyGranted case systemPrompt @@ -73,6 +71,27 @@ class WindowPositionManager { static func revealAppInFinder() { guard let appURL = Bundle.main.bundleURL as URL? else { return } NSWorkspace.shared.activateFileViewerSelecting([appURL]) + print("🔑 Finder opened this build — drag it into Accessibility / Input Monitoring: \(appURL.path)") + } + + /// Opens Finder + Settings so the user can drag the current debug build in. + static func prepareAccessibilityReGrantFromFinder() { + hasAttemptedAccessibilitySystemPromptDuringCurrentLaunch = false + revealAppInFinder() + openAccessibilitySettings() + } + + /// Opens Finder + Settings so the user can drag the current debug build in. + static func prepareInputMonitoringReGrantFromFinder() { + hasAttemptedInputMonitoringSystemPromptDuringCurrentLaunch = false + revealAppInFinder() + openInputMonitoringSettings() + } + + /// Re-checks accessibility trust without showing the prompt. + static func refreshAccessibilityTrustCache() { + let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue(): false] as CFDictionary + _ = AXIsProcessTrustedWithOptions(options) } // MARK: - Input Monitoring Permission @@ -114,9 +133,14 @@ class WindowPositionManager { // MARK: - Screen Recording Permission + /// Live TCC check — does not use cached UserDefaults fallbacks. + static func isScreenCapturePreflightGranted() -> Bool { + CGPreflightScreenCaptureAccess() + } + /// Returns true if Screen Recording permission is granted. static func hasScreenRecordingPermission() -> Bool { - let hasScreenRecordingPermissionNow = CGPreflightScreenCaptureAccess() + let hasScreenRecordingPermissionNow = isScreenCapturePreflightGranted() if hasScreenRecordingPermissionNow { ClickyDefaults.shared.set(true, forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) } @@ -145,6 +169,18 @@ class WindowPositionManager { ClickyDefaults.shared.removeObject(forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) } + static func clearCachedScreenContentPermission() { + ClickyDefaults.shared.removeObject(forKey: "hasScreenContentPermission") + } + + /// One-time startup diagnostics when permissions are incomplete. + static func logPermissionDiagnosticsSnapshot() { + let bundlePath = Bundle.main.bundlePath + print("🔑 Permission diagnostics — enable THIS build in System Settings:") + print("🔑 \(bundlePath)") + print("🔑 TCC snapshot — accessibility: \(hasAccessibilityPermission()), inputMonitoring: \(hasInputMonitoringPermission()), screenRecordingPreflight: \(CGPreflightScreenCaptureAccess())") + } + /// Prompts the system dialog for Screen Recording permission. /// Uses the system prompt once, then opens System Settings on later attempts so /// the user never gets the prompt and the Settings pane at the same time. diff --git a/leanring-buddy/leanring_buddyApp.swift b/leanring-buddy/leanring_buddyApp.swift index dff03cc06..db01542f5 100644 --- a/leanring-buddy/leanring_buddyApp.swift +++ b/leanring-buddy/leanring_buddyApp.swift @@ -37,11 +37,12 @@ final class CompanionAppDelegate: NSObject, NSApplicationDelegate { print("🎯 Clicky: Starting...") print("🎯 Clicky: Version \(Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "unknown")") - UserDefaults.standard.register(defaults: ["NSInitialToolTipDelay": 0]) + ClickyDefaults.shared.register(defaults: ["NSInitialToolTipDelay": 0]) ClickyAnalytics.configure() ClickyAnalytics.trackAppOpened() ClickyE2EConfiguration.applyLaunchOverrides() + MacOSScreenshotFloatingThumbnailSuppression.enableForAppLifetime() menuBarPanelManager = MenuBarPanelManager(companionManager: companionManager) companionManager.start() @@ -61,8 +62,15 @@ final class CompanionAppDelegate: NSObject, NSApplicationDelegate { // startSparkleUpdater() } + func applicationDidBecomeActive(_ notification: Notification) { + // Re-check TCC after the user returns from System Settings. + companionManager.refreshAllPermissions() + companionManager.schedulePermissionRefreshBurstAfterReturningFromSettings() + } + func applicationWillTerminate(_ notification: Notification) { companionManager.stop() + MacOSScreenshotFloatingThumbnailSuppression.restoreAfterAppTermination() } /// Registers the app as a login item so it launches automatically on From 41a48b92e40a973e382eb32c3a88ed0f4fe89207 Mon Sep 17 00:00:00 2001 From: octavi42 Date: Mon, 8 Jun 2026 13:28:31 +0300 Subject: [PATCH 10/14] Add script to bootstrap isolated git worktrees for parallel development. new-worktree.sh creates a branch worktree with its own CLICKY_HOME, UserDefaults suite, and optional push-to-talk disable, and patches that worktree's Xcode scheme so multiple Clicky instances can run side by side without sharing ~/.clicky or fighting over the global hotkey. --- .gitignore | 1 + scripts/README.md | 23 +++++- scripts/new-worktree.sh | 169 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 192 insertions(+), 1 deletion(-) create mode 100755 scripts/new-worktree.sh diff --git a/.gitignore b/.gitignore index 832e80a1a..4029e55bb 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ build/ releases/ .claude/ coding-plans/ +.clicky-worktree.env diff --git a/scripts/README.md b/scripts/README.md index 9b2f0b348..9554e0bf0 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,4 +1,25 @@ -# Release Scripts +# Scripts + +## `new-worktree.sh` — Isolated parallel development + +Creates a git worktree with its own Clicky data home, UserDefaults suite, and +(usually) disabled global push-to-talk so multiple agents/branches can run side +by side without clobbering `~/.clicky` or fighting over Control+Option. + +```bash +# Secondary worktree (no global hotkey) +./scripts/new-worktree.sh feature/memory-routines + +# Primary dev worktree (keeps push-to-talk) +./scripts/new-worktree.sh feature/teaching-skills-upstream --primary +``` + +Each worktree patches **only its local** Xcode scheme with: +- `CLICKY_HOME=/tmp/clicky-` +- `-CLICKY_DEFAULTS_SUITE=clicky-` +- `-CLICKY_DISABLE_GLOBAL_PTT=1` (unless `--primary`) + +--- ## `release.sh` — Ship a new version of makesomething diff --git a/scripts/new-worktree.sh b/scripts/new-worktree.sh new file mode 100755 index 000000000..3f1e1cea4 --- /dev/null +++ b/scripts/new-worktree.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Create an isolated git worktree for parallel Clicky development. +# +# Usage: +# ./scripts/new-worktree.sh [--primary] [--path ] +# +# Examples: +# ./scripts/new-worktree.sh feature/memory-routines +# ./scripts/new-worktree.sh memory-routines --primary +# ./scripts/new-worktree.sh feature/skills-v2 --path ../clicky-skills-v2 +# +# Each worktree gets its own data home, UserDefaults suite, and (by default) +# disables the global push-to-talk hotkey so multiple instances do not conflict. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BASE_BRANCH="${CLICKY_WORKTREE_BASE:-origin/feature/teaching-skills-upstream}" + +usage() { + sed -n '3,12p' "$0" + exit 1 +} + +BRANCH_NAME="" +WORKTREE_PATH="" +IS_PRIMARY="0" + +while [[ $# -gt 0 ]]; do + case "$1" in + --primary) + IS_PRIMARY="1" + shift + ;; + --path) + WORKTREE_PATH="${2:-}" + [[ -n "$WORKTREE_PATH" ]] || usage + shift 2 + ;; + -h|--help) + usage + ;; + *) + if [[ -z "$BRANCH_NAME" ]]; then + BRANCH_NAME="$1" + shift + else + echo "Unknown argument: $1" >&2 + usage + fi + ;; + esac +done + +[[ -n "$BRANCH_NAME" ]] || usage + +if [[ "$BRANCH_NAME" != */* ]]; then + BRANCH_NAME="feature/$BRANCH_NAME" +fi + +slug="${BRANCH_NAME#feature/}" +slug="${slug//\//-}" +slug="$(echo "$slug" | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]-')" + +CLICKY_HOME="/tmp/clicky-${slug}" +CLICKY_DEFAULTS_SUITE="clicky-${slug}" + +if [[ -z "$WORKTREE_PATH" ]]; then + WORKTREE_PATH="$(dirname "$ROOT_DIR")/clicky-${slug}" +fi + +if [[ ! "$WORKTREE_PATH" = /* ]]; then + WORKTREE_PATH="$(cd "$ROOT_DIR" && cd "$(dirname "$WORKTREE_PATH")" && pwd)/$(basename "$WORKTREE_PATH")" +fi + +echo "Fetching latest $BASE_BRANCH..." +git -C "$ROOT_DIR" fetch origin --prune + +if git -C "$ROOT_DIR" show-ref --verify --quiet "refs/heads/$BRANCH_NAME"; then + echo "Branch already exists locally: $BRANCH_NAME" + git -C "$ROOT_DIR" worktree add "$WORKTREE_PATH" "$BRANCH_NAME" +else + git -C "$ROOT_DIR" worktree add -b "$BRANCH_NAME" "$WORKTREE_PATH" "$BASE_BRANCH" +fi + +mkdir -p "$CLICKY_HOME/skills" + +cat >"$WORKTREE_PATH/.clicky-worktree.env" </dev/null || true + +cat < Date: Mon, 8 Jun 2026 14:28:25 +0300 Subject: [PATCH 11/14] Persist voice sessions to disk on session boundary Save session JSON under ~/.clicky/sessions when a voice session ends via confirm phrase, 30s idle, or panel close. Includes 7-day retention cleanup, SessionStore unit tests, and cross-suite test isolation for ClickyPaths. --- AGENTS.md | 2 + SESSION_PERSISTENCE_PLAN.md | 204 ++++++++++++++++++ .../xcschemes/leanring-buddy.xcscheme | 17 +- leanring-buddy/ClickyPaths.swift | 23 +- leanring-buddy/CompanionManager.swift | 120 +++++++++++ leanring-buddy/MenuBarPanelManager.swift | 2 + leanring-buddy/PersistedSession.swift | 24 +++ leanring-buddy/SessionStore.swift | 107 +++++++++ leanring-buddy/TeachingSkill.swift | 2 +- leanring-buddyTests/ClickyPathsTests.swift | 127 ++++++----- .../ClickyTestHomeIsolation.swift | 45 ++++ leanring-buddyTests/SessionStoreTests.swift | 164 ++++++++++++++ 12 files changed, 775 insertions(+), 62 deletions(-) create mode 100644 SESSION_PERSISTENCE_PLAN.md create mode 100644 leanring-buddy/PersistedSession.swift create mode 100644 leanring-buddy/SessionStore.swift create mode 100644 leanring-buddyTests/ClickyTestHomeIsolation.swift create mode 100644 leanring-buddyTests/SessionStoreTests.swift diff --git a/AGENTS.md b/AGENTS.md index 5800ab181..071e82cd4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,6 +75,8 @@ Worker vars: `ELEVENLABS_VOICE_ID` | `WindowPositionManager.swift` | ~262 | Window placement logic, Screen Recording permission flow, and accessibility permission helpers. | | `AppBundleConfiguration.swift` | ~28 | Runtime configuration reader for keys stored in the app bundle Info.plist. | | `TeachingTopicHistoryStore.swift` | ~95 | Persists teaching topic tokens, bundle IDs, and timestamps to `~/.clicky/topic-history.json` for cross-session repeat detection. | +| `PersistedSession.swift` | ~25 | Codable on-disk session model (`sessionId`, `startedAt`, `endedAt`, `outcome`, `privacyOptOut`, `appsUsed`, `turns`) for the memory pipeline capture step. | +| `SessionStore.swift` | ~100 | Filesystem store for voice sessions at `~/.clicky/sessions//.json` with ISO8601 JSON encoding and 7-day retention cleanup. | | `TeachingSkillStore.swift` | ~100 | Create/read/update/delete teaching skills at `~/.clicky/skills//SKILL.md`. | | `TeachingSkill.swift` | ~280 | Model and YAML frontmatter parsing for teaching skills. | | `SkillMatcher.swift` | ~170 | App/topic matching and duplicate skill pair detection. | diff --git a/SESSION_PERSISTENCE_PLAN.md b/SESSION_PERSISTENCE_PLAN.md new file mode 100644 index 000000000..fd1ae72fd --- /dev/null +++ b/SESSION_PERSISTENCE_PLAN.md @@ -0,0 +1,204 @@ +# Session Persistence — Implementation Plan + +**Branch:** `feature/session-persistence` +**Worktree:** `/Users/cristeaoctavian/Projects/clicky-session-persistence` +**Data home (isolated):** `/tmp/clicky-session-persistence` +**Scope:** Step 1 of the memory pipeline only — **sessions get saved to disk correctly.** Do NOT build the gate, distill, or consent steps here. Keep this PR small and reviewable. + +Reference: [`docs/architecture/MEMORY_PIPELINE.md`](docs/architecture/MEMORY_PIPELINE.md) (esp. "Session boundary" and "Capture" sections) and [`docs/architecture/schemas/session.example.json`](docs/architecture/schemas/session.example.json). + +--- + +## Why this is first + +Today sessions live only in RAM as `CompanionManager.sessionTrace` (`[SessionTraceEntry]`, capped at 20 entries, and **wiped** by `maybeWriteTeachingSkill` on a successful skill write — see `CompanionManager.swift:377`). The downstream gate/distill/consent steps all need a saved session JSON to operate on. Nothing downstream can exist without this. + +--- + +## Context: existing code you will build on + +| Thing | Location | Notes | +|-------|----------|-------| +| `SessionTraceEntry` | `leanring-buddy/TeachingSkill.swift:326` | `timestamp: Date`, `userTranscript: String`, `assistantResponse: String`, `bundleId: String?`, `pointed: Bool`. Currently `Equatable` only — **add `Codable`**. | +| In-RAM trace | `CompanionManager.swift:85` | `private var sessionTrace: [SessionTraceEntry] = []` | +| Capture hook | `CompanionManager.swift:284` `recordSessionExchange(...)` | Appends a turn; caps at 20; records topic history. Called at `CompanionManager.swift:1100`. | +| Skill-write hook | `CompanionManager.swift:312` `maybeWriteTeachingSkill(after:)` | Called at `CompanionManager.swift:1105`, right after `recordSessionExchange`. On success it calls `sessionTrace.removeAll()` at line 377. **This is the wipe you must persist ahead of.** | +| Confirmation detection | `SkillTriggerEvaluator.isConfirmationTranscript(_:)` | Already exists. Reuse for the confirm-phrase session-end trigger. | +| Screen-teaching detection | `SkillTriggerEvaluator.isScreenTeachingSession(_:)` | Reuse if needed for outcome heuristics. | +| Isolated paths | `leanring-buddy/ClickyPaths.swift` | `ClickyPaths.home` (already worktree-isolated via `CLICKY_HOME`). Add a `sessions` URL here. | +| Path tests pattern | `leanring-buddyTests/ClickyPathsTests.swift` | Uses `ClickyPaths.overrideHomeForTesting`. Mirror this for `SessionStore` round-trip tests. | +| Panel close hook | `MenuBarPanelManager.swift:143` `hidePanel()` + `.clickyDismissPanel` notification (`MenuBarPanelManager.swift:17`) | Use to detect "panel closed" session boundary. | +| Launch lifecycle | `CompanionManager.swift:454` `start()` → `bootstrapTeachingSkills()` (`:257`) | Add the 7-day cleanup call here. | +| Learning toggle | `CompanionManager.isLearningFromSessionsEnabled` (`:93`) | Respect it. When disabled, treat the session as `privacyOptOut = true` (still persist, but flagged) — see Open Questions. | + +**Xcode project note:** the target uses `PBXFileSystemSynchronizedRootGroup`, so any new `.swift` file dropped into `leanring-buddy/` (or `leanring-buddyTests/`) is auto-included in the build. **No `project.pbxproj` editing required.** + +**Persistence target path:** `ClickyPaths.home/sessions//.json` (per MEMORY_PIPELINE.md "Capture" → Persistence). The date folder is the session's `startedAt` in local time. + +--- + +## What to build (4 small steps) + +### Step 1 — Model + store + +**1a. Make `SessionTraceEntry` `Codable`** (`TeachingSkill.swift:326`). +Change `struct SessionTraceEntry: Equatable` → `struct SessionTraceEntry: Equatable, Codable`. All stored properties are already `Codable`-compatible. This is the only edit to that file. + +**1b. New file `leanring-buddy/PersistedSession.swift`** — matches `schemas/session.example.json`: + +```swift +struct PersistedSession: Codable, Equatable { + let sessionId: UUID + let startedAt: Date + let endedAt: Date + let outcome: SessionOutcome + let privacyOptOut: Bool + let appsUsed: [String] // unique bundle IDs, in first-seen order + let turns: [SessionTraceEntry] +} + +enum SessionOutcome: String, Codable { + case success + case abandoned + case unknown +} +``` + +Notes: +- The schema uses string values `"success" | "abandoned" | "unknown"`; `SessionOutcome: String` serializes to exactly those. Verify the encoder produces the bare string (it does for `RawRepresentable String` enums). +- Dates in the schema are ISO8601 (`2026-06-05T14:22:10Z`). Configure the `JSONEncoder`/`JSONDecoder` in `SessionStore` with `.iso8601` date strategy so output matches the schema (the default `.deferredToDate` emits numeric timestamps — **do not** use the default). + +**1c. Add `sessions` path to `ClickyPaths`** (`ClickyPaths.swift`, alongside `skills` / `topicHistory`): + +```swift +static var sessions: URL { + home.appendingPathComponent("sessions", isDirectory: true) +} +``` + +**1d. New file `leanring-buddy/SessionStore.swift`** — filesystem store, mirror `TeachingSkillStore` style: + +```swift +final class SessionStore { + static var sessionsRootURL: URL { ClickyPaths.sessions } + + @discardableResult + func save(_ session: PersistedSession) throws -> URL // writes //.json + func deleteSessionsOlderThan(days: Int, now: Date = Date()) // 7-day cleanup + // (optional helpers) func loadAllSessions() -> [PersistedSession] +} +``` + +Implementation details: +- Use a dedicated `JSONEncoder` with `dateEncodingStrategy = .iso8601` and `outputFormatting = [.prettyPrinted, .sortedKeys]`. +- Date-folder name: format `startedAt` with a fixed `DateFormatter` (`yyyy-MM-dd`, `Locale(identifier: "en_US_POSIX")`, current time zone). Create intermediate directories. +- Write atomically (`.atomic`). +- `deleteSessionsOlderThan`: walk `sessionsRootURL`, parse each `` folder (or use file modification date as a fallback) and remove folders/files older than the cutoff. Be defensive — never throw out of cleanup; log and continue. + +### Step 2 — Session-end detection in `CompanionManager` + +The boundary (MEMORY_PIPELINE.md "Session boundary") fires on **any** of: +1. Confirmation phrase on the latest turn — `SkillTriggerEvaluator.isConfirmationTranscript(transcript)`. +2. **30 seconds** idle after the last exchange. +3. Companion panel closed. + +Add: +- `private let sessionStore = SessionStore()` +- `private var sessionStartedAt: Date?` — set on the first `recordSessionExchange` of a new session (i.e. when `sessionTrace` was empty before the append). +- `private var sessionIdleTimer: Timer?` — (re)scheduled at 30s on every `recordSessionExchange`; on fire, call `finalizeAndPersistSession(outcome:)`. Must be created/invalidated on the main actor. +- A guard so a session is only persisted once (e.g. set `sessionStartedAt = nil` after finalize, and bail early if there are no turns). + +Wire the three triggers: +- **Confirm phrase:** in `recordSessionExchange` (or right after it at `:1100`), if `isConfirmationTranscript(transcript)` is true, finalize with `outcome: .success`. +- **Idle:** the `sessionIdleTimer` callback finalizes with `outcome` derived (see Step 3). +- **Panel close:** observe `.clickyDismissPanel` (or add a hook in `MenuBarPanelManager.hidePanel()`), finalize the in-flight session. Prefer subscribing to the existing `.clickyDismissPanel` notification in `CompanionManager` so `MenuBarPanelManager` stays untouched — but note that notification is also posted during onboarding restart (`triggerOnboarding`), so only finalize when `sessionTrace` is non-empty. + +### Step 3 — Finalize + persist + +New `private func finalizeAndPersistSession(outcome: SessionOutcome? = nil)`: +1. Guard: `!sessionTrace.isEmpty`, else return (and clear timer/state). +2. Compute `outcome` if not explicitly passed: + - confirmation phrase seen on any/last turn → `.success` + - idle/panel-close with at least one `pointed` turn or screen-teaching → `.unknown` + - otherwise → `.abandoned` + (Keep this heuristic simple and documented in code comments; it can be refined by the later gate PR.) +3. `appsUsed`: unique `bundleId`s from `sessionTrace.compactMap(\.bundleId)`, preserving first-seen order. +4. `privacyOptOut`: `!isLearningFromSessionsEnabled` (see Open Questions). +5. Build `PersistedSession(sessionId: UUID(), startedAt: sessionStartedAt ?? firstTurnTimestamp, endedAt: Date(), …, turns: sessionTrace)`. +6. `try? sessionStore.save(session)` (log on failure; never crash the voice pipeline). +7. Reset: invalidate `sessionIdleTimer`, `sessionStartedAt = nil`, `sessionTrace.removeAll()`. + +**Critical ordering vs. skill write (the line 377 wipe):** +`maybeWriteTeachingSkill` is `async` (spawns `skillWriteTask`) and wipes `sessionTrace` on success. To avoid losing turns: +- **Preferred:** persist the session *before* `maybeWriteTeachingSkill` can wipe it. The simplest correct approach: have the confirm-phrase finalize run at the call site (`:1100`–`:1105`) **before** `maybeWriteTeachingSkill(after:)`, OR snapshot the trace into the `PersistedSession` synchronously at finalize time (finalize takes a value copy of `sessionTrace`, so even if the async task later wipes the live array, the persisted copy is intact). +- Because `finalizeAndPersistSession` copies `sessionTrace` by value into `PersistedSession.turns` synchronously, persistence is safe regardless of the async wipe **as long as finalize runs**. The risk is double-wipe / lost session if only the skill path runs and finalize never fires. Ensure the confirm-phrase path always calls finalize. +- Do not remove or change `maybeWriteTeachingSkill`'s existing behavior. This PR only *adds* persistence. + +### Step 4 — 7-day cleanup on launch + +In `bootstrapTeachingSkills()` (`CompanionManager.swift:257`) or directly in `start()` (`:454`), add: +```swift +sessionStore.deleteSessionsOlderThan(days: 7) +``` +Spec: "Delete after successful distillation or after 7 days." Distill-deletion is out of scope for this PR; only the 7-day sweep is implemented here. + +--- + +## Tests + +Add `leanring-buddyTests/SessionStoreTests.swift` (Swift Testing, mirror `ClickyPathsTests.swift`): +- Use `ClickyPaths.overrideHomeForTesting = ` with `@Suite(.serialized)` (the override is a shared global — serialize like `ClickyPathsTests`). +- **Round-trip:** save a `PersistedSession` with 2 turns, read the JSON back off disk, decode, `#expect` equality of all fields (including ISO8601 dates and outcome string). +- **Path shape:** assert the file lands at `/sessions//.json`. +- **JSON shape:** decode the raw file and confirm `outcome` serializes as a bare string and dates are ISO8601 strings (guards against the default numeric date strategy regressing). +- **Cleanup:** create a session folder dated 8 days ago and one dated today; call `deleteSessionsOlderThan(days: 7)`; assert old removed, recent kept. +- Optionally add a `SessionTraceEntry` Codable round-trip test. + +Run tests from **Xcode** (Cmd+U) — **do NOT run `xcodebuild` from the terminal** (invalidates TCC permissions per AGENTS.md). + +--- + +## Manual verification + +1. Open `/Users/cristeaoctavian/Projects/clicky-session-persistence/leanring-buddy.xcodeproj` in Xcode (this worktree is `--primary`, so push-to-talk is enabled and scheme args set `CLICKY_HOME=/tmp/clicky-session-persistence`). +2. Cmd+R, grant TCC permissions to this build if asked. +3. Do a voice session (push-to-talk), ask something, then say "thanks" / "got it". +4. Confirm a JSON file appears under `/tmp/clicky-session-persistence/sessions//`, and its contents match the `session.example.json` shape. +5. Test idle path: do an exchange, wait 30s without speaking, confirm a session JSON is written with a non-success outcome. +6. Test panel-close path: do an exchange, click the menu bar icon to close the panel, confirm persistence. + +--- + +## Files touched / added + +**Add:** +- `leanring-buddy/PersistedSession.swift` +- `leanring-buddy/SessionStore.swift` +- `leanring-buddyTests/SessionStoreTests.swift` + +**Edit:** +- `leanring-buddy/TeachingSkill.swift` — add `Codable` to `SessionTraceEntry` (one line). +- `leanring-buddy/ClickyPaths.swift` — add `sessions` URL. +- `leanring-buddy/CompanionManager.swift` — `sessionStore`, `sessionStartedAt`, `sessionIdleTimer`, session-end triggers, `finalizeAndPersistSession`, 7-day cleanup on launch. + +**Docs (per AGENTS.md self-update):** +- Update the "Key Files" table in `AGENTS.md` with `PersistedSession.swift` and `SessionStore.swift`. +- Update `docs/architecture/MEMORY_PIPELINE.md` "Existing code map" — flip "Session persist" from ❌ to ✅. + +--- + +## Scope guardrails (do NOT do here) + +- No `MemoryGate`, no distill, no consent UI, no preferences/routines/activity log. +- Do not refactor `maybeWriteTeachingSkill` or `recordSessionExchange` beyond what is needed to hook finalize. +- Do not fix the known non-blocking warnings (Swift 6 concurrency, deprecated `onChange`). +- Do not rename the project/scheme ("leanring" typo is intentional). +- Do not run `xcodebuild` from the terminal. + +--- + +## Open questions (decide, note your choice in the PR) + +1. **`privacyOptOut` semantics.** When `isLearningFromSessionsEnabled == false`: (a) persist with `privacyOptOut = true` (recommended — keeps the boundary logic uniform, lets the future gate skip it), or (b) skip persistence entirely. Recommend (a). The MEMORY_PIPELINE "do NOT save when `privacyOptOut`" rule applies at the **gate** stage, not capture. +2. **Multiple end-triggers firing close together** (e.g. confirm phrase then panel close). Guard with `sessionStartedAt == nil` after finalize so only one JSON is written per session. +3. **Outcome heuristic granularity.** Keep it coarse here; the gate PR will refine. Document whatever you pick inline. diff --git a/leanring-buddy.xcodeproj/xcshareddata/xcschemes/leanring-buddy.xcscheme b/leanring-buddy.xcodeproj/xcshareddata/xcschemes/leanring-buddy.xcscheme index 1b6cd72c9..e8f7fb13b 100644 --- a/leanring-buddy.xcodeproj/xcshareddata/xcschemes/leanring-buddy.xcscheme +++ b/leanring-buddy.xcodeproj/xcshareddata/xcschemes/leanring-buddy.xcscheme @@ -27,12 +27,12 @@ buildConfiguration = "Debug" selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB" selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB" - shouldUseLaunchSchemeArgsEnv = "YES" + shouldUseLaunchSchemeArgsEnv = "NO" shouldAutocreateTestPlan = "YES"> + parallelizable = "NO"> + + + + + + + + ? private var curatorLLMTask: Task? @@ -257,6 +264,7 @@ final class CompanionManager: ObservableObject { private func bootstrapTeachingSkills() { teachingSkillStore.loadSkills() topicHistoryStore.load() + sessionStore.deleteSessionsOlderThan(days: 7) SkillCurator.curate(store: teachingSkillStore) teachingSkills = teachingSkillStore.skills runCuratorLLMPassesIfNeeded() @@ -286,6 +294,8 @@ final class CompanionManager: ObservableObject { spokenResponse: String, pointed: Bool ) { + let wasEmptyBeforeAppend = sessionTrace.isEmpty + sessionTrace.append( SessionTraceEntry( timestamp: Date(), @@ -296,10 +306,16 @@ final class CompanionManager: ObservableObject { ) ) + if wasEmptyBeforeAppend { + sessionStartedAt = Date() + } + if sessionTrace.count > 20 { sessionTrace.removeFirst(sessionTrace.count - 20) } + restartSessionIdleTimer() + if !SkillTriggerEvaluator.isConfirmationTranscript(transcript) { let topic = SkillTriggerEvaluator.deriveTopic(fromQuestion: transcript) topicHistoryStore.recordTopic( @@ -309,6 +325,81 @@ final class CompanionManager: ObservableObject { } } + private func restartSessionIdleTimer() { + sessionIdleTimer?.invalidate() + sessionIdleTimer = Timer.scheduledTimer(withTimeInterval: 30, repeats: false) { [weak self] _ in + Task { @MainActor in + self?.finalizeAndPersistSession() + } + } + } + + private func finalizeAndPersistSession(outcome explicitOutcome: SessionOutcome? = nil) { + sessionIdleTimer?.invalidate() + sessionIdleTimer = nil + + guard sessionStartedAt != nil else { return } + + let turnsSnapshot = sessionTrace + guard !turnsSnapshot.isEmpty else { + sessionStartedAt = nil + return + } + + let resolvedOutcome = explicitOutcome ?? deriveSessionOutcome(from: turnsSnapshot) + let appsUsed = orderedUniqueBundleIds(from: turnsSnapshot) + let privacyOptOut = !isLearningFromSessionsEnabled + + let session = PersistedSession( + sessionId: UUID(), + startedAt: sessionStartedAt ?? turnsSnapshot.first!.timestamp, + endedAt: Date(), + outcome: resolvedOutcome, + privacyOptOut: privacyOptOut, + appsUsed: appsUsed, + turns: turnsSnapshot + ) + + do { + let savedURL = try sessionStore.save(session) + print("💾 Persisted session to \(savedURL.path)") + } catch { + print("⚠️ Failed to persist session: \(error)") + } + + sessionStartedAt = nil + sessionTrace.removeAll() + } + + /// Coarse outcome heuristic for capture-time persistence. The future memory gate + /// can refine this; keep the rules simple and predictable here. + private func deriveSessionOutcome(from turns: [SessionTraceEntry]) -> SessionOutcome { + if let lastTurn = turns.last, + SkillTriggerEvaluator.isConfirmationTranscript(lastTurn.userTranscript) { + return .success + } + + if turns.contains(where: \.pointed) || + SkillTriggerEvaluator.isScreenTeachingSession(turns) { + return .unknown + } + + return .abandoned + } + + private func orderedUniqueBundleIds(from turns: [SessionTraceEntry]) -> [String] { + var seenBundleIds = Set() + var orderedBundleIds: [String] = [] + + for bundleId in turns.compactMap(\.bundleId) { + if seenBundleIds.insert(bundleId).inserted { + orderedBundleIds.append(bundleId) + } + } + + return orderedBundleIds + } + private func maybeWriteTeachingSkill(after transcript: String) { guard isLearningFromSessionsEnabled else { return } guard SkillTriggerEvaluator.isScreenTeachingSession(sessionTrace) else { return } @@ -453,6 +544,7 @@ final class CompanionManager: ObservableObject { func start() { bootstrapTeachingSkills() + bindPanelClosedObservation() refreshAllPermissions() print("🔑 Clicky start — accessibility: \(hasAccessibilityPermission), inputMonitoring: \(hasInputMonitoringPermission), pushToTalkActive: \(isPushToTalkHotkeyActive), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission), onboarded: \(hasCompletedOnboarding)") startPermissionPolling() @@ -578,6 +670,12 @@ final class CompanionManager: ObservableObject { currentResponseTask = nil skillWriteTask?.cancel() skillWriteTask = nil + sessionIdleTimer?.invalidate() + sessionIdleTimer = nil + if let panelClosedObserver { + NotificationCenter.default.removeObserver(panelClosedObserver) + self.panelClosedObserver = nil + } shortcutTransitionCancellable?.cancel() voiceStateCancellable?.cancel() audioPowerCancellable?.cancel() @@ -829,12 +927,28 @@ final class CompanionManager: ObservableObject { // the brief idle gap between recording and processing // would prematurely hide the overlay. if self.currentResponseTask == nil { + self.isPushToTalkInteractionActive = false self.scheduleTransientHideIfNeeded() } } } } + private func bindPanelClosedObservation() { + panelClosedObserver = NotificationCenter.default.addObserver( + forName: .clickyCompanionPanelDidClose, + object: nil, + queue: .main + ) { [weak self] _ in + Task { @MainActor in + guard let self else { return } + guard !self.isPushToTalkInteractionActive else { return } + guard !self.sessionTrace.isEmpty else { return } + self.finalizeAndPersistSession() + } + } + } + private func bindShortcutTransitions() { shortcutTransitionCancellable = globalPushToTalkShortcutMonitor .shortcutTransitionPublisher @@ -851,6 +965,8 @@ final class CompanionManager: ObservableObject { // Don't register push-to-talk while the onboarding video is playing guard !showOnboardingVideo else { return } + isPushToTalkInteractionActive = true + // Cancel any pending transient hide so the overlay stays visible transientHideTask?.cancel() transientHideTask = nil @@ -1103,6 +1219,9 @@ final class CompanionManager: ObservableObject { pointed: hasPointCoordinate ) maybeWriteTeachingSkill(after: transcript) + if SkillTriggerEvaluator.isConfirmationTranscript(transcript) { + finalizeAndPersistSession(outcome: .success) + } ClickyAnalytics.trackAIResponseReceived(response: spokenText) @@ -1132,6 +1251,7 @@ final class CompanionManager: ObservableObject { if !Task.isCancelled { voiceState = .idle + isPushToTalkInteractionActive = false scheduleTransientHideIfNeeded() } } diff --git a/leanring-buddy/MenuBarPanelManager.swift b/leanring-buddy/MenuBarPanelManager.swift index 3a56d5fed..67616c551 100644 --- a/leanring-buddy/MenuBarPanelManager.swift +++ b/leanring-buddy/MenuBarPanelManager.swift @@ -16,6 +16,7 @@ import SwiftUI extension Notification.Name { static let clickyDismissPanel = Notification.Name("clickyDismissPanel") + static let clickyCompanionPanelDidClose = Notification.Name("clickyCompanionPanelDidClose") } /// Custom NSPanel subclass that can become the key window even with @@ -143,6 +144,7 @@ final class MenuBarPanelManager: NSObject { private func hidePanel() { panel?.orderOut(nil) removeClickOutsideMonitor() + NotificationCenter.default.post(name: .clickyCompanionPanelDidClose, object: nil) } private func createPanel() { diff --git a/leanring-buddy/PersistedSession.swift b/leanring-buddy/PersistedSession.swift new file mode 100644 index 000000000..7b56adfb0 --- /dev/null +++ b/leanring-buddy/PersistedSession.swift @@ -0,0 +1,24 @@ +// +// PersistedSession.swift +// leanring-buddy +// +// On-disk session JSON written by SessionStore under ~/.clicky/sessions/. +// + +import Foundation + +struct PersistedSession: Codable, Equatable { + let sessionId: UUID + let startedAt: Date + let endedAt: Date + let outcome: SessionOutcome + let privacyOptOut: Bool + let appsUsed: [String] + let turns: [SessionTraceEntry] +} + +enum SessionOutcome: String, Codable { + case success + case abandoned + case unknown +} diff --git a/leanring-buddy/SessionStore.swift b/leanring-buddy/SessionStore.swift new file mode 100644 index 000000000..84f65fe8d --- /dev/null +++ b/leanring-buddy/SessionStore.swift @@ -0,0 +1,107 @@ +// +// SessionStore.swift +// leanring-buddy +// +// Filesystem store for persisted voice sessions under ClickyPaths.sessions. +// + +import Foundation + +final class SessionStore { + static var sessionsRootURL: URL { + ClickyPaths.sessions + } + + private let encoder: JSONEncoder = { + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + return encoder + }() + + private let decoder: JSONDecoder = { + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + return decoder + }() + + private let dayFolderFormatter: DateFormatter = { + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.dateFormat = "yyyy-MM-dd" + return formatter + }() + + @discardableResult + func save(_ session: PersistedSession) throws -> URL { + let fileManager = FileManager.default + let dayFolderName = dayFolderFormatter.string(from: session.startedAt) + let dayFolderURL = Self.sessionsRootURL + .appendingPathComponent(dayFolderName, isDirectory: true) + try fileManager.createDirectory(at: dayFolderURL, withIntermediateDirectories: true) + + let sessionFileURL = dayFolderURL + .appendingPathComponent("\(session.sessionId.uuidString).json") + let encodedData = try encoder.encode(session) + try encodedData.write(to: sessionFileURL, options: .atomic) + return sessionFileURL + } + + func loadAllSessions() -> [PersistedSession] { + let fileManager = FileManager.default + guard let dayFolders = try? fileManager.contentsOfDirectory( + at: Self.sessionsRootURL, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + ) else { + return [] + } + + var loadedSessions: [PersistedSession] = [] + for dayFolder in dayFolders where (try? dayFolder.resourceValues(forKeys: [.isDirectoryKey]).isDirectory) == true { + guard let sessionFiles = try? fileManager.contentsOfDirectory( + at: dayFolder, + includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles] + ) else { + continue + } + + for sessionFile in sessionFiles where sessionFile.pathExtension == "json" { + guard let fileData = try? Data(contentsOf: sessionFile), + let session = try? decoder.decode(PersistedSession.self, from: fileData) else { + continue + } + loadedSessions.append(session) + } + } + + return loadedSessions.sorted { $0.startedAt < $1.startedAt } + } + + func deleteSessionsOlderThan(days: Int, now: Date = Date()) { + let fileManager = FileManager.default + let cutoffDate = Calendar.current.date(byAdding: .day, value: -days, to: now) ?? now + + guard let dayFolders = try? fileManager.contentsOfDirectory( + at: Self.sessionsRootURL, + includingPropertiesForKeys: [.contentModificationDateKey, .isDirectoryKey], + options: [.skipsHiddenFiles] + ) else { + return + } + + for dayFolder in dayFolders { + let resourceValues = try? dayFolder.resourceValues(forKeys: [.isDirectoryKey, .contentModificationDateKey]) + guard resourceValues?.isDirectory == true else { continue } + + let folderDate = dayFolderFormatter.date(from: dayFolder.lastPathComponent) + ?? resourceValues?.contentModificationDate + ?? .distantPast + + guard folderDate < cutoffDate else { continue } + + try? fileManager.removeItem(at: dayFolder) + } + } +} diff --git a/leanring-buddy/TeachingSkill.swift b/leanring-buddy/TeachingSkill.swift index 9a08a3372..73cf4cabe 100644 --- a/leanring-buddy/TeachingSkill.swift +++ b/leanring-buddy/TeachingSkill.swift @@ -323,7 +323,7 @@ struct TeachingSkill: Identifiable, Equatable { }() } -struct SessionTraceEntry: Equatable { +struct SessionTraceEntry: Equatable, Codable { let timestamp: Date let userTranscript: String let assistantResponse: String diff --git a/leanring-buddyTests/ClickyPathsTests.swift b/leanring-buddyTests/ClickyPathsTests.swift index d9fb5196d..2bb63ea17 100644 --- a/leanring-buddyTests/ClickyPathsTests.swift +++ b/leanring-buddyTests/ClickyPathsTests.swift @@ -13,87 +13,108 @@ import Testing @Suite(.serialized) struct ClickyPathsTests { @Test func defaultHomeUsesClickyDirectoryInUserHome() { - ClickyPaths.overrideHomeForTesting = nil + ClickyTestHomeIsolation.withSerializedHomeAccess { + ClickyPaths.overrideHomeForTesting = nil + ClickyPaths.ignoreConfiguredHomeForTesting = true + defer { ClickyPaths.ignoreConfiguredHomeForTesting = false } - let expectedHome = FileManager.default.homeDirectoryForCurrentUser - .appendingPathComponent(".clicky", isDirectory: true) + let expectedHome = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent(".clicky", isDirectory: true) - #expect(ClickyPaths.home == expectedHome) + #expect(ClickyPaths.home == expectedHome) + } + } + + @Test func clickyHomeEnvironmentVariableRedirectsHome() { + ClickyTestHomeIsolation.withSerializedHomeAccess { + ClickyPaths.overrideHomeForTesting = nil + ClickyPaths.ignoreConfiguredHomeForTesting = false + + let configuredHomePath = ProcessInfo.processInfo.environment["CLICKY_HOME"] + let clickyHomeLaunchArgument = ClickyLaunchArguments.value(forPrefix: "-CLICKY_HOME=") + + if let clickyHomeLaunchArgument, !clickyHomeLaunchArgument.isEmpty { + let expectedHome = URL(fileURLWithPath: clickyHomeLaunchArgument, isDirectory: true) + #expect(ClickyPaths.home == expectedHome) + return + } + + if let configuredHomePath, !configuredHomePath.isEmpty { + let expectedHome = URL(fileURLWithPath: configuredHomePath, isDirectory: true) + #expect(ClickyPaths.home == expectedHome) + return + } + + // No worktree isolation configured in this test run. + } } @Test func overrideHomeRedirectsSkillsAndTopicHistory() throws { let temporaryHome = FileManager.default.temporaryDirectory .appendingPathComponent("clicky-test-\(UUID().uuidString)", isDirectory: true) - defer { - ClickyPaths.overrideHomeForTesting = nil - try? FileManager.default.removeItem(at: temporaryHome) - } + defer { try? FileManager.default.removeItem(at: temporaryHome) } - try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) - ClickyPaths.overrideHomeForTesting = temporaryHome + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) - #expect(ClickyPaths.home == temporaryHome) - #expect(ClickyPaths.skills == temporaryHome.appendingPathComponent("skills", isDirectory: true)) - #expect(ClickyPaths.topicHistory == temporaryHome.appendingPathComponent("topic-history.json")) + #expect(ClickyPaths.home == temporaryHome) + #expect(ClickyPaths.skills == temporaryHome.appendingPathComponent("skills", isDirectory: true)) + #expect(ClickyPaths.sessions == temporaryHome.appendingPathComponent("sessions", isDirectory: true)) + #expect(ClickyPaths.topicHistory == temporaryHome.appendingPathComponent("topic-history.json")) + } } @Test func teachingSkillStoreUsesClickyPathsSkillsDirectory() throws { let temporaryHome = FileManager.default.temporaryDirectory .appendingPathComponent("clicky-test-\(UUID().uuidString)", isDirectory: true) - defer { - ClickyPaths.overrideHomeForTesting = nil - try? FileManager.default.removeItem(at: temporaryHome) - } + defer { try? FileManager.default.removeItem(at: temporaryHome) } - try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) - ClickyPaths.overrideHomeForTesting = temporaryHome + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) - #expect(TeachingSkillStore.skillsRootURL == ClickyPaths.skills) + #expect(TeachingSkillStore.skillsRootURL == ClickyPaths.skills) + } } @Test func teachingTopicHistoryStoreUsesClickyPathsTopicHistoryFile() throws { let temporaryHome = FileManager.default.temporaryDirectory .appendingPathComponent("clicky-test-\(UUID().uuidString)", isDirectory: true) - defer { - ClickyPaths.overrideHomeForTesting = nil - try? FileManager.default.removeItem(at: temporaryHome) - } + defer { try? FileManager.default.removeItem(at: temporaryHome) } - try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) - ClickyPaths.overrideHomeForTesting = temporaryHome + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) - #expect(TeachingTopicHistoryStore.historyFileURL == ClickyPaths.topicHistory) + #expect(TeachingTopicHistoryStore.historyFileURL == ClickyPaths.topicHistory) + } } @Test func teachingSkillStoreWritesToIsolatedHome() throws { let temporaryHome = FileManager.default.temporaryDirectory .appendingPathComponent("clicky-test-\(UUID().uuidString)", isDirectory: true) - defer { - ClickyPaths.overrideHomeForTesting = nil - try? FileManager.default.removeItem(at: temporaryHome) + defer { try? FileManager.default.removeItem(at: temporaryHome) } + + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + + let skill = TeachingSkill( + id: "teach-textedit-save", + name: "teach-textedit-save", + description: "Save in TextEdit", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: nil, + usageCount: 0, + isPinned: false, + taskSlug: nil, + body: "step one: press command-s." + ) + + let store = TeachingSkillStore() + _ = try store.saveSkill(skill) + + let skillFile = temporaryHome + .appendingPathComponent("skills/teach-textedit-save/SKILL.md") + #expect(FileManager.default.fileExists(atPath: skillFile.path)) } - - try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) - ClickyPaths.overrideHomeForTesting = temporaryHome - - let skill = TeachingSkill( - id: "teach-textedit-save", - name: "teach-textedit-save", - description: "Save in TextEdit", - bundleIds: ["com.apple.TextEdit"], - status: .active, - lastUsed: nil, - usageCount: 0, - isPinned: false, - taskSlug: nil, - body: "step one: press command-s." - ) - - let store = TeachingSkillStore() - _ = try store.saveSkill(skill) - - let skillFile = temporaryHome - .appendingPathComponent("skills/teach-textedit-save/SKILL.md") - #expect(FileManager.default.fileExists(atPath: skillFile.path)) } } diff --git a/leanring-buddyTests/ClickyTestHomeIsolation.swift b/leanring-buddyTests/ClickyTestHomeIsolation.swift new file mode 100644 index 000000000..8ff5a3c1d --- /dev/null +++ b/leanring-buddyTests/ClickyTestHomeIsolation.swift @@ -0,0 +1,45 @@ +// +// ClickyTestHomeIsolation.swift +// leanring-buddyTests +// +// Serializes access to the process-wide `ClickyPaths.overrideHomeForTesting` +// global across ALL test suites. `@Suite(.serialized)` only serializes tests +// within a single suite, but Swift Testing still runs separate suites in +// parallel in-process. Without a shared lock, one suite can reset the override +// while another suite is mid-test (e.g. between `save` and `loadAllSessions`), +// causing the store to resolve a different home directory than it wrote to. +// + +import Foundation +@testable import leanring_buddy + +enum ClickyTestHomeIsolation { + private static let lock = NSLock() + + /// Runs `body` with `ClickyPaths.overrideHomeForTesting` set to `home`, + /// holding a process-wide lock so no other suite can mutate the global + /// for the duration. The override is always cleared afterwards. + static func withIsolatedHome( + _ home: URL, + _ body: () throws -> ResultType + ) rethrows -> ResultType { + lock.lock() + defer { lock.unlock() } + + let previousOverride = ClickyPaths.overrideHomeForTesting + ClickyPaths.overrideHomeForTesting = home + defer { ClickyPaths.overrideHomeForTesting = previousOverride } + + return try body() + } + + /// Runs `body` while holding the same lock without setting an override, + /// for tests that assert the production default home resolution. + static func withSerializedHomeAccess( + _ body: () throws -> ResultType + ) rethrows -> ResultType { + lock.lock() + defer { lock.unlock() } + return try body() + } +} diff --git a/leanring-buddyTests/SessionStoreTests.swift b/leanring-buddyTests/SessionStoreTests.swift new file mode 100644 index 000000000..8a30b9ada --- /dev/null +++ b/leanring-buddyTests/SessionStoreTests.swift @@ -0,0 +1,164 @@ +// +// SessionStoreTests.swift +// leanring-buddyTests +// + +import Foundation +import Testing +@testable import leanring_buddy + +@Suite(.serialized) +struct SessionStoreTests { + private func makeTemporaryHome() -> URL { + FileManager.default.temporaryDirectory + .appendingPathComponent("clicky-session-test-\(UUID().uuidString)", isDirectory: true) + } + + private func dayFolderFormatter() -> DateFormatter { + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.dateFormat = "yyyy-MM-dd" + return formatter + } + + private func sampleSession( + sessionId: UUID = UUID(), + startedAt: Date = Date(timeIntervalSince1970: 1_747_000_000), + endedAt: Date = Date(timeIntervalSince1970: 1_747_000_120) + ) -> PersistedSession { + PersistedSession( + sessionId: sessionId, + startedAt: startedAt, + endedAt: endedAt, + outcome: .success, + privacyOptOut: false, + appsUsed: ["com.apple.TextEdit", "com.apple.Safari"], + turns: [ + SessionTraceEntry( + timestamp: startedAt, + userTranscript: "how do i save in textedit?", + assistantResponse: "press command-s to save.", + bundleId: "com.apple.TextEdit", + pointed: true + ), + SessionTraceEntry( + timestamp: endedAt, + userTranscript: "got it", + assistantResponse: "nice.", + bundleId: "com.apple.TextEdit", + pointed: false + ) + ] + ) + } + + @Test func sessionStoreRoundTripPreservesAllFields() throws { + let temporaryHome = makeTemporaryHome() + defer { try? FileManager.default.removeItem(at: temporaryHome) } + + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + + let session = sampleSession() + let store = SessionStore() + let savedURL = try store.save(session) + + let fileData = try Data(contentsOf: savedURL) + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + let decodedSession = try decoder.decode(PersistedSession.self, from: fileData) + + #expect(decodedSession == session) + #expect(store.loadAllSessions() == [session]) + } + } + + @Test func sessionStoreWritesExpectedPathShape() throws { + let temporaryHome = makeTemporaryHome() + defer { try? FileManager.default.removeItem(at: temporaryHome) } + + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + + let sessionId = UUID() + let startedAt = Date(timeIntervalSince1970: 1_747_000_000) + let session = sampleSession(sessionId: sessionId, startedAt: startedAt) + let store = SessionStore() + let savedURL = try store.save(session) + + let expectedDayFolder = dayFolderFormatter().string(from: startedAt) + let expectedURL = temporaryHome + .appendingPathComponent("sessions", isDirectory: true) + .appendingPathComponent(expectedDayFolder, isDirectory: true) + .appendingPathComponent("\(sessionId.uuidString).json") + + #expect(savedURL == expectedURL) + #expect(FileManager.default.fileExists(atPath: expectedURL.path)) + } + } + + @Test func sessionStoreWritesISO8601DatesAndStringOutcome() throws { + let temporaryHome = makeTemporaryHome() + defer { try? FileManager.default.removeItem(at: temporaryHome) } + + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + + let session = sampleSession() + let store = SessionStore() + let savedURL = try store.save(session) + + let jsonObject = try JSONSerialization.jsonObject(with: Data(contentsOf: savedURL)) as? [String: Any] + #expect(jsonObject != nil) + #expect(jsonObject?["outcome"] as? String == "success") + #expect(jsonObject?["startedAt"] is String) + #expect(jsonObject?["endedAt"] is String) + #expect((jsonObject?["startedAt"] as? String)?.contains("T") == true) + } + } + + @Test func sessionStoreDeletesSessionsOlderThanSevenDays() throws { + let temporaryHome = makeTemporaryHome() + defer { try? FileManager.default.removeItem(at: temporaryHome) } + + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + + let formatter = dayFolderFormatter() + let now = Date() + let eightDaysAgo = Calendar.current.date(byAdding: .day, value: -8, to: now)! + let oldDayFolder = ClickyPaths.sessions.appendingPathComponent(formatter.string(from: eightDaysAgo), isDirectory: true) + let todayFolder = ClickyPaths.sessions.appendingPathComponent(formatter.string(from: now), isDirectory: true) + + try FileManager.default.createDirectory(at: oldDayFolder, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: todayFolder, withIntermediateDirectories: true) + try Data("{}".utf8).write(to: oldDayFolder.appendingPathComponent("old.json")) + try Data("{}".utf8).write(to: todayFolder.appendingPathComponent("recent.json")) + + SessionStore().deleteSessionsOlderThan(days: 7, now: now) + + #expect(FileManager.default.fileExists(atPath: oldDayFolder.path) == false) + #expect(FileManager.default.fileExists(atPath: todayFolder.path)) + } + } + + @Test func sessionTraceEntryIsCodable() throws { + let entry = SessionTraceEntry( + timestamp: Date(timeIntervalSince1970: 1_747_000_000), + userTranscript: "hello", + assistantResponse: "hi", + bundleId: "com.apple.TextEdit", + pointed: true + ) + + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + + let encodedData = try encoder.encode(entry) + let decodedEntry = try decoder.decode(SessionTraceEntry.self, from: encodedData) + + #expect(decodedEntry == entry) + } +} From c3e5bb2e36d355d9432203912e961c82a84bd09e Mon Sep 17 00:00:00 2001 From: octavi42 Date: Mon, 8 Jun 2026 14:33:52 +0300 Subject: [PATCH 12/14] Guard async skill-write wipe against clobbering a new session maybeWriteTeachingSkill's in-flight task ended with an unconditional sessionTrace.removeAll(). Since synthesis is async, it could fire after a new session had already started and wipe its turns before persistence. Only clear the trace when it still matches the synthesized snapshot. --- leanring-buddy/CompanionManager.swift | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index 91bfb50eb..6e0825b6a 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -465,7 +465,15 @@ final class CompanionManager: ObservableObject { bundleId: targetBundleId, skillId: skill.id ) - sessionTrace.removeAll() + // Only wipe the live trace if it still holds the same turns this + // task synthesized. Skill synthesis is async (a multi-second Claude + // call); by the time it finishes, finalizeAndPersistSession may have + // already cleared the trace and a NEW session may have started. An + // unconditional removeAll() here would wipe that new session's turns + // before they can be persisted. + if sessionTrace == traceSnapshot { + sessionTrace.removeAll() + } ClickyAnalytics.trackTeachingSkillSaved( skillID: skill.id, From d540a74edd9a6f777ec6fde095fe204cd088f643 Mon Sep 17 00:00:00 2001 From: octavi42 Date: Mon, 8 Jun 2026 14:39:22 +0300 Subject: [PATCH 13/14] Harden session finalize against save failure and TTS-overlap - Keep the in-memory trace and re-arm the idle timer when sessionStore.save throws, so a transient I/O error retries instead of dropping the capture. - Arm the 30s idle boundary after TTS playback completes rather than in recordSessionExchange, so a long spoken reply no longer fires the timer mid-speech and splits one conversation into two persisted sessions. --- leanring-buddy/CompanionManager.swift | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index 6e0825b6a..7dea8aca9 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -314,7 +314,10 @@ final class CompanionManager: ObservableObject { sessionTrace.removeFirst(sessionTrace.count - 20) } - restartSessionIdleTimer() + // The 30s idle timer is intentionally NOT armed here. recordSessionExchange + // runs before the assistant's TTS playback, so arming now could let the timer + // fire mid-speech on a long reply and split one conversation into two sessions. + // It is armed when the response task returns to idle (after TTS) instead. if !SkillTriggerEvaluator.isConfirmationTranscript(transcript) { let topic = SkillTriggerEvaluator.deriveTopic(fromQuestion: transcript) @@ -363,12 +366,15 @@ final class CompanionManager: ObservableObject { do { let savedURL = try sessionStore.save(session) print("💾 Persisted session to \(savedURL.path)") + // Only discard the in-memory session once it is safely on disk. + sessionStartedAt = nil + sessionTrace.removeAll() } catch { - print("⚠️ Failed to persist session: \(error)") + // Keep the trace and re-arm the idle timer so a transient I/O error + // gets another chance to persist instead of silently losing the capture. + print("⚠️ Failed to persist session, will retry on next idle: \(error)") + restartSessionIdleTimer() } - - sessionStartedAt = nil - sessionTrace.removeAll() } /// Coarse outcome heuristic for capture-time persistence. The future memory gate @@ -1260,6 +1266,12 @@ final class CompanionManager: ObservableObject { if !Task.isCancelled { voiceState = .idle isPushToTalkInteractionActive = false + // Arm the idle boundary now that the assistant has finished speaking, + // so the 30s countdown measures genuine user inactivity rather than + // overlapping with TTS playback. Only relevant while a session is open. + if sessionStartedAt != nil { + restartSessionIdleTimer() + } scheduleTransientHideIfNeeded() } } From 0c40e7c14d107153e897d3ba7e4778f8a27da0b0 Mon Sep 17 00:00:00 2001 From: octavi42 Date: Mon, 8 Jun 2026 14:47:08 +0300 Subject: [PATCH 14/14] Defer idle finalize while TTS is still playing If the 30s idle timer fires while the assistant is mid-speech, re-arm instead of finalizing so a long spoken reply cannot split one conversation into two persisted sessions. --- leanring-buddy/CompanionManager.swift | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index 7dea8aca9..f581fd5a7 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -332,7 +332,15 @@ final class CompanionManager: ObservableObject { sessionIdleTimer?.invalidate() sessionIdleTimer = Timer.scheduledTimer(withTimeInterval: 30, repeats: false) { [weak self] _ in Task { @MainActor in - self?.finalizeAndPersistSession() + guard let self else { return } + // If the assistant is still speaking when the timer fires, defer the + // idle boundary by re-arming. This keeps a long TTS reply from ending + // the session mid-speech and splitting one conversation into two. + if self.elevenLabsTTSClient.isPlaying { + self.restartSessionIdleTimer() + return + } + self.finalizeAndPersistSession() } } }