diff --git a/.github/workflows/e2e-teaching-skills.yml b/.github/workflows/e2e-teaching-skills.yml new file mode 100644 index 000000000..4b64a3b26 --- /dev/null +++ b/.github/workflows/e2e-teaching-skills.yml @@ -0,0 +1,46 @@ +name: E2E Teaching Skills + +on: + push: + branches: + - main + - feature/teaching-skills + - feature/teaching-skills-upstream + pull_request: + branches: + - main + - feature/teaching-skills + - feature/teaching-skills-upstream + +jobs: + teaching-skills-e2e: + name: Teaching skills E2E (macOS) + runs-on: macos-15 + timeout-minutes: 30 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Run teaching-skills E2E + run: | + chmod +x tests/e2e/run-all.sh tests/e2e/teaching-skills.sh tests/e2e/skills-library.sh tests/e2e/lib/common.sh + ./tests/e2e/run-all.sh + + - name: Upload E2E logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: teaching-skills-e2e-logs + if-no-files-found: ignore + path: | + /tmp/clicky-e2e-app.log + /tmp/clicky-e2e-app-read.log + /tmp/clicky-e2e-worker.log + /tmp/clicky-e2e-build.log + ~/.clicky/e2e-last-system-prompt.txt diff --git a/.gitignore b/.gitignore index 832e80a1a..4029e55bb 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ build/ releases/ .claude/ coding-plans/ +.clicky-worktree.env diff --git a/AGENTS.md b/AGENTS.md index 6946d4419..071e82cd4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,9 +53,9 @@ Worker vars: `ELEVENLABS_VOICE_ID` | File | Lines | Purpose | |------|-------|---------| | `leanring_buddyApp.swift` | ~89 | Menu bar app entry point. Uses `@NSApplicationDelegateAdaptor` with `CompanionAppDelegate` which creates `MenuBarPanelManager` and starts `CompanionManager`. No main window — the app lives entirely in the status bar. | -| `CompanionManager.swift` | ~1026 | Central state machine. Owns dictation, shortcut monitoring, screen capture, Claude API, ElevenLabs TTS, and overlay management. Tracks voice state (idle/listening/processing/responding), conversation history, model selection, and cursor visibility. Coordinates the full push-to-talk → screenshot → Claude → TTS → pointing pipeline. | +| `CompanionManager.swift` | ~1070 | Central state machine. Owns dictation, shortcut monitoring, screen capture, Claude API, ElevenLabs TTS, overlay management, teaching skills, and niche discovery. Tracks voice state (idle/listening/processing/responding), conversation history, model selection, and cursor visibility. Coordinates the full push-to-talk → screenshot → Claude → TTS → pointing pipeline. | | `MenuBarPanelManager.swift` | ~243 | NSStatusItem + custom NSPanel lifecycle. Creates the menu bar icon, manages the floating companion panel (show/hide/position), installs click-outside-to-dismiss monitor. | -| `CompanionPanelView.swift` | ~761 | SwiftUI panel content for the menu bar dropdown. Shows companion status, push-to-talk instructions, model picker (Sonnet/Opus), permissions UI, DM feedback button, and quit button. Dark aesthetic using `DS` design system. | +| `CompanionPanelView.swift` | ~850 | SwiftUI panel content for the menu bar dropdown. Shows companion status, push-to-talk instructions, niche onboarding/suggestions, teaching skills UI, model picker (Sonnet/Opus), permissions UI, DM feedback button, and quit button. Dark aesthetic using `DS` design system. | | `OverlayWindow.swift` | ~881 | Full-screen transparent overlay hosting the blue cursor, response text, waveform, and spinner. Handles cursor animation, element pointing with bezier arcs, multi-monitor coordinate mapping, and fade-out transitions. | | `CompanionResponseOverlay.swift` | ~217 | SwiftUI view for the response text bubble and waveform displayed next to the cursor in the overlay. | | `CompanionScreenCaptureUtility.swift` | ~132 | Multi-monitor screenshot capture using ScreenCaptureKit. Returns labeled image data for each connected display. | @@ -71,9 +71,26 @@ Worker vars: `ELEVENLABS_VOICE_ID` | `ElevenLabsTTSClient.swift` | ~81 | ElevenLabs TTS client. Sends text to the Worker proxy, plays back audio via `AVAudioPlayer`. Exposes `isPlaying` for transient cursor scheduling. | | `ElementLocationDetector.swift` | ~335 | Detects UI element locations in screenshots for cursor pointing. | | `DesignSystem.swift` | ~880 | Design system tokens — colors, corner radii, shared styles. All UI references `DS.Colors`, `DS.CornerRadius`, etc. | -| `ClickyAnalytics.swift` | ~121 | PostHog analytics integration for usage tracking. | +| `ClickyAnalytics.swift` | ~175 | PostHog analytics integration for usage tracking. | | `WindowPositionManager.swift` | ~262 | Window placement logic, Screen Recording permission flow, and accessibility permission helpers. | | `AppBundleConfiguration.swift` | ~28 | Runtime configuration reader for keys stored in the app bundle Info.plist. | +| `TeachingTopicHistoryStore.swift` | ~95 | Persists teaching topic tokens, bundle IDs, and timestamps to `~/.clicky/topic-history.json` for cross-session repeat detection. | +| `PersistedSession.swift` | ~25 | Codable on-disk session model (`sessionId`, `startedAt`, `endedAt`, `outcome`, `privacyOptOut`, `appsUsed`, `turns`) for the memory pipeline capture step. | +| `SessionStore.swift` | ~100 | Filesystem store for voice sessions at `~/.clicky/sessions//.json` with ISO8601 JSON encoding and 7-day retention cleanup. | +| `TeachingSkillStore.swift` | ~100 | Create/read/update/delete teaching skills at `~/.clicky/skills//SKILL.md`. | +| `TeachingSkill.swift` | ~280 | Model and YAML frontmatter parsing for teaching skills. | +| `SkillMatcher.swift` | ~170 | App/topic matching and duplicate skill pair detection. | +| `SkillCurator.swift` | ~175 | Time-based stale/archive lifecycle plus throttled LLM merge/patch passes. | +| `SkillSynthesizer.swift` | ~120 | Post-session skill drafting via Claude API. | +| `SkillTriggerEvaluator.swift` | ~120 | Decides when to create or update a teaching skill from session traces. | +| `TeachingPromptBuilder.swift` | ~35 | Composes voice response system prompt with matched teaching skills. | +| `TeachingSkillsLibraryView.swift` | ~260 | Full skills library UI with status filters, detail view, pin/delete/restore. | +| `ClickyE2EConfiguration.swift` | ~75 | Launch flags and E2E debug artifact paths for automated tests. | +| `NicheDiscoveryManager.swift` | ~175 | Niche onboarding, static/app-aware suggestion cards, local override loading. | +| `UserNiche.swift` | ~35 | User niche enum (general, content creator, developer, student, designer). | +| `Resources/niche-examples.json` | ~80 | Bundled niche suggestion packs and app-specific prompt maps. | +| `leanring-buddyTests/TeachingSkillTests.swift` | ~320 | Unit tests for skill parsing, matching, triggers, prompt injection, duplicate detection. | +| `tests/e2e/full-stack/` | — | Scaffold for full user-perspective E2E (BlackHole, PTT simulation, Peekaboo). | | `worker/src/index.ts` | ~142 | Cloudflare Worker proxy. Three routes: `/chat` (Claude), `/tts` (ElevenLabs), `/transcribe-token` (AssemblyAI temp token). | ## Build & Run diff --git a/FUTURE_IMPLEMENTATION.md b/FUTURE_IMPLEMENTATION.md new file mode 100644 index 000000000..41f6e07f8 --- /dev/null +++ b/FUTURE_IMPLEMENTATION.md @@ -0,0 +1,153 @@ +# Future Implementation — Clicky Fork + +Planned features captured from product brainstorming. Not implemented yet. + +**Implementation order & E2E test plan:** see [`IMPLEMENTATION_AND_E2E_PLAN.md`](./IMPLEMENTATION_AND_E2E_PLAN.md) — implement Evolving Teaching Skills first, then run automated user-perspective E2E tests. + +--- + +## 1. Evolving Teaching Skills (Auto-Learning) + +**Problem:** Clicky starts from zero every session. It does not accumulate knowledge about how to teach specific apps or workflows. + +**Concept:** After successful tutoring interactions, Clicky writes reusable teaching skills locally — inspired by Hermes Agent's learning loop, adapted for screen-native teaching (not CLI automation). + +### What a teaching skill is + +A folder under `~/.clicky/skills//SKILL.md` using the [agentskills.io](https://agentskills.io) format: + +- App or workflow name (e.g. `teach-final-cut-color`, `teach-xcode-source-control`) +- Step order and common user mistakes +- UI vocabulary (button labels, menu paths, shortcuts) +- Pointing heuristics (where to point first, what to avoid) +- Completion signals ("user said got it", step confirmed, correct UI state visible) + +### Learning loop + +1. User completes a tutoring session (push-to-talk + screen + pointing). +2. Clicky evaluates whether the session is worth persisting (multi-step help, user confirmed success, or repeated topic). +3. Clicky writes or updates a `SKILL.md` with what worked. +4. Next session: relevant skills are loaded into the system prompt when matching apps or topics appear on screen. + +### Curator (maintenance) + +Prevent skill bloat and stale UI knowledge: + +- Track usage: viewed, used, patched +- Auto-transition: `active → stale → archived` after inactivity (e.g. 30 / 90 days) +- Optional LLM review pass to merge duplicates or patch drift when app UI changes +- User can pin important skills so they are never archived + +### Triggers (when to create/update a skill) + +- Multi-step guidance completed successfully +- User explicitly confirms ("got it", "thanks that worked") +- Same topic asked 2+ times in one week +- NOT: generic Q&A unrelated to screen UI + +### Implementation notes + +| Layer | Approach | +|-------|----------| +| Storage | `~/.clicky/skills/` (local only, privacy-first) | +| Format | `SKILL.md` with YAML frontmatter (`name`, `description`, optional `paths` for app bundles) | +| Injection | Extend `companionVoiceResponseSystemPrompt` with matched skills at request time | +| Worker | Optional `/skill-synthesize` route for post-session skill drafting (keeps API keys off-device) | +| Swift | New `TeachingSkillStore`, `SkillCurator`, hook in `CompanionManager` after successful responses | +| Phase 2 | GEPA-style optimization from traces (Hermes self-evolution pattern) — research/demo only | + +### Success criteria + +- Repeat questions on the same app get faster, more accurate pointing +- Skills stay small and relevant (Curator prevents catalog pollution) +- Demo: "Clicky remembered how to teach me Xcode commits from last week" + +--- + +## 2. Niche Discovery (Onboarding + Suggestions) + +**Problem:** Many users do not know what Clicky is for or what to ask. Blank-slate AI assistants fail the "first 60 seconds" test. + +**Concept:** Help users discover Clicky through niche-specific, concrete examples — not generic "I'm an AI helper" messaging. + +### Core UX + +1. **Onboarding:** Ask what the user does (content creator, developer, student, designer, etc.) or infer from screen. +2. **Suggestion cards:** Show 3–5 actionable example prompts in the menu bar panel. +3. **Spoken intro (optional):** Clicky briefly explains what it can do *for their niche* via TTS. +4. **Live suggestions:** When screen context matches (e.g. Final Cut, OBS, Premiere), surface timely prompts. + +### Example — Content creators + +- "Show me where to add captions in this timeline" +- "Walk me through export settings for YouTube" +- "How do I color grade this clip?" +- "Where is the transition menu?" + +Each suggestion should imply: **voice + screen + pointing**, not plain chat. + +### Niche packs (optional, pairs with teaching skills) + +``` +~/.clicky/niches/ + content-creator/ + examples.json # suggested prompts + intro-script.txt # optional spoken onboarding + developer/ + examples.json + student/ + examples.json +``` + +Niche selection can seed which teaching skills to prioritize and which apps to watch for. + +### Detection strategies + +| Method | Effort | Notes | +|--------|--------|-------| +| User picks niche in onboarding | Low | Store in UserDefaults (`selectedUserNiche`) | +| Frontmost app bundle ID | Medium | Map `com.apple.FinalCut` → content creator suggestions | +| Screenshot + Claude one-shot classify | Medium | Fallback when app mapping unknown | +| Proactive idle suggestions | Higher | Tutor mode synergy — suggest when user pauses in a known app | + +### Implementation notes + +| Layer | Approach | +|-------|----------| +| Swift | `NicheDiscoveryManager`, extend `CompanionPanelView` with suggestion UI | +| Persistence | `UserDefaults` for niche; optional `~/.clicky/profile.md` for richer profile | +| Prompts | Niche-specific example strings + optional niche clause in system prompt | +| Screen | Reuse `CompanionScreenCaptureUtility` + frontmost app from `NSWorkspace` | +| Analytics | Track which suggestions get tapped / spoken (PostHog via existing `ClickyAnalytics`) | + +### Success criteria + +- New users ask a useful question within first session (measurable via analytics) +- Reduced "what do I use this for?" support confusion +- Niche feels personal: content creators see creator examples, not developer ones + +--- + +## Suggested build order + +1. **Niche discovery (light)** — onboarding picker + static suggestion cards per niche +2. **App-aware suggestions** — detect frontmost app, swap examples dynamically +3. **Teaching skills (write)** — post-session skill creation for successful multi-step help +4. **Teaching skills (read + Curator)** — load skills into prompts, maintain library over time +5. **Combine** — niche picks starting suggestions; skills deepen tutoring in that niche over time + +--- + +## Out of scope (for this fork) + +- Full Hermes Agent port (Telegram, VPS, 20+ tools) +- Cloud skill sync / marketplace (local-first unless explicitly added later) +- Replacing push-to-talk with Realtime API (separate future idea) + +--- + +## References + +- Hermes Agent skills + Curator: https://github.com/NousResearch/hermes-agent +- Agent Skills standard: https://agentskills.io +- Related fork ideas: Skilly (teaching skills), moonmidas/clicky (Tutor mode) diff --git a/IMPLEMENTATION_AND_E2E_PLAN.md b/IMPLEMENTATION_AND_E2E_PLAN.md new file mode 100644 index 000000000..25e5dca6f --- /dev/null +++ b/IMPLEMENTATION_AND_E2E_PLAN.md @@ -0,0 +1,165 @@ +# Evolving Teaching Skills — Implementation & E2E Plan + +Captured from product/eng discussion. **Order of work: implement the feature first, then run full automated E2E tests.** + +Related: [`FUTURE_IMPLEMENTATION.md`](./FUTURE_IMPLEMENTATION.md) (feature spec) + +--- + +## Goal + +Clicky learns from successful tutoring sessions and reuses that knowledge in future sessions via local `SKILL.md` files under `~/.clicky/skills/`. + +**Demo success:** *"I taught Clicky how to walk me through Xcode commits yesterday — today it remembers and points better on the first try."* + +--- + +## Phase 1 — Foundation (implement first) + +1. **`TeachingSkillStore.swift`** + - Create/read/update skills at `~/.clicky/skills//SKILL.md` + - Parse YAML frontmatter: `name`, `description`, `bundleIds`, `status`, `lastUsed`, `usageCount` + - Index skills on app launch + +2. **Session trace capture** + - Extend `CompanionManager` beyond in-RAM `conversationHistory` (~10 turns) + - Persist lightweight session log: transcript, response, frontmost app bundle ID, pointing used or not + - Hook after each successful exchange in `sendTranscriptToClaudeWithScreenshot` + +3. **Skill matching at request time** + - Frontmost app via `NSWorkspace.shared.frontmostApplication` + - Match by bundle ID + keyword overlap with user transcript + - Load top 1–3 skills into system prompt (small token budget) + +4. **Prompt injection** + - Replace static `companionVoiceResponseSystemPrompt` with a builder: + - base Clicky prompt + - + matched teaching skills + - Pass composed prompt to `claudeAPI.analyzeImageStreaming` + +--- + +## Phase 2 — Write loop (evolving) + +5. **Trigger evaluator** + - After session / N exchanges, decide create-or-update: + - multi-step help (2+ exchanges with pointing) + - user confirms ("got it", "thanks", "that worked") + - same topic repeated within 7 days + - Skip generic off-screen Q&A + +6. **Skill synthesizer** + - Option A: local Claude call via existing `/chat` worker with synthesis prompt + - Option B: new worker route `POST /skill-synthesize` + - Input: session trace → Output: `SKILL.md` (steps, UI labels, pointing tips, mistakes) + +7. **Create vs update** + - Merge into existing similar skill instead of duplicating + - Bump `lastUsed`, increment `usageCount` + +--- + +## Phase 3 — Curator + +8. **`SkillCurator.swift`** + - Track: viewed, used, patched, archived + - Auto-archive: 30d inactive → stale, 90d → archived + - User pin (never archive) + - Optional LLM pass to merge duplicates / patch stale UI labels + +--- + +## Phase 4 — UX (recommended) + +9. **Minimal UI in `CompanionPanelView`** + - List skills, pin/delete, toggle "learn from sessions" + +10. **Analytics** + - Track: skill matched, created, used (via `ClickyAnalytics` / PostHog) + +--- + +## Files to touch + +| File | Role | +|------|------| +| `CompanionManager.swift` | trace capture, triggers, prompt builder, post-session hook | +| **New** `TeachingSkillStore.swift` | filesystem + parsing | +| **New** `SkillCurator.swift` | lifecycle + dedup | +| **New** `SkillMatcher.swift` | app/topic matching | +| `CompanionPanelView.swift` | skills UI | +| `worker/src/index.ts` | optional `/skill-synthesize` | + +--- + +## E2E automation — full user-perspective stack + +Pure XCUITest alone is not enough for Clicky (mic, screen capture, global shortcuts, pointing overlay, LLM). Use this macOS automation stack instead. + +| Layer | Tool | Purpose | +|-------|------|---------| +| Permissions | [`@guidepup/setup`](https://github.com/guidepup/setup) or [`jacobsalmela/tccutil`](https://github.com/jacobsalmela/tccutil) | Pre-grant Screen Recording, Mic, Accessibility in TCC.db | +| Push-to-talk | [`NaryaAI/voice-testing-tools`](https://github.com/NaryaAI/voice-testing-tools) `simulate-keypress.swift` | Hold/release `ctrl+option` (Clicky's shortcut) via HID events | +| Voice input | Same repo `tts.mjs` + **BlackHole 2ch** | Deterministic TTS → virtual mic → real STT path | +| Screen + UI | [**Peekaboo**](https://github.com/steipete/Peekaboo) `.peekaboo.json` or [**axcli**](https://github.com/andelf/axcli) | Launch app, screenshots, verify overlay/cursor, AX tree | +| Deterministic AI | Mock worker with recorded Claude fixtures | Repeatable pass/fail (real API is flaky) | + +### Prerequisites + +1. **Stable code signing** — ad-hoc builds reset TCC on every rebuild; use dev/Developer ID cert so permissions persist. +2. **Self-hosted Mac test machine** — one-time TCC pre-seed via `@guidepup/setup`; fully unattended cloud CI without setup is unreliable. +3. **BlackHole as mic input** during tests to exercise real transcription. +4. **Mock worker** for E2E assertions (record/replay fixtures). + +### Example E2E flow + +```bash +# One-time setup +npx @guidepup/setup --ci +brew install blackhole-2ch sox switchaudio-osx + +# Launch Clicky + fixture app +open /path/to/Clicky.app +open -a TextEdit + +# Session 1: ask for help +node tools/tts.mjs "how do I save this document?" & +swift tools/simulate-keypress.swift down:ctrl down:option wait:3000 up:option up:ctrl + +# Confirm success (triggers skill write) +node tools/tts.mjs "got it thanks that worked" & +swift tools/simulate-keypress.swift down:ctrl down:option wait:2000 up:option up:ctrl + +# Assert skill file exists +test -f ~/.clicky/skills/teach-textedit-save/SKILL.md + +# Session 2: repeat question, verify skill loaded + pointing +peekaboo run tests/e2e/teaching-skills.peekaboo.json --json +``` + +### Clicky test hooks to add (small, E2E-friendly) + +- Launch flag `-CLICKY_E2E=1`: skip onboarding, point at test worker URL, faster timeouts +- Optional `--inject-transcript` fallback if STT flakes (still runs response + skill loop) +- Expose for assertions: `lastSystemPrompt`, `lastSkillMatched`, overlay accessibility identifiers + +--- + +## Execution order + +1. Implement Phases 1–2 (read + write loop) — minimum viable evolving skills +2. Add test hooks + mock worker fixtures +3. Scaffold `tests/e2e/` (voice-testing-tools + Peekaboo scripts) +4. One-time Mac test machine setup (TCC, BlackHole, signing) +5. Run full E2E regression +6. Phase 3–4 (Curator + UI) as follow-up + +--- + +## References + +- Hermes Agent skills + Curator: https://github.com/NousResearch/hermes-agent +- Agent Skills standard: https://agentskills.io +- Voice E2E tools: https://github.com/NaryaAI/voice-testing-tools +- macOS GUI automation: https://github.com/steipete/Peekaboo +- TCC automation: https://github.com/guidepup/setup diff --git a/SESSION_PERSISTENCE_PLAN.md b/SESSION_PERSISTENCE_PLAN.md new file mode 100644 index 000000000..fd1ae72fd --- /dev/null +++ b/SESSION_PERSISTENCE_PLAN.md @@ -0,0 +1,204 @@ +# Session Persistence — Implementation Plan + +**Branch:** `feature/session-persistence` +**Worktree:** `/Users/cristeaoctavian/Projects/clicky-session-persistence` +**Data home (isolated):** `/tmp/clicky-session-persistence` +**Scope:** Step 1 of the memory pipeline only — **sessions get saved to disk correctly.** Do NOT build the gate, distill, or consent steps here. Keep this PR small and reviewable. + +Reference: [`docs/architecture/MEMORY_PIPELINE.md`](docs/architecture/MEMORY_PIPELINE.md) (esp. "Session boundary" and "Capture" sections) and [`docs/architecture/schemas/session.example.json`](docs/architecture/schemas/session.example.json). + +--- + +## Why this is first + +Today sessions live only in RAM as `CompanionManager.sessionTrace` (`[SessionTraceEntry]`, capped at 20 entries, and **wiped** by `maybeWriteTeachingSkill` on a successful skill write — see `CompanionManager.swift:377`). The downstream gate/distill/consent steps all need a saved session JSON to operate on. Nothing downstream can exist without this. + +--- + +## Context: existing code you will build on + +| Thing | Location | Notes | +|-------|----------|-------| +| `SessionTraceEntry` | `leanring-buddy/TeachingSkill.swift:326` | `timestamp: Date`, `userTranscript: String`, `assistantResponse: String`, `bundleId: String?`, `pointed: Bool`. Currently `Equatable` only — **add `Codable`**. | +| In-RAM trace | `CompanionManager.swift:85` | `private var sessionTrace: [SessionTraceEntry] = []` | +| Capture hook | `CompanionManager.swift:284` `recordSessionExchange(...)` | Appends a turn; caps at 20; records topic history. Called at `CompanionManager.swift:1100`. | +| Skill-write hook | `CompanionManager.swift:312` `maybeWriteTeachingSkill(after:)` | Called at `CompanionManager.swift:1105`, right after `recordSessionExchange`. On success it calls `sessionTrace.removeAll()` at line 377. **This is the wipe you must persist ahead of.** | +| Confirmation detection | `SkillTriggerEvaluator.isConfirmationTranscript(_:)` | Already exists. Reuse for the confirm-phrase session-end trigger. | +| Screen-teaching detection | `SkillTriggerEvaluator.isScreenTeachingSession(_:)` | Reuse if needed for outcome heuristics. | +| Isolated paths | `leanring-buddy/ClickyPaths.swift` | `ClickyPaths.home` (already worktree-isolated via `CLICKY_HOME`). Add a `sessions` URL here. | +| Path tests pattern | `leanring-buddyTests/ClickyPathsTests.swift` | Uses `ClickyPaths.overrideHomeForTesting`. Mirror this for `SessionStore` round-trip tests. | +| Panel close hook | `MenuBarPanelManager.swift:143` `hidePanel()` + `.clickyDismissPanel` notification (`MenuBarPanelManager.swift:17`) | Use to detect "panel closed" session boundary. | +| Launch lifecycle | `CompanionManager.swift:454` `start()` → `bootstrapTeachingSkills()` (`:257`) | Add the 7-day cleanup call here. | +| Learning toggle | `CompanionManager.isLearningFromSessionsEnabled` (`:93`) | Respect it. When disabled, treat the session as `privacyOptOut = true` (still persist, but flagged) — see Open Questions. | + +**Xcode project note:** the target uses `PBXFileSystemSynchronizedRootGroup`, so any new `.swift` file dropped into `leanring-buddy/` (or `leanring-buddyTests/`) is auto-included in the build. **No `project.pbxproj` editing required.** + +**Persistence target path:** `ClickyPaths.home/sessions//.json` (per MEMORY_PIPELINE.md "Capture" → Persistence). The date folder is the session's `startedAt` in local time. + +--- + +## What to build (4 small steps) + +### Step 1 — Model + store + +**1a. Make `SessionTraceEntry` `Codable`** (`TeachingSkill.swift:326`). +Change `struct SessionTraceEntry: Equatable` → `struct SessionTraceEntry: Equatable, Codable`. All stored properties are already `Codable`-compatible. This is the only edit to that file. + +**1b. New file `leanring-buddy/PersistedSession.swift`** — matches `schemas/session.example.json`: + +```swift +struct PersistedSession: Codable, Equatable { + let sessionId: UUID + let startedAt: Date + let endedAt: Date + let outcome: SessionOutcome + let privacyOptOut: Bool + let appsUsed: [String] // unique bundle IDs, in first-seen order + let turns: [SessionTraceEntry] +} + +enum SessionOutcome: String, Codable { + case success + case abandoned + case unknown +} +``` + +Notes: +- The schema uses string values `"success" | "abandoned" | "unknown"`; `SessionOutcome: String` serializes to exactly those. Verify the encoder produces the bare string (it does for `RawRepresentable String` enums). +- Dates in the schema are ISO8601 (`2026-06-05T14:22:10Z`). Configure the `JSONEncoder`/`JSONDecoder` in `SessionStore` with `.iso8601` date strategy so output matches the schema (the default `.deferredToDate` emits numeric timestamps — **do not** use the default). + +**1c. Add `sessions` path to `ClickyPaths`** (`ClickyPaths.swift`, alongside `skills` / `topicHistory`): + +```swift +static var sessions: URL { + home.appendingPathComponent("sessions", isDirectory: true) +} +``` + +**1d. New file `leanring-buddy/SessionStore.swift`** — filesystem store, mirror `TeachingSkillStore` style: + +```swift +final class SessionStore { + static var sessionsRootURL: URL { ClickyPaths.sessions } + + @discardableResult + func save(_ session: PersistedSession) throws -> URL // writes //.json + func deleteSessionsOlderThan(days: Int, now: Date = Date()) // 7-day cleanup + // (optional helpers) func loadAllSessions() -> [PersistedSession] +} +``` + +Implementation details: +- Use a dedicated `JSONEncoder` with `dateEncodingStrategy = .iso8601` and `outputFormatting = [.prettyPrinted, .sortedKeys]`. +- Date-folder name: format `startedAt` with a fixed `DateFormatter` (`yyyy-MM-dd`, `Locale(identifier: "en_US_POSIX")`, current time zone). Create intermediate directories. +- Write atomically (`.atomic`). +- `deleteSessionsOlderThan`: walk `sessionsRootURL`, parse each `` folder (or use file modification date as a fallback) and remove folders/files older than the cutoff. Be defensive — never throw out of cleanup; log and continue. + +### Step 2 — Session-end detection in `CompanionManager` + +The boundary (MEMORY_PIPELINE.md "Session boundary") fires on **any** of: +1. Confirmation phrase on the latest turn — `SkillTriggerEvaluator.isConfirmationTranscript(transcript)`. +2. **30 seconds** idle after the last exchange. +3. Companion panel closed. + +Add: +- `private let sessionStore = SessionStore()` +- `private var sessionStartedAt: Date?` — set on the first `recordSessionExchange` of a new session (i.e. when `sessionTrace` was empty before the append). +- `private var sessionIdleTimer: Timer?` — (re)scheduled at 30s on every `recordSessionExchange`; on fire, call `finalizeAndPersistSession(outcome:)`. Must be created/invalidated on the main actor. +- A guard so a session is only persisted once (e.g. set `sessionStartedAt = nil` after finalize, and bail early if there are no turns). + +Wire the three triggers: +- **Confirm phrase:** in `recordSessionExchange` (or right after it at `:1100`), if `isConfirmationTranscript(transcript)` is true, finalize with `outcome: .success`. +- **Idle:** the `sessionIdleTimer` callback finalizes with `outcome` derived (see Step 3). +- **Panel close:** observe `.clickyDismissPanel` (or add a hook in `MenuBarPanelManager.hidePanel()`), finalize the in-flight session. Prefer subscribing to the existing `.clickyDismissPanel` notification in `CompanionManager` so `MenuBarPanelManager` stays untouched — but note that notification is also posted during onboarding restart (`triggerOnboarding`), so only finalize when `sessionTrace` is non-empty. + +### Step 3 — Finalize + persist + +New `private func finalizeAndPersistSession(outcome: SessionOutcome? = nil)`: +1. Guard: `!sessionTrace.isEmpty`, else return (and clear timer/state). +2. Compute `outcome` if not explicitly passed: + - confirmation phrase seen on any/last turn → `.success` + - idle/panel-close with at least one `pointed` turn or screen-teaching → `.unknown` + - otherwise → `.abandoned` + (Keep this heuristic simple and documented in code comments; it can be refined by the later gate PR.) +3. `appsUsed`: unique `bundleId`s from `sessionTrace.compactMap(\.bundleId)`, preserving first-seen order. +4. `privacyOptOut`: `!isLearningFromSessionsEnabled` (see Open Questions). +5. Build `PersistedSession(sessionId: UUID(), startedAt: sessionStartedAt ?? firstTurnTimestamp, endedAt: Date(), …, turns: sessionTrace)`. +6. `try? sessionStore.save(session)` (log on failure; never crash the voice pipeline). +7. Reset: invalidate `sessionIdleTimer`, `sessionStartedAt = nil`, `sessionTrace.removeAll()`. + +**Critical ordering vs. skill write (the line 377 wipe):** +`maybeWriteTeachingSkill` is `async` (spawns `skillWriteTask`) and wipes `sessionTrace` on success. To avoid losing turns: +- **Preferred:** persist the session *before* `maybeWriteTeachingSkill` can wipe it. The simplest correct approach: have the confirm-phrase finalize run at the call site (`:1100`–`:1105`) **before** `maybeWriteTeachingSkill(after:)`, OR snapshot the trace into the `PersistedSession` synchronously at finalize time (finalize takes a value copy of `sessionTrace`, so even if the async task later wipes the live array, the persisted copy is intact). +- Because `finalizeAndPersistSession` copies `sessionTrace` by value into `PersistedSession.turns` synchronously, persistence is safe regardless of the async wipe **as long as finalize runs**. The risk is double-wipe / lost session if only the skill path runs and finalize never fires. Ensure the confirm-phrase path always calls finalize. +- Do not remove or change `maybeWriteTeachingSkill`'s existing behavior. This PR only *adds* persistence. + +### Step 4 — 7-day cleanup on launch + +In `bootstrapTeachingSkills()` (`CompanionManager.swift:257`) or directly in `start()` (`:454`), add: +```swift +sessionStore.deleteSessionsOlderThan(days: 7) +``` +Spec: "Delete after successful distillation or after 7 days." Distill-deletion is out of scope for this PR; only the 7-day sweep is implemented here. + +--- + +## Tests + +Add `leanring-buddyTests/SessionStoreTests.swift` (Swift Testing, mirror `ClickyPathsTests.swift`): +- Use `ClickyPaths.overrideHomeForTesting = ` with `@Suite(.serialized)` (the override is a shared global — serialize like `ClickyPathsTests`). +- **Round-trip:** save a `PersistedSession` with 2 turns, read the JSON back off disk, decode, `#expect` equality of all fields (including ISO8601 dates and outcome string). +- **Path shape:** assert the file lands at `/sessions//.json`. +- **JSON shape:** decode the raw file and confirm `outcome` serializes as a bare string and dates are ISO8601 strings (guards against the default numeric date strategy regressing). +- **Cleanup:** create a session folder dated 8 days ago and one dated today; call `deleteSessionsOlderThan(days: 7)`; assert old removed, recent kept. +- Optionally add a `SessionTraceEntry` Codable round-trip test. + +Run tests from **Xcode** (Cmd+U) — **do NOT run `xcodebuild` from the terminal** (invalidates TCC permissions per AGENTS.md). + +--- + +## Manual verification + +1. Open `/Users/cristeaoctavian/Projects/clicky-session-persistence/leanring-buddy.xcodeproj` in Xcode (this worktree is `--primary`, so push-to-talk is enabled and scheme args set `CLICKY_HOME=/tmp/clicky-session-persistence`). +2. Cmd+R, grant TCC permissions to this build if asked. +3. Do a voice session (push-to-talk), ask something, then say "thanks" / "got it". +4. Confirm a JSON file appears under `/tmp/clicky-session-persistence/sessions//`, and its contents match the `session.example.json` shape. +5. Test idle path: do an exchange, wait 30s without speaking, confirm a session JSON is written with a non-success outcome. +6. Test panel-close path: do an exchange, click the menu bar icon to close the panel, confirm persistence. + +--- + +## Files touched / added + +**Add:** +- `leanring-buddy/PersistedSession.swift` +- `leanring-buddy/SessionStore.swift` +- `leanring-buddyTests/SessionStoreTests.swift` + +**Edit:** +- `leanring-buddy/TeachingSkill.swift` — add `Codable` to `SessionTraceEntry` (one line). +- `leanring-buddy/ClickyPaths.swift` — add `sessions` URL. +- `leanring-buddy/CompanionManager.swift` — `sessionStore`, `sessionStartedAt`, `sessionIdleTimer`, session-end triggers, `finalizeAndPersistSession`, 7-day cleanup on launch. + +**Docs (per AGENTS.md self-update):** +- Update the "Key Files" table in `AGENTS.md` with `PersistedSession.swift` and `SessionStore.swift`. +- Update `docs/architecture/MEMORY_PIPELINE.md` "Existing code map" — flip "Session persist" from ❌ to ✅. + +--- + +## Scope guardrails (do NOT do here) + +- No `MemoryGate`, no distill, no consent UI, no preferences/routines/activity log. +- Do not refactor `maybeWriteTeachingSkill` or `recordSessionExchange` beyond what is needed to hook finalize. +- Do not fix the known non-blocking warnings (Swift 6 concurrency, deprecated `onChange`). +- Do not rename the project/scheme ("leanring" typo is intentional). +- Do not run `xcodebuild` from the terminal. + +--- + +## Open questions (decide, note your choice in the PR) + +1. **`privacyOptOut` semantics.** When `isLearningFromSessionsEnabled == false`: (a) persist with `privacyOptOut = true` (recommended — keeps the boundary logic uniform, lets the future gate skip it), or (b) skip persistence entirely. Recommend (a). The MEMORY_PIPELINE "do NOT save when `privacyOptOut`" rule applies at the **gate** stage, not capture. +2. **Multiple end-triggers firing close together** (e.g. confirm phrase then panel close). Guard with `sessionStartedAt == nil` after finalize so only one JSON is written per session. +3. **Outcome heuristic granularity.** Keep it coarse here; the gate PR will refine. Document whatever you pick inline. diff --git a/leanring-buddy.xcodeproj/project.pbxproj b/leanring-buddy.xcodeproj/project.pbxproj index 75e572618..56a30bac5 100644 --- a/leanring-buddy.xcodeproj/project.pbxproj +++ b/leanring-buddy.xcodeproj/project.pbxproj @@ -416,6 +416,7 @@ ENABLE_HARDENED_RUNTIME = YES; ENABLE_OUTGOING_NETWORK_CONNECTIONS = YES; ENABLE_PREVIEWS = YES; + ENABLE_TESTABILITY = YES; ENABLE_USER_SELECTED_FILES = readonly; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_FILE = "leanring-buddy/Info.plist"; @@ -429,6 +430,7 @@ ); MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = "com.yourcompany.leanring-buddy"; + PRODUCT_MODULE_NAME = leanring_buddy; PRODUCT_NAME = Clicky; REGISTER_APP_GROUPS = YES; STRING_CATALOG_GENERATE_SYMBOLS = YES; @@ -454,6 +456,7 @@ ENABLE_HARDENED_RUNTIME = YES; ENABLE_OUTGOING_NETWORK_CONNECTIONS = YES; ENABLE_PREVIEWS = YES; + ENABLE_TESTABILITY = YES; ENABLE_USER_SELECTED_FILES = readonly; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_FILE = "leanring-buddy/Info.plist"; @@ -467,6 +470,7 @@ ); MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = "com.yourcompany.leanring-buddy"; + PRODUCT_MODULE_NAME = leanring_buddy; PRODUCT_NAME = Clicky; REGISTER_APP_GROUPS = YES; STRING_CATALOG_GENERATE_SYMBOLS = YES; @@ -495,7 +499,7 @@ SWIFT_EMIT_LOC_STRINGS = NO; SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; SWIFT_VERSION = 5.0; - TEST_HOST = "$(BUILT_PRODUCTS_DIR)/leanring-buddy.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/leanring-buddy"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/Clicky.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Clicky"; }; name = Debug; }; @@ -516,7 +520,7 @@ SWIFT_EMIT_LOC_STRINGS = NO; SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; SWIFT_VERSION = 5.0; - TEST_HOST = "$(BUILT_PRODUCTS_DIR)/leanring-buddy.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/leanring-buddy"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/Clicky.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Clicky"; }; name = Release; }; diff --git a/leanring-buddy.xcodeproj/xcshareddata/xcschemes/leanring-buddy.xcscheme b/leanring-buddy.xcodeproj/xcshareddata/xcschemes/leanring-buddy.xcscheme new file mode 100644 index 000000000..e8f7fb13b --- /dev/null +++ b/leanring-buddy.xcodeproj/xcshareddata/xcschemes/leanring-buddy.xcscheme @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/leanring-buddy/AppBundleConfiguration.swift b/leanring-buddy/AppBundleConfiguration.swift index 6fd233757..c7671af04 100644 --- a/leanring-buddy/AppBundleConfiguration.swift +++ b/leanring-buddy/AppBundleConfiguration.swift @@ -8,6 +8,18 @@ import Foundation enum AppBundleConfiguration { + private static let placeholderWorkerBaseURL = "https://your-worker-name.your-subdomain.workers.dev" + + /// Cloudflare Worker base URL for Claude, TTS, and AssemblyAI token routes. + /// Set `ClickyWorkerBaseURL` in Info.plist (no trailing slash). + static var workerBaseURL: String { + stringValue(forKey: "ClickyWorkerBaseURL") ?? placeholderWorkerBaseURL + } + + static var isWorkerBaseURLConfigured: Bool { + workerBaseURL != placeholderWorkerBaseURL + } + static func stringValue(forKey key: String) -> String? { if let value = Bundle.main.object(forInfoDictionaryKey: key) as? String { let trimmedValue = value.trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/leanring-buddy/AssemblyAIStreamingTranscriptionProvider.swift b/leanring-buddy/AssemblyAIStreamingTranscriptionProvider.swift index d21286b65..bab81a1a2 100644 --- a/leanring-buddy/AssemblyAIStreamingTranscriptionProvider.swift +++ b/leanring-buddy/AssemblyAIStreamingTranscriptionProvider.swift @@ -19,7 +19,9 @@ struct AssemblyAIStreamingTranscriptionProviderError: LocalizedError { final class AssemblyAIStreamingTranscriptionProvider: BuddyTranscriptionProvider { /// URL for the Cloudflare Worker endpoint that returns a short-lived /// AssemblyAI streaming token. The real API key never leaves the server. - private static let tokenProxyURL = "https://your-worker-name.your-subdomain.workers.dev/transcribe-token" + private static var tokenProxyURL: String { + "\(AppBundleConfiguration.workerBaseURL)/transcribe-token" + } let displayName = "AssemblyAI" let requiresSpeechRecognitionPermission = false diff --git a/leanring-buddy/ClaudeAPI.swift b/leanring-buddy/ClaudeAPI.swift index 0c7070b56..4f5f528ee 100644 --- a/leanring-buddy/ClaudeAPI.swift +++ b/leanring-buddy/ClaudeAPI.swift @@ -288,4 +288,51 @@ class ClaudeAPI { let duration = Date().timeIntervalSince(startTime) return (text: text, duration: duration) } + + /// Sends a text-only request to Claude. Used for skill synthesis and other + /// non-vision tasks that still route through the worker proxy. + func sendTextMessage( + systemPrompt: String, + userPrompt: String, + maxTokens: Int = 1024 + ) async throws -> (text: String, duration: TimeInterval) { + let startTime = Date() + + var request = makeAPIRequest() + let body: [String: Any] = [ + "model": model, + "max_tokens": maxTokens, + "system": systemPrompt, + "messages": [ + ["role": "user", "content": userPrompt] + ] + ] + + request.httpBody = try JSONSerialization.data(withJSONObject: body) + let (data, response) = try await session.data(for: request) + + guard let httpResponse = response as? HTTPURLResponse, + (200...299).contains(httpResponse.statusCode) else { + let responseString = String(data: data, encoding: .utf8) ?? "Unknown error" + throw NSError( + domain: "ClaudeAPI", + code: (response as? HTTPURLResponse)?.statusCode ?? -1, + userInfo: [NSLocalizedDescriptionKey: "API Error: \(responseString)"] + ) + } + + let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] + guard let content = json?["content"] as? [[String: Any]], + let textBlock = content.first(where: { ($0["type"] as? String) == "text" }), + let text = textBlock["text"] as? String else { + throw NSError( + domain: "ClaudeAPI", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "Invalid response format"] + ) + } + + let duration = Date().timeIntervalSince(startTime) + return (text: text, duration: duration) + } } diff --git a/leanring-buddy/ClickyAnalytics.swift b/leanring-buddy/ClickyAnalytics.swift index 29e26138e..84ac27da7 100644 --- a/leanring-buddy/ClickyAnalytics.swift +++ b/leanring-buddy/ClickyAnalytics.swift @@ -118,4 +118,64 @@ enum ClickyAnalytics { "error": error ]) } + + // MARK: - Teaching Skills + + static func trackTeachingSkillsMatched(skillIDs: [String], bundleID: String?) { + PostHogSDK.shared.capture("teaching_skills_matched", properties: [ + "skill_ids": skillIDs, + "bundle_id": bundleID ?? "unknown", + "count": skillIDs.count + ]) + } + + static func trackTeachingSkillWriteTriggered(reason: String, topic: String) { + PostHogSDK.shared.capture("teaching_skill_write_triggered", properties: [ + "reason": reason, + "topic": topic + ]) + } + + static func trackTeachingSkillSaved(skillID: String, reason: String, updatedExisting: Bool) { + PostHogSDK.shared.capture("teaching_skill_saved", properties: [ + "skill_id": skillID, + "reason": reason, + "updated_existing": updatedExisting + ]) + } + + static func trackTeachingSkillDeleted(skillID: String) { + PostHogSDK.shared.capture("teaching_skill_deleted", properties: [ + "skill_id": skillID + ]) + } + + static func trackTeachingSkillMerged(primarySkillID: String, duplicateSkillID: String) { + PostHogSDK.shared.capture("teaching_skill_merged", properties: [ + "primary_skill_id": primarySkillID, + "duplicate_skill_id": duplicateSkillID + ]) + } + + static func trackTeachingSkillPatched(skillID: String) { + PostHogSDK.shared.capture("teaching_skill_patched", properties: [ + "skill_id": skillID + ]) + } + + // MARK: - Niche Discovery + + static func trackNicheSelected(niche: String) { + PostHogSDK.shared.capture("niche_selected", properties: [ + "niche": niche + ]) + } + + static func trackNicheSuggestionTapped(suggestion: String, niche: String, bundleID: String?) { + PostHogSDK.shared.capture("niche_suggestion_tapped", properties: [ + "suggestion": suggestion, + "niche": niche, + "bundle_id": bundleID ?? "unknown" + ]) + } } diff --git a/leanring-buddy/ClickyE2EConfiguration.swift b/leanring-buddy/ClickyE2EConfiguration.swift new file mode 100644 index 000000000..25332957d --- /dev/null +++ b/leanring-buddy/ClickyE2EConfiguration.swift @@ -0,0 +1,109 @@ +// +// ClickyE2EConfiguration.swift +// leanring-buddy +// +// Launch flags used by automated end-to-end tests. +// + +import Foundation + +enum ClickyE2EConfiguration { + static var isEnabled: Bool { + ProcessInfo.processInfo.arguments.contains("-CLICKY_E2E=1") + } + + static var workerBaseURL: String? { + argumentValue(for: "-CLICKY_WORKER_URL=") + } + + static var injectTranscript: String? { + argumentValue(for: "-CLICKY_INJECT_TRANSCRIPT=") + } + + static var injectTranscript2: String? { + argumentValue(for: "-CLICKY_INJECT_TRANSCRIPT_2=") + } + + static var injectTranscript3: String? { + argumentValue(for: "-CLICKY_INJECT_TRANSCRIPT_3=") + } + + static var restoreSkillID: String? { + argumentValue(for: "-CLICKY_E2E_RESTORE_SKILL=") + } + + static var clickyDirectoryURL: URL { + ClickyPaths.home + } + + static var lastSystemPromptFileURL: URL { + clickyDirectoryURL.appendingPathComponent("e2e-last-system-prompt.txt") + } + + static var lastMatchedSkillIDFileURL: URL { + clickyDirectoryURL.appendingPathComponent("e2e-last-matched-skill-id.txt") + } + + static var skillsCountFileURL: URL { + clickyDirectoryURL.appendingPathComponent("e2e-skills-count.txt") + } + + static var skillLibraryStateFileURL: URL { + clickyDirectoryURL.appendingPathComponent("e2e-skill-library-state.txt") + } + + static func applyLaunchOverrides() { + guard isEnabled else { return } + + ClickyDefaults.shared.set(true, forKey: "hasCompletedOnboarding") + ClickyDefaults.shared.set(true, forKey: "hasSubmittedEmail") + ClickyDefaults.shared.set(true, forKey: "isClickyCursorEnabled") + } + + static func writeLastSystemPromptForE2E(_ systemPrompt: String) { + guard isEnabled else { return } + writeText(systemPrompt, to: lastSystemPromptFileURL) + } + + static func writeLastMatchedSkillIDForE2E(_ skillID: String?) { + guard isEnabled else { return } + + if let skillID { + writeText(skillID, to: lastMatchedSkillIDFileURL) + } else { + try? FileManager.default.removeItem(at: lastMatchedSkillIDFileURL) + } + } + + static func writeSkillsCountForE2E(_ skillsCount: Int) { + guard isEnabled else { return } + writeText(String(skillsCount), to: skillsCountFileURL) + } + + static func writeSkillLibraryStateForE2E(_ skills: [TeachingSkill]) { + guard isEnabled else { return } + + let libraryEntries = skills.map { skill in + [ + "id": skill.id, + "status": skill.status.rawValue, + "pinned": skill.isPinned + ] as [String: Any] + } + + guard let jsonData = try? JSONSerialization.data(withJSONObject: libraryEntries, options: [.prettyPrinted]), + let jsonString = String(data: jsonData, encoding: .utf8) else { + return + } + writeText(jsonString, to: skillLibraryStateFileURL) + } + + private static func writeText(_ text: String, to fileURL: URL) { + try? FileManager.default.createDirectory(at: clickyDirectoryURL, withIntermediateDirectories: true) + try? text.write(to: fileURL, atomically: true, encoding: .utf8) + } + + private static func argumentValue(for prefix: String) -> String? { + ClickyLaunchArguments.value(forPrefix: prefix) + } +} diff --git a/leanring-buddy/ClickyPaths.swift b/leanring-buddy/ClickyPaths.swift new file mode 100644 index 000000000..9bb329cdd --- /dev/null +++ b/leanring-buddy/ClickyPaths.swift @@ -0,0 +1,71 @@ +// +// ClickyPaths.swift +// leanring-buddy +// +// Centralizes Clicky data home, launch arguments, and isolated UserDefaults. +// Enables parallel worktree testing via CLICKY_HOME / -CLICKY_HOME=. +// + +import Foundation + +enum ClickyLaunchArguments { + static func value(forPrefix prefix: String) -> String? { + ProcessInfo.processInfo.arguments + .first { $0.hasPrefix(prefix) } + .map { String($0.dropFirst(prefix.count)) } + .flatMap { $0.isEmpty ? nil : $0 } + } + + static func isPresent(_ flag: String) -> Bool { + ProcessInfo.processInfo.arguments.contains(flag) + } +} + +enum ClickyPaths { + /// Test-only override. Production never sets this. + static var overrideHomeForTesting: URL? + + /// When true, `home` skips `-CLICKY_HOME=` and `CLICKY_HOME` so tests can + /// assert the production default (`~/.clicky`) even when Xcode inherits + /// worktree env vars from the scheme or parent process. + static var ignoreConfiguredHomeForTesting = false + + static var home: URL { + if let overrideHomeForTesting { + return overrideHomeForTesting + } + if !ignoreConfiguredHomeForTesting { + if let argumentPath = ClickyLaunchArguments.value(forPrefix: "-CLICKY_HOME=") { + return URL(fileURLWithPath: argumentPath, isDirectory: true) + } + if let environmentPath = ProcessInfo.processInfo.environment["CLICKY_HOME"], + !environmentPath.isEmpty { + return URL(fileURLWithPath: environmentPath, isDirectory: true) + } + } + return FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent(".clicky", isDirectory: true) + } + + static var skills: URL { + home.appendingPathComponent("skills", isDirectory: true) + } + + static var topicHistory: URL { + home.appendingPathComponent("topic-history.json") + } + + static var sessions: URL { + home.appendingPathComponent("sessions", isDirectory: true) + } +} + +enum ClickyDefaults { + static let shared: UserDefaults = { + if let suiteName = ClickyLaunchArguments.value(forPrefix: "-CLICKY_DEFAULTS_SUITE="), + let suiteDefaults = UserDefaults(suiteName: suiteName) { + return suiteDefaults + } + return .standard + }() +} diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index 0234cf19f..f581fd5a7 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -8,10 +8,10 @@ // import AVFoundation +import AppKit import Combine import Foundation import PostHog -import ScreenCaptureKit import SwiftUI enum CompanionVoiceState { @@ -27,6 +27,7 @@ final class CompanionManager: ObservableObject { @Published private(set) var lastTranscript: String? @Published private(set) var currentAudioPowerLevel: CGFloat = 0 @Published private(set) var hasAccessibilityPermission = false + @Published private(set) var hasInputMonitoringPermission = false @Published private(set) var hasScreenRecordingPermission = false @Published private(set) var hasMicrophonePermission = false @Published private(set) var hasScreenContentPermission = false @@ -70,7 +71,40 @@ final class CompanionManager: ObservableObject { /// Base URL for the Cloudflare Worker proxy. All API requests route /// through this so keys never ship in the app binary. - private static let workerBaseURL = "https://your-worker-name.your-subdomain.workers.dev" + private static var workerBaseURL: String { + ClickyE2EConfiguration.workerBaseURL ?? AppBundleConfiguration.workerBaseURL + } + + /// True when the global CGEvent tap is active and can detect Control+Option. + var isPushToTalkHotkeyActive: Bool { + hasInputMonitoringPermission && globalPushToTalkShortcutMonitor.isEventTapInstalled + } + + private let teachingSkillStore = TeachingSkillStore() + private let topicHistoryStore = TeachingTopicHistoryStore() + private let sessionStore = SessionStore() + private var sessionTrace: [SessionTraceEntry] = [] + private var sessionStartedAt: Date? + private var sessionIdleTimer: Timer? + /// True from push-to-talk press until the voice pipeline returns to idle. + /// Prevents panel-close finalization while the menu bar panel is auto-dismissed on PTT. + private var isPushToTalkInteractionActive = false + private var panelClosedObserver: NSObjectProtocol? + private var skillWriteTask: Task? + private var curatorLLMTask: Task? + + /// Skills currently on disk, exposed for the panel UI. + @Published private(set) var teachingSkills: [TeachingSkill] = [] + + /// When disabled, Clicky still reads skills but will not create new ones. + @Published var isLearningFromSessionsEnabled: Bool = ClickyDefaults.shared.object(forKey: "isLearningFromSessionsEnabled") == nil + ? true + : ClickyDefaults.shared.bool(forKey: "isLearningFromSessionsEnabled") + + /// Exposed for automated E2E assertions. + @Published private(set) var lastSystemPrompt: String? + @Published private(set) var lastMatchedSkillNames: [String] = [] + @Published private(set) var lastSkillWriteTrigger: String? private lazy var claudeAPI: ClaudeAPI = { return ClaudeAPI(proxyURL: "\(Self.workerBaseURL)/chat", model: selectedModel) @@ -92,15 +126,26 @@ final class CompanionManager: ObservableObject { private var voiceStateCancellable: AnyCancellable? private var audioPowerCancellable: AnyCancellable? private var accessibilityCheckTimer: Timer? + private var workspaceActivationObserver: NSObjectProtocol? + private var permissionRefreshBurstTask: Task? + private var hasLoggedPermissionDiagnostics = false private var pendingKeyboardShortcutStartTask: Task? /// Scheduled hide for transient cursor mode — cancelled if the user /// speaks again before the delay elapses. private var transientHideTask: Task? - /// True when all three required permissions (accessibility, screen recording, - /// microphone) are granted. Used by the panel to show a single "all good" state. + /// Path to the Clicky.app bundle for this run. Shown when TCC must target this build. + var runningApplicationBundlePath: String { + Bundle.main.bundlePath + } + + /// True when all required permissions are granted. Used by the panel to show a single "all good" state. var allPermissionsGranted: Bool { - hasAccessibilityPermission && hasScreenRecordingPermission && hasMicrophonePermission && hasScreenContentPermission + hasAccessibilityPermission + && hasInputMonitoringPermission + && hasScreenRecordingPermission + && hasMicrophonePermission + && hasScreenContentPermission } /// Whether the blue cursor overlay is currently visible on screen. @@ -108,24 +153,371 @@ final class CompanionManager: ObservableObject { @Published private(set) var isOverlayVisible: Bool = false /// The Claude model used for voice responses. Persisted to UserDefaults. - @Published var selectedModel: String = UserDefaults.standard.string(forKey: "selectedClaudeModel") ?? "claude-sonnet-4-6" + @Published var selectedModel: String = ClickyDefaults.shared.string(forKey: "selectedClaudeModel") ?? "claude-sonnet-4-6" + + func setLearningFromSessionsEnabled(_ enabled: Bool) { + isLearningFromSessionsEnabled = enabled + ClickyDefaults.shared.set(enabled, forKey: "isLearningFromSessionsEnabled") + } + + func refreshTeachingSkills() { + teachingSkillStore.loadSkills() + teachingSkills = teachingSkillStore.skills + } + + func deleteTeachingSkill(id: String) { + do { + try teachingSkillStore.deleteSkill(id: id) + teachingSkills = teachingSkillStore.skills + ClickyAnalytics.trackTeachingSkillDeleted(skillID: id) + } catch { + print("⚠️ Failed to delete teaching skill \(id): \(error)") + } + } + + func setTeachingSkillPinned(id: String, pinned: Bool) { + do { + try teachingSkillStore.setPinned(id: id, pinned: pinned) + teachingSkills = teachingSkillStore.skills + } catch { + print("⚠️ Failed to pin teaching skill \(id): \(error)") + } + } + + func restoreTeachingSkill(id: String) { + do { + try teachingSkillStore.restoreSkill(id: id) + teachingSkills = teachingSkillStore.skills + writeE2EArtifactsIfNeeded() + } catch { + print("⚠️ Failed to restore teaching skill \(id): \(error)") + } + } + + func teachingSkills(withStatus status: TeachingSkillStatus?) -> [TeachingSkill] { + teachingSkillStore.skills(withStatus: status) + } + + /// Allows E2E tests to bypass microphone/STT and exercise the response + skill loop directly. + func injectTranscriptForE2E(_ transcript: String) async { + await withCheckedContinuation { (continuation: CheckedContinuation) in + sendTranscriptToClaudeWithScreenshot(transcript: transcript) { + continuation.resume() + } + } + } + + func runE2EInjectSequenceIfNeeded() { + guard ClickyE2EConfiguration.isEnabled else { return } + + Task { + try? await Task.sleep(nanoseconds: 2_000_000_000) + + if let firstTranscript = ClickyE2EConfiguration.injectTranscript { + print("🧪 E2E inject 1: \(firstTranscript)") + await injectTranscriptForE2E(firstTranscript) + + if let secondTranscript = ClickyE2EConfiguration.injectTranscript2 { + try? await Task.sleep(nanoseconds: 2_000_000_000) + print("🧪 E2E inject 2: \(secondTranscript)") + await injectTranscriptForE2E(secondTranscript) + } + + if let thirdTranscript = ClickyE2EConfiguration.injectTranscript3 { + try? await Task.sleep(nanoseconds: 2_000_000_000) + print("🧪 E2E inject 3: \(thirdTranscript)") + await injectTranscriptForE2E(thirdTranscript) + try? await Task.sleep(nanoseconds: 3_000_000_000) + } + } else if let thirdTranscript = ClickyE2EConfiguration.injectTranscript3 { + print("🧪 E2E inject read-path: \(thirdTranscript)") + await injectTranscriptForE2E(thirdTranscript) + } + } + } + + func runE2EBootstrapActionsIfNeeded() { + guard ClickyE2EConfiguration.isEnabled else { return } + + Task { + try? await Task.sleep(nanoseconds: 500_000_000) + + if let restoreSkillID = ClickyE2EConfiguration.restoreSkillID { + restoreTeachingSkill(id: restoreSkillID) + } + + writeE2EArtifactsIfNeeded() + } + } + + func writeE2EArtifactsIfNeeded() { + guard ClickyE2EConfiguration.isEnabled else { return } + + ClickyE2EConfiguration.writeSkillsCountForE2E(teachingSkillStore.skills.count) + ClickyE2EConfiguration.writeSkillLibraryStateForE2E(teachingSkillStore.skills) + } + + private func frontmostApplicationBundleId() -> String? { + NSWorkspace.shared.frontmostApplication?.bundleIdentifier + } + + private func bootstrapTeachingSkills() { + teachingSkillStore.loadSkills() + topicHistoryStore.load() + sessionStore.deleteSessionsOlderThan(days: 7) + SkillCurator.curate(store: teachingSkillStore) + teachingSkills = teachingSkillStore.skills + runCuratorLLMPassesIfNeeded() + writeE2EArtifactsIfNeeded() + } + + private func runCuratorLLMPassesIfNeeded() { + curatorLLMTask?.cancel() + curatorLLMTask = Task { + await SkillCurator.curateWithLLMPasses(store: teachingSkillStore, claudeAPI: claudeAPI) + teachingSkills = teachingSkillStore.skills + } + } + + private func matchedSkills(for transcript: String) -> [TeachingSkill] { + let bundleId = TeachingSkill.detectBundleId(in: transcript) ?? frontmostApplicationBundleId() + let matches = SkillMatcher.matchSkills( + from: teachingSkillStore.skills, + bundleId: bundleId, + transcript: transcript + ) + return matches.map(\.skill) + } + + private func recordSessionExchange( + transcript: String, + spokenResponse: String, + pointed: Bool + ) { + let wasEmptyBeforeAppend = sessionTrace.isEmpty + + sessionTrace.append( + SessionTraceEntry( + timestamp: Date(), + userTranscript: transcript, + assistantResponse: spokenResponse, + bundleId: frontmostApplicationBundleId(), + pointed: pointed + ) + ) + + if wasEmptyBeforeAppend { + sessionStartedAt = Date() + } + + if sessionTrace.count > 20 { + sessionTrace.removeFirst(sessionTrace.count - 20) + } + + // The 30s idle timer is intentionally NOT armed here. recordSessionExchange + // runs before the assistant's TTS playback, so arming now could let the timer + // fire mid-speech on a long reply and split one conversation into two sessions. + // It is armed when the response task returns to idle (after TTS) instead. + + if !SkillTriggerEvaluator.isConfirmationTranscript(transcript) { + let topic = SkillTriggerEvaluator.deriveTopic(fromQuestion: transcript) + topicHistoryStore.recordTopic( + topic: topic, + bundleId: frontmostApplicationBundleId() + ) + } + } + + private func restartSessionIdleTimer() { + sessionIdleTimer?.invalidate() + sessionIdleTimer = Timer.scheduledTimer(withTimeInterval: 30, repeats: false) { [weak self] _ in + Task { @MainActor in + guard let self else { return } + // If the assistant is still speaking when the timer fires, defer the + // idle boundary by re-arming. This keeps a long TTS reply from ending + // the session mid-speech and splitting one conversation into two. + if self.elevenLabsTTSClient.isPlaying { + self.restartSessionIdleTimer() + return + } + self.finalizeAndPersistSession() + } + } + } + + private func finalizeAndPersistSession(outcome explicitOutcome: SessionOutcome? = nil) { + sessionIdleTimer?.invalidate() + sessionIdleTimer = nil + + guard sessionStartedAt != nil else { return } + + let turnsSnapshot = sessionTrace + guard !turnsSnapshot.isEmpty else { + sessionStartedAt = nil + return + } + + let resolvedOutcome = explicitOutcome ?? deriveSessionOutcome(from: turnsSnapshot) + let appsUsed = orderedUniqueBundleIds(from: turnsSnapshot) + let privacyOptOut = !isLearningFromSessionsEnabled + + let session = PersistedSession( + sessionId: UUID(), + startedAt: sessionStartedAt ?? turnsSnapshot.first!.timestamp, + endedAt: Date(), + outcome: resolvedOutcome, + privacyOptOut: privacyOptOut, + appsUsed: appsUsed, + turns: turnsSnapshot + ) + + do { + let savedURL = try sessionStore.save(session) + print("💾 Persisted session to \(savedURL.path)") + // Only discard the in-memory session once it is safely on disk. + sessionStartedAt = nil + sessionTrace.removeAll() + } catch { + // Keep the trace and re-arm the idle timer so a transient I/O error + // gets another chance to persist instead of silently losing the capture. + print("⚠️ Failed to persist session, will retry on next idle: \(error)") + restartSessionIdleTimer() + } + } + + /// Coarse outcome heuristic for capture-time persistence. The future memory gate + /// can refine this; keep the rules simple and predictable here. + private func deriveSessionOutcome(from turns: [SessionTraceEntry]) -> SessionOutcome { + if let lastTurn = turns.last, + SkillTriggerEvaluator.isConfirmationTranscript(lastTurn.userTranscript) { + return .success + } + + if turns.contains(where: \.pointed) || + SkillTriggerEvaluator.isScreenTeachingSession(turns) { + return .unknown + } + + return .abandoned + } + + private func orderedUniqueBundleIds(from turns: [SessionTraceEntry]) -> [String] { + var seenBundleIds = Set() + var orderedBundleIds: [String] = [] + + for bundleId in turns.compactMap(\.bundleId) { + if seenBundleIds.insert(bundleId).inserted { + orderedBundleIds.append(bundleId) + } + } + + return orderedBundleIds + } + + private func maybeWriteTeachingSkill(after transcript: String) { + guard isLearningFromSessionsEnabled else { return } + guard SkillTriggerEvaluator.isScreenTeachingSession(sessionTrace) else { return } + guard let trigger = SkillTriggerEvaluator.shouldWriteSkill( + sessionTrace: sessionTrace, + latestTranscript: transcript, + topicHistory: topicHistoryStore.entries + ) else { + return + } + + lastSkillWriteTrigger = trigger.reason.rawValue + ClickyAnalytics.trackTeachingSkillWriteTriggered(reason: trigger.reason.rawValue, topic: trigger.topic) + + let traceSnapshot = sessionTrace + let targetBundleId = SkillTargetAppResolver.resolveTargetBundleId( + from: traceSnapshot, + frontmostBundleId: frontmostApplicationBundleId() + ) + let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: traceSnapshot) ?? trigger.topic + skillWriteTask?.cancel() + skillWriteTask = Task { + do { + let existingSkill = SkillMatcher.findSkillForUpdate( + in: teachingSkillStore.skills, + targetBundleId: targetBundleId, + primaryQuestion: primaryQuestion + ) + + let synthesized = try await SkillSynthesizer.synthesizeSkillContent( + sessionTrace: traceSnapshot, + trigger: trigger, + existingSkill: existingSkill, + targetBundleId: targetBundleId, + claudeAPI: claudeAPI + ) + + guard !Task.isCancelled else { return } + + let metadata = SkillSynthesizer.buildSkillMetadata( + sessionTrace: traceSnapshot, + trigger: trigger, + targetBundleId: targetBundleId + ) + + let skill = SkillSynthesizer.buildSkill( + id: existingSkill?.id ?? metadata.id, + name: synthesized.name, + description: synthesized.description, + body: synthesized.body, + targetBundleId: targetBundleId, + taskSlug: metadata.taskSlug, + primaryQuestion: primaryQuestion, + existingSkill: existingSkill + ) + + _ = try teachingSkillStore.saveSkill(skill) + SkillCurator.curate(store: teachingSkillStore) + teachingSkills = teachingSkillStore.skills + runCuratorLLMPassesIfNeeded() + topicHistoryStore.recordTopic( + topic: trigger.topic, + bundleId: targetBundleId, + skillId: skill.id + ) + // Only wipe the live trace if it still holds the same turns this + // task synthesized. Skill synthesis is async (a multi-second Claude + // call); by the time it finishes, finalizeAndPersistSession may have + // already cleared the trace and a NEW session may have started. An + // unconditional removeAll() here would wipe that new session's turns + // before they can be persisted. + if sessionTrace == traceSnapshot { + sessionTrace.removeAll() + } + + ClickyAnalytics.trackTeachingSkillSaved( + skillID: skill.id, + reason: trigger.reason.rawValue, + updatedExisting: existingSkill != nil + ) + writeE2EArtifactsIfNeeded() + print("📚 Saved teaching skill: \(skill.id)") + } catch { + print("⚠️ Failed to synthesize teaching skill: \(error)") + } + } + } func setSelectedModel(_ model: String) { selectedModel = model - UserDefaults.standard.set(model, forKey: "selectedClaudeModel") + ClickyDefaults.shared.set(model, forKey: "selectedClaudeModel") claudeAPI.model = model } /// User preference for whether the Clicky cursor should be shown. /// When toggled off, the overlay is hidden and push-to-talk is disabled. /// Persisted to UserDefaults so the choice survives app restarts. - @Published var isClickyCursorEnabled: Bool = UserDefaults.standard.object(forKey: "isClickyCursorEnabled") == nil + @Published var isClickyCursorEnabled: Bool = ClickyDefaults.shared.object(forKey: "isClickyCursorEnabled") == nil ? true - : UserDefaults.standard.bool(forKey: "isClickyCursorEnabled") + : ClickyDefaults.shared.bool(forKey: "isClickyCursorEnabled") func setClickyCursorEnabled(_ enabled: Bool) { isClickyCursorEnabled = enabled - UserDefaults.standard.set(enabled, forKey: "isClickyCursorEnabled") + ClickyDefaults.shared.set(enabled, forKey: "isClickyCursorEnabled") transientHideTask?.cancel() transientHideTask = nil @@ -142,12 +534,12 @@ final class CompanionManager: ObservableObject { /// Whether the user has completed onboarding at least once. Persisted /// to UserDefaults so the Start button only appears on first launch. var hasCompletedOnboarding: Bool { - get { UserDefaults.standard.bool(forKey: "hasCompletedOnboarding") } - set { UserDefaults.standard.set(newValue, forKey: "hasCompletedOnboarding") } + get { ClickyDefaults.shared.bool(forKey: "hasCompletedOnboarding") } + set { ClickyDefaults.shared.set(newValue, forKey: "hasCompletedOnboarding") } } /// Whether the user has submitted their email during onboarding. - @Published var hasSubmittedEmail: Bool = UserDefaults.standard.bool(forKey: "hasSubmittedEmail") + @Published var hasSubmittedEmail: Bool = ClickyDefaults.shared.bool(forKey: "hasSubmittedEmail") /// Submits the user's email to FormSpark and identifies them in PostHog. func submitEmail(_ email: String) { @@ -155,7 +547,7 @@ final class CompanionManager: ObservableObject { guard !trimmedEmail.isEmpty else { return } hasSubmittedEmail = true - UserDefaults.standard.set(true, forKey: "hasSubmittedEmail") + ClickyDefaults.shared.set(true, forKey: "hasSubmittedEmail") // Identify user in PostHog PostHogSDK.shared.identify(trimmedEmail, userProperties: [ @@ -173,9 +565,12 @@ final class CompanionManager: ObservableObject { } func start() { + bootstrapTeachingSkills() + bindPanelClosedObservation() refreshAllPermissions() - print("🔑 Clicky start — accessibility: \(hasAccessibilityPermission), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission), onboarded: \(hasCompletedOnboarding)") + print("🔑 Clicky start — accessibility: \(hasAccessibilityPermission), inputMonitoring: \(hasInputMonitoringPermission), pushToTalkActive: \(isPushToTalkHotkeyActive), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission), onboarded: \(hasCompletedOnboarding)") startPermissionPolling() + startWorkspaceActivationObservation() bindVoiceStateObservation() bindAudioPowerLevel() bindShortcutTransitions() @@ -295,106 +690,174 @@ final class CompanionManager: ObservableObject { currentResponseTask?.cancel() currentResponseTask = nil + skillWriteTask?.cancel() + skillWriteTask = nil + sessionIdleTimer?.invalidate() + sessionIdleTimer = nil + if let panelClosedObserver { + NotificationCenter.default.removeObserver(panelClosedObserver) + self.panelClosedObserver = nil + } shortcutTransitionCancellable?.cancel() voiceStateCancellable?.cancel() audioPowerCancellable?.cancel() accessibilityCheckTimer?.invalidate() accessibilityCheckTimer = nil + permissionRefreshBurstTask?.cancel() + permissionRefreshBurstTask = nil + if let workspaceActivationObserver { + NSWorkspace.shared.notificationCenter.removeObserver(workspaceActivationObserver) + self.workspaceActivationObserver = nil + } + } + + func requestAccessibilityPermissionFromPanel() { + _ = WindowPositionManager.requestAccessibilityPermission() + schedulePermissionRefreshBurst() + } + + func requestInputMonitoringPermissionFromPanel() { + _ = WindowPositionManager.requestInputMonitoringPermission() + schedulePermissionRefreshBurst() + } + + func requestScreenRecordingPermissionFromPanel() { + _ = WindowPositionManager.requestScreenRecordingPermission() + schedulePermissionRefreshBurst() + } + + func schedulePermissionRefreshBurstAfterReturningFromSettings() { + schedulePermissionRefreshBurst() } func refreshAllPermissions() { + clearStalePermissionUserDefaultsIfLiveTCCDenied() + let previouslyHadAccessibility = hasAccessibilityPermission + let previouslyHadInputMonitoring = hasInputMonitoringPermission let previouslyHadScreenRecording = hasScreenRecordingPermission let previouslyHadMicrophone = hasMicrophonePermission let previouslyHadAll = allPermissionsGranted + WindowPositionManager.refreshAccessibilityTrustCache() let currentlyHasAccessibility = WindowPositionManager.hasAccessibilityPermission() hasAccessibilityPermission = currentlyHasAccessibility - if currentlyHasAccessibility { + let currentlyHasInputMonitoring = WindowPositionManager.hasInputMonitoringPermission() + hasInputMonitoringPermission = currentlyHasInputMonitoring + + if currentlyHasInputMonitoring { globalPushToTalkShortcutMonitor.start() } else { globalPushToTalkShortcutMonitor.stop() } - hasScreenRecordingPermission = WindowPositionManager.hasScreenRecordingPermission() + hasScreenRecordingPermission = WindowPositionManager + .shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch() let micAuthStatus = AVCaptureDevice.authorizationStatus(for: .audio) hasMicrophonePermission = micAuthStatus == .authorized // Debug: log permission state on changes if previouslyHadAccessibility != hasAccessibilityPermission + || previouslyHadInputMonitoring != hasInputMonitoringPermission || previouslyHadScreenRecording != hasScreenRecordingPermission || previouslyHadMicrophone != hasMicrophonePermission { - print("🔑 Permissions — accessibility: \(hasAccessibilityPermission), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission)") + print("🔑 Permissions — accessibility: \(hasAccessibilityPermission), inputMonitoring: \(hasInputMonitoringPermission), pushToTalkActive: \(isPushToTalkHotkeyActive), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission)") } // Track individual permission grants as they happen if !previouslyHadAccessibility && hasAccessibilityPermission { ClickyAnalytics.trackPermissionGranted(permission: "accessibility") } + if !previouslyHadInputMonitoring && hasInputMonitoringPermission { + ClickyAnalytics.trackPermissionGranted(permission: "input_monitoring") + } if !previouslyHadScreenRecording && hasScreenRecordingPermission { ClickyAnalytics.trackPermissionGranted(permission: "screen_recording") } if !previouslyHadMicrophone && hasMicrophonePermission { ClickyAnalytics.trackPermissionGranted(permission: "microphone") } - // Screen content permission is persisted — once the user has approved the - // SCShareableContent picker, we don't need to re-check it. - if !hasScreenContentPermission { - hasScreenContentPermission = UserDefaults.standard.bool(forKey: "hasScreenContentPermission") - } + refreshScreenContentPermissionFromCaptureTest() if !previouslyHadAll && allPermissionsGranted { ClickyAnalytics.trackAllPermissionsGranted() } + + if !allPermissionsGranted && !hasLoggedPermissionDiagnostics { + hasLoggedPermissionDiagnostics = true + WindowPositionManager.logPermissionDiagnosticsSnapshot() + } } - /// Triggers the macOS screen content picker by performing a dummy - /// screenshot capture. Once the user approves, we persist the grant - /// so they're never asked again during onboarding. + /// Screen content uses the same macOS screen-recording permission as capture. + /// This avoids ScreenCaptureKit, which on Tahoe triggers screencaptureui popups. @Published private(set) var isRequestingScreenContent = false func requestScreenContentPermission() { guard !isRequestingScreenContent else { return } isRequestingScreenContent = true + + if !hasScreenRecordingPermission { + _ = WindowPositionManager.requestScreenRecordingPermission() + } + Task { - do { - let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) - guard let display = content.displays.first else { - await MainActor.run { isRequestingScreenContent = false } - return - } - let filter = SCContentFilter(display: display, excludingWindows: []) - let config = SCStreamConfiguration() - config.width = 320 - config.height = 240 - let image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config) - // Verify the capture actually returned real content — a 0x0 or - // fully-empty image means the user denied the prompt. - let didCapture = image.width > 0 && image.height > 0 - print("🔑 Screen content capture result — width: \(image.width), height: \(image.height), didCapture: \(didCapture)") - await MainActor.run { - isRequestingScreenContent = false - guard didCapture else { return } - hasScreenContentPermission = true - UserDefaults.standard.set(true, forKey: "hasScreenContentPermission") + try? await Task.sleep(nanoseconds: 500_000_000) + await MainActor.run { + self.isRequestingScreenContent = false + self.refreshScreenContentPermissionFromCaptureTest() + + if self.hasScreenContentPermission { ClickyAnalytics.trackPermissionGranted(permission: "screen_content") + } - // If onboarding was already completed, show the cursor overlay now - if hasCompletedOnboarding && allPermissionsGranted && !isOverlayVisible && isClickyCursorEnabled { - overlayWindowManager.hasShownOverlayBefore = true - overlayWindowManager.showOverlay(onScreens: NSScreen.screens, companionManager: self) - isOverlayVisible = true - } + if self.hasCompletedOnboarding + && self.allPermissionsGranted + && !self.isOverlayVisible + && self.isClickyCursorEnabled { + self.overlayWindowManager.hasShownOverlayBefore = true + self.overlayWindowManager.showOverlay(onScreens: NSScreen.screens, companionManager: self) + self.isOverlayVisible = true } - } catch { - print("⚠️ Screen content permission request failed: \(error)") - await MainActor.run { isRequestingScreenContent = false } } } } + private func refreshScreenContentPermissionFromCaptureTest() { + guard WindowPositionManager.isScreenCapturePreflightGranted() else { + hasScreenContentPermission = false + return + } + + if ClickyDefaults.shared.bool(forKey: "hasScreenContentPermission") { + hasScreenContentPermission = true + return + } + + guard hasScreenRecordingPermission else { + hasScreenContentPermission = false + return + } + guard CompanionScreenCaptureUtility.verifyScreenCaptureAccess() else { + hasScreenContentPermission = false + return + } + + hasScreenContentPermission = true + ClickyDefaults.shared.set(true, forKey: "hasScreenContentPermission") + } + + /// Old builds can leave screen-permission prefs set while this binary has no TCC access. + private func clearStalePermissionUserDefaultsIfLiveTCCDenied() { + guard !WindowPositionManager.isScreenCapturePreflightGranted() else { return } + + WindowPositionManager.clearPreviouslyConfirmedScreenRecordingPermission() + WindowPositionManager.clearCachedScreenContentPermission() + hasScreenContentPermission = false + } + // MARK: - Private /// Triggers the system microphone prompt if the user has never been asked. @@ -412,11 +875,42 @@ final class CompanionManager: ObservableObject { /// user grants them in System Settings. Screen Recording is the exception — /// macOS requires an app restart for that one to take effect. private func startPermissionPolling() { - accessibilityCheckTimer = Timer.scheduledTimer(withTimeInterval: 1.5, repeats: true) { [weak self] _ in + let permissionPollingTimer = Timer(timeInterval: 1.5, repeats: true) { [weak self] _ in Task { @MainActor [weak self] in self?.refreshAllPermissions() } } + // .common keeps polling alive while the user is in System Settings. + RunLoop.main.add(permissionPollingTimer, forMode: .common) + accessibilityCheckTimer = permissionPollingTimer + } + + private func startWorkspaceActivationObservation() { + workspaceActivationObserver = NSWorkspace.shared.notificationCenter.addObserver( + forName: NSWorkspace.didActivateApplicationNotification, + object: nil, + queue: .main + ) { [weak self] notification in + guard let self else { return } + guard let activatedApplication = notification.userInfo?[NSWorkspace.applicationUserInfoKey] as? NSRunningApplication, + activatedApplication.bundleIdentifier == Bundle.main.bundleIdentifier else { + return + } + self.refreshAllPermissions() + } + } + + /// Polls several times after a Grant tap or return from System Settings. + private func schedulePermissionRefreshBurst() { + permissionRefreshBurstTask?.cancel() + permissionRefreshBurstTask = Task { @MainActor [weak self] in + guard let self else { return } + for _ in 0..<12 { + if Task.isCancelled { return } + refreshAllPermissions() + try? await Task.sleep(nanoseconds: 1_000_000_000) + } + } } private func bindAudioPowerLevel() { @@ -455,12 +949,28 @@ final class CompanionManager: ObservableObject { // the brief idle gap between recording and processing // would prematurely hide the overlay. if self.currentResponseTask == nil { + self.isPushToTalkInteractionActive = false self.scheduleTransientHideIfNeeded() } } } } + private func bindPanelClosedObservation() { + panelClosedObserver = NotificationCenter.default.addObserver( + forName: .clickyCompanionPanelDidClose, + object: nil, + queue: .main + ) { [weak self] _ in + Task { @MainActor in + guard let self else { return } + guard !self.isPushToTalkInteractionActive else { return } + guard !self.sessionTrace.isEmpty else { return } + self.finalizeAndPersistSession() + } + } + } + private func bindShortcutTransitions() { shortcutTransitionCancellable = globalPushToTalkShortcutMonitor .shortcutTransitionPublisher @@ -477,6 +987,8 @@ final class CompanionManager: ObservableObject { // Don't register push-to-talk while the onboarding video is playing guard !showOnboardingVideo else { return } + isPushToTalkInteractionActive = true + // Cancel any pending transient hide so the overlay stays visible transientHideTask?.cancel() transientHideTask = nil @@ -583,11 +1095,15 @@ final class CompanionManager: ObservableObject { /// the spinner/processing state until TTS audio begins playing. /// Claude's response may include a [POINT:x,y:label] tag which triggers /// the buddy to fly to that element on screen. - private func sendTranscriptToClaudeWithScreenshot(transcript: String) { + private func sendTranscriptToClaudeWithScreenshot( + transcript: String, + onComplete: (@MainActor () -> Void)? = nil + ) { currentResponseTask?.cancel() elevenLabsTTSClient.stopPlayback() currentResponseTask = Task { + defer { onComplete?() } // Stay in processing (spinner) state — no streaming text displayed voiceState = .processing @@ -610,9 +1126,33 @@ final class CompanionManager: ObservableObject { (userPlaceholder: entry.userTranscript, assistantResponse: entry.assistantResponse) } + let matchedTeachingSkills = matchedSkills(for: transcript) + lastMatchedSkillNames = matchedTeachingSkills.map(\.name) + let systemPrompt = TeachingPromptBuilder.buildVoiceResponsePrompt( + basePrompt: Self.companionVoiceResponseSystemPrompt, + matchedSkills: matchedTeachingSkills + ) + lastSystemPrompt = systemPrompt + ClickyE2EConfiguration.writeLastSystemPromptForE2E(systemPrompt) + + for skill in matchedTeachingSkills { + _ = try? teachingSkillStore.markUsed(skill) + } + teachingSkills = teachingSkillStore.skills + + if !matchedTeachingSkills.isEmpty { + ClickyAnalytics.trackTeachingSkillsMatched( + skillIDs: matchedTeachingSkills.map(\.id), + bundleID: frontmostApplicationBundleId() + ) + ClickyE2EConfiguration.writeLastMatchedSkillIDForE2E(matchedTeachingSkills.first?.id) + } else { + ClickyE2EConfiguration.writeLastMatchedSkillIDForE2E(nil) + } + let (fullResponseText, _) = try await claudeAPI.analyzeImageStreaming( images: labeledImages, - systemPrompt: Self.companionVoiceResponseSystemPrompt, + systemPrompt: systemPrompt, conversationHistory: historyForAPI, userPrompt: transcript, onTextChunk: { _ in @@ -695,19 +1235,32 @@ final class CompanionManager: ObservableObject { print("🧠 Conversation history: \(conversationHistory.count) exchanges") + recordSessionExchange( + transcript: transcript, + spokenResponse: spokenText, + pointed: hasPointCoordinate + ) + maybeWriteTeachingSkill(after: transcript) + if SkillTriggerEvaluator.isConfirmationTranscript(transcript) { + finalizeAndPersistSession(outcome: .success) + } + ClickyAnalytics.trackAIResponseReceived(response: spokenText) // Play the response via TTS. Keep the spinner (processing state) // until the audio actually starts playing, then switch to responding. if !spokenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { - do { - try await elevenLabsTTSClient.speakText(spokenText) - // speakText returns after player.play() — audio is now playing - voiceState = .responding - } catch { - ClickyAnalytics.trackTTSError(error: error.localizedDescription) - print("⚠️ ElevenLabs TTS error: \(error)") - speakCreditsErrorFallback() + if ClickyE2EConfiguration.isEnabled { + voiceState = .idle + } else { + do { + try await elevenLabsTTSClient.speakText(spokenText) + voiceState = .responding + } catch { + ClickyAnalytics.trackTTSError(error: error.localizedDescription) + print("⚠️ ElevenLabs TTS error: \(error)") + speakCreditsErrorFallback() + } } } } catch is CancellationError { @@ -720,6 +1273,13 @@ final class CompanionManager: ObservableObject { if !Task.isCancelled { voiceState = .idle + isPushToTalkInteractionActive = false + // Arm the idle boundary now that the assistant has finished speaking, + // so the 30s countdown measures genuine user inactivity rather than + // overlapping with TTS playback. Only relevant while a session is open. + if sessionStartedAt != nil { + restartSessionIdleTimer() + } scheduleTransientHideIfNeeded() } } diff --git a/leanring-buddy/CompanionPanelView.swift b/leanring-buddy/CompanionPanelView.swift index 76789b4c6..910e57f75 100644 --- a/leanring-buddy/CompanionPanelView.swift +++ b/leanring-buddy/CompanionPanelView.swift @@ -13,8 +13,21 @@ import SwiftUI struct CompanionPanelView: View { @ObservedObject var companionManager: CompanionManager @State private var emailInput: String = "" + @State private var isShowingTeachingSkillsLibrary = false var body: some View { + if isShowingTeachingSkillsLibrary { + TeachingSkillsLibraryView(companionManager: companionManager) { + isShowingTeachingSkillsLibrary = false + } + .frame(width: 320) + .background(panelBackground) + } else { + mainPanelContent + } + } + + private var mainPanelContent: some View { VStack(alignment: .leading, spacing: 0) { panelHeader Divider() @@ -25,6 +38,14 @@ struct CompanionPanelView: View { .padding(.top, 16) .padding(.horizontal, 16) + if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { + Spacer() + .frame(height: 12) + + teachingSkillsSection + .padding(.horizontal, 16) + } + if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { Spacer() .frame(height: 12) @@ -127,10 +148,31 @@ struct CompanionPanelView: View { @ViewBuilder private var permissionsCopySection: some View { if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { - Text("Hold Control+Option to talk.") - .font(.system(size: 12, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) - .frame(maxWidth: .infinity, alignment: .leading) + VStack(alignment: .leading, spacing: 6) { + Text("Hold ^ Control + ⌥ Option to talk.") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + + Text("Use Control (not ⌘ Command). Speak while holding, then release.") + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textTertiary) + .fixedSize(horizontal: false, vertical: true) + + if !companionManager.isPushToTalkHotkeyActive { + Text("Push-to-talk is inactive. Enable Input Monitoring for Clicky, then quit and reopen the app.") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.warning) + .fixedSize(horizontal: false, vertical: true) + } + + if !AppBundleConfiguration.isWorkerBaseURLConfigured { + Text("API proxy is not configured. Set ClickyWorkerBaseURL in Info.plist to your Cloudflare Worker URL.") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.warning) + .fixedSize(horizontal: false, vertical: true) + } + } + .frame(maxWidth: .infinity, alignment: .leading) } else if companionManager.allPermissionsGranted && !companionManager.hasSubmittedEmail { VStack(alignment: .leading, spacing: 4) { Text("Drop your email to get started.") @@ -153,10 +195,20 @@ struct CompanionPanelView: View { .font(.system(size: 12, weight: .bold)) .foregroundColor(DS.Colors.textSecondary) - Text("Some permissions were revoked. Grant all four below to keep using Clicky.") + Text("Some permissions were revoked. Grant all five below to keep using Clicky.") .font(.system(size: 11)) .foregroundColor(DS.Colors.textTertiary) .fixedSize(horizontal: false, vertical: true) + + Text("Enable THIS build: tap Find App, drag Clicky.app into the list, then quit and reopen Clicky.") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.warning) + .fixedSize(horizontal: false, vertical: true) + + Text(companionManager.runningApplicationBundlePath) + .font(.system(size: 9, design: .monospaced)) + .foregroundColor(DS.Colors.textTertiary) + .fixedSize(horizontal: false, vertical: true) } .frame(maxWidth: .infinity, alignment: .leading) } else { @@ -254,6 +306,8 @@ struct CompanionPanelView: View { accessibilityPermissionRow + inputMonitoringPermissionRow + screenRecordingPermissionRow if companionManager.hasScreenRecordingPermission { @@ -272,9 +326,17 @@ struct CompanionPanelView: View { .foregroundColor(isGranted ? DS.Colors.textTertiary : DS.Colors.warning) .frame(width: 16) - Text("Accessibility") - .font(.system(size: 13, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) + VStack(alignment: .leading, spacing: 1) { + Text("Accessibility") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + + if !isGranted { + Text("Use Find App, enable this build, then quit and reopen") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + } + } } Spacer() @@ -293,7 +355,7 @@ struct CompanionPanelView: View { Button(action: { // Triggers the system accessibility prompt (AXIsProcessTrustedWithOptions) // on first attempt, then opens System Settings on subsequent attempts. - WindowPositionManager.requestAccessibilityPermission() + companionManager.requestAccessibilityPermissionFromPanel() }) { Text("Grant") .font(.system(size: 11, weight: .semibold)) @@ -312,8 +374,79 @@ struct CompanionPanelView: View { // Reveals the app in Finder so the user can drag it into // the Accessibility list if it doesn't appear automatically // (common with unsigned dev builds). - WindowPositionManager.revealAppInFinder() - WindowPositionManager.openAccessibilitySettings() + WindowPositionManager.prepareAccessibilityReGrantFromFinder() + }) { + Text("Find App") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(DS.Colors.textSecondary) + .padding(.horizontal, 10) + .padding(.vertical, 4) + .background( + Capsule() + .stroke(DS.Colors.borderSubtle, lineWidth: 0.8) + ) + } + .buttonStyle(.plain) + .pointerCursor() + } + } + } + .padding(.vertical, 6) + } + + private var inputMonitoringPermissionRow: some View { + let isGranted = companionManager.hasInputMonitoringPermission + return HStack { + HStack(spacing: 8) { + Image(systemName: "keyboard") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(isGranted ? DS.Colors.textTertiary : DS.Colors.warning) + .frame(width: 16) + + VStack(alignment: .leading, spacing: 1) { + Text("Input Monitoring") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + + Text(isGranted + ? "Required for Control+Option push-to-talk" + : "Use Find App, enable this build, then quit and reopen") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + } + } + + Spacer() + + if isGranted { + HStack(spacing: 4) { + Circle() + .fill(DS.Colors.success) + .frame(width: 6, height: 6) + Text("Granted") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.success) + } + } else { + HStack(spacing: 6) { + Button(action: { + companionManager.requestInputMonitoringPermissionFromPanel() + }) { + Text("Grant") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(DS.Colors.textOnAccent) + .padding(.horizontal, 10) + .padding(.vertical, 4) + .background( + Capsule() + .fill(DS.Colors.accent) + ) + } + .buttonStyle(.plain) + .pointerCursor() + + Button(action: { + WindowPositionManager.prepareInputMonitoringReGrantFromFinder() }) { Text("Find App") .font(.system(size: 11, weight: .semibold)) @@ -371,7 +504,7 @@ struct CompanionPanelView: View { // Triggers the native macOS screen recording prompt on first // attempt (auto-adds app to the list), then opens System Settings // on subsequent attempts. - WindowPositionManager.requestScreenRecordingPermission() + companionManager.requestScreenRecordingPermissionFromPanel() }) { Text("Grant") .font(.system(size: 11, weight: .semibold)) @@ -596,6 +729,97 @@ struct CompanionPanelView: View { .padding(.vertical, 4) } + // MARK: - Teaching Skills + + private var teachingSkillsSection: some View { + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Teaching Skills") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + + Spacer() + + Toggle("", isOn: Binding( + get: { companionManager.isLearningFromSessionsEnabled }, + set: { companionManager.setLearningFromSessionsEnabled($0) } + )) + .toggleStyle(.switch) + .labelsHidden() + .tint(DS.Colors.accent) + .scaleEffect(0.8) + .accessibilityIdentifier("clicky.panel.teaching-skills.learn-toggle") + } + + Text(companionManager.isLearningFromSessionsEnabled + ? "Clicky learns from successful tutoring sessions." + : "Learning paused — saved skills still apply.") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + + if companionManager.teachingSkills.isEmpty { + Text("No skills yet. Teach Clicky something on screen and confirm it worked.") + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textTertiary) + .fixedSize(horizontal: false, vertical: true) + } else { + ForEach(companionManager.teachingSkills.prefix(4)) { skill in + teachingSkillRow(skill) + } + + Button(action: { + isShowingTeachingSkillsLibrary = true + }) { + Text("View all") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.accent) + } + .buttonStyle(.plain) + .pointerCursor() + .accessibilityIdentifier("clicky.panel.teaching-skills.view-all") + } + } + .padding(.vertical, 4) + } + + private func teachingSkillRow(_ skill: TeachingSkill) -> some View { + HStack(spacing: 8) { + VStack(alignment: .leading, spacing: 2) { + Text(skill.name) + .font(.system(size: 12, weight: .semibold)) + .foregroundColor(DS.Colors.textSecondary) + .lineLimit(1) + + Text("\(skill.usageCount) uses • \(skill.status.rawValue)") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + } + + Spacer() + + Button(action: { + companionManager.setTeachingSkillPinned(id: skill.id, pinned: !skill.isPinned) + }) { + Image(systemName: skill.isPinned ? "pin.fill" : "pin") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(skill.isPinned ? DS.Colors.accent : DS.Colors.textTertiary) + } + .buttonStyle(.plain) + .pointerCursor() + + Button(action: { + companionManager.deleteTeachingSkill(id: skill.id) + }) { + Image(systemName: "trash") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + } + .buttonStyle(.plain) + .pointerCursor() + } + .padding(.vertical, 4) + } + // MARK: - Model Picker private var modelPickerRow: some View { diff --git a/leanring-buddy/CompanionScreenCaptureUtility.swift b/leanring-buddy/CompanionScreenCaptureUtility.swift index 797841783..dbd5c98c4 100644 --- a/leanring-buddy/CompanionScreenCaptureUtility.swift +++ b/leanring-buddy/CompanionScreenCaptureUtility.swift @@ -8,7 +8,6 @@ // import AppKit -import ScreenCaptureKit struct CompanionScreenCapture { let imageData: Data @@ -23,110 +22,90 @@ struct CompanionScreenCapture { @MainActor enum CompanionScreenCaptureUtility { + private static let maxScreenshotDimensionInPixels = 1280 /// Captures all connected displays as JPEG data, labeling each with /// whether the user's cursor is on that screen. This gives the AI /// full context across multiple monitors. static func captureAllScreensAsJPEG() async throws -> [CompanionScreenCapture] { - let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) - - guard !content.displays.isEmpty else { - throw NSError(domain: "CompanionScreenCapture", code: -1, - userInfo: [NSLocalizedDescriptionKey: "No display available for capture"]) + guard WindowPositionManager.hasScreenRecordingPermission() else { + throw NSError( + domain: "CompanionScreenCapture", + code: -4, + userInfo: [NSLocalizedDescriptionKey: "Screen recording permission is required"] + ) } - let mouseLocation = NSEvent.mouseLocation - - // Exclude all windows belonging to this app so the AI sees - // only the user's content, not our overlays or panels. - let ownBundleIdentifier = Bundle.main.bundleIdentifier - let ownAppWindows = content.windows.filter { window in - window.owningApplication?.bundleIdentifier == ownBundleIdentifier - } + MacOSScreenshotFloatingThumbnailDismisser.dismissIfNeeded() - // Build a lookup from display ID to NSScreen so we can use AppKit-coordinate - // frames instead of CG-coordinate frames. NSEvent.mouseLocation and NSScreen.frame - // both use AppKit coordinates (bottom-left origin), while SCDisplay.frame uses - // Core Graphics coordinates (top-left origin). On multi-display setups, the Y - // origins differ for secondary displays, which breaks cursor-contains checks - // and downstream coordinate conversions. - var nsScreenByDisplayID: [CGDirectDisplayID: NSScreen] = [:] - for screen in NSScreen.screens { - if let screenNumber = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID { - nsScreenByDisplayID[screenNumber] = screen - } - } - - // Sort displays so the cursor screen is always first - let sortedDisplays = content.displays.sorted { displayA, displayB in - let frameA = nsScreenByDisplayID[displayA.displayID]?.frame ?? displayA.frame - let frameB = nsScreenByDisplayID[displayB.displayID]?.frame ?? displayB.frame - let aContainsCursor = frameA.contains(mouseLocation) - let bContainsCursor = frameB.contains(mouseLocation) + let mouseLocation = NSEvent.mouseLocation + let sortedScreens = NSScreen.screens.sorted { screenA, screenB in + let aContainsCursor = screenA.frame.contains(mouseLocation) + let bContainsCursor = screenB.frame.contains(mouseLocation) if aContainsCursor != bContainsCursor { return aContainsCursor } return false } + guard !sortedScreens.isEmpty else { + throw NSError( + domain: "CompanionScreenCapture", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "No display available for capture"] + ) + } + var capturedScreens: [CompanionScreenCapture] = [] - for (displayIndex, display) in sortedDisplays.enumerated() { - // Use NSScreen.frame (AppKit coordinates, bottom-left origin) so - // displayFrame is in the same coordinate system as NSEvent.mouseLocation - // and the overlay window's screenFrame in BlueCursorView. - let displayFrame = nsScreenByDisplayID[display.displayID]?.frame - ?? CGRect(x: display.frame.origin.x, y: display.frame.origin.y, - width: CGFloat(display.width), height: CGFloat(display.height)) + for (displayIndex, screen) in sortedScreens.enumerated() { + let displayFrame = screen.frame let isCursorScreen = displayFrame.contains(mouseLocation) + let belowWindowID = CoreGraphicsScreenCaptureUtility.topmostOwnedWindowID(on: screen) - let filter = SCContentFilter(display: display, excludingWindows: ownAppWindows) - - let configuration = SCStreamConfiguration() - let maxDimension = 1280 - let aspectRatio = CGFloat(display.width) / CGFloat(display.height) - if display.width >= display.height { - configuration.width = maxDimension - configuration.height = Int(CGFloat(maxDimension) / aspectRatio) - } else { - configuration.height = maxDimension - configuration.width = Int(CGFloat(maxDimension) * aspectRatio) - } - - let cgImage = try await SCScreenshotManager.captureImage( - contentFilter: filter, - configuration: configuration - ) - - guard let jpegData = NSBitmapImageRep(cgImage: cgImage) - .representation(using: .jpeg, properties: [.compressionFactor: 0.8]) else { + guard let capture = CoreGraphicsScreenCaptureUtility.captureScreenAsJPEG( + screen: screen, + belowWindowID: belowWindowID, + maxDimension: maxScreenshotDimensionInPixels + ) else { continue } let screenLabel: String - if sortedDisplays.count == 1 { + if sortedScreens.count == 1 { screenLabel = "user's screen (cursor is here)" } else if isCursorScreen { - screenLabel = "screen \(displayIndex + 1) of \(sortedDisplays.count) — cursor is on this screen (primary focus)" + screenLabel = "screen \(displayIndex + 1) of \(sortedScreens.count) — cursor is on this screen (primary focus)" } else { - screenLabel = "screen \(displayIndex + 1) of \(sortedDisplays.count) — secondary screen" + screenLabel = "screen \(displayIndex + 1) of \(sortedScreens.count) — secondary screen" } capturedScreens.append(CompanionScreenCapture( - imageData: jpegData, + imageData: capture.imageData, label: screenLabel, isCursorScreen: isCursorScreen, displayWidthInPoints: Int(displayFrame.width), displayHeightInPoints: Int(displayFrame.height), displayFrame: displayFrame, - screenshotWidthInPixels: configuration.width, - screenshotHeightInPixels: configuration.height + screenshotWidthInPixels: capture.screenshotWidthInPixels, + screenshotHeightInPixels: capture.screenshotHeightInPixels )) } + MacOSScreenshotFloatingThumbnailDismisser.dismissIfNeeded() + MacOSScreenshotFloatingThumbnailDismisser.cleanupTemporaryScreenshotArtifacts() + guard !capturedScreens.isEmpty else { - throw NSError(domain: "CompanionScreenCapture", code: -2, - userInfo: [NSLocalizedDescriptionKey: "Failed to capture any screen"]) + throw NSError( + domain: "CompanionScreenCapture", + code: -2, + userInfo: [NSLocalizedDescriptionKey: "Failed to capture any screen"] + ) } return capturedScreens } + + /// Verifies screen capture works without invoking ScreenCaptureKit. + static func verifyScreenCaptureAccess(maxDimension: Int = 64) -> Bool { + CoreGraphicsScreenCaptureUtility.canCaptureAnyScreen(maxDimension: maxDimension) + } } diff --git a/leanring-buddy/CoreGraphicsScreenCaptureUtility.swift b/leanring-buddy/CoreGraphicsScreenCaptureUtility.swift new file mode 100644 index 000000000..17f20b054 --- /dev/null +++ b/leanring-buddy/CoreGraphicsScreenCaptureUtility.swift @@ -0,0 +1,159 @@ +// +// CoreGraphicsScreenCaptureUtility.swift +// leanring-buddy +// +// Captures the screen with CGWindowListCreateImage instead of ScreenCaptureKit. +// On macOS Tahoe, ScreenCaptureKit screenshot/stream APIs route through +// screencaptureui and show the floating screenshot thumbnail popup. +// + +import AppKit +import CoreGraphics +import Darwin + +struct CoreGraphicsScreenJPEGCapture { + let imageData: Data + let screenshotWidthInPixels: Int + let screenshotHeightInPixels: Int +} + +enum CoreGraphicsScreenCaptureUtility { + static func captureScreenAsJPEG( + screen: NSScreen, + belowWindowID: CGWindowID?, + maxDimension: Int + ) -> CoreGraphicsScreenJPEGCapture? { + let quartzCaptureFrame = screen.quartzCaptureFrame + let captureOptions: CGWindowListOption = belowWindowID == nil + ? .optionOnScreenOnly + : .optionOnScreenBelowWindow + let referenceWindowID = belowWindowID ?? kCGNullWindowID + + guard let capturedImage = LegacyCoreGraphicsWindowCapture.createImage( + in: quartzCaptureFrame, + listOption: captureOptions, + windowID: referenceWindowID, + imageOptions: [.bestResolution, .boundsIgnoreFraming] + ) else { + return nil + } + + guard let resizedImage = resizeImage(capturedImage, maxDimension: maxDimension), + let jpegData = NSBitmapImageRep(cgImage: resizedImage) + .representation(using: .jpeg, properties: [.compressionFactor: 0.8]) else { + return nil + } + + return CoreGraphicsScreenJPEGCapture( + imageData: jpegData, + screenshotWidthInPixels: resizedImage.width, + screenshotHeightInPixels: resizedImage.height + ) + } + + static func topmostOwnedWindowID(on screen: NSScreen) -> CGWindowID? { + let visibleOwnedWindows = NSApp.windows.filter { window in + window.isVisible + && window.alphaValue > 0 + && screen.frame.intersects(window.frame) + } + + guard let topmostOwnedWindow = visibleOwnedWindows.max(by: { $0.windowNumber < $1.windowNumber }) else { + return nil + } + + // NSWindow.windowNumber is an Int that can be negative or exceed UInt32 on + // recent macOS (e.g. status bar windows), so convert safely instead of + // force-trapping with the non-failable CGWindowID initializer. + guard topmostOwnedWindow.windowNumber > 0, + let topmostOwnedWindowID = CGWindowID(exactly: topmostOwnedWindow.windowNumber) else { + return nil + } + + return topmostOwnedWindowID + } + + static func canCaptureAnyScreen(maxDimension: Int = 64) -> Bool { + guard let screen = NSScreen.main ?? NSScreen.screens.first else { return false } + return captureScreenAsJPEG( + screen: screen, + belowWindowID: topmostOwnedWindowID(on: screen), + maxDimension: maxDimension + ) != nil + } + + private static func resizeImage(_ image: CGImage, maxDimension: Int) -> CGImage? { + let sourceWidth = image.width + let sourceHeight = image.height + guard sourceWidth > 0, sourceHeight > 0 else { return nil } + + let aspectRatio = CGFloat(sourceWidth) / CGFloat(sourceHeight) + let targetWidth: Int + let targetHeight: Int + + if sourceWidth >= sourceHeight { + targetWidth = maxDimension + targetHeight = max(1, Int(CGFloat(maxDimension) / aspectRatio)) + } else { + targetHeight = maxDimension + targetWidth = max(1, Int(CGFloat(maxDimension) * aspectRatio)) + } + + guard let context = CGContext( + data: nil, + width: targetWidth, + height: targetHeight, + bitsPerComponent: 8, + bytesPerRow: 0, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) else { + return nil + } + + context.interpolationQuality = .high + context.draw(image, in: CGRect(x: 0, y: 0, width: targetWidth, height: targetHeight)) + return context.makeImage() + } +} + +/// Runtime wrapper around CGWindowListCreateImage. The symbol is marked +/// unavailable in the macOS 15 SDK, but it still works and does not route +/// through Tahoe's screencaptureui floating thumbnail. +private enum LegacyCoreGraphicsWindowCapture { + private typealias CGWindowListCreateImageFunction = @convention(c) ( + CGRect, + CGWindowListOption, + CGWindowID, + CGWindowImageOption + ) -> Unmanaged? + + private static let createImageFunction: CGWindowListCreateImageFunction? = { + guard let symbol = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "CGWindowListCreateImage") else { + return nil + } + return unsafeBitCast(symbol, to: CGWindowListCreateImageFunction.self) + }() + + static func createImage( + in screenBounds: CGRect, + listOption: CGWindowListOption, + windowID: CGWindowID, + imageOptions: CGWindowImageOption + ) -> CGImage? { + createImageFunction?(screenBounds, listOption, windowID, imageOptions)?.takeRetainedValue() + } +} + +private extension NSScreen { + var quartzCaptureFrame: CGRect { + let screenFrame = frame + guard let primaryScreen = NSScreen.screens.first else { return screenFrame } + return CGRect( + x: screenFrame.origin.x, + y: primaryScreen.frame.height - screenFrame.origin.y - screenFrame.height, + width: screenFrame.width, + height: screenFrame.height + ) + } +} diff --git a/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift b/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift index 8020269b4..02b9a72eb 100644 --- a/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift +++ b/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift @@ -11,12 +11,21 @@ import AppKit import Combine import CoreGraphics import Foundation +import os final class GlobalPushToTalkShortcutMonitor: ObservableObject { + private static let logger = Logger( + subsystem: Bundle.main.bundleIdentifier ?? "com.yourcompany.leanring-buddy", + category: "PushToTalk" + ) + let shortcutTransitionPublisher = PassthroughSubject() private var globalEventTap: CFMachPort? private var globalEventTapRunLoopSource: CFRunLoopSource? + /// True once a CGEvent tap is installed and enabled for this process. + @Published private(set) var isEventTapInstalled = false + /// Mutated exclusively from the CGEvent tap callback, which runs on /// `CFRunLoopGetMain()` and therefore always executes on the main thread. /// Published so the overlay can hide immediately on key release without @@ -28,11 +37,26 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { } func start() { - // If the event tap is already running, don't restart it. - // Restarting resets isShortcutCurrentlyPressed, which would kill - // the waveform overlay mid-press when the permission poller calls - // refreshAllPermissions → start() every few seconds. - guard globalEventTap == nil else { return } + if ClickyLaunchArguments.isPresent("-CLICKY_DISABLE_GLOBAL_PTT=1") { + Self.logger.notice("Global push-to-talk disabled via launch flag") + return + } + + guard WindowPositionManager.hasInputMonitoringPermission() else { + Self.logger.error("Input Monitoring permission missing — push-to-talk hotkey inactive") + stop() + return + } + + if let globalEventTap, !CGEvent.tapIsEnabled(tap: globalEventTap) { + Self.logger.notice("Recreating disabled CGEvent tap") + stop() + } + + guard globalEventTap == nil else { + isEventTapInstalled = true + return + } let monitoredEventTypes: [CGEventType] = [.flagsChanged, .keyDown, .keyUp] let eventMask = monitoredEventTypes.reduce(CGEventMask(0)) { currentMask, eventType in @@ -62,7 +86,14 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { callback: eventTapCallback, userInfo: Unmanaged.passUnretained(self).toOpaque() ) else { - print("⚠️ Global push-to-talk: couldn't create CGEvent tap") + if !CGPreflightListenEventAccess() { + Self.logger.error("Input Monitoring permission required for push-to-talk") + print("⚠️ Global push-to-talk: Input Monitoring permission is required (System Settings → Privacy & Security → Input Monitoring)") + } else { + Self.logger.error("CGEvent tapCreate returned nil despite Input Monitoring being granted") + print("⚠️ Global push-to-talk: couldn't create CGEvent tap — quit and reopen Clicky") + } + isEventTapInstalled = false return } @@ -72,7 +103,9 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { 0 ) else { CFMachPortInvalidate(globalEventTap) + Self.logger.error("Couldn't create CGEvent tap run loop source") print("⚠️ Global push-to-talk: couldn't create event tap run loop source") + isEventTapInstalled = false return } @@ -81,10 +114,13 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { CFRunLoopAddSource(CFRunLoopGetMain(), globalEventTapRunLoopSource, .commonModes) CGEvent.tapEnable(tap: globalEventTap, enable: true) + isEventTapInstalled = CGEvent.tapIsEnabled(tap: globalEventTap) + Self.logger.notice("CGEvent tap installed (enabled=\(self.isEventTapInstalled, privacy: .public))") } func stop() { isShortcutCurrentlyPressed = false + isEventTapInstalled = false if let globalEventTapRunLoopSource { CFRunLoopRemoveSource(CFRunLoopGetMain(), globalEventTapRunLoopSource, .commonModes) @@ -102,8 +138,10 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { event: CGEvent ) -> Unmanaged? { if eventType == .tapDisabledByTimeout || eventType == .tapDisabledByUserInput { + Self.logger.notice("CGEvent tap disabled by system — re-enabling") if let globalEventTap { CGEvent.tapEnable(tap: globalEventTap, enable: true) + isEventTapInstalled = CGEvent.tapIsEnabled(tap: globalEventTap) } return Unmanaged.passUnretained(event) } @@ -120,9 +158,11 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { case .none: break case .pressed: + Self.logger.notice("Push-to-talk shortcut pressed") isShortcutCurrentlyPressed = true shortcutTransitionPublisher.send(.pressed) case .released: + Self.logger.notice("Push-to-talk shortcut released") isShortcutCurrentlyPressed = false shortcutTransitionPublisher.send(.released) } diff --git a/leanring-buddy/Info.plist b/leanring-buddy/Info.plist index e3d2b455a..578241346 100644 --- a/leanring-buddy/Info.plist +++ b/leanring-buddy/Info.plist @@ -2,6 +2,8 @@ + ClickyWorkerBaseURL + https://clicky-proxy.c-z-tavi.workers.dev LSUIElement SUFeedURL @@ -10,6 +12,8 @@ /l3d2rw5ZZFRU3AadP/w2Zf8FHfhA6bKv16BQOV5OSk= VoiceTranscriptionProvider assemblyai + NSAccessibilityUsageDescription + Clicky needs Accessibility access to help position windows and point at on-screen elements. NSMicrophoneUsageDescription Clicky uses your microphone so you can talk to it NSScreenCaptureUsageDescription diff --git a/leanring-buddy/MacOSScreenshotFloatingThumbnailDismisser.swift b/leanring-buddy/MacOSScreenshotFloatingThumbnailDismisser.swift new file mode 100644 index 000000000..99647fe02 --- /dev/null +++ b/leanring-buddy/MacOSScreenshotFloatingThumbnailDismisser.swift @@ -0,0 +1,58 @@ +// +// MacOSScreenshotFloatingThumbnailDismisser.swift +// leanring-buddy +// +// macOS Tahoe keeps screencaptureui alive and shows floating screenshot +// thumbnails when ScreenCaptureKit takes captures. Clicky now captures via +// Core Graphics, but we still tear down leftover thumbnail UI if it appears. +// + +import Foundation + +enum MacOSScreenshotFloatingThumbnailDismisser { + private static let screencaptureUIProcessNames = [ + "screencaptureui", + ] + + static func dismissIfNeeded() { + for processName in screencaptureUIProcessNames { + terminateProcess(named: processName) + } + } + + static func cleanupTemporaryScreenshotArtifacts() { + let temporaryItemsDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent("TemporaryItems", isDirectory: true) + + guard let temporaryItemURLs = try? FileManager.default.contentsOfDirectory( + at: temporaryItemsDirectory, + includingPropertiesForKeys: nil + ) else { + return + } + + for temporaryItemURL in temporaryItemURLs where temporaryItemURL.lastPathComponent + .localizedCaseInsensitiveContains("screencaptureui") { + try? FileManager.default.removeItem(at: temporaryItemURL) + } + + scheduleFollowUpDismissal() + } + + private static func scheduleFollowUpDismissal() { + Task { + for _ in 0..<3 { + try? await Task.sleep(nanoseconds: 150_000_000) + dismissIfNeeded() + } + } + } + + private static func terminateProcess(named processName: String) { + let terminateProcess = Process() + terminateProcess.executableURL = URL(fileURLWithPath: "/usr/bin/killall") + terminateProcess.arguments = ["-9", processName] + try? terminateProcess.run() + terminateProcess.waitUntilExit() + } +} diff --git a/leanring-buddy/MacOSScreenshotFloatingThumbnailSuppression.swift b/leanring-buddy/MacOSScreenshotFloatingThumbnailSuppression.swift new file mode 100644 index 000000000..6d3079844 --- /dev/null +++ b/leanring-buddy/MacOSScreenshotFloatingThumbnailSuppression.swift @@ -0,0 +1,117 @@ +// +// MacOSScreenshotFloatingThumbnailSuppression.swift +// leanring-buddy +// +// Keeps macOS from showing screenshot floating thumbnails while Clicky is +// running. Restores the user's prior preference when the app quits. +// + +import CoreFoundation +import Foundation + +enum MacOSScreenshotFloatingThumbnailSuppression { + private static let screencapturePreferenceDomain = "com.apple.screencapture" as CFString + private static let screencaptureUIPreferenceDomain = "com.apple.screencaptureui" as CFString + private static let showThumbnailPreferenceKey = "show-thumbnail" as CFString + private static let thumbnailExpirationPreferenceKey = "thumbnailExpiration" as CFString + private static var savedShowThumbnailPreference: CFPropertyList? + private static var savedThumbnailExpirationPreference: CFPropertyList? + private static var isSuppressingForAppLifetime = false + + static func enableForAppLifetime() { + guard !isSuppressingForAppLifetime else { return } + + savedShowThumbnailPreference = CFPreferencesCopyAppValue( + showThumbnailPreferenceKey, + screencapturePreferenceDomain + ) + savedThumbnailExpirationPreference = CFPreferencesCopyAppValue( + thumbnailExpirationPreferenceKey, + screencaptureUIPreferenceDomain + ) + + CFPreferencesSetValue( + showThumbnailPreferenceKey, + kCFBooleanFalse, + screencapturePreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + CFPreferencesSetValue( + thumbnailExpirationPreferenceKey, + 0.001 as CFPropertyList, + screencaptureUIPreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + + CFPreferencesSynchronize( + screencapturePreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + CFPreferencesSynchronize( + screencaptureUIPreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + + isSuppressingForAppLifetime = true + MacOSScreenshotFloatingThumbnailDismisser.dismissIfNeeded() + } + + static func restoreAfterAppTermination() { + guard isSuppressingForAppLifetime else { return } + + if let savedShowThumbnailPreference { + CFPreferencesSetValue( + showThumbnailPreferenceKey, + savedShowThumbnailPreference, + screencapturePreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + } else { + CFPreferencesSetValue( + showThumbnailPreferenceKey, + nil, + screencapturePreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + } + + if let savedThumbnailExpirationPreference { + CFPreferencesSetValue( + thumbnailExpirationPreferenceKey, + savedThumbnailExpirationPreference, + screencaptureUIPreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + } else { + CFPreferencesSetValue( + thumbnailExpirationPreferenceKey, + nil, + screencaptureUIPreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + } + + CFPreferencesSynchronize( + screencapturePreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + CFPreferencesSynchronize( + screencaptureUIPreferenceDomain, + kCFPreferencesAnyUser, + kCFPreferencesCurrentHost + ) + + savedShowThumbnailPreference = nil + savedThumbnailExpirationPreference = nil + isSuppressingForAppLifetime = false + } +} diff --git a/leanring-buddy/MenuBarPanelManager.swift b/leanring-buddy/MenuBarPanelManager.swift index e5eb98de7..67616c551 100644 --- a/leanring-buddy/MenuBarPanelManager.swift +++ b/leanring-buddy/MenuBarPanelManager.swift @@ -16,6 +16,7 @@ import SwiftUI extension Notification.Name { static let clickyDismissPanel = Notification.Name("clickyDismissPanel") + static let clickyCompanionPanelDidClose = Notification.Name("clickyCompanionPanelDidClose") } /// Custom NSPanel subclass that can become the key window even with @@ -131,6 +132,8 @@ final class MenuBarPanelManager: NSObject { createPanel() } + companionManager.refreshAllPermissions() + positionPanelBelowStatusItem() panel?.makeKeyAndOrderFront(nil) @@ -141,6 +144,7 @@ final class MenuBarPanelManager: NSObject { private func hidePanel() { panel?.orderOut(nil) removeClickOutsideMonitor() + NotificationCenter.default.post(name: .clickyCompanionPanelDidClose, object: nil) } private func createPanel() { diff --git a/leanring-buddy/PersistedSession.swift b/leanring-buddy/PersistedSession.swift new file mode 100644 index 000000000..7b56adfb0 --- /dev/null +++ b/leanring-buddy/PersistedSession.swift @@ -0,0 +1,24 @@ +// +// PersistedSession.swift +// leanring-buddy +// +// On-disk session JSON written by SessionStore under ~/.clicky/sessions/. +// + +import Foundation + +struct PersistedSession: Codable, Equatable { + let sessionId: UUID + let startedAt: Date + let endedAt: Date + let outcome: SessionOutcome + let privacyOptOut: Bool + let appsUsed: [String] + let turns: [SessionTraceEntry] +} + +enum SessionOutcome: String, Codable { + case success + case abandoned + case unknown +} diff --git a/leanring-buddy/SessionStore.swift b/leanring-buddy/SessionStore.swift new file mode 100644 index 000000000..84f65fe8d --- /dev/null +++ b/leanring-buddy/SessionStore.swift @@ -0,0 +1,107 @@ +// +// SessionStore.swift +// leanring-buddy +// +// Filesystem store for persisted voice sessions under ClickyPaths.sessions. +// + +import Foundation + +final class SessionStore { + static var sessionsRootURL: URL { + ClickyPaths.sessions + } + + private let encoder: JSONEncoder = { + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + return encoder + }() + + private let decoder: JSONDecoder = { + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + return decoder + }() + + private let dayFolderFormatter: DateFormatter = { + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.dateFormat = "yyyy-MM-dd" + return formatter + }() + + @discardableResult + func save(_ session: PersistedSession) throws -> URL { + let fileManager = FileManager.default + let dayFolderName = dayFolderFormatter.string(from: session.startedAt) + let dayFolderURL = Self.sessionsRootURL + .appendingPathComponent(dayFolderName, isDirectory: true) + try fileManager.createDirectory(at: dayFolderURL, withIntermediateDirectories: true) + + let sessionFileURL = dayFolderURL + .appendingPathComponent("\(session.sessionId.uuidString).json") + let encodedData = try encoder.encode(session) + try encodedData.write(to: sessionFileURL, options: .atomic) + return sessionFileURL + } + + func loadAllSessions() -> [PersistedSession] { + let fileManager = FileManager.default + guard let dayFolders = try? fileManager.contentsOfDirectory( + at: Self.sessionsRootURL, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + ) else { + return [] + } + + var loadedSessions: [PersistedSession] = [] + for dayFolder in dayFolders where (try? dayFolder.resourceValues(forKeys: [.isDirectoryKey]).isDirectory) == true { + guard let sessionFiles = try? fileManager.contentsOfDirectory( + at: dayFolder, + includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles] + ) else { + continue + } + + for sessionFile in sessionFiles where sessionFile.pathExtension == "json" { + guard let fileData = try? Data(contentsOf: sessionFile), + let session = try? decoder.decode(PersistedSession.self, from: fileData) else { + continue + } + loadedSessions.append(session) + } + } + + return loadedSessions.sorted { $0.startedAt < $1.startedAt } + } + + func deleteSessionsOlderThan(days: Int, now: Date = Date()) { + let fileManager = FileManager.default + let cutoffDate = Calendar.current.date(byAdding: .day, value: -days, to: now) ?? now + + guard let dayFolders = try? fileManager.contentsOfDirectory( + at: Self.sessionsRootURL, + includingPropertiesForKeys: [.contentModificationDateKey, .isDirectoryKey], + options: [.skipsHiddenFiles] + ) else { + return + } + + for dayFolder in dayFolders { + let resourceValues = try? dayFolder.resourceValues(forKeys: [.isDirectoryKey, .contentModificationDateKey]) + guard resourceValues?.isDirectory == true else { continue } + + let folderDate = dayFolderFormatter.date(from: dayFolder.lastPathComponent) + ?? resourceValues?.contentModificationDate + ?? .distantPast + + guard folderDate < cutoffDate else { continue } + + try? fileManager.removeItem(at: dayFolder) + } + } +} diff --git a/leanring-buddy/SkillCurator.swift b/leanring-buddy/SkillCurator.swift new file mode 100644 index 000000000..11a2f1161 --- /dev/null +++ b/leanring-buddy/SkillCurator.swift @@ -0,0 +1,176 @@ +// +// SkillCurator.swift +// leanring-buddy +// +// Maintains the teaching skill library over time. +// + +import Foundation + +enum SkillCurator { + private static let staleAfterDays = 30 + private static let archiveAfterDays = 90 + private static let maxMergePassesPerLaunch = 1 + private static let maxPatchPassesPerLaunch = 1 + + private static var mergePassesUsedThisLaunch = 0 + private static var patchPassesUsedThisLaunch = 0 + + static func curate(store: TeachingSkillStore, now: Date = Date()) { + curateTimeBasedLifecycle(store: store, now: now) + } + + static func curateTimeBasedLifecycle(store: TeachingSkillStore, now: Date = Date()) { + let calendar = Calendar.current + + for skill in store.skills { + guard !skill.isPinned else { continue } + guard let lastUsed = skill.lastUsed else { continue } + + let daysSinceUse = calendar.dateComponents([.day], from: lastUsed, to: now).day ?? 0 + var updated = skill + var skillChanged = false + + if daysSinceUse >= archiveAfterDays, updated.status != .archived { + updated.status = .archived + skillChanged = true + } else if daysSinceUse >= staleAfterDays, updated.status == .active { + updated.status = .stale + skillChanged = true + } + + if skillChanged { + try? store.saveSkill(updated) + } + } + } + + static func curateWithLLMPasses(store: TeachingSkillStore, claudeAPI: ClaudeAPI) async { + await mergeOneDuplicatePairIfNeeded(store: store, claudeAPI: claudeAPI) + await patchOneStaleSkillIfNeeded(store: store, claudeAPI: claudeAPI) + } + + private static func mergeOneDuplicatePairIfNeeded( + store: TeachingSkillStore, + claudeAPI: ClaudeAPI + ) async { + guard mergePassesUsedThisLaunch < maxMergePassesPerLaunch else { return } + + let duplicatePairs = SkillMatcher.findDuplicateSkillPairs(in: store.skills) + guard let duplicatePair = duplicatePairs.first else { return } + + do { + let mergedSkill = try await mergeDuplicateSkills( + primarySkill: duplicatePair.primarySkill, + duplicateSkill: duplicatePair.duplicateSkill, + claudeAPI: claudeAPI + ) + _ = try store.saveSkill(mergedSkill) + try store.deleteSkill(id: duplicatePair.duplicateSkill.id) + mergePassesUsedThisLaunch += 1 + ClickyAnalytics.trackTeachingSkillMerged( + primarySkillID: duplicatePair.primarySkill.id, + duplicateSkillID: duplicatePair.duplicateSkill.id + ) + print("📚 Curator merged duplicate skills: \(duplicatePair.duplicateSkill.id) → \(duplicatePair.primarySkill.id)") + } catch { + print("⚠️ Curator failed to merge duplicate skills: \(error)") + } + } + + private static func patchOneStaleSkillIfNeeded( + store: TeachingSkillStore, + claudeAPI: ClaudeAPI + ) async { + guard patchPassesUsedThisLaunch < maxPatchPassesPerLaunch else { return } + + guard let staleSkill = store.skills.first(where: { $0.status == .stale && !$0.isPinned }) else { + return + } + + do { + let patchedSkill = try await patchStaleSkill(staleSkill, claudeAPI: claudeAPI) + _ = try store.saveSkill(patchedSkill) + patchPassesUsedThisLaunch += 1 + ClickyAnalytics.trackTeachingSkillPatched(skillID: staleSkill.id) + print("📚 Curator patched stale skill: \(staleSkill.id)") + } catch { + print("⚠️ Curator failed to patch stale skill \(staleSkill.id): \(error)") + } + } + + private static let mergeSystemPrompt = """ + you merge duplicate teaching skills for clicky, a screen-native voice tutor. + + given two similar skills, produce a single merged markdown body only. do not include yaml frontmatter. + + preserve the best steps, ui labels, pointing tips, and completion signals from both skills. + remove redundancy. keep it concise and practical. all lowercase. + """ + + private static let patchSystemPrompt = """ + you refresh stale teaching skills for clicky when macos app ui may have changed. + + given an existing teaching skill, produce an updated markdown body only. do not include yaml frontmatter. + + keep the same workflow intent but update ui labels, menu paths, and pointing heuristics for current macos ui. + note any labels that may have changed. all lowercase. + """ + + private static func mergeDuplicateSkills( + primarySkill: TeachingSkill, + duplicateSkill: TeachingSkill, + claudeAPI: ClaudeAPI + ) async throws -> TeachingSkill { + let userPrompt = """ + merge these duplicate teaching skills into one improved skill body. + + primary skill (\(primarySkill.name)): + \(primarySkill.body) + + duplicate skill (\(duplicateSkill.name)): + \(duplicateSkill.body) + """ + + let response = try await claudeAPI.sendTextMessage( + systemPrompt: mergeSystemPrompt, + userPrompt: userPrompt, + maxTokens: 1200 + ) + + var mergedSkill = primarySkill + mergedSkill.body = response.text.trimmingCharacters(in: .whitespacesAndNewlines) + mergedSkill.usageCount = primarySkill.usageCount + duplicateSkill.usageCount + mergedSkill.lastUsed = Date() + mergedSkill.status = .active + return mergedSkill + } + + private static func patchStaleSkill( + _ staleSkill: TeachingSkill, + claudeAPI: ClaudeAPI + ) async throws -> TeachingSkill { + let userPrompt = """ + refresh this stale teaching skill for current macos ui: + + skill name: \(staleSkill.name) + description: \(staleSkill.description) + bundle ids: \(staleSkill.bundleIds.joined(separator: ", ")) + + existing body: + \(staleSkill.body) + """ + + let response = try await claudeAPI.sendTextMessage( + systemPrompt: patchSystemPrompt, + userPrompt: userPrompt, + maxTokens: 1200 + ) + + var patchedSkill = staleSkill + patchedSkill.body = response.text.trimmingCharacters(in: .whitespacesAndNewlines) + patchedSkill.status = .active + patchedSkill.lastUsed = Date() + return patchedSkill + } +} diff --git a/leanring-buddy/SkillMatcher.swift b/leanring-buddy/SkillMatcher.swift new file mode 100644 index 000000000..e5013a851 --- /dev/null +++ b/leanring-buddy/SkillMatcher.swift @@ -0,0 +1,217 @@ +// +// SkillMatcher.swift +// leanring-buddy +// +// Matches teaching skills to the current app and user transcript. +// + +import Foundation + +struct SkillMatch: Equatable { + let skill: TeachingSkill + let score: Int +} + +struct DuplicateSkillPair: Equatable { + let primarySkill: TeachingSkill + let duplicateSkill: TeachingSkill + let overlapScore: Int +} + +enum SkillMatcher { + static let topicStopwords: Set = [ + "how", "do", "does", "did", "can", "could", "would", "should", "what", "where", "when", "why", "which", + "the", "this", "that", "these", "those", "a", "an", "my", "your", "me", "you", "and", "or", "but", + "is", "are", "was", "were", "be", "been", "being", "to", "in", "on", "at", "for", "of", "with", "from", + "please", "help", "show", "tell", "explain", "use", "using", "got", "thanks", "thank", "worked", "perfect" + ] + + static func meaningfulTokens(_ text: String) -> [String] { + tokenize(text).filter { !topicStopwords.contains($0) } + } + + static func matchSkills( + from skills: [TeachingSkill], + bundleId: String?, + transcript: String, + limit: Int = 3 + ) -> [SkillMatch] { + let queryTokens = tokenize(transcript) + let eligibleSkills = skills.filter { skill in + skill.status != .archived || skill.isPinned + } + + let scored = eligibleSkills.compactMap { skill -> SkillMatch? in + var score = 0 + + if let bundleId, skill.bundleIds.contains(bundleId) { + score += 12 + } + + let haystackTokens = Set( + tokenize(skill.name) + + tokenize(skill.description) + + tokenize(skill.body) + ) + score += queryTokens.filter { haystackTokens.contains($0) }.count + + if skill.status == .active { + score += 1 + } + score += min(skill.usageCount, 5) + + return score > 0 ? SkillMatch(skill: skill, score: score) : nil + } + + return Array( + scored + .sorted { lhs, rhs in + if lhs.score != rhs.score { return lhs.score > rhs.score } + return (lhs.skill.lastUsed ?? .distantPast) > (rhs.skill.lastUsed ?? .distantPast) + } + .prefix(limit) + ) + } + + static func findSimilarSkill( + in skills: [TeachingSkill], + bundleId: String?, + topic: String + ) -> TeachingSkill? { + findSkillForUpdate( + in: skills, + targetBundleId: bundleId, + primaryQuestion: topic + ) + } + + static func findSkillForUpdate( + in skills: [TeachingSkill], + targetBundleId: String?, + primaryQuestion: String + ) -> TeachingSkill? { + let resolvedTargetBundleId = TeachingSkill.detectBundleId(in: primaryQuestion) ?? targetBundleId + let expectedSkillId = TeachingSkill.stableSkillId( + bundleId: resolvedTargetBundleId, + primaryQuestion: primaryQuestion + ) + let expectedTaskSlug = TeachingSkill.taskSlug(from: primaryQuestion) + + if let exactMatch = skills.first(where: { $0.id == expectedSkillId }) { + return exactMatch + } + + if let resolvedTargetBundleId { + let bundleMatches = skills.filter { skill in + skill.bundleIds.contains(resolvedTargetBundleId) && + (skill.taskSlug == expectedTaskSlug || skill.id.hasSuffix("-\(expectedTaskSlug)")) + } + + if let bestBundleMatch = bundleMatches.max(by: { lhs, rhs in + if lhs.usageCount != rhs.usageCount { return lhs.usageCount < rhs.usageCount } + return (lhs.lastUsed ?? .distantPast) < (rhs.lastUsed ?? .distantPast) + }) { + return bestBundleMatch + } + } + + let topicTokens = Set(meaningfulTokens(primaryQuestion)) + guard !topicTokens.isEmpty else { return nil } + + let candidates = skills.filter { skill in + guard let resolvedTargetBundleId else { return true } + return skill.bundleIds.isEmpty || skill.bundleIds.contains(resolvedTargetBundleId) + } + + return candidates.max { lhs, rhs in + overlapScore(lhs, topicTokens: topicTokens) < overlapScore(rhs, topicTokens: topicTokens) + } + .flatMap { candidate in + overlapScore(candidate, topicTokens: topicTokens) >= 2 ? candidate : nil + } + } + + static func tokenOverlapScore(between leftSkill: TeachingSkill, and rightSkill: TeachingSkill) -> Int { + let leftTokens = Set( + tokenize(leftSkill.name) + + tokenize(leftSkill.description) + + tokenize(leftSkill.body) + ) + let rightTokens = Set( + tokenize(rightSkill.name) + + tokenize(rightSkill.description) + + tokenize(rightSkill.body) + ) + return leftTokens.filter { rightTokens.contains($0) }.count + } + + static func findDuplicateSkillPairs( + in skills: [TeachingSkill], + minimumOverlapScore: Int = 3 + ) -> [DuplicateSkillPair] { + let eligibleSkills = skills.filter { skill in + !skill.isPinned && skill.status != .archived + } + + var duplicatePairs: [DuplicateSkillPair] = [] + + for leftIndex in 0..= minimumOverlapScore else { continue } + + let primarySkill: TeachingSkill + let duplicateSkill: TeachingSkill + if leftSkill.usageCount != rightSkill.usageCount { + if leftSkill.usageCount > rightSkill.usageCount { + primarySkill = leftSkill + duplicateSkill = rightSkill + } else { + primarySkill = rightSkill + duplicateSkill = leftSkill + } + } else if (leftSkill.lastUsed ?? .distantPast) >= (rightSkill.lastUsed ?? .distantPast) { + primarySkill = leftSkill + duplicateSkill = rightSkill + } else { + primarySkill = rightSkill + duplicateSkill = leftSkill + } + + duplicatePairs.append( + DuplicateSkillPair( + primarySkill: primarySkill, + duplicateSkill: duplicateSkill, + overlapScore: overlapScore + ) + ) + } + } + + return duplicatePairs + } + + private static func overlapScore(_ skill: TeachingSkill, topicTokens: Set) -> Int { + let skillTokens = Set( + tokenize(skill.name) + + tokenize(skill.description) + + tokenize(skill.body) + ) + return topicTokens.filter { skillTokens.contains($0) }.count + } + + static func tokenize(_ text: String) -> [String] { + text + .lowercased() + .components(separatedBy: CharacterSet.alphanumerics.inverted) + .filter { $0.count >= 3 } + } +} diff --git a/leanring-buddy/SkillSynthesizer.swift b/leanring-buddy/SkillSynthesizer.swift new file mode 100644 index 000000000..fd01d71eb --- /dev/null +++ b/leanring-buddy/SkillSynthesizer.swift @@ -0,0 +1,164 @@ +// +// SkillSynthesizer.swift +// leanring-buddy +// +// Drafts or updates teaching skills from a completed tutoring session trace. +// + +import Foundation + +enum SkillSynthesizer { + private static let synthesisSystemPrompt = """ + you write teaching skills for clicky, a screen-native voice tutor that points at ui elements. + + given a tutoring session transcript, produce a markdown teaching skill body only. do not include yaml frontmatter. + + include: + - the app/workflow being taught + - ordered steps + - exact ui labels, menu paths, and shortcuts when known + - pointing heuristics (what to point at first, what to avoid) + - common mistakes the user made or might make + - user preferences and constraints from the session (e.g. keyboard only, no menu) + - completion signals ("user said got it", visible ui state) + + keep it concise, practical, and specific to macos ui teaching. all lowercase. + """ + + private static let patchSystemPrompt = """ + you update an existing teaching skill for clicky, a screen-native voice tutor that points at ui elements. + + given the existing skill body and a new tutoring session, produce an updated markdown body only. do not include yaml frontmatter. + + patch-first rules: + - preserve guidance that still works + - merge new constraints and refinements from the latest session + - do not duplicate steps or create a parallel workflow + - honor user preferences such as "keyboard only" or "avoid the menu" + - keep ordered steps, ui labels, shortcuts, pointing heuristics, and completion signals + + keep it concise, practical, and specific to macos ui teaching. all lowercase. + """ + + static func synthesizeSkillContent( + sessionTrace: [SessionTraceEntry], + trigger: SkillWriteTrigger, + existingSkill: TeachingSkill?, + targetBundleId: String?, + claudeAPI: ClaudeAPI + ) async throws -> (name: String, description: String, body: String) { + let sessionSummary = renderSessionSummary( + sessionTrace, + trigger: trigger, + existingSkill: existingSkill, + targetBundleId: targetBundleId + ) + let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: sessionTrace) ?? trigger.topic + let metadata = TeachingSkill.buildMetadata(primaryQuestion: primaryQuestion, bundleId: targetBundleId) + + let userPrompt: String + let systemPrompt: String + if let existingSkill { + systemPrompt = patchSystemPrompt + userPrompt = """ + patch this existing teaching skill using the new session. preserve useful existing guidance and merge refinements: + + existing skill id: \(existingSkill.id) + existing skill body: + \(existingSkill.body) + + new session: + \(sessionSummary) + """ + } else { + systemPrompt = synthesisSystemPrompt + userPrompt = "create a new teaching skill from this session:\n\n\(sessionSummary)" + } + + let response = try await claudeAPI.sendTextMessage( + systemPrompt: systemPrompt, + userPrompt: userPrompt, + maxTokens: 1200 + ) + + let cleanedBody = response.text.trimmingCharacters(in: .whitespacesAndNewlines) + let name = existingSkill?.name ?? metadata.name + let description = existingSkill?.description ?? metadata.description + return (name: name, description: description, body: cleanedBody) + } + + static func buildSkillMetadata( + sessionTrace: [SessionTraceEntry], + trigger: SkillWriteTrigger, + targetBundleId: String? + ) -> TeachingSkill.Metadata { + let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: sessionTrace) ?? trigger.topic + return TeachingSkill.buildMetadata(primaryQuestion: primaryQuestion, bundleId: targetBundleId) + } + + static func buildSkill( + id: String?, + name: String, + description: String, + body: String, + targetBundleId: String?, + taskSlug: String?, + primaryQuestion: String, + existingSkill: TeachingSkill? + ) -> TeachingSkill { + let resolvedID = id + ?? existingSkill?.id + ?? TeachingSkill.stableSkillId(bundleId: targetBundleId, primaryQuestion: primaryQuestion) + let resolvedTaskSlug = taskSlug ?? existingSkill?.taskSlug ?? TeachingSkill.taskSlug(from: primaryQuestion) + + let bundleIds: [String] + if let targetBundleId { + bundleIds = [targetBundleId] + } else if let existingSkill, !existingSkill.bundleIds.isEmpty { + bundleIds = existingSkill.bundleIds + } else { + bundleIds = [] + } + + return TeachingSkill( + id: resolvedID, + name: name, + description: description, + bundleIds: bundleIds, + status: .active, + lastUsed: Date(), + usageCount: existingSkill?.usageCount ?? 0, + isPinned: existingSkill?.isPinned ?? false, + taskSlug: resolvedTaskSlug, + body: body + ) + } + + private static func renderSessionSummary( + _ sessionTrace: [SessionTraceEntry], + trigger: SkillWriteTrigger, + existingSkill: TeachingSkill?, + targetBundleId: String? + ) -> String { + var lines: [String] = [] + lines.append("trigger: \(trigger.reason.rawValue)") + lines.append("topic: \(trigger.topic)") + lines.append("target bundle id: \(targetBundleId ?? "unknown")") + if let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: sessionTrace) { + lines.append("primary question: \(primaryQuestion)") + } + if let existingSkill { + lines.append("existing skill id: \(existingSkill.id)") + } + + for (index, entry) in sessionTrace.enumerated() { + lines.append("exchange \(index + 1):") + lines.append("user: \(entry.userTranscript)") + lines.append("assistant: \(entry.assistantResponse)") + lines.append("frontmost bundle id: \(entry.bundleId ?? "unknown")") + lines.append("pointed: \(entry.pointed ? "yes" : "no")") + } + + return lines.joined(separator: "\n") + } +} diff --git a/leanring-buddy/SkillTargetAppResolver.swift b/leanring-buddy/SkillTargetAppResolver.swift new file mode 100644 index 000000000..18d0d5ff5 --- /dev/null +++ b/leanring-buddy/SkillTargetAppResolver.swift @@ -0,0 +1,36 @@ +// +// SkillTargetAppResolver.swift +// leanring-buddy +// +// Resolves which app a tutoring session is teaching, which may differ from the +// frontmost app (e.g. terminal focused while TextEdit is visible on screen). +// + +import Foundation + +enum SkillTargetAppResolver { + static func resolveTargetBundleId( + from sessionTrace: [SessionTraceEntry], + frontmostBundleId: String?, + existingSkill: TeachingSkill? = nil + ) -> String? { + if let primaryQuestion = SkillTriggerEvaluator.primaryTeachingQuestion(from: sessionTrace), + let bundleIdFromPrimaryQuestion = TeachingSkill.detectBundleId(in: primaryQuestion) { + return bundleIdFromPrimaryQuestion + } + + for entry in sessionTrace { + for text in [entry.userTranscript, entry.assistantResponse] { + if let bundleId = TeachingSkill.detectBundleId(in: text) { + return bundleId + } + } + } + + if let existingSkill, let existingBundleId = existingSkill.bundleIds.first { + return existingBundleId + } + + return frontmostBundleId + } +} diff --git a/leanring-buddy/SkillTriggerEvaluator.swift b/leanring-buddy/SkillTriggerEvaluator.swift new file mode 100644 index 000000000..5ada47617 --- /dev/null +++ b/leanring-buddy/SkillTriggerEvaluator.swift @@ -0,0 +1,83 @@ +// +// SkillTriggerEvaluator.swift +// leanring-buddy +// +// Decides when a tutoring session should create or update a teaching skill. +// + +import Foundation + +struct SkillWriteTrigger: Equatable { + enum Reason: String, Equatable { + case userConfirmed + } + + let reason: Reason + let topic: String +} + +enum SkillTriggerEvaluator { + private static let confirmationPhrases = [ + "got it", + "that worked", + "thanks that worked", + "thank you that worked", + "perfect", + "helpful", + "makes sense now", + "that helps" + ] + + /// Hermes-style write gate: persist skills only after explicit user confirmation. + static func shouldWriteSkill( + sessionTrace: [SessionTraceEntry], + latestTranscript: String, + topicHistory: [TeachingTopicHistoryEntry] = [] + ) -> SkillWriteTrigger? { + guard !sessionTrace.isEmpty else { return nil } + + let normalizedTranscript = latestTranscript + .lowercased() + .trimmingCharacters(in: .whitespacesAndNewlines) + + guard confirmationPhrases.contains(where: { normalizedTranscript.contains($0) }) else { + return nil + } + + let topic = deriveTopic(from: sessionTrace) + return SkillWriteTrigger(reason: .userConfirmed, topic: topic) + } + + static func primaryTeachingQuestion(from sessionTrace: [SessionTraceEntry]) -> String? { + sessionTrace + .map(\.userTranscript) + .first { !isConfirmationTranscript($0) } + } + + static func isConfirmationTranscript(_ transcript: String) -> Bool { + let normalized = transcript + .lowercased() + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !normalized.isEmpty else { return true } + return confirmationPhrases.contains { normalized.contains($0) } + } + + static func deriveTopic(from sessionTrace: [SessionTraceEntry]) -> String { + guard let primaryQuestion = primaryTeachingQuestion(from: sessionTrace) else { return "" } + return deriveTopic(fromQuestion: primaryQuestion) + } + + static func deriveTopic(fromQuestion question: String) -> String { + let tokens = SkillMatcher.meaningfulTokens(question) + return tokens.prefix(6).joined(separator: " ") + } + + static func isScreenTeachingSession(_ sessionTrace: [SessionTraceEntry]) -> Bool { + sessionTrace.contains(where: \.pointed) || + sessionTrace.contains(where: { $0.bundleId != nil }) || + sessionTrace.contains { entry in + TeachingSkill.detectBundleId(in: entry.userTranscript) != nil || + TeachingSkill.detectBundleId(in: entry.assistantResponse) != nil + } + } +} diff --git a/leanring-buddy/TeachingPromptBuilder.swift b/leanring-buddy/TeachingPromptBuilder.swift new file mode 100644 index 000000000..4cfcf8dfb --- /dev/null +++ b/leanring-buddy/TeachingPromptBuilder.swift @@ -0,0 +1,41 @@ +// +// TeachingPromptBuilder.swift +// leanring-buddy +// +// Composes the voice response system prompt with matched teaching skills. +// + +import Foundation + +enum TeachingPromptBuilder { + static func buildVoiceResponsePrompt( + basePrompt: String, + matchedSkills: [TeachingSkill], + maxSkillCharacters: Int = 3500 + ) -> String { + guard !matchedSkills.isEmpty else { return basePrompt } + + var usedCharacters = 0 + var renderedSkills: [String] = [] + + for skill in matchedSkills { + let rendered = skill.renderedMarkdown() + if usedCharacters + rendered.count > maxSkillCharacters, !renderedSkills.isEmpty { + break + } + renderedSkills.append(rendered) + usedCharacters += rendered.count + } + + guard !renderedSkills.isEmpty else { return basePrompt } + + return """ + \(basePrompt) + + teaching skills: + the following local teaching notes were learned from earlier successful tutoring sessions on this mac. reuse them when they clearly apply. prefer the saved ui labels, menu paths, shortcuts, and pointing order over guessing. + + \(renderedSkills.joined(separator: "\n\n")) + """ + } +} diff --git a/leanring-buddy/TeachingSkill.swift b/leanring-buddy/TeachingSkill.swift new file mode 100644 index 000000000..73cf4cabe --- /dev/null +++ b/leanring-buddy/TeachingSkill.swift @@ -0,0 +1,332 @@ +// +// TeachingSkill.swift +// leanring-buddy +// +// Model and parsing for local teaching skills stored as SKILL.md files. +// + +import Foundation + +enum TeachingSkillStatus: String, Codable, CaseIterable { + case active + case stale + case archived +} + +struct TeachingSkill: Identifiable, Equatable { + let id: String + var name: String + var description: String + var bundleIds: [String] + var status: TeachingSkillStatus + var lastUsed: Date? + var usageCount: Int + var isPinned: Bool + var taskSlug: String? + var body: String + + struct Metadata: Equatable { + let id: String + let name: String + let description: String + let taskSlug: String + } + + private static let knownAppNames: [String: String] = [ + "com.apple.TextEdit": "TextEdit", + "com.apple.dt.Xcode": "Xcode", + "com.apple.finder": "Finder", + "com.apple.Safari": "Safari", + "com.apple.mail": "Mail", + "com.apple.Notes": "Notes", + "com.apple.systempreferences": "System Settings", + "com.apple.Preview": "Preview" + ] + + var folderURL: URL { + TeachingSkillStore.skillsRootURL.appendingPathComponent(id, isDirectory: true) + } + + var fileURL: URL { + folderURL.appendingPathComponent("SKILL.md") + } + + func renderedMarkdown(maxBodyCharacters: Int = 2500) -> String { + var content = body.trimmingCharacters(in: .whitespacesAndNewlines) + if content.count > maxBodyCharacters { + let endIndex = content.index(content.startIndex, offsetBy: maxBodyCharacters) + content = String(content[.. String { + let lastUsedValue = lastUsed.map { TeachingSkill.dateFormatter.string(from: $0) } ?? "" + let bundleLines = bundleIds.map { " - \($0)" }.joined(separator: "\n") + return """ + --- + name: \(name) + description: \(TeachingSkill.yamlEscape(description)) + bundleIds: + \(bundleLines.isEmpty ? " []" : bundleLines) + status: \(status.rawValue) + lastUsed: \(lastUsedValue) + usageCount: \(usageCount) + pinned: \(isPinned) + taskSlug: \(taskSlug ?? "") + --- + + \(body.trimmingCharacters(in: .whitespacesAndNewlines)) + + """ + } + + static func slug(from name: String) -> String { + let lowered = name.lowercased() + let allowed = lowered.map { character -> Character in + if character.isLetter || character.isNumber { return character } + if character == " " || character == "-" || character == "_" { return "-" } + return "-" + } + let collapsed = String(allowed) + .replacingOccurrences(of: "-+", with: "-", options: .regularExpression) + .trimmingCharacters(in: CharacterSet(charactersIn: "-")) + return collapsed.isEmpty ? "teaching-skill" : collapsed + } + + static func displayName(forBundleId bundleId: String?) -> String? { + guard let bundleId else { return nil } + if let knownName = knownAppNames[bundleId] { + return knownName + } + guard let lastComponent = bundleId.split(separator: ".").last else { return nil } + return String(lastComponent) + } + + static func detectBundleId(in text: String) -> String? { + let loweredText = text.lowercased() + + for (bundleId, displayName) in knownAppNames { + let loweredDisplayName = displayName.lowercased() + if loweredText.contains(loweredDisplayName) { + return bundleId + } + + let compactDisplayName = loweredDisplayName.replacingOccurrences(of: " ", with: "") + if compactDisplayName.count >= 4, loweredText.contains(compactDisplayName) { + return bundleId + } + } + + if loweredText.contains("text edit") || loweredText.contains("textedit") { + return "com.apple.TextEdit" + } + + return nil + } + + static func taskSlug(from primaryQuestion: String) -> String { + let actionTokens = SkillMatcher.meaningfulTokens(primaryQuestion) + let primaryActionToken = actionTokens.first ?? "task" + return slug(from: primaryActionToken) + } + + static func stableSkillId(bundleId: String?, primaryQuestion: String) -> String { + let resolvedTaskSlug = taskSlug(from: primaryQuestion) + let appName = displayName(forBundleId: bundleId) + + if let appName { + return "teach-\(slug(from: appName))-\(resolvedTaskSlug)" + } + + let actionPhrase = SkillMatcher.meaningfulTokens(primaryQuestion).prefix(3).joined(separator: " ") + if actionPhrase.isEmpty { + return "teach-\(resolvedTaskSlug)" + } + + return "teach-\(slug(from: actionPhrase))" + } + + static func buildMetadata(primaryQuestion: String, bundleId: String?) -> Metadata { + let actionTokens = SkillMatcher.meaningfulTokens(primaryQuestion) + let primaryActionToken = actionTokens.first ?? "task" + let resolvedTaskSlug = taskSlug(from: primaryQuestion) + let appName = displayName(forBundleId: bundleId) + let skillID = stableSkillId(bundleId: bundleId, primaryQuestion: primaryQuestion) + + let skillName: String + if let appName { + skillName = "\(capitalizeWord(primaryActionToken)) in \(appName)" + } else { + let actionPhrase = actionTokens.prefix(3).joined(separator: " ") + skillName = capitalizeWords(actionPhrase.isEmpty ? primaryActionToken : actionPhrase) + } + + let skillDescription = descriptionFromQuestion(primaryQuestion) + return Metadata(id: skillID, name: skillName, description: skillDescription, taskSlug: resolvedTaskSlug) + } + + private static func descriptionFromQuestion(_ question: String) -> String { + var cleanedQuestion = question.trimmingCharacters(in: .whitespacesAndNewlines) + if cleanedQuestion.hasSuffix("?") { + cleanedQuestion.removeLast() + } + + let loweredQuestion = cleanedQuestion.lowercased() + let questionPrefixes = [ + "how do i ", + "how can i ", + "how to ", + "what is the way to ", + "what is ", + "where do i ", + "where is ", + "where can i " + ] + + for prefix in questionPrefixes where loweredQuestion.hasPrefix(prefix) { + cleanedQuestion = String(cleanedQuestion.dropFirst(prefix.count)) + break + } + + let meaningfulTokens = SkillMatcher.meaningfulTokens(cleanedQuestion) + if !meaningfulTokens.isEmpty { + return "Walk the user through \(meaningfulTokens.joined(separator: " "))" + } + + cleanedQuestion = cleanedQuestion + .trimmingCharacters(in: .whitespacesAndNewlines) + .replacingOccurrences(of: " ", with: " ") + + if cleanedQuestion.isEmpty { + return "Walk the user through this task" + } + + return "Walk the user through \(cleanedQuestion)" + } + + private static func capitalizeWord(_ word: String) -> String { + guard let firstCharacter = word.first else { return word } + return String(firstCharacter).uppercased() + word.dropFirst() + } + + private static func capitalizeWords(_ phrase: String) -> String { + phrase + .split(separator: " ") + .map { capitalizeWord(String($0)) } + .joined(separator: " ") + } + + static func parse(id: String, markdown: String) -> TeachingSkill? { + let trimmed = markdown.trimmingCharacters(in: .whitespacesAndNewlines) + guard trimmed.hasPrefix("---") else { return nil } + + let components = trimmed.components(separatedBy: "---") + guard components.count >= 3 else { return nil } + + let frontmatter = components[1] + let body = components.dropFirst(2).joined(separator: "---").trimmingCharacters(in: .whitespacesAndNewlines) + let metadata = parseFrontmatter(frontmatter) + + let name = metadata["name"] ?? id + let description = metadata["description"] ?? "" + let bundleIds = metadata["bundleIds"]? + .split(separator: ",") + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } ?? [] + let status = TeachingSkillStatus(rawValue: metadata["status"] ?? "active") ?? .active + let lastUsed = metadata["lastUsed"].flatMap { dateFormatter.date(from: $0) } + let usageCount = Int(metadata["usageCount"] ?? "0") ?? 0 + let isPinned = (metadata["pinned"] ?? "false").lowercased() == "true" + let taskSlug = metadata["taskSlug"]?.trimmingCharacters(in: .whitespacesAndNewlines) + let resolvedTaskSlug = (taskSlug?.isEmpty == false) ? taskSlug : nil + + return TeachingSkill( + id: id, + name: name, + description: description, + bundleIds: bundleIds, + status: status, + lastUsed: lastUsed, + usageCount: usageCount, + isPinned: isPinned, + taskSlug: resolvedTaskSlug, + body: body + ) + } + + private static func parseFrontmatter(_ frontmatter: String) -> [String: String] { + var result: [String: String] = [:] + var currentListKey: String? + var listValues: [String] = [] + + func flushList() { + guard let key = currentListKey else { return } + result[key] = listValues.joined(separator: ",") + currentListKey = nil + listValues = [] + } + + for line in frontmatter.split(separator: "\n", omittingEmptySubsequences: false) { + let trimmedLine = line.trimmingCharacters(in: .whitespaces) + if trimmedLine.hasPrefix("- ") { + listValues.append(String(trimmedLine.dropFirst(2)).trimmingCharacters(in: .whitespaces)) + continue + } + + flushList() + + guard let separatorIndex = trimmedLine.firstIndex(of: ":") else { continue } + let key = String(trimmedLine[.. String { + if value.contains(":") || value.contains("\"") || value.contains("\n") { + return "\"\(value.replacingOccurrences(of: "\"", with: "\\\""))\"" + } + return value + } + + private static func unyamlEscape(_ value: String) -> String { + if value.hasPrefix("\""), value.hasSuffix("\""), value.count >= 2 { + let inner = value.dropFirst().dropLast() + return inner.replacingOccurrences(of: "\\\"", with: "\"") + } + return value + } + + static let dateFormatter: DateFormatter = { + let formatter = DateFormatter() + formatter.calendar = Calendar(identifier: .gregorian) + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.timeZone = TimeZone(secondsFromGMT: 0) + formatter.dateFormat = "yyyy-MM-dd" + return formatter + }() +} + +struct SessionTraceEntry: Equatable, Codable { + let timestamp: Date + let userTranscript: String + let assistantResponse: String + let bundleId: String? + let pointed: Bool +} diff --git a/leanring-buddy/TeachingSkillStore.swift b/leanring-buddy/TeachingSkillStore.swift new file mode 100644 index 000000000..86a1488d4 --- /dev/null +++ b/leanring-buddy/TeachingSkillStore.swift @@ -0,0 +1,100 @@ +// +// TeachingSkillStore.swift +// leanring-buddy +// +// Local filesystem storage for teaching skills under ~/.clicky/skills/. +// + +import Foundation + +final class TeachingSkillStore { + static var skillsRootURL: URL { + ClickyPaths.skills + } + + private(set) var skills: [TeachingSkill] = [] + + func loadSkills() { + let fileManager = FileManager.default + try? fileManager.createDirectory(at: Self.skillsRootURL, withIntermediateDirectories: true) + + guard let folders = try? fileManager.contentsOfDirectory( + at: Self.skillsRootURL, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + ) else { + skills = [] + return + } + + var loaded: [TeachingSkill] = [] + for folder in folders where (try? folder.resourceValues(forKeys: [.isDirectoryKey]).isDirectory) == true { + let skillFile = folder.appendingPathComponent("SKILL.md") + guard let markdown = try? String(contentsOf: skillFile, encoding: .utf8), + let skill = TeachingSkill.parse(id: folder.lastPathComponent, markdown: markdown) else { + continue + } + loaded.append(skill) + } + + skills = loaded.sorted { lhs, rhs in + if lhs.usageCount != rhs.usageCount { return lhs.usageCount > rhs.usageCount } + return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending + } + } + + @discardableResult + func saveSkill(_ skill: TeachingSkill) throws -> TeachingSkill { + let fileManager = FileManager.default + try fileManager.createDirectory(at: skill.folderURL, withIntermediateDirectories: true) + try skill.serialize().write(to: skill.fileURL, atomically: true, encoding: .utf8) + if let index = skills.firstIndex(where: { $0.id == skill.id }) { + skills[index] = skill + } else { + skills.append(skill) + } + skills.sort { lhs, rhs in + if lhs.usageCount != rhs.usageCount { return lhs.usageCount > rhs.usageCount } + return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending + } + return skill + } + + func deleteSkill(id: String) throws { + let folder = Self.skillsRootURL.appendingPathComponent(id, isDirectory: true) + try FileManager.default.removeItem(at: folder) + skills.removeAll { $0.id == id } + } + + func skill(withID id: String) -> TeachingSkill? { + skills.first { $0.id == id } + } + + func markUsed(_ skill: TeachingSkill) throws -> TeachingSkill { + var updated = skill + updated.lastUsed = Date() + updated.usageCount += 1 + if updated.status == .stale { + updated.status = .active + } + return try saveSkill(updated) + } + + func setPinned(id: String, pinned: Bool) throws { + guard var skill = skill(withID: id) else { return } + skill.isPinned = pinned + _ = try saveSkill(skill) + } + + func restoreSkill(id: String) throws { + guard var skill = skill(withID: id) else { return } + skill.status = .active + skill.lastUsed = Date() + _ = try saveSkill(skill) + } + + func skills(withStatus status: TeachingSkillStatus?) -> [TeachingSkill] { + guard let status else { return skills } + return skills.filter { $0.status == status } + } +} diff --git a/leanring-buddy/TeachingSkillsLibraryView.swift b/leanring-buddy/TeachingSkillsLibraryView.swift new file mode 100644 index 000000000..d3dcb1cf3 --- /dev/null +++ b/leanring-buddy/TeachingSkillsLibraryView.swift @@ -0,0 +1,264 @@ +// +// TeachingSkillsLibraryView.swift +// leanring-buddy +// +// Full teaching skills library with status filters, detail view, and lifecycle actions. +// + +import SwiftUI + +enum TeachingSkillsLibraryFilter: String, CaseIterable, Identifiable { + case all + case active + case stale + case archived + + var id: String { rawValue } + + var label: String { + switch self { + case .all: return "All" + case .active: return "Active" + case .stale: return "Stale" + case .archived: return "Archived" + } + } + + var status: TeachingSkillStatus? { + switch self { + case .all: return nil + case .active: return .active + case .stale: return .stale + case .archived: return .archived + } + } +} + +struct TeachingSkillsLibraryView: View { + @ObservedObject var companionManager: CompanionManager + let onBack: () -> Void + + @State private var selectedFilter: TeachingSkillsLibraryFilter = .all + @State private var selectedSkill: TeachingSkill? + + private var filteredSkills: [TeachingSkill] { + companionManager.teachingSkills(withStatus: selectedFilter.status) + } + + var body: some View { + VStack(alignment: .leading, spacing: 0) { + libraryHeader + + Divider() + .background(DS.Colors.borderSubtle) + .padding(.horizontal, 16) + + if let selectedSkill { + skillDetailView(selectedSkill) + } else { + filterPicker + .padding(.horizontal, 16) + .padding(.top, 12) + + skillsList + } + } + } + + private var libraryHeader: some View { + HStack(spacing: 8) { + Button(action: { + if selectedSkill != nil { + selectedSkill = nil + } else { + onBack() + } + }) { + Image(systemName: "chevron.left") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(DS.Colors.textTertiary) + .frame(width: 24, height: 24) + .background( + Circle() + .fill(Color.white.opacity(0.08)) + ) + } + .buttonStyle(.plain) + .pointerCursor() + + Text(selectedSkill?.name ?? "Teaching Skills") + .font(.system(size: 14, weight: .semibold)) + .foregroundColor(DS.Colors.textSecondary) + .lineLimit(1) + + Spacer() + } + .padding(.horizontal, 16) + .padding(.vertical, 14) + } + + private var filterPicker: some View { + HStack(spacing: 4) { + ForEach(TeachingSkillsLibraryFilter.allCases) { filter in + Button(action: { + selectedFilter = filter + }) { + Text(filter.label) + .font(.system(size: 11, weight: .medium)) + .foregroundColor(selectedFilter == filter ? DS.Colors.textOnAccent : DS.Colors.textTertiary) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background( + RoundedRectangle(cornerRadius: DS.CornerRadius.small, style: .continuous) + .fill(selectedFilter == filter ? DS.Colors.accent : Color.white.opacity(0.06)) + ) + } + .buttonStyle(.plain) + .pointerCursor() + .accessibilityIdentifier("clicky.panel.skills-library.filter.\(filter.rawValue)") + } + } + } + + private var skillsList: some View { + ScrollView { + if filteredSkills.isEmpty { + Text("No \(selectedFilter == .all ? "" : selectedFilter.label.lowercased() + " ")skills.") + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textTertiary) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 16) + .padding(.top, 12) + } else { + VStack(spacing: 0) { + ForEach(filteredSkills) { skill in + skillRow(skill) + Divider() + .background(DS.Colors.borderSubtle.opacity(0.5)) + .padding(.leading, 16) + } + } + .padding(.top, 8) + } + } + .frame(maxHeight: 280) + } + + private func skillRow(_ skill: TeachingSkill) -> some View { + HStack(spacing: 8) { + Button(action: { + selectedSkill = skill + }) { + VStack(alignment: .leading, spacing: 2) { + HStack(spacing: 4) { + Text(skill.name) + .font(.system(size: 12, weight: .semibold)) + .foregroundColor(DS.Colors.textSecondary) + .lineLimit(1) + + if skill.isPinned { + Image(systemName: "pin.fill") + .font(.system(size: 9)) + .foregroundColor(DS.Colors.accent) + } + } + + Text("\(skill.usageCount) uses • \(skill.status.rawValue)") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + } + .frame(maxWidth: .infinity, alignment: .leading) + } + .buttonStyle(.plain) + .pointerCursor() + + if skill.status == .archived || skill.status == .stale { + Button(action: { + companionManager.restoreTeachingSkill(id: skill.id) + }) { + Image(systemName: "arrow.uturn.backward") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + } + .buttonStyle(.plain) + .pointerCursor() + } + + Button(action: { + companionManager.setTeachingSkillPinned(id: skill.id, pinned: !skill.isPinned) + }) { + Image(systemName: skill.isPinned ? "pin.fill" : "pin") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(skill.isPinned ? DS.Colors.accent : DS.Colors.textTertiary) + } + .buttonStyle(.plain) + .pointerCursor() + + Button(action: { + if selectedSkill?.id == skill.id { + selectedSkill = nil + } + companionManager.deleteTeachingSkill(id: skill.id) + }) { + Image(systemName: "trash") + .font(.system(size: 11, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + } + .buttonStyle(.plain) + .pointerCursor() + } + .padding(.horizontal, 16) + .padding(.vertical, 8) + .accessibilityIdentifier("clicky.panel.skills-library.row.\(skill.id)") + } + + private func skillDetailView(_ skill: TeachingSkill) -> some View { + ScrollView { + VStack(alignment: .leading, spacing: 12) { + VStack(alignment: .leading, spacing: 4) { + Text(skill.description) + .font(.system(size: 12)) + .foregroundColor(DS.Colors.textSecondary) + + Text("\(skill.usageCount) uses • \(skill.status.rawValue)") + .font(.system(size: 10)) + .foregroundColor(DS.Colors.textTertiary) + + if !skill.bundleIds.isEmpty { + Text(skill.bundleIds.joined(separator: ", ")) + .font(.system(size: 10, design: .monospaced)) + .foregroundColor(DS.Colors.textTertiary) + } + } + + Divider() + .background(DS.Colors.borderSubtle) + + skillMarkdownBody(skill.body) + } + .padding(.horizontal, 16) + .padding(.vertical, 12) + } + .frame(maxHeight: 300) + } + + @ViewBuilder + private func skillMarkdownBody(_ body: String) -> some View { + if let attributedBody = try? AttributedString( + markdown: body, + options: AttributedString.MarkdownParsingOptions(interpretedSyntax: .inlineOnlyPreservingWhitespace) + ) { + Text(attributedBody) + .font(.system(size: 11)) + .foregroundColor(DS.Colors.textSecondary) + .frame(maxWidth: .infinity, alignment: .leading) + .textSelection(.enabled) + } else { + Text(body) + .font(.system(size: 11, design: .monospaced)) + .foregroundColor(DS.Colors.textSecondary) + .frame(maxWidth: .infinity, alignment: .leading) + .textSelection(.enabled) + } + } +} diff --git a/leanring-buddy/TeachingTopicHistoryStore.swift b/leanring-buddy/TeachingTopicHistoryStore.swift new file mode 100644 index 000000000..bc5ce2d94 --- /dev/null +++ b/leanring-buddy/TeachingTopicHistoryStore.swift @@ -0,0 +1,99 @@ +// +// TeachingTopicHistoryStore.swift +// leanring-buddy +// +// Persists lightweight teaching topic history under ~/.clicky/topic-history.json. +// + +import Foundation + +struct TeachingTopicHistoryEntry: Codable, Equatable { + let topicTokens: [String] + let bundleId: String? + let timestamp: Date + var skillId: String? +} + +final class TeachingTopicHistoryStore { + static var historyFileURL: URL { + ClickyPaths.topicHistory + } + + private(set) var entries: [TeachingTopicHistoryEntry] = [] + private let maxEntryCount = 200 + private let historyFileURL: URL + + init(historyFileURL: URL = TeachingTopicHistoryStore.historyFileURL) { + self.historyFileURL = historyFileURL + } + + func load() { + guard FileManager.default.fileExists(atPath: historyFileURL.path), + let data = try? Data(contentsOf: historyFileURL), + let decoded = try? JSONDecoder().decode([TeachingTopicHistoryEntry].self, from: data) else { + entries = [] + return + } + entries = decoded + } + + func save() { + let directory = historyFileURL.deletingLastPathComponent() + try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + guard let data = try? JSONEncoder().encode(entries) else { return } + try? data.write(to: historyFileURL, options: .atomic) + } + + func recordTopic(topic: String, bundleId: String?, skillId: String? = nil) { + let topicTokens = SkillMatcher.meaningfulTokens(topic) + guard !topicTokens.isEmpty else { return } + + entries.append( + TeachingTopicHistoryEntry( + topicTokens: topicTokens, + bundleId: bundleId, + timestamp: Date(), + skillId: skillId + ) + ) + + if entries.count > maxEntryCount { + entries.removeFirst(entries.count - maxEntryCount) + } + save() + } + + func hasRepeatedTopic( + topic: String, + bundleId: String?, + withinDays: Int = 7, + now: Date = Date() + ) -> Bool { + let topicTokens = Set(SkillMatcher.meaningfulTokens(topic)) + guard topicTokens.count >= 1 else { return false } + + let cutoff = Calendar.current.date(byAdding: .day, value: -withinDays, to: now) ?? .distantPast + let matchingEntries = entries.filter { entry in + entry.timestamp >= cutoff && + bundleIdsMatch(entry.bundleId, bundleId) && + tokenOverlapCount(Set(entry.topicTokens), topicTokens) >= 2 + } + + return matchingEntries.count >= 2 + } + + private func bundleIdsMatch(_ lhs: String?, _ rhs: String?) -> Bool { + switch (lhs, rhs) { + case (nil, nil): + return true + case (let left?, let right?): + return left == right + case (nil, _), (_, nil): + return true + } + } + + private func tokenOverlapCount(_ lhs: Set, _ rhs: Set) -> Int { + lhs.intersection(rhs).count + } +} diff --git a/leanring-buddy/WindowPositionManager.swift b/leanring-buddy/WindowPositionManager.swift index 8f6edf650..d283cb527 100644 --- a/leanring-buddy/WindowPositionManager.swift +++ b/leanring-buddy/WindowPositionManager.swift @@ -8,8 +8,7 @@ import AppKit import ApplicationServices -import ScreenCaptureKit - +import CoreGraphics enum PermissionRequestPresentationDestination: Equatable { case alreadyGranted case systemPrompt @@ -19,6 +18,7 @@ enum PermissionRequestPresentationDestination: Equatable { @MainActor class WindowPositionManager { private static var hasAttemptedAccessibilitySystemPromptDuringCurrentLaunch = false + private static var hasAttemptedInputMonitoringSystemPromptDuringCurrentLaunch = false private static var hasAttemptedScreenRecordingSystemPromptDuringCurrentLaunch = false private static let hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey = "com.learningbuddy.hasPreviouslyConfirmedScreenRecordingPermission" @@ -71,15 +71,78 @@ class WindowPositionManager { static func revealAppInFinder() { guard let appURL = Bundle.main.bundleURL as URL? else { return } NSWorkspace.shared.activateFileViewerSelecting([appURL]) + print("🔑 Finder opened this build — drag it into Accessibility / Input Monitoring: \(appURL.path)") + } + + /// Opens Finder + Settings so the user can drag the current debug build in. + static func prepareAccessibilityReGrantFromFinder() { + hasAttemptedAccessibilitySystemPromptDuringCurrentLaunch = false + revealAppInFinder() + openAccessibilitySettings() + } + + /// Opens Finder + Settings so the user can drag the current debug build in. + static func prepareInputMonitoringReGrantFromFinder() { + hasAttemptedInputMonitoringSystemPromptDuringCurrentLaunch = false + revealAppInFinder() + openInputMonitoringSettings() + } + + /// Re-checks accessibility trust without showing the prompt. + static func refreshAccessibilityTrustCache() { + let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue(): false] as CFDictionary + _ = AXIsProcessTrustedWithOptions(options) + } + + // MARK: - Input Monitoring Permission + + /// Returns true if the app can install the global CGEvent tap used for push-to-talk. + /// `nonisolated` because it only reads the live TCC state via a pure C preflight call + /// (no main-actor state), so the nonisolated push-to-talk monitor can call it directly. + /// Without this, Xcode 16.4 rejects the call from `GlobalPushToTalkShortcutMonitor.start()`. + nonisolated static func hasInputMonitoringPermission() -> Bool { + CGPreflightListenEventAccess() + } + + /// Presents the system Input Monitoring prompt once, then opens System Settings. + @discardableResult + static func requestInputMonitoringPermission() -> PermissionRequestPresentationDestination { + let presentationDestination = permissionRequestPresentationDestination( + hasPermissionNow: hasInputMonitoringPermission(), + hasAttemptedSystemPrompt: hasAttemptedInputMonitoringSystemPromptDuringCurrentLaunch + ) + + switch presentationDestination { + case .alreadyGranted: + return .alreadyGranted + case .systemPrompt: + hasAttemptedInputMonitoringSystemPromptDuringCurrentLaunch = true + _ = CGRequestListenEventAccess() + case .systemSettings: + openInputMonitoringSettings() + } + + return presentationDestination + } + + /// Opens System Settings to the Input Monitoring pane. + static func openInputMonitoringSettings() { + guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent") else { return } + NSWorkspace.shared.open(url) } // MARK: - Screen Recording Permission + /// Live TCC check — does not use cached UserDefaults fallbacks. + static func isScreenCapturePreflightGranted() -> Bool { + CGPreflightScreenCaptureAccess() + } + /// Returns true if Screen Recording permission is granted. static func hasScreenRecordingPermission() -> Bool { - let hasScreenRecordingPermissionNow = CGPreflightScreenCaptureAccess() + let hasScreenRecordingPermissionNow = isScreenCapturePreflightGranted() if hasScreenRecordingPermissionNow { - UserDefaults.standard.set(true, forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) + ClickyDefaults.shared.set(true, forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) } return hasScreenRecordingPermissionNow } @@ -91,7 +154,7 @@ class WindowPositionManager { static func shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch() -> Bool { shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch( hasScreenRecordingPermissionNow: hasScreenRecordingPermission(), - hasPreviouslyConfirmedScreenRecordingPermission: UserDefaults.standard.bool(forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) + hasPreviouslyConfirmedScreenRecordingPermission: ClickyDefaults.shared.bool(forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) ) } @@ -103,7 +166,19 @@ class WindowPositionManager { } static func clearPreviouslyConfirmedScreenRecordingPermission() { - UserDefaults.standard.removeObject(forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) + ClickyDefaults.shared.removeObject(forKey: hasPreviouslyConfirmedScreenRecordingPermissionUserDefaultsKey) + } + + static func clearCachedScreenContentPermission() { + ClickyDefaults.shared.removeObject(forKey: "hasScreenContentPermission") + } + + /// One-time startup diagnostics when permissions are incomplete. + static func logPermissionDiagnosticsSnapshot() { + let bundlePath = Bundle.main.bundlePath + print("🔑 Permission diagnostics — enable THIS build in System Settings:") + print("🔑 \(bundlePath)") + print("🔑 TCC snapshot — accessibility: \(hasAccessibilityPermission()), inputMonitoring: \(hasInputMonitoringPermission()), screenRecordingPreflight: \(CGPreflightScreenCaptureAccess())") } /// Prompts the system dialog for Screen Recording permission. diff --git a/leanring-buddy/leanring_buddyApp.swift b/leanring-buddy/leanring_buddyApp.swift index b004a8960..db01542f5 100644 --- a/leanring-buddy/leanring_buddyApp.swift +++ b/leanring-buddy/leanring_buddyApp.swift @@ -37,24 +37,40 @@ final class CompanionAppDelegate: NSObject, NSApplicationDelegate { print("🎯 Clicky: Starting...") print("🎯 Clicky: Version \(Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "unknown")") - UserDefaults.standard.register(defaults: ["NSInitialToolTipDelay": 0]) + ClickyDefaults.shared.register(defaults: ["NSInitialToolTipDelay": 0]) ClickyAnalytics.configure() ClickyAnalytics.trackAppOpened() + ClickyE2EConfiguration.applyLaunchOverrides() + MacOSScreenshotFloatingThumbnailSuppression.enableForAppLifetime() menuBarPanelManager = MenuBarPanelManager(companionManager: companionManager) companionManager.start() + + companionManager.runE2EInjectSequenceIfNeeded() + companionManager.runE2EBootstrapActionsIfNeeded() // Auto-open the panel if the user still needs to do something: // either they haven't onboarded yet, or permissions were revoked. - if !companionManager.hasCompletedOnboarding || !companionManager.allPermissionsGranted { + // Skip during headless E2E — the ad-hoc E2E build has a different code + // signature than the Xcode Debug app, so TCC checks (accessibility, etc.) + // will look "denied" even when the dev build is already approved. + if !ClickyE2EConfiguration.isEnabled, + !companionManager.hasCompletedOnboarding || !companionManager.allPermissionsGranted { menuBarPanelManager?.showPanelOnLaunch() } registerAsLoginItemIfNeeded() // startSparkleUpdater() } + func applicationDidBecomeActive(_ notification: Notification) { + // Re-check TCC after the user returns from System Settings. + companionManager.refreshAllPermissions() + companionManager.schedulePermissionRefreshBurstAfterReturningFromSettings() + } + func applicationWillTerminate(_ notification: Notification) { companionManager.stop() + MacOSScreenshotFloatingThumbnailSuppression.restoreAfterAppTermination() } /// Registers the app as a login item so it launches automatically on diff --git a/leanring-buddyTests/ClickyPathsTests.swift b/leanring-buddyTests/ClickyPathsTests.swift new file mode 100644 index 000000000..2bb63ea17 --- /dev/null +++ b/leanring-buddyTests/ClickyPathsTests.swift @@ -0,0 +1,120 @@ +// +// ClickyPathsTests.swift +// leanring-buddyTests +// + +import Foundation +import Testing +@testable import leanring_buddy + +// Serialized because these tests mutate the shared `ClickyPaths.overrideHomeForTesting` +// global. Without this, Swift Testing's default parallel execution lets them clobber +// each other's override and produces flaky failures. +@Suite(.serialized) +struct ClickyPathsTests { + @Test func defaultHomeUsesClickyDirectoryInUserHome() { + ClickyTestHomeIsolation.withSerializedHomeAccess { + ClickyPaths.overrideHomeForTesting = nil + ClickyPaths.ignoreConfiguredHomeForTesting = true + defer { ClickyPaths.ignoreConfiguredHomeForTesting = false } + + let expectedHome = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent(".clicky", isDirectory: true) + + #expect(ClickyPaths.home == expectedHome) + } + } + + @Test func clickyHomeEnvironmentVariableRedirectsHome() { + ClickyTestHomeIsolation.withSerializedHomeAccess { + ClickyPaths.overrideHomeForTesting = nil + ClickyPaths.ignoreConfiguredHomeForTesting = false + + let configuredHomePath = ProcessInfo.processInfo.environment["CLICKY_HOME"] + let clickyHomeLaunchArgument = ClickyLaunchArguments.value(forPrefix: "-CLICKY_HOME=") + + if let clickyHomeLaunchArgument, !clickyHomeLaunchArgument.isEmpty { + let expectedHome = URL(fileURLWithPath: clickyHomeLaunchArgument, isDirectory: true) + #expect(ClickyPaths.home == expectedHome) + return + } + + if let configuredHomePath, !configuredHomePath.isEmpty { + let expectedHome = URL(fileURLWithPath: configuredHomePath, isDirectory: true) + #expect(ClickyPaths.home == expectedHome) + return + } + + // No worktree isolation configured in this test run. + } + } + + @Test func overrideHomeRedirectsSkillsAndTopicHistory() throws { + let temporaryHome = FileManager.default.temporaryDirectory + .appendingPathComponent("clicky-test-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: temporaryHome) } + + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + + #expect(ClickyPaths.home == temporaryHome) + #expect(ClickyPaths.skills == temporaryHome.appendingPathComponent("skills", isDirectory: true)) + #expect(ClickyPaths.sessions == temporaryHome.appendingPathComponent("sessions", isDirectory: true)) + #expect(ClickyPaths.topicHistory == temporaryHome.appendingPathComponent("topic-history.json")) + } + } + + @Test func teachingSkillStoreUsesClickyPathsSkillsDirectory() throws { + let temporaryHome = FileManager.default.temporaryDirectory + .appendingPathComponent("clicky-test-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: temporaryHome) } + + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + + #expect(TeachingSkillStore.skillsRootURL == ClickyPaths.skills) + } + } + + @Test func teachingTopicHistoryStoreUsesClickyPathsTopicHistoryFile() throws { + let temporaryHome = FileManager.default.temporaryDirectory + .appendingPathComponent("clicky-test-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: temporaryHome) } + + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + + #expect(TeachingTopicHistoryStore.historyFileURL == ClickyPaths.topicHistory) + } + } + + @Test func teachingSkillStoreWritesToIsolatedHome() throws { + let temporaryHome = FileManager.default.temporaryDirectory + .appendingPathComponent("clicky-test-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: temporaryHome) } + + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + + let skill = TeachingSkill( + id: "teach-textedit-save", + name: "teach-textedit-save", + description: "Save in TextEdit", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: nil, + usageCount: 0, + isPinned: false, + taskSlug: nil, + body: "step one: press command-s." + ) + + let store = TeachingSkillStore() + _ = try store.saveSkill(skill) + + let skillFile = temporaryHome + .appendingPathComponent("skills/teach-textedit-save/SKILL.md") + #expect(FileManager.default.fileExists(atPath: skillFile.path)) + } + } +} diff --git a/leanring-buddyTests/ClickyTestHomeIsolation.swift b/leanring-buddyTests/ClickyTestHomeIsolation.swift new file mode 100644 index 000000000..8ff5a3c1d --- /dev/null +++ b/leanring-buddyTests/ClickyTestHomeIsolation.swift @@ -0,0 +1,45 @@ +// +// ClickyTestHomeIsolation.swift +// leanring-buddyTests +// +// Serializes access to the process-wide `ClickyPaths.overrideHomeForTesting` +// global across ALL test suites. `@Suite(.serialized)` only serializes tests +// within a single suite, but Swift Testing still runs separate suites in +// parallel in-process. Without a shared lock, one suite can reset the override +// while another suite is mid-test (e.g. between `save` and `loadAllSessions`), +// causing the store to resolve a different home directory than it wrote to. +// + +import Foundation +@testable import leanring_buddy + +enum ClickyTestHomeIsolation { + private static let lock = NSLock() + + /// Runs `body` with `ClickyPaths.overrideHomeForTesting` set to `home`, + /// holding a process-wide lock so no other suite can mutate the global + /// for the duration. The override is always cleared afterwards. + static func withIsolatedHome( + _ home: URL, + _ body: () throws -> ResultType + ) rethrows -> ResultType { + lock.lock() + defer { lock.unlock() } + + let previousOverride = ClickyPaths.overrideHomeForTesting + ClickyPaths.overrideHomeForTesting = home + defer { ClickyPaths.overrideHomeForTesting = previousOverride } + + return try body() + } + + /// Runs `body` while holding the same lock without setting an override, + /// for tests that assert the production default home resolution. + static func withSerializedHomeAccess( + _ body: () throws -> ResultType + ) rethrows -> ResultType { + lock.lock() + defer { lock.unlock() } + return try body() + } +} diff --git a/leanring-buddyTests/SessionStoreTests.swift b/leanring-buddyTests/SessionStoreTests.swift new file mode 100644 index 000000000..8a30b9ada --- /dev/null +++ b/leanring-buddyTests/SessionStoreTests.swift @@ -0,0 +1,164 @@ +// +// SessionStoreTests.swift +// leanring-buddyTests +// + +import Foundation +import Testing +@testable import leanring_buddy + +@Suite(.serialized) +struct SessionStoreTests { + private func makeTemporaryHome() -> URL { + FileManager.default.temporaryDirectory + .appendingPathComponent("clicky-session-test-\(UUID().uuidString)", isDirectory: true) + } + + private func dayFolderFormatter() -> DateFormatter { + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.dateFormat = "yyyy-MM-dd" + return formatter + } + + private func sampleSession( + sessionId: UUID = UUID(), + startedAt: Date = Date(timeIntervalSince1970: 1_747_000_000), + endedAt: Date = Date(timeIntervalSince1970: 1_747_000_120) + ) -> PersistedSession { + PersistedSession( + sessionId: sessionId, + startedAt: startedAt, + endedAt: endedAt, + outcome: .success, + privacyOptOut: false, + appsUsed: ["com.apple.TextEdit", "com.apple.Safari"], + turns: [ + SessionTraceEntry( + timestamp: startedAt, + userTranscript: "how do i save in textedit?", + assistantResponse: "press command-s to save.", + bundleId: "com.apple.TextEdit", + pointed: true + ), + SessionTraceEntry( + timestamp: endedAt, + userTranscript: "got it", + assistantResponse: "nice.", + bundleId: "com.apple.TextEdit", + pointed: false + ) + ] + ) + } + + @Test func sessionStoreRoundTripPreservesAllFields() throws { + let temporaryHome = makeTemporaryHome() + defer { try? FileManager.default.removeItem(at: temporaryHome) } + + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + + let session = sampleSession() + let store = SessionStore() + let savedURL = try store.save(session) + + let fileData = try Data(contentsOf: savedURL) + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + let decodedSession = try decoder.decode(PersistedSession.self, from: fileData) + + #expect(decodedSession == session) + #expect(store.loadAllSessions() == [session]) + } + } + + @Test func sessionStoreWritesExpectedPathShape() throws { + let temporaryHome = makeTemporaryHome() + defer { try? FileManager.default.removeItem(at: temporaryHome) } + + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + + let sessionId = UUID() + let startedAt = Date(timeIntervalSince1970: 1_747_000_000) + let session = sampleSession(sessionId: sessionId, startedAt: startedAt) + let store = SessionStore() + let savedURL = try store.save(session) + + let expectedDayFolder = dayFolderFormatter().string(from: startedAt) + let expectedURL = temporaryHome + .appendingPathComponent("sessions", isDirectory: true) + .appendingPathComponent(expectedDayFolder, isDirectory: true) + .appendingPathComponent("\(sessionId.uuidString).json") + + #expect(savedURL == expectedURL) + #expect(FileManager.default.fileExists(atPath: expectedURL.path)) + } + } + + @Test func sessionStoreWritesISO8601DatesAndStringOutcome() throws { + let temporaryHome = makeTemporaryHome() + defer { try? FileManager.default.removeItem(at: temporaryHome) } + + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + + let session = sampleSession() + let store = SessionStore() + let savedURL = try store.save(session) + + let jsonObject = try JSONSerialization.jsonObject(with: Data(contentsOf: savedURL)) as? [String: Any] + #expect(jsonObject != nil) + #expect(jsonObject?["outcome"] as? String == "success") + #expect(jsonObject?["startedAt"] is String) + #expect(jsonObject?["endedAt"] is String) + #expect((jsonObject?["startedAt"] as? String)?.contains("T") == true) + } + } + + @Test func sessionStoreDeletesSessionsOlderThanSevenDays() throws { + let temporaryHome = makeTemporaryHome() + defer { try? FileManager.default.removeItem(at: temporaryHome) } + + try ClickyTestHomeIsolation.withIsolatedHome(temporaryHome) { + try FileManager.default.createDirectory(at: temporaryHome, withIntermediateDirectories: true) + + let formatter = dayFolderFormatter() + let now = Date() + let eightDaysAgo = Calendar.current.date(byAdding: .day, value: -8, to: now)! + let oldDayFolder = ClickyPaths.sessions.appendingPathComponent(formatter.string(from: eightDaysAgo), isDirectory: true) + let todayFolder = ClickyPaths.sessions.appendingPathComponent(formatter.string(from: now), isDirectory: true) + + try FileManager.default.createDirectory(at: oldDayFolder, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: todayFolder, withIntermediateDirectories: true) + try Data("{}".utf8).write(to: oldDayFolder.appendingPathComponent("old.json")) + try Data("{}".utf8).write(to: todayFolder.appendingPathComponent("recent.json")) + + SessionStore().deleteSessionsOlderThan(days: 7, now: now) + + #expect(FileManager.default.fileExists(atPath: oldDayFolder.path) == false) + #expect(FileManager.default.fileExists(atPath: todayFolder.path)) + } + } + + @Test func sessionTraceEntryIsCodable() throws { + let entry = SessionTraceEntry( + timestamp: Date(timeIntervalSince1970: 1_747_000_000), + userTranscript: "hello", + assistantResponse: "hi", + bundleId: "com.apple.TextEdit", + pointed: true + ) + + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + + let encodedData = try encoder.encode(entry) + let decodedEntry = try decoder.decode(SessionTraceEntry.self, from: encodedData) + + #expect(decodedEntry == entry) + } +} diff --git a/leanring-buddyTests/TeachingSkillTests.swift b/leanring-buddyTests/TeachingSkillTests.swift new file mode 100644 index 000000000..85c4aaf37 --- /dev/null +++ b/leanring-buddyTests/TeachingSkillTests.swift @@ -0,0 +1,425 @@ +// +// TeachingSkillTests.swift +// leanring-buddyTests +// + +import Foundation +import Testing +@testable import leanring_buddy + +struct TeachingSkillTests { + @Test func parsesSkillFrontmatterAndBody() throws { + let markdown = """ + --- + name: teach-xcode-source-control + description: Walk through committing in Xcode + bundleIds: + - com.apple.dt.Xcode + status: active + lastUsed: 2026-05-24 + usageCount: 3 + pinned: true + taskSlug: commit + --- + + step one: open source control. + step two: click commit. + """ + + let skill = try #require(TeachingSkill.parse(id: "teach-xcode-source-control", markdown: markdown)) + #expect(skill.name == "teach-xcode-source-control") + #expect(skill.description == "Walk through committing in Xcode") + #expect(skill.bundleIds == ["com.apple.dt.Xcode"]) + #expect(skill.usageCount == 3) + #expect(skill.isPinned) + #expect(skill.taskSlug == "commit") + #expect(skill.body.contains("step one")) + } + + @Test func matchesSkillsByBundleAndKeywords() { + let skill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 2, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s" + ) + + let matches = SkillMatcher.matchSkills( + from: [skill], + bundleId: "com.apple.TextEdit", + transcript: "how do I save this document?" + ) + + #expect(matches.count == 1) + #expect(matches.first?.skill.id == "teach-textedit-save") + } + + @Test func triggerFiresOnUserConfirmation() { + let trace = [ + SessionTraceEntry( + timestamp: Date(), + userTranscript: "how do I save this?", + assistantResponse: "click file then save", + bundleId: "com.apple.TextEdit", + pointed: true + ) + ] + + let trigger = SkillTriggerEvaluator.shouldWriteSkill( + sessionTrace: trace, + latestTranscript: "got it thanks that worked" + ) + + #expect(trigger?.reason == .userConfirmed) + } + + @Test func doesNotWriteBeforeUserConfirmation() { + let trace = [ + SessionTraceEntry( + timestamp: Date(), + userTranscript: "how do I save this document?", + assistantResponse: "click file then save", + bundleId: "com.apple.TextEdit", + pointed: true + ), + SessionTraceEntry( + timestamp: Date(), + userTranscript: "where is the save button?", + assistantResponse: "pointing at file menu", + bundleId: "com.apple.TextEdit", + pointed: true + ) + ] + + let trigger = SkillTriggerEvaluator.shouldWriteSkill( + sessionTrace: trace, + latestTranscript: "where is the save button?" + ) + + #expect(trigger == nil) + } + + @Test func topicIgnoresConfirmationPhrase() { + let trace = [ + SessionTraceEntry( + timestamp: Date(), + userTranscript: "how do I save this document?", + assistantResponse: "click file then save", + bundleId: "com.apple.TextEdit", + pointed: true + ), + SessionTraceEntry( + timestamp: Date(), + userTranscript: "got it thanks that worked", + assistantResponse: "great", + bundleId: "com.apple.TextEdit", + pointed: false + ) + ] + + let topic = SkillTriggerEvaluator.deriveTopic(from: trace) + #expect(topic == "save document") + #expect(SkillTriggerEvaluator.primaryTeachingQuestion(from: trace) == "how do I save this document?") + } + + @Test func slugAndNameAreCleanForSaveQuestionWithConfirmation() throws { + let trace = [ + SessionTraceEntry( + timestamp: Date(), + userTranscript: "how do I save this document?", + assistantResponse: "click file then save", + bundleId: "com.apple.TextEdit", + pointed: true + ), + SessionTraceEntry( + timestamp: Date(), + userTranscript: "got it thanks that worked", + assistantResponse: "great", + bundleId: "com.apple.TextEdit", + pointed: false + ) + ] + + let trigger = SkillTriggerEvaluator.shouldWriteSkill( + sessionTrace: trace, + latestTranscript: "got it thanks that worked" + ) + let metadata = SkillSynthesizer.buildSkillMetadata( + sessionTrace: trace, + trigger: try #require(trigger), + targetBundleId: "com.apple.TextEdit" + ) + + #expect(metadata.id == "teach-textedit-save") + #expect(metadata.name == "Save in TextEdit") + #expect(metadata.description == "Walk the user through save document") + #expect(metadata.taskSlug == "save") + #expect(!metadata.id.contains("got")) + #expect(!metadata.id.contains("thanks")) + #expect(!metadata.id.contains("worked")) + } + + @Test func crossSessionRepeatDoesNotAutoWriteWithoutConfirmation() throws { + let tempHistoryURL = FileManager.default.temporaryDirectory + .appendingPathComponent("clicky-topic-history-test-\(UUID().uuidString).json") + defer { try? FileManager.default.removeItem(at: tempHistoryURL) } + + let topicHistoryStore = TeachingTopicHistoryStore(historyFileURL: tempHistoryURL) + topicHistoryStore.load() + + let bundleId = "com.apple.TextEdit" + let topic = "save document" + + topicHistoryStore.recordTopic(topic: topic, bundleId: bundleId) + + let trace = [ + SessionTraceEntry( + timestamp: Date(), + userTranscript: "how do I save this document?", + assistantResponse: "click file then save", + bundleId: bundleId, + pointed: true + ) + ] + + topicHistoryStore.recordTopic(topic: topic, bundleId: bundleId) + + let trigger = SkillTriggerEvaluator.shouldWriteSkill( + sessionTrace: trace, + latestTranscript: "how do I save this document?", + topicHistory: topicHistoryStore.entries + ) + + #expect(trigger == nil) + } + + @Test func resolvesTargetAppFromMentionedAppNotFrontmostBundle() { + let trace = [ + SessionTraceEntry( + timestamp: Date(), + userTranscript: "how do I save this document in TextEdit?", + assistantResponse: "use command s", + bundleId: "com.mitchellh.ghostty", + pointed: true + ) + ] + + let targetBundleId = SkillTargetAppResolver.resolveTargetBundleId( + from: trace, + frontmostBundleId: "com.mitchellh.ghostty" + ) + + #expect(targetBundleId == "com.apple.TextEdit") + } + + @Test func findSkillForUpdateUsesStableIdentity() { + let existingSkill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 2, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s" + ) + + let matchedSkill = SkillMatcher.findSkillForUpdate( + in: [existingSkill], + targetBundleId: "com.apple.TextEdit", + primaryQuestion: "how do I save this document?" + ) + + #expect(matchedSkill?.id == "teach-textedit-save") + } + + @Test func findSkillForUpdateMatchesRefinementToExistingSkill() { + let existingSkill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 2, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s" + ) + + let matchedSkill = SkillMatcher.findSkillForUpdate( + in: [existingSkill], + targetBundleId: "com.mitchellh.ghostty", + primaryQuestion: "how do I save this document in TextEdit?" + ) + + #expect(matchedSkill?.id == "teach-textedit-save") + } + + @Test func promptBuilderInjectsMatchedSkills() { + let skill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: nil, + usageCount: 0, + isPinned: false, + taskSlug: "save", + body: "use file > save" + ) + + let prompt = TeachingPromptBuilder.buildVoiceResponsePrompt( + basePrompt: "base prompt", + matchedSkills: [skill] + ) + + #expect(prompt.contains("base prompt")) + #expect(prompt.contains("teaching skills:")) + #expect(prompt.contains("Save in TextEdit")) + #expect(prompt.contains("use file > save")) + } + + @Test func matchedSkillAppearsInComposedPrompt() { + let skill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: nil, + usageCount: 0, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s" + ) + + let matches = SkillMatcher.matchSkills( + from: [skill], + bundleId: "com.apple.TextEdit", + transcript: "how do I save this document?" + ) + let matchedSkills = matches.map(\.skill) + let prompt = TeachingPromptBuilder.buildVoiceResponsePrompt( + basePrompt: "companion voice prompt", + matchedSkills: matchedSkills + ) + + #expect(matchedSkills.map(\.name) == ["Save in TextEdit"]) + #expect(prompt.contains("Save in TextEdit")) + #expect(prompt.contains("click file then save or use command s")) + } + + @Test func detectsDuplicateSkillsWithOverlappingContent() { + let saveSkill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 3, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s shortcut" + ) + let duplicateSaveSkill = TeachingSkill( + id: "teach-textedit-save-document", + name: "Save document in TextEdit", + description: "Help save the current document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date().addingTimeInterval(-86400), + usageCount: 1, + isPinned: false, + taskSlug: "save", + body: "open file menu then choose save for the document" + ) + + let duplicatePairs = SkillMatcher.findDuplicateSkillPairs( + in: [saveSkill, duplicateSaveSkill], + minimumOverlapScore: 3 + ) + + #expect(duplicatePairs.count == 1) + #expect(duplicatePairs.first?.primarySkill.id == "teach-textedit-save") + #expect(duplicatePairs.first?.duplicateSkill.id == "teach-textedit-save-document") + } + + @Test func ignoresPinnedSkillsForDuplicateDetection() { + let pinnedSkill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 3, + isPinned: true, + taskSlug: "save", + body: "click file then save or use command s shortcut" + ) + let overlappingSkill = TeachingSkill( + id: "teach-textedit-save-copy", + name: "Save document in TextEdit", + description: "Help save the current document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 1, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s shortcut" + ) + + let duplicatePairs = SkillMatcher.findDuplicateSkillPairs( + in: [pinnedSkill, overlappingSkill], + minimumOverlapScore: 3 + ) + + #expect(duplicatePairs.isEmpty) + } + + @Test func ignoresSkillsWithDifferentBundleIdsForDuplicateDetection() { + let textEditSkill = TeachingSkill( + id: "teach-textedit-save", + name: "Save in TextEdit", + description: "Walk the user through saving a document", + bundleIds: ["com.apple.TextEdit"], + status: .active, + lastUsed: Date(), + usageCount: 3, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s shortcut" + ) + let xcodeSkill = TeachingSkill( + id: "teach-xcode-save", + name: "Save in Xcode", + description: "Walk the user through saving a file", + bundleIds: ["com.apple.dt.Xcode"], + status: .active, + lastUsed: Date(), + usageCount: 1, + isPinned: false, + taskSlug: "save", + body: "click file then save or use command s shortcut" + ) + + let duplicatePairs = SkillMatcher.findDuplicateSkillPairs( + in: [textEditSkill, xcodeSkill], + minimumOverlapScore: 3 + ) + + #expect(duplicatePairs.isEmpty) + } +} diff --git a/leanring-buddyTests/leanring_buddyTests.swift b/leanring-buddyTests/leanring_buddyTests.swift index 188fe7ae0..73ff6f0b7 100644 --- a/leanring-buddyTests/leanring_buddyTests.swift +++ b/leanring-buddyTests/leanring_buddyTests.swift @@ -8,6 +8,7 @@ import Testing @testable import leanring_buddy +@MainActor struct leanring_buddyTests { @Test func firstPermissionRequestUsesSystemPromptOnly() async throws { diff --git a/scripts/README.md b/scripts/README.md index 9b2f0b348..9554e0bf0 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,4 +1,25 @@ -# Release Scripts +# Scripts + +## `new-worktree.sh` — Isolated parallel development + +Creates a git worktree with its own Clicky data home, UserDefaults suite, and +(usually) disabled global push-to-talk so multiple agents/branches can run side +by side without clobbering `~/.clicky` or fighting over Control+Option. + +```bash +# Secondary worktree (no global hotkey) +./scripts/new-worktree.sh feature/memory-routines + +# Primary dev worktree (keeps push-to-talk) +./scripts/new-worktree.sh feature/teaching-skills-upstream --primary +``` + +Each worktree patches **only its local** Xcode scheme with: +- `CLICKY_HOME=/tmp/clicky-` +- `-CLICKY_DEFAULTS_SUITE=clicky-` +- `-CLICKY_DISABLE_GLOBAL_PTT=1` (unless `--primary`) + +--- ## `release.sh` — Ship a new version of makesomething diff --git a/scripts/new-worktree.sh b/scripts/new-worktree.sh new file mode 100755 index 000000000..3f1e1cea4 --- /dev/null +++ b/scripts/new-worktree.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Create an isolated git worktree for parallel Clicky development. +# +# Usage: +# ./scripts/new-worktree.sh [--primary] [--path ] +# +# Examples: +# ./scripts/new-worktree.sh feature/memory-routines +# ./scripts/new-worktree.sh memory-routines --primary +# ./scripts/new-worktree.sh feature/skills-v2 --path ../clicky-skills-v2 +# +# Each worktree gets its own data home, UserDefaults suite, and (by default) +# disables the global push-to-talk hotkey so multiple instances do not conflict. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BASE_BRANCH="${CLICKY_WORKTREE_BASE:-origin/feature/teaching-skills-upstream}" + +usage() { + sed -n '3,12p' "$0" + exit 1 +} + +BRANCH_NAME="" +WORKTREE_PATH="" +IS_PRIMARY="0" + +while [[ $# -gt 0 ]]; do + case "$1" in + --primary) + IS_PRIMARY="1" + shift + ;; + --path) + WORKTREE_PATH="${2:-}" + [[ -n "$WORKTREE_PATH" ]] || usage + shift 2 + ;; + -h|--help) + usage + ;; + *) + if [[ -z "$BRANCH_NAME" ]]; then + BRANCH_NAME="$1" + shift + else + echo "Unknown argument: $1" >&2 + usage + fi + ;; + esac +done + +[[ -n "$BRANCH_NAME" ]] || usage + +if [[ "$BRANCH_NAME" != */* ]]; then + BRANCH_NAME="feature/$BRANCH_NAME" +fi + +slug="${BRANCH_NAME#feature/}" +slug="${slug//\//-}" +slug="$(echo "$slug" | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]-')" + +CLICKY_HOME="/tmp/clicky-${slug}" +CLICKY_DEFAULTS_SUITE="clicky-${slug}" + +if [[ -z "$WORKTREE_PATH" ]]; then + WORKTREE_PATH="$(dirname "$ROOT_DIR")/clicky-${slug}" +fi + +if [[ ! "$WORKTREE_PATH" = /* ]]; then + WORKTREE_PATH="$(cd "$ROOT_DIR" && cd "$(dirname "$WORKTREE_PATH")" && pwd)/$(basename "$WORKTREE_PATH")" +fi + +echo "Fetching latest $BASE_BRANCH..." +git -C "$ROOT_DIR" fetch origin --prune + +if git -C "$ROOT_DIR" show-ref --verify --quiet "refs/heads/$BRANCH_NAME"; then + echo "Branch already exists locally: $BRANCH_NAME" + git -C "$ROOT_DIR" worktree add "$WORKTREE_PATH" "$BRANCH_NAME" +else + git -C "$ROOT_DIR" worktree add -b "$BRANCH_NAME" "$WORKTREE_PATH" "$BASE_BRANCH" +fi + +mkdir -p "$CLICKY_HOME/skills" + +cat >"$WORKTREE_PATH/.clicky-worktree.env" </dev/null || true + +cat <-save/SKILL.md +--- skill preview --- +... +PASS: skill slug is clean (teach--save) +Phase B: relaunch Clicky and verify saved skill is injected into prompt... +PASS: saved skill content found in composed system prompt +--- prompt preview --- +teaching skills: +... + +E2E PASS: Phase A (write) + Phase B (read-path) succeeded +``` + +Exit code `0` on success, non-zero on any failure. + +## What the test verifies + +### Phase A — Write path + +1. Builds Clicky unsigned (`CODE_SIGN_IDENTITY="-"`, `CODE_SIGNING_ALLOWED=NO`) +2. Starts `tests/e2e/mock-worker.mjs` on port 8787 +3. Clears `~/.clicky/skills/` and `~/.clicky/e2e-last-system-prompt.txt` +4. Launches Clicky with injected transcripts (question + confirmation) +5. Asserts within 30s: + - `~/.clicky/skills/*/SKILL.md` exists + - Skill slug contains `save` + - Slug does **not** contain `got`, `thanks`, or `worked` + +### Phase B — Read path + +1. Kills Clicky from Phase A (skills remain on disk) +2. Relaunches with `-CLICKY_INJECT_TRANSCRIPT_3="how do I save this document?"` +3. Asserts within 30s: + - `~/.clicky/e2e-last-system-prompt.txt` exists + - File contains `teaching skills:` + - File contains `save` (saved skill injected into system prompt) + +## CI + +GitHub Actions workflow: [`.github/workflows/e2e-teaching-skills.yml`](../../.github/workflows/e2e-teaching-skills.yml) + +- Triggers on `push` and `pull_request` to `main` +- Runner: `macos-14` +- Runs `./tests/e2e/run-all.sh` +- Uploads `/tmp/clicky-e2e-*.log` and `~/.clicky/e2e-last-system-prompt.txt` as artifacts on failure + +Optional CI badge (after the workflow has run on GitHub): + +```markdown +![E2E Teaching Skills](https://github.com///actions/workflows/e2e-teaching-skills.yml/badge.svg) +``` + +### Unit tests in CI + +The workflow attempts `xcodebuild build-for-testing` before E2E. The shared Xcode scheme currently does not include `leanring-buddyTests`, so this step is best-effort (`continue-on-error: true`) and does not block E2E. Unit tests live in `leanring-buddyTests/TeachingSkillTests.swift` and can be run from Xcode until the scheme is wired. + +## Troubleshooting + +Log files (overwritten each run): + +| Log | Contents | +|-----|----------| +| `/tmp/clicky-e2e-build.log` | Xcode build output | +| `/tmp/clicky-e2e-worker.log` | Mock worker stdout | +| `/tmp/clicky-e2e-app.log` | Clicky stdout during Phase A | +| `/tmp/clicky-e2e-app-read.log` | Clicky stdout during Phase B | + +Debug artifacts under `~/.clicky/`: + +| Path | Purpose | +|------|---------| +| `~/.clicky/skills//SKILL.md` | Saved teaching skill | +| `~/.clicky/e2e-last-system-prompt.txt` | Last composed system prompt (E2E mode) | + +Common failures: + +- **Build fails** — check `/tmp/clicky-e2e-build.log`; requires Xcode and macOS 14.2+ SDK +- **No skill written in 30s** — check `/tmp/clicky-e2e-app.log` and worker log; mock worker must be reachable at `127.0.0.1:8787` +- **Read path fails** — skill from Phase A must still exist under `~/.clicky/skills/`; check `/tmp/clicky-e2e-app-read.log` + +## E2E launch flags + +Defined in `leanring-buddy/ClickyE2EConfiguration.swift`: + +- `-CLICKY_E2E=1` — skip onboarding defaults, faster test startup +- `-CLICKY_WORKER_URL=` — point API calls at mock worker +- `-CLICKY_INJECT_TRANSCRIPT=` — first injected transcript (write path) +- `-CLICKY_INJECT_TRANSCRIPT_2=` — second injected transcript (confirmation) +- `-CLICKY_INJECT_TRANSCRIPT_3=` — read-path-only launch (skills already on disk) + +Writes `~/.clicky/e2e-last-system-prompt.txt` on each injected response for headless assertions. + +## Mock worker + +`tests/e2e/mock-worker.mjs` serves deterministic `/chat` and `/tts` responses on `http://127.0.0.1:8787`. Streaming responses include a `[POINT:...]` tag; non-streaming synthesis responses produce skill body content. + +```bash +node tests/e2e/mock-worker.mjs +``` + +## Manual inject (without full E2E script) + +Write path only: + +```bash +open build/E2E/Clicky.app --args \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL=http://127.0.0.1:8787 \ + -CLICKY_INJECT_TRANSCRIPT="how do I save this document?" \ + -CLICKY_INJECT_TRANSCRIPT_2="got it thanks that worked" +``` + +Read path only (after a skill exists): + +```bash +open build/E2E/Clicky.app --args \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL=http://127.0.0.1:8787 \ + -CLICKY_INJECT_TRANSCRIPT_3="how do I save this document?" + +grep "teaching skills:" ~/.clicky/e2e-last-system-prompt.txt +``` + +## Full user-perspective path (future) + +Not covered by the current headless E2E script: + +| Layer | Tool | Purpose | +|-------|------|---------| +| Permissions | `@guidepup/setup` or `tccutil` | Pre-grant Screen Recording, Mic, Accessibility | +| Push-to-talk | `NaryaAI/voice-testing-tools` | Simulate `ctrl+option` via HID events | +| Voice input | `tts.mjs` + BlackHole 2ch | Deterministic TTS into virtual mic | +| Screen/UI | Peekaboo or `axcli` | Launch apps, verify overlay/cursor | + +TCC permission seeding in CI is documented as a follow-up — it can be flaky on ephemeral runners. + +## Related unit tests + +`leanring-buddyTests/TeachingSkillTests.swift` covers topic extraction, slug generation, cross-session repeat detection, and prompt injection logic without launching the app. diff --git a/tests/e2e/lib/common.sh b/tests/e2e/lib/common.sh new file mode 100755 index 000000000..ce9329d19 --- /dev/null +++ b/tests/e2e/lib/common.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Shared helpers for Clicky headless E2E scripts. + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +CLICKY_APP="${CLICKY_APP:-$ROOT_DIR/build/E2E/Clicky.app}" +WORKER_URL="${CLICKY_WORKER_URL:-http://127.0.0.1:8787}" +CLICKY_DIR="${CLICKY_HOME:-$HOME/.clicky}" +SKILLS_DIR="$CLICKY_DIR/skills" +export CLICKY_HOME="$CLICKY_DIR" + +E2E_PROMPT_FILE="$CLICKY_DIR/e2e-last-system-prompt.txt" +E2E_MATCHED_SKILL_FILE="$CLICKY_DIR/e2e-last-matched-skill-id.txt" +E2E_SUGGESTIONS_FILE="$CLICKY_DIR/e2e-last-suggestions.txt" +E2E_SELECTED_NICHE_FILE="$CLICKY_DIR/e2e-selected-niche.txt" +E2E_SKILLS_COUNT_FILE="$CLICKY_DIR/e2e-skills-count.txt" +E2E_LIBRARY_STATE_FILE="$CLICKY_DIR/e2e-skill-library-state.txt" + +MOCK_WORKER_PID="" +CLICKY_PID="" + +e2e_cleanup() { + if [[ -n "$MOCK_WORKER_PID" ]]; then + kill "$MOCK_WORKER_PID" 2>/dev/null || true + MOCK_WORKER_PID="" + fi + if [[ -n "$CLICKY_PID" ]]; then + kill "$CLICKY_PID" 2>/dev/null || true + CLICKY_PID="" + fi +} + +ensure_clicky_built() { + if [[ "${SKIP_E2E_BUILD:-}" == "1" ]] && [[ -d "$CLICKY_APP" ]]; then + echo "Using existing Clicky build at $CLICKY_APP" + return 0 + fi + + echo "Building Clicky for E2E..." + mkdir -p "$ROOT_DIR/build/E2E" + xcodebuild \ + -project "$ROOT_DIR/leanring-buddy.xcodeproj" \ + -scheme leanring-buddy \ + -destination 'platform=macOS' \ + -derivedDataPath "$ROOT_DIR/build/E2E/DerivedData" \ + CODE_SIGN_IDENTITY="-" \ + CODE_SIGNING_ALLOWED=NO \ + build >/tmp/clicky-e2e-build.log 2>&1 + + local built_app="$ROOT_DIR/build/E2E/DerivedData/Build/Products/Debug/Clicky.app" + rm -rf "$CLICKY_APP" + ditto "$built_app" "$CLICKY_APP" + echo "Built Clicky at $CLICKY_APP" +} + +worker_port_from_url() { + local url="$1" + local port="8787" + + if [[ "$url" =~ :([0-9]+)(/|$|\?) ]]; then + port="${BASH_REMATCH[1]}" + fi + + echo "$port" +} + +start_mock_worker() { + local worker_port + worker_port="$(worker_port_from_url "$WORKER_URL")" + echo "Starting mock worker on $WORKER_URL (port $worker_port)..." + MOCK_WORKER_PORT="$worker_port" node "$ROOT_DIR/tests/e2e/mock-worker.mjs" >/tmp/clicky-e2e-worker.log 2>&1 & + MOCK_WORKER_PID=$! + sleep 1 +} + +launch_clicky() { + local log_file="${1:-/tmp/clicky-e2e-app.log}" + shift || true + + "$CLICKY_APP/Contents/MacOS/Clicky" -CLICKY_HOME="$CLICKY_DIR" "$@" >"$log_file" 2>&1 & + CLICKY_PID=$! +} + +wait_for_file() { + local file_path="$1" + local timeout_seconds="${2:-30}" + local description="${3:-$file_path}" + + for _ in $(seq 1 "$timeout_seconds"); do + if [[ -f "$file_path" ]]; then + return 0 + fi + sleep 1 + done + + echo "FAIL: timed out waiting for $description ($file_path)" + return 1 +} + +wait_for_file_content() { + local file_path="$1" + local pattern="$2" + local timeout_seconds="${3:-30}" + local description="${4:-$file_path}" + + for _ in $(seq 1 "$timeout_seconds"); do + if [[ -f "$file_path" ]] && grep -q "$pattern" "$file_path"; then + return 0 + fi + sleep 1 + done + + echo "FAIL: timed out waiting for '$pattern' in $description" + return 1 +} + +reset_e2e_artifacts() { + rm -f \ + "$E2E_PROMPT_FILE" \ + "$E2E_MATCHED_SKILL_FILE" \ + "$E2E_SUGGESTIONS_FILE" \ + "$E2E_SELECTED_NICHE_FILE" \ + "$E2E_SKILLS_COUNT_FILE" \ + "$E2E_LIBRARY_STATE_FILE" +} + +seed_teaching_skill() { + local skill_id="$1" + local skill_status="$2" + local skill_pinned="${3:-false}" + local skill_name="$4" + local skill_body="$5" + + mkdir -p "$SKILLS_DIR/$skill_id" + cat >"$SKILLS_DIR/$skill_id/SKILL.md" </dev/null || true + echo "--- worker log ---" + tail -20 /tmp/clicky-e2e-worker.log 2>/dev/null || true +} diff --git a/tests/e2e/mock-worker.mjs b/tests/e2e/mock-worker.mjs new file mode 100644 index 000000000..bebb0073e --- /dev/null +++ b/tests/e2e/mock-worker.mjs @@ -0,0 +1,115 @@ +#!/usr/bin/env node +import http from "node:http"; + +const PORT = Number(process.env.MOCK_WORKER_PORT ?? 8787); + +function jsonResponse(res, status, body) { + res.writeHead(status, { "content-type": "application/json" }); + res.end(JSON.stringify(body)); +} + +function streamVoiceResponse(res) { + const text = + "click file then save, or use command s. [POINT:120,40:file menu]"; + const chunks = text.match(/.{1,12}/g) ?? [text]; + + res.writeHead(200, { + "content-type": "text/event-stream", + "cache-control": "no-cache", + }); + + for (const chunk of chunks) { + const payload = { + type: "content_block_delta", + delta: { type: "text_delta", text: chunk }, + }; + res.write(`data: ${JSON.stringify(payload)}\n\n`); + } + res.write("data: [DONE]\n\n"); + res.end(); +} + +function synthesisResponse(res, parsed) { + const messages = parsed.messages ?? []; + const lastMessage = messages.at(-1) ?? {}; + const content = lastMessage.content; + const promptText = typeof content === "string" + ? content + : Array.isArray(content) + ? content.map((part) => part.text ?? "").join("\n") + : ""; + const isPatch = promptText.includes("patch this existing teaching skill"); + + const text = isPatch + ? [ + "workflow: save a document in textedit", + "preference: keyboard shortcut only, avoid the file menu", + "step one: press command s to save", + "pointing: do not point at the file menu unless the user asks for menu steps", + "completion: user says got it or thanks that worked", + ].join("\n") + : [ + "workflow: save a document in textedit", + "step one: click file in the menu bar", + "step two: choose save or press command s", + "pointing: start at the file menu near the top left", + "common mistake: looking for a floppy disk icon instead of file > save", + "completion: user says got it or thanks that worked", + ].join("\n"); + + jsonResponse(res, 200, { + content: [ + { + type: "text", + text, + }, + ], + }); +} + +const server = http.createServer(async (req, res) => { + if (req.method !== "POST") { + res.writeHead(405); + res.end("Method not allowed"); + return; + } + + const body = await new Promise((resolve, reject) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + req.on("error", reject); + }); + + if (req.url === "/tts") { + res.writeHead(200, { "content-type": "audio/mpeg" }); + res.end(Buffer.from([0xff, 0xfb, 0x90, 0x00])); + return; + } + + if (req.url === "/chat") { + let parsed = {}; + try { + parsed = JSON.parse(body); + } catch { + res.writeHead(400); + res.end("invalid json"); + return; + } + + if (parsed.stream === true) { + streamVoiceResponse(res); + return; + } + + synthesisResponse(res, parsed); + return; + } + + res.writeHead(404); + res.end("Not found"); +}); + +server.listen(PORT, "127.0.0.1", () => { + console.log(`mock worker listening on http://127.0.0.1:${PORT}`); +}); diff --git a/tests/e2e/run-all.sh b/tests/e2e/run-all.sh new file mode 100755 index 000000000..2ea9e6cff --- /dev/null +++ b/tests/e2e/run-all.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/common.sh +source "$SCRIPT_DIR/lib/common.sh" + +ensure_clicky_built +export SKIP_E2E_BUILD=1 + +HEADLESS_SCRIPTS=( + teaching-skills.sh + skills-library.sh +) + +echo "=== Clicky E2E run-all ===" +echo "" + +for headless_script in "${HEADLESS_SCRIPTS[@]}"; do + echo "----------------------------------------" + echo "Running $headless_script" + echo "----------------------------------------" + bash "$SCRIPT_DIR/$headless_script" + echo "" +done + +echo "=== E2E run-all PASS ===" diff --git a/tests/e2e/skills-library.sh b/tests/e2e/skills-library.sh new file mode 100755 index 000000000..9f62477f8 --- /dev/null +++ b/tests/e2e/skills-library.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/common.sh +source "$SCRIPT_DIR/lib/common.sh" + +ACTIVE_SKILL_ID="teach-e2e-active-test" +ARCHIVED_SKILL_ID="teach-e2e-archived-test" + +trap e2e_cleanup EXIT + +ensure_clicky_built +start_mock_worker + +echo "Seeding teaching skills for library E2E..." +rm -rf "$SKILLS_DIR" +mkdir -p "$SKILLS_DIR" +reset_e2e_artifacts + +seed_teaching_skill "$ACTIVE_SKILL_ID" "active" "false" "E2E Active Skill" "active skill body for e2e library test" +seed_teaching_skill "$ARCHIVED_SKILL_ID" "archived" "false" "E2E Archived Skill" "archived skill body for e2e library test" + +echo "Phase A: write skill library snapshot..." +launch_clicky /tmp/clicky-e2e-library-a.log \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL="$WORKER_URL" + +if ! wait_for_file "$E2E_LIBRARY_STATE_FILE" 30 "skill library state artifact"; then + print_failure_logs /tmp/clicky-e2e-library-a.log + exit 1 +fi + +if ! grep -q "\"id\" : \"$ACTIVE_SKILL_ID\"" "$E2E_LIBRARY_STATE_FILE"; then + echo "FAIL: library state missing active skill $ACTIVE_SKILL_ID" + cat "$E2E_LIBRARY_STATE_FILE" + exit 1 +fi + +if ! grep -q "\"id\" : \"$ARCHIVED_SKILL_ID\"" "$E2E_LIBRARY_STATE_FILE"; then + echo "FAIL: library state missing archived skill $ARCHIVED_SKILL_ID" + cat "$E2E_LIBRARY_STATE_FILE" + exit 1 +fi + +if ! grep -q "\"status\" : \"active\"" "$E2E_LIBRARY_STATE_FILE"; then + echo "FAIL: library state missing active status" + exit 1 +fi + +if ! grep -q "\"status\" : \"archived\"" "$E2E_LIBRARY_STATE_FILE"; then + echo "FAIL: library state missing archived status" + exit 1 +fi + +echo "PASS: library snapshot contains both seeded skills with correct statuses" +echo "--- library state preview ---" +head -20 "$E2E_LIBRARY_STATE_FILE" + +kill "$CLICKY_PID" 2>/dev/null || true +CLICKY_PID="" +sleep 1 + +echo "" +echo "Phase B: restore archived skill via E2E hook..." +rm -f "$E2E_LIBRARY_STATE_FILE" + +launch_clicky /tmp/clicky-e2e-library-b.log \ + -CLICKY_E2E=1 \ + -CLICKY_E2E_RESTORE_SKILL="$ARCHIVED_SKILL_ID" \ + -CLICKY_WORKER_URL="$WORKER_URL" + +if ! wait_for_file "$E2E_LIBRARY_STATE_FILE" 30 "updated library state artifact"; then + print_failure_logs /tmp/clicky-e2e-library-b.log + exit 1 +fi + +if ! python3 - <<'PY' "$E2E_LIBRARY_STATE_FILE" "$ARCHIVED_SKILL_ID" +import json, sys +path, skill_id = sys.argv[1], sys.argv[2] +entries = json.load(open(path)) +match = next((entry for entry in entries if entry.get("id") == skill_id), None) +if not match: + raise SystemExit(f"missing skill {skill_id}") +if match.get("status") != "active": + raise SystemExit(f"expected active after restore, got {match.get('status')}") +print("ok") +PY +then + echo "FAIL: archived skill was not restored to active" + cat "$E2E_LIBRARY_STATE_FILE" + exit 1 +fi + +echo "PASS: archived skill restored to active in library snapshot" +echo "" +echo "E2E PASS: skills-library (A + B) succeeded" diff --git a/tests/e2e/teaching-skills.sh b/tests/e2e/teaching-skills.sh new file mode 100755 index 000000000..1c80c4d50 --- /dev/null +++ b/tests/e2e/teaching-skills.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/common.sh +source "$SCRIPT_DIR/lib/common.sh" + +trap e2e_cleanup EXIT + +ensure_clicky_built +start_mock_worker + +echo "Resetting prior teaching skills..." +rm -rf "$SKILLS_DIR" +mkdir -p "$SKILLS_DIR" +rm -f "$E2E_PROMPT_FILE" + +echo "Phase A: launch Clicky and teach a save workflow..." +launch_clicky /tmp/clicky-e2e-app.log \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL="$WORKER_URL" \ + -CLICKY_INJECT_TRANSCRIPT="how do I save this document in TextEdit?" \ + -CLICKY_INJECT_TRANSCRIPT_2="got it thanks that worked" + +SKILL_FILE="" +for _ in $(seq 1 90); do + if compgen -G "$SKILLS_DIR/*/SKILL.md" >/dev/null; then + SKILL_FILE="$(ls "$SKILLS_DIR"/*/SKILL.md | head -1)" + break + fi + sleep 1 +done + +if [[ -z "$SKILL_FILE" ]]; then + echo "FAIL: no teaching skill written within 90s" + echo "--- app log ---" + tail -40 /tmp/clicky-e2e-app.log || true + echo "--- worker log ---" + tail -20 /tmp/clicky-e2e-worker.log || true + exit 1 +fi + +SKILL_ID="$(basename "$(dirname "$SKILL_FILE")")" +echo "PASS: teaching skill written to $SKILL_FILE" +echo "--- skill preview ---" +head -20 "$SKILL_FILE" + +if [[ "$SKILL_ID" != "teach-textedit-save" ]]; then + echo "FAIL: expected skill id teach-textedit-save, got '$SKILL_ID'" + exit 1 +fi + +if [[ "$SKILL_ID" == *got* ]] || [[ "$SKILL_ID" == *thanks* ]] || [[ "$SKILL_ID" == *worked* ]]; then + echo "FAIL: skill slug '$SKILL_ID' contains confirmation phrase tokens" + exit 1 +fi + +echo "PASS: skill slug is clean ($SKILL_ID)" + +kill "$CLICKY_PID" 2>/dev/null || true +CLICKY_PID="" +sleep 1 +rm -f "$E2E_PROMPT_FILE" + +echo "Phase B: relaunch Clicky and verify saved skill is injected into prompt..." +launch_clicky /tmp/clicky-e2e-app-read.log \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL="$WORKER_URL" \ + -CLICKY_INJECT_TRANSCRIPT_3="how do I save this document in TextEdit?" + +for _ in $(seq 1 90); do + if [[ -f "$E2E_PROMPT_FILE" ]] && grep -q "teaching skills:" "$E2E_PROMPT_FILE"; then + if grep -qi "save" "$E2E_PROMPT_FILE"; then + echo "PASS: saved skill content found in composed system prompt" + echo "--- prompt preview ---" + grep -A 8 "teaching skills:" "$E2E_PROMPT_FILE" | head -12 + break + fi + fi + sleep 1 +done + +if [[ ! -f "$E2E_PROMPT_FILE" ]] || ! grep -q "teaching skills:" "$E2E_PROMPT_FILE"; then + echo "FAIL: composed system prompt did not include saved skill content within 90s" + echo "--- app log ---" + tail -40 /tmp/clicky-e2e-app-read.log || true + if [[ -f "$E2E_PROMPT_FILE" ]]; then + echo "--- prompt file ---" + head -40 "$E2E_PROMPT_FILE" || true + else + echo "prompt file missing: $E2E_PROMPT_FILE" + fi + exit 1 +fi + +kill "$CLICKY_PID" 2>/dev/null || true +CLICKY_PID="" +sleep 1 + +SKILL_COUNT_BEFORE_PATCH="$(find "$SKILLS_DIR" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" + +echo "Phase C: patch existing skill after refinement + confirmation..." +launch_clicky /tmp/clicky-e2e-app-patch.log \ + -CLICKY_E2E=1 \ + -CLICKY_WORKER_URL="$WORKER_URL" \ + -CLICKY_INJECT_TRANSCRIPT="how do I save this document in TextEdit?" \ + -CLICKY_INJECT_TRANSCRIPT_2="only use the keyboard shortcut not the file menu" \ + -CLICKY_INJECT_TRANSCRIPT_3="got it thanks that worked" + +for _ in $(seq 1 90); do + if [[ -f "$SKILL_FILE" ]] && grep -qi "keyboard shortcut only\|avoid the file menu" "$SKILL_FILE"; then + break + fi + sleep 1 +done + +SKILL_COUNT_AFTER_PATCH="$(find "$SKILLS_DIR" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" + +if [[ "$SKILL_COUNT_AFTER_PATCH" != "$SKILL_COUNT_BEFORE_PATCH" ]]; then + echo "FAIL: expected $SKILL_COUNT_BEFORE_PATCH skill folder(s) after patch, found $SKILL_COUNT_AFTER_PATCH" + ls -la "$SKILLS_DIR" || true + exit 1 +fi + +if [[ "$(basename "$(dirname "$SKILL_FILE")")" != "teach-textedit-save" ]]; then + echo "FAIL: expected patched skill id teach-textedit-save, got $(basename "$(dirname "$SKILL_FILE")")" + exit 1 +fi + +if ! grep -qi "keyboard shortcut only\|avoid the file menu" "$SKILL_FILE"; then + echo "FAIL: patched skill body did not include keyboard-only refinement" + echo "--- skill body ---" + tail -20 "$SKILL_FILE" || true + exit 1 +fi + +echo "PASS: existing skill patched in place ($SKILL_ID)" +echo "" +echo "E2E PASS: Phase A (write) + Phase B (read-path) + Phase C (patch) succeeded" +exit 0