From ae3e7d594c7853c76533fbd2210ac072cda9830c Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:04:51 +0200 Subject: [PATCH 01/25] feat: add multi-client support for Cursor, Codex, and Gemini CLI Expand the plugin to support Cursor, Codex, and Gemini CLI alongside Claude Code. Add onboard-confidence and migrate-optimizely skills and commands. Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude-plugin/plugin.json | 1 + .codex-plugin/plugin.json | 39 + .cursor-plugin/plugin.json | 19 + .mcp.json | 10 +- CLAUDE.md | 4 + GEMINI.md | 15 + README.md | 67 +- commands/migrate-optimizely.md | 9 + commands/onboard-confidence.md | 9 + gemini-extension.json | 38 + skills/migrate-optimizely/SKILL.md | 1271 +++++++++++++++++++++++ skills/onboard-confidence/SKILL.md | 1545 ++++++++++++++++++++++++++++ 12 files changed, 3018 insertions(+), 9 deletions(-) create mode 100644 .codex-plugin/plugin.json create mode 100644 .cursor-plugin/plugin.json create mode 100644 GEMINI.md create mode 100644 commands/migrate-optimizely.md create mode 100644 commands/onboard-confidence.md create mode 100644 gemini-extension.json create mode 100644 skills/migrate-optimizely/SKILL.md create mode 100644 skills/onboard-confidence/SKILL.md diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index c327e8e..4312627 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -8,6 +8,7 @@ }, "homepage": "https://confidence.spotify.com", "repository": "https://github.com/spotify/confidence-ai-plugins", + "license": "Apache-2.0", "keywords": [ "feature-flags", "experiments", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json new file mode 100644 index 0000000..83404f4 --- /dev/null +++ b/.codex-plugin/plugin.json @@ -0,0 +1,39 @@ +{ + "name": "confidence", + "version": "0.2.3", + "description": "Access Confidence feature flags, experiments, and migration tools directly from Codex.", + "author": { + "name": "Spotify Confidence", + "url": "https://confidence.spotify.com" + }, + "homepage": "https://confidence.spotify.com", + "repository": "https://github.com/spotify/confidence-ai-plugins", + "license": "Apache-2.0", + "keywords": [ + "feature-flags", + "experiments", + "a/b-testing", + "migration", + "openfeature" + ], + "skills": "./skills/", + "mcpServers": "./.mcp.json", + "interface": { + "displayName": "Confidence", + "shortDescription": "Feature flags, experiments, and migration tools", + "longDescription": "Access Confidence feature flags, experiments, and migration tools directly from Codex. Create, list, resolve, and target feature flags. Migrate from PostHog or Optimizely to Confidence SDK.", + "developerName": "Spotify Confidence", + "category": "Productivity", + "capabilities": [ + "Read", + "Write" + ], + "websiteURL": "https://confidence.spotify.com", + "defaultPrompt": [ + "List my feature flags", + "Create a flag called new-checkout with a boolean schema", + "Migrate my PostHog flags to Confidence" + ], + "logo": "./assets/logo.svg" + } +} diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json new file mode 100644 index 0000000..4a5ab8e --- /dev/null +++ b/.cursor-plugin/plugin.json @@ -0,0 +1,19 @@ +{ + "name": "confidence", + "displayName": "Confidence", + "version": "0.2.3", + "description": "Access Confidence feature flags, experiments, and migration tools directly from Cursor.", + "author": { + "name": "Spotify Confidence", + "url": "https://confidence.spotify.com" + }, + "license": "Apache-2.0", + "keywords": [ + "feature-flags", + "experiments", + "a/b-testing", + "migration", + "openfeature" + ], + "logo": "assets/logo.svg" +} diff --git a/.mcp.json b/.mcp.json index d5b9567..18fcc8f 100644 --- a/.mcp.json +++ b/.mcp.json @@ -2,11 +2,17 @@ "mcpServers": { "confidence-flags": { "type": "http", - "url": "https://mcp.confidence.dev/mcp/flags" + "url": "https://mcp.confidence.dev/mcp/flags", + "headers": { + "x-confidence-mcp-consumer": "plugin" + } }, "confidence-docs": { "type": "http", - "url": "https://mcp.confidence.dev/mcp/docs" + "url": "https://mcp.confidence.dev/mcp/docs", + "headers": { + "x-confidence-mcp-consumer": "plugin" + } } } } diff --git a/CLAUDE.md b/CLAUDE.md index c2e9a1b..0e30362 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,10 +5,14 @@ This plugin integrates Confidence with Claude Code, providing tools for feature ## Commands - `/confidence:migrate-posthog >` — Migrate feature flags from PostHog to Confidence SDK +- `/confidence:migrate-optimizely >` — Migrate feature flags from Optimizely to Confidence SDK +- `/confidence:onboard-confidence ` — Create accounts, onboard users, set up SDK clients, configure warehouses, and learn experimentation concepts ## Skills - **migrate-posthog** — Auto-triggers when the user asks to migrate PostHog flags or transform SDK code to Confidence +- **migrate-optimizely** — Auto-triggers when the user asks to migrate Optimizely flags or transform SDK code to Confidence +- **onboard-confidence** — Auto-triggers when the user asks to create a Confidence account, invite users, set up SDK clients, configure warehouses, run the setup wizard, or learn about experimentation ## MCP Servers diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 0000000..50026ea --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,15 @@ +# Confidence Extension + +You are a helpful assistant that can manage Confidence feature flags and experiments using the Confidence MCP tools. + +## Available Tool Categories + +- **Feature Flags** — Create, list, update, archive, resolve, and target feature flags +- **Documentation** — Search Confidence docs and SDK integration guides + +## Guidelines + +- Always check that the user is authenticated before performing flag operations. +- Use the confidence-docs tools to answer questions about SDK integration, OpenFeature setup, and best practices. +- When creating flags, confirm the flag name and schema with the user before proceeding. +- For migrations from PostHog or Optimizely, guide the user through the migration plan before executing changes. diff --git a/README.md b/README.md index 3736891..430ce6a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Confidence AI Plugin -Official Confidence plugin for AI clients. Access feature flags, experiments, and migration tools directly from your AI coding tool. +Official Confidence plugin for AI coding tools. Access feature flags, experiments, and migration tools directly from Claude Code, Cursor, Codex, and Gemini CLI. ## Installation @@ -24,12 +24,52 @@ Official Confidence plugin for AI clients. Access feature flags, experiments, an ``` Follow the browser prompts to log in. -### Updating +#### Updating ```bash claude plugin update confidence:confidence ``` +### Cursor + +#### From the Marketplace + +1. Open **Cursor Settings** > **Plugins** +2. Search for **Confidence** +3. Click **Install** + +#### Manual setup + +Add the MCP servers to `.cursor/mcp.json` in your project (or `~/.cursor/mcp.json` globally): + +```json +{ + "mcpServers": { + "confidence-flags": { + "url": "https://mcp.confidence.dev/mcp/flags" + }, + "confidence-docs": { + "url": "https://mcp.confidence.dev/mcp/docs" + } + } +} +``` + +### Codex + +```bash +codex plugin marketplace add spotify/confidence-ai-plugins +codex +/plugins +# Select Confidence and install +``` + +### Gemini CLI + +```bash +gemini extensions install https://github.com/spotify/confidence-ai-plugins +``` + ### Local Development ```bash @@ -41,20 +81,24 @@ claude --plugin-dir ./confidence-ai-plugins This plugin provides access to Confidence tools across these categories: -- **Feature flags** - Create, list, update, archive, and resolve feature flags -- **Migration** - Migrate feature flags from PostHog to Confidence +- **Feature flags** — Create, list, update, archive, resolve, and target feature flags +- **Documentation** — Search Confidence docs and SDK integration guides +- **Migration** — Migrate feature flags from PostHog or Optimizely to Confidence ## Slash Commands -- `/confidence:migrate-posthog` - Migrate feature flags from PostHog to Confidence SDK +- `/confidence:migrate-posthog` — Migrate feature flags from PostHog to Confidence SDK +- `/confidence:migrate-optimizely` — Migrate feature flags from Optimizely to Confidence SDK +- `/confidence:onboard-confidence` — Create Confidence accounts and onboard users ## Example Usage ``` > List my feature flags > Create a flag called new-checkout with a boolean schema -> /migrate-posthog plan flag -> /migrate-posthog plan code +> /confidence:migrate-posthog plan flag +> /confidence:migrate-posthog plan code +> /confidence:migrate-optimizely ``` ## MCP Servers @@ -64,6 +108,15 @@ This plugin provides access to Confidence tools across these categories: | `confidence-flags` | `https://mcp.confidence.spotify.com/mcp/flags` | Feature flag management | | `confidence-docs` | `https://mcp.confidence.spotify.com/mcp/docs` | Confidence documentation | +## Supported Clients + +| Client | Config | Marketplace | +|--------|--------|-------------| +| Claude Code | `.claude-plugin/` | Community marketplace | +| Cursor | `.cursor-plugin/` | Cursor Marketplace | +| Codex | `.codex-plugin/` | Via marketplace command | +| Gemini CLI | `gemini-extension.json` | Direct from repo | + ## Documentation - [Confidence documentation](https://confidence.spotify.com/docs) diff --git a/commands/migrate-optimizely.md b/commands/migrate-optimizely.md new file mode 100644 index 0000000..a0fe22f --- /dev/null +++ b/commands/migrate-optimizely.md @@ -0,0 +1,9 @@ +--- +name: migrate-optimizely +description: Migrate feature flags from Optimizely to Confidence +argument-hint: [plan flag | plan code | execute ] +--- + +All migration instructions are maintained in `skills/migrate-optimizely/SKILL.md` to prevent divergence. + +**Before doing anything else**, use the Read tool to read `skills/migrate-optimizely/SKILL.md` and follow those instructions to handle this command. diff --git a/commands/onboard-confidence.md b/commands/onboard-confidence.md new file mode 100644 index 0000000..ba937a7 --- /dev/null +++ b/commands/onboard-confidence.md @@ -0,0 +1,9 @@ +--- +name: onboard-confidence +description: Create Confidence accounts and onboard users +argument-hint: [create-account | invite-user | create-client | setup-wizard | setup-warehouse | learn | status] +--- + +All onboarding instructions are maintained in `skills/onboard-confidence/SKILL.md` to prevent divergence. + +**Before doing anything else**, use the Read tool to read `skills/onboard-confidence/SKILL.md` and follow those instructions to handle this command. diff --git a/gemini-extension.json b/gemini-extension.json new file mode 100644 index 0000000..9d3872a --- /dev/null +++ b/gemini-extension.json @@ -0,0 +1,38 @@ +{ + "name": "confidence", + "version": "0.2.3", + "description": "Access Confidence feature flags, experiments, and migration tools directly from Gemini CLI.", + "mcpServers": { + "confidence-flags": { + "command": "npx", + "args": [ + "mcp-remote@latest", + "${CONFIDENCE_MCP_FLAGS_URL:-https://mcp.confidence.dev/mcp/flags}", + "--header", + "x-confidence-mcp-consumer:plugin" + ] + }, + "confidence-docs": { + "command": "npx", + "args": [ + "mcp-remote@latest", + "${CONFIDENCE_MCP_DOCS_URL:-https://mcp.confidence.dev/mcp/docs}", + "--header", + "x-confidence-mcp-consumer:plugin" + ] + } + }, + "contextFileName": "GEMINI.md", + "settings": [ + { + "name": "Confidence MCP Flags URL", + "description": "Confidence MCP flags endpoint URL (default: https://mcp.confidence.dev/mcp/flags).", + "envVar": "CONFIDENCE_MCP_FLAGS_URL" + }, + { + "name": "Confidence MCP Docs URL", + "description": "Confidence MCP docs endpoint URL (default: https://mcp.confidence.dev/mcp/docs).", + "envVar": "CONFIDENCE_MCP_DOCS_URL" + } + ] +} diff --git a/skills/migrate-optimizely/SKILL.md b/skills/migrate-optimizely/SKILL.md new file mode 100644 index 0000000..dfda792 --- /dev/null +++ b/skills/migrate-optimizely/SKILL.md @@ -0,0 +1,1271 @@ +--- +description: Migrate feature flags from Optimizely to Confidence SDK. Use when the user says /migrate-optimizely, asks to migrate Optimizely flags, or transform SDK code to Confidence. +--- + +# Optimizely to Confidence Migration + +MCP-driven, self-sufficient migration from Optimizely to Confidence. + +## Migration Flow + +The migration happens in two phases: **flags first, then code**. + +``` +Phase 1: Flag Definitions + plan flags → Scan Optimizely, choose environment & client & entity, generate plan + execute → Create each flag in Confidence with targeting rules + +Phase 2: Code Transformation + plan code → Scan codebase, fetch SDK guide, generate transform rules + execute → Transform code flag by flag, each flag = one PR +``` + +**Why flags first?** The flags need to exist in Confidence before the +code can resolve them. Once flags are live in Confidence, you migrate +the code that evaluates them — one flag at a time, one PR at a time. + +**Each code PR is scoped to a single flag.** This keeps PRs small, +reviewable, and independently shippable. If one flag's migration has +issues, it doesn't block the others. + +## Commands + +| Command | Description | +|---------|-------------| +| `/migrate-optimizely plan flags` | Phase 1: plan flag definitions migration | +| `/migrate-optimizely plan code` | Phase 2: plan code transformation | +| `/migrate-optimizely execute ` | Execute a plan interactively | + +--- + +## Migration Overview (MUST display at start of `plan flags` or `plan code`) + +**Every time** the user runs `plan flags` or `plan code`, display this +overview FIRST — before doing any work. This orients the user on where +they are in the full migration journey. + +``` +═══════════════════════════════════════════════════════════════ + Optimizely → Confidence Migration +═══════════════════════════════════════════════════════════════ + + The migration happens in two phases: flags first, then code. + + ┌─────────────────────────────────────────────────────────┐ + │ PHASE 1 — Flag Definitions │ + │ │ + │ Move all flags from Optimizely to Confidence with │ + │ their targeting rules, rollout percentages, variants, │ + │ and variables. │ + │ │ + │ Steps: │ + │ 1. Scan all flags in Optimizely │ + │ 2. Choose which Optimizely environment to migrate │ + │ 3. Choose a Confidence client (your app) │ + │ 4. Map randomization units (user_id, etc.) │ + │ 5. Generate migration plan with targeting rules │ + │ 6. Execute: create each flag in Confidence │ + │ │ + │ Result: All flags live in Confidence, ready to resolve│ + ├─────────────────────────────────────────────────────────┤ + │ PHASE 2 — Code Transformation │ + │ │ + │ Once flags exist in Confidence, migrate the code that │ + │ evaluates them. Each flag = one PR. │ + │ │ + │ Steps: │ + │ 1. Detect language & framework │ + │ 2. Fetch Confidence SDK guide │ + │ 3. Scan codebase for Optimizely usage │ + │ 4. Generate transform rules (Optimizely→Confidence) │ + │ 5. Generate plan grouped by flag │ + │ 6. Execute: transform code flag by flag, one PR each│ + │ │ + │ Result: Code uses Confidence SDK, Optimizely removed │ + └─────────────────────────────────────────────────────────┘ + + Why flags first? + Flags must exist in Confidence before code can resolve them. + + Why one PR per flag? + Keeps changes small, reviewable, and independently shippable. + If one flag's migration has issues, it doesn't block the others. + +═══════════════════════════════════════════════════════════════ +``` + +After displaying the overview, indicate which phase the user is about +to enter: + +- For `plan flags`: "Starting **Phase 1** — Flag Definitions" +- For `plan code`: "Starting **Phase 2** — Code Transformation. + Make sure Phase 1 (flag definitions) is complete first — the flags + need to exist in Confidence before the code can resolve them." + +Then proceed with the normal workflow for that phase. + +--- + +## SDK Preference + +**ALWAYS prefer OpenFeature with local resolve.** + +| Priority | Approach | When to use | +|----------|----------|-------------| +| 1st | Local resolve | Default for all new integrations | +| 2nd | Remote resolve | Only if local resolve not supported for platform | +| Avoid | Direct SDK | Being phased out | + +--- + +## Plan Philosophy + +**Plans must be MCP-boxed, self-sufficient, and agent-agnostic.** + +| Principle | Meaning | +|-----------|---------| +| **MCP-boxed** | Every external data fetch uses explicit MCP tool calls | +| **Self-sufficient** | Plan contains ALL information needed - no "query MCP for X" | +| **Agent-agnostic** | Any agent with MCPs can execute without prior context | +| **Language-agnostic** | Detect framework, fetch SDK guide from MCP dynamically | + +--- + +## Prerequisites + +Before starting any workflow, check that required MCP servers are available. +Try calling a simple tool from each. If it fails, install the missing MCP. + +### Optimizely MCP + +The Optimizely Feature Experimentation MCP server provides tools to list +projects, flags, experiments, environments, audiences, and manage them. + +**Discovery:** The exact tool names depend on the MCP server version. After +installing, use ToolSearch to discover available tools matching `optimizely`. +Common patterns: list projects, list flags, get flag, list environments, +list audiences, get ruleset. + +If not available, install it: +``` +claude mcp add optimizely-exp --transport http --url https://exp.mcp.opal.optimizely.com/mcp +``` + +The user will be prompted to authenticate via Opti ID (OAuth) in their browser. + +### Confidence MCP + +Test: `mcp__confidence-flags__listClients` + +If not available, install it: +``` +claude mcp add confidence-flags --transport http --url https://mcp.confidence.dev/mcp/flags +``` + +The user will be prompted to authenticate via OAuth in their browser. + +### Confidence Docs MCP (for `plan code` only) + +Test: `mcp__confidence-docs__searchDocumentation` + +If not available, install it: +``` +claude mcp add confidence-docs --transport http --url https://mcp.confidence.dev/mcp/docs +``` + +--- + +## User-Facing Communication Rules + +**NEVER expose internal technical details to the user.** The user should see +human-readable descriptions of what's happening, not internal implementation +details like targeting payload formats, rule types, or operator names. + +- Do NOT say "creating plan based on eqRule / rangeRule / setRule" etc. +- Do NOT show raw targeting payloads or JSON structures in conversation +- DO say things like: "Creating flag with rule: plan equals 'pro' AND country is US or UK" +- DO describe rules in plain English: "age between 18 and 65", "plan is not free" +- The plan FILE may contain MCP command payloads (for machine execution), + but conversation output must be human-friendly + +**Step Tracker:** Display a visual step tracker at every phase transition. +The tracker shows all phases, marks completed ones, highlights the current +one, and shows remaining ones. Update and re-display it each time you move +to a new phase. + +### Plan Flags Step Tracker + +Display this at the START and after EACH step completes (updating status): + +``` +───── Plan Flags ────────────────────────────────────────── + [1] Scan Optimizely ○ pending + [2] Choose environment ○ pending + [3] Choose client ○ pending + [4] Map entities ○ pending + [5] Generate plan ○ pending +──────────────────────────────────────────────────────────── +``` + +Status markers: +- `○ pending` — not started yet +- `◉ in progress` — currently running +- `⏸ awaiting user` — blocked on user input (e.g. picking a client or entity) +- `✓ done` — completed (add brief user-facing result) +- `⊘ skipped` — skipped by user + +Use `⏸ awaiting user` whenever the workflow has asked a question and is +waiting for an explicit reply. This makes "I'm blocked on you" visible +to both agent and user, and prevents the agent from drifting into +auto-progression while a question is open. + +**IMPORTANT:** Never expose internal/technical details in the tracker. +No pagination info, no API page counts, no internal field names. +Show only what matters to the user. + +Example after Step 1 completes: +``` +───── Plan Flags ────────────────────────────────────────── + [1] Scan Optimizely ✓ 12 flags found + [2] Choose environment ◉ in progress + [3] Choose client ○ pending + [4] Map entities ○ pending + [5] Generate plan ○ pending +──────────────────────────────────────────────────────────── +``` + +### Execute Step Tracker + +Display this at the START and update after EACH flag: + +``` +───── Execute Migration ─────────────────────────────────── + Client: test | Entity: user_id | Flags: 12 + Progress: [░░░░░░░░░░░░░░░░░░░░] 0/12 +──────────────────────────────────────────────────────────── +``` + +Update the progress bar as flags are processed. Use `█` for completed +and `░` for remaining. The bar should be 20 characters wide. + +Examples at various stages: +``` + Progress: [██████░░░░░░░░░░░░░░] 4/12 (1 skipped) + Current: checkout-redesign +``` + +``` + Progress: [████████████████████] 12/12 done + Result: 11 migrated, 1 skipped +``` + +After each flag completes, show: +``` + ✓ checkout-redesign — MATCH (enabled) +``` + +After a skip: +``` + ⊘ legacy-banner — skipped +``` + +### Final Summary (Execute) + +At the end of execution, show a complete summary: + +``` +───── Migration Complete ────────────────────────────────── + Progress: [████████████████████] 12/12 done + Migrated: 11 | Skipped: 1 | Failed: 0 + + ✓ checkout-redesign 100% user_id + ✓ pricing-experiment 50/50 user_id + ⊘ legacy-banner — skipped + ✓ dark-mode 25% user_id + ... +──────────────────────────────────────────────────────────── +``` + +--- + +## Confidence Naming Rules + +- **Flag names:** lowercase letters, digits, and hyphens only (`[a-z0-9-]`) +- **Optimizely flag keys** may contain underscores — convert them to hyphens + when creating in Confidence (e.g. `product_sort` → `product-sort`) +- **Entity references:** Confidence entity names do NOT support underscores. + The entity reference (e.g. `entities/company`) is separate from the context + field name (e.g. `company_id`). When creating entity fields with + `addContextField`, always provide an explicit `entityReference` with a + clean name (no underscores). If omitted, the tool auto-generates one from + the field name which will fail. + + | Field name | Entity reference | Works? | + |------------|-----------------|--------| + | `user_id` | `entities/user` | Yes | + | `company_id` | `entities/company` | Yes | + | `visitor_id` | `entities/visitor` | Yes | + | `company_id` | *(omitted — auto: `entities/company_id`)* | **No** | + +--- + +## Plan Code: Workflow + +### Resume Check (MUST do first) + +Same as Plan Flag: check for existing `.claude/plans/optimizely-code-migration-*.md`. +If found with incomplete `Generation Status`, resume from the last +incomplete step. If complete, ask user if they want to start fresh. +If not found, start fresh. + +The plan file uses the same progressive pattern: created at Step 1, +updated after each step, with a `## Generation Status` section. + +### Step 1: Detect Language & Framework + +``` +Grep: pattern="optimizely|Optimizely" -> Find Optimizely usage +Glob: pattern="package.json" or "build.gradle" or "Cargo.toml" etc +Read: dependency file -> Determine language/framework +``` + +**Optimizely SDK packages to detect:** + +| Language | Package/Import | +|----------|---------------| +| JS/TS (Node) | `@optimizely/optimizely-sdk` | +| React | `@optimizely/react-sdk` | +| React Native | `@optimizely/react-native-sdk` | +| Python | `optimizely` (PyPI: `optimizely-sdk`) | +| Java | `com.optimizely.ab:core-api` or `com.optimizely.ab.*` | +| Go | `github.com/optimizely/go-sdk` or `github.com/optimizely/go-sdk/v2` | +| Ruby | `optimizely-sdk` (gem) | +| PHP | `optimizely/php-sdk` | +| C# | `Optimizely.Sdk` | +| Swift | `OptimizelySwiftSdk` | +| Android | `com.optimizely.ab:android-sdk` | +| Flutter | `optimizely_flutter_sdk` | + +### Step 2: Fetch SDK Guide from MCP + +**Query confidence-docs MCP based on detected language:** + +``` +mcp__confidence-docs__getCodeSnippetAndSdkIntegrationTips + sdk: "" +``` + +``` +mcp__confidence-docs__searchDocumentation + query: "OpenFeature local resolve " +``` + +``` +mcp__confidence-docs__getFullSource + source: "https://confidence.spotify.com/docs/sdks/server/" +``` + +**CRITICAL:** Include the ACTUAL response in the plan, not a reference to fetch it. + +### Step 3: Scan Codebase for Optimizely Usage + +``` +Grep: pattern="" -> Find all usages +``` + +**Must detect BOTH modern and legacy Optimizely API patterns:** + +**Modern API (recommended — `decide` method):** + +| Pattern | Language | +|---------|----------| +| `createInstance(` | JS/TS | +| `createUserContext(` | All languages | +| `.decide(` / `user.decide(` | All languages | +| `decision.enabled` | All languages | +| `decision.variationKey` / `decision.variation_key` | JS / Python | +| `decision.variables` | All languages | +| `user.trackEvent(` / `user.track_event(` | All languages | +| `OptimizelyProvider` | React | +| `useDecision(` | React | + +**Legacy API (deprecated but still common):** + +| Pattern | Modern equivalent | +|---------|------------------| +| `isFeatureEnabled(flagKey, userId, attrs)` | `user.decide(flagKey).enabled` | +| `activate(experimentKey, userId, attrs)` | `user.decide(flagKey).variationKey` | +| `getVariation(experimentKey, userId, attrs)` | `user.decide(flagKey).variationKey` | +| `getFeatureVariableString(flagKey, varKey, userId, attrs)` | `user.decide(flagKey).variables[varKey]` | +| `getFeatureVariableInteger(...)` | `user.decide(flagKey).variables[varKey]` | +| `getFeatureVariableBoolean(...)` | `user.decide(flagKey).variables[varKey]` | +| `getFeatureVariableDouble(...)` | `user.decide(flagKey).variables[varKey]` | +| `getFeatureVariableJSON(...)` | `user.decide(flagKey).variables[varKey]` | +| `getAllFeatureVariables(flagKey, userId, attrs)` | `user.decide(flagKey).variables` | +| `track(eventKey, userId, attrs)` | `user.trackEvent(eventKey)` | + +Group files by flag key they reference. + +### Step 4: Generate Transform Rules + +Based on SDK guide from MCP: +- Extract install commands +- Extract initialization code +- Extract flag evaluation API +- Generate find/replace rules matching Optimizely → Confidence patterns + +### Step 5: Generate Plan + +Save to `.claude/plans/optimizely-code-migration-.md` + +--- + +## Plan Code: Template + +```markdown +# Optimizely to Confidence Code Migration Plan + +**Created:** +**Scope:** Code transformation only +**Language:** +**Framework:** + +--- + +## 1. SDK Setup + +### Install + + + +### API Reference (from MCP: confidence-docs) + + + +### Create Confidence Wrapper + +**File:** + +**Must match Optimizely API surface:** + +| Method | Signature | +|--------|-----------| + + +--- + +## 2. Transform Rules + +### Source Files + +| Find | Replace | +|------|---------| +| | | +| | | + +### Test Files + +| Find | Replace | +|------|---------| +| | | + +--- + +## 3. Files to Transform + + + +--- + +## 4. Progress + +| # | Item | Status | +|---|------|--------| +| 0 | SDK Setup | :white_circle: | + +``` + +--- + +## Plan Flag: Workflow + +### Resume Check (MUST do first) + +Before starting, check for an existing in-progress plan: + +``` +Glob: .claude/plans/optimizely-flag-migration-*.md +``` + +If a plan file exists, read its `## Generation Status` section: +- If status is `complete` → tell user a plan already exists, ask if + they want to start fresh or use the existing one +- If status is NOT `complete` → **resume from the last incomplete step** + Tell the user: "Found an in-progress plan. Resuming from step ." +- If no plan file exists → start fresh + +### Progressive Plan File + +The plan file is created at the START (Step 1) and updated after EACH +step. This means if the session closes, the file has partial progress +that can be resumed. + +**File path:** `.claude/plans/optimizely-flag-migration-.md` + +The plan file MUST include a `## Generation Status` section at the top +(right after the title) that tracks which steps are done: + +```markdown +## Generation Status + +| Step | Status | Result | +|------|--------|--------| +| 1. Scan Optimizely | ✓ complete | 12 flags | +| 2. Choose environment | ✓ complete | production | +| 3. Choose client | ✓ complete | test | +| 4. Map entities | ○ not started | | +| 5. Generate rules | ○ not started | | +``` + +Status values: `✓ complete`, `◉ in progress`, `○ not started` + +**After each step completes**, update the status table AND write that +step's data to the plan file. Do NOT wait until the end to write. + +### Step 1: Scan Optimizely Flags + +Use the Optimizely MCP to list all flags in the project. + +**Discovery:** Use ToolSearch to find available Optimizely MCP tools. +Look for tools that list flags/features in a project. Typical patterns: +- List all flags in a project +- Get flag details (variations, variables) +- List environments +- List audiences +- Get ruleset for a flag in an environment + +**CRITICAL: Paginate until ALL flags are fetched.** If the MCP supports +pagination, keep fetching until all flags are returned. + +For each flag found, gather: +- Flag key and name/description +- Variations (names, keys, variable overrides) +- Variables (name, type, default value) +- Available environments + +**After scan completes:** Write the flag data to the plan file and +update Generation Status step 1 to `✓ complete`. + +### Step 2: Choose Optimizely Environment + +Optimizely flags have different rules per environment (e.g. production, +development, staging). The user must choose which environment's rules +to migrate. + +**EDUCATE then ASK the user:** + +> **What is an environment?** +> In Optimizely, each flag can have different targeting rules and rollout +> percentages per environment. For example, a flag might be 100% rolled +> out in development but only 10% in production. +> +> Which environment's rules should I migrate? +> +> Your environments: +> 1. production +> 2. development +> 3. + +**Wait for an explicit pick.** Set the step to `⏸ awaiting user` and +stop. A re-run of `/migrate-optimizely`, an empty message, or any reply +that is not a valid choice is **not** consent — NEVER infer from silence. +If the reply is ambiguous, re-ask. + +After the user picks an environment, fetch the ruleset for each flag +in that environment. For each flag, extract: +- Rule type: **Targeted Delivery** (rollout) or **A/B Test** (experiment) +- Audience conditions (targeting rules) +- Traffic allocation (variant percentages) +- Whether the ruleset is enabled + +**After environment selected and rulesets fetched:** Write Section 1b +(Environment & Rules) to plan file and update Generation Status step 2 +to `✓ complete`. + +### Step 3: Select Confidence Client + +``` +mcp__confidence-flags__listClients +``` + +**EDUCATE then ASK the user:** + +> **What is a client?** +> A client represents the application that resolves flags — your website, +> backend service, or mobile app. Each client has its own secret for +> authentication and can be scoped to environments (dev, staging, prod). +> Flags are associated with one or more clients, so Confidence knows which +> application should receive which flags. +> +> Think of it like: "Where will these flags be evaluated?" +> +> Your existing clients: +> 1. +> 2. +> ... +> N. Create a new client +> +> Which client should I use as the default for all flags? +> You can always rearrange them later in the Confidence UI. + +**Wait for an explicit pick.** Set the step to `⏸ awaiting user` and +stop. A re-run of `/migrate-optimizely`, an empty message, or any reply +that is not a number from the list / `new ` is **not** consent — +NEVER infer the recommendation from silence. If the reply is ambiguous, +re-ask, listing the choices again. + +- If user picks existing → use it +- If user wants new → ASK for name → `mcp__confidence-flags__createClient` + +**After client selected:** Write Section 1 (Default Client) to plan +file and update Generation Status step 3 to `✓ complete`. + +### Step 4: Map Randomization Units + +``` +mcp__confidence-flags__getContextSchema clientName: "" +``` + +Show the user entity fields (fields marked as entity in the schema). + +This step maps Optimizely's `userId` to Confidence entity fields. + +**EDUCATE then ASK:** + +> **What is a randomization unit (entity)?** +> An entity is the "thing" that gets randomly assigned to a variant — +> usually a user. The entity field (like `user_id` or `visitor_id`) is +> the identifier Confidence uses to ensure **consistent assignment**: the +> same user always sees the same variant. +> +> In Confidence, it maps to the `targeting_key` in the evaluation context. + +> All of your flags randomize per user. In Optimizely, each user is +> identified by `userId` (passed to `createUserContext`). In Confidence, +> you need to pick which field represents the same user identifier. +> +> Common choices: +> - **user_id** — if your flags target authenticated users +> - **visitor_id** — if targeting anonymous visitors (auto-generated by +> Confidence client SDKs) +> +> Your client's existing entity fields: +> 1. +> 2. +> ... +> N. Create a new field +> +> Which Confidence field represents the same user as Optimizely's `userId`? + +**Wait for an explicit pick.** Same rule as Step 3 — set the step to +`⏸ awaiting user` and stop. Silence, a re-run, or any non-listed reply +is **not** consent. Re-ask if the reply is ambiguous. + +- If user picks existing → use it as `targetingKey` for all flags +- If user wants new → ASK for name + type → `mcp__confidence-flags__addContextField` + +**Note on Optimizely group experiments:** Optimizely does not have native +group bucketing like PostHog's `aggregation_group_type_index`. If an +Optimizely project uses group-level experiments (passing a group ID as +`userId`), the user should create a separate entity field for that group +identifier. Flag the user if any flags appear to use non-user IDs. + +**Step 4 only creates entity fields.** Attribute fields used in +targeting rules (`country`, `plan`, `age`, etc.) MUST NOT be created +here. Record them in Section 3 "Need to Create" and let `execute` +create them — that way, if the user later skips a flag, no orphan +schema fields are left in Confidence. + +**After entity mapped:** Write Section 2 (Randomization Mapping) to +plan file, reconcile and write Section 3 (Context Schema), and update +Generation Status step 4 to `✓ complete`. + +### Step 5: Generate MCP Commands + +**Confirmation gate (MUST pass before generating).** Before writing +Section 4, summarize chosen environment, client + entity in chat and ask: + +> Plan will migrate rules from Optimizely environment `` to +> Confidence client `` with randomization entity ``. +> All flags will be defaulted to `[ ] Migrate [ ] Skip` +> (neither pre-checked) — you'll opt each one in during review. +> Confirm or change? + +Set the step to `⏸ awaiting user` and stop. Only proceed on an +explicit `yes` / `confirm` / equivalent. A re-run or ambiguous reply +is **not** confirmation. + +For each flag in Section 4, generate the MCP command payloads +(createFlag, addFlagToClient, addTargetingRule, resolveFlag) using the +Operator Mapping Reference (below). Write them into each flag's section. + +**After all commands generated:** Update Generation Status step 5 to +`✓ complete` and set the overall status to `complete`. Write the +Progress table (Section 6). + +**Tell the user:** +> Plan generated! Review it at `.claude/plans/optimizely-flag-migration-.md` +> +> Migration is **opt-in**: every flag starts with both checkboxes +> empty. Tick `[x] Migrate` or `[x] Skip` for each flag — `execute` +> will refuse any flag with neither box set. +> When you're ready, run: `/migrate-optimizely execute ` + +--- + +## Optimizely Concepts Reference (agent-internal, do NOT show to user) + +### Optimizely Flag Structure + +Each Optimizely flag has: +- **Key:** Alphanumeric + hyphens/underscores (max 64 chars) +- **Variations:** Named variants with keys and variable overrides +- **Variables:** Typed config values (string, integer, double, boolean, JSON) + with defaults and per-variation overrides +- **Rules per environment:** Ordered list of Targeted Delivery or A/B Test rules +- **Audiences:** Reusable targeting definitions with boolean conditions + +### Optimizely Traffic Allocation + +Optimizely uses a 0-10,000 bucket range (basis points): +- `endOfRange: 5000` = 50% of traffic +- Buckets are assigned via MurmurHash3 on `(userId + experimentId)` +- If total allocation < 10,000, remaining users are excluded + +**Conversion to Confidence:** Divide by 100 to get percentage. For +non-round percentages, round to nearest integer (Confidence uses +whole percentages that must sum to 100). + +### Optimizely Audience Conditions Format + +Conditions use a nested list format: +```json +["and", + {"type": "custom_attribute", "name": "country", "match": "exact", "value": "US"}, + {"type": "custom_attribute", "name": "age", "match": "gt", "value": 18} +] +``` + +Combinators: `"and"`, `"or"`, `"not"` as first element of a list. + +Individual condition: +```json +{ + "type": "custom_attribute", + "name": "", + "match": "", + "value": +} +``` + +Match type defaults: `exact` if a value is provided, `exists` if no value. + +--- + +## Operator Mapping Reference (agent-internal, do NOT show to user) + +This is how Optimizely operators map to Confidence targeting payloads. +Use this when generating `addTargetingRule` payloads in the plan file. + +**CRITICAL: Confidence Targeting Payload Format** + +The payload uses a `criteria` + `expression` pattern. Criteria are named +references (`ref-0`, `ref-1`, ...) that define individual conditions. +The `expression` combines them with boolean logic (`and`, `or`, `not`, `ref`). + +```json +{ + "criteria": { + "ref-0": { + "attribute": { + "attributeName": "", + "": { ... } + } + } + }, + "expression": { "ref": "ref-0" } +} +``` + +**DO NOT use nested rule objects like `{"or": {"operands": [{"eqRule": ...}]}}` +at the top level.** That format is silently parsed as empty targeting +(matching ALL contexts) due to `ignoringUnknownFields()` in the proto parser. + +### Criterion Rules + +| Optimizely Match | Confidence Criterion | +|-----------------|---------------------| +| `exact: "X"` (string) | `"eqRule": { "value": { "stringValue": "X" } }` | +| `exact: N` (number) | `"eqRule": { "value": { "numberValue": N } }` | +| `exact: true/false` | `"eqRule": { "value": { "boolValue": true } }` | +| `ge: N` | `"rangeRule": { "startInclusive": { "numberValue": N } }` | +| `gt: N` | `"rangeRule": { "startExclusive": { "numberValue": N } }` | +| `lt: N` | `"rangeRule": { "endExclusive": { "numberValue": N } }` | +| `le: N` | `"rangeRule": { "endInclusive": { "numberValue": N } }` | + +### Expression Combinators + +| Pattern | Expression | +|---------|-----------| +| Single condition | `{ "ref": "ref-0" }` | +| AND | `{ "and": { "operands": [{ "ref": "ref-0" }, { "ref": "ref-1" }] } }` | +| OR | `{ "or": { "operands": [{ "ref": "ref-0" }, { "ref": "ref-1" }] } }` | +| NOT | `{ "not": { "ref": "ref-0" } }` | + +### Optimizely Operator Mapping + +| Optimizely | Confidence Payload Strategy | +|-----------|---------------------------| +| `exact: "X"` | One criterion with `eqRule`, expression: `ref` | +| `NOT` + `exact: "X"` | One criterion with `eqRule`, expression: `not` wrapping `ref` | +| `exact: "A"` OR `exact: "B"` | One criterion per value with `eqRule`, expression: `or` of `ref`s | +| `ge: N` | One criterion with `rangeRule` (startInclusive), expression: `ref` | +| `gt: N` | One criterion with `rangeRule` (startExclusive), expression: `ref` | +| `lt: N` | One criterion with `rangeRule` (endExclusive), expression: `ref` | +| `le: N` | One criterion with `rangeRule` (endInclusive), expression: `ref` | + +**Blocked (manual review required):** + +| Optimizely Match | Reason | +|-----------------|--------| +| `substring` | No Confidence equivalent (contains/substring not supported) | +| `exists` | No Confidence equivalent (field-presence check not supported) | +| Semver comparisons | No Confidence equivalent (version type not supported) | + +When a flag uses a blocked operator, mark it in the plan with a warning: +> ⚠ This flag uses `substring` matching which has no Confidence equivalent. +> Manual review required — consider converting to `startsWith`/`endsWith` +> if the pattern allows, or implement in application code. + +### AND / OR Combinations + +**AND conditions:** Optimizely `["and", cond1, cond2]`. +Create one criterion per condition, combine with `and` expression. + +**OR conditions:** Optimizely `["or", cond1, cond2]`. +Create one criterion per condition, combine with `or` expression. + +**Nested combinations:** Optimizely supports arbitrary nesting: +`["and", cond1, ["or", cond2, cond3]]`. Map directly to nested +Confidence expressions. + +### Complete Examples + +**Single equality (country = "US"):** +```json +{ + "criteria": { + "ref-0": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "US" } } } } + }, + "expression": { "ref": "ref-0" } +} +``` + +**AND (plan = "pro" AND country = "US"):** +```json +{ + "criteria": { + "ref-0": { "attribute": { "attributeName": "plan", "eqRule": { "value": { "stringValue": "pro" } } } }, + "ref-1": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "US" } } } } + }, + "expression": { "and": { "operands": [{ "ref": "ref-0" }, { "ref": "ref-1" }] } } +} +``` + +**OR (country = "US" OR country = "UK"):** +```json +{ + "criteria": { + "ref-0": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "US" } } } }, + "ref-1": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "UK" } } } } + }, + "expression": { "or": { "operands": [{ "ref": "ref-0" }, { "ref": "ref-1" }] } } +} +``` + +**NOT (country != "DE"):** +```json +{ + "criteria": { + "ref-0": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "DE" } } } } + }, + "expression": { "not": { "ref": "ref-0" } } +} +``` + +**Range (age >= 18):** +```json +{ + "criteria": { + "ref-0": { "attribute": { "attributeName": "age", "rangeRule": { "startInclusive": { "numberValue": 18 } } } } + }, + "expression": { "ref": "ref-0" } +} +``` + +**Nested AND/OR (plan = "pro" AND (country = "US" OR country = "UK")):** +```json +{ + "criteria": { + "ref-0": { "attribute": { "attributeName": "plan", "eqRule": { "value": { "stringValue": "pro" } } } }, + "ref-1": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "US" } } } }, + "ref-2": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "UK" } } } } + }, + "expression": { "and": { "operands": [{ "ref": "ref-0" }, { "or": { "operands": [{ "ref": "ref-1" }, { "ref": "ref-2" }] } }] } } +} +``` + +### Multivariant A/B Split Handling + +**CRITICAL:** A single Confidence targeting rule CAN assign multiple +variants at different split percentages. Use ONE rule per targeting +condition, listing all variants and their shares in that rule. + +**How to map Optimizely traffic allocation to Confidence rules:** + +Optimizely uses 0-10,000 basis points. Convert to percentages: + +For a 2-variant experiment (e.g. endOfRange: 5000, 10000): +- Variation 1: 5000/10000 = 50% +- Variation 2: (10000-5000)/10000 = 50% +- Add ONE rule with: variation-1 at 50%, variation-2 at 50% + +For partial rollout (e.g. endOfRange: 2500, 5000 out of 10000): +- Variation 1: 2500/10000 = 25% +- Variation 2: (5000-2500)/10000 = 25% +- Remaining: 50% unallocated +- Add ONE rule with appropriate variant allocations + +**Do NOT create separate rules per variant.** One targeting rule = +one set of targeting conditions, with the variant split defined +inside that rule. The `rolloutPercentage` on the rule controls +what fraction of users who match the targeting conditions enter the +rule at all (use 100% unless you want a partial rollout on top of +the targeting). The variant percentages within the rule control the +split among those who enter. + +### Variable Mapping to Confidence Schema + +Optimizely typed variables map to Confidence flag schema: + +| Optimizely Type | Confidence Schema Type | +|----------------|----------------------| +| `string` | `"string"` | +| `integer` | `"integer"` | +| `double` | `"double"` | +| `boolean` | `"boolean"` | +| `json` | Flatten to individual fields or use `"string"` | + +When creating a flag with variables, use `schemaObject` to define the +schema and include variable values in each variant's `value` object. + +Example: Optimizely flag with variables `sort_method` (string) and +`items_per_page` (integer): + +``` +createFlag + flagName: "product-sort" + schemaObject: {"sort_method": "string", "items_per_page": "integer"} + variants: [ + {"name": "control", "value": {"sort_method": "relevance", "items_per_page": 20}}, + {"name": "treatment", "value": {"sort_method": "popularity", "items_per_page": 30}} + ] +``` + +--- + +## Plan Flag: Template + +```markdown +# Optimizely to Confidence Flag Migration Plan + +**Created:** +**Scope:** Flag definitions only +**Optimizely Project:** + +--- + +## Generation Status + +| Step | Status | Result | +|------|--------|--------| +| 1. Scan Optimizely | ○ not started | | +| 2. Choose environment | ○ not started | | +| 3. Choose client | ○ not started | | +| 4. Map entities | ○ not started | | +| 5. Generate rules | ○ not started | | + +**Overall:** in progress + +--- + +## 1. Default Client + +A client represents the application that resolves flags (e.g. your +website, backend service, or mobile app). Each client authenticates +with its own secret and can be scoped to environments (dev, staging, +prod). Flags are associated with clients so Confidence knows which +application receives which flags. + +**Available Clients:** + +**Selected:** `` + +--- + +## 1b. Optimizely Environment + +In Optimizely, each flag has different rules per environment. This plan +migrates the rules from a single environment. + +**Available Environments:** + +**Selected:** `` + +--- + +## 2. Randomization Mapping + +An entity is the "thing" being randomly assigned to a variant — usually +a user. The entity field (like `user_id` or `visitor_id`) is the +identifier Confidence uses for consistent assignment: the same user +always sees the same variant. + +### Per-user flags (Optimizely `userId`) + +Optimizely's `userId` (per-user identifier) is mapped to: **``** + +**Available Entity Fields:** + +--- + +## 3. Context Schema + +The context schema defines what fields Confidence expects in the +evaluation context when resolving flags — things like `country`, +`plan`, or `age` that targeting rules use to decide who gets what. + +Below is a reconciliation of what Optimizely flags need vs what already +exists in the Confidence client's schema. + +### Already in Confidence + +These fields are already defined in the `` client and match +Optimizely targeting attributes. No action needed. + +| Field | Type | Entity | Optimizely Attribute | +|-------|------|--------|---------------------| + + +### Need to Create + +These fields are used in Optimizely audience conditions but don't exist +yet in the Confidence client. They will be created during execution +using `addContextField`. + +| Field | Type | Entity | Optimizely Attribute | +|-------|------|--------|---------------------| + + +### Confidence-only (not in Optimizely) + +These fields exist in Confidence but aren't used by any Optimizely flag. +Listed for reference — no action needed. + +| Field | Type | Entity | +|-------|------|--------| + + +--- + +## 4. Flags to Migrate + +Below are the flags we're planning to migrate, along with their +targeting rules described in plain language. + +**Migration is opt-in.** Each flag starts with both checkboxes empty. +Tick `[x] Migrate` for every flag you want to bring across, or +`[x] Skip` to drop it. Flags with neither box ticked will be refused +by `execute` — no implicit defaults. + +During execution, each flag will be created one by one, interactively. + +### Flag: `` + +**Description:** +**Optimizely key:** +**Rule type:** +**Rules:** +**Rollout:** +**Variants:** +**Variables:** +**Confidence entity:** +**Action:** [ ] Migrate [ ] Skip + +**MCP Commands:** + + + +--- + +## 5. Blocked Flags + +Flags using Optimizely features that cannot be automatically migrated. + +| Flag | Blocked Reason | Recommendation | +|------|---------------|----------------| + + +--- + +## 6. Progress + +| # | Flag | Status | +|---|------|--------| +| 1 | | :white_circle: | + +``` + +--- + +## Execute: How It Works + +**`execute ` walks through the plan interactively, step by step.** + +### For Code Plans + +**Each flag = one PR.** The code migration creates a separate pull +request for each flag, keeping changes small and reviewable. + +``` +1. READ the plan file +2. SDK SETUP (Section 1 of plan) — one-time, before any flag + - Show install command from plan + - ASK: "Install SDK now? [Yes / Skip / I already did]" + - If Yes -> run install command + - Show wrapper file path + API surface from plan + - ASK: "Create the Confidence wrapper now? [Yes / Skip / I already did]" + - If Yes -> create the file using plan's API reference +3. FOR EACH FLAG in the files list: + a. Create a branch: `migrate/-to-confidence` + b. Show flag name + all files using it + c. ASK: "Transform this flag's files? [Yes / Skip / Pause]" + d. If Yes -> apply transform rules from plan to all files for this flag + e. Run lint + typecheck on changed files + f. Commit changes + g. Create PR with title: "feat: migrate from Optimizely to Confidence" + h. Show PR link + i. CHECKPOINT: "PR created. [Continue to next flag / Pause]?" + j. Wait for user response +4. COMPLETION + - Show summary: migrated vs skipped + - List all PRs created with links +``` + +### For Flag Plans + +``` +1. READ the plan file + - Client is already in the plan — use it, do NOT re-ask + - Entity (randomization unit) is already in the plan as the default + - REFUSE TO PROCEED if any flag has neither `[x] Migrate` nor + `[x] Skip` ticked. List those flags back to the user and ask + them to tick a box for each before re-running execute. Migration + is opt-in — never assume a default. +2. FOR EACH FLAG marked [x] Migrate: + - Show flag name, description, and rules in plain English + - ASK: "Create this flag in Confidence? [Yes / Skip / Pause]" + - If Yes -> run the flag setup sequence (see below) + - CHECKPOINT: "Flag done. [Continue / Pause]?" + - Wait for user response +3. COMPLETION + - Show summary: created vs skipped +``` + +**Flag Setup Sequence (MUST complete all steps before resolving):** + +Each flag MUST go through these steps in order. Do NOT call +`resolveFlag` until ALL prior steps succeed. + +``` +STEP 0: addContextField (if needed) + → Create any attribute fields required by this flag's targeting rules + that don't yet exist in Confidence (from Section 3 "Need to Create") + +STEP 1: createFlag + → If flag already exists, check the response for which clients + it's enabled on. + +STEP 2: Ensure flag is active and on the correct client + → If createFlag response does NOT list the target client: + a. Try addFlagToClient + b. If that fails with "Cannot update an archived flag": + → unarchiveFlag first, then retry addFlagToClient + → If createFlag response lists the target client: proceed + +STEP 3: addTargetingRule + → Add the targeting rule from the plan + → IMPORTANT: targeting rules added while a flag is archived OR + immediately after unarchiving may become inactive. Always complete + steps 1-2 fully (createFlag, unarchive, addFlagToClient) BEFORE + calling addTargetingRule. Do NOT add rules between createFlag and + unarchiveFlag — they will be inactive and you'll have to re-add. + +STEP 4: resolveFlag (verification) + → Only NOW resolve to verify the flag works + → MUST test BOTH positive AND negative cases: + a. Resolve with a context that SHOULD match the targeting rule + → Verify the expected variant is returned + b. Resolve with a context that SHOULD NOT match + → Verify no variant / default is returned + → For attribute-based targeting (country, plan, etc.), the resolve + call MUST include those attributes in the evaluation context. + Without them, the targeting conditions cannot be evaluated and + may appear to match when they wouldn't in production. + → If resolve fails with "No active flags found": + something went wrong in steps 1-2 — diagnose, don't skip + → If all rules show "Rule is inactive" / no match: + targeting rules were likely added while flag was archived. + Re-add the targeting rule now that the flag is active. + → Do NOT report a flag as successfully migrated until both + positive and negative resolve tests pass. +``` + +**Why this matters:** Confidence flags can be in states that +`createFlag` won't fix: archived, or enabled for a different client +only. The setup sequence handles all edge cases so resolves never +fail for avoidable reasons. + +### Rules + +- **NEVER auto-continue** — always wait for user at each checkpoint +- **Flag-by-flag** — each flag is one unit (its files + tests) +- **PR checkpoints** — offer to create PR after each flag or batch +- **Resumable** — update Progress table in plan file after each step + +--- + +## Required MCPs + +### For `plan code` + +| MCP | Tools Used | +|-----|------------| +| `confidence-docs` | `getCodeSnippetAndSdkIntegrationTips`, `searchDocumentation`, `getFullSource` | + +### For `plan flag` + +| MCP | Tools Used | +|-----|------------| +| `optimizely-exp` | List flags, get flag details, list environments, list audiences, get rulesets | +| `confidence-flags` | `listClients`, `getContextSchema`, `createFlag`, `addTargetingRule`, `resolveFlag` | diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md new file mode 100644 index 0000000..bf7f65a --- /dev/null +++ b/skills/onboard-confidence/SKILL.md @@ -0,0 +1,1545 @@ +--- +description: Create Confidence accounts and onboard users. Use when the user asks to create an account, invite users, onboard to Confidence, or check account status. +--- + +# Confidence Onboarding + +Create accounts, invite users, and get started with Confidence — all from the CLI. + +## Commands + +| Command | Description | +|---------|-------------| +| `/onboard-confidence create-account` | Create a new Confidence account | +| `/onboard-confidence invite-user` | Invite a user to an account | +| `/onboard-confidence create-client` | Create an SDK client and generate credentials | +| `/onboard-confidence setup-wizard` | Guided walkthrough: client → flag → targeting → resolve | +| `/onboard-confidence setup-warehouse` | Configure data warehouse, connectors, and assignment tables | +| `/onboard-confidence learn` | Interactive learning about experimentation concepts | +| `/onboard-confidence status` | Check current user/account status | + +--- + +## Authentication + +**Browser-based Auth0 login.** The skill opens a browser for Auth0 login (Google, email/password, SSO) and captures the token automatically. The user never touches a token. + +### Auth0 Configuration (agent-internal) + +| Parameter | Signup (create-account) | Existing account (all other commands) | +|-----------|-------------------------|---------------------------------------| +| Domain | `auth.confidence.dev` | `auth.confidence.dev` | +| Client ID | `82qMvwZvqd3t3S0gRDvs8R53TehQXSJY` | `2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w` | +| Audience | `https://confidence.dev/` | `https://confidence.dev/` | +| Scope | `openid profile email offline_access` | `openid profile email offline_access` | + +### Auth script + +Write the following to `$TMPDIR/confidence_auth.py`, substituting CLIENT_ID and optional ORGANIZATION parameter. Run with `python3 $TMPDIR/confidence_auth.py`. Outputs `TOKEN:` on success. + +```python +import http.server, urllib.parse, json, sys, subprocess, hashlib, base64, secrets, string + +code_verifier = ''.join(secrets.choice(string.ascii_letters + string.digits + '-._~') for _ in range(43)) +code_challenge = base64.urlsafe_b64encode(hashlib.sha256(code_verifier.encode()).digest()).rstrip(b'=').decode() + +port = 8084 # Fixed — must match Auth0 Allowed Callback URLs +CLIENT_ID = '' +ORGANIZATION = '' # Set after account creation, empty for signup +REDIRECT_URI = f'http://localhost:{port}/callback' +auth_code = None +error = None + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + global auth_code, error + q = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query) + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.end_headers() + if 'code' in q: + auth_code = q['code'][0] + self.wfile.write(b'

Login successful!

You can close this tab.

') + else: + error = q.get('error', ['unknown'])[0] + self.wfile.write(b'

Login failed

Please try again.

') + def log_message(self, format, *args): + pass + +params = { + 'client_id': CLIENT_ID, + 'redirect_uri': REDIRECT_URI, + 'response_type': 'code', + 'scope': 'openid profile email offline_access', + 'audience': 'https://confidence.dev/', + 'code_challenge': code_challenge, + 'code_challenge_method': 'S256', +} +if ORGANIZATION: + params['organization'] = ORGANIZATION +else: + params['screen_hint'] = 'signup' + params['prompt'] = 'login' + +authorize_url = 'https://auth.confidence.dev/authorize?' + urllib.parse.urlencode(params) +subprocess.Popen(['open', authorize_url]) +print('WAITING_FOR_LOGIN', flush=True) + +server = http.server.HTTPServer(('127.0.0.1', port), Handler) +server.timeout = 120 +while auth_code is None and error is None: + server.handle_request() +server.server_close() + +if error: + print(f'AUTH_ERROR:{error}', flush=True) + sys.exit(1) + +import urllib.request +token_data = json.dumps({ + 'grant_type': 'authorization_code', + 'client_id': CLIENT_ID, + 'code': auth_code, + 'redirect_uri': REDIRECT_URI, + 'code_verifier': code_verifier +}).encode() +req = urllib.request.Request( + 'https://auth.confidence.dev/oauth/token', + data=token_data, + headers={'Content-Type': 'application/json'} +) +try: + with urllib.request.urlopen(req) as resp: + token_response = json.loads(resp.read()) + print(f'TOKEN:{token_response["access_token"]}', flush=True) +except Exception as e: + print(f'TOKEN_ERROR:{e}', flush=True) + sys.exit(1) +``` + +**Key details:** +- Port is fixed at **8084** (must match Auth0 Allowed Callback URLs) +- For signup (`create-account`): no `organization`, add `screen_hint=signup` + `prompt=login` +- For existing account (all other commands): include `organization=` — auto-completes if browser session exists +- After `create-account`, automatically re-auth with `organization` param to get org-scoped token (browser auto-redirects, no interaction) +- If port 8084 is busy: `lsof -ti:8084 | xargs kill -9 2>/dev/null` +- All network commands require `dangerouslyDisableSandbox: true` + +### Token persistence + +After a successful login, **save the token to disk** so it survives across sessions: + +```bash +mkdir -p ~/.confidence +echo "$TOKEN" > ~/.confidence/.auth_token +chmod 600 ~/.confidence/.auth_token +``` + +**On every sub-command start**, check for a saved token before prompting login: + +```bash +if [ -f ~/.confidence/.auth_token ]; then + TOKEN=$(cat ~/.confidence/.auth_token) + # Decode JWT exp claim (handle base64 padding) + PAYLOAD=$(echo "$TOKEN" | cut -d. -f2) + EXP=$(echo "$PAYLOAD" | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d.get('exp', 0)) +") + NOW=$(date +%s) + if [ "$EXP" -gt "$NOW" ]; then + echo "VALID" # Token still good — skip login + else + echo "EXPIRED" # Token expired — re-authenticate + rm ~/.confidence/.auth_token + fi +fi +``` + +**Extract region from token** to determine API base URLs: + +```bash +REGION=$(echo "$PAYLOAD" | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d.get('https://confidence.dev/region', 'EU')) +") +``` + +Then use `${REGION,,}` (lowercase) for URL prefix: `iam.eu.confidence.dev`, `flags.eu.confidence.dev`, etc. + +If the token is valid, skip the login step entirely. If expired or missing, run the auth flow. + +### Important: gRPC-REST transcoding rules + +The Confidence APIs use gRPC with REST transcoding. The `body` field in the proto HTTP binding determines the JSON structure: + +- **`body: "client"`** → send the client object directly: `{"display_name": "iOS App"}` +- **`body: "flag"`** → send the flag object directly: `{}` +- **`body: "*"`** → send the full request message: `{"account": {...}, "billingDetails": {...}}` + +Fields NOT in the body (like `flag_id`, `parent`) become **query parameters**. + +**Field names are `snake_case`** in requests. Responses may use `camelCase`. + +### Common notes + +- All network commands require `dangerouslyDisableSandbox: true` +- Never show the token value to the user +- Always use region-specific URLs (e.g., `iam.eu.confidence.dev` not `iam.confidence.dev`) + +--- + +## User-Facing Communication Rules + +**NEVER expose internal technical details to the user.** + +- Do NOT show raw JSON request/response bodies in conversation +- Do NOT show Auth0 configuration details, token values, or OAuth internals +- DO show human-readable status updates: "Opening browser for login...", "Creating your workspace...", "Invitation sent!" +- DO describe results in plain English +- The agent handles all auth/API complexity silently + +**Step Tracker:** Display a visual step tracker at every phase transition. Update and re-display it each time you move to a new step. + +--- + +## Sub-command: create-account + +### Step Tracker + +Display at START and after EACH step completes (updating status): + +``` +───── Create Account ────────────────────────────────────── + [1] Log in ○ pending + [2] Workspace name ○ pending + [3] Account details ○ pending + [4] Create account ○ pending + [5] Connect tools ○ pending + [6] Done ○ pending +──────────────────────────────────────────────────────────── +``` + +Use `●` for completed, `▶` for in-progress, `○` for pending. + +### Step 1: Log in + +Write the auth script to `$TMPDIR/confidence_auth.py` with the **signup client ID** (`82qMvwZvqd3t3S0gRDvs8R53TehQXSJY`). Run it and parse the TOKEN from stdout. + +Tell the user: +> Opening your browser to log in. Sign up with Google or create an account with email and password. + +If login fails, show the error in plain English and offer to retry. + +### Step 2: Workspace name + +EDUCATE then ASK: + +> Your workspace name is the unique identifier for your Confidence account. +> It appears in URLs and is used to log in. +> +> **Rules:** 3-21 characters, lowercase letters, digits, and hyphens. Must start with a letter and end with a letter or digit. + +Wait for user input. Then: + +1. **Validate locally** against regex `^[a-z][a-z0-9-]{1,19}[a-z0-9]$` +2. **Check availability:** +```bash +curl -s "https://onboarding.confidence.dev/v1/loginIdAvailability:check?login_id=${LOGIN_ID}" +``` +Response: `{ "available": true/false, "message": "..." }` + +If taken, inform the user and suggest alternatives (append numbers, abbreviations). Re-ask. + +### Step 3: Account details + +Collect interactively, one field at a time: + +1. **Display name** — the human-readable name for the workspace (company name). + Validate: 3-21 characters, starts with a letter, alphanumeric + spaces + hyphens. + +2. **Region** — present as a choice: + > Where should your data be stored? This **cannot be changed later**. + > 1. EU (Europe) + > 2. US (United States) + +3. **Authentication method** — present as a choice: + > How should users log in to your workspace? + > 1. Google + > 2. Email + password + > 3. Both + +4. **Admin email** — the email of the first admin user. Must be a **work email** — free email providers (Gmail, Yahoo, etc.) are rejected by the API. + +5. **Allowed login email domains** — optional. Ask if they want to restrict login to a specific email domain (e.g., `@company.com`). + +### Step 4: Create account + +Build and send the request: + +```bash +curl -s -w "\n%{http_code}" -X POST "https://onboarding.confidence.dev/v1/accounts" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "account": { + "displayName": "", + "loginId": "", + "region": "", + "authConnections": [], + "adminEmail": "", + "allowedLoginEmailDomains": [] + } + }' +``` + +**Auth connections format:** +- Google: `[{"googleAuthConnection": {}}]` +- Password: `[{"passwordAuthConnection": {}}]` +- Both: `[{"googleAuthConnection": {}}, {"passwordAuthConnection": {}}]` + +**Success response (HTTP 200):** +```json +{ "name": "accounts/...", "externalId": "...", "loginId": "my-workspace", "displayName": "My Workspace" } +``` + +Tell the user: +> Your workspace **** has been created! +> Workspace ID: `` +> Region: +> +> You can access it at: https://confidence.spotify.com + +**Error handling:** + +| HTTP Status | Meaning | User message | +|---|---|---| +| 400 + "work email" | Free email rejected | "Confidence requires a work email address. Free providers like Gmail aren't allowed." | +| 400 + "already have an account" | Logged-in Auth0 user already has account | "This login already has a Confidence account. Log in with a different email to create a new workspace." → re-run Step 1 | +| 400 | Other validation error | Parse `.message`, show in plain English, re-collect the invalid field | +| 401 | Token expired/invalid | "Session expired. Let me log you in again." → re-run Step 1 | +| 409 | Name already taken | "That workspace name was just taken. Let's pick another." → re-run Step 2 | +| 500+ | Server error | "Something went wrong on our end. Let me try again in a moment." | + +### Step 5: Get account-scoped token + +The token from Step 1 has no `org_id` (it was issued before the account existed). Re-auth with the **regular client ID** and the `organization` parameter set to the `externalId` returned in Step 4. + +Run the auth script again with: +- `CLIENT_ID = '2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w'` (regular client) +- `ORGANIZATION = ''` + +This auto-completes in the browser — no login form, just a redirect. The new token will have `org_id`, `account_name`, and `region` claims. + +Save this token to `~/.confidence/.auth_token`. This is the token used for all subsequent commands. + +Tell the user: +> Activating your account... (browser will briefly flash) + +Then suggest connecting MCP: +> To connect Confidence tools for flag management, type `/mcp` and authenticate **confidence-flags**. +> Your browser session will auto-complete it — no extra login. + +### Step 6: Done + +Show a summary and next steps: + +``` +═══════════════════════════════════════════════════════════════ + Welcome to Confidence! +═══════════════════════════════════════════════════════════════ + + Workspace: () + Region: + Admin: + URL: https://confidence.spotify.com + + Next steps: + • Invite team members: /onboard-confidence invite-user + • Create a feature flag: Ask me to create a flag, or use + the Confidence UI + • Integrate your app: Ask me for SDK setup instructions + +═══════════════════════════════════════════════════════════════ +``` + +--- + +## Sub-command: invite-user + +### Step Tracker + +``` +───── Invite User ───────────────────────────────────────── + [1] Authenticate ○ pending + [2] Target account ○ pending + [3] Invitation details ○ pending + [4] Send invitation ○ pending +──────────────────────────────────────────────────────────── +``` + +### Step 1: Authenticate + +Check if a token is available from a prior `create-account` run in this session. + +If not, write the auth script with the **regular client ID** (`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w`) — this user already has an account. + +Validate the token works by calling: +```bash +curl -s "https://iam.confidence.dev/v1/currentUser" \ + -H "Authorization: Bearer $TOKEN" +``` + +### Step 2: Target account + +Try to identify the account automatically: + +1. If MCP is connected, call `mcp__confidence-flags__getIdentityInfo` (no args) — returns current user's identity and account +2. If MCP isn't connected, use the `/v1/currentUser` REST response +3. If the user has multiple account memberships, ask which one + +Tell the user which account will receive the invitation. + +### Step 3: Invitation details + +Ask for: + +1. **Email address(es)** — required. Accept a single email or a comma-separated list for batch invites. + Validate email format locally. + +2. **Send invitation email?** — default yes. + > Should Confidence send an invitation email? (yes/no, default: yes) + +### Step 4: Send invitation + +For each email address: + +```bash +curl -s -w "\n%{http_code}" -X POST "https://iam.confidence.dev/v1/userInvitations" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "userInvitation": { + "invitedEmail": "", + "disableInvitationEmail": + } + }' +``` + +**Success response:** +```json +{ + "name": "userInvitations/abc123", + "invitedEmail": "user@example.com", + "inviter": "Admin Name", + "expirationTime": "2026-06-03T10:00:00Z", + "invitationUri": "https://confidence.spotify.com/...", + "invitationToken": "..." +} +``` + +For single invite, tell the user: +> Invitation sent to **user@example.com**! +> They'll receive an email with instructions to join. +> The invitation expires on . + +For batch invites, show a summary table: +``` +Invitations sent: + ✓ alice@example.com — expires Jun 3 + ✓ bob@example.com — expires Jun 3 + ✗ charlie@invalid — invalid email address +``` + +**Error handling:** + +| HTTP Status | Meaning | User message | +|---|---|---| +| 400 | Invalid email | "That email address doesn't look right. Can you check it?" | +| 401 | Token expired | Re-authenticate (Step 1) | +| 403 | No permission | "You don't have permission to invite users. You need the admin role." | +| 409 | Already invited | "That user has already been invited." | + +--- + +## Sub-command: create-client + +Create an SDK client for flag resolution and generate its credentials. Uses REST APIs — no MCP needed. + +### Step Tracker + +``` +───── Create Client ─────────────────────────────────────── + [1] Client name ○ pending + [2] Create client ○ pending + [3] Get credentials ○ pending +──────────────────────────────────────────────────────────── +``` + +### Step 1: Client name + +Ask the user what to name the client. Suggest based on platform: + +> What should we call this client? (e.g., "iOS App", "Web Frontend", "Backend Service") + +### Step 2: Create client + +Body is the client object directly (proto `body: "client"`): +```bash +curl -s -w "\n%{http_code}" -X POST "https://iam.${REGION}.confidence.dev/v1/clients" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"display_name": ""}' +``` + +Response includes `name` (e.g., `clients/kqr3nc9dh70cwt5e2vws`). Save this for Step 3. + +### Step 3: Get credentials + +Body is the credential object directly (proto `body: "client_credential"`): +```bash +curl -s -w "\n%{http_code}" -X POST "https://iam.${REGION}.confidence.dev/v1/${CLIENT_NAME}/credentials" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"display_name": "Default Secret"}' +``` + +The `clientSecret.secret` is only returned once on creation — show it to the user. + +``` +═══════════════════════════════════════════════════════════════ + Client Created +═══════════════════════════════════════════════════════════════ + + Name: + Secret: + + Use this secret in your SDK configuration to resolve flags. + Keep it safe — you can regenerate it, but the old one will + stop working. + + Next: Ask me for SDK integration instructions, or run + /onboard-confidence setup-wizard + +═══════════════════════════════════════════════════════════════ +``` + +--- + +## Sub-command: setup-wizard + +Guided walkthrough of the full onboarding checklist. Uses REST APIs — no MCP needed. + +### Prerequisites + +Requires an authenticated token. If none saved, run login flow first. + +Determine the region from the token or ask the user — this sets the API base URLs: +- EU: `flags.eu.confidence.dev`, `resolver.eu.confidence.dev`, `iam.eu.confidence.dev` +- US: `flags.us.confidence.dev`, `resolver.us.confidence.dev`, `iam.us.confidence.dev` + +### Step Tracker + +``` +───── Setup Wizard ──────────────────────────────────────── + [1] Create client ○ pending + [2] Create flag ○ pending + [3] Add variants ○ pending + [4] Add targeting ○ pending + [5] Test resolve ○ pending + [6] Done ○ pending +──────────────────────────────────────────────────────────── +``` + +### Step 1: Create client + +Check if the user already has a client: +```bash +curl -s "https://iam.${REGION}.confidence.dev/v1/clients" \ + -H "Authorization: Bearer $TOKEN" +``` + +If clients exist, ask which one to use. If none, run the `create-client` flow (REST). + +Save the client `name` (e.g., `clients/abc123`) and the `clientSecret` for resolve in Step 5. If using an existing client, fetch its credentials: +```bash +curl -s "https://iam.${REGION}.confidence.dev/v1/${CLIENT_NAME}/credentials" \ + -H "Authorization: Bearer $TOKEN" +``` + +### Step 2: Create flag + +EDUCATE then ASK: +> A feature flag controls a piece of functionality. Let's create your first one. +> What should it be called? (e.g., "new-checkout-flow", "dark-mode") + +Validate: 4-63 chars, `[a-z0-9-]`. + +`flag_id` is a **query parameter**, body is the flag object (proto `body: "flag"`): +```bash +curl -s -w "\n%{http_code}" -X POST "https://flags.${REGION}.confidence.dev/v1/flags?flag_id=" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{}' +``` + +**Do NOT attach flag to client yet** — the schema update in Step 3 clears the client list. Attach after variants are added. + +### Step 3: Add variants + +EDUCATE: +> Variants are the different values a flag can have. For a simple on/off flag, you'd have "on" and "off" variants. + +Ask the user: +> What variants should this flag have? +> 1. Simple on/off (boolean) +> 2. Custom variants (I'll name them) + +**IMPORTANT: Set the flag schema BEFORE adding variants with values.** Variant values must match the schema. + +For on/off, first set schema: +```bash +curl -s -X PATCH "https://flags.${REGION}.confidence.dev/v1/flags/" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"schema": {"schema": {"enabled": {"boolSchema": {}}}}}' +``` + +For custom variants, infer the schema from the value types the user describes and set it first. + +Then create each variant (body is the variant object directly, proto `body: "variant"`): +```bash +curl -s -w "\n%{http_code}" -X POST "https://flags.${REGION}.confidence.dev/v1/flags//variants" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"name": "flags//variants/", "value": {}}' +``` + +For on/off: create "on" with `{"enabled": true}` and "off" with `{"enabled": false}`. + +**After all variants are created**, attach the flag to the client: +```bash +curl -s -X POST "https://flags.${REGION}.confidence.dev/v1/flags/:addFlagClient" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"client": "", "flag": "flags/"}' +``` + +### Step 4: Add targeting + +EDUCATE: +> Targeting rules control who sees which variant. Let's set a default — you can add more rules later. + +Ask: +> Which variant should be the default? + +**First, create a catch-all segment** (if one doesn't exist) and allocate it to 100%: +```bash +curl -s -X POST "https://flags.${REGION}.confidence.dev/v1/segments?segment_id=everyone" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"display_name": "Everyone"}' + +curl -s -X PATCH "https://flags.${REGION}.confidence.dev/v1/segments/everyone" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"allocation": {"proportion": {"value": "1"}}}' + +curl -s -X POST "https://flags.${REGION}.confidence.dev/v1/segments/everyone:allocate" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{}' +``` + +**IMPORTANT:** Segment proportion must be > 0 and `:allocate` must be called, otherwise resolve returns empty. + +Then create a rule referencing the segment (body is rule object, proto `body: "rule"`): +```bash +curl -s -w "\n%{http_code}" -X POST "https://flags.${REGION}.confidence.dev/v1/flags//rules" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "segment": "segments/everyone", + "assignment_spec": { + "bucket_count": 100, + "assignments": [{ + "assignment_id": "", + "variant": {"variant": "flags//variants/"}, + "bucket_ranges": [{"lower": 0, "upper": 100}] + }] + }, + "targeting_key_selector": "targeting_key", + "enabled": true + }' +``` + +### Step 5: Test resolve + +EDUCATE: +> Let's verify the flag works by resolving it. + +Use the **resolver API** with the client secret (not Bearer token): +```bash +curl -s -w "\n%{http_code}" -X POST "https://resolver.${REGION}.confidence.dev/v1/flags:resolve" \ + -H "Content-Type: application/json" \ + -d '{ + "flags": ["flags/"], + "evaluationContext": { + "targeting_key": "test-user-1" + }, + "clientSecret": "", + "apply": true + }' +``` + +Parse the response and show in plain English: +> Flag **** resolved to variant **** — it works! + +If resolve fails, check that the flag is attached to the client and has at least one enabled rule. + +### Step 6: Done + +``` +═══════════════════════════════════════════════════════════════ + Setup Complete! +═══════════════════════════════════════════════════════════════ + + Client: + Secret: + Flag: + Variants: + Default: + + Your flag is live and resolving. Next steps: + • Integrate the SDK: Ask me for setup instructions + • Create more flags: Ask me or use the Confidence UI + • Set up experiments: /onboard-confidence learn + +═══════════════════════════════════════════════════════════════ +``` + +--- + +## Sub-command: setup-warehouse + +Configure a data warehouse, event connectors, and assignment tables for experimentation analytics. Uses REST APIs only. + +### Prerequisites + +Requires an authenticated token. If none saved, run the Auth0 login flow first. + +**API Base URLs** (region-specific): +- Metrics: `https://metrics.confidence.dev/v1` (or `metrics.eu.` / `metrics.us.`) +- Connectors: `https://connectors.confidence.dev/v1` (or `connectors.eu.` / `connectors.us.`) + +### Step Tracker + +``` +───── Setup Warehouse ───────────────────────────────────── + [1] Choose warehouse ○ pending + [2] Configure ○ pending + [3] Validate ○ pending + [4] Create warehouse ○ pending + [5] Create connectors ○ pending + [6] Assignment table ○ pending + [7] Verify pipeline ○ pending + [8] Done ○ pending +──────────────────────────────────────────────────────────── +``` + +### Step 1: Choose warehouse type + +> Which data warehouse do you use? +> 1. BigQuery +> 2. Snowflake +> 3. Databricks +> 4. Redshift + +### Step 2: Configure + +Collect configuration based on type. Explain each field briefly. + +**BigQuery:** +- GCP project ID — the project where BigQuery datasets live +- Dataset name (default: `confidence`) +- Service account email — must have BigQuery permissions + +**Snowflake:** +- Account — your Snowflake account identifier +- User — Snowflake user for Confidence to use +- Authentication key — crypto key reference for JWT signing +- Role — Snowflake role with necessary permissions +- Warehouse — SQL warehouse for query execution +- Exposure database — database for exposure tables +- Exposure schema — schema for exposure tables + +**Databricks:** +- Host — Databricks workspace URL +- SQL warehouse ID +- Service principal client ID + secret +- Exposure schema — schema for exposure tables + +**Redshift:** +- Cluster — Redshift cluster identifier +- AWS region — select from available regions +- IAM role ARN — role Confidence assumes +- Database name +- Schema name +- Authentication — AWS access key + secret, or web identity + +### Step 3: Validate configuration + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.confidence.dev/v1/dataWarehouseConfig:validate" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "Config": { } + }' +``` + +**Response:** +```json +{ + "validation": [{ "key": "...", "description": "...", "success": true/false, "error": "..." }], + "successful": true/false, + "configurationResponse": { /* type-specific: available schemas, databases, roles */ } +} +``` + +If `successful` is false, explain each failure in plain English and **ask the user how they want to proceed:** + +> Some permissions need to be configured on your GCP project. I can fix this automatically if you have `gcloud` set up, or I can show you the exact commands to run yourself. +> +> 1. Fix it for me (requires gcloud CLI) +> 2. Show me the commands + +**If the user chooses 1 (fix it for me):** + +First check gcloud is available: `which gcloud`. If not, fall back to option 2. + +Extract the account ID from the token claim `https://confidence.dev/account_name` (e.g., `accounts/my-workspace` → `my-workspace`). The Confidence SA is: `account-${ACCOUNT_ID}@spotify-confidence.iam.gserviceaccount.com` + +For each failure, **confirm before each action:** + +**"Unable to create access token" (SERVICE_ACCOUNT):** +> Confidence needs permission to access your service account. Can I grant that now? +```bash +CONFIDENCE_SA="account-${ACCOUNT_ID}@spotify-confidence.iam.gserviceaccount.com" +gcloud iam service-accounts add-iam-policy-binding ${CUSTOMER_SA} \ + --project=${PROJECT} \ + --member="serviceAccount:${CONFIDENCE_SA}" \ + --role="roles/iam.workloadIdentityUser" --quiet +gcloud iam service-accounts add-iam-policy-binding ${CUSTOMER_SA} \ + --project=${PROJECT} \ + --member="serviceAccount:${CONFIDENCE_SA}" \ + --role="roles/iam.serviceAccountTokenCreator" --quiet +``` + +**"Missing permission 'bigquery.jobs.create'" (PERMISSIONS):** +> Your service account needs BigQuery Job User permissions. Can I grant that? +```bash +gcloud projects add-iam-policy-binding ${PROJECT} \ + --member="serviceAccount:${CUSTOMER_SA}" \ + --role="roles/bigquery.jobUser" --quiet +``` + +**"Could not find dataset" or dataset errors (DATASET):** +> The BigQuery dataset needs to be created or permissions updated. Can I do that? +```bash +bq mk --project_id=${PROJECT} --dataset --location=${REGION} ${DATASET} +bq update --project_id=${PROJECT} --source /dev/stdin ${DATASET} << EOF +{"access": [ + {"role": "WRITER", "userByEmail": "${CUSTOMER_SA}"}, + {"role": "OWNER", "specialGroup": "projectOwners"}, + {"role": "WRITER", "specialGroup": "projectWriters"}, + {"role": "READER", "specialGroup": "projectReaders"} +]} +EOF +``` + +**"free tier" / "Streaming insert is not allowed":** +> BigQuery streaming requires billing enabled on your GCP project. Can I link a billing account? +```bash +gcloud billing accounts list +gcloud billing projects link ${PROJECT} --billing-account=${BILLING_ACCOUNT} +``` +Note: billing propagation to BigQuery can take up to 15 minutes. + +After fixing, re-validate. If still failing (e.g., IAM propagation), inform the user and offer to retry. + +**If the user chooses 2 (show me the commands):** + +Show the exact gcloud/bq commands they need to run, with their specific values filled in. Format as a copyable script block: + +``` +Here's what needs to be configured on your GCP project: + +# 1. Grant Confidence access to your service account +gcloud iam service-accounts add-iam-policy-binding \ + \ + --project= \ + --member="serviceAccount:account-@spotify-confidence.iam.gserviceaccount.com" \ + --role="roles/iam.workloadIdentityUser" + +gcloud iam service-accounts add-iam-policy-binding \ + \ + --project= \ + --member="serviceAccount:account-@spotify-confidence.iam.gserviceaccount.com" \ + --role="roles/iam.serviceAccountTokenCreator" + +# 2. Grant BigQuery Job User +gcloud projects add-iam-policy-binding \ + --member="serviceAccount:" \ + --role="roles/bigquery.jobUser" + +# 3. Enable billing (if not already) +gcloud billing projects link --billing-account= + +Run these commands, then let me know and I'll retry validation. +``` + +If `configurationResponse` contains available options (schemas, roles) — present these as choices to help the user. + +### Step 4: Create warehouse + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.confidence.dev/v1/dataWarehouses" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "dataWarehouse": { + "config": { + "": { } + } + } + }' +``` + +Save the returned `name` (e.g., `dataWarehouses/...`) for reference. + +### Step 5: Create connectors + +Create both connectors: + +**Flag Applied Connection** (assignment data → warehouse): +```bash +curl -s -w "\n%{http_code}" -X POST "https://connectors.confidence.dev/v1/flagAppliedConnections" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "flagAppliedConnection": { + "bigQuery": { + "bigQueryConfig": { "serviceAccount": "...", "project": "...", "dataset": "..." }, + "table": "assignments" + } + } + }' +``` + +Adapt the destination field per warehouse type: +- BigQuery: `"bigQuery": { "bigQueryConfig": {...}, "table": "assignments" }` +- Snowflake: `"snowflake": { "snowflakeConfig": {...}, "table": "assignments" }` +- Databricks: `"databricks": { "databricksConfig": {...}, "table": "assignments" }` +- Redshift: `"redshift": { "redshiftConfig": {...}, "s3Config": {...}, "batchFileConfig": {...}, "table": "assignments" }` + +**Event Connection** (events → warehouse): +```bash +curl -s -w "\n%{http_code}" -X POST "https://connectors.confidence.dev/v1/eventConnections" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "eventConnection": { + "bigQuery": { + "bigQueryConfig": { "serviceAccount": "...", "project": "...", "dataset": "..." }, + "tablePrefix": "events_" + } + } + }' +``` + +Same destination pattern as above, but with `tablePrefix` instead of `table`. + +**Redshift/Databricks require additional config:** +- `s3Config`: `{ "bucket": "...", "region": "...", "roleArn": "..." }` — staging bucket +- `batchFileConfig`: `{ "maxEventsPerFile": 10000, "maxFileAge": "300s", "maxFileSize": 104857600 }` — batching params + +Collect these if the user chose Redshift or Databricks. + +### Step 6: Assignment table + +Create an assignment table so Confidence can analyze experiment assignments: + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.confidence.dev/v1/assignmentTables" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "assignmentTable": { + "displayName": "Assignments", + "sql": "SELECT targeting_key, rule, assignment_id, assignment_time FROM `..assignments`", + "entityColumn": { "name": "targeting_key" }, + "timestampColumn": { "name": "assignment_time" }, + "exposureKeyColumn": { "name": "rule" }, + "variantKeyColumn": { "name": "assignment_id" }, + "dataDeliveredUntilUpdateStrategyConfig": { + "strategy": "AUTOMATIC", + "automaticUpdateConfig": { + "commitDelay": "300s" + } + } + } + }' +``` + +Adapt the SQL query per warehouse type — BigQuery uses backtick-qualified tables, Snowflake uses `database.schema.table`, etc. + +### Step 7: Verify data pipeline + +Verify both connectors by generating test data and checking it lands in the warehouse. + +**7a. Verify flag assignments** + +Resolve a flag to generate assignment data (use an existing flag + client secret): +```bash +curl -s -X POST "https://resolver.${REGION}.confidence.dev/v1/flags:resolve" \ + -H "Content-Type: application/json" \ + -d '{ + "flags": ["flags/"], + "evaluation_context": {"targeting_key": "warehouse-verify-user"}, + "client_secret": "", + "apply": true + }' +``` + +If no flags exist yet, tell the user: +> No flags to test with. Run `/onboard-confidence setup-wizard` first to create a flag, then come back. + +**7b. Verify events** + +First check for an event definition to use: +```bash +curl -s "https://events.${REGION}.confidence.dev/v1/eventDefinitions" \ + -H "Authorization: Bearer $TOKEN" +``` + +If no event definitions exist, create one with a schema: +```bash +curl -s -X POST "https://events.${REGION}.confidence.dev/v1/eventDefinitions?event_definition_id=test-event" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"schema": {"action": {"stringSchema": {}}, "page": {"stringSchema": {}}}}' +``` + +If an event definition exists but has an empty schema, update it so payload data flows through: +```bash +curl -s -X PATCH "https://events.${REGION}.confidence.dev/v1/eventDefinitions/" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"schema": {"action": {"stringSchema": {}}, "page": {"stringSchema": {}}}}' +``` + +Then publish test events (uses client secret, NOT Bearer token): +```bash +NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +curl -s -X POST "https://events.${REGION}.confidence.dev/v1/events:publish" \ + -H "Content-Type: application/json" \ + -d '{ + "client_secret": "", + "events": [ + { + "event_definition": "eventDefinitions/", + "payload": {"action": "clicked_button", "page": "homepage"}, + "event_time": "'$NOW'" + } + ], + "send_time": "'$NOW'" + }' +``` + +Check response: `{"errors": []}` means success. If `EVENT_DEFINITION_NOT_FOUND`, the definition doesn't exist. If `EVENT_SCHEMA_VALIDATION_FAILED`, the payload doesn't match the schema. + +**7c. Check data in warehouse** + +Ask the user: "Want me to check the data (needs `bq` CLI), or show you the queries?" + +If user has `bq`: +```bash +echo "=== ASSIGNMENTS ===" && \ +bq query --project_id=${PROJECT} --use_legacy_sql=false \ + 'SELECT targeting_key, rule, assignment_id, assignment_time + FROM `${PROJECT}.${DATASET}.assignments` + ORDER BY assignment_time DESC LIMIT 5' && \ +echo "=== EVENTS ===" && \ +bq query --project_id=${PROJECT} --use_legacy_sql=false \ + 'SELECT * FROM `${PROJECT}.${DATASET}.events_*` + ORDER BY _event_time DESC LIMIT 5' +``` + +If user doesn't have `bq`, show the queries: +> Run these in the BigQuery console (https://console.cloud.google.com/bigquery): +> ```sql +> -- Assignments +> SELECT targeting_key, rule, assignment_id, assignment_time +> FROM `..assignments` +> ORDER BY assignment_time DESC LIMIT 5; +> +> -- Events +> SELECT * FROM `..events_*` +> ORDER BY _event_time DESC LIMIT 5; +> ``` + +**Show results:** +``` + ● Assignments: rows — data flowing + → () + ● Events: rows — data flowing + on () +``` + +**If no rows after a few seconds**, tell the user: +> Streaming can take up to a minute. Check again shortly, or verify in the BigQuery console. + +### Step 8: Done + +``` +═══════════════════════════════════════════════════════════════ + Data Warehouse Connected & Verified +═══════════════════════════════════════════════════════════════ + + Warehouse: () + Dataset: + Connectors: + ● Flag assignments → assignments table (verified) + ● Events → events_* tables (running) + Assignment: + ● Assignment table configured (auto-updating) + + Flag assignment and event data is flowing to your + warehouse. Experiment analysis is ready. + +═══════════════════════════════════════════════════════════════ +``` + +--- + +## Sub-command: learn + +Interactive learning about experimentation concepts. The skill teaches, asks questions, and the user answers — like a guided course. + +### Topics + +| Topic | Category | What it covers | +|-------|----------|----------------| +| Statistics | STATS | Statistical significance, p-values, confidence intervals, sample size | +| Experiment Design | DESIGN | Hypothesis formation, control/treatment, randomization, bias | +| Feature Flags | FLAGS | Flag types, targeting rules, rollouts, kill switches | +| Metrics | METRICS | Metric types, guardrails, primary/secondary metrics, SRM | +| Coordination | COORDINATION | Mutual exclusion, layered experiments, interaction effects | + +### Flow + +1. **Pick a topic:** + > What would you like to learn about? + > 1. Statistics fundamentals + > 2. Experiment design + > 3. Feature flags + > 4. Metrics + > 5. Coordination + +2. **Fetch content** — use `mcp__confidence-docs__searchDocumentation` to get relevant Confidence documentation for the chosen topic. + +3. **Teach** — present a concept from the docs in 2-3 clear paragraphs. Use examples relevant to the user's product. + +4. **Ask a question** — pose a comprehension question with multiple-choice answers: + > **Question:** When running an A/B test, why is it important to determine sample size before starting? + > 1. To make the test run faster + > 2. To ensure you have enough statistical power to detect the expected effect + > 3. To reduce server costs + > 4. It's not important — you can stop whenever + +5. **Evaluate the answer** — if correct, explain why. If wrong, explain the right answer and the reasoning. + +6. **Track progress** — call the Learning API to record the user's answer: + ```bash + curl -s -X POST "https://onboarding.confidence.dev/v1/learningProgress:answerQuestions" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "course": "courses/", + "questionUpdates": [{ + "lessonIndex": , + "questionIndex": , + "currentAnswerIndex": + }] + }' + ``` + +7. **Continue or finish** — after each question, ask if they want to continue or switch topics. + +8. **Show progress** — at any time, fetch and display progress: + ```bash + curl -s "https://onboarding.confidence.dev/v1/learningProgress" \ + -H "Authorization: Bearer $TOKEN" + ``` + + ``` + ───── Learning Progress ──────────────────────────────────── + Statistics: ██████░░░░ 3/5 lessons + Design: ████████░░ 4/5 lessons + Feature Flags: ██████████ 5/5 complete! + Metrics: ░░░░░░░░░░ not started + Coordination: ░░░░░░░░░░ not started + ──────────────────────────────────────────────────────────── + ``` + +### Key principles + +- **Be conversational** — this is a dialogue, not a textbook +- **Use real examples** — tie concepts to the user's product/domain when possible +- **Encourage exploration** — if the user asks follow-up questions, answer them before moving on +- **Track everything** — every answer gets recorded via the Learning API so progress persists across sessions + +--- + +## Sub-command: status + +This is a lightweight command. Try MCP first (no REST auth needed if MCP is connected). + +**If MCP is connected:** + +1. Call `mcp__confidence-flags__getIdentityInfo` (no args) +2. Call `mcp__confidence-flags__listClients` +3. Display: + +``` +═══════════════════════════════════════════════════════════════ + Confidence Account Status +═══════════════════════════════════════════════════════════════ + + Identity: () + Account: + Clients: + + MCP Status: + confidence-flags: ● connected + confidence-docs: ● connected + +═══════════════════════════════════════════════════════════════ +``` + +**If MCP is NOT connected:** + +1. Check if a token is available from a prior command in this session +2. If yes, call `GET https://iam.confidence.dev/v1/currentUser` and display the result +3. If no token, tell the user: + > No active session. Run `/onboard-confidence create-account` to get started, or `/mcp` to authenticate Confidence tools. + +--- + +## API Reference (agent-internal — do NOT show to user) + +### Base URLs + +All APIs except onboarding and Auth0 require **region-specific URLs**. Extract region from the JWT token claim `https://confidence.dev/region` (value: `EU` or `US`), lowercase it, and use as prefix. + +``` +AUTH0_DOMAIN: auth.confidence.dev +ONBOARDING_API: https://onboarding.confidence.dev/v1 (no region prefix) +IAM_API: https://iam.${region}.confidence.dev/v1 (e.g., iam.eu.confidence.dev) +FLAGS_API: https://flags.${region}.confidence.dev/v1 +RESOLVER_API: https://resolver.${region}.confidence.dev/v1 +EVENTS_API: https://events.${region}.confidence.dev/v1 +CONNECTORS_API: https://connectors.${region}.confidence.dev/v1 +METRICS_API: https://metrics.${region}.confidence.dev/v1 +``` + +### Endpoints + +**Check login ID availability (no auth):** +``` +GET ${ONBOARDING_API}/loginIdAvailability:check?login_id={id} +→ { "available": bool, "message": string } +``` + +**Check region availability (no auth):** +``` +GET ${ONBOARDING_API}/country:validate +→ { "allowed": bool } +``` + +**Create account (Bearer token):** +``` +POST ${ONBOARDING_API}/accounts +Body: { + "account": { + "displayName": string, + "loginId": string, + "region": "REGION_EU" | "REGION_US", + "authConnections": [ {"googleAuthConnection":{}} | {"passwordAuthConnection":{}} ], + "adminEmail": string (must be work email — free providers rejected), + "allowedLoginEmailDomains": [string] (optional) + }, + "marketingOptIn": bool (optional), + "userRole": string (optional), + "userGoals": [string] (optional) +} +→ { "name": string, "externalId": string, "loginId": string, "displayName": string } +``` + +**Create user invitation (Bearer token + admin permission):** +``` +POST ${IAM_API}/userInvitations +Body: { + "userInvitation": { + "invitedEmail": string, + "ttl": { "seconds": int } (optional, default 7 days), + "disableInvitationEmail": bool (optional, default false), + "labels": { string: string } (optional) + } +} +→ { + "name": string, + "invitedEmail": string, + "inviter": string, + "expirationTime": string, + "invitationUri": string, + "invitationToken": string +} +``` + +**List user invitations (Bearer token):** +``` +GET ${IAM_API}/userInvitations +→ { "userInvitations": [...], "nextPageToken": string } +``` + +**Get current user (Bearer token):** +``` +GET ${IAM_API}/currentUser +→ { + "user": { "name", "fullName", "email", ... }, + "accountMemberships": [{ "account", "displayName", "loginId", "region" }], + "account": string, + "identity": { "name", "displayName", ... } +} +``` + +**Create client (Bearer token, body: "client"):** +``` +POST ${IAM_API}/clients +Body (direct client object): { "display_name": string } +→ { "name": "clients/...", "displayName": string, ... } +``` + +**Create client credential (Bearer token, body: "client_credential"):** +``` +POST ${IAM_API}/${clientName}/credentials +Body (direct credential object): { "display_name": string } +→ { "name": "clients/.../clientCredentials/...", "clientSecret": { "secret": string }, ... } + NOTE: secret only returned once on creation +``` + +**List clients (Bearer token):** +``` +GET ${IAM_API}/clients +→ { "clients": [...], "nextPageToken": string } +``` + +**Create flag (Bearer token, body: "flag", flag_id is query param):** +``` +POST ${FLAGS_API}/flags?flag_id= +Body (direct flag object): {} + flag_id: 4-63 chars, [a-z0-9-] +→ Flag object +``` + +**Update flag schema (Bearer token, body: "flag"):** +``` +PATCH ${FLAGS_API}/flags/ +Body: { "schema": { "schema": { "": { "boolSchema": {} | "stringSchema": {} | "intSchema": {} | "doubleSchema": {} } } } } +→ Flag object + NOTE: schema MUST be set before adding variants with values +``` + +**Add flag to client (Bearer token, body: "*"):** +``` +POST ${FLAGS_API}/flags/:addFlagClient +Body: { "client": "clients/", "flag": "flags/" } +→ Flag object +``` + +**Create variant (Bearer token, body: "variant"):** +``` +POST ${FLAGS_API}/flags//variants +Body (direct variant object): { "name": "flags//variants/", "value": { ... } } +→ Variant object + NOTE: value fields must match the flag schema +``` + +**Create rule (Bearer token, body: "rule"):** +``` +POST ${FLAGS_API}/flags//rules +Body (direct rule object): { "assignment_spec": { ... }, "targeting_key_selector": "targeting_key", "enabled": true } +→ Rule object +``` + +**Resolve flags (client secret — NOT Bearer token):** +``` +POST ${RESOLVER_API}/flags:resolve +Body: { + "flags": ["flags/"], + "evaluationContext": { "targeting_key": string, ... }, + "clientSecret": string, + "apply": bool +} +→ { "resolvedFlags": [{ "flag": string, "variant": string, "value": {...}, "reason": string }] } +``` + +**List event definitions (Bearer token):** +``` +GET https://events.${region}.confidence.dev/v1/eventDefinitions +→ { "eventDefinitions": [...], "nextPageToken": string } +``` + +**Create event definition (Bearer token):** +``` +POST https://events.${region}.confidence.dev/v1/eventDefinitions?event_definition_id= +Body (direct object): { "schema": { "": { "stringSchema": {} | "intSchema": {} | "doubleSchema": {} | "boolSchema": {} } } } +→ EventDefinition object +``` + +**Update event definition schema (Bearer token):** +``` +PATCH https://events.${region}.confidence.dev/v1/eventDefinitions/ +Body: { "schema": { "": { "stringSchema": {} } } } +→ EventDefinition object + NOTE: schema fields determine which payload fields appear as columns in warehouse +``` + +**Publish events (client secret — NOT Bearer token):** +``` +POST https://events.${region}.confidence.dev/v1/events:publish +Body: { + "client_secret": string, + "events": [{ "event_definition": "eventDefinitions/", "payload": {...}, "event_time": "ISO8601" }], + "send_time": "ISO8601" +} +→ { "errors": [{ "index": int, "reason": string, "message": string }] } + Empty errors array = success +``` + +**Create data warehouse (Bearer token):** +``` +POST ${METRICS_API}/dataWarehouses +Body: { "dataWarehouse": { "config": { "Config": {...} } } } +→ DataWarehouse object +``` + +**Validate warehouse config (Bearer token):** +``` +POST ${METRICS_API}/dataWarehouseConfig:validate +Body: { "Config": {...} } +→ { "validation": [...], "successful": bool, "configurationResponse": {...} } +``` + +**Check warehouse exists (Bearer token):** +``` +GET ${METRICS_API}/dataWarehouses:exists +→ { "exists": bool } +``` + +**Create flag applied connection (Bearer token):** +``` +POST ${CONNECTORS_API}/flagAppliedConnections +Body: { "flagAppliedConnection": { "": { "Config": {...}, "table": "..." } } } +→ FlagAppliedConnection object +``` + +**Create event connection (Bearer token):** +``` +POST ${CONNECTORS_API}/eventConnections +Body: { "eventConnection": { "": { "Config": {...}, "tablePrefix": "..." } } } +→ EventConnection object +``` + +**Create assignment table (Bearer token):** +``` +POST ${METRICS_API}/assignmentTables +Body: { "assignmentTable": { "displayName": str, "sql": str, "entityColumn": {...}, "timestampColumn": {...}, "exposureKeyColumn": {...}, "variantKeyColumn": {...}, "dataDeliveredUntilUpdateStrategyConfig": {...} } } +→ AssignmentTable object +``` + +**Get learning progress (Bearer token):** +``` +GET https://onboarding.confidence.dev/v1/learningProgress +→ { "courseProgresses": [...], "completedCourses": int } +``` + +**Answer questions (Bearer token):** +``` +POST https://onboarding.confidence.dev/v1/learningProgress:answerQuestions +Body: { "course": "courses/", "questionUpdates": [{ "lessonIndex": int, "questionIndex": int, "currentAnswerIndex": int }] } +→ LearningProgress object +``` + +### Validation Rules + +| Field | Rule | Regex | +|-------|------|-------| +| `loginId` | 3-21 chars, lowercase, digits, hyphens. Starts with letter, ends with letter/digit | `^[a-z][a-z0-9-]{1,19}[a-z0-9]$` | +| `displayName` | 3-21 chars, letters, digits, spaces, hyphens. Starts with letter, ends with letter/digit | `^[a-zA-Z][a-zA-Z0-9\s-]{1,19}[a-zA-Z0-9]$` | +| `region` | Exactly `REGION_EU` or `REGION_US` | — | +| `authConnections` | At least one required | — | +| `adminEmail` | Must be a work email. Free providers (Gmail, Yahoo, Hotmail, etc.) are rejected | — | + +--- + +## Error Handling Reference (agent-internal) + +### Common HTTP errors + +| Status | Meaning | Recovery | +|--------|---------|----------| +| 400 | Validation error | Parse `.message`, show plain English, re-collect invalid field | +| 401 | Invalid/expired token | Re-trigger Auth0 login | +| 403 | Insufficient permissions | Explain needed role/permission | +| 404 | Resource not found | Check account/resource exists | +| 409 | Conflict (already exists) | Name taken or user already invited | +| 429 | Rate limited | Wait briefly and retry | +| 500+ | Server error | Inform user, suggest retry | + +### Sandbox note + +All `curl`, `open`, and `python3` commands that access external hosts (`auth.confidence.dev`, `onboarding.confidence.dev`, `iam.confidence.dev`) require `dangerouslyDisableSandbox: true`. On first occurrence, briefly explain to the user that network access outside the sandbox is needed for API calls. + +--- + +## Required MCP Tools (optional — only for `status` and `learn`) + +Most sub-commands use REST APIs and do NOT require MCP. MCP is only used as a convenience: + +| Tool | Used by | Purpose | +|------|---------|---------| +| `mcp__confidence-flags__getIdentityInfo` | `status` | Get current identity (convenience) | +| `mcp__confidence-flags__listClients` | `status` | List available clients (convenience) | +| `mcp__confidence-docs__searchDocumentation` | `learn` | Fetch educational content | + +**All other sub-commands (`create-account`, `invite-user`, `create-client`, `setup-wizard`, `setup-warehouse`) work entirely via REST APIs with the saved auth token.** + +--- + +## Known Limitations + +- **MCP auth cannot be triggered programmatically** — user must run `/mcp` to authenticate MCP servers. The Auth0 browser session from the login step makes this instant (no second login). +- **Port 8084 must be free** — the Auth0 callback server uses a fixed port. If busy, kill the process first. +- **Auth0 Allowed Callback URLs** — both Auth0 clients must have `http://localhost:8084/callback` in their Allowed Callback URLs, Allowed Logout URLs, and Allowed Web Origins. +- **Learning API** — REST-only (gRPC on epx-onboarding). Course content is generated by the skill using docs MCP; the API only tracks progress indices. +- **`learn` sub-command** — uses docs MCP for content. If MCP not connected, the skill can still teach using its own knowledge but won't have the latest docs. +- **Region-specific API URLs** — flags/resolver APIs use region prefixes (`flags.eu.confidence.dev` vs `flags.us.confidence.dev`). Determine region from the JWT token or from the account creation step. From 5df96475ee9da601ca6dbc3d98c8376b7afe77e3 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:06:08 +0200 Subject: [PATCH 02/25] refactor: remove optimizely from ai-onboarding branch Keep this branch focused on onboarding skill and multi-client support. Co-Authored-By: Claude Opus 4.6 (1M context) --- CLAUDE.md | 2 - README.md | 4 +- commands/migrate-optimizely.md | 9 - skills/migrate-optimizely/SKILL.md | 1271 ---------------------------- 4 files changed, 1 insertion(+), 1285 deletions(-) delete mode 100644 commands/migrate-optimizely.md delete mode 100644 skills/migrate-optimizely/SKILL.md diff --git a/CLAUDE.md b/CLAUDE.md index 0e30362..cc81e35 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,13 +5,11 @@ This plugin integrates Confidence with Claude Code, providing tools for feature ## Commands - `/confidence:migrate-posthog >` — Migrate feature flags from PostHog to Confidence SDK -- `/confidence:migrate-optimizely >` — Migrate feature flags from Optimizely to Confidence SDK - `/confidence:onboard-confidence ` — Create accounts, onboard users, set up SDK clients, configure warehouses, and learn experimentation concepts ## Skills - **migrate-posthog** — Auto-triggers when the user asks to migrate PostHog flags or transform SDK code to Confidence -- **migrate-optimizely** — Auto-triggers when the user asks to migrate Optimizely flags or transform SDK code to Confidence - **onboard-confidence** — Auto-triggers when the user asks to create a Confidence account, invite users, set up SDK clients, configure warehouses, run the setup wizard, or learn about experimentation ## MCP Servers diff --git a/README.md b/README.md index 430ce6a..46a2316 100644 --- a/README.md +++ b/README.md @@ -83,12 +83,11 @@ This plugin provides access to Confidence tools across these categories: - **Feature flags** — Create, list, update, archive, resolve, and target feature flags - **Documentation** — Search Confidence docs and SDK integration guides -- **Migration** — Migrate feature flags from PostHog or Optimizely to Confidence +- **Migration** — Migrate feature flags from PostHog to Confidence ## Slash Commands - `/confidence:migrate-posthog` — Migrate feature flags from PostHog to Confidence SDK -- `/confidence:migrate-optimizely` — Migrate feature flags from Optimizely to Confidence SDK - `/confidence:onboard-confidence` — Create Confidence accounts and onboard users ## Example Usage @@ -98,7 +97,6 @@ This plugin provides access to Confidence tools across these categories: > Create a flag called new-checkout with a boolean schema > /confidence:migrate-posthog plan flag > /confidence:migrate-posthog plan code -> /confidence:migrate-optimizely ``` ## MCP Servers diff --git a/commands/migrate-optimizely.md b/commands/migrate-optimizely.md deleted file mode 100644 index a0fe22f..0000000 --- a/commands/migrate-optimizely.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: migrate-optimizely -description: Migrate feature flags from Optimizely to Confidence -argument-hint: [plan flag | plan code | execute ] ---- - -All migration instructions are maintained in `skills/migrate-optimizely/SKILL.md` to prevent divergence. - -**Before doing anything else**, use the Read tool to read `skills/migrate-optimizely/SKILL.md` and follow those instructions to handle this command. diff --git a/skills/migrate-optimizely/SKILL.md b/skills/migrate-optimizely/SKILL.md deleted file mode 100644 index dfda792..0000000 --- a/skills/migrate-optimizely/SKILL.md +++ /dev/null @@ -1,1271 +0,0 @@ ---- -description: Migrate feature flags from Optimizely to Confidence SDK. Use when the user says /migrate-optimizely, asks to migrate Optimizely flags, or transform SDK code to Confidence. ---- - -# Optimizely to Confidence Migration - -MCP-driven, self-sufficient migration from Optimizely to Confidence. - -## Migration Flow - -The migration happens in two phases: **flags first, then code**. - -``` -Phase 1: Flag Definitions - plan flags → Scan Optimizely, choose environment & client & entity, generate plan - execute → Create each flag in Confidence with targeting rules - -Phase 2: Code Transformation - plan code → Scan codebase, fetch SDK guide, generate transform rules - execute → Transform code flag by flag, each flag = one PR -``` - -**Why flags first?** The flags need to exist in Confidence before the -code can resolve them. Once flags are live in Confidence, you migrate -the code that evaluates them — one flag at a time, one PR at a time. - -**Each code PR is scoped to a single flag.** This keeps PRs small, -reviewable, and independently shippable. If one flag's migration has -issues, it doesn't block the others. - -## Commands - -| Command | Description | -|---------|-------------| -| `/migrate-optimizely plan flags` | Phase 1: plan flag definitions migration | -| `/migrate-optimizely plan code` | Phase 2: plan code transformation | -| `/migrate-optimizely execute ` | Execute a plan interactively | - ---- - -## Migration Overview (MUST display at start of `plan flags` or `plan code`) - -**Every time** the user runs `plan flags` or `plan code`, display this -overview FIRST — before doing any work. This orients the user on where -they are in the full migration journey. - -``` -═══════════════════════════════════════════════════════════════ - Optimizely → Confidence Migration -═══════════════════════════════════════════════════════════════ - - The migration happens in two phases: flags first, then code. - - ┌─────────────────────────────────────────────────────────┐ - │ PHASE 1 — Flag Definitions │ - │ │ - │ Move all flags from Optimizely to Confidence with │ - │ their targeting rules, rollout percentages, variants, │ - │ and variables. │ - │ │ - │ Steps: │ - │ 1. Scan all flags in Optimizely │ - │ 2. Choose which Optimizely environment to migrate │ - │ 3. Choose a Confidence client (your app) │ - │ 4. Map randomization units (user_id, etc.) │ - │ 5. Generate migration plan with targeting rules │ - │ 6. Execute: create each flag in Confidence │ - │ │ - │ Result: All flags live in Confidence, ready to resolve│ - ├─────────────────────────────────────────────────────────┤ - │ PHASE 2 — Code Transformation │ - │ │ - │ Once flags exist in Confidence, migrate the code that │ - │ evaluates them. Each flag = one PR. │ - │ │ - │ Steps: │ - │ 1. Detect language & framework │ - │ 2. Fetch Confidence SDK guide │ - │ 3. Scan codebase for Optimizely usage │ - │ 4. Generate transform rules (Optimizely→Confidence) │ - │ 5. Generate plan grouped by flag │ - │ 6. Execute: transform code flag by flag, one PR each│ - │ │ - │ Result: Code uses Confidence SDK, Optimizely removed │ - └─────────────────────────────────────────────────────────┘ - - Why flags first? - Flags must exist in Confidence before code can resolve them. - - Why one PR per flag? - Keeps changes small, reviewable, and independently shippable. - If one flag's migration has issues, it doesn't block the others. - -═══════════════════════════════════════════════════════════════ -``` - -After displaying the overview, indicate which phase the user is about -to enter: - -- For `plan flags`: "Starting **Phase 1** — Flag Definitions" -- For `plan code`: "Starting **Phase 2** — Code Transformation. - Make sure Phase 1 (flag definitions) is complete first — the flags - need to exist in Confidence before the code can resolve them." - -Then proceed with the normal workflow for that phase. - ---- - -## SDK Preference - -**ALWAYS prefer OpenFeature with local resolve.** - -| Priority | Approach | When to use | -|----------|----------|-------------| -| 1st | Local resolve | Default for all new integrations | -| 2nd | Remote resolve | Only if local resolve not supported for platform | -| Avoid | Direct SDK | Being phased out | - ---- - -## Plan Philosophy - -**Plans must be MCP-boxed, self-sufficient, and agent-agnostic.** - -| Principle | Meaning | -|-----------|---------| -| **MCP-boxed** | Every external data fetch uses explicit MCP tool calls | -| **Self-sufficient** | Plan contains ALL information needed - no "query MCP for X" | -| **Agent-agnostic** | Any agent with MCPs can execute without prior context | -| **Language-agnostic** | Detect framework, fetch SDK guide from MCP dynamically | - ---- - -## Prerequisites - -Before starting any workflow, check that required MCP servers are available. -Try calling a simple tool from each. If it fails, install the missing MCP. - -### Optimizely MCP - -The Optimizely Feature Experimentation MCP server provides tools to list -projects, flags, experiments, environments, audiences, and manage them. - -**Discovery:** The exact tool names depend on the MCP server version. After -installing, use ToolSearch to discover available tools matching `optimizely`. -Common patterns: list projects, list flags, get flag, list environments, -list audiences, get ruleset. - -If not available, install it: -``` -claude mcp add optimizely-exp --transport http --url https://exp.mcp.opal.optimizely.com/mcp -``` - -The user will be prompted to authenticate via Opti ID (OAuth) in their browser. - -### Confidence MCP - -Test: `mcp__confidence-flags__listClients` - -If not available, install it: -``` -claude mcp add confidence-flags --transport http --url https://mcp.confidence.dev/mcp/flags -``` - -The user will be prompted to authenticate via OAuth in their browser. - -### Confidence Docs MCP (for `plan code` only) - -Test: `mcp__confidence-docs__searchDocumentation` - -If not available, install it: -``` -claude mcp add confidence-docs --transport http --url https://mcp.confidence.dev/mcp/docs -``` - ---- - -## User-Facing Communication Rules - -**NEVER expose internal technical details to the user.** The user should see -human-readable descriptions of what's happening, not internal implementation -details like targeting payload formats, rule types, or operator names. - -- Do NOT say "creating plan based on eqRule / rangeRule / setRule" etc. -- Do NOT show raw targeting payloads or JSON structures in conversation -- DO say things like: "Creating flag with rule: plan equals 'pro' AND country is US or UK" -- DO describe rules in plain English: "age between 18 and 65", "plan is not free" -- The plan FILE may contain MCP command payloads (for machine execution), - but conversation output must be human-friendly - -**Step Tracker:** Display a visual step tracker at every phase transition. -The tracker shows all phases, marks completed ones, highlights the current -one, and shows remaining ones. Update and re-display it each time you move -to a new phase. - -### Plan Flags Step Tracker - -Display this at the START and after EACH step completes (updating status): - -``` -───── Plan Flags ────────────────────────────────────────── - [1] Scan Optimizely ○ pending - [2] Choose environment ○ pending - [3] Choose client ○ pending - [4] Map entities ○ pending - [5] Generate plan ○ pending -──────────────────────────────────────────────────────────── -``` - -Status markers: -- `○ pending` — not started yet -- `◉ in progress` — currently running -- `⏸ awaiting user` — blocked on user input (e.g. picking a client or entity) -- `✓ done` — completed (add brief user-facing result) -- `⊘ skipped` — skipped by user - -Use `⏸ awaiting user` whenever the workflow has asked a question and is -waiting for an explicit reply. This makes "I'm blocked on you" visible -to both agent and user, and prevents the agent from drifting into -auto-progression while a question is open. - -**IMPORTANT:** Never expose internal/technical details in the tracker. -No pagination info, no API page counts, no internal field names. -Show only what matters to the user. - -Example after Step 1 completes: -``` -───── Plan Flags ────────────────────────────────────────── - [1] Scan Optimizely ✓ 12 flags found - [2] Choose environment ◉ in progress - [3] Choose client ○ pending - [4] Map entities ○ pending - [5] Generate plan ○ pending -──────────────────────────────────────────────────────────── -``` - -### Execute Step Tracker - -Display this at the START and update after EACH flag: - -``` -───── Execute Migration ─────────────────────────────────── - Client: test | Entity: user_id | Flags: 12 - Progress: [░░░░░░░░░░░░░░░░░░░░] 0/12 -──────────────────────────────────────────────────────────── -``` - -Update the progress bar as flags are processed. Use `█` for completed -and `░` for remaining. The bar should be 20 characters wide. - -Examples at various stages: -``` - Progress: [██████░░░░░░░░░░░░░░] 4/12 (1 skipped) - Current: checkout-redesign -``` - -``` - Progress: [████████████████████] 12/12 done - Result: 11 migrated, 1 skipped -``` - -After each flag completes, show: -``` - ✓ checkout-redesign — MATCH (enabled) -``` - -After a skip: -``` - ⊘ legacy-banner — skipped -``` - -### Final Summary (Execute) - -At the end of execution, show a complete summary: - -``` -───── Migration Complete ────────────────────────────────── - Progress: [████████████████████] 12/12 done - Migrated: 11 | Skipped: 1 | Failed: 0 - - ✓ checkout-redesign 100% user_id - ✓ pricing-experiment 50/50 user_id - ⊘ legacy-banner — skipped - ✓ dark-mode 25% user_id - ... -──────────────────────────────────────────────────────────── -``` - ---- - -## Confidence Naming Rules - -- **Flag names:** lowercase letters, digits, and hyphens only (`[a-z0-9-]`) -- **Optimizely flag keys** may contain underscores — convert them to hyphens - when creating in Confidence (e.g. `product_sort` → `product-sort`) -- **Entity references:** Confidence entity names do NOT support underscores. - The entity reference (e.g. `entities/company`) is separate from the context - field name (e.g. `company_id`). When creating entity fields with - `addContextField`, always provide an explicit `entityReference` with a - clean name (no underscores). If omitted, the tool auto-generates one from - the field name which will fail. - - | Field name | Entity reference | Works? | - |------------|-----------------|--------| - | `user_id` | `entities/user` | Yes | - | `company_id` | `entities/company` | Yes | - | `visitor_id` | `entities/visitor` | Yes | - | `company_id` | *(omitted — auto: `entities/company_id`)* | **No** | - ---- - -## Plan Code: Workflow - -### Resume Check (MUST do first) - -Same as Plan Flag: check for existing `.claude/plans/optimizely-code-migration-*.md`. -If found with incomplete `Generation Status`, resume from the last -incomplete step. If complete, ask user if they want to start fresh. -If not found, start fresh. - -The plan file uses the same progressive pattern: created at Step 1, -updated after each step, with a `## Generation Status` section. - -### Step 1: Detect Language & Framework - -``` -Grep: pattern="optimizely|Optimizely" -> Find Optimizely usage -Glob: pattern="package.json" or "build.gradle" or "Cargo.toml" etc -Read: dependency file -> Determine language/framework -``` - -**Optimizely SDK packages to detect:** - -| Language | Package/Import | -|----------|---------------| -| JS/TS (Node) | `@optimizely/optimizely-sdk` | -| React | `@optimizely/react-sdk` | -| React Native | `@optimizely/react-native-sdk` | -| Python | `optimizely` (PyPI: `optimizely-sdk`) | -| Java | `com.optimizely.ab:core-api` or `com.optimizely.ab.*` | -| Go | `github.com/optimizely/go-sdk` or `github.com/optimizely/go-sdk/v2` | -| Ruby | `optimizely-sdk` (gem) | -| PHP | `optimizely/php-sdk` | -| C# | `Optimizely.Sdk` | -| Swift | `OptimizelySwiftSdk` | -| Android | `com.optimizely.ab:android-sdk` | -| Flutter | `optimizely_flutter_sdk` | - -### Step 2: Fetch SDK Guide from MCP - -**Query confidence-docs MCP based on detected language:** - -``` -mcp__confidence-docs__getCodeSnippetAndSdkIntegrationTips - sdk: "" -``` - -``` -mcp__confidence-docs__searchDocumentation - query: "OpenFeature local resolve " -``` - -``` -mcp__confidence-docs__getFullSource - source: "https://confidence.spotify.com/docs/sdks/server/" -``` - -**CRITICAL:** Include the ACTUAL response in the plan, not a reference to fetch it. - -### Step 3: Scan Codebase for Optimizely Usage - -``` -Grep: pattern="" -> Find all usages -``` - -**Must detect BOTH modern and legacy Optimizely API patterns:** - -**Modern API (recommended — `decide` method):** - -| Pattern | Language | -|---------|----------| -| `createInstance(` | JS/TS | -| `createUserContext(` | All languages | -| `.decide(` / `user.decide(` | All languages | -| `decision.enabled` | All languages | -| `decision.variationKey` / `decision.variation_key` | JS / Python | -| `decision.variables` | All languages | -| `user.trackEvent(` / `user.track_event(` | All languages | -| `OptimizelyProvider` | React | -| `useDecision(` | React | - -**Legacy API (deprecated but still common):** - -| Pattern | Modern equivalent | -|---------|------------------| -| `isFeatureEnabled(flagKey, userId, attrs)` | `user.decide(flagKey).enabled` | -| `activate(experimentKey, userId, attrs)` | `user.decide(flagKey).variationKey` | -| `getVariation(experimentKey, userId, attrs)` | `user.decide(flagKey).variationKey` | -| `getFeatureVariableString(flagKey, varKey, userId, attrs)` | `user.decide(flagKey).variables[varKey]` | -| `getFeatureVariableInteger(...)` | `user.decide(flagKey).variables[varKey]` | -| `getFeatureVariableBoolean(...)` | `user.decide(flagKey).variables[varKey]` | -| `getFeatureVariableDouble(...)` | `user.decide(flagKey).variables[varKey]` | -| `getFeatureVariableJSON(...)` | `user.decide(flagKey).variables[varKey]` | -| `getAllFeatureVariables(flagKey, userId, attrs)` | `user.decide(flagKey).variables` | -| `track(eventKey, userId, attrs)` | `user.trackEvent(eventKey)` | - -Group files by flag key they reference. - -### Step 4: Generate Transform Rules - -Based on SDK guide from MCP: -- Extract install commands -- Extract initialization code -- Extract flag evaluation API -- Generate find/replace rules matching Optimizely → Confidence patterns - -### Step 5: Generate Plan - -Save to `.claude/plans/optimizely-code-migration-.md` - ---- - -## Plan Code: Template - -```markdown -# Optimizely to Confidence Code Migration Plan - -**Created:** -**Scope:** Code transformation only -**Language:** -**Framework:** - ---- - -## 1. SDK Setup - -### Install - - - -### API Reference (from MCP: confidence-docs) - - - -### Create Confidence Wrapper - -**File:** - -**Must match Optimizely API surface:** - -| Method | Signature | -|--------|-----------| - - ---- - -## 2. Transform Rules - -### Source Files - -| Find | Replace | -|------|---------| -| | | -| | | - -### Test Files - -| Find | Replace | -|------|---------| -| | | - ---- - -## 3. Files to Transform - - - ---- - -## 4. Progress - -| # | Item | Status | -|---|------|--------| -| 0 | SDK Setup | :white_circle: | - -``` - ---- - -## Plan Flag: Workflow - -### Resume Check (MUST do first) - -Before starting, check for an existing in-progress plan: - -``` -Glob: .claude/plans/optimizely-flag-migration-*.md -``` - -If a plan file exists, read its `## Generation Status` section: -- If status is `complete` → tell user a plan already exists, ask if - they want to start fresh or use the existing one -- If status is NOT `complete` → **resume from the last incomplete step** - Tell the user: "Found an in-progress plan. Resuming from step ." -- If no plan file exists → start fresh - -### Progressive Plan File - -The plan file is created at the START (Step 1) and updated after EACH -step. This means if the session closes, the file has partial progress -that can be resumed. - -**File path:** `.claude/plans/optimizely-flag-migration-.md` - -The plan file MUST include a `## Generation Status` section at the top -(right after the title) that tracks which steps are done: - -```markdown -## Generation Status - -| Step | Status | Result | -|------|--------|--------| -| 1. Scan Optimizely | ✓ complete | 12 flags | -| 2. Choose environment | ✓ complete | production | -| 3. Choose client | ✓ complete | test | -| 4. Map entities | ○ not started | | -| 5. Generate rules | ○ not started | | -``` - -Status values: `✓ complete`, `◉ in progress`, `○ not started` - -**After each step completes**, update the status table AND write that -step's data to the plan file. Do NOT wait until the end to write. - -### Step 1: Scan Optimizely Flags - -Use the Optimizely MCP to list all flags in the project. - -**Discovery:** Use ToolSearch to find available Optimizely MCP tools. -Look for tools that list flags/features in a project. Typical patterns: -- List all flags in a project -- Get flag details (variations, variables) -- List environments -- List audiences -- Get ruleset for a flag in an environment - -**CRITICAL: Paginate until ALL flags are fetched.** If the MCP supports -pagination, keep fetching until all flags are returned. - -For each flag found, gather: -- Flag key and name/description -- Variations (names, keys, variable overrides) -- Variables (name, type, default value) -- Available environments - -**After scan completes:** Write the flag data to the plan file and -update Generation Status step 1 to `✓ complete`. - -### Step 2: Choose Optimizely Environment - -Optimizely flags have different rules per environment (e.g. production, -development, staging). The user must choose which environment's rules -to migrate. - -**EDUCATE then ASK the user:** - -> **What is an environment?** -> In Optimizely, each flag can have different targeting rules and rollout -> percentages per environment. For example, a flag might be 100% rolled -> out in development but only 10% in production. -> -> Which environment's rules should I migrate? -> -> Your environments: -> 1. production -> 2. development -> 3. - -**Wait for an explicit pick.** Set the step to `⏸ awaiting user` and -stop. A re-run of `/migrate-optimizely`, an empty message, or any reply -that is not a valid choice is **not** consent — NEVER infer from silence. -If the reply is ambiguous, re-ask. - -After the user picks an environment, fetch the ruleset for each flag -in that environment. For each flag, extract: -- Rule type: **Targeted Delivery** (rollout) or **A/B Test** (experiment) -- Audience conditions (targeting rules) -- Traffic allocation (variant percentages) -- Whether the ruleset is enabled - -**After environment selected and rulesets fetched:** Write Section 1b -(Environment & Rules) to plan file and update Generation Status step 2 -to `✓ complete`. - -### Step 3: Select Confidence Client - -``` -mcp__confidence-flags__listClients -``` - -**EDUCATE then ASK the user:** - -> **What is a client?** -> A client represents the application that resolves flags — your website, -> backend service, or mobile app. Each client has its own secret for -> authentication and can be scoped to environments (dev, staging, prod). -> Flags are associated with one or more clients, so Confidence knows which -> application should receive which flags. -> -> Think of it like: "Where will these flags be evaluated?" -> -> Your existing clients: -> 1. -> 2. -> ... -> N. Create a new client -> -> Which client should I use as the default for all flags? -> You can always rearrange them later in the Confidence UI. - -**Wait for an explicit pick.** Set the step to `⏸ awaiting user` and -stop. A re-run of `/migrate-optimizely`, an empty message, or any reply -that is not a number from the list / `new ` is **not** consent — -NEVER infer the recommendation from silence. If the reply is ambiguous, -re-ask, listing the choices again. - -- If user picks existing → use it -- If user wants new → ASK for name → `mcp__confidence-flags__createClient` - -**After client selected:** Write Section 1 (Default Client) to plan -file and update Generation Status step 3 to `✓ complete`. - -### Step 4: Map Randomization Units - -``` -mcp__confidence-flags__getContextSchema clientName: "" -``` - -Show the user entity fields (fields marked as entity in the schema). - -This step maps Optimizely's `userId` to Confidence entity fields. - -**EDUCATE then ASK:** - -> **What is a randomization unit (entity)?** -> An entity is the "thing" that gets randomly assigned to a variant — -> usually a user. The entity field (like `user_id` or `visitor_id`) is -> the identifier Confidence uses to ensure **consistent assignment**: the -> same user always sees the same variant. -> -> In Confidence, it maps to the `targeting_key` in the evaluation context. - -> All of your flags randomize per user. In Optimizely, each user is -> identified by `userId` (passed to `createUserContext`). In Confidence, -> you need to pick which field represents the same user identifier. -> -> Common choices: -> - **user_id** — if your flags target authenticated users -> - **visitor_id** — if targeting anonymous visitors (auto-generated by -> Confidence client SDKs) -> -> Your client's existing entity fields: -> 1. -> 2. -> ... -> N. Create a new field -> -> Which Confidence field represents the same user as Optimizely's `userId`? - -**Wait for an explicit pick.** Same rule as Step 3 — set the step to -`⏸ awaiting user` and stop. Silence, a re-run, or any non-listed reply -is **not** consent. Re-ask if the reply is ambiguous. - -- If user picks existing → use it as `targetingKey` for all flags -- If user wants new → ASK for name + type → `mcp__confidence-flags__addContextField` - -**Note on Optimizely group experiments:** Optimizely does not have native -group bucketing like PostHog's `aggregation_group_type_index`. If an -Optimizely project uses group-level experiments (passing a group ID as -`userId`), the user should create a separate entity field for that group -identifier. Flag the user if any flags appear to use non-user IDs. - -**Step 4 only creates entity fields.** Attribute fields used in -targeting rules (`country`, `plan`, `age`, etc.) MUST NOT be created -here. Record them in Section 3 "Need to Create" and let `execute` -create them — that way, if the user later skips a flag, no orphan -schema fields are left in Confidence. - -**After entity mapped:** Write Section 2 (Randomization Mapping) to -plan file, reconcile and write Section 3 (Context Schema), and update -Generation Status step 4 to `✓ complete`. - -### Step 5: Generate MCP Commands - -**Confirmation gate (MUST pass before generating).** Before writing -Section 4, summarize chosen environment, client + entity in chat and ask: - -> Plan will migrate rules from Optimizely environment `` to -> Confidence client `` with randomization entity ``. -> All flags will be defaulted to `[ ] Migrate [ ] Skip` -> (neither pre-checked) — you'll opt each one in during review. -> Confirm or change? - -Set the step to `⏸ awaiting user` and stop. Only proceed on an -explicit `yes` / `confirm` / equivalent. A re-run or ambiguous reply -is **not** confirmation. - -For each flag in Section 4, generate the MCP command payloads -(createFlag, addFlagToClient, addTargetingRule, resolveFlag) using the -Operator Mapping Reference (below). Write them into each flag's section. - -**After all commands generated:** Update Generation Status step 5 to -`✓ complete` and set the overall status to `complete`. Write the -Progress table (Section 6). - -**Tell the user:** -> Plan generated! Review it at `.claude/plans/optimizely-flag-migration-.md` -> -> Migration is **opt-in**: every flag starts with both checkboxes -> empty. Tick `[x] Migrate` or `[x] Skip` for each flag — `execute` -> will refuse any flag with neither box set. -> When you're ready, run: `/migrate-optimizely execute ` - ---- - -## Optimizely Concepts Reference (agent-internal, do NOT show to user) - -### Optimizely Flag Structure - -Each Optimizely flag has: -- **Key:** Alphanumeric + hyphens/underscores (max 64 chars) -- **Variations:** Named variants with keys and variable overrides -- **Variables:** Typed config values (string, integer, double, boolean, JSON) - with defaults and per-variation overrides -- **Rules per environment:** Ordered list of Targeted Delivery or A/B Test rules -- **Audiences:** Reusable targeting definitions with boolean conditions - -### Optimizely Traffic Allocation - -Optimizely uses a 0-10,000 bucket range (basis points): -- `endOfRange: 5000` = 50% of traffic -- Buckets are assigned via MurmurHash3 on `(userId + experimentId)` -- If total allocation < 10,000, remaining users are excluded - -**Conversion to Confidence:** Divide by 100 to get percentage. For -non-round percentages, round to nearest integer (Confidence uses -whole percentages that must sum to 100). - -### Optimizely Audience Conditions Format - -Conditions use a nested list format: -```json -["and", - {"type": "custom_attribute", "name": "country", "match": "exact", "value": "US"}, - {"type": "custom_attribute", "name": "age", "match": "gt", "value": 18} -] -``` - -Combinators: `"and"`, `"or"`, `"not"` as first element of a list. - -Individual condition: -```json -{ - "type": "custom_attribute", - "name": "", - "match": "", - "value": -} -``` - -Match type defaults: `exact` if a value is provided, `exists` if no value. - ---- - -## Operator Mapping Reference (agent-internal, do NOT show to user) - -This is how Optimizely operators map to Confidence targeting payloads. -Use this when generating `addTargetingRule` payloads in the plan file. - -**CRITICAL: Confidence Targeting Payload Format** - -The payload uses a `criteria` + `expression` pattern. Criteria are named -references (`ref-0`, `ref-1`, ...) that define individual conditions. -The `expression` combines them with boolean logic (`and`, `or`, `not`, `ref`). - -```json -{ - "criteria": { - "ref-0": { - "attribute": { - "attributeName": "", - "": { ... } - } - } - }, - "expression": { "ref": "ref-0" } -} -``` - -**DO NOT use nested rule objects like `{"or": {"operands": [{"eqRule": ...}]}}` -at the top level.** That format is silently parsed as empty targeting -(matching ALL contexts) due to `ignoringUnknownFields()` in the proto parser. - -### Criterion Rules - -| Optimizely Match | Confidence Criterion | -|-----------------|---------------------| -| `exact: "X"` (string) | `"eqRule": { "value": { "stringValue": "X" } }` | -| `exact: N` (number) | `"eqRule": { "value": { "numberValue": N } }` | -| `exact: true/false` | `"eqRule": { "value": { "boolValue": true } }` | -| `ge: N` | `"rangeRule": { "startInclusive": { "numberValue": N } }` | -| `gt: N` | `"rangeRule": { "startExclusive": { "numberValue": N } }` | -| `lt: N` | `"rangeRule": { "endExclusive": { "numberValue": N } }` | -| `le: N` | `"rangeRule": { "endInclusive": { "numberValue": N } }` | - -### Expression Combinators - -| Pattern | Expression | -|---------|-----------| -| Single condition | `{ "ref": "ref-0" }` | -| AND | `{ "and": { "operands": [{ "ref": "ref-0" }, { "ref": "ref-1" }] } }` | -| OR | `{ "or": { "operands": [{ "ref": "ref-0" }, { "ref": "ref-1" }] } }` | -| NOT | `{ "not": { "ref": "ref-0" } }` | - -### Optimizely Operator Mapping - -| Optimizely | Confidence Payload Strategy | -|-----------|---------------------------| -| `exact: "X"` | One criterion with `eqRule`, expression: `ref` | -| `NOT` + `exact: "X"` | One criterion with `eqRule`, expression: `not` wrapping `ref` | -| `exact: "A"` OR `exact: "B"` | One criterion per value with `eqRule`, expression: `or` of `ref`s | -| `ge: N` | One criterion with `rangeRule` (startInclusive), expression: `ref` | -| `gt: N` | One criterion with `rangeRule` (startExclusive), expression: `ref` | -| `lt: N` | One criterion with `rangeRule` (endExclusive), expression: `ref` | -| `le: N` | One criterion with `rangeRule` (endInclusive), expression: `ref` | - -**Blocked (manual review required):** - -| Optimizely Match | Reason | -|-----------------|--------| -| `substring` | No Confidence equivalent (contains/substring not supported) | -| `exists` | No Confidence equivalent (field-presence check not supported) | -| Semver comparisons | No Confidence equivalent (version type not supported) | - -When a flag uses a blocked operator, mark it in the plan with a warning: -> ⚠ This flag uses `substring` matching which has no Confidence equivalent. -> Manual review required — consider converting to `startsWith`/`endsWith` -> if the pattern allows, or implement in application code. - -### AND / OR Combinations - -**AND conditions:** Optimizely `["and", cond1, cond2]`. -Create one criterion per condition, combine with `and` expression. - -**OR conditions:** Optimizely `["or", cond1, cond2]`. -Create one criterion per condition, combine with `or` expression. - -**Nested combinations:** Optimizely supports arbitrary nesting: -`["and", cond1, ["or", cond2, cond3]]`. Map directly to nested -Confidence expressions. - -### Complete Examples - -**Single equality (country = "US"):** -```json -{ - "criteria": { - "ref-0": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "US" } } } } - }, - "expression": { "ref": "ref-0" } -} -``` - -**AND (plan = "pro" AND country = "US"):** -```json -{ - "criteria": { - "ref-0": { "attribute": { "attributeName": "plan", "eqRule": { "value": { "stringValue": "pro" } } } }, - "ref-1": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "US" } } } } - }, - "expression": { "and": { "operands": [{ "ref": "ref-0" }, { "ref": "ref-1" }] } } -} -``` - -**OR (country = "US" OR country = "UK"):** -```json -{ - "criteria": { - "ref-0": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "US" } } } }, - "ref-1": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "UK" } } } } - }, - "expression": { "or": { "operands": [{ "ref": "ref-0" }, { "ref": "ref-1" }] } } -} -``` - -**NOT (country != "DE"):** -```json -{ - "criteria": { - "ref-0": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "DE" } } } } - }, - "expression": { "not": { "ref": "ref-0" } } -} -``` - -**Range (age >= 18):** -```json -{ - "criteria": { - "ref-0": { "attribute": { "attributeName": "age", "rangeRule": { "startInclusive": { "numberValue": 18 } } } } - }, - "expression": { "ref": "ref-0" } -} -``` - -**Nested AND/OR (plan = "pro" AND (country = "US" OR country = "UK")):** -```json -{ - "criteria": { - "ref-0": { "attribute": { "attributeName": "plan", "eqRule": { "value": { "stringValue": "pro" } } } }, - "ref-1": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "US" } } } }, - "ref-2": { "attribute": { "attributeName": "country", "eqRule": { "value": { "stringValue": "UK" } } } } - }, - "expression": { "and": { "operands": [{ "ref": "ref-0" }, { "or": { "operands": [{ "ref": "ref-1" }, { "ref": "ref-2" }] } }] } } -} -``` - -### Multivariant A/B Split Handling - -**CRITICAL:** A single Confidence targeting rule CAN assign multiple -variants at different split percentages. Use ONE rule per targeting -condition, listing all variants and their shares in that rule. - -**How to map Optimizely traffic allocation to Confidence rules:** - -Optimizely uses 0-10,000 basis points. Convert to percentages: - -For a 2-variant experiment (e.g. endOfRange: 5000, 10000): -- Variation 1: 5000/10000 = 50% -- Variation 2: (10000-5000)/10000 = 50% -- Add ONE rule with: variation-1 at 50%, variation-2 at 50% - -For partial rollout (e.g. endOfRange: 2500, 5000 out of 10000): -- Variation 1: 2500/10000 = 25% -- Variation 2: (5000-2500)/10000 = 25% -- Remaining: 50% unallocated -- Add ONE rule with appropriate variant allocations - -**Do NOT create separate rules per variant.** One targeting rule = -one set of targeting conditions, with the variant split defined -inside that rule. The `rolloutPercentage` on the rule controls -what fraction of users who match the targeting conditions enter the -rule at all (use 100% unless you want a partial rollout on top of -the targeting). The variant percentages within the rule control the -split among those who enter. - -### Variable Mapping to Confidence Schema - -Optimizely typed variables map to Confidence flag schema: - -| Optimizely Type | Confidence Schema Type | -|----------------|----------------------| -| `string` | `"string"` | -| `integer` | `"integer"` | -| `double` | `"double"` | -| `boolean` | `"boolean"` | -| `json` | Flatten to individual fields or use `"string"` | - -When creating a flag with variables, use `schemaObject` to define the -schema and include variable values in each variant's `value` object. - -Example: Optimizely flag with variables `sort_method` (string) and -`items_per_page` (integer): - -``` -createFlag - flagName: "product-sort" - schemaObject: {"sort_method": "string", "items_per_page": "integer"} - variants: [ - {"name": "control", "value": {"sort_method": "relevance", "items_per_page": 20}}, - {"name": "treatment", "value": {"sort_method": "popularity", "items_per_page": 30}} - ] -``` - ---- - -## Plan Flag: Template - -```markdown -# Optimizely to Confidence Flag Migration Plan - -**Created:** -**Scope:** Flag definitions only -**Optimizely Project:** - ---- - -## Generation Status - -| Step | Status | Result | -|------|--------|--------| -| 1. Scan Optimizely | ○ not started | | -| 2. Choose environment | ○ not started | | -| 3. Choose client | ○ not started | | -| 4. Map entities | ○ not started | | -| 5. Generate rules | ○ not started | | - -**Overall:** in progress - ---- - -## 1. Default Client - -A client represents the application that resolves flags (e.g. your -website, backend service, or mobile app). Each client authenticates -with its own secret and can be scoped to environments (dev, staging, -prod). Flags are associated with clients so Confidence knows which -application receives which flags. - -**Available Clients:** - -**Selected:** `` - ---- - -## 1b. Optimizely Environment - -In Optimizely, each flag has different rules per environment. This plan -migrates the rules from a single environment. - -**Available Environments:** - -**Selected:** `` - ---- - -## 2. Randomization Mapping - -An entity is the "thing" being randomly assigned to a variant — usually -a user. The entity field (like `user_id` or `visitor_id`) is the -identifier Confidence uses for consistent assignment: the same user -always sees the same variant. - -### Per-user flags (Optimizely `userId`) - -Optimizely's `userId` (per-user identifier) is mapped to: **``** - -**Available Entity Fields:** - ---- - -## 3. Context Schema - -The context schema defines what fields Confidence expects in the -evaluation context when resolving flags — things like `country`, -`plan`, or `age` that targeting rules use to decide who gets what. - -Below is a reconciliation of what Optimizely flags need vs what already -exists in the Confidence client's schema. - -### Already in Confidence - -These fields are already defined in the `` client and match -Optimizely targeting attributes. No action needed. - -| Field | Type | Entity | Optimizely Attribute | -|-------|------|--------|---------------------| - - -### Need to Create - -These fields are used in Optimizely audience conditions but don't exist -yet in the Confidence client. They will be created during execution -using `addContextField`. - -| Field | Type | Entity | Optimizely Attribute | -|-------|------|--------|---------------------| - - -### Confidence-only (not in Optimizely) - -These fields exist in Confidence but aren't used by any Optimizely flag. -Listed for reference — no action needed. - -| Field | Type | Entity | -|-------|------|--------| - - ---- - -## 4. Flags to Migrate - -Below are the flags we're planning to migrate, along with their -targeting rules described in plain language. - -**Migration is opt-in.** Each flag starts with both checkboxes empty. -Tick `[x] Migrate` for every flag you want to bring across, or -`[x] Skip` to drop it. Flags with neither box ticked will be refused -by `execute` — no implicit defaults. - -During execution, each flag will be created one by one, interactively. - -### Flag: `` - -**Description:** -**Optimizely key:** -**Rule type:** -**Rules:** -**Rollout:** -**Variants:** -**Variables:** -**Confidence entity:** -**Action:** [ ] Migrate [ ] Skip - -**MCP Commands:** - - - ---- - -## 5. Blocked Flags - -Flags using Optimizely features that cannot be automatically migrated. - -| Flag | Blocked Reason | Recommendation | -|------|---------------|----------------| - - ---- - -## 6. Progress - -| # | Flag | Status | -|---|------|--------| -| 1 | | :white_circle: | - -``` - ---- - -## Execute: How It Works - -**`execute ` walks through the plan interactively, step by step.** - -### For Code Plans - -**Each flag = one PR.** The code migration creates a separate pull -request for each flag, keeping changes small and reviewable. - -``` -1. READ the plan file -2. SDK SETUP (Section 1 of plan) — one-time, before any flag - - Show install command from plan - - ASK: "Install SDK now? [Yes / Skip / I already did]" - - If Yes -> run install command - - Show wrapper file path + API surface from plan - - ASK: "Create the Confidence wrapper now? [Yes / Skip / I already did]" - - If Yes -> create the file using plan's API reference -3. FOR EACH FLAG in the files list: - a. Create a branch: `migrate/-to-confidence` - b. Show flag name + all files using it - c. ASK: "Transform this flag's files? [Yes / Skip / Pause]" - d. If Yes -> apply transform rules from plan to all files for this flag - e. Run lint + typecheck on changed files - f. Commit changes - g. Create PR with title: "feat: migrate from Optimizely to Confidence" - h. Show PR link - i. CHECKPOINT: "PR created. [Continue to next flag / Pause]?" - j. Wait for user response -4. COMPLETION - - Show summary: migrated vs skipped - - List all PRs created with links -``` - -### For Flag Plans - -``` -1. READ the plan file - - Client is already in the plan — use it, do NOT re-ask - - Entity (randomization unit) is already in the plan as the default - - REFUSE TO PROCEED if any flag has neither `[x] Migrate` nor - `[x] Skip` ticked. List those flags back to the user and ask - them to tick a box for each before re-running execute. Migration - is opt-in — never assume a default. -2. FOR EACH FLAG marked [x] Migrate: - - Show flag name, description, and rules in plain English - - ASK: "Create this flag in Confidence? [Yes / Skip / Pause]" - - If Yes -> run the flag setup sequence (see below) - - CHECKPOINT: "Flag done. [Continue / Pause]?" - - Wait for user response -3. COMPLETION - - Show summary: created vs skipped -``` - -**Flag Setup Sequence (MUST complete all steps before resolving):** - -Each flag MUST go through these steps in order. Do NOT call -`resolveFlag` until ALL prior steps succeed. - -``` -STEP 0: addContextField (if needed) - → Create any attribute fields required by this flag's targeting rules - that don't yet exist in Confidence (from Section 3 "Need to Create") - -STEP 1: createFlag - → If flag already exists, check the response for which clients - it's enabled on. - -STEP 2: Ensure flag is active and on the correct client - → If createFlag response does NOT list the target client: - a. Try addFlagToClient - b. If that fails with "Cannot update an archived flag": - → unarchiveFlag first, then retry addFlagToClient - → If createFlag response lists the target client: proceed - -STEP 3: addTargetingRule - → Add the targeting rule from the plan - → IMPORTANT: targeting rules added while a flag is archived OR - immediately after unarchiving may become inactive. Always complete - steps 1-2 fully (createFlag, unarchive, addFlagToClient) BEFORE - calling addTargetingRule. Do NOT add rules between createFlag and - unarchiveFlag — they will be inactive and you'll have to re-add. - -STEP 4: resolveFlag (verification) - → Only NOW resolve to verify the flag works - → MUST test BOTH positive AND negative cases: - a. Resolve with a context that SHOULD match the targeting rule - → Verify the expected variant is returned - b. Resolve with a context that SHOULD NOT match - → Verify no variant / default is returned - → For attribute-based targeting (country, plan, etc.), the resolve - call MUST include those attributes in the evaluation context. - Without them, the targeting conditions cannot be evaluated and - may appear to match when they wouldn't in production. - → If resolve fails with "No active flags found": - something went wrong in steps 1-2 — diagnose, don't skip - → If all rules show "Rule is inactive" / no match: - targeting rules were likely added while flag was archived. - Re-add the targeting rule now that the flag is active. - → Do NOT report a flag as successfully migrated until both - positive and negative resolve tests pass. -``` - -**Why this matters:** Confidence flags can be in states that -`createFlag` won't fix: archived, or enabled for a different client -only. The setup sequence handles all edge cases so resolves never -fail for avoidable reasons. - -### Rules - -- **NEVER auto-continue** — always wait for user at each checkpoint -- **Flag-by-flag** — each flag is one unit (its files + tests) -- **PR checkpoints** — offer to create PR after each flag or batch -- **Resumable** — update Progress table in plan file after each step - ---- - -## Required MCPs - -### For `plan code` - -| MCP | Tools Used | -|-----|------------| -| `confidence-docs` | `getCodeSnippetAndSdkIntegrationTips`, `searchDocumentation`, `getFullSource` | - -### For `plan flag` - -| MCP | Tools Used | -|-----|------------| -| `optimizely-exp` | List flags, get flag details, list environments, list audiences, get rulesets | -| `confidence-flags` | `listClients`, `getContextSchema`, `createFlag`, `addTargetingRule`, `resolveFlag` | From f94a954bcfcc3380167918d5584754e8287cb9b7 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:09:26 +0200 Subject: [PATCH 03/25] feat: add multi-warehouse verification to setup-warehouse skill Add Snowflake, Databricks, and Redshift query support in the data pipeline verification step. Previously only BigQuery was covered. Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 82 ++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 10 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index bf7f65a..40c49a9 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -998,7 +998,11 @@ curl -s -w "\n%{http_code}" -X POST "https://metrics.confidence.dev/v1/assignmen }' ``` -Adapt the SQL query per warehouse type — BigQuery uses backtick-qualified tables, Snowflake uses `database.schema.table`, etc. +Adapt the SQL query per warehouse type: +- **BigQuery:** `` SELECT targeting_key, rule, assignment_id, assignment_time FROM `..assignments` `` +- **Snowflake:** `SELECT targeting_key, rule, assignment_id, assignment_time FROM ..ASSIGNMENTS` +- **Databricks:** `SELECT targeting_key, rule, assignment_id, assignment_time FROM .assignments` +- **Redshift:** `SELECT targeting_key, rule, assignment_id, assignment_time FROM .assignments` ### Step 7: Verify data pipeline @@ -1067,9 +1071,11 @@ Check response: `{"errors": []}` means success. If `EVENT_DEFINITION_NOT_FOUND`, **7c. Check data in warehouse** -Ask the user: "Want me to check the data (needs `bq` CLI), or show you the queries?" +Verification approach depends on warehouse type. Ask the user: "Want me to check the data, or show you the queries?" -If user has `bq`: +**BigQuery:** + +If user has `bq` CLI: ```bash echo "=== ASSIGNMENTS ===" && \ bq query --project_id=${PROJECT} --use_legacy_sql=false \ @@ -1082,20 +1088,76 @@ bq query --project_id=${PROJECT} --use_legacy_sql=false \ ORDER BY _event_time DESC LIMIT 5' ``` -If user doesn't have `bq`, show the queries: -> Run these in the BigQuery console (https://console.cloud.google.com/bigquery): +If no `bq`, show queries for BigQuery console. + +**Snowflake:** + +If user has `snowsql` CLI: +```bash +snowsql -a ${SNOWFLAKE_ACCOUNT} -u ${SNOWFLAKE_USER} -r ${SNOWFLAKE_ROLE} -w ${SNOWFLAKE_WAREHOUSE} -d ${SNOWFLAKE_DATABASE} -s ${SNOWFLAKE_SCHEMA} -q " +SELECT targeting_key, rule, assignment_id, assignment_time +FROM ${SNOWFLAKE_DATABASE}.${SNOWFLAKE_SCHEMA}.ASSIGNMENTS +ORDER BY assignment_time DESC LIMIT 5; +" +``` + +If no `snowsql`, use the Snowflake SQL REST API: +```bash +# Get a JWT token for Snowflake (using keypair auth) or prompt user for password +# Then query via the SQL API: +curl -s -X POST "https://${SNOWFLAKE_ACCOUNT}.snowflakecomputing.com/api/v2/statements" \ + -H "Authorization: Bearer ${SNOWFLAKE_JWT}" \ + -H "Content-Type: application/json" \ + -H "X-Snowflake-Authorization-Token-Type: KEYPAIR_JWT" \ + -d '{ + "statement": "SELECT targeting_key, rule, assignment_id, assignment_time FROM '${SNOWFLAKE_DATABASE}'.'${SNOWFLAKE_SCHEMA}'.ASSIGNMENTS ORDER BY assignment_time DESC LIMIT 5", + "warehouse": "'${SNOWFLAKE_WAREHOUSE}'", + "database": "'${SNOWFLAKE_DATABASE}'", + "schema": "'${SNOWFLAKE_SCHEMA}'", + "role": "'${SNOWFLAKE_ROLE}'" + }' +``` + +If neither available, show the queries for the Snowflake worksheet (https://app.snowflake.com): > ```sql > -- Assignments > SELECT targeting_key, rule, assignment_id, assignment_time -> FROM `..assignments` +> FROM ..ASSIGNMENTS > ORDER BY assignment_time DESC LIMIT 5; > -> -- Events -> SELECT * FROM `..events_*` +> -- Events (list event tables first, then query) +> SHOW TABLES LIKE 'EVENTS_%' IN .; +> SELECT * FROM .. > ORDER BY _event_time DESC LIMIT 5; > ``` -**Show results:** +**Databricks:** + +Show queries for the Databricks SQL editor: +> ```sql +> SELECT targeting_key, rule, assignment_id, assignment_time +> FROM .assignments +> ORDER BY assignment_time DESC LIMIT 5; +> +> SHOW TABLES IN LIKE 'events_*'; +> SELECT * FROM . +> ORDER BY _event_time DESC LIMIT 5; +> ``` + +**Redshift:** + +If user has `psql` or `aws redshift-data`: +```bash +aws redshift-data execute-statement \ + --cluster-identifier ${CLUSTER} \ + --database ${DATABASE} \ + --db-user ${DB_USER} \ + --sql "SELECT targeting_key, rule, assignment_id, assignment_time FROM ${SCHEMA}.assignments ORDER BY assignment_time DESC LIMIT 5" +``` + +Otherwise, show queries for the Redshift query editor. + +**Show results (all warehouse types):** ``` ● Assignments: rows — data flowing → () @@ -1104,7 +1166,7 @@ If user doesn't have `bq`, show the queries: ``` **If no rows after a few seconds**, tell the user: -> Streaming can take up to a minute. Check again shortly, or verify in the BigQuery console. +> Data delivery can take up to a few minutes depending on your warehouse. Check again shortly, or verify in your warehouse console. ### Step 8: Done From 931dde0f00b8926c403f5a96483796d97f2bb00a Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:31:43 +0200 Subject: [PATCH 04/25] fix: never assume partial success from ambiguous validation errors Report exact error messages from the warehouse validation API instead of optimistically splitting them into "works" + "but". Adds Snowflake-specific remediation guidance for auth and permission errors. Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index 40c49a9..c539d37 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -813,13 +813,29 @@ curl -s -w "\n%{http_code}" -X POST "https://metrics.confidence.dev/v1/dataWareh } ``` -If `successful` is false, explain each failure in plain English and **ask the user how they want to proceed:** +If the response is an error (HTTP 400/500) or `successful` is false: + +**IMPORTANT: Never assume partial success from an ambiguous error.** If the API returns an error like "X does not exist or not authorized", report the exact error message — do NOT split it into "connection works but X is missing". The error may indicate an auth failure, a missing resource, or both. Show the user the exact error and let them determine the cause. + +For each validation failure, show: +> Validation failed: `` + +Then offer remediation based on warehouse type. + +**For BigQuery failures**, ask the user how they want to proceed: > Some permissions need to be configured on your GCP project. I can fix this automatically if you have `gcloud` set up, or I can show you the exact commands to run yourself. > > 1. Fix it for me (requires gcloud CLI) > 2. Show me the commands +**For Snowflake failures**, show the SQL commands needed: +- Auth failures → the crypto key's public key needs to be registered with the Snowflake user via `ALTER USER ... SET RSA_PUBLIC_KEY='...'` +- Database/schema missing → `CREATE DATABASE` / `CREATE SCHEMA` commands +- Permission errors → `GRANT` commands + +**For Databricks/Redshift failures**, show the relevant remediation steps for that platform. + **If the user chooses 1 (fix it for me):** First check gcloud is available: `which gcloud`. If not, fall back to option 2. From 94c099fac8685babb774447452ddcf288cc7c77e Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:53:35 +0200 Subject: [PATCH 05/25] fix: correct gRPC transcoding for warehouse, connector, and assignment table APIs - Warehouse create: body is the object directly, not wrapped in dataWarehouse key - Connectors: body is the object directly, not wrapped in flagAppliedConnection/eventConnection - Assignment table: body is the object directly, not wrapped in assignmentTable - Snowflake connectors require database and schema fields in snowflakeConfig - Pipeline verify step now lists clients and lets user pick before creating credentials - Snowflake remediation generates SQL and copies to clipboard Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 131 ++++++++++++++++++++--------- 1 file changed, 92 insertions(+), 39 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index c539d37..1970e5d 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -829,10 +829,39 @@ Then offer remediation based on warehouse type. > 1. Fix it for me (requires gcloud CLI) > 2. Show me the commands -**For Snowflake failures**, show the SQL commands needed: -- Auth failures → the crypto key's public key needs to be registered with the Snowflake user via `ALTER USER ... SET RSA_PUBLIC_KEY='...'` -- Database/schema missing → `CREATE DATABASE` / `CREATE SCHEMA` commands -- Permission errors → `GRANT` commands +**For Snowflake failures**, generate the full remediation SQL, **copy it to clipboard via `pbcopy`**, and tell the user to paste it in the Snowflake worksheet (https://app.snowflake.com): + +1. **Fetch the crypto key's public key** from the IAM API: + ```bash + curl -s "https://iam.${REGION}.confidence.dev/v1/cryptoKeys/" -H "Authorization: Bearer $TOKEN" + ``` + Strip the PEM headers (`-----BEGIN/END PUBLIC KEY-----`) and newlines to get the raw base64 string for Snowflake. + +2. **Generate SQL based on the error:** + + Auth failures → register the public key: + ```sql + -- If this is the only Confidence account using this Snowflake user: + ALTER USER SET RSA_PUBLIC_KEY=''; + -- If another Confidence account already uses RSA_PUBLIC_KEY, use key 2: + ALTER USER SET RSA_PUBLIC_KEY_2=''; + ``` + **IMPORTANT:** Always ask the user if other Confidence accounts share this Snowflake user. If yes, use `RSA_PUBLIC_KEY_2` to avoid breaking existing connections. Snowflake accepts auth from either key. + + Database/schema missing: + ```sql + CREATE DATABASE IF NOT EXISTS ; + CREATE SCHEMA IF NOT EXISTS .; + GRANT USAGE ON DATABASE TO ROLE ; + GRANT USAGE ON SCHEMA . TO ROLE ; + GRANT ALL ON SCHEMA . TO ROLE ; + ``` + +3. **Copy to clipboard and tell the user:** + ```bash + echo "" | pbcopy + ``` + > The SQL commands have been copied to your clipboard. Paste them in the Snowflake worksheet at https://app.snowflake.com and run them. Let me know when done and I'll retry validation. **For Databricks/Redshift failures**, show the relevant remediation steps for that platform. @@ -925,15 +954,15 @@ If `configurationResponse` contains available options (schemas, roles) — prese ### Step 4: Create warehouse +**IMPORTANT:** The body is the data warehouse object directly (gRPC transcoding `body: "data_warehouse"`), NOT wrapped in a `dataWarehouse` key. + ```bash -curl -s -w "\n%{http_code}" -X POST "https://metrics.confidence.dev/v1/dataWarehouses" \ +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/dataWarehouses" \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{ - "dataWarehouse": { - "config": { - "": { } - } + "config": { + "": { } } }' ``` @@ -944,38 +973,40 @@ Save the returned `name` (e.g., `dataWarehouses/...`) for reference. Create both connectors: -**Flag Applied Connection** (assignment data → warehouse): +**Flag Applied Connection** (assignment data → warehouse). + +**IMPORTANT:** The body is the connection object directly (gRPC transcoding `body: "flag_applied_connection"`), NOT wrapped. + ```bash -curl -s -w "\n%{http_code}" -X POST "https://connectors.confidence.dev/v1/flagAppliedConnections" \ +curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev/v1/flagAppliedConnections" \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{ - "flagAppliedConnection": { - "bigQuery": { - "bigQueryConfig": { "serviceAccount": "...", "project": "...", "dataset": "..." }, - "table": "assignments" - } + "bigQuery": { + "bigQueryConfig": { "serviceAccount": "...", "project": "...", "dataset": "..." }, + "table": "assignments" } }' ``` Adapt the destination field per warehouse type: - BigQuery: `"bigQuery": { "bigQueryConfig": {...}, "table": "assignments" }` -- Snowflake: `"snowflake": { "snowflakeConfig": {...}, "table": "assignments" }` +- Snowflake: `"snowflake": { "snowflakeConfig": {..., "database": "...", "schema": "..."}, "table": "ASSIGNMENTS" }` — **Snowflake requires `database` and `schema` fields in snowflakeConfig for connectors** - Databricks: `"databricks": { "databricksConfig": {...}, "table": "assignments" }` - Redshift: `"redshift": { "redshiftConfig": {...}, "s3Config": {...}, "batchFileConfig": {...}, "table": "assignments" }` -**Event Connection** (events → warehouse): +**Event Connection** (events → warehouse). + +**IMPORTANT:** The body is the connection object directly (gRPC transcoding `body: "event_connection"`), NOT wrapped. + ```bash -curl -s -w "\n%{http_code}" -X POST "https://connectors.confidence.dev/v1/eventConnections" \ +curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev/v1/eventConnections" \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{ - "eventConnection": { - "bigQuery": { - "bigQueryConfig": { "serviceAccount": "...", "project": "...", "dataset": "..." }, - "tablePrefix": "events_" - } + "bigQuery": { + "bigQueryConfig": { "serviceAccount": "...", "project": "...", "dataset": "..." }, + "tablePrefix": "events_" } }' ``` @@ -990,25 +1021,25 @@ Collect these if the user chose Redshift or Databricks. ### Step 6: Assignment table -Create an assignment table so Confidence can analyze experiment assignments: +Create an assignment table so Confidence can analyze experiment assignments. + +**IMPORTANT:** The body is the assignment table object directly (gRPC transcoding `body: "assignment_table"`), NOT wrapped in an `assignmentTable` key. ```bash -curl -s -w "\n%{http_code}" -X POST "https://metrics.confidence.dev/v1/assignmentTables" \ +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/assignmentTables" \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{ - "assignmentTable": { - "displayName": "Assignments", - "sql": "SELECT targeting_key, rule, assignment_id, assignment_time FROM `..assignments`", - "entityColumn": { "name": "targeting_key" }, - "timestampColumn": { "name": "assignment_time" }, - "exposureKeyColumn": { "name": "rule" }, - "variantKeyColumn": { "name": "assignment_id" }, - "dataDeliveredUntilUpdateStrategyConfig": { - "strategy": "AUTOMATIC", - "automaticUpdateConfig": { - "commitDelay": "300s" - } + "displayName": "Assignments", + "sql": "", + "entityColumn": { "name": "targeting_key" }, + "timestampColumn": { "name": "assignment_time" }, + "exposureKeyColumn": { "name": "rule" }, + "variantKeyColumn": { "name": "assignment_id" }, + "dataDeliveredUntilUpdateStrategyConfig": { + "strategy": "AUTOMATIC", + "automaticUpdateConfig": { + "commitDelay": "300s" } } }' @@ -1024,7 +1055,29 @@ Adapt the SQL query per warehouse type: Verify both connectors by generating test data and checking it lands in the warehouse. -**7a. Verify flag assignments** +**7a. Get a client secret for testing** + +The resolver and events APIs require a **client secret** (not a Bearer token). + +1. **List the user's clients** and show them: + ```bash + curl -s "https://iam.${REGION}.confidence.dev/v1/clients" -H "Authorization: Bearer $TOKEN" + ``` + Display each client with its name and last-seen time. If only one client exists, confirm it with the user. If multiple, let them pick. + +2. **Ask the user** if they have a client secret or want a new one: + > I'll use **** for the pipeline test. Do you have the client secret, or should I create a new credential? + +3. If the user wants a new credential, create one on the chosen client: + ```bash + curl -s -X POST "https://iam.${REGION}.confidence.dev/v1//credentials" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"display_name": "Pipeline Test"}' + ``` + Save the secret to a temp file for pipeline use. **Never print the secret to the user's terminal.** + +**7b. Verify flag assignments** Resolve a flag to generate assignment data (use an existing flag + client secret): ```bash From 102faf0ff6d4d9f729bb94682f33ceddb785a109 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:54:28 +0200 Subject: [PATCH 06/25] fix: auto-create crypto key for Snowflake setup instead of asking user Users don't know what a crypto key reference is. The skill now: - Creates the crypto key via IAM API automatically - Extracts the public key and generates ALTER USER SQL - Copies the SQL to clipboard for the user to run in Snowflake - Generates CREATE DATABASE/SCHEMA SQL if needed Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 52 ++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index 1970e5d..7ef5032 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -771,13 +771,51 @@ Collect configuration based on type. Explain each field briefly. - Service account email — must have BigQuery permissions **Snowflake:** -- Account — your Snowflake account identifier -- User — Snowflake user for Confidence to use -- Authentication key — crypto key reference for JWT signing -- Role — Snowflake role with necessary permissions -- Warehouse — SQL warehouse for query execution -- Exposure database — database for exposure tables -- Exposure schema — schema for exposure tables + +Ask the user for these fields (explain each briefly): +- Account — Snowflake account identifier (e.g., `zlvpqre-wr49874`) +- User — Snowflake user for Confidence to connect as +- Role — Snowflake role (default: `ACCOUNTADMIN`) +- Warehouse — SQL warehouse for query execution (default: `COMPUTE_WH`) +- Exposure database — database for exposure tables (default: `CONFIDENCE`) +- Exposure schema — schema for exposure tables (default: `EXPOSURE`) + +**Then create a crypto key automatically** — the user does NOT provide this. The skill creates it via the IAM API: + +```bash +curl -s -w "\n%{http_code}" -X POST "https://iam.${REGION}.confidence.dev/v1/cryptoKeys?crypto_key_id=snowflake-key" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"kind": "SNOWFLAKE"}' +``` + +If the key already exists (HTTP 409), fetch it instead: +```bash +curl -s "https://iam.${REGION}.confidence.dev/v1/cryptoKeys/snowflake-key" \ + -H "Authorization: Bearer $TOKEN" +``` + +Extract the `publicKey` from the response, strip PEM headers and newlines to get raw base64. Then generate the Snowflake SQL to register the key, **copy it to clipboard**, and tell the user: + +> I've created an authentication key for Snowflake. You need to register it with your Snowflake user. +> The SQL has been copied to your clipboard — paste it in the Snowflake worksheet and run it. + +The SQL should be: +```sql +ALTER USER SET RSA_PUBLIC_KEY=''; +``` + +If the user says other Confidence accounts share this Snowflake user, use `RSA_PUBLIC_KEY_2` instead. + +Also generate SQL for creating the database/schema if the user says they don't exist yet: +```sql +CREATE DATABASE IF NOT EXISTS ; +CREATE SCHEMA IF NOT EXISTS .; +GRANT USAGE ON DATABASE TO ROLE ; +GRANT ALL ON SCHEMA . TO ROLE ; +``` + +Save the crypto key name (e.g., `cryptoKeys/snowflake-key`) for use in the warehouse config. **Databricks:** - Host — Databricks workspace URL From f157fde105498a4474ff1514a8795e25763fac2e Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:00:06 +0200 Subject: [PATCH 07/25] fix: add plain-language setup guides for all warehouse types Each config field now explains what it is, where to find it in the warehouse UI, and what to do if it doesn't exist yet. Covers BigQuery, Snowflake, Databricks, and Redshift. Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 65 ++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index 7ef5032..ee3ec90 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -766,9 +766,17 @@ Requires an authenticated token. If none saved, run the Auth0 login flow first. Collect configuration based on type. Explain each field briefly. **BigQuery:** -- GCP project ID — the project where BigQuery datasets live -- Dataset name (default: `confidence`) -- Service account email — must have BigQuery permissions + +Guide the user through each field with plain-language explanations and where to find the value: + +1. **GCP Project ID** — the Google Cloud project where your data lives. + > Go to **Google Cloud Console** (console.cloud.google.com). Your project ID is shown in the top bar next to "Google Cloud". It looks like `my-company-prod` or `project-12345`. + +2. **Dataset name** — where Confidence stores its tables (default: `confidence`). + > A dataset is like a folder in BigQuery. If you don't have one yet, the skill can create it for you via `bq mk`. + +3. **Service account email** — a robot account that Confidence uses to write data. + > Go to **Google Cloud Console → IAM & Admin → Service Accounts**. Create one (e.g., `confidence-connector@.iam.gserviceaccount.com`) or pick an existing one. It needs BigQuery Data Editor and BigQuery Job User roles. **Snowflake:** @@ -818,18 +826,49 @@ GRANT ALL ON SCHEMA . TO ROLE ; Save the crypto key name (e.g., `cryptoKeys/snowflake-key`) for use in the warehouse config. **Databricks:** -- Host — Databricks workspace URL -- SQL warehouse ID -- Service principal client ID + secret -- Exposure schema — schema for exposure tables + +Guide the user through each field with plain-language explanations and where to find the value: + +1. **Host** — your Databricks workspace URL. + > This is the URL you see in your browser when you open Databricks. It looks like `https://dbc-xxxxx.cloud.databricks.com`. You can find it in **Databricks → Settings → Workspace URL**, or just copy it from your browser address bar. + +2. **SQL Warehouse ID** — the compute resource Confidence uses to run queries. + > Go to **Databricks → SQL → SQL Warehouses**. Pick a warehouse (or create one) and copy its ID from the **Connection details** tab. It looks like a hex string, e.g., `1a2b3c4d5e6f7890`. + > If you don't have a SQL Warehouse yet, guide the user: **SQL → SQL Warehouses → Create SQL Warehouse** → pick "Serverless" (simplest), size Small. + +3. **Service principal client ID** — how Confidence authenticates to Databricks. + > A service principal is like a robot account. Go to **Databricks → Settings → Identity and access → Service principals → Add service principal → Add new**. Give it a name like "Confidence". After creation, copy the **Application (client) ID**. + > Then create a secret: click the service principal → **Secrets → Generate secret**. Copy the **Secret** value (shown only once). + +4. **Service principal client secret** — the secret you just generated above. + +5. **Catalog** — the Databricks Unity Catalog where Confidence stores its tables (default: `confidence`). + > A catalog is like a top-level folder for your data. If you don't have one, guide the user: **Databricks → Catalog → Create Catalog** → name it `confidence`. + > Grant the service principal access: `GRANT USE CATALOG ON CATALOG confidence TO \`\`` + +6. **Schema** — the schema inside the catalog for Confidence tables (default: `exposure`). + > A schema is a subfolder inside the catalog. If it doesn't exist, guide the user to create it: + > `CREATE SCHEMA IF NOT EXISTS confidence.exposure` + > Grant access: `GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence.exposure TO \`\`` **Redshift:** -- Cluster — Redshift cluster identifier -- AWS region — select from available regions -- IAM role ARN — role Confidence assumes -- Database name -- Schema name -- Authentication — AWS access key + secret, or web identity + +Guide the user through each field with plain-language explanations and where to find the value: + +1. **Cluster** — your Redshift cluster identifier. + > Go to **AWS Console → Amazon Redshift → Clusters**. The cluster name is in the list (e.g., `my-analytics-cluster`). + +2. **AWS Region** — where your cluster runs (e.g., `us-east-1`, `eu-west-1`). + > Shown in the top-right corner of your AWS Console, or in the cluster details page. + +3. **IAM Role ARN** — the role Confidence assumes to access Redshift. + > Go to **AWS Console → IAM → Roles**. Create or pick a role with Redshift access. The ARN looks like `arn:aws:iam::123456789012:role/ConfidenceRedshift`. + +4. **Database name** — the Redshift database (default: `dev` or your main database). + +5. **Schema name** — where Confidence stores its tables (default: `confidence`). + +6. **Authentication** — AWS access key + secret, or web identity federation. ### Step 3: Validate configuration From dd81a45a09ab970cd4580b80b77873aa9d999709 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:41:52 +0200 Subject: [PATCH 08/25] fix: Databricks validate not supported, fix connector format - Note that validate endpoint only supports BigQuery and Snowflake - Databricks connectors need connectionConfig wrapper + batchFileConfig - Skip validation for Databricks/Redshift and go straight to create Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 35 +++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index ee3ec90..f461b98 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -872,12 +872,16 @@ Guide the user through each field with plain-language explanations and where to ### Step 3: Validate configuration +**NOTE:** The validate endpoint only supports BigQuery and Snowflake. For Databricks and Redshift, skip validation and proceed directly to Step 4 (Create warehouse). Inform the user: +> Validation isn't available for Databricks/Redshift — I'll create the warehouse and we'll verify the connection when testing the pipeline. + +For BigQuery/Snowflake: ```bash -curl -s -w "\n%{http_code}" -X POST "https://metrics.confidence.dev/v1/dataWarehouseConfig:validate" \ +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/dataWarehouseConfig:validate" \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{ - "Config": { } + "": { } }' ``` @@ -1067,10 +1071,29 @@ curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev ``` Adapt the destination field per warehouse type: -- BigQuery: `"bigQuery": { "bigQueryConfig": {...}, "table": "assignments" }` -- Snowflake: `"snowflake": { "snowflakeConfig": {..., "database": "...", "schema": "..."}, "table": "ASSIGNMENTS" }` — **Snowflake requires `database` and `schema` fields in snowflakeConfig for connectors** -- Databricks: `"databricks": { "databricksConfig": {...}, "table": "assignments" }` -- Redshift: `"redshift": { "redshiftConfig": {...}, "s3Config": {...}, "batchFileConfig": {...}, "table": "assignments" }` +- **BigQuery:** `"bigQuery": { "bigQueryConfig": {...}, "table": "assignments" }` +- **Snowflake:** `"snowflake": { "snowflakeConfig": {..., "database": "...", "schema": "..."}, "table": "ASSIGNMENTS" }` — Snowflake requires `database` and `schema` fields in snowflakeConfig for connectors +- **Databricks:** Databricks connectors use a nested `connectionConfig` for auth and require `batchFileConfig`: + ```json + "databricks": { + "databricksConfig": { + "connectionConfig": { + "host": "...", + "warehouseId": "...", + "clientId": "...", + "clientSecret": "..." + }, + "schema": ".", + "batchFileConfig": { + "maxEventsPerFile": 10000, + "maxFileAge": "300s", + "maxFileSize": 104857600 + } + }, + "table": "assignments" + } + ``` +- **Redshift:** `"redshift": { "redshiftConfig": {...}, "s3Config": {...}, "batchFileConfig": {...}, "table": "assignments" }` **Event Connection** (events → warehouse). From ad7f4c39fef22b6fcde66da688b7f63f1307ed41 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:50:49 +0200 Subject: [PATCH 09/25] fix: document Databricks validate as backend limitation, not field name issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tested every field name variation — the validate endpoint genuinely only supports BigQuery and Snowflake. Skill now explains this honestly rather than silently skipping. Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index f461b98..a61226f 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -872,8 +872,8 @@ Guide the user through each field with plain-language explanations and where to ### Step 3: Validate configuration -**NOTE:** The validate endpoint only supports BigQuery and Snowflake. For Databricks and Redshift, skip validation and proceed directly to Step 4 (Create warehouse). Inform the user: -> Validation isn't available for Databricks/Redshift — I'll create the warehouse and we'll verify the connection when testing the pipeline. +**NOTE:** The validate endpoint only supports BigQuery (`bigQueryConfig`) and Snowflake (`snowflakeConfig`). The Confidence backend does not recognize Databricks or Redshift configs for validation (returns "configuration must be set" for any field name variant). For Databricks and Redshift, skip validation and proceed directly to Step 4 (Create warehouse). Tell the user honestly: +> Pre-validation isn't available yet for Databricks/Redshift. I'll create the warehouse now and we'll verify the connection works end-to-end in the pipeline test step. For BigQuery/Snowflake: ```bash From 3768965ba1e14abe76645e98416aec7a53f5a927 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:57:37 +0200 Subject: [PATCH 10/25] fix: document Databricks connector 500 as known backend issue Connector returns 500 even with valid credentials (verified OAuth directly). Note as known issue and suggest contacting support. Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index a61226f..caa3c80 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -1073,7 +1073,7 @@ curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev Adapt the destination field per warehouse type: - **BigQuery:** `"bigQuery": { "bigQueryConfig": {...}, "table": "assignments" }` - **Snowflake:** `"snowflake": { "snowflakeConfig": {..., "database": "...", "schema": "..."}, "table": "ASSIGNMENTS" }` — Snowflake requires `database` and `schema` fields in snowflakeConfig for connectors -- **Databricks:** Databricks connectors use a nested `connectionConfig` for auth and require `batchFileConfig`: +- **Databricks:** Databricks connectors use a nested `connectionConfig` for auth and require `batchFileConfig`. **Known issue:** the connector backend may return 500 even with valid credentials — if this happens, inform the user and suggest contacting Confidence support. ```json "databricks": { "databricksConfig": { @@ -1083,11 +1083,9 @@ Adapt the destination field per warehouse type: "clientId": "...", "clientSecret": "..." }, - "schema": ".", + "schema": "", "batchFileConfig": { - "maxEventsPerFile": 10000, - "maxFileAge": "300s", - "maxFileSize": 104857600 + "maxFileAge": "300s" } }, "table": "assignments" From 691169d019444fdd454ccf882e4d2ee6e1b5ab15 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:09:47 +0200 Subject: [PATCH 11/25] fix: Databricks connectors require S3 staging bucket Root cause found in logs: IllegalArgumentException "S3BucketConfig needs to be set". Databricks connectors batch-write to S3 then load into Databricks. Added s3BucketConfig (bucket, region, roleArn) to connector format and Step 2 config collection. Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index caa3c80..c322b48 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -851,6 +851,11 @@ Guide the user through each field with plain-language explanations and where to > `CREATE SCHEMA IF NOT EXISTS confidence.exposure` > Grant access: `GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence.exposure TO \`\`` +7. **S3 staging bucket** — Confidence writes data to S3 first, then loads into Databricks. + > You need an S3 bucket for Confidence to use as a staging area. Go to **AWS Console → S3 → Create bucket**. Name it something like `confidence-databricks-staging`. + > Then create an IAM role with write access to the bucket and provide the **Role ARN** (e.g., `arn:aws:iam::123456789012:role/ConfidenceDatabricksStaging`). + > This is required even if your Databricks runs on GCP or Azure — the connector uses S3 for staging. + **Redshift:** Guide the user through each field with plain-language explanations and where to find the value: @@ -1073,7 +1078,7 @@ curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev Adapt the destination field per warehouse type: - **BigQuery:** `"bigQuery": { "bigQueryConfig": {...}, "table": "assignments" }` - **Snowflake:** `"snowflake": { "snowflakeConfig": {..., "database": "...", "schema": "..."}, "table": "ASSIGNMENTS" }` — Snowflake requires `database` and `schema` fields in snowflakeConfig for connectors -- **Databricks:** Databricks connectors use a nested `connectionConfig` for auth and require `batchFileConfig`. **Known issue:** the connector backend may return 500 even with valid credentials — if this happens, inform the user and suggest contacting Confidence support. +- **Databricks:** Databricks connectors use a nested `connectionConfig` for auth, require an **S3 staging bucket** for batch writes, and `batchFileConfig`: ```json "databricks": { "databricksConfig": { @@ -1084,6 +1089,11 @@ Adapt the destination field per warehouse type: "clientSecret": "..." }, "schema": "", + "s3BucketConfig": { + "bucket": "", + "region": "", + "roleArn": "" + }, "batchFileConfig": { "maxFileAge": "300s" } @@ -1091,6 +1101,8 @@ Adapt the destination field per warehouse type: "table": "assignments" } ``` + **IMPORTANT:** Databricks connectors require an S3 staging bucket — Confidence writes data in batches to S3, then loads into Databricks. The user needs to provide an S3 bucket, AWS region, and IAM role ARN with write access to the bucket. Explain this to the user: + > Confidence writes data to a staging bucket first, then loads it into Databricks. You'll need an S3 bucket and an IAM role that allows Confidence to write to it. - **Redshift:** `"redshift": { "redshiftConfig": {...}, "s3Config": {...}, "batchFileConfig": {...}, "table": "assignments" }` **Event Connection** (events → warehouse). From 3b35ff38fbc0c5035bb4b1e02772a1196e672f5d Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:21:02 +0200 Subject: [PATCH 12/25] feat: rewrite Databricks setup with full step-by-step guide Explains upfront that Databricks requires AWS S3 staging bucket, Databricks admin access, and a schema. Walks through each step with exact UI paths, commands, and explains why each piece is needed. Covers trust policy errors and Unity Catalog vs hive_metastore. Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 95 ++++++++++++++++++++++-------- 1 file changed, 72 insertions(+), 23 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index c322b48..40111ce 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -827,34 +827,83 @@ Save the crypto key name (e.g., `cryptoKeys/snowflake-key`) for use in the wareh **Databricks:** -Guide the user through each field with plain-language explanations and where to find the value: - -1. **Host** — your Databricks workspace URL. - > This is the URL you see in your browser when you open Databricks. It looks like `https://dbc-xxxxx.cloud.databricks.com`. You can find it in **Databricks → Settings → Workspace URL**, or just copy it from your browser address bar. +Before collecting details, explain what's needed upfront so the user knows the full picture: -2. **SQL Warehouse ID** — the compute resource Confidence uses to run queries. - > Go to **Databricks → SQL → SQL Warehouses**. Pick a warehouse (or create one) and copy its ID from the **Connection details** tab. It looks like a hex string, e.g., `1a2b3c4d5e6f7890`. - > If you don't have a SQL Warehouse yet, guide the user: **SQL → SQL Warehouses → Create SQL Warehouse** → pick "Serverless" (simplest), size Small. - -3. **Service principal client ID** — how Confidence authenticates to Databricks. - > A service principal is like a robot account. Go to **Databricks → Settings → Identity and access → Service principals → Add service principal → Add new**. Give it a name like "Confidence". After creation, copy the **Application (client) ID**. - > Then create a secret: click the service principal → **Secrets → Generate secret**. Copy the **Secret** value (shown only once). +> Setting up Databricks with Confidence requires three things: +> +> 1. **A Databricks workspace** with admin access (to create a service principal) +> 2. **An AWS account** with an S3 bucket (Confidence stages data in S3 before loading into Databricks — this is required even if Databricks runs on GCP or Azure) +> 3. **A schema in Databricks** where Confidence will create tables +> +> If you don't have an AWS account, you'll need to create one (free tier works) or ask your infrastructure team for an S3 bucket and IAM role. +> +> Here's how data flows: **Confidence → S3 staging bucket → Databricks tables** -4. **Service principal client secret** — the secret you just generated above. +Then collect the details step by step. Ask one at a time, explain each field, and tell the user exactly where to find it: -5. **Catalog** — the Databricks Unity Catalog where Confidence stores its tables (default: `confidence`). - > A catalog is like a top-level folder for your data. If you don't have one, guide the user: **Databricks → Catalog → Create Catalog** → name it `confidence`. - > Grant the service principal access: `GRANT USE CATALOG ON CATALOG confidence TO \`\`` +**Part 1: Databricks connection** -6. **Schema** — the schema inside the catalog for Confidence tables (default: `exposure`). - > A schema is a subfolder inside the catalog. If it doesn't exist, guide the user to create it: - > `CREATE SCHEMA IF NOT EXISTS confidence.exposure` - > Grant access: `GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence.exposure TO \`\`` +1. **Host** — your Databricks workspace URL. + > This is the URL in your browser when you open Databricks. It looks like `https://dbc-xxxxx.cloud.databricks.com` or `https://1234567890.7.gcp.databricks.com`. Just copy it from your address bar — I only need the hostname, not the full URL. -7. **S3 staging bucket** — Confidence writes data to S3 first, then loads into Databricks. - > You need an S3 bucket for Confidence to use as a staging area. Go to **AWS Console → S3 → Create bucket**. Name it something like `confidence-databricks-staging`. - > Then create an IAM role with write access to the bucket and provide the **Role ARN** (e.g., `arn:aws:iam::123456789012:role/ConfidenceDatabricksStaging`). - > This is required even if your Databricks runs on GCP or Azure — the connector uses S3 for staging. +2. **SQL Warehouse ID** — the compute resource Confidence uses to run queries. + > Go to **Databricks → SQL Warehouses** in the left sidebar. Click on a warehouse, then open the **Connection details** tab. Copy the ID — it's a hex string like `ccf7028466008a3c`. + > If you don't have a SQL Warehouse: click **Create SQL Warehouse** → name it anything → pick **Serverless** type, **Small** size → **Create**. Then copy the ID from Connection details. + +3. **Service principal** — a robot account that Confidence uses to authenticate. + > You need **workspace admin access** for this step. Go to **Databricks → Settings** (gear icon top right) → **Identity and access** → **Service principals**. + > - Click **Add service principal → Add new** + > - Name it "Confidence" + > - After creation, copy the **Application (client) ID** (a UUID like `85cc292a-c1d2-453f-85ec-f4230e99238f`) + > - Click into the service principal → **Secrets → Generate secret** + > - Copy the **Secret** value — it's shown only once + > + > If you see "Access denied" or can't find Identity and access, you don't have admin access. Ask your Databricks workspace admin to create the service principal for you. + +**Part 2: S3 staging bucket (requires AWS)** + +Explain why this is needed: +> Confidence doesn't write directly to Databricks tables. Instead, it writes data files to an S3 bucket, then tells Databricks to load them. This is how most tools integrate with Databricks at scale — it's faster and more reliable than row-by-row inserts. +> +> You'll need an AWS account for this, even if your Databricks runs on GCP or Azure. + +4. **S3 bucket name** — the staging bucket. + > Go to **AWS Console → S3 → Create bucket**. + > - Name: something like `confidence-staging-` (must be globally unique) + > - Region: pick the same region as your Databricks workspace (e.g., `eu-west-1` for EU) + > - Leave all other settings as default → **Create bucket** + > + > If you already have a bucket you want to reuse, that works too — just give me the name. + +5. **AWS Region** — where the S3 bucket lives (e.g., `eu-west-1`, `us-east-1`). + +6. **IAM Role ARN** — an AWS role that grants Confidence permission to write to the bucket. + > Go to **AWS Console → IAM → Roles → Create role**. + > - Trusted entity: **Web identity** + > - Identity provider: add `accounts.google.com` + > - Audience: the Confidence service account for your account (`account-@spotify-confidence.iam.gserviceaccount.com` — the skill should compute this from the JWT token claim `https://confidence.dev/account_name`) + > - Click **Next** → attach the policy **AmazonS3FullAccess** (or a custom policy scoped to your bucket) + > - Name the role (e.g., `confidence-databricks-staging`) → **Create role** + > - Copy the **Role ARN** (looks like `arn:aws:iam::123456789012:role/confidence-databricks-staging`) + > + > **Important:** The role's trust policy must allow the Confidence service account to assume it via web identity federation. If Confidence gets "Not authorized to perform sts:AssumeRoleWithWebIdentity", the trust policy is wrong — check that the Confidence service account is listed as a trusted principal. + +**Part 3: Databricks schema** + +7. **Schema** — where Confidence creates its tables (default: `confidence`). + > In Databricks SQL editor, run: + > ```sql + > CREATE SCHEMA IF NOT EXISTS confidence; + > GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence TO ``; + > ``` + > If your workspace uses Unity Catalog, you may need to specify a catalog too: + > ```sql + > CREATE CATALOG IF NOT EXISTS confidence; + > CREATE SCHEMA IF NOT EXISTS confidence.confidence; + > GRANT USE CATALOG ON CATALOG confidence TO ``; + > GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence.confidence TO ``; + > ``` + > Copy the SQL to clipboard for the user. **Redshift:** From 603c9a3d196354ab5f8bb179634d89ce0671bc16 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:23:42 +0200 Subject: [PATCH 13/25] feat: add aws CLI automation for Databricks S3 staging setup Skill can now create S3 bucket, IAM role, and trust policy automatically via aws CLI. Falls back to manual console steps if aws CLI not available. Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 108 +++++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 6 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index 40111ce..0e7abe5 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -867,8 +867,93 @@ Explain why this is needed: > > You'll need an AWS account for this, even if your Databricks runs on GCP or Azure. +Ask the user: +> Do you have the `aws` CLI set up, or would you prefer manual steps? +> 1. Set it up for me (requires `aws` CLI) +> 2. Show me the steps + +**If the user picks 1 (aws CLI):** + +First check: `which aws`. If not found, offer to install: `brew install awscli` (macOS) or guide them to https://aws.amazon.com/cli/. + +Then check they're logged in: `aws sts get-caller-identity`. If not, tell them: +> Run `aws configure` or `aws sso login` to log into your AWS account first. + +Extract the Confidence service account from the token: +```bash +ACCOUNT_ID=$(echo "$TOKEN" | cut -d. -f2 | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d['https://confidence.dev/account_name'].split('/')[-1]) +") +CONFIDENCE_SA="account-${ACCOUNT_ID}@spotify-confidence.iam.gserviceaccount.com" +``` + +Ask the user for a bucket name (suggest `confidence-staging-`) and region (suggest `eu-west-1`). + +Then run these commands, confirming each step: + +```bash +# 1. Create S3 bucket +aws s3api create-bucket --bucket ${BUCKET_NAME} --region ${AWS_REGION} \ + --create-bucket-configuration LocationConstraint=${AWS_REGION} + +# 2. Create the trust policy file +cat > $TMPDIR/trust-policy.json << EOF +{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Federated": "accounts.google.com"}, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "accounts.google.com:email": "${CONFIDENCE_SA}" + } + } + }] +} +EOF + +# 3. Create IAM role +aws iam create-role --role-name confidence-databricks-staging \ + --assume-role-policy-document file://$TMPDIR/trust-policy.json + +# 4. Create and attach S3 access policy +cat > $TMPDIR/s3-policy.json << EOF +{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:PutObject", "s3:GetObject", "s3:DeleteObject", "s3:ListBucket"], + "Resource": [ + "arn:aws:s3:::${BUCKET_NAME}", + "arn:aws:s3:::${BUCKET_NAME}/*" + ] + }] +} +EOF +aws iam put-role-policy --role-name confidence-databricks-staging \ + --policy-name S3Access --policy-document file://$TMPDIR/s3-policy.json + +# 5. Get the role ARN +ROLE_ARN=$(aws iam get-role --role-name confidence-databricks-staging --query 'Role.Arn' --output text) +echo "ROLE_ARN: $ROLE_ARN" +``` + +After completion, show the user: +> AWS setup complete! +> - Bucket: `` in `` +> - Role: `` +> +> Continuing with connector setup... + +**If the user picks 2 (manual steps):** + 4. **S3 bucket name** — the staging bucket. - > Go to **AWS Console → S3 → Create bucket**. + > Go to **AWS Console** (https://console.aws.amazon.com) → **S3 → Create bucket**. > - Name: something like `confidence-staging-` (must be globally unique) > - Region: pick the same region as your Databricks workspace (e.g., `eu-west-1` for EU) > - Leave all other settings as default → **Create bucket** @@ -880,13 +965,24 @@ Explain why this is needed: 6. **IAM Role ARN** — an AWS role that grants Confidence permission to write to the bucket. > Go to **AWS Console → IAM → Roles → Create role**. > - Trusted entity: **Web identity** - > - Identity provider: add `accounts.google.com` - > - Audience: the Confidence service account for your account (`account-@spotify-confidence.iam.gserviceaccount.com` — the skill should compute this from the JWT token claim `https://confidence.dev/account_name`) - > - Click **Next** → attach the policy **AmazonS3FullAccess** (or a custom policy scoped to your bucket) - > - Name the role (e.g., `confidence-databricks-staging`) → **Create role** + > - Identity provider: select **accounts.google.com** (add it first if not listed under Identity providers) + > - Audience: `account-@spotify-confidence.iam.gserviceaccount.com` + > (the skill should compute the account ID from the JWT token and fill this in for the user) + > - Click **Next** → **Create policy** → JSON tab → paste this: + > ```json + > { + > "Version": "2012-10-17", + > "Statement": [{ + > "Effect": "Allow", + > "Action": ["s3:PutObject", "s3:GetObject", "s3:DeleteObject", "s3:ListBucket"], + > "Resource": ["arn:aws:s3:::", "arn:aws:s3:::/*"] + > }] + > } + > ``` + > - Attach the policy → name the role (e.g., `confidence-databricks-staging`) → **Create role** > - Copy the **Role ARN** (looks like `arn:aws:iam::123456789012:role/confidence-databricks-staging`) > - > **Important:** The role's trust policy must allow the Confidence service account to assume it via web identity federation. If Confidence gets "Not authorized to perform sts:AssumeRoleWithWebIdentity", the trust policy is wrong — check that the Confidence service account is listed as a trusted principal. + > **If you get "Not authorized to perform sts:AssumeRoleWithWebIdentity" later:** the trust policy is wrong — the Confidence service account email must exactly match what's in the role's trust policy. **Part 3: Databricks schema** From 565ba7fe58ce4a3a12aaa5185e93cd2c59e5a45e Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:39:19 +0200 Subject: [PATCH 14/25] fix: AWS trust policy must use numeric SA ID, not email Three bugs found during Databricks testing: 1. accounts.google.com:email rejected by AWS ("requires application id") 2. accounts.google.com:sub with email string accepted but fails at runtime ("Not authorized to perform sts:AssumeRoleWithWebIdentity") 3. Only accounts.google.com:sub with the numeric unique ID works Skill now fetches numeric ID via gcloud and documents the pitfall. Also: write aws credentials directly instead of interactive configure, install awscli via brew if missing, open console for login. Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index 0e7abe5..8ee0ce7 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -879,7 +879,7 @@ First check: `which aws`. If not found, offer to install: `brew install awscli` Then check they're logged in: `aws sts get-caller-identity`. If not, tell them: > Run `aws configure` or `aws sso login` to log into your AWS account first. -Extract the Confidence service account from the token: +Extract the Confidence service account and its numeric unique ID (required for AWS trust policy): ```bash ACCOUNT_ID=$(echo "$TOKEN" | cut -d. -f2 | python3 -c " import sys, json, base64 @@ -889,10 +889,24 @@ d = json.loads(base64.b64decode(p)) print(d['https://confidence.dev/account_name'].split('/')[-1]) ") CONFIDENCE_SA="account-${ACCOUNT_ID}@spotify-confidence.iam.gserviceaccount.com" + +# CRITICAL: AWS trust policy needs the NUMERIC unique ID, not the email. +# The email won't work — AWS requires accounts.google.com:sub which is the numeric ID. +SA_UNIQUE_ID=$(gcloud iam service-accounts describe ${CONFIDENCE_SA} \ + --project=spotify-confidence --format="value(uniqueId)") ``` +If `gcloud` can't access `spotify-confidence` project, the user needs to contact Confidence support to get the numeric service account ID. + Ask the user for a bucket name (suggest `confidence-staging-`) and region (suggest `eu-west-1`). +If `aws` CLI is not installed, install it: `brew install awscli` (macOS). + +If `aws` CLI is not configured, the skill should: +1. Open the AWS console login: `open "https://console.aws.amazon.com"` +2. Guide user to create access key: **click your name top right → Security credentials → Access keys → Create access key** +3. Write the credentials directly to `~/.aws/credentials` and `~/.aws/config` (don't use interactive `aws configure`) + Then run these commands, confirming each step: ```bash @@ -901,6 +915,9 @@ aws s3api create-bucket --bucket ${BUCKET_NAME} --region ${AWS_REGION} \ --create-bucket-configuration LocationConstraint=${AWS_REGION} # 2. Create the trust policy file +# IMPORTANT: Use accounts.google.com:sub with the NUMERIC service account ID. +# Using :email will fail with "MalformedPolicyDocument". +# Using the email string as :sub will fail at runtime with "Not authorized to perform sts:AssumeRoleWithWebIdentity". cat > $TMPDIR/trust-policy.json << EOF { "Version": "2012-10-17", @@ -910,7 +927,7 @@ cat > $TMPDIR/trust-policy.json << EOF "Action": "sts:AssumeRoleWithWebIdentity", "Condition": { "StringEquals": { - "accounts.google.com:email": "${CONFIDENCE_SA}" + "accounts.google.com:sub": "${SA_UNIQUE_ID}" } } }] From f4baefecc792bdb216c6701c719738b0c64de83e Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:48:36 +0200 Subject: [PATCH 15/25] fix: update Databricks skill with verified pipeline findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Data flows via GCS not S3 (S3 is fallback), fixed flow diagram - Added ~5 min batch delay warning - Databricks verification now uses SQL Statement API directly instead of just showing queries for the user to run - Removed stale "known backend issue" — pipeline works end-to-end Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 35 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index 8ee0ce7..c021a43 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -837,7 +837,9 @@ Before collecting details, explain what's needed upfront so the user knows the f > > If you don't have an AWS account, you'll need to create one (free tier works) or ask your infrastructure team for an S3 bucket and IAM role. > -> Here's how data flows: **Confidence → S3 staging bucket → Databricks tables** +> Here's how data flows: **Confidence → GCS staging → Databricks tables** (the S3 bucket is used as a fallback/alternative staging path, but GCS is the primary path for EU accounts). +> +> Data is batched and delivered every ~5 minutes. After creating the connectors, you'll need to wait for the first batch before data appears in Databricks. Then collect the details step by step. Ask one at a time, explain each field, and tell the user exactly where to find it: @@ -1474,16 +1476,27 @@ If neither available, show the queries for the Snowflake worksheet (https://app. **Databricks:** -Show queries for the Databricks SQL editor: -> ```sql -> SELECT targeting_key, rule, assignment_id, assignment_time -> FROM .assignments -> ORDER BY assignment_time DESC LIMIT 5; -> -> SHOW TABLES IN LIKE 'events_*'; -> SELECT * FROM . -> ORDER BY _event_time DESC LIMIT 5; -> ``` +Use the Databricks SQL Statement API to query directly (the skill already has the service principal credentials): +```bash +DB_TOKEN=$(curl -s -X POST "https://${DATABRICKS_HOST}/oidc/v1/token" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=client_credentials&client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&scope=all-apis" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])") + +curl -s -X POST "https://${DATABRICKS_HOST}/api/2.0/sql/statements" \ + -H "Authorization: Bearer $DB_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "warehouse_id": "'${WAREHOUSE_ID}'", + "statement": "SELECT targeting_key, rule, assignment_id, assignment_time FROM '${SCHEMA}'.assignments ORDER BY assignment_time DESC LIMIT 5", + "wait_timeout": "30s" + }' +``` + +**IMPORTANT:** Data is batched every ~5 minutes. If the table doesn't exist yet, wait and retry. Tell the user: +> Data delivery takes about 5 minutes. Let me check again... + +If `TABLE_OR_VIEW_NOT_FOUND` after 10 minutes, check the connector logs for errors. **Redshift:** From eaa2884e975902be3cf160fd74194c437c6a87f0 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:50:46 +0200 Subject: [PATCH 16/25] feat: rewrite Databricks config as conversational step-by-step guide - Ask one question at a time, confirm before moving on - Explain every field in plain language with exact UI paths - Service principal: full step-by-step with fallback for non-admins - S3 staging: explain the "mailbox" metaphor for why it's needed - Schema: detect Unity Catalog vs hive_metastore, copy SQL to clipboard - Accurate data flow explanation (GCS staging, 5-min batches) Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 121 +++++++++++++++++++---------- 1 file changed, 80 insertions(+), 41 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index c021a43..1eb9f09 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -827,47 +827,75 @@ Save the crypto key name (e.g., `cryptoKeys/snowflake-key`) for use in the wareh **Databricks:** -Before collecting details, explain what's needed upfront so the user knows the full picture: +Before collecting details, explain the full picture so the user knows what they need: > Setting up Databricks with Confidence requires three things: > -> 1. **A Databricks workspace** with admin access (to create a service principal) -> 2. **An AWS account** with an S3 bucket (Confidence stages data in S3 before loading into Databricks — this is required even if Databricks runs on GCP or Azure) -> 3. **A schema in Databricks** where Confidence will create tables +> 1. **A Databricks workspace** — you need admin access to create a service principal (a robot account) +> 2. **An AWS account with an S3 bucket** — Confidence needs this as a staging area for loading data into Databricks. This is required even if your Databricks runs on GCP or Azure +> 3. **A schema in Databricks** — a place for Confidence to create tables (e.g., `confidence`) > -> If you don't have an AWS account, you'll need to create one (free tier works) or ask your infrastructure team for an S3 bucket and IAM role. +> **How data flows:** +> Your flag assignments and events are collected by Confidence, staged in a cloud bucket, then batch-loaded into your Databricks tables every ~5 minutes. After setup, you'll need to wait a few minutes before data appears. > -> Here's how data flows: **Confidence → GCS staging → Databricks tables** (the S3 bucket is used as a fallback/alternative staging path, but GCS is the primary path for EU accounts). -> -> Data is batched and delivered every ~5 minutes. After creating the connectors, you'll need to wait for the first batch before data appears in Databricks. +> **Don't have an AWS account?** You'll need one for the S3 staging bucket. AWS free tier works fine. I can set it up for you if you have the `aws` CLI, or walk you through the AWS Console. + +Then collect the details **one at a time**. After each answer, confirm it before moving to the next. Don't dump all questions at once. -Then collect the details step by step. Ask one at a time, explain each field, and tell the user exactly where to find it: +**Part 1: Databricks connection** (3 things needed) -**Part 1: Databricks connection** +1. **Host** — ask the user: + > What's your Databricks workspace URL? Just paste the URL from your browser address bar. -1. **Host** — your Databricks workspace URL. - > This is the URL in your browser when you open Databricks. It looks like `https://dbc-xxxxx.cloud.databricks.com` or `https://1234567890.7.gcp.databricks.com`. Just copy it from your address bar — I only need the hostname, not the full URL. + Extract the hostname from whatever they paste (strip `https://`, trailing paths, query params). Valid examples: + - `dbc-a1b2c3d4-e5f6.cloud.databricks.com` + - `1234567890.7.gcp.databricks.com` + - `adb-1234567890.12.azuredatabricks.net` -2. **SQL Warehouse ID** — the compute resource Confidence uses to run queries. - > Go to **Databricks → SQL Warehouses** in the left sidebar. Click on a warehouse, then open the **Connection details** tab. Copy the ID — it's a hex string like `ccf7028466008a3c`. - > If you don't have a SQL Warehouse: click **Create SQL Warehouse** → name it anything → pick **Serverless** type, **Small** size → **Create**. Then copy the ID from Connection details. + Confirm: "Got it — your Databricks workspace is at ``." -3. **Service principal** — a robot account that Confidence uses to authenticate. - > You need **workspace admin access** for this step. Go to **Databricks → Settings** (gear icon top right) → **Identity and access** → **Service principals**. - > - Click **Add service principal → Add new** - > - Name it "Confidence" - > - After creation, copy the **Application (client) ID** (a UUID like `85cc292a-c1d2-453f-85ec-f4230e99238f`) - > - Click into the service principal → **Secrets → Generate secret** - > - Copy the **Secret** value — it's shown only once +2. **SQL Warehouse ID** — ask the user: + > I need a SQL Warehouse ID. Here's how to find it: + > 1. In Databricks, click **SQL Warehouses** in the left sidebar + > 2. Click on a warehouse name + > 3. Open the **Connection details** tab + > 4. Copy the **HTTP Path** — the ID is the last part after `/sql/1.0/warehouses/` + > + > It looks like a hex string, e.g., `ccf7028466008a3c` > - > If you see "Access denied" or can't find Identity and access, you don't have admin access. Ask your Databricks workspace admin to create the service principal for you. + > **Don't have a SQL Warehouse?** Click **Create SQL Warehouse** → name it "Confidence" → pick **Serverless**, size **Small** → **Create**. Then copy the ID. -**Part 2: S3 staging bucket (requires AWS)** + Confirm: "Using warehouse ``." + +3. **Service principal** — ask the user: + > I need a service principal — this is a robot account that Confidence uses to connect to Databricks. + > + > **To create one:** + > 1. Click the **gear icon** (⚙️) at the top of Databricks → **Settings** + > 2. Under **Identity and access**, click **Service principals** + > 3. Click **Add service principal → Add new** + > 4. Name it "Confidence" → **Add** + > 5. Click into the new service principal + > 6. Copy the **Application ID** (a UUID like `85cc292a-c1d2-...`) + > 7. Go to the **Secrets** tab → **Generate secret** + > 8. Copy both the **Secret** (shown only once!) and the **Client ID** + > + > Paste the **Client ID** and **Secret** here. -Explain why this is needed: -> Confidence doesn't write directly to Databricks tables. Instead, it writes data files to an S3 bucket, then tells Databricks to load them. This is how most tools integrate with Databricks at scale — it's faster and more reliable than row-by-row inserts. + If the user says they can't access Settings or service principals: + > You need workspace admin access for this step. Ask your Databricks admin to: + > 1. Create a service principal named "Confidence" + > 2. Generate a secret for it + > 3. Send you the Client ID and Secret + + Confirm: "Service principal configured." + +**Part 2: S3 staging bucket** (requires AWS account) + +Explain why: +> Confidence loads data into Databricks in batches via a staging bucket. The Confidence connector API requires an S3 bucket configuration, even when your Databricks runs on GCP or Azure. Think of it as a mailbox — Confidence drops files there, and Databricks picks them up. > -> You'll need an AWS account for this, even if your Databricks runs on GCP or Azure. +> You need an AWS account for this. If you don't have one, I can help you set one up. Ask the user: > Do you have the `aws` CLI set up, or would you prefer manual steps? @@ -1005,20 +1033,31 @@ After completion, show the user: **Part 3: Databricks schema** -7. **Schema** — where Confidence creates its tables (default: `confidence`). - > In Databricks SQL editor, run: - > ```sql - > CREATE SCHEMA IF NOT EXISTS confidence; - > GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence TO ``; - > ``` - > If your workspace uses Unity Catalog, you may need to specify a catalog too: - > ```sql - > CREATE CATALOG IF NOT EXISTS confidence; - > CREATE SCHEMA IF NOT EXISTS confidence.confidence; - > GRANT USE CATALOG ON CATALOG confidence TO ``; - > GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence.confidence TO ``; - > ``` - > Copy the SQL to clipboard for the user. +Ask the user: +> Last thing — where should Confidence create its tables in Databricks? I need a schema name. +> The default is `confidence`. If you already have a schema you'd like to use, let me know. + +Then check if the schema exists and the service principal has access. Generate the SQL and **copy to clipboard**: + +> I'll set up the schema and permissions. Here's what I'm running — copied to your clipboard. Paste it in the **Databricks SQL Editor** (left sidebar → SQL Editor) and run it. + +For workspaces **without Unity Catalog** (hive_metastore): +```sql +CREATE SCHEMA IF NOT EXISTS confidence; +GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence TO ``; +``` + +For workspaces **with Unity Catalog**: +```sql +CREATE CATALOG IF NOT EXISTS confidence; +CREATE SCHEMA IF NOT EXISTS confidence.confidence; +GRANT USE CATALOG ON CATALOG confidence TO ``; +GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence.confidence TO ``; +``` + +**How to tell which one:** If the user sees **Catalog** in the Databricks left sidebar, they have Unity Catalog. If they only see **Data**, they're on hive_metastore. + +After the user runs it, confirm: "Schema ready. Moving on to create the warehouse." **Redshift:** From 733cc57549e71158e67c7476fa34bae5acbc9ffe Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:52:15 +0200 Subject: [PATCH 17/25] =?UTF-8?q?fix:=20correct=20Databricks=20data=20flow?= =?UTF-8?q?=20=E2=80=94=20S3=20IS=20used,=20not=20just=20GCS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified S3 bucket has parquet files: assignments and events staged there before Databricks COPY INTO. Actual flow is: Confidence → S3 bucket (customer) → Databricks tables. GCS is internal only. Removed contradictory "GCS primary" claim. Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index 1eb9f09..401df01 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -836,7 +836,11 @@ Before collecting details, explain the full picture so the user knows what they > 3. **A schema in Databricks** — a place for Confidence to create tables (e.g., `confidence`) > > **How data flows:** -> Your flag assignments and events are collected by Confidence, staged in a cloud bucket, then batch-loaded into your Databricks tables every ~5 minutes. After setup, you'll need to wait a few minutes before data appears. +> Confidence collects your flag assignments and events internally, then writes parquet files to an S3 bucket you provide, and finally loads them into Databricks tables. This happens in batches every ~5 minutes. +> +> ``` +> Confidence (collects data) → S3 bucket (staging) → Databricks (tables) +> ``` > > **Don't have an AWS account?** You'll need one for the S3 staging bucket. AWS free tier works fine. I can set it up for you if you have the `aws` CLI, or walk you through the AWS Console. @@ -893,7 +897,7 @@ Then collect the details **one at a time**. After each answer, confirm it before **Part 2: S3 staging bucket** (requires AWS account) Explain why: -> Confidence loads data into Databricks in batches via a staging bucket. The Confidence connector API requires an S3 bucket configuration, even when your Databricks runs on GCP or Azure. Think of it as a mailbox — Confidence drops files there, and Databricks picks them up. +> Confidence writes parquet files to an S3 bucket, then Databricks loads them via COPY INTO. Think of it as a mailbox — Confidence drops files there, and Databricks picks them up. **This is required even if your Databricks runs on GCP or Azure.** > > You need an AWS account for this. If you don't have one, I can help you set one up. From 542998713e74b756527f14f08bddb1ebcfcc8878 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Wed, 10 Jun 2026 10:23:43 +0200 Subject: [PATCH 18/25] refactor: remove multi-client plugin files from ai-onboarding Move Cursor, Codex, and Gemini CLI plugin configs to the dedicated feat/multi-client-plugins branch for a separate PR. Co-Authored-By: Claude Opus 4.6 (1M context) --- .codex-plugin/plugin.json | 39 ----------------------- .cursor-plugin/plugin.json | 19 ----------- .mcp.json | 10 ++---- GEMINI.md | 15 --------- README.md | 65 ++++---------------------------------- gemini-extension.json | 38 ---------------------- 6 files changed, 9 insertions(+), 177 deletions(-) delete mode 100644 .codex-plugin/plugin.json delete mode 100644 .cursor-plugin/plugin.json delete mode 100644 GEMINI.md delete mode 100644 gemini-extension.json diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json deleted file mode 100644 index 83404f4..0000000 --- a/.codex-plugin/plugin.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "name": "confidence", - "version": "0.2.3", - "description": "Access Confidence feature flags, experiments, and migration tools directly from Codex.", - "author": { - "name": "Spotify Confidence", - "url": "https://confidence.spotify.com" - }, - "homepage": "https://confidence.spotify.com", - "repository": "https://github.com/spotify/confidence-ai-plugins", - "license": "Apache-2.0", - "keywords": [ - "feature-flags", - "experiments", - "a/b-testing", - "migration", - "openfeature" - ], - "skills": "./skills/", - "mcpServers": "./.mcp.json", - "interface": { - "displayName": "Confidence", - "shortDescription": "Feature flags, experiments, and migration tools", - "longDescription": "Access Confidence feature flags, experiments, and migration tools directly from Codex. Create, list, resolve, and target feature flags. Migrate from PostHog or Optimizely to Confidence SDK.", - "developerName": "Spotify Confidence", - "category": "Productivity", - "capabilities": [ - "Read", - "Write" - ], - "websiteURL": "https://confidence.spotify.com", - "defaultPrompt": [ - "List my feature flags", - "Create a flag called new-checkout with a boolean schema", - "Migrate my PostHog flags to Confidence" - ], - "logo": "./assets/logo.svg" - } -} diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json deleted file mode 100644 index 4a5ab8e..0000000 --- a/.cursor-plugin/plugin.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "confidence", - "displayName": "Confidence", - "version": "0.2.3", - "description": "Access Confidence feature flags, experiments, and migration tools directly from Cursor.", - "author": { - "name": "Spotify Confidence", - "url": "https://confidence.spotify.com" - }, - "license": "Apache-2.0", - "keywords": [ - "feature-flags", - "experiments", - "a/b-testing", - "migration", - "openfeature" - ], - "logo": "assets/logo.svg" -} diff --git a/.mcp.json b/.mcp.json index 18fcc8f..d5b9567 100644 --- a/.mcp.json +++ b/.mcp.json @@ -2,17 +2,11 @@ "mcpServers": { "confidence-flags": { "type": "http", - "url": "https://mcp.confidence.dev/mcp/flags", - "headers": { - "x-confidence-mcp-consumer": "plugin" - } + "url": "https://mcp.confidence.dev/mcp/flags" }, "confidence-docs": { "type": "http", - "url": "https://mcp.confidence.dev/mcp/docs", - "headers": { - "x-confidence-mcp-consumer": "plugin" - } + "url": "https://mcp.confidence.dev/mcp/docs" } } } diff --git a/GEMINI.md b/GEMINI.md deleted file mode 100644 index 50026ea..0000000 --- a/GEMINI.md +++ /dev/null @@ -1,15 +0,0 @@ -# Confidence Extension - -You are a helpful assistant that can manage Confidence feature flags and experiments using the Confidence MCP tools. - -## Available Tool Categories - -- **Feature Flags** — Create, list, update, archive, resolve, and target feature flags -- **Documentation** — Search Confidence docs and SDK integration guides - -## Guidelines - -- Always check that the user is authenticated before performing flag operations. -- Use the confidence-docs tools to answer questions about SDK integration, OpenFeature setup, and best practices. -- When creating flags, confirm the flag name and schema with the user before proceeding. -- For migrations from PostHog or Optimizely, guide the user through the migration plan before executing changes. diff --git a/README.md b/README.md index 46a2316..3736891 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Confidence AI Plugin -Official Confidence plugin for AI coding tools. Access feature flags, experiments, and migration tools directly from Claude Code, Cursor, Codex, and Gemini CLI. +Official Confidence plugin for AI clients. Access feature flags, experiments, and migration tools directly from your AI coding tool. ## Installation @@ -24,52 +24,12 @@ Official Confidence plugin for AI coding tools. Access feature flags, experiment ``` Follow the browser prompts to log in. -#### Updating +### Updating ```bash claude plugin update confidence:confidence ``` -### Cursor - -#### From the Marketplace - -1. Open **Cursor Settings** > **Plugins** -2. Search for **Confidence** -3. Click **Install** - -#### Manual setup - -Add the MCP servers to `.cursor/mcp.json` in your project (or `~/.cursor/mcp.json` globally): - -```json -{ - "mcpServers": { - "confidence-flags": { - "url": "https://mcp.confidence.dev/mcp/flags" - }, - "confidence-docs": { - "url": "https://mcp.confidence.dev/mcp/docs" - } - } -} -``` - -### Codex - -```bash -codex plugin marketplace add spotify/confidence-ai-plugins -codex -/plugins -# Select Confidence and install -``` - -### Gemini CLI - -```bash -gemini extensions install https://github.com/spotify/confidence-ai-plugins -``` - ### Local Development ```bash @@ -81,22 +41,20 @@ claude --plugin-dir ./confidence-ai-plugins This plugin provides access to Confidence tools across these categories: -- **Feature flags** — Create, list, update, archive, resolve, and target feature flags -- **Documentation** — Search Confidence docs and SDK integration guides -- **Migration** — Migrate feature flags from PostHog to Confidence +- **Feature flags** - Create, list, update, archive, and resolve feature flags +- **Migration** - Migrate feature flags from PostHog to Confidence ## Slash Commands -- `/confidence:migrate-posthog` — Migrate feature flags from PostHog to Confidence SDK -- `/confidence:onboard-confidence` — Create Confidence accounts and onboard users +- `/confidence:migrate-posthog` - Migrate feature flags from PostHog to Confidence SDK ## Example Usage ``` > List my feature flags > Create a flag called new-checkout with a boolean schema -> /confidence:migrate-posthog plan flag -> /confidence:migrate-posthog plan code +> /migrate-posthog plan flag +> /migrate-posthog plan code ``` ## MCP Servers @@ -106,15 +64,6 @@ This plugin provides access to Confidence tools across these categories: | `confidence-flags` | `https://mcp.confidence.spotify.com/mcp/flags` | Feature flag management | | `confidence-docs` | `https://mcp.confidence.spotify.com/mcp/docs` | Confidence documentation | -## Supported Clients - -| Client | Config | Marketplace | -|--------|--------|-------------| -| Claude Code | `.claude-plugin/` | Community marketplace | -| Cursor | `.cursor-plugin/` | Cursor Marketplace | -| Codex | `.codex-plugin/` | Via marketplace command | -| Gemini CLI | `gemini-extension.json` | Direct from repo | - ## Documentation - [Confidence documentation](https://confidence.spotify.com/docs) diff --git a/gemini-extension.json b/gemini-extension.json deleted file mode 100644 index 9d3872a..0000000 --- a/gemini-extension.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "name": "confidence", - "version": "0.2.3", - "description": "Access Confidence feature flags, experiments, and migration tools directly from Gemini CLI.", - "mcpServers": { - "confidence-flags": { - "command": "npx", - "args": [ - "mcp-remote@latest", - "${CONFIDENCE_MCP_FLAGS_URL:-https://mcp.confidence.dev/mcp/flags}", - "--header", - "x-confidence-mcp-consumer:plugin" - ] - }, - "confidence-docs": { - "command": "npx", - "args": [ - "mcp-remote@latest", - "${CONFIDENCE_MCP_DOCS_URL:-https://mcp.confidence.dev/mcp/docs}", - "--header", - "x-confidence-mcp-consumer:plugin" - ] - } - }, - "contextFileName": "GEMINI.md", - "settings": [ - { - "name": "Confidence MCP Flags URL", - "description": "Confidence MCP flags endpoint URL (default: https://mcp.confidence.dev/mcp/flags).", - "envVar": "CONFIDENCE_MCP_FLAGS_URL" - }, - { - "name": "Confidence MCP Docs URL", - "description": "Confidence MCP docs endpoint URL (default: https://mcp.confidence.dev/mcp/docs).", - "envVar": "CONFIDENCE_MCP_DOCS_URL" - } - ] -} From 8669c1ec22029cd0b2c566ee8a3b75ac579cf484 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Wed, 10 Jun 2026 13:44:46 +0200 Subject: [PATCH 19/25] feat: rewrite Redshift setup with full step-by-step guide Findings from end-to-end Redshift testing: - Validate endpoint DOES support Redshift (fixed incorrect note) - One IAM role must be trusted by both Google OIDC and Redshift service - Role must be attached to cluster for COPY command to work - GRANT USAGE ON SCHEMA TO PUBLIC required or validation returns "not found" - Redshift Serverless not supported (needs provisioned cluster identifier) - S3 staging bucket required (same pattern as Databricks) - Full aws CLI automation for creating cluster, role, bucket, schema Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 118 ++++++++++++++++++++++++++--- 1 file changed, 106 insertions(+), 12 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index 401df01..c8af62f 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -1065,27 +1065,121 @@ After the user runs it, confirm: "Schema ready. Moving on to create the warehous **Redshift:** -Guide the user through each field with plain-language explanations and where to find the value: +Before collecting details, explain the full picture: -1. **Cluster** — your Redshift cluster identifier. - > Go to **AWS Console → Amazon Redshift → Clusters**. The cluster name is in the list (e.g., `my-analytics-cluster`). +> Setting up Redshift with Confidence requires: +> +> 1. **A Redshift cluster** (provisioned, not Serverless — Confidence uses the Redshift Data API with cluster identifiers) +> 2. **An S3 bucket** — same staging pattern as Databricks: Confidence writes data to S3, then Redshift loads it via COPY +> 3. **One IAM role** that serves double duty — both Confidence (via Google OIDC) and Redshift (for COPY) need to assume it +> 4. **A schema** with public grants so Confidence can see it +> +> **How data flows:** +> ``` +> Confidence → S3 bucket (staging) → Redshift COPY INTO → tables +> ``` +> +> If you already have a Redshift cluster, I just need the details. If not, I can create one via the `aws` CLI. -2. **AWS Region** — where your cluster runs (e.g., `us-east-1`, `eu-west-1`). - > Shown in the top-right corner of your AWS Console, or in the cluster details page. +Then collect step by step, one at a time: -3. **IAM Role ARN** — the role Confidence assumes to access Redshift. - > Go to **AWS Console → IAM → Roles**. Create or pick a role with Redshift access. The ARN looks like `arn:aws:iam::123456789012:role/ConfidenceRedshift`. +**Part 1: Redshift cluster** -4. **Database name** — the Redshift database (default: `dev` or your main database). +1. **Cluster identifier** — ask the user: + > What's your Redshift cluster name? Go to **AWS Console → Amazon Redshift → Clusters**. The name is in the list. + > + > **Don't have one?** I can create a single-node `ra3.large` cluster for you (cheapest option, ~$0.25/hr). Note: Redshift Serverless won't work — Confidence needs a provisioned cluster identifier. + +2. **AWS Region** — e.g., `eu-west-1`. Usually same region as the cluster. + +3. **Database name** — default is `dev`. -5. **Schema name** — where Confidence stores its tables (default: `confidence`). +**Part 2: IAM role (one role for both Confidence and Redshift)** -6. **Authentication** — AWS access key + secret, or web identity federation. +4. **IAM Role** — this single role must be trusted by both Google OIDC (so Confidence can write to S3 and call Redshift Data API) AND the Redshift service (so COPY can read from S3). + + If using `aws` CLI, create it automatically: + ```bash + # Get the Confidence SA numeric ID + SA_UNIQUE_ID=$(gcloud iam service-accounts describe ${CONFIDENCE_SA} \ + --project=spotify-confidence --format="value(uniqueId)") + + # Create role with dual trust policy + cat > $TMPDIR/redshift-trust.json << EOF + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Federated": "accounts.google.com"}, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "accounts.google.com:sub": "${SA_UNIQUE_ID}" + } + } + }, + { + "Effect": "Allow", + "Principal": {"Service": "redshift.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] + } + EOF + aws iam create-role --role-name confidence-redshift \ + --assume-role-policy-document file://$TMPDIR/redshift-trust.json + ``` + + Then attach permissions (S3 access + Redshift Data API): + ```bash + # S3 access + aws iam put-role-policy --role-name confidence-redshift \ + --policy-name S3Access --policy-document file://$TMPDIR/s3-policy.json + + # Redshift Data API access + cat > $TMPDIR/redshift-data-policy.json << EOF + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["redshift-data:*", "redshift:GetClusterCredentials", + "redshift:GetClusterCredentialsWithIAM", "redshift:DescribeClusters"], + "Resource": "*" + }] + } + EOF + aws iam put-role-policy --role-name confidence-redshift \ + --policy-name RedshiftAccess --policy-document file://$TMPDIR/redshift-data-policy.json + ``` + + **CRITICAL:** Attach the role to the Redshift cluster (required for COPY command): + ```bash + aws redshift modify-cluster-iam-roles \ + --cluster-identifier ${CLUSTER} \ + --add-iam-roles ${ROLE_ARN} --region ${AWS_REGION} + ``` + Wait for status `in-sync` before proceeding. + +**Part 3: S3 staging bucket** + +5. Same as Databricks — create an S3 bucket for staging. Can reuse the same bucket if user already set one up for Databricks. + +**Part 4: Schema** + +6. **Schema** — default `confidence`. Create it and grant public access: + ```bash + aws redshift-data execute-statement \ + --cluster-identifier ${CLUSTER} --database ${DATABASE} --db-user admin \ + --sql "CREATE SCHEMA IF NOT EXISTS confidence; GRANT USAGE ON SCHEMA confidence TO PUBLIC; GRANT CREATE ON SCHEMA confidence TO PUBLIC;" \ + --region ${AWS_REGION} + ``` + **IMPORTANT:** `GRANT USAGE ON SCHEMA ... TO PUBLIC` is required — without it, Confidence's validation returns "Schema not found" even though the schema exists. ### Step 3: Validate configuration -**NOTE:** The validate endpoint only supports BigQuery (`bigQueryConfig`) and Snowflake (`snowflakeConfig`). The Confidence backend does not recognize Databricks or Redshift configs for validation (returns "configuration must be set" for any field name variant). For Databricks and Redshift, skip validation and proceed directly to Step 4 (Create warehouse). Tell the user honestly: -> Pre-validation isn't available yet for Databricks/Redshift. I'll create the warehouse now and we'll verify the connection works end-to-end in the pipeline test step. +**NOTE:** The validate endpoint supports BigQuery (`bigQueryConfig`), Snowflake (`snowflakeConfig`), and Redshift (`redshiftConfig`). It does NOT support Databricks (returns "configuration must be set" for any field name variant). For Databricks, skip validation and proceed directly to Step 4 (Create warehouse). Tell the user: +> Pre-validation isn't available yet for Databricks. I'll create the warehouse now and we'll verify the connection works end-to-end in the pipeline test step. For BigQuery/Snowflake: ```bash From bafa97684fc8b94fbe35b36e745781064a3455dd Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Wed, 10 Jun 2026 13:48:07 +0200 Subject: [PATCH 20/25] feat: rewrite Redshift as conversational guide + per-warehouse step trackers Redshift: - Full step-by-step for someone who's never used AWS or Redshift - Explains what Redshift is, why S3 is needed, what IAM roles do - One question at a time, confirm before moving on - Auto-creates cluster, bucket, role, schema via aws CLI - Manual AWS Console fallback path - Explains GRANT USAGE requirement - Explains why Serverless won't work Step trackers: - Each warehouse type gets its own tracker showing actual sub-steps - BigQuery: 10 steps, Snowflake: 12, Databricks: 13, Redshift: 13 - Tracker updates after every step so user sees progress Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 244 +++++++++++++++++++++++------ 1 file changed, 200 insertions(+), 44 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index c8af62f..52a583d 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -740,19 +740,89 @@ Requires an authenticated token. If none saved, run the Auth0 login flow first. ### Step Tracker +Show the initial tracker at start. After the user picks a warehouse type in Step 1, **replace it** with the warehouse-specific tracker that shows the actual sub-steps. Update and re-display after each step. + +**Initial (before warehouse choice):** ``` ───── Setup Warehouse ───────────────────────────────────── - [1] Choose warehouse ○ pending - [2] Configure ○ pending - [3] Validate ○ pending - [4] Create warehouse ○ pending - [5] Create connectors ○ pending - [6] Assignment table ○ pending - [7] Verify pipeline ○ pending - [8] Done ○ pending + [1] Choose warehouse ▶ in-progress +──────────────────────────────────────────────────────────── +``` + +**BigQuery tracker (after choosing BigQuery):** +``` +───── Setup Warehouse (BigQuery) ────────────────────────── + [1] Choose warehouse ● done + [2] GCP project ID ○ pending + [3] Dataset name ○ pending + [4] Service account ○ pending + [5] Validate & fix ○ pending + [6] Create warehouse ○ pending + [7] Create connectors ○ pending + [8] Assignment table ○ pending + [9] Verify pipeline ○ pending + [10] Done ○ pending +──────────────────────────────────────────────────────────── +``` + +**Snowflake tracker (after choosing Snowflake):** +``` +───── Setup Warehouse (Snowflake) ───────────────────────── + [1] Choose warehouse ● done + [2] Account & user ○ pending + [3] Role & warehouse ○ pending + [4] Database & schema ○ pending + [5] Create crypto key ○ pending + [6] Register key in SF ○ pending + [7] Validate ○ pending + [8] Create warehouse ○ pending + [9] Create connectors ○ pending + [10] Assignment table ○ pending + [11] Verify pipeline ○ pending + [12] Done ○ pending ──────────────────────────────────────────────────────────── ``` +**Databricks tracker (after choosing Databricks):** +``` +───── Setup Warehouse (Databricks) ──────────────────────── + [1] Choose warehouse ● done + [2] Workspace URL ○ pending + [3] SQL Warehouse ID ○ pending + [4] Service principal ○ pending + [5] AWS account & CLI ○ pending + [6] S3 bucket ○ pending + [7] IAM role ○ pending + [8] Databricks schema ○ pending + [9] Create warehouse ○ pending + [10] Create connectors ○ pending + [11] Assignment table ○ pending + [12] Verify pipeline ○ pending + [13] Done ○ pending +──────────────────────────────────────────────────────────── +``` + +**Redshift tracker (after choosing Redshift):** +``` +───── Setup Warehouse (Redshift) ────────────────────────── + [1] Choose warehouse ● done + [2] AWS account & CLI ○ pending + [3] Redshift cluster ○ pending + [4] S3 bucket ○ pending + [5] IAM role ○ pending + [6] Attach role ○ pending + [7] Schema & grants ○ pending + [8] Validate ○ pending + [9] Create warehouse ○ pending + [10] Create connectors ○ pending + [11] Assignment table ○ pending + [12] Verify pipeline ○ pending + [13] Done ○ pending +──────────────────────────────────────────────────────────── +``` + +Use `●` for completed, `▶` for in-progress, `○` for pending. Re-display the full tracker after every step transition so the user always sees their progress. + ### Step 1: Choose warehouse type > Which data warehouse do you use? @@ -1065,46 +1135,89 @@ After the user runs it, confirm: "Schema ready. Moving on to create the warehous **Redshift:** -Before collecting details, explain the full picture: +Before collecting details, explain the full picture so the user knows what they're signing up for: -> Setting up Redshift with Confidence requires: +> Setting up Redshift with Confidence requires an **AWS account**. Here's what we'll set up: > -> 1. **A Redshift cluster** (provisioned, not Serverless — Confidence uses the Redshift Data API with cluster identifiers) -> 2. **An S3 bucket** — same staging pattern as Databricks: Confidence writes data to S3, then Redshift loads it via COPY -> 3. **One IAM role** that serves double duty — both Confidence (via Google OIDC) and Redshift (for COPY) need to assume it -> 4. **A schema** with public grants so Confidence can see it +> 1. **A Redshift cluster** — a data warehouse that stores your experiment data +> 2. **An S3 bucket** — a staging area where Confidence drops data files before loading them into Redshift +> 3. **An IAM role** — permissions that let Confidence write to S3 and load into Redshift +> 4. **A schema** — a folder inside Redshift where Confidence creates its tables > > **How data flows:** > ``` -> Confidence → S3 bucket (staging) → Redshift COPY INTO → tables +> Confidence → S3 bucket (staging) → Redshift COPY → your tables > ``` > -> If you already have a Redshift cluster, I just need the details. If not, I can create one via the `aws` CLI. +> I can set up everything automatically if you have the `aws` CLI, or walk you through the AWS Console step by step. +> +> **Don't have an AWS account?** You'll need one. I can open the signup page for you. AWS free tier covers S3, but Redshift clusters cost ~$0.25/hr while running. You can delete it after testing. + +Ask the user: +> Do you have the `aws` CLI set up, or would you prefer manual steps? +> 1. Set it up for me (requires `aws` CLI) +> 2. Show me the steps + +**If the user picks 1 (aws CLI):** + +Check `which aws`. If not found: `brew install awscli` (macOS). +Check `aws sts get-caller-identity`. If not logged in, open the AWS console login (`open "https://console.aws.amazon.com"`), guide them to create access keys (**click name top right → Security credentials → Access keys → Create**), then write the credentials to `~/.aws/credentials` and `~/.aws/config`. -Then collect step by step, one at a time: +Then ask one question at a time: **Part 1: Redshift cluster** -1. **Cluster identifier** — ask the user: - > What's your Redshift cluster name? Go to **AWS Console → Amazon Redshift → Clusters**. The name is in the list. +1. Ask the user: + > Do you already have a Redshift cluster, or should I create one? + + If they have one: + > What's the cluster name? Go to **AWS Console → Amazon Redshift → Clusters**. The name is in the first column. + + If they need one, explain: + > I'll create a single-node Redshift cluster. This is a data warehouse — like a powerful database optimized for analytics. + > - **Cost:** ~$0.25/hour while running. Delete it when you're done testing. + > - **Type:** `ra3.large` (cheapest option that supports single-node) + > - **Region:** `eu-west-1` (Europe) — should match where your Confidence account is > - > **Don't have one?** I can create a single-node `ra3.large` cluster for you (cheapest option, ~$0.25/hr). Note: Redshift Serverless won't work — Confidence needs a provisioned cluster identifier. + > **Important:** Redshift Serverless won't work — Confidence needs a provisioned cluster. I'll create the right type. + + Create it: + ```bash + aws redshift create-cluster \ + --cluster-identifier confidence-redshift-${ACCOUNT_ID} \ + --cluster-type single-node \ + --node-type ra3.large \ + --master-username admin \ + --master-user-password '' \ + --db-name dev \ + --region eu-west-1 \ + --publicly-accessible + ``` -2. **AWS Region** — e.g., `eu-west-1`. Usually same region as the cluster. + Wait for status `available` (takes ~1–2 minutes): + ```bash + aws redshift wait cluster-available --cluster-identifier ${CLUSTER} --region ${AWS_REGION} + ``` -3. **Database name** — default is `dev`. + Confirm: "Redshift cluster `` is running." -**Part 2: IAM role (one role for both Confidence and Redshift)** +**Part 2: IAM role** -4. **IAM Role** — this single role must be trusted by both Google OIDC (so Confidence can write to S3 and call Redshift Data API) AND the Redshift service (so COPY can read from S3). +2. Explain: + > Now I need to set up permissions. Confidence needs a single IAM role that does two things: + > - Lets Confidence write data files to S3 and query Redshift + > - Lets Redshift read those files from S3 (via the COPY command) + > + > I'll create this role automatically. - If using `aws` CLI, create it automatically: + Get the Confidence service account numeric ID: ```bash - # Get the Confidence SA numeric ID SA_UNIQUE_ID=$(gcloud iam service-accounts describe ${CONFIDENCE_SA} \ --project=spotify-confidence --format="value(uniqueId)") + ``` - # Create role with dual trust policy + Create the role with dual trust (Google OIDC + Redshift): + ```bash cat > $TMPDIR/redshift-trust.json << EOF { "Version": "2012-10-17", @@ -1131,50 +1244,93 @@ Then collect step by step, one at a time: --assume-role-policy-document file://$TMPDIR/redshift-trust.json ``` - Then attach permissions (S3 access + Redshift Data API): + Attach S3 + Redshift Data API permissions: ```bash - # S3 access + # S3 write access + cat > $TMPDIR/s3-policy.json << EOF + {"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:PutObject","s3:GetObject","s3:DeleteObject","s3:ListBucket"],"Resource":["arn:aws:s3:::${BUCKET_NAME}","arn:aws:s3:::${BUCKET_NAME}/*"]}]} + EOF aws iam put-role-policy --role-name confidence-redshift \ --policy-name S3Access --policy-document file://$TMPDIR/s3-policy.json # Redshift Data API access cat > $TMPDIR/redshift-data-policy.json << EOF - { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": ["redshift-data:*", "redshift:GetClusterCredentials", - "redshift:GetClusterCredentialsWithIAM", "redshift:DescribeClusters"], - "Resource": "*" - }] - } + {"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["redshift-data:*","redshift:GetClusterCredentials","redshift:GetClusterCredentialsWithIAM","redshift:DescribeClusters"],"Resource":"*"}]} EOF aws iam put-role-policy --role-name confidence-redshift \ --policy-name RedshiftAccess --policy-document file://$TMPDIR/redshift-data-policy.json ``` - **CRITICAL:** Attach the role to the Redshift cluster (required for COPY command): + **CRITICAL:** Attach the role to the Redshift cluster — without this, the COPY command can't read from S3: ```bash aws redshift modify-cluster-iam-roles \ --cluster-identifier ${CLUSTER} \ --add-iam-roles ${ROLE_ARN} --region ${AWS_REGION} ``` - Wait for status `in-sync` before proceeding. + Wait for `in-sync`: + ```bash + aws redshift describe-clusters --cluster-identifier ${CLUSTER} --region ${AWS_REGION} \ + --query "Clusters[0].IamRoles[*].{Role:IamRoleArn,Status:ApplyStatus}" --output table + ``` + + Confirm: "IAM role created and attached to cluster." **Part 3: S3 staging bucket** -5. Same as Databricks — create an S3 bucket for staging. Can reuse the same bucket if user already set one up for Databricks. +3. Ask the user: + > Do you have an S3 bucket I should use, or should I create one? + + If they already did Databricks setup: + > You already have `` from the Databricks setup. Want to reuse it? + + Otherwise, create one: + ```bash + aws s3api create-bucket --bucket confidence-redshift-${ACCOUNT_ID} \ + --region ${AWS_REGION} \ + --create-bucket-configuration LocationConstraint=${AWS_REGION} + ``` + + Confirm: "S3 bucket `` created in ``." **Part 4: Schema** -6. **Schema** — default `confidence`. Create it and grant public access: +4. Ask the user: + > What should the schema be called? The default is `confidence`. + + Create the schema and grant permissions so Confidence can see it: ```bash aws redshift-data execute-statement \ --cluster-identifier ${CLUSTER} --database ${DATABASE} --db-user admin \ - --sql "CREATE SCHEMA IF NOT EXISTS confidence; GRANT USAGE ON SCHEMA confidence TO PUBLIC; GRANT CREATE ON SCHEMA confidence TO PUBLIC;" \ + --sql "CREATE SCHEMA IF NOT EXISTS ${SCHEMA}; GRANT USAGE ON SCHEMA ${SCHEMA} TO PUBLIC; GRANT CREATE ON SCHEMA ${SCHEMA} TO PUBLIC;" \ --region ${AWS_REGION} ``` - **IMPORTANT:** `GRANT USAGE ON SCHEMA ... TO PUBLIC` is required — without it, Confidence's validation returns "Schema not found" even though the schema exists. + + **IMPORTANT:** `GRANT USAGE ON SCHEMA ... TO PUBLIC` is required — without it, Confidence's validation returns "Schema not found" even though the schema exists. This is because Confidence connects via IAM, not as the `admin` user. + + Confirm: "Schema `` created with permissions." + +**If the user picks 2 (manual steps):** + +Walk them through the AWS Console for each step: + +1. **Redshift cluster:** Go to **AWS Console → Amazon Redshift → Create cluster** → single-node, ra3.large, database `dev`, publicly accessible. + +2. **S3 bucket:** Go to **AWS Console → S3 → Create bucket** → name it, pick same region as cluster. + +3. **IAM role:** Go to **AWS Console → IAM → Roles → Create role** → two trust steps: + - Add **Web identity** trust with `accounts.google.com`, sub = `` (compute and display for the user) + - Add **AWS service** trust for `redshift.amazonaws.com` + - Attach policies: custom S3 policy scoped to bucket + `AmazonRedshiftDataFullAccess` + - Copy the **Role ARN** + - Go back to **Redshift → Clusters → your cluster → Properties → Manage IAM roles → Add the new role** + +4. **Schema:** Go to **Redshift → Query editor v2** → connect to cluster → run: + ```sql + CREATE SCHEMA IF NOT EXISTS confidence; + GRANT USAGE ON SCHEMA confidence TO PUBLIC; + GRANT CREATE ON SCHEMA confidence TO PUBLIC; + ``` + Copy the SQL to clipboard for the user. ### Step 3: Validate configuration From 6df2b6018bc4e880ea54d7bc9cab93b6d3665aa1 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Wed, 10 Jun 2026 14:29:42 +0200 Subject: [PATCH 21/25] feat: split warehouse skills, remove token persistence, add dry-run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes: 1. Split setup-warehouse into per-type skills for LLM efficiency: - setup-warehouse/ (thin dispatcher, ~190 lines) - setup-warehouse-bigquery/ (~700 lines) - setup-warehouse-snowflake/ (~750 lines) - setup-warehouse-databricks/ (~860 lines) - setup-warehouse-redshift/ (~870 lines) Each skill is self-contained with auth, API refs, and step tracker. Onboard-confidence SKILL.md reduced from 2254 to 1152 lines. 2. Remove token persistence — tokens kept in session only, never written to disk. Browser login on every new session or expiry. 3. Add dry-run skill (~1380 lines) for testing UX without real APIs. Simulates all sub-commands with mock data, [DRY RUN] prefix, same step trackers and questions as real skills. Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence-dry-run/SKILL.md | 1381 ++++++++++++++++++++ skills/onboard-confidence/SKILL.md | 1139 +--------------- skills/setup-warehouse-bigquery/SKILL.md | 705 ++++++++++ skills/setup-warehouse-databricks/SKILL.md | 860 ++++++++++++ skills/setup-warehouse-redshift/SKILL.md | 869 ++++++++++++ skills/setup-warehouse-snowflake/SKILL.md | 753 +++++++++++ skills/setup-warehouse/SKILL.md | 187 +++ 7 files changed, 4771 insertions(+), 1123 deletions(-) create mode 100644 skills/onboard-confidence-dry-run/SKILL.md create mode 100644 skills/setup-warehouse-bigquery/SKILL.md create mode 100644 skills/setup-warehouse-databricks/SKILL.md create mode 100644 skills/setup-warehouse-redshift/SKILL.md create mode 100644 skills/setup-warehouse-snowflake/SKILL.md create mode 100644 skills/setup-warehouse/SKILL.md diff --git a/skills/onboard-confidence-dry-run/SKILL.md b/skills/onboard-confidence-dry-run/SKILL.md new file mode 100644 index 0000000..6b958ae --- /dev/null +++ b/skills/onboard-confidence-dry-run/SKILL.md @@ -0,0 +1,1381 @@ +--- +description: Dry-run the Confidence onboarding flow to test UX without real API calls. Use when the user says "dry run", "test onboarding", "demo onboarding", or wants to preview the onboarding experience. +--- + +# Confidence Onboarding — Dry Run + +This skill runs the full onboarding experience with simulated API responses. No real accounts, flags, or warehouses are created. Use it to test the UX flow, demo to stakeholders, or train new users. + +## How it works + +- Every API call is simulated with realistic mock responses +- The step trackers, questions, confirmations, and explanations are identical to the real skill +- Browser login is skipped — a mock token is used +- All warehouse types can be tested without needing actual AWS/GCP/Snowflake/Databricks accounts + +## Commands + +| Command | What it simulates | +|---------|-------------------| +| `/onboard-confidence-dry-run create-account` | Account creation flow | +| `/onboard-confidence-dry-run invite-user` | User invitation flow | +| `/onboard-confidence-dry-run create-client` | SDK client creation flow | +| `/onboard-confidence-dry-run setup-wizard` | Full setup wizard (client → flag → variants → targeting → resolve) | +| `/onboard-confidence-dry-run setup-warehouse` | Warehouse setup dispatcher | +| `/onboard-confidence-dry-run setup-warehouse-bigquery` | BigQuery warehouse setup | +| `/onboard-confidence-dry-run setup-warehouse-snowflake` | Snowflake warehouse setup | +| `/onboard-confidence-dry-run setup-warehouse-databricks` | Databricks warehouse setup | +| `/onboard-confidence-dry-run setup-warehouse-redshift` | Redshift warehouse setup | + +--- + +## Dry Run Rules + +1. **Show the exact same UX** as the real skill — same step trackers, same questions, same confirmations, same tone +2. **Display `[DRY RUN]` prefix** on every status update so the user knows it's simulated +3. **Simulate API responses** — don't make real HTTP calls. Instead, print what would happen and show mock response data +4. **Still ask the user for input** at every step (workspace name, flag name, warehouse type, etc.) — the point is to test the interaction flow +5. **Skip browser login** — use a mock token with mock claims: + ``` + Mock token claims: + - account_name: accounts/dry-run-demo + - region: EU + - org_id: org_DryRunDemo123 + - identity: identities/udryrun123 + ``` +6. **Use realistic mock data** for API responses. Examples are listed in the Mock Data Reference section below. +7. **For warehouse-specific dry runs**, simulate the full flow including: + - Snowflake: mock crypto key creation, show the ALTER USER SQL that would be generated + - Databricks: mock S3 bucket creation, IAM role, show the trust policy that would be created + - Redshift: mock cluster creation, show the dual trust policy, GRANT statements + - BigQuery: mock gcloud commands that would run +8. **At the end of each dry run**, show the dry-run summary banner (see Dry Run Summary section) +9. **No sandbox overrides** — since no real network calls are made, the skill never needs `dangerouslyDisableSandbox: true` +10. **No token persistence** — never write anything to `~/.confidence/` or `$TMPDIR` + +--- + +## Mock Data Reference + +Use these mock responses when simulating API calls. Substitute user-provided values (workspace name, flag name, etc.) where indicated with ``. + +### Authentication mock + +Skip all browser-based Auth0 login. Instead, tell the user: + +> [DRY RUN] Skipping browser login — using mock credentials. + +Mock token claims to use throughout the session: + +``` +account_name: accounts/dry-run-demo +region: EU +org_id: org_DryRunDemo123 +identity: identities/udryrun123 +email: dryrun@example.com +``` + +Region derived from mock token: `eu` (lowercase). All mock API URLs use `eu` prefix (e.g., `iam.eu.confidence.dev`). + +### Create account response + +```json +{ + "name": "accounts/dry-run-demo", + "externalId": "org_DryRunDemo123", + "loginId": "", + "displayName": "" +} +``` + +### Check login ID availability + +```json +{"available": true, "message": ""} +``` + +If the user enters `taken-name` (for testing), return: +```json +{"available": false, "message": "This workspace name is already in use."} +``` + +### Create client response + +```json +{ + "name": "clients/dry-run-client", + "displayName": "" +} +``` + +### Client secret (mock) + +``` +dryrn_sk_mock1234567890abcdef +``` + +### Create flag response + +```json +{ + "name": "flags/", + "schema": {} +} +``` + +### Update flag schema response + +```json +{ + "name": "flags/", + "schema": { + "schema": { + "enabled": {"boolSchema": {}} + } + } +} +``` + +### Create variant response + +```json +{ + "name": "flags//variants/", + "value": {"enabled": true} +} +``` + +### Add flag to client response + +```json +{ + "name": "flags/", + "clients": ["clients/dry-run-client"] +} +``` + +### Create segment response + +```json +{ + "name": "segments/everyone", + "displayName": "Everyone", + "allocation": {"proportion": {"value": "1"}} +} +``` + +### Create rule response + +```json +{ + "name": "flags//rules/rule1", + "segment": "segments/everyone", + "enabled": true +} +``` + +### Resolve response + +```json +{ + "resolvedFlags": [ + { + "flag": "flags/", + "variant": "flags//variants/", + "value": {"enabled": true}, + "reason": "RESOLVE_REASON_MATCH" + } + ] +} +``` + +### User invitation response + +```json +{ + "name": "userInvitations/dry-run-inv-001", + "invitedEmail": "", + "inviter": "Dry Run Admin", + "expirationTime": "2026-06-17T10:00:00Z", + "invitationUri": "https://confidence.spotify.com/invite/mock-token", + "invitationToken": "mock-invitation-token" +} +``` + +### Current user response + +```json +{ + "user": { + "name": "users/dry-run-user", + "fullName": "Dry Run User", + "email": "dryrun@example.com" + }, + "accountMemberships": [ + { + "account": "accounts/dry-run-demo", + "displayName": "Dry Run Demo", + "loginId": "dry-run-demo", + "region": "EU" + } + ], + "account": "accounts/dry-run-demo", + "identity": { + "name": "identities/udryrun123", + "displayName": "Dry Run User" + } +} +``` + +### Validate warehouse (BigQuery) + +```json +{ + "validation": [ + {"key": "SERVICE_ACCOUNT", "description": "Service account access", "success": true}, + {"key": "PERMISSIONS", "description": "BigQuery permissions", "success": true}, + {"key": "DATASET", "description": "Dataset access", "success": true} + ], + "successful": true +} +``` + +### Validate warehouse (Snowflake) + +```json +{ + "validation": [ + {"key": "AUTHENTICATION", "description": "Key-pair authentication", "success": true}, + {"key": "ROLE", "description": "Role access", "success": true}, + {"key": "WAREHOUSE", "description": "Warehouse access", "success": true}, + {"key": "DATABASE", "description": "Database access", "success": true}, + {"key": "SCHEMA", "description": "Schema access", "success": true} + ], + "successful": true +} +``` + +### Validate warehouse (Redshift) + +```json +{ + "validation": [ + {"key": "CLUSTER", "description": "Cluster connectivity", "success": true}, + {"key": "IAM_ROLE", "description": "IAM role assumption", "success": true}, + {"key": "SCHEMA", "description": "Schema access", "success": true} + ], + "successful": true +} +``` + +### Create warehouse + +```json +{"name": "dataWarehouses/dry-run-wh-123"} +``` + +### Create crypto key (Snowflake) + +```json +{ + "name": "cryptoKeys/snowflake-key", + "kind": "SNOWFLAKE", + "publicKey": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0mock1234567890abcd\nefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ01234567\n89mockkeydatafordryruntestingpurposes0123456789abcdefghijklmnopqrst\nuvwxyz==\n-----END PUBLIC KEY-----" +} +``` + +### Create flag applied connection + +```json +{"name": "flagAppliedConnections/dry-run-connector", "state": "STATE_RUNNING"} +``` + +### Create event connection + +```json +{"name": "eventConnections/dry-run-events", "state": "STATE_RUNNING"} +``` + +### Create assignment table + +```json +{"name": "assignmentTables/dry-run-assignments", "displayName": "Assignments"} +``` + +### Verify pipeline — assignments + +``` +targeting_key | rule | assignment_id | assignment_time +dry-run-user | flags/my-test-flag/rules/rule1 | on | 2026-06-10T12:00:00Z +``` + +### Verify pipeline — events + +``` +_event_time | user_action | page +2026-06-10T12:00:00Z | clicked_button | homepage +``` + +### Publish events response + +```json +{"errors": []} +``` + +### Learning progress response + +```json +{ + "courseProgresses": [ + {"course": "courses/STATS", "completedLessons": 0, "totalLessons": 5}, + {"course": "courses/DESIGN", "completedLessons": 0, "totalLessons": 5}, + {"course": "courses/FLAGS", "completedLessons": 0, "totalLessons": 5}, + {"course": "courses/METRICS", "completedLessons": 0, "totalLessons": 5}, + {"course": "courses/COORDINATION", "completedLessons": 0, "totalLessons": 5} + ], + "completedCourses": 0 +} +``` + +--- + +## Dry Run Summary + +At the end of every sub-command dry run, display this banner: + +``` +═══════════════════════════════════════════════════════════════ + [DRY RUN] Complete +═══════════════════════════════════════════════════════════════ + + This was a simulated run. No real resources were created. + + To run for real: + • /onboard-confidence + • /onboard-confidence:setup-warehouse- + +═══════════════════════════════════════════════════════════════ +``` + +--- + +## Sub-command: create-account + +### Step Tracker + +Display at START and after EACH step completes (updating status). Prefix the title with `[DRY RUN]`. + +``` +───── [DRY RUN] Create Account ─────────────────────────────── + [1] Log in ○ pending + [2] Workspace name ○ pending + [3] Account details ○ pending + [4] Create account ○ pending + [5] Connect tools ○ pending + [6] Done ○ pending +────────────────────────────────────────────────────────────── +``` + +Use `●` for completed, `▶` for in-progress, `○` for pending. + +### Step 1: Log in + +**Skip browser login entirely.** Display: + +> [DRY RUN] Skipping browser login — using mock credentials. +> [DRY RUN] Authenticated as dryrun@example.com + +Mark step 1 as `●`. + +### Step 2: Workspace name + +Same UX as the real skill. EDUCATE then ASK: + +> Your workspace name is the unique identifier for your Confidence account. +> It appears in URLs and is used to log in. +> +> **Rules:** 3-21 characters, lowercase letters, digits, and hyphens. Must start with a letter and end with a letter or digit. + +**Wait for user input.** Validate locally against regex `^[a-z][a-z0-9-]{1,19}[a-z0-9]$`. If invalid, explain and re-ask — exactly like the real skill. + +Then simulate the availability check: + +> [DRY RUN] Checking availability... `` is available! + +If the user enters `taken-name`, simulate: + +> [DRY RUN] Checking availability... `taken-name` is already taken. Try another name. + +### Step 3: Account details + +Same UX as the real skill. Collect interactively, one field at a time: + +1. **Display name** — ask, validate (3-21 chars, starts with letter). + +2. **Region** — present as a choice: + > Where should your data be stored? This **cannot be changed later**. + > 1. EU (Europe) + > 2. US (United States) + +3. **Authentication method** — present as a choice: + > How should users log in to your workspace? + > 1. Google + > 2. Email + password + > 3. Both + +4. **Admin email** — ask. Validate work email. If free email (gmail, yahoo, etc.), reject: + > Confidence requires a work email address. Free providers like Gmail aren't allowed. + +5. **Allowed login email domains** — optional. Ask if they want to restrict. + +### Step 4: Create account + +Display what would happen: + +> [DRY RUN] Would call `POST https://onboarding.confidence.dev/v1/accounts` +> [DRY RUN] Creating workspace ****... + +Then show mock success: + +> [DRY RUN] Your workspace **** has been created! +> Workspace ID: `` +> Region: +> +> You can access it at: https://confidence.spotify.com + +### Step 5: Connect tools + +> [DRY RUN] Would re-authenticate with org-scoped token (browser auto-redirect). +> [DRY RUN] Skipping — mock token already has org scope. + +Then suggest MCP: + +> To connect Confidence tools for flag management, type `/mcp` and authenticate **confidence-flags**. + +### Step 6: Done + +Show the same summary as the real skill, but with `[DRY RUN]` in the banner: + +``` +═══════════════════════════════════════════════════════════════ + [DRY RUN] Welcome to Confidence! +═══════════════════════════════════════════════════════════════ + + Workspace: () + Region: + Admin: + URL: https://confidence.spotify.com + + Next steps: + • Invite team members: /onboard-confidence invite-user + • Create a feature flag: Ask me to create a flag, or use + the Confidence UI + • Integrate your app: Ask me for SDK setup instructions + +═══════════════════════════════════════════════════════════════ +``` + +Then show the Dry Run Summary banner. + +--- + +## Sub-command: invite-user + +### Step Tracker + +``` +───── [DRY RUN] Invite User ────────────────────────────────── + [1] Authenticate ○ pending + [2] Target account ○ pending + [3] Invitation details ○ pending + [4] Send invitation ○ pending +────────────────────────────────────────────────────────────── +``` + +### Step 1: Authenticate + +> [DRY RUN] Skipping browser login — using mock credentials. +> [DRY RUN] Authenticated as dryrun@example.com + +> [DRY RUN] Would call `GET https://iam.eu.confidence.dev/v1/currentUser` +> [DRY RUN] Current user: Dry Run User (dryrun@example.com) + +### Step 2: Target account + +> [DRY RUN] Account: **Dry Run Demo** (dry-run-demo) + +### Step 3: Invitation details + +Same UX as the real skill. Ask for: + +1. **Email address(es)** — required. Accept single or comma-separated. Validate format locally. +2. **Send invitation email?** — default yes. + +### Step 4: Send invitation + +For each email: + +> [DRY RUN] Would call `POST https://iam.eu.confidence.dev/v1/userInvitations` + +For single invite: +> [DRY RUN] Invitation sent to ****! +> They'll receive an email with instructions to join. +> The invitation expires on Jun 17, 2026. + +For batch invites, show a summary table: +``` +[DRY RUN] Invitations sent: + ✓ alice@example.com — expires Jun 17 + ✓ bob@example.com — expires Jun 17 +``` + +If any email fails local validation: +``` + ✗ charlie@invalid — invalid email address +``` + +Then show the Dry Run Summary banner. + +--- + +## Sub-command: create-client + +### Step Tracker + +``` +───── [DRY RUN] Create Client ──────────────────────────────── + [1] Client name ○ pending + [2] Create client ○ pending + [3] Get credentials ○ pending +────────────────────────────────────────────────────────────── +``` + +### Step 1: Client name + +Same UX as the real skill: + +> What should we call this client? (e.g., "iOS App", "Web Frontend", "Backend Service") + +Wait for user input. + +### Step 2: Create client + +> [DRY RUN] Would call `POST https://iam.eu.confidence.dev/v1/clients` +> [DRY RUN] Client **** created. + +### Step 3: Get credentials + +> [DRY RUN] Would call `POST https://iam.eu.confidence.dev/v1/clients/dry-run-client/credentials` + +``` +═══════════════════════════════════════════════════════════════ + [DRY RUN] Client Created +═══════════════════════════════════════════════════════════════ + + Name: + Secret: dryrn_sk_mock1234567890abcdef + + Use this secret in your SDK configuration to resolve flags. + Keep it safe — you can regenerate it, but the old one will + stop working. + + Next: Ask me for SDK integration instructions, or run + /onboard-confidence setup-wizard + +═══════════════════════════════════════════════════════════════ +``` + +Then show the Dry Run Summary banner. + +--- + +## Sub-command: setup-wizard + +### Step Tracker + +``` +───── [DRY RUN] Setup Wizard ───────────────────────────────── + [1] Create client ○ pending + [2] Create flag ○ pending + [3] Add variants ○ pending + [4] Add targeting ○ pending + [5] Test resolve ○ pending + [6] Done ○ pending +────────────────────────────────────────────────────────────── +``` + +### Prerequisites + +> [DRY RUN] Skipping browser login — using mock credentials. +> [DRY RUN] Region: EU (from mock token) + +### Step 1: Create client + +> [DRY RUN] Would call `GET https://iam.eu.confidence.dev/v1/clients` +> [DRY RUN] No existing clients found. Creating one now. + +Ask user for client name (same UX as real skill). + +> [DRY RUN] Would call `POST https://iam.eu.confidence.dev/v1/clients` +> [DRY RUN] Client **** created. +> [DRY RUN] Would call `POST https://iam.eu.confidence.dev/v1/clients/dry-run-client/credentials` +> [DRY RUN] Client secret generated. + +### Step 2: Create flag + +Same EDUCATE then ASK flow: + +> A feature flag controls a piece of functionality. Let's create your first one. +> What should it be called? (e.g., "new-checkout-flow", "dark-mode") + +Wait for user input. Validate: 4-63 chars, `[a-z0-9-]`. + +> [DRY RUN] Would call `POST https://flags.eu.confidence.dev/v1/flags?flag_id=` +> [DRY RUN] Flag **** created. + +### Step 3: Add variants + +Same EDUCATE flow: + +> Variants are the different values a flag can have. For a simple on/off flag, you'd have "on" and "off" variants. +> +> What variants should this flag have? +> 1. Simple on/off (boolean) +> 2. Custom variants (I'll name them) + +Wait for user input. + +> [DRY RUN] Would call `PATCH https://flags.eu.confidence.dev/v1/flags/` (set schema) +> [DRY RUN] Schema set. + +For each variant: +> [DRY RUN] Would call `POST https://flags.eu.confidence.dev/v1/flags//variants` +> [DRY RUN] Variant **** created with value ``. + +After all variants: +> [DRY RUN] Would call `POST https://flags.eu.confidence.dev/v1/flags/:addFlagClient` +> [DRY RUN] Flag attached to client. + +### Step 4: Add targeting + +Same EDUCATE flow: + +> Targeting rules control who sees which variant. Let's set a default — you can add more rules later. +> Which variant should be the default? + +Wait for user input. + +> [DRY RUN] Would call `POST https://flags.eu.confidence.dev/v1/segments?segment_id=everyone` +> [DRY RUN] Segment "Everyone" created. +> [DRY RUN] Would call `PATCH https://flags.eu.confidence.dev/v1/segments/everyone` (set allocation) +> [DRY RUN] Would call `POST https://flags.eu.confidence.dev/v1/segments/everyone:allocate` +> [DRY RUN] Segment allocated at 100%. +> [DRY RUN] Would call `POST https://flags.eu.confidence.dev/v1/flags//rules` +> [DRY RUN] Rule created — all users get variant ****. + +### Step 5: Test resolve + +> Let's verify the flag works by resolving it. + +> [DRY RUN] Would call `POST https://resolver.eu.confidence.dev/v1/flags:resolve` +> [DRY RUN] Flag **** resolved to variant **** — it works! + +Show mock resolve response: +``` +[DRY RUN] Mock resolve result: + Flag: + Variant: + Value: {"enabled": true} + Reason: RESOLVE_REASON_MATCH +``` + +### Step 6: Done + +``` +═══════════════════════════════════════════════════════════════ + [DRY RUN] Setup Complete! +═══════════════════════════════════════════════════════════════ + + Client: + Secret: dryrn_sk_mock1234567890abcdef + Flag: + Variants: + Default: + + Your flag is live and resolving. Next steps: + • Integrate the SDK: Ask me for setup instructions + • Create more flags: Ask me or use the Confidence UI + • Set up experiments: /onboard-confidence learn + +═══════════════════════════════════════════════════════════════ +``` + +Then show the Dry Run Summary banner. + +--- + +## Sub-command: setup-warehouse + +### Flow + +Show the 4 warehouse options (same as real skill): + +> Which data warehouse do you use? +> 1. BigQuery +> 2. Snowflake +> 3. Databricks +> 4. Redshift + +After the user picks, run the corresponding warehouse-specific dry run below. + +--- + +## Sub-command: setup-warehouse-bigquery + +### Step Tracker + +``` +───── [DRY RUN] Setup Warehouse (BigQuery) ─────────────────── + [1] Choose warehouse ● done + [2] GCP project ID ○ pending + [3] Dataset name ○ pending + [4] Service account ○ pending + [5] Validate & fix ○ pending + [6] Create warehouse ○ pending + [7] Create connectors ○ pending + [8] Assignment table ○ pending + [9] Verify pipeline ○ pending + [10] Done ○ pending +────────────────────────────────────────────────────────────── +``` + +### Step 1: Choose warehouse (already done) + +Mark as `●`. + +### Step 2: GCP Project ID + +Same UX as real skill: + +> What's your GCP project ID? Go to **Google Cloud Console** (console.cloud.google.com). Your project ID is shown in the top bar next to "Google Cloud". It looks like `my-company-prod` or `project-12345`. + +Wait for user input. + +### Step 3: Dataset name + +Same UX: + +> A dataset is like a folder in BigQuery where Confidence stores its tables. The default is `confidence`. +> If you don't have one yet, I can create it for you via `bq mk`. + +Wait for user input (or accept default). + +### Step 4: Service account + +Same UX: + +> A service account is a robot account that Confidence uses to write data to your BigQuery dataset. +> Go to **Google Cloud Console -> IAM & Admin -> Service Accounts**. Create one (e.g., `confidence-connector@.iam.gserviceaccount.com`) or pick an existing one. +> It needs **BigQuery Data Editor** and **BigQuery Job User** roles. + +Wait for user input. + +### Step 5: Validate & fix + +> [DRY RUN] Would call `POST https://metrics.eu.confidence.dev/v1/dataWarehouseConfig:validate` +> [DRY RUN] Validation passed! All checks succeeded: +> - SERVICE_ACCOUNT: Service account access ✓ +> - PERMISSIONS: BigQuery permissions ✓ +> - DATASET: Dataset access ✓ + +Then show what gcloud commands would have been run if validation had failed: + +> [DRY RUN] If validation had failed, these commands would fix it: +> ``` +> # Grant Confidence access to your service account +> gcloud iam service-accounts add-iam-policy-binding \ +> --project= \ +> --member="serviceAccount:account-dry-run-demo@spotify-confidence.iam.gserviceaccount.com" \ +> --role="roles/iam.workloadIdentityUser" +> +> # Grant BigQuery Job User +> gcloud projects add-iam-policy-binding \ +> --member="serviceAccount:" \ +> --role="roles/bigquery.jobUser" +> ``` + +### Step 6: Create warehouse + +> [DRY RUN] Would call `POST https://metrics.eu.confidence.dev/v1/dataWarehouses` +> [DRY RUN] Warehouse created: `dataWarehouses/dry-run-wh-123` + +### Step 7: Create connectors + +> [DRY RUN] Would call `POST https://connectors.eu.confidence.dev/v1/flagAppliedConnections` +> [DRY RUN] Flag assignment connector created: `flagAppliedConnections/dry-run-connector` (STATE_RUNNING) +> +> [DRY RUN] Would call `POST https://connectors.eu.confidence.dev/v1/eventConnections` +> [DRY RUN] Event connector created: `eventConnections/dry-run-events` (STATE_RUNNING) + +### Step 8: Assignment table + +> [DRY RUN] Would call `POST https://metrics.eu.confidence.dev/v1/assignmentTables` +> [DRY RUN] Assignment table created: `assignmentTables/dry-run-assignments` + +Show the SQL that would be used: +```sql +SELECT targeting_key, rule, assignment_id, assignment_time +FROM `..assignments` +``` + +### Step 9: Verify pipeline + +> [DRY RUN] Would resolve a flag and publish test events to verify data flow. + +Show mock pipeline results: + +``` +[DRY RUN] Pipeline verification: + ● Assignments: 1 row — data flowing + dry-run-user -> on (2026-06-10T12:00:00Z) + ● Events: 1 row — data flowing + clicked_button on homepage (2026-06-10T12:00:00Z) +``` + +### Step 10: Done + +``` +═══════════════════════════════════════════════════════════════ + [DRY RUN] Data Warehouse Connected & Verified +═══════════════════════════════════════════════════════════════ + + Warehouse: BigQuery () + Dataset: + Connectors: + ● Flag assignments -> assignments table (verified) + ● Events -> events_* tables (running) + Assignment: + ● Assignment table configured (auto-updating) + + Flag assignment and event data is flowing to your + warehouse. Experiment analysis is ready. + +═══════════════════════════════════════════════════════════════ +``` + +Then show the Dry Run Summary banner. + +--- + +## Sub-command: setup-warehouse-snowflake + +### Step Tracker + +``` +───── [DRY RUN] Setup Warehouse (Snowflake) ────────────────── + [1] Choose warehouse ● done + [2] Account & user ○ pending + [3] Role & warehouse ○ pending + [4] Database & schema ○ pending + [5] Create crypto key ○ pending + [6] Register key in SF ○ pending + [7] Validate ○ pending + [8] Create warehouse ○ pending + [9] Create connectors ○ pending + [10] Assignment table ○ pending + [11] Verify pipeline ○ pending + [12] Done ○ pending +────────────────────────────────────────────────────────────── +``` + +### Step 2: Account & user + +Same UX as real skill. Ask for: +- **Account** — Snowflake account identifier (e.g., `zlvpqre-wr49874`) +- **User** — Snowflake user for Confidence to connect as + +### Step 3: Role & warehouse + +- **Role** — default `ACCOUNTADMIN` +- **Warehouse** — default `COMPUTE_WH` + +### Step 4: Database & schema + +- **Exposure database** — default `CONFIDENCE` +- **Exposure schema** — default `EXPOSURE` + +Show the SQL that would be needed if the database/schema don't exist: +```sql +CREATE DATABASE IF NOT EXISTS ; +CREATE SCHEMA IF NOT EXISTS .; +GRANT USAGE ON DATABASE TO ROLE ; +GRANT ALL ON SCHEMA . TO ROLE ; +``` + +### Step 5: Create crypto key + +> [DRY RUN] Would call `POST https://iam.eu.confidence.dev/v1/cryptoKeys?crypto_key_id=snowflake-key` +> [DRY RUN] Crypto key created: `cryptoKeys/snowflake-key` +> [DRY RUN] Public key generated (mock RSA 2048-bit) + +### Step 6: Register key in Snowflake + +Show the ALTER USER SQL that would be generated: + +> [DRY RUN] In the real flow, this SQL would be copied to your clipboard: +> ```sql +> ALTER USER SET RSA_PUBLIC_KEY='MIIBIjANBgkqhkiG9w0BAQE...mockkey...'; +> ``` + +Ask: "Does another Confidence account share this Snowflake user?" (same as real skill). If yes, show `RSA_PUBLIC_KEY_2` variant. + +### Step 7: Validate + +> [DRY RUN] Would call `POST https://metrics.eu.confidence.dev/v1/dataWarehouseConfig:validate` +> [DRY RUN] Validation passed! All checks succeeded: +> - AUTHENTICATION: Key-pair authentication ✓ +> - ROLE: Role access ✓ +> - WAREHOUSE: Warehouse access ✓ +> - DATABASE: Database access ✓ +> - SCHEMA: Schema access ✓ + +### Step 8: Create warehouse + +> [DRY RUN] Would call `POST https://metrics.eu.confidence.dev/v1/dataWarehouses` +> [DRY RUN] Warehouse created: `dataWarehouses/dry-run-wh-123` + +### Step 9: Create connectors + +> [DRY RUN] Would call `POST https://connectors.eu.confidence.dev/v1/flagAppliedConnections` +> [DRY RUN] Flag assignment connector created (Snowflake -> ..ASSIGNMENTS) +> +> [DRY RUN] Would call `POST https://connectors.eu.confidence.dev/v1/eventConnections` +> [DRY RUN] Event connector created (Snowflake -> ..EVENTS_*) + +### Step 10: Assignment table + +> [DRY RUN] Would call `POST https://metrics.eu.confidence.dev/v1/assignmentTables` +> [DRY RUN] Assignment table created. + +Show the SQL: +```sql +SELECT targeting_key, rule, assignment_id, assignment_time +FROM ..ASSIGNMENTS +``` + +### Step 11: Verify pipeline + +Show mock pipeline results: + +``` +[DRY RUN] Pipeline verification: + ● Assignments: 1 row — data flowing + dry-run-user -> on (2026-06-10T12:00:00Z) + ● Events: 1 row — data flowing + clicked_button on homepage (2026-06-10T12:00:00Z) +``` + +### Step 12: Done + +``` +═══════════════════════════════════════════════════════════════ + [DRY RUN] Data Warehouse Connected & Verified +═══════════════════════════════════════════════════════════════ + + Warehouse: Snowflake () + Database: + Schema: + Connectors: + ● Flag assignments -> ASSIGNMENTS table (verified) + ● Events -> EVENTS_* tables (running) + Assignment: + ● Assignment table configured (auto-updating) + + Flag assignment and event data is flowing to your + warehouse. Experiment analysis is ready. + +═══════════════════════════════════════════════════════════════ +``` + +Then show the Dry Run Summary banner. + +--- + +## Sub-command: setup-warehouse-databricks + +### Step Tracker + +``` +───── [DRY RUN] Setup Warehouse (Databricks) ───────────────── + [1] Choose warehouse ● done + [2] Workspace URL ○ pending + [3] SQL Warehouse ID ○ pending + [4] Service principal ○ pending + [5] AWS account & CLI ○ pending + [6] S3 bucket ○ pending + [7] IAM role ○ pending + [8] Databricks schema ○ pending + [9] Create warehouse ○ pending + [10] Create connectors ○ pending + [11] Assignment table ○ pending + [12] Verify pipeline ○ pending + [13] Done ○ pending +────────────────────────────────────────────────────────────── +``` + +### Overview + +Same overview as real skill: + +> Setting up Databricks with Confidence requires three things: +> +> 1. **A Databricks workspace** — you need admin access to create a service principal (a robot account) +> 2. **An AWS account with an S3 bucket** — Confidence needs this as a staging area for loading data into Databricks. This is required even if your Databricks runs on GCP or Azure +> 3. **A schema in Databricks** — a place for Confidence to create tables (e.g., `confidence`) +> +> **How data flows:** +> Confidence collects your flag assignments and events internally, then writes parquet files to an S3 bucket you provide, and finally loads them into Databricks tables. This happens in batches every ~5 minutes. + +### Step 2: Workspace URL + +Same UX: ask for URL, extract hostname, confirm. + +### Step 3: SQL Warehouse ID + +Same UX: explain how to find it, ask for the ID. + +### Step 4: Service principal + +Same UX: explain how to create one, ask for Client ID and Secret. + +For dry run, accept any values. Display: +> [DRY RUN] Service principal configured (mock credentials accepted). + +### Step 5: AWS account & CLI + +Same choice: +> Do you have the `aws` CLI set up, or would you prefer manual steps? +> 1. Set it up for me (requires `aws` CLI) +> 2. Show me the steps + +> [DRY RUN] Skipping AWS CLI check — mock mode. + +### Step 6: S3 bucket + +Ask for bucket name (suggest `confidence-staging-dry-run-demo`) and region. + +> [DRY RUN] Would run: `aws s3api create-bucket --bucket --region ` +> [DRY RUN] S3 bucket `` created in ``. + +### Step 7: IAM role + +Show the trust policy that would be created: + +> [DRY RUN] Would create IAM role with this trust policy: +> ```json +> { +> "Version": "2012-10-17", +> "Statement": [{ +> "Effect": "Allow", +> "Principal": {"Federated": "accounts.google.com"}, +> "Action": "sts:AssumeRoleWithWebIdentity", +> "Condition": { +> "StringEquals": { +> "accounts.google.com:sub": "123456789012345678901" +> } +> } +> }] +> } +> ``` +> +> [DRY RUN] Would create S3 access policy scoped to ``. +> [DRY RUN] IAM role created: `arn:aws:iam::123456789012:role/confidence-databricks-staging` + +### Step 8: Databricks schema + +Same UX: ask for schema name (default `confidence`). + +Show the SQL that would need to be run: + +> [DRY RUN] In the real flow, this SQL would be copied to your clipboard: +> ```sql +> CREATE SCHEMA IF NOT EXISTS confidence; +> GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence TO ``; +> ``` + +### Step 9: Create warehouse + +> [DRY RUN] Note: Pre-validation is not available for Databricks. +> [DRY RUN] Would call `POST https://metrics.eu.confidence.dev/v1/dataWarehouses` +> [DRY RUN] Warehouse created: `dataWarehouses/dry-run-wh-123` + +### Step 10: Create connectors + +> [DRY RUN] Would call `POST https://connectors.eu.confidence.dev/v1/flagAppliedConnections` +> [DRY RUN] Flag assignment connector created (Databricks -> .assignments, S3 staging: ) +> +> [DRY RUN] Would call `POST https://connectors.eu.confidence.dev/v1/eventConnections` +> [DRY RUN] Event connector created (Databricks -> .events_*, S3 staging: ) + +### Step 11: Assignment table + +> [DRY RUN] Would call `POST https://metrics.eu.confidence.dev/v1/assignmentTables` +> [DRY RUN] Assignment table created. + +Show the SQL: +```sql +SELECT targeting_key, rule, assignment_id, assignment_time +FROM .assignments +``` + +### Step 12: Verify pipeline + +``` +[DRY RUN] Pipeline verification: + ● Assignments: 1 row — data flowing + dry-run-user -> on (2026-06-10T12:00:00Z) + ● Events: 1 row — data flowing + clicked_button on homepage (2026-06-10T12:00:00Z) +``` + +### Step 13: Done + +``` +═══════════════════════════════════════════════════════════════ + [DRY RUN] Data Warehouse Connected & Verified +═══════════════════════════════════════════════════════════════ + + Warehouse: Databricks () + Schema: + S3 Bucket: () + Connectors: + ● Flag assignments -> assignments table (verified) + ● Events -> events_* tables (running) + Assignment: + ● Assignment table configured (auto-updating) + + Flag assignment and event data is flowing to your + warehouse. Experiment analysis is ready. + + Note: Data is delivered in ~5 minute batches. + +═══════════════════════════════════════════════════════════════ +``` + +Then show the Dry Run Summary banner. + +--- + +## Sub-command: setup-warehouse-redshift + +### Step Tracker + +``` +───── [DRY RUN] Setup Warehouse (Redshift) ─────────────────── + [1] Choose warehouse ● done + [2] AWS account & CLI ○ pending + [3] Redshift cluster ○ pending + [4] S3 bucket ○ pending + [5] IAM role ○ pending + [6] Attach role ○ pending + [7] Schema & grants ○ pending + [8] Validate ○ pending + [9] Create warehouse ○ pending + [10] Create connectors ○ pending + [11] Assignment table ○ pending + [12] Verify pipeline ○ pending + [13] Done ○ pending +────────────────────────────────────────────────────────────── +``` + +### Overview + +Same overview as real skill: + +> Setting up Redshift with Confidence requires an **AWS account**. Here's what we'll set up: +> +> 1. **A Redshift cluster** — a data warehouse that stores your experiment data +> 2. **An S3 bucket** — a staging area where Confidence drops data files before loading them into Redshift +> 3. **An IAM role** — permissions that let Confidence write to S3 and load into Redshift +> 4. **A schema** — a folder inside Redshift where Confidence creates its tables +> +> **Important: Redshift Serverless won't work** — Confidence needs a provisioned cluster. + +### Step 2: AWS account & CLI + +Same choice as real skill. In dry run: + +> [DRY RUN] Skipping AWS CLI check — mock mode. + +### Step 3: Redshift cluster + +Ask if they have one or want to create one. Same UX as real skill. + +If creating: +> [DRY RUN] Would run: +> ``` +> aws redshift create-cluster \ +> --cluster-identifier confidence-redshift-dry-run-demo \ +> --cluster-type single-node \ +> --node-type ra3.large \ +> --master-username admin \ +> --master-user-password \ +> --db-name dev \ +> --region eu-west-1 \ +> --publicly-accessible +> ``` +> [DRY RUN] Cluster `confidence-redshift-dry-run-demo` is running. + +If using existing, ask for cluster name. + +### Step 4: S3 bucket + +Ask for bucket name and region (same UX). + +> [DRY RUN] Would run: `aws s3api create-bucket --bucket --region ` +> [DRY RUN] S3 bucket `` created in ``. + +### Step 5: IAM role + +Show the dual trust policy that would be created: + +> [DRY RUN] Would create IAM role with dual trust policy: +> ```json +> { +> "Version": "2012-10-17", +> "Statement": [ +> { +> "Effect": "Allow", +> "Principal": {"Federated": "accounts.google.com"}, +> "Action": "sts:AssumeRoleWithWebIdentity", +> "Condition": { +> "StringEquals": { +> "accounts.google.com:sub": "123456789012345678901" +> } +> } +> }, +> { +> "Effect": "Allow", +> "Principal": {"Service": "redshift.amazonaws.com"}, +> "Action": "sts:AssumeRole" +> } +> ] +> } +> ``` +> +> [DRY RUN] Would create S3 access policy scoped to ``. +> [DRY RUN] Would create Redshift Data API policy. +> [DRY RUN] IAM role created: `arn:aws:iam::123456789012:role/confidence-redshift` + +### Step 6: Attach role + +> [DRY RUN] Would run: +> ``` +> aws redshift modify-cluster-iam-roles \ +> --cluster-identifier \ +> --add-iam-roles arn:aws:iam::123456789012:role/confidence-redshift +> ``` +> [DRY RUN] IAM role attached to cluster. + +### Step 7: Schema & grants + +Ask for schema name (default `confidence`). + +Show the GRANT statements: + +> [DRY RUN] In the real flow, these would be run against Redshift: +> ```sql +> CREATE SCHEMA IF NOT EXISTS ; +> GRANT USAGE ON SCHEMA TO PUBLIC; +> GRANT CREATE ON SCHEMA TO PUBLIC; +> ``` + +### Step 8: Validate + +> [DRY RUN] Would call `POST https://metrics.eu.confidence.dev/v1/dataWarehouseConfig:validate` +> [DRY RUN] Validation passed! All checks succeeded: +> - CLUSTER: Cluster connectivity ✓ +> - IAM_ROLE: IAM role assumption ✓ +> - SCHEMA: Schema access ✓ + +### Step 9: Create warehouse + +> [DRY RUN] Would call `POST https://metrics.eu.confidence.dev/v1/dataWarehouses` +> [DRY RUN] Warehouse created: `dataWarehouses/dry-run-wh-123` + +### Step 10: Create connectors + +> [DRY RUN] Would call `POST https://connectors.eu.confidence.dev/v1/flagAppliedConnections` +> [DRY RUN] Flag assignment connector created (Redshift -> .assignments, S3 staging: ) +> +> [DRY RUN] Would call `POST https://connectors.eu.confidence.dev/v1/eventConnections` +> [DRY RUN] Event connector created (Redshift -> .events_*, S3 staging: ) + +### Step 11: Assignment table + +> [DRY RUN] Would call `POST https://metrics.eu.confidence.dev/v1/assignmentTables` +> [DRY RUN] Assignment table created. + +Show the SQL: +```sql +SELECT targeting_key, rule, assignment_id, assignment_time +FROM .assignments +``` + +### Step 12: Verify pipeline + +``` +[DRY RUN] Pipeline verification: + ● Assignments: 1 row — data flowing + dry-run-user -> on (2026-06-10T12:00:00Z) + ● Events: 1 row — data flowing + clicked_button on homepage (2026-06-10T12:00:00Z) +``` + +### Step 13: Done + +``` +═══════════════════════════════════════════════════════════════ + [DRY RUN] Data Warehouse Connected & Verified +═══════════════════════════════════════════════════════════════ + + Warehouse: Redshift () + Database: + Schema: + S3 Bucket: () + Connectors: + ● Flag assignments -> assignments table (verified) + ● Events -> events_* tables (running) + Assignment: + ● Assignment table configured (auto-updating) + + Flag assignment and event data is flowing to your + warehouse. Experiment analysis is ready. + +═══════════════════════════════════════════════════════════════ +``` + +Then show the Dry Run Summary banner. + +--- + +## User-Facing Communication Rules + +Follow the same rules as the real onboarding skill: + +- **NEVER expose internal technical details** — but since this is a dry run, you DO show the mock API endpoints being "called" and mock response data. This is the point of the dry run. +- **DO show `[DRY RUN]` prefix** on every simulated action +- **DO show human-readable status updates** alongside the mock data +- **Step Tracker:** Display the step tracker at every phase transition, with `[DRY RUN]` in the title. Update it after each step completes. +- **Be conversational** — same tone as the real skill +- **Ask for real input** — workspace names, flag names, warehouse config values, etc. The user should experience the full interaction flow. + +--- + +## Important: What NOT to do + +- **Do NOT make any real HTTP calls** — no `curl`, no `open`, no `python3` auth scripts +- **Do NOT write files to disk** — no `$TMPDIR/confidence_auth.py`, no `~/.confidence/.auth_token` +- **Do NOT require `dangerouslyDisableSandbox: true`** — there are no external network calls +- **Do NOT use any MCP tools** — everything is simulated +- **Do NOT skip user input steps** — the entire point is to test the interaction flow diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index 52a583d..d670b50 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -125,22 +125,14 @@ except Exception as e: - If port 8084 is busy: `lsof -ti:8084 | xargs kill -9 2>/dev/null` - All network commands require `dangerouslyDisableSandbox: true` -### Token persistence +### Session-only token management -After a successful login, **save the token to disk** so it survives across sessions: +The token is kept in the current session only and is never saved to disk. If the session ends or the token expires, the skill will open your browser to log in again. -```bash -mkdir -p ~/.confidence -echo "$TOKEN" > ~/.confidence/.auth_token -chmod 600 ~/.confidence/.auth_token -``` - -**On every sub-command start**, check for a saved token before prompting login: +**On every sub-command start**, check if the `TOKEN` variable is set and not expired: ```bash -if [ -f ~/.confidence/.auth_token ]; then - TOKEN=$(cat ~/.confidence/.auth_token) - # Decode JWT exp claim (handle base64 padding) +if [ -n "$TOKEN" ]; then PAYLOAD=$(echo "$TOKEN" | cut -d. -f2) EXP=$(echo "$PAYLOAD" | python3 -c " import sys, json, base64 @@ -154,14 +146,17 @@ print(d.get('exp', 0)) echo "VALID" # Token still good — skip login else echo "EXPIRED" # Token expired — re-authenticate - rm ~/.confidence/.auth_token + unset TOKEN fi fi ``` +If `TOKEN` is unset or expired, run the browser auth flow to get a new token. Store the result in the `TOKEN` shell variable only. **NEVER write the token to disk. NEVER reference `~/.confidence/`.** + **Extract region from token** to determine API base URLs: ```bash +PAYLOAD=$(echo "$TOKEN" | cut -d. -f2) REGION=$(echo "$PAYLOAD" | python3 -c " import sys, json, base64 p = sys.stdin.read().strip() @@ -337,7 +332,7 @@ Run the auth script again with: This auto-completes in the browser — no login form, just a redirect. The new token will have `org_id`, `account_name`, and `region` claims. -Save this token to `~/.confidence/.auth_token`. This is the token used for all subsequent commands. +Store this token in the `TOKEN` shell variable. This is the token used for all subsequent commands in this session. **Do NOT save to disk.** Tell the user: > Activating your account... (browser will briefly flash) @@ -538,7 +533,7 @@ Guided walkthrough of the full onboarding checklist. Uses REST APIs — no MCP n ### Prerequisites -Requires an authenticated token. If none saved, run login flow first. +Requires an authenticated token. If none available in the current session, run login flow first. Determine the region from the token or ask the user — this sets the API base URLs: - EU: `flags.eu.confidence.dev`, `resolver.eu.confidence.dev`, `iam.eu.confidence.dev` @@ -728,1113 +723,11 @@ If resolve fails, check that the flag is attached to the client and has at least ## Sub-command: setup-warehouse -Configure a data warehouse, event connectors, and assignment tables for experimentation analytics. Uses REST APIs only. - -### Prerequisites - -Requires an authenticated token. If none saved, run the Auth0 login flow first. - -**API Base URLs** (region-specific): -- Metrics: `https://metrics.confidence.dev/v1` (or `metrics.eu.` / `metrics.us.`) -- Connectors: `https://connectors.confidence.dev/v1` (or `connectors.eu.` / `connectors.us.`) - -### Step Tracker - -Show the initial tracker at start. After the user picks a warehouse type in Step 1, **replace it** with the warehouse-specific tracker that shows the actual sub-steps. Update and re-display after each step. - -**Initial (before warehouse choice):** -``` -───── Setup Warehouse ───────────────────────────────────── - [1] Choose warehouse ▶ in-progress -──────────────────────────────────────────────────────────── -``` - -**BigQuery tracker (after choosing BigQuery):** -``` -───── Setup Warehouse (BigQuery) ────────────────────────── - [1] Choose warehouse ● done - [2] GCP project ID ○ pending - [3] Dataset name ○ pending - [4] Service account ○ pending - [5] Validate & fix ○ pending - [6] Create warehouse ○ pending - [7] Create connectors ○ pending - [8] Assignment table ○ pending - [9] Verify pipeline ○ pending - [10] Done ○ pending -──────────────────────────────────────────────────────────── -``` - -**Snowflake tracker (after choosing Snowflake):** -``` -───── Setup Warehouse (Snowflake) ───────────────────────── - [1] Choose warehouse ● done - [2] Account & user ○ pending - [3] Role & warehouse ○ pending - [4] Database & schema ○ pending - [5] Create crypto key ○ pending - [6] Register key in SF ○ pending - [7] Validate ○ pending - [8] Create warehouse ○ pending - [9] Create connectors ○ pending - [10] Assignment table ○ pending - [11] Verify pipeline ○ pending - [12] Done ○ pending -──────────────────────────────────────────────────────────── -``` - -**Databricks tracker (after choosing Databricks):** -``` -───── Setup Warehouse (Databricks) ──────────────────────── - [1] Choose warehouse ● done - [2] Workspace URL ○ pending - [3] SQL Warehouse ID ○ pending - [4] Service principal ○ pending - [5] AWS account & CLI ○ pending - [6] S3 bucket ○ pending - [7] IAM role ○ pending - [8] Databricks schema ○ pending - [9] Create warehouse ○ pending - [10] Create connectors ○ pending - [11] Assignment table ○ pending - [12] Verify pipeline ○ pending - [13] Done ○ pending -──────────────────────────────────────────────────────────── -``` - -**Redshift tracker (after choosing Redshift):** -``` -───── Setup Warehouse (Redshift) ────────────────────────── - [1] Choose warehouse ● done - [2] AWS account & CLI ○ pending - [3] Redshift cluster ○ pending - [4] S3 bucket ○ pending - [5] IAM role ○ pending - [6] Attach role ○ pending - [7] Schema & grants ○ pending - [8] Validate ○ pending - [9] Create warehouse ○ pending - [10] Create connectors ○ pending - [11] Assignment table ○ pending - [12] Verify pipeline ○ pending - [13] Done ○ pending -──────────────────────────────────────────────────────────── -``` - -Use `●` for completed, `▶` for in-progress, `○` for pending. Re-display the full tracker after every step transition so the user always sees their progress. - -### Step 1: Choose warehouse type - -> Which data warehouse do you use? -> 1. BigQuery -> 2. Snowflake -> 3. Databricks -> 4. Redshift - -### Step 2: Configure - -Collect configuration based on type. Explain each field briefly. - -**BigQuery:** - -Guide the user through each field with plain-language explanations and where to find the value: - -1. **GCP Project ID** — the Google Cloud project where your data lives. - > Go to **Google Cloud Console** (console.cloud.google.com). Your project ID is shown in the top bar next to "Google Cloud". It looks like `my-company-prod` or `project-12345`. - -2. **Dataset name** — where Confidence stores its tables (default: `confidence`). - > A dataset is like a folder in BigQuery. If you don't have one yet, the skill can create it for you via `bq mk`. - -3. **Service account email** — a robot account that Confidence uses to write data. - > Go to **Google Cloud Console → IAM & Admin → Service Accounts**. Create one (e.g., `confidence-connector@.iam.gserviceaccount.com`) or pick an existing one. It needs BigQuery Data Editor and BigQuery Job User roles. - -**Snowflake:** - -Ask the user for these fields (explain each briefly): -- Account — Snowflake account identifier (e.g., `zlvpqre-wr49874`) -- User — Snowflake user for Confidence to connect as -- Role — Snowflake role (default: `ACCOUNTADMIN`) -- Warehouse — SQL warehouse for query execution (default: `COMPUTE_WH`) -- Exposure database — database for exposure tables (default: `CONFIDENCE`) -- Exposure schema — schema for exposure tables (default: `EXPOSURE`) - -**Then create a crypto key automatically** — the user does NOT provide this. The skill creates it via the IAM API: - -```bash -curl -s -w "\n%{http_code}" -X POST "https://iam.${REGION}.confidence.dev/v1/cryptoKeys?crypto_key_id=snowflake-key" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"kind": "SNOWFLAKE"}' -``` - -If the key already exists (HTTP 409), fetch it instead: -```bash -curl -s "https://iam.${REGION}.confidence.dev/v1/cryptoKeys/snowflake-key" \ - -H "Authorization: Bearer $TOKEN" -``` - -Extract the `publicKey` from the response, strip PEM headers and newlines to get raw base64. Then generate the Snowflake SQL to register the key, **copy it to clipboard**, and tell the user: - -> I've created an authentication key for Snowflake. You need to register it with your Snowflake user. -> The SQL has been copied to your clipboard — paste it in the Snowflake worksheet and run it. - -The SQL should be: -```sql -ALTER USER SET RSA_PUBLIC_KEY=''; -``` - -If the user says other Confidence accounts share this Snowflake user, use `RSA_PUBLIC_KEY_2` instead. - -Also generate SQL for creating the database/schema if the user says they don't exist yet: -```sql -CREATE DATABASE IF NOT EXISTS ; -CREATE SCHEMA IF NOT EXISTS .; -GRANT USAGE ON DATABASE TO ROLE ; -GRANT ALL ON SCHEMA . TO ROLE ; -``` - -Save the crypto key name (e.g., `cryptoKeys/snowflake-key`) for use in the warehouse config. - -**Databricks:** - -Before collecting details, explain the full picture so the user knows what they need: - -> Setting up Databricks with Confidence requires three things: -> -> 1. **A Databricks workspace** — you need admin access to create a service principal (a robot account) -> 2. **An AWS account with an S3 bucket** — Confidence needs this as a staging area for loading data into Databricks. This is required even if your Databricks runs on GCP or Azure -> 3. **A schema in Databricks** — a place for Confidence to create tables (e.g., `confidence`) -> -> **How data flows:** -> Confidence collects your flag assignments and events internally, then writes parquet files to an S3 bucket you provide, and finally loads them into Databricks tables. This happens in batches every ~5 minutes. -> -> ``` -> Confidence (collects data) → S3 bucket (staging) → Databricks (tables) -> ``` -> -> **Don't have an AWS account?** You'll need one for the S3 staging bucket. AWS free tier works fine. I can set it up for you if you have the `aws` CLI, or walk you through the AWS Console. - -Then collect the details **one at a time**. After each answer, confirm it before moving to the next. Don't dump all questions at once. - -**Part 1: Databricks connection** (3 things needed) - -1. **Host** — ask the user: - > What's your Databricks workspace URL? Just paste the URL from your browser address bar. - - Extract the hostname from whatever they paste (strip `https://`, trailing paths, query params). Valid examples: - - `dbc-a1b2c3d4-e5f6.cloud.databricks.com` - - `1234567890.7.gcp.databricks.com` - - `adb-1234567890.12.azuredatabricks.net` - - Confirm: "Got it — your Databricks workspace is at ``." - -2. **SQL Warehouse ID** — ask the user: - > I need a SQL Warehouse ID. Here's how to find it: - > 1. In Databricks, click **SQL Warehouses** in the left sidebar - > 2. Click on a warehouse name - > 3. Open the **Connection details** tab - > 4. Copy the **HTTP Path** — the ID is the last part after `/sql/1.0/warehouses/` - > - > It looks like a hex string, e.g., `ccf7028466008a3c` - > - > **Don't have a SQL Warehouse?** Click **Create SQL Warehouse** → name it "Confidence" → pick **Serverless**, size **Small** → **Create**. Then copy the ID. - - Confirm: "Using warehouse ``." - -3. **Service principal** — ask the user: - > I need a service principal — this is a robot account that Confidence uses to connect to Databricks. - > - > **To create one:** - > 1. Click the **gear icon** (⚙️) at the top of Databricks → **Settings** - > 2. Under **Identity and access**, click **Service principals** - > 3. Click **Add service principal → Add new** - > 4. Name it "Confidence" → **Add** - > 5. Click into the new service principal - > 6. Copy the **Application ID** (a UUID like `85cc292a-c1d2-...`) - > 7. Go to the **Secrets** tab → **Generate secret** - > 8. Copy both the **Secret** (shown only once!) and the **Client ID** - > - > Paste the **Client ID** and **Secret** here. - - If the user says they can't access Settings or service principals: - > You need workspace admin access for this step. Ask your Databricks admin to: - > 1. Create a service principal named "Confidence" - > 2. Generate a secret for it - > 3. Send you the Client ID and Secret - - Confirm: "Service principal configured." - -**Part 2: S3 staging bucket** (requires AWS account) - -Explain why: -> Confidence writes parquet files to an S3 bucket, then Databricks loads them via COPY INTO. Think of it as a mailbox — Confidence drops files there, and Databricks picks them up. **This is required even if your Databricks runs on GCP or Azure.** -> -> You need an AWS account for this. If you don't have one, I can help you set one up. - -Ask the user: -> Do you have the `aws` CLI set up, or would you prefer manual steps? -> 1. Set it up for me (requires `aws` CLI) -> 2. Show me the steps - -**If the user picks 1 (aws CLI):** - -First check: `which aws`. If not found, offer to install: `brew install awscli` (macOS) or guide them to https://aws.amazon.com/cli/. - -Then check they're logged in: `aws sts get-caller-identity`. If not, tell them: -> Run `aws configure` or `aws sso login` to log into your AWS account first. - -Extract the Confidence service account and its numeric unique ID (required for AWS trust policy): -```bash -ACCOUNT_ID=$(echo "$TOKEN" | cut -d. -f2 | python3 -c " -import sys, json, base64 -p = sys.stdin.read().strip() -p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' -d = json.loads(base64.b64decode(p)) -print(d['https://confidence.dev/account_name'].split('/')[-1]) -") -CONFIDENCE_SA="account-${ACCOUNT_ID}@spotify-confidence.iam.gserviceaccount.com" - -# CRITICAL: AWS trust policy needs the NUMERIC unique ID, not the email. -# The email won't work — AWS requires accounts.google.com:sub which is the numeric ID. -SA_UNIQUE_ID=$(gcloud iam service-accounts describe ${CONFIDENCE_SA} \ - --project=spotify-confidence --format="value(uniqueId)") -``` - -If `gcloud` can't access `spotify-confidence` project, the user needs to contact Confidence support to get the numeric service account ID. - -Ask the user for a bucket name (suggest `confidence-staging-`) and region (suggest `eu-west-1`). - -If `aws` CLI is not installed, install it: `brew install awscli` (macOS). - -If `aws` CLI is not configured, the skill should: -1. Open the AWS console login: `open "https://console.aws.amazon.com"` -2. Guide user to create access key: **click your name top right → Security credentials → Access keys → Create access key** -3. Write the credentials directly to `~/.aws/credentials` and `~/.aws/config` (don't use interactive `aws configure`) - -Then run these commands, confirming each step: - -```bash -# 1. Create S3 bucket -aws s3api create-bucket --bucket ${BUCKET_NAME} --region ${AWS_REGION} \ - --create-bucket-configuration LocationConstraint=${AWS_REGION} - -# 2. Create the trust policy file -# IMPORTANT: Use accounts.google.com:sub with the NUMERIC service account ID. -# Using :email will fail with "MalformedPolicyDocument". -# Using the email string as :sub will fail at runtime with "Not authorized to perform sts:AssumeRoleWithWebIdentity". -cat > $TMPDIR/trust-policy.json << EOF -{ - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Principal": {"Federated": "accounts.google.com"}, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "accounts.google.com:sub": "${SA_UNIQUE_ID}" - } - } - }] -} -EOF - -# 3. Create IAM role -aws iam create-role --role-name confidence-databricks-staging \ - --assume-role-policy-document file://$TMPDIR/trust-policy.json - -# 4. Create and attach S3 access policy -cat > $TMPDIR/s3-policy.json << EOF -{ - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": ["s3:PutObject", "s3:GetObject", "s3:DeleteObject", "s3:ListBucket"], - "Resource": [ - "arn:aws:s3:::${BUCKET_NAME}", - "arn:aws:s3:::${BUCKET_NAME}/*" - ] - }] -} -EOF -aws iam put-role-policy --role-name confidence-databricks-staging \ - --policy-name S3Access --policy-document file://$TMPDIR/s3-policy.json - -# 5. Get the role ARN -ROLE_ARN=$(aws iam get-role --role-name confidence-databricks-staging --query 'Role.Arn' --output text) -echo "ROLE_ARN: $ROLE_ARN" -``` - -After completion, show the user: -> AWS setup complete! -> - Bucket: `` in `` -> - Role: `` -> -> Continuing with connector setup... - -**If the user picks 2 (manual steps):** - -4. **S3 bucket name** — the staging bucket. - > Go to **AWS Console** (https://console.aws.amazon.com) → **S3 → Create bucket**. - > - Name: something like `confidence-staging-` (must be globally unique) - > - Region: pick the same region as your Databricks workspace (e.g., `eu-west-1` for EU) - > - Leave all other settings as default → **Create bucket** - > - > If you already have a bucket you want to reuse, that works too — just give me the name. - -5. **AWS Region** — where the S3 bucket lives (e.g., `eu-west-1`, `us-east-1`). - -6. **IAM Role ARN** — an AWS role that grants Confidence permission to write to the bucket. - > Go to **AWS Console → IAM → Roles → Create role**. - > - Trusted entity: **Web identity** - > - Identity provider: select **accounts.google.com** (add it first if not listed under Identity providers) - > - Audience: `account-@spotify-confidence.iam.gserviceaccount.com` - > (the skill should compute the account ID from the JWT token and fill this in for the user) - > - Click **Next** → **Create policy** → JSON tab → paste this: - > ```json - > { - > "Version": "2012-10-17", - > "Statement": [{ - > "Effect": "Allow", - > "Action": ["s3:PutObject", "s3:GetObject", "s3:DeleteObject", "s3:ListBucket"], - > "Resource": ["arn:aws:s3:::", "arn:aws:s3:::/*"] - > }] - > } - > ``` - > - Attach the policy → name the role (e.g., `confidence-databricks-staging`) → **Create role** - > - Copy the **Role ARN** (looks like `arn:aws:iam::123456789012:role/confidence-databricks-staging`) - > - > **If you get "Not authorized to perform sts:AssumeRoleWithWebIdentity" later:** the trust policy is wrong — the Confidence service account email must exactly match what's in the role's trust policy. - -**Part 3: Databricks schema** - -Ask the user: -> Last thing — where should Confidence create its tables in Databricks? I need a schema name. -> The default is `confidence`. If you already have a schema you'd like to use, let me know. - -Then check if the schema exists and the service principal has access. Generate the SQL and **copy to clipboard**: - -> I'll set up the schema and permissions. Here's what I'm running — copied to your clipboard. Paste it in the **Databricks SQL Editor** (left sidebar → SQL Editor) and run it. - -For workspaces **without Unity Catalog** (hive_metastore): -```sql -CREATE SCHEMA IF NOT EXISTS confidence; -GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence TO ``; -``` - -For workspaces **with Unity Catalog**: -```sql -CREATE CATALOG IF NOT EXISTS confidence; -CREATE SCHEMA IF NOT EXISTS confidence.confidence; -GRANT USE CATALOG ON CATALOG confidence TO ``; -GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence.confidence TO ``; -``` - -**How to tell which one:** If the user sees **Catalog** in the Databricks left sidebar, they have Unity Catalog. If they only see **Data**, they're on hive_metastore. - -After the user runs it, confirm: "Schema ready. Moving on to create the warehouse." - -**Redshift:** - -Before collecting details, explain the full picture so the user knows what they're signing up for: - -> Setting up Redshift with Confidence requires an **AWS account**. Here's what we'll set up: -> -> 1. **A Redshift cluster** — a data warehouse that stores your experiment data -> 2. **An S3 bucket** — a staging area where Confidence drops data files before loading them into Redshift -> 3. **An IAM role** — permissions that let Confidence write to S3 and load into Redshift -> 4. **A schema** — a folder inside Redshift where Confidence creates its tables -> -> **How data flows:** -> ``` -> Confidence → S3 bucket (staging) → Redshift COPY → your tables -> ``` -> -> I can set up everything automatically if you have the `aws` CLI, or walk you through the AWS Console step by step. -> -> **Don't have an AWS account?** You'll need one. I can open the signup page for you. AWS free tier covers S3, but Redshift clusters cost ~$0.25/hr while running. You can delete it after testing. - -Ask the user: -> Do you have the `aws` CLI set up, or would you prefer manual steps? -> 1. Set it up for me (requires `aws` CLI) -> 2. Show me the steps - -**If the user picks 1 (aws CLI):** - -Check `which aws`. If not found: `brew install awscli` (macOS). -Check `aws sts get-caller-identity`. If not logged in, open the AWS console login (`open "https://console.aws.amazon.com"`), guide them to create access keys (**click name top right → Security credentials → Access keys → Create**), then write the credentials to `~/.aws/credentials` and `~/.aws/config`. - -Then ask one question at a time: - -**Part 1: Redshift cluster** - -1. Ask the user: - > Do you already have a Redshift cluster, or should I create one? - - If they have one: - > What's the cluster name? Go to **AWS Console → Amazon Redshift → Clusters**. The name is in the first column. - - If they need one, explain: - > I'll create a single-node Redshift cluster. This is a data warehouse — like a powerful database optimized for analytics. - > - **Cost:** ~$0.25/hour while running. Delete it when you're done testing. - > - **Type:** `ra3.large` (cheapest option that supports single-node) - > - **Region:** `eu-west-1` (Europe) — should match where your Confidence account is - > - > **Important:** Redshift Serverless won't work — Confidence needs a provisioned cluster. I'll create the right type. - - Create it: - ```bash - aws redshift create-cluster \ - --cluster-identifier confidence-redshift-${ACCOUNT_ID} \ - --cluster-type single-node \ - --node-type ra3.large \ - --master-username admin \ - --master-user-password '' \ - --db-name dev \ - --region eu-west-1 \ - --publicly-accessible - ``` - - Wait for status `available` (takes ~1–2 minutes): - ```bash - aws redshift wait cluster-available --cluster-identifier ${CLUSTER} --region ${AWS_REGION} - ``` - - Confirm: "Redshift cluster `` is running." - -**Part 2: IAM role** - -2. Explain: - > Now I need to set up permissions. Confidence needs a single IAM role that does two things: - > - Lets Confidence write data files to S3 and query Redshift - > - Lets Redshift read those files from S3 (via the COPY command) - > - > I'll create this role automatically. - - Get the Confidence service account numeric ID: - ```bash - SA_UNIQUE_ID=$(gcloud iam service-accounts describe ${CONFIDENCE_SA} \ - --project=spotify-confidence --format="value(uniqueId)") - ``` - - Create the role with dual trust (Google OIDC + Redshift): - ```bash - cat > $TMPDIR/redshift-trust.json << EOF - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": {"Federated": "accounts.google.com"}, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "accounts.google.com:sub": "${SA_UNIQUE_ID}" - } - } - }, - { - "Effect": "Allow", - "Principal": {"Service": "redshift.amazonaws.com"}, - "Action": "sts:AssumeRole" - } - ] - } - EOF - aws iam create-role --role-name confidence-redshift \ - --assume-role-policy-document file://$TMPDIR/redshift-trust.json - ``` - - Attach S3 + Redshift Data API permissions: - ```bash - # S3 write access - cat > $TMPDIR/s3-policy.json << EOF - {"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:PutObject","s3:GetObject","s3:DeleteObject","s3:ListBucket"],"Resource":["arn:aws:s3:::${BUCKET_NAME}","arn:aws:s3:::${BUCKET_NAME}/*"]}]} - EOF - aws iam put-role-policy --role-name confidence-redshift \ - --policy-name S3Access --policy-document file://$TMPDIR/s3-policy.json - - # Redshift Data API access - cat > $TMPDIR/redshift-data-policy.json << EOF - {"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["redshift-data:*","redshift:GetClusterCredentials","redshift:GetClusterCredentialsWithIAM","redshift:DescribeClusters"],"Resource":"*"}]} - EOF - aws iam put-role-policy --role-name confidence-redshift \ - --policy-name RedshiftAccess --policy-document file://$TMPDIR/redshift-data-policy.json - ``` - - **CRITICAL:** Attach the role to the Redshift cluster — without this, the COPY command can't read from S3: - ```bash - aws redshift modify-cluster-iam-roles \ - --cluster-identifier ${CLUSTER} \ - --add-iam-roles ${ROLE_ARN} --region ${AWS_REGION} - ``` - Wait for `in-sync`: - ```bash - aws redshift describe-clusters --cluster-identifier ${CLUSTER} --region ${AWS_REGION} \ - --query "Clusters[0].IamRoles[*].{Role:IamRoleArn,Status:ApplyStatus}" --output table - ``` - - Confirm: "IAM role created and attached to cluster." - -**Part 3: S3 staging bucket** - -3. Ask the user: - > Do you have an S3 bucket I should use, or should I create one? - - If they already did Databricks setup: - > You already have `` from the Databricks setup. Want to reuse it? - - Otherwise, create one: - ```bash - aws s3api create-bucket --bucket confidence-redshift-${ACCOUNT_ID} \ - --region ${AWS_REGION} \ - --create-bucket-configuration LocationConstraint=${AWS_REGION} - ``` - - Confirm: "S3 bucket `` created in ``." - -**Part 4: Schema** - -4. Ask the user: - > What should the schema be called? The default is `confidence`. - - Create the schema and grant permissions so Confidence can see it: - ```bash - aws redshift-data execute-statement \ - --cluster-identifier ${CLUSTER} --database ${DATABASE} --db-user admin \ - --sql "CREATE SCHEMA IF NOT EXISTS ${SCHEMA}; GRANT USAGE ON SCHEMA ${SCHEMA} TO PUBLIC; GRANT CREATE ON SCHEMA ${SCHEMA} TO PUBLIC;" \ - --region ${AWS_REGION} - ``` - - **IMPORTANT:** `GRANT USAGE ON SCHEMA ... TO PUBLIC` is required — without it, Confidence's validation returns "Schema not found" even though the schema exists. This is because Confidence connects via IAM, not as the `admin` user. - - Confirm: "Schema `` created with permissions." - -**If the user picks 2 (manual steps):** - -Walk them through the AWS Console for each step: - -1. **Redshift cluster:** Go to **AWS Console → Amazon Redshift → Create cluster** → single-node, ra3.large, database `dev`, publicly accessible. - -2. **S3 bucket:** Go to **AWS Console → S3 → Create bucket** → name it, pick same region as cluster. - -3. **IAM role:** Go to **AWS Console → IAM → Roles → Create role** → two trust steps: - - Add **Web identity** trust with `accounts.google.com`, sub = `` (compute and display for the user) - - Add **AWS service** trust for `redshift.amazonaws.com` - - Attach policies: custom S3 policy scoped to bucket + `AmazonRedshiftDataFullAccess` - - Copy the **Role ARN** - - Go back to **Redshift → Clusters → your cluster → Properties → Manage IAM roles → Add the new role** - -4. **Schema:** Go to **Redshift → Query editor v2** → connect to cluster → run: - ```sql - CREATE SCHEMA IF NOT EXISTS confidence; - GRANT USAGE ON SCHEMA confidence TO PUBLIC; - GRANT CREATE ON SCHEMA confidence TO PUBLIC; - ``` - Copy the SQL to clipboard for the user. - -### Step 3: Validate configuration - -**NOTE:** The validate endpoint supports BigQuery (`bigQueryConfig`), Snowflake (`snowflakeConfig`), and Redshift (`redshiftConfig`). It does NOT support Databricks (returns "configuration must be set" for any field name variant). For Databricks, skip validation and proceed directly to Step 4 (Create warehouse). Tell the user: -> Pre-validation isn't available yet for Databricks. I'll create the warehouse now and we'll verify the connection works end-to-end in the pipeline test step. - -For BigQuery/Snowflake: -```bash -curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/dataWarehouseConfig:validate" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "": { } - }' -``` - -**Response:** -```json -{ - "validation": [{ "key": "...", "description": "...", "success": true/false, "error": "..." }], - "successful": true/false, - "configurationResponse": { /* type-specific: available schemas, databases, roles */ } -} -``` - -If the response is an error (HTTP 400/500) or `successful` is false: - -**IMPORTANT: Never assume partial success from an ambiguous error.** If the API returns an error like "X does not exist or not authorized", report the exact error message — do NOT split it into "connection works but X is missing". The error may indicate an auth failure, a missing resource, or both. Show the user the exact error and let them determine the cause. - -For each validation failure, show: -> Validation failed: `` - -Then offer remediation based on warehouse type. - -**For BigQuery failures**, ask the user how they want to proceed: - -> Some permissions need to be configured on your GCP project. I can fix this automatically if you have `gcloud` set up, or I can show you the exact commands to run yourself. -> -> 1. Fix it for me (requires gcloud CLI) -> 2. Show me the commands - -**For Snowflake failures**, generate the full remediation SQL, **copy it to clipboard via `pbcopy`**, and tell the user to paste it in the Snowflake worksheet (https://app.snowflake.com): - -1. **Fetch the crypto key's public key** from the IAM API: - ```bash - curl -s "https://iam.${REGION}.confidence.dev/v1/cryptoKeys/" -H "Authorization: Bearer $TOKEN" - ``` - Strip the PEM headers (`-----BEGIN/END PUBLIC KEY-----`) and newlines to get the raw base64 string for Snowflake. - -2. **Generate SQL based on the error:** - - Auth failures → register the public key: - ```sql - -- If this is the only Confidence account using this Snowflake user: - ALTER USER SET RSA_PUBLIC_KEY=''; - -- If another Confidence account already uses RSA_PUBLIC_KEY, use key 2: - ALTER USER SET RSA_PUBLIC_KEY_2=''; - ``` - **IMPORTANT:** Always ask the user if other Confidence accounts share this Snowflake user. If yes, use `RSA_PUBLIC_KEY_2` to avoid breaking existing connections. Snowflake accepts auth from either key. - - Database/schema missing: - ```sql - CREATE DATABASE IF NOT EXISTS ; - CREATE SCHEMA IF NOT EXISTS .; - GRANT USAGE ON DATABASE TO ROLE ; - GRANT USAGE ON SCHEMA . TO ROLE ; - GRANT ALL ON SCHEMA . TO ROLE ; - ``` - -3. **Copy to clipboard and tell the user:** - ```bash - echo "" | pbcopy - ``` - > The SQL commands have been copied to your clipboard. Paste them in the Snowflake worksheet at https://app.snowflake.com and run them. Let me know when done and I'll retry validation. - -**For Databricks/Redshift failures**, show the relevant remediation steps for that platform. - -**If the user chooses 1 (fix it for me):** - -First check gcloud is available: `which gcloud`. If not, fall back to option 2. - -Extract the account ID from the token claim `https://confidence.dev/account_name` (e.g., `accounts/my-workspace` → `my-workspace`). The Confidence SA is: `account-${ACCOUNT_ID}@spotify-confidence.iam.gserviceaccount.com` - -For each failure, **confirm before each action:** - -**"Unable to create access token" (SERVICE_ACCOUNT):** -> Confidence needs permission to access your service account. Can I grant that now? -```bash -CONFIDENCE_SA="account-${ACCOUNT_ID}@spotify-confidence.iam.gserviceaccount.com" -gcloud iam service-accounts add-iam-policy-binding ${CUSTOMER_SA} \ - --project=${PROJECT} \ - --member="serviceAccount:${CONFIDENCE_SA}" \ - --role="roles/iam.workloadIdentityUser" --quiet -gcloud iam service-accounts add-iam-policy-binding ${CUSTOMER_SA} \ - --project=${PROJECT} \ - --member="serviceAccount:${CONFIDENCE_SA}" \ - --role="roles/iam.serviceAccountTokenCreator" --quiet -``` - -**"Missing permission 'bigquery.jobs.create'" (PERMISSIONS):** -> Your service account needs BigQuery Job User permissions. Can I grant that? -```bash -gcloud projects add-iam-policy-binding ${PROJECT} \ - --member="serviceAccount:${CUSTOMER_SA}" \ - --role="roles/bigquery.jobUser" --quiet -``` - -**"Could not find dataset" or dataset errors (DATASET):** -> The BigQuery dataset needs to be created or permissions updated. Can I do that? -```bash -bq mk --project_id=${PROJECT} --dataset --location=${REGION} ${DATASET} -bq update --project_id=${PROJECT} --source /dev/stdin ${DATASET} << EOF -{"access": [ - {"role": "WRITER", "userByEmail": "${CUSTOMER_SA}"}, - {"role": "OWNER", "specialGroup": "projectOwners"}, - {"role": "WRITER", "specialGroup": "projectWriters"}, - {"role": "READER", "specialGroup": "projectReaders"} -]} -EOF -``` - -**"free tier" / "Streaming insert is not allowed":** -> BigQuery streaming requires billing enabled on your GCP project. Can I link a billing account? -```bash -gcloud billing accounts list -gcloud billing projects link ${PROJECT} --billing-account=${BILLING_ACCOUNT} -``` -Note: billing propagation to BigQuery can take up to 15 minutes. - -After fixing, re-validate. If still failing (e.g., IAM propagation), inform the user and offer to retry. - -**If the user chooses 2 (show me the commands):** - -Show the exact gcloud/bq commands they need to run, with their specific values filled in. Format as a copyable script block: - -``` -Here's what needs to be configured on your GCP project: - -# 1. Grant Confidence access to your service account -gcloud iam service-accounts add-iam-policy-binding \ - \ - --project= \ - --member="serviceAccount:account-@spotify-confidence.iam.gserviceaccount.com" \ - --role="roles/iam.workloadIdentityUser" - -gcloud iam service-accounts add-iam-policy-binding \ - \ - --project= \ - --member="serviceAccount:account-@spotify-confidence.iam.gserviceaccount.com" \ - --role="roles/iam.serviceAccountTokenCreator" - -# 2. Grant BigQuery Job User -gcloud projects add-iam-policy-binding \ - --member="serviceAccount:" \ - --role="roles/bigquery.jobUser" - -# 3. Enable billing (if not already) -gcloud billing projects link --billing-account= - -Run these commands, then let me know and I'll retry validation. -``` - -If `configurationResponse` contains available options (schemas, roles) — present these as choices to help the user. - -### Step 4: Create warehouse - -**IMPORTANT:** The body is the data warehouse object directly (gRPC transcoding `body: "data_warehouse"`), NOT wrapped in a `dataWarehouse` key. - -```bash -curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/dataWarehouses" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "config": { - "": { } - } - }' -``` - -Save the returned `name` (e.g., `dataWarehouses/...`) for reference. - -### Step 5: Create connectors - -Create both connectors: - -**Flag Applied Connection** (assignment data → warehouse). - -**IMPORTANT:** The body is the connection object directly (gRPC transcoding `body: "flag_applied_connection"`), NOT wrapped. - -```bash -curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev/v1/flagAppliedConnections" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "bigQuery": { - "bigQueryConfig": { "serviceAccount": "...", "project": "...", "dataset": "..." }, - "table": "assignments" - } - }' -``` - -Adapt the destination field per warehouse type: -- **BigQuery:** `"bigQuery": { "bigQueryConfig": {...}, "table": "assignments" }` -- **Snowflake:** `"snowflake": { "snowflakeConfig": {..., "database": "...", "schema": "..."}, "table": "ASSIGNMENTS" }` — Snowflake requires `database` and `schema` fields in snowflakeConfig for connectors -- **Databricks:** Databricks connectors use a nested `connectionConfig` for auth, require an **S3 staging bucket** for batch writes, and `batchFileConfig`: - ```json - "databricks": { - "databricksConfig": { - "connectionConfig": { - "host": "...", - "warehouseId": "...", - "clientId": "...", - "clientSecret": "..." - }, - "schema": "", - "s3BucketConfig": { - "bucket": "", - "region": "", - "roleArn": "" - }, - "batchFileConfig": { - "maxFileAge": "300s" - } - }, - "table": "assignments" - } - ``` - **IMPORTANT:** Databricks connectors require an S3 staging bucket — Confidence writes data in batches to S3, then loads into Databricks. The user needs to provide an S3 bucket, AWS region, and IAM role ARN with write access to the bucket. Explain this to the user: - > Confidence writes data to a staging bucket first, then loads it into Databricks. You'll need an S3 bucket and an IAM role that allows Confidence to write to it. -- **Redshift:** `"redshift": { "redshiftConfig": {...}, "s3Config": {...}, "batchFileConfig": {...}, "table": "assignments" }` - -**Event Connection** (events → warehouse). - -**IMPORTANT:** The body is the connection object directly (gRPC transcoding `body: "event_connection"`), NOT wrapped. - -```bash -curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev/v1/eventConnections" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "bigQuery": { - "bigQueryConfig": { "serviceAccount": "...", "project": "...", "dataset": "..." }, - "tablePrefix": "events_" - } - }' -``` - -Same destination pattern as above, but with `tablePrefix` instead of `table`. - -**Redshift/Databricks require additional config:** -- `s3Config`: `{ "bucket": "...", "region": "...", "roleArn": "..." }` — staging bucket -- `batchFileConfig`: `{ "maxEventsPerFile": 10000, "maxFileAge": "300s", "maxFileSize": 104857600 }` — batching params - -Collect these if the user chose Redshift or Databricks. - -### Step 6: Assignment table - -Create an assignment table so Confidence can analyze experiment assignments. - -**IMPORTANT:** The body is the assignment table object directly (gRPC transcoding `body: "assignment_table"`), NOT wrapped in an `assignmentTable` key. - -```bash -curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/assignmentTables" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "displayName": "Assignments", - "sql": "", - "entityColumn": { "name": "targeting_key" }, - "timestampColumn": { "name": "assignment_time" }, - "exposureKeyColumn": { "name": "rule" }, - "variantKeyColumn": { "name": "assignment_id" }, - "dataDeliveredUntilUpdateStrategyConfig": { - "strategy": "AUTOMATIC", - "automaticUpdateConfig": { - "commitDelay": "300s" - } - } - }' -``` - -Adapt the SQL query per warehouse type: -- **BigQuery:** `` SELECT targeting_key, rule, assignment_id, assignment_time FROM `..assignments` `` -- **Snowflake:** `SELECT targeting_key, rule, assignment_id, assignment_time FROM ..ASSIGNMENTS` -- **Databricks:** `SELECT targeting_key, rule, assignment_id, assignment_time FROM .assignments` -- **Redshift:** `SELECT targeting_key, rule, assignment_id, assignment_time FROM .assignments` - -### Step 7: Verify data pipeline - -Verify both connectors by generating test data and checking it lands in the warehouse. - -**7a. Get a client secret for testing** - -The resolver and events APIs require a **client secret** (not a Bearer token). - -1. **List the user's clients** and show them: - ```bash - curl -s "https://iam.${REGION}.confidence.dev/v1/clients" -H "Authorization: Bearer $TOKEN" - ``` - Display each client with its name and last-seen time. If only one client exists, confirm it with the user. If multiple, let them pick. - -2. **Ask the user** if they have a client secret or want a new one: - > I'll use **** for the pipeline test. Do you have the client secret, or should I create a new credential? - -3. If the user wants a new credential, create one on the chosen client: - ```bash - curl -s -X POST "https://iam.${REGION}.confidence.dev/v1//credentials" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"display_name": "Pipeline Test"}' - ``` - Save the secret to a temp file for pipeline use. **Never print the secret to the user's terminal.** - -**7b. Verify flag assignments** - -Resolve a flag to generate assignment data (use an existing flag + client secret): -```bash -curl -s -X POST "https://resolver.${REGION}.confidence.dev/v1/flags:resolve" \ - -H "Content-Type: application/json" \ - -d '{ - "flags": ["flags/"], - "evaluation_context": {"targeting_key": "warehouse-verify-user"}, - "client_secret": "", - "apply": true - }' -``` - -If no flags exist yet, tell the user: -> No flags to test with. Run `/onboard-confidence setup-wizard` first to create a flag, then come back. - -**7b. Verify events** - -First check for an event definition to use: -```bash -curl -s "https://events.${REGION}.confidence.dev/v1/eventDefinitions" \ - -H "Authorization: Bearer $TOKEN" -``` - -If no event definitions exist, create one with a schema: -```bash -curl -s -X POST "https://events.${REGION}.confidence.dev/v1/eventDefinitions?event_definition_id=test-event" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"schema": {"action": {"stringSchema": {}}, "page": {"stringSchema": {}}}}' -``` - -If an event definition exists but has an empty schema, update it so payload data flows through: -```bash -curl -s -X PATCH "https://events.${REGION}.confidence.dev/v1/eventDefinitions/" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"schema": {"action": {"stringSchema": {}}, "page": {"stringSchema": {}}}}' -``` - -Then publish test events (uses client secret, NOT Bearer token): -```bash -NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -curl -s -X POST "https://events.${REGION}.confidence.dev/v1/events:publish" \ - -H "Content-Type: application/json" \ - -d '{ - "client_secret": "", - "events": [ - { - "event_definition": "eventDefinitions/", - "payload": {"action": "clicked_button", "page": "homepage"}, - "event_time": "'$NOW'" - } - ], - "send_time": "'$NOW'" - }' -``` - -Check response: `{"errors": []}` means success. If `EVENT_DEFINITION_NOT_FOUND`, the definition doesn't exist. If `EVENT_SCHEMA_VALIDATION_FAILED`, the payload doesn't match the schema. - -**7c. Check data in warehouse** - -Verification approach depends on warehouse type. Ask the user: "Want me to check the data, or show you the queries?" - -**BigQuery:** - -If user has `bq` CLI: -```bash -echo "=== ASSIGNMENTS ===" && \ -bq query --project_id=${PROJECT} --use_legacy_sql=false \ - 'SELECT targeting_key, rule, assignment_id, assignment_time - FROM `${PROJECT}.${DATASET}.assignments` - ORDER BY assignment_time DESC LIMIT 5' && \ -echo "=== EVENTS ===" && \ -bq query --project_id=${PROJECT} --use_legacy_sql=false \ - 'SELECT * FROM `${PROJECT}.${DATASET}.events_*` - ORDER BY _event_time DESC LIMIT 5' -``` - -If no `bq`, show queries for BigQuery console. - -**Snowflake:** - -If user has `snowsql` CLI: -```bash -snowsql -a ${SNOWFLAKE_ACCOUNT} -u ${SNOWFLAKE_USER} -r ${SNOWFLAKE_ROLE} -w ${SNOWFLAKE_WAREHOUSE} -d ${SNOWFLAKE_DATABASE} -s ${SNOWFLAKE_SCHEMA} -q " -SELECT targeting_key, rule, assignment_id, assignment_time -FROM ${SNOWFLAKE_DATABASE}.${SNOWFLAKE_SCHEMA}.ASSIGNMENTS -ORDER BY assignment_time DESC LIMIT 5; -" -``` - -If no `snowsql`, use the Snowflake SQL REST API: -```bash -# Get a JWT token for Snowflake (using keypair auth) or prompt user for password -# Then query via the SQL API: -curl -s -X POST "https://${SNOWFLAKE_ACCOUNT}.snowflakecomputing.com/api/v2/statements" \ - -H "Authorization: Bearer ${SNOWFLAKE_JWT}" \ - -H "Content-Type: application/json" \ - -H "X-Snowflake-Authorization-Token-Type: KEYPAIR_JWT" \ - -d '{ - "statement": "SELECT targeting_key, rule, assignment_id, assignment_time FROM '${SNOWFLAKE_DATABASE}'.'${SNOWFLAKE_SCHEMA}'.ASSIGNMENTS ORDER BY assignment_time DESC LIMIT 5", - "warehouse": "'${SNOWFLAKE_WAREHOUSE}'", - "database": "'${SNOWFLAKE_DATABASE}'", - "schema": "'${SNOWFLAKE_SCHEMA}'", - "role": "'${SNOWFLAKE_ROLE}'" - }' -``` - -If neither available, show the queries for the Snowflake worksheet (https://app.snowflake.com): -> ```sql -> -- Assignments -> SELECT targeting_key, rule, assignment_id, assignment_time -> FROM ..ASSIGNMENTS -> ORDER BY assignment_time DESC LIMIT 5; -> -> -- Events (list event tables first, then query) -> SHOW TABLES LIKE 'EVENTS_%' IN .; -> SELECT * FROM .. -> ORDER BY _event_time DESC LIMIT 5; -> ``` - -**Databricks:** - -Use the Databricks SQL Statement API to query directly (the skill already has the service principal credentials): -```bash -DB_TOKEN=$(curl -s -X POST "https://${DATABRICKS_HOST}/oidc/v1/token" \ - -H "Content-Type: application/x-www-form-urlencoded" \ - -d "grant_type=client_credentials&client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&scope=all-apis" \ - | python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])") - -curl -s -X POST "https://${DATABRICKS_HOST}/api/2.0/sql/statements" \ - -H "Authorization: Bearer $DB_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "warehouse_id": "'${WAREHOUSE_ID}'", - "statement": "SELECT targeting_key, rule, assignment_id, assignment_time FROM '${SCHEMA}'.assignments ORDER BY assignment_time DESC LIMIT 5", - "wait_timeout": "30s" - }' -``` - -**IMPORTANT:** Data is batched every ~5 minutes. If the table doesn't exist yet, wait and retry. Tell the user: -> Data delivery takes about 5 minutes. Let me check again... - -If `TABLE_OR_VIEW_NOT_FOUND` after 10 minutes, check the connector logs for errors. - -**Redshift:** - -If user has `psql` or `aws redshift-data`: -```bash -aws redshift-data execute-statement \ - --cluster-identifier ${CLUSTER} \ - --database ${DATABASE} \ - --db-user ${DB_USER} \ - --sql "SELECT targeting_key, rule, assignment_id, assignment_time FROM ${SCHEMA}.assignments ORDER BY assignment_time DESC LIMIT 5" -``` - -Otherwise, show queries for the Redshift query editor. - -**Show results (all warehouse types):** -``` - ● Assignments: rows — data flowing - → () - ● Events: rows — data flowing - on () -``` - -**If no rows after a few seconds**, tell the user: -> Data delivery can take up to a few minutes depending on your warehouse. Check again shortly, or verify in your warehouse console. - -### Step 8: Done - -``` -═══════════════════════════════════════════════════════════════ - Data Warehouse Connected & Verified -═══════════════════════════════════════════════════════════════ - - Warehouse: () - Dataset: - Connectors: - ● Flag assignments → assignments table (verified) - ● Events → events_* tables (running) - Assignment: - ● Assignment table configured (auto-updating) - - Flag assignment and event data is flowing to your - warehouse. Experiment analysis is ready. - -═══════════════════════════════════════════════════════════════ -``` +This command has been split into dedicated skills for each warehouse type. When the user asks to set up a warehouse, use `/onboard-confidence:setup-warehouse` which will guide them to the right one: +- `/onboard-confidence:setup-warehouse-bigquery` +- `/onboard-confidence:setup-warehouse-snowflake` +- `/onboard-confidence:setup-warehouse-databricks` +- `/onboard-confidence:setup-warehouse-redshift` --- @@ -2240,7 +1133,7 @@ Most sub-commands use REST APIs and do NOT require MCP. MCP is only used as a co | `mcp__confidence-flags__listClients` | `status` | List available clients (convenience) | | `mcp__confidence-docs__searchDocumentation` | `learn` | Fetch educational content | -**All other sub-commands (`create-account`, `invite-user`, `create-client`, `setup-wizard`, `setup-warehouse`) work entirely via REST APIs with the saved auth token.** +**All other sub-commands (`create-account`, `invite-user`, `create-client`, `setup-wizard`, `setup-warehouse`) work entirely via REST APIs with the session auth token.** --- diff --git a/skills/setup-warehouse-bigquery/SKILL.md b/skills/setup-warehouse-bigquery/SKILL.md new file mode 100644 index 0000000..26b1c72 --- /dev/null +++ b/skills/setup-warehouse-bigquery/SKILL.md @@ -0,0 +1,705 @@ +--- +description: Set up BigQuery as a data warehouse for Confidence. Use when the user chose BigQuery for warehouse setup. +--- + +# Setup Warehouse: BigQuery + +Configure BigQuery as the data warehouse for Confidence experimentation analytics. This skill handles the full end-to-end setup: collect GCP config, validate permissions, create the warehouse, set up connectors, create the assignment table, and verify the pipeline. + +--- + +## Authentication + +**Browser-based Auth0 login.** The skill opens a browser for Auth0 login (Google, email/password, SSO) and captures the token automatically. The user never touches a token. + +### Session-only token management + +The token is kept in the current session only and is never saved to disk. If the session ends or the token expires, the skill will open your browser to log in again. + +On every sub-command start, check if the `TOKEN` variable is set and not expired: + +```bash +if [ -n "$TOKEN" ]; then + PAYLOAD=$(echo "$TOKEN" | cut -d. -f2) + EXP=$(echo "$PAYLOAD" | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d.get('exp', 0)) +") + NOW=$(date +%s) + if [ "$EXP" -gt "$NOW" ]; then + echo "VALID" + else + echo "EXPIRED" + unset TOKEN + fi +fi +``` + +If `TOKEN` is unset or expired, run the Auth0 login flow with the **regular client ID** (`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w`) and the user's `organization` parameter. Store the result in the `TOKEN` shell variable only. **NEVER write the token to disk. NEVER reference `~/.confidence/`.** + +### Auth script + +Write the following to `$TMPDIR/confidence_auth.py` with CLIENT_ID=`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w` and ORGANIZATION from the token. Run with `python3 $TMPDIR/confidence_auth.py`. Outputs `TOKEN:` on success. + +```python +import http.server, urllib.parse, json, sys, subprocess, hashlib, base64, secrets, string + +code_verifier = ''.join(secrets.choice(string.ascii_letters + string.digits + '-._~') for _ in range(43)) +code_challenge = base64.urlsafe_b64encode(hashlib.sha256(code_verifier.encode()).digest()).rstrip(b'=').decode() + +port = 8084 +CLIENT_ID = '' +ORGANIZATION = '' +REDIRECT_URI = f'http://localhost:{port}/callback' +auth_code = None +error = None + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + global auth_code, error + q = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query) + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.end_headers() + if 'code' in q: + auth_code = q['code'][0] + self.wfile.write(b'

Login successful!

You can close this tab.

') + else: + error = q.get('error', ['unknown'])[0] + self.wfile.write(b'

Login failed

Please try again.

') + def log_message(self, format, *args): + pass + +params = { + 'client_id': CLIENT_ID, + 'redirect_uri': REDIRECT_URI, + 'response_type': 'code', + 'scope': 'openid profile email offline_access', + 'audience': 'https://confidence.dev/', + 'code_challenge': code_challenge, + 'code_challenge_method': 'S256', +} +if ORGANIZATION: + params['organization'] = ORGANIZATION + +authorize_url = 'https://auth.confidence.dev/authorize?' + urllib.parse.urlencode(params) +subprocess.Popen(['open', authorize_url]) +print('WAITING_FOR_LOGIN', flush=True) + +server = http.server.HTTPServer(('127.0.0.1', port), Handler) +server.timeout = 120 +while auth_code is None and error is None: + server.handle_request() +server.server_close() + +if error: + print(f'AUTH_ERROR:{error}', flush=True) + sys.exit(1) + +import urllib.request +token_data = json.dumps({ + 'grant_type': 'authorization_code', + 'client_id': CLIENT_ID, + 'code': auth_code, + 'redirect_uri': REDIRECT_URI, + 'code_verifier': code_verifier +}).encode() +req = urllib.request.Request( + 'https://auth.confidence.dev/oauth/token', + data=token_data, + headers={'Content-Type': 'application/json'} +) +try: + with urllib.request.urlopen(req) as resp: + token_response = json.loads(resp.read()) + print(f'TOKEN:{token_response["access_token"]}', flush=True) +except Exception as e: + print(f'TOKEN_ERROR:{e}', flush=True) + sys.exit(1) +``` + +### Extract region from token + +```bash +REGION=$(echo "$PAYLOAD" | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d.get('https://confidence.dev/region', 'EU')) +") +``` + +Then use `${REGION,,}` (lowercase) for URL prefix: `iam.eu.confidence.dev`, `metrics.eu.confidence.dev`, etc. + +### Common notes + +- Port is fixed at **8084** (must match Auth0 Allowed Callback URLs) +- If port 8084 is busy: `lsof -ti:8084 | xargs kill -9 2>/dev/null` +- All network commands require `dangerouslyDisableSandbox: true` +- Never show the token value to the user +- Always use region-specific URLs (e.g., `iam.eu.confidence.dev` not `iam.confidence.dev`) + +### Important: gRPC-REST transcoding rules + +The Confidence APIs use gRPC with REST transcoding. The `body` field in the proto HTTP binding determines the JSON structure: + +- **`body: "data_warehouse"`** -> send the data warehouse object directly: `{"config": {...}}` +- **`body: "flag_applied_connection"`** -> send the connection object directly: `{"bigQuery": {...}}` +- **`body: "event_connection"`** -> send the connection object directly: `{"bigQuery": {...}}` +- **`body: "assignment_table"`** -> send the assignment table object directly: `{"displayName": "...", "sql": "...", ...}` +- **`body: "*"`** -> send the full request message + +The body is the object directly, NOT wrapped in an outer key. + +Fields NOT in the body (like `flag_id`, `parent`) become **query parameters**. + +**Field names are `snake_case`** in requests. Responses may use `camelCase`. + +--- + +## User-Facing Communication Rules + +**NEVER expose internal technical details to the user.** + +- Do NOT show raw JSON request/response bodies in conversation +- Do NOT show Auth0 configuration details, token values, or OAuth internals +- DO show human-readable status updates: "Opening browser for login...", "Creating your warehouse...", "Connectors configured!" +- DO describe results in plain English +- The agent handles all auth/API complexity silently + +**Step Tracker:** Display a visual step tracker at every phase transition. Update and re-display it each time you move to a new step. + +--- + +## Step Tracker + +Display at START and after EACH step completes (updating status): + +``` +───── Setup Warehouse (BigQuery) ────────────────────────── + [1] Choose warehouse ● done + [2] GCP project ID ○ pending + [3] Dataset name ○ pending + [4] Service account ○ pending + [5] Validate & fix ○ pending + [6] Create warehouse ○ pending + [7] Create connectors ○ pending + [8] Assignment table ○ pending + [9] Verify pipeline ○ pending + [10] Done ○ pending +──────────────────────────────────────────────────────────── +``` + +Use `●` for completed, `▶` for in-progress, `○` for pending. Re-display the full tracker after every step transition. + +--- + +## Step 1: Choose warehouse (already done) + +The user has already chosen BigQuery. Mark step 1 as done. + +--- + +## Step 2: GCP Project ID + +Guide the user: + +> What's your GCP project ID? Go to **Google Cloud Console** (console.cloud.google.com). Your project ID is shown in the top bar next to "Google Cloud". It looks like `my-company-prod` or `project-12345`. + +--- + +## Step 3: Dataset name + +> A dataset is like a folder in BigQuery where Confidence stores its tables. The default is `confidence`. +> If you don't have one yet, I can create it for you via `bq mk`. + +Default: `confidence` + +--- + +## Step 4: Service account + +> A service account is a robot account that Confidence uses to write data to your BigQuery dataset. +> Go to **Google Cloud Console -> IAM & Admin -> Service Accounts**. Create one (e.g., `confidence-connector@.iam.gserviceaccount.com`) or pick an existing one. +> It needs **BigQuery Data Editor** and **BigQuery Job User** roles. + +--- + +## Step 5: Validate & fix permissions + +Run the validation endpoint: + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/dataWarehouseConfig:validate" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "bigQueryConfig": { + "serviceAccount": "", + "project": "", + "dataset": "" + } + }' +``` + +**Response:** +```json +{ + "validation": [{ "key": "...", "description": "...", "success": true/false, "error": "..." }], + "successful": true/false, + "configurationResponse": { /* available schemas, etc. */ } +} +``` + +If `successful` is true, move to Step 6. + +**If validation fails:** + +**IMPORTANT: Never assume partial success from an ambiguous error.** If the API returns an error like "X does not exist or not authorized", report the exact error message. Do NOT split it into "connection works but X is missing". Show the user the exact error and let them determine the cause. + +For each validation failure, show: +> Validation failed: `` + +Then offer remediation: + +> Some permissions need to be configured on your GCP project. I can fix this automatically if you have `gcloud` set up, or I can show you the exact commands to run yourself. +> +> 1. Fix it for me (requires gcloud CLI) +> 2. Show me the commands + +### Fix it automatically (gcloud) + +First check gcloud is available: `which gcloud`. If not found, fall back to option 2. + +Extract the account ID from the token claim `https://confidence.dev/account_name` (e.g., `accounts/my-workspace` -> `my-workspace`). The Confidence SA is: `account-${ACCOUNT_ID}@spotify-confidence.iam.gserviceaccount.com` + +For each failure, **confirm before each action:** + +**"Unable to create access token" (SERVICE_ACCOUNT):** +> Confidence needs permission to access your service account. Can I grant that now? +```bash +CONFIDENCE_SA="account-${ACCOUNT_ID}@spotify-confidence.iam.gserviceaccount.com" +gcloud iam service-accounts add-iam-policy-binding ${CUSTOMER_SA} \ + --project=${PROJECT} \ + --member="serviceAccount:${CONFIDENCE_SA}" \ + --role="roles/iam.workloadIdentityUser" --quiet +gcloud iam service-accounts add-iam-policy-binding ${CUSTOMER_SA} \ + --project=${PROJECT} \ + --member="serviceAccount:${CONFIDENCE_SA}" \ + --role="roles/iam.serviceAccountTokenCreator" --quiet +``` + +**"Missing permission 'bigquery.jobs.create'" (PERMISSIONS):** +> Your service account needs BigQuery Job User permissions. Can I grant that? +```bash +gcloud projects add-iam-policy-binding ${PROJECT} \ + --member="serviceAccount:${CUSTOMER_SA}" \ + --role="roles/bigquery.jobUser" --quiet +``` + +**"Could not find dataset" or dataset errors (DATASET):** +> The BigQuery dataset needs to be created or permissions updated. Can I do that? +```bash +bq mk --project_id=${PROJECT} --dataset --location=${REGION} ${DATASET} +bq update --project_id=${PROJECT} --source /dev/stdin ${DATASET} << EOF +{"access": [ + {"role": "WRITER", "userByEmail": "${CUSTOMER_SA}"}, + {"role": "OWNER", "specialGroup": "projectOwners"}, + {"role": "WRITER", "specialGroup": "projectWriters"}, + {"role": "READER", "specialGroup": "projectReaders"} +]} +EOF +``` + +**"free tier" / "Streaming insert is not allowed":** +> BigQuery streaming requires billing enabled on your GCP project. Can I link a billing account? +```bash +gcloud billing accounts list +gcloud billing projects link ${PROJECT} --billing-account=${BILLING_ACCOUNT} +``` +Note: billing propagation to BigQuery can take up to 15 minutes. + +After fixing, re-validate. If still failing (e.g., IAM propagation), inform the user and offer to retry. + +### Show commands (manual) + +Show the exact gcloud/bq commands they need to run, with their specific values filled in: + +``` +Here's what needs to be configured on your GCP project: + +# 1. Grant Confidence access to your service account +gcloud iam service-accounts add-iam-policy-binding \ + \ + --project= \ + --member="serviceAccount:account-@spotify-confidence.iam.gserviceaccount.com" \ + --role="roles/iam.workloadIdentityUser" + +gcloud iam service-accounts add-iam-policy-binding \ + \ + --project= \ + --member="serviceAccount:account-@spotify-confidence.iam.gserviceaccount.com" \ + --role="roles/iam.serviceAccountTokenCreator" + +# 2. Grant BigQuery Job User +gcloud projects add-iam-policy-binding \ + --member="serviceAccount:" \ + --role="roles/bigquery.jobUser" + +# 3. Enable billing (if not already) +gcloud billing projects link --billing-account= + +Run these commands, then let me know and I'll retry validation. +``` + +If `configurationResponse` contains available options (schemas, roles), present these as choices to help the user. + +--- + +## Step 6: Create warehouse + +**IMPORTANT:** The body is the data warehouse object directly (gRPC transcoding `body: "data_warehouse"`), NOT wrapped in a `dataWarehouse` key. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/dataWarehouses" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "config": { + "bigQueryConfig": { + "serviceAccount": "", + "project": "", + "dataset": "" + } + } + }' +``` + +Save the returned `name` (e.g., `dataWarehouses/...`) for reference. + +--- + +## Step 7: Create connectors + +Create both connectors: + +### Flag Applied Connection (assignment data -> warehouse) + +**IMPORTANT:** The body is the connection object directly (gRPC transcoding `body: "flag_applied_connection"`), NOT wrapped. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev/v1/flagAppliedConnections" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "bigQuery": { + "bigQueryConfig": { + "serviceAccount": "", + "project": "", + "dataset": "" + }, + "table": "assignments" + } + }' +``` + +### Event Connection (events -> warehouse) + +**IMPORTANT:** The body is the connection object directly (gRPC transcoding `body: "event_connection"`), NOT wrapped. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev/v1/eventConnections" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "bigQuery": { + "bigQueryConfig": { + "serviceAccount": "", + "project": "", + "dataset": "" + }, + "tablePrefix": "events_" + } + }' +``` + +--- + +## Step 8: Assignment table + +Create an assignment table so Confidence can analyze experiment assignments. + +**IMPORTANT:** The body is the assignment table object directly (gRPC transcoding `body: "assignment_table"`), NOT wrapped in an `assignmentTable` key. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/assignmentTables" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "displayName": "Assignments", + "sql": "SELECT targeting_key, rule, assignment_id, assignment_time FROM `..assignments`", + "entityColumn": { "name": "targeting_key" }, + "timestampColumn": { "name": "assignment_time" }, + "exposureKeyColumn": { "name": "rule" }, + "variantKeyColumn": { "name": "assignment_id" }, + "dataDeliveredUntilUpdateStrategyConfig": { + "strategy": "AUTOMATIC", + "automaticUpdateConfig": { + "commitDelay": "300s" + } + } + }' +``` + +--- + +## Step 9: Verify data pipeline + +Verify both connectors by generating test data and checking it lands in the warehouse. + +### 9a. Get a client secret for testing + +The resolver and events APIs require a **client secret** (not a Bearer token). + +1. **List the user's clients** and show them: + ```bash + curl -s "https://iam.${REGION}.confidence.dev/v1/clients" -H "Authorization: Bearer $TOKEN" + ``` + Display each client with its name and last-seen time. If only one client exists, confirm it with the user. If multiple, let them pick. + +2. **Ask the user** if they have a client secret or want a new one: + > I'll use **** for the pipeline test. Do you have the client secret, or should I create a new credential? + +3. If the user wants a new credential, create one on the chosen client: + ```bash + curl -s -X POST "https://iam.${REGION}.confidence.dev/v1//credentials" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"display_name": "Pipeline Test"}' + ``` + Save the secret to a temp file for pipeline use. **Never print the secret to the user's terminal.** + +### 9b. Verify flag assignments + +Resolve a flag to generate assignment data (use an existing flag + client secret): +```bash +curl -s -X POST "https://resolver.${REGION}.confidence.dev/v1/flags:resolve" \ + -H "Content-Type: application/json" \ + -d '{ + "flags": ["flags/"], + "evaluation_context": {"targeting_key": "warehouse-verify-user"}, + "client_secret": "", + "apply": true + }' +``` + +If no flags exist yet, tell the user: +> No flags to test with. Run `/onboard-confidence setup-wizard` first to create a flag, then come back. + +### 9c. Verify events + +First check for an event definition to use: +```bash +curl -s "https://events.${REGION}.confidence.dev/v1/eventDefinitions" \ + -H "Authorization: Bearer $TOKEN" +``` + +If no event definitions exist, create one with a schema: +```bash +curl -s -X POST "https://events.${REGION}.confidence.dev/v1/eventDefinitions?event_definition_id=test-event" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"schema": {"action": {"stringSchema": {}}, "page": {"stringSchema": {}}}}' +``` + +If an event definition exists but has an empty schema, update it so payload data flows through: +```bash +curl -s -X PATCH "https://events.${REGION}.confidence.dev/v1/eventDefinitions/" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"schema": {"action": {"stringSchema": {}}, "page": {"stringSchema": {}}}}' +``` + +Then publish test events (uses client secret, NOT Bearer token): +```bash +NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +curl -s -X POST "https://events.${REGION}.confidence.dev/v1/events:publish" \ + -H "Content-Type: application/json" \ + -d '{ + "client_secret": "", + "events": [ + { + "event_definition": "eventDefinitions/", + "payload": {"action": "clicked_button", "page": "homepage"}, + "event_time": "'$NOW'" + } + ], + "send_time": "'$NOW'" + }' +``` + +Check response: `{"errors": []}` means success. If `EVENT_DEFINITION_NOT_FOUND`, the definition doesn't exist. If `EVENT_SCHEMA_VALIDATION_FAILED`, the payload doesn't match the schema. + +### 9d. Check data in BigQuery + +Ask the user: "Want me to check the data, or show you the queries?" + +If user has `bq` CLI: +```bash +echo "=== ASSIGNMENTS ===" && \ +bq query --project_id=${PROJECT} --use_legacy_sql=false \ + 'SELECT targeting_key, rule, assignment_id, assignment_time + FROM `${PROJECT}.${DATASET}.assignments` + ORDER BY assignment_time DESC LIMIT 5' && \ +echo "=== EVENTS ===" && \ +bq query --project_id=${PROJECT} --use_legacy_sql=false \ + 'SELECT * FROM `${PROJECT}.${DATASET}.events_*` + ORDER BY _event_time DESC LIMIT 5' +``` + +If no `bq`, show queries for BigQuery console. + +**Show results:** +``` + ● Assignments: rows -- data flowing + -> () + ● Events: rows -- data flowing + on () +``` + +**If no rows after a few seconds**, tell the user: +> Data delivery can take up to a few minutes depending on your warehouse. Check again shortly, or verify in your BigQuery console. + +--- + +## Step 10: Done + +``` +═══════════════════════════════════════════════════════════════ + Data Warehouse Connected & Verified +═══════════════════════════════════════════════════════════════ + + Warehouse: BigQuery () + Dataset: + Connectors: + ● Flag assignments -> assignments table (verified) + ● Events -> events_* tables (running) + Assignment: + ● Assignment table configured (auto-updating) + + Flag assignment and event data is flowing to your + warehouse. Experiment analysis is ready. + +═══════════════════════════════════════════════════════════════ +``` + +--- + +## API Reference (agent-internal -- do NOT show to user) + +### Base URLs + +All APIs require **region-specific URLs**. Extract region from the JWT token claim `https://confidence.dev/region` (value: `EU` or `US`), lowercase it, and use as prefix. + +``` +IAM_API: https://iam.${region}.confidence.dev/v1 +RESOLVER_API: https://resolver.${region}.confidence.dev/v1 +EVENTS_API: https://events.${region}.confidence.dev/v1 +CONNECTORS_API: https://connectors.${region}.confidence.dev/v1 +METRICS_API: https://metrics.${region}.confidence.dev/v1 +``` + +### Endpoints + +**Validate warehouse config (Bearer token):** +``` +POST ${METRICS_API}/dataWarehouseConfig:validate +Body: { "bigQueryConfig": {...} } +-> { "validation": [...], "successful": bool, "configurationResponse": {...} } +``` + +**Check warehouse exists (Bearer token):** +``` +GET ${METRICS_API}/dataWarehouses:exists +-> { "exists": bool } +``` + +**Create data warehouse (Bearer token, body: "data_warehouse"):** +``` +POST ${METRICS_API}/dataWarehouses +Body (direct object): { "config": { "bigQueryConfig": {...} } } +-> DataWarehouse object +``` + +**Create flag applied connection (Bearer token, body: "flag_applied_connection"):** +``` +POST ${CONNECTORS_API}/flagAppliedConnections +Body (direct object): { "bigQuery": { "bigQueryConfig": {...}, "table": "assignments" } } +-> FlagAppliedConnection object +``` + +**Create event connection (Bearer token, body: "event_connection"):** +``` +POST ${CONNECTORS_API}/eventConnections +Body (direct object): { "bigQuery": { "bigQueryConfig": {...}, "tablePrefix": "events_" } } +-> EventConnection object +``` + +**Create assignment table (Bearer token, body: "assignment_table"):** +``` +POST ${METRICS_API}/assignmentTables +Body (direct object): { "displayName": str, "sql": str, "entityColumn": {...}, "timestampColumn": {...}, "exposureKeyColumn": {...}, "variantKeyColumn": {...}, "dataDeliveredUntilUpdateStrategyConfig": {...} } +-> AssignmentTable object +``` + +**List clients (Bearer token):** +``` +GET ${IAM_API}/clients +-> { "clients": [...], "nextPageToken": string } +``` + +**Create client credential (Bearer token, body: "client_credential"):** +``` +POST ${IAM_API}/${clientName}/credentials +Body (direct object): { "display_name": string } +-> { "name": "...", "clientSecret": { "secret": string }, ... } + NOTE: secret only returned once on creation +``` + +**Resolve flags (client secret -- NOT Bearer token):** +``` +POST ${RESOLVER_API}/flags:resolve +Body: { "flags": ["flags/"], "evaluationContext": {...}, "clientSecret": string, "apply": bool } +-> { "resolvedFlags": [...] } +``` + +**Publish events (client secret -- NOT Bearer token):** +``` +POST ${EVENTS_API}/events:publish +Body: { "client_secret": string, "events": [...], "send_time": "ISO8601" } +-> { "errors": [...] } +``` + +--- + +## Error Handling Reference (agent-internal) + +### Common HTTP errors + +| Status | Meaning | Recovery | +|--------|---------|----------| +| 400 | Validation error | Parse `.message`, show plain English, re-collect invalid field | +| 401 | Invalid/expired token | Re-trigger Auth0 login | +| 403 | Insufficient permissions | Explain needed role/permission | +| 404 | Resource not found | Check account/resource exists | +| 409 | Conflict (already exists) | Resource already created | +| 429 | Rate limited | Wait briefly and retry | +| 500+ | Server error | Inform user, suggest retry | + +### Sandbox note + +All `curl`, `open`, and `python3` commands that access external hosts (`auth.confidence.dev`, `metrics.confidence.dev`, `connectors.confidence.dev`, etc.) require `dangerouslyDisableSandbox: true`. On first occurrence, briefly explain to the user that network access outside the sandbox is needed for API calls. diff --git a/skills/setup-warehouse-databricks/SKILL.md b/skills/setup-warehouse-databricks/SKILL.md new file mode 100644 index 0000000..49a6e15 --- /dev/null +++ b/skills/setup-warehouse-databricks/SKILL.md @@ -0,0 +1,860 @@ +--- +description: Set up Databricks as a data warehouse for Confidence. Use when the user chose Databricks for warehouse setup. +--- + +# Setup Warehouse: Databricks + +Configure Databricks as the data warehouse for Confidence experimentation analytics. This skill handles the full end-to-end setup: collect Databricks connection details, set up an S3 staging bucket with IAM, configure the schema, create the warehouse, set up connectors, create the assignment table, and verify the pipeline. + +--- + +## Authentication + +**Browser-based Auth0 login.** The skill opens a browser for Auth0 login (Google, email/password, SSO) and captures the token automatically. The user never touches a token. + +### Session-only token management + +The token is kept in the current session only and is never saved to disk. If the session ends or the token expires, the skill will open your browser to log in again. + +On every sub-command start, check if the `TOKEN` variable is set and not expired: + +```bash +if [ -n "$TOKEN" ]; then + PAYLOAD=$(echo "$TOKEN" | cut -d. -f2) + EXP=$(echo "$PAYLOAD" | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d.get('exp', 0)) +") + NOW=$(date +%s) + if [ "$EXP" -gt "$NOW" ]; then + echo "VALID" + else + echo "EXPIRED" + unset TOKEN + fi +fi +``` + +If `TOKEN` is unset or expired, run the Auth0 login flow with the **regular client ID** (`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w`) and the user's `organization` parameter. Store the result in the `TOKEN` shell variable only. **NEVER write the token to disk. NEVER reference `~/.confidence/`.** + +### Auth script + +Write the following to `$TMPDIR/confidence_auth.py` with CLIENT_ID=`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w` and ORGANIZATION from the token. Run with `python3 $TMPDIR/confidence_auth.py`. Outputs `TOKEN:` on success. + +```python +import http.server, urllib.parse, json, sys, subprocess, hashlib, base64, secrets, string + +code_verifier = ''.join(secrets.choice(string.ascii_letters + string.digits + '-._~') for _ in range(43)) +code_challenge = base64.urlsafe_b64encode(hashlib.sha256(code_verifier.encode()).digest()).rstrip(b'=').decode() + +port = 8084 +CLIENT_ID = '' +ORGANIZATION = '' +REDIRECT_URI = f'http://localhost:{port}/callback' +auth_code = None +error = None + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + global auth_code, error + q = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query) + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.end_headers() + if 'code' in q: + auth_code = q['code'][0] + self.wfile.write(b'

Login successful!

You can close this tab.

') + else: + error = q.get('error', ['unknown'])[0] + self.wfile.write(b'

Login failed

Please try again.

') + def log_message(self, format, *args): + pass + +params = { + 'client_id': CLIENT_ID, + 'redirect_uri': REDIRECT_URI, + 'response_type': 'code', + 'scope': 'openid profile email offline_access', + 'audience': 'https://confidence.dev/', + 'code_challenge': code_challenge, + 'code_challenge_method': 'S256', +} +if ORGANIZATION: + params['organization'] = ORGANIZATION + +authorize_url = 'https://auth.confidence.dev/authorize?' + urllib.parse.urlencode(params) +subprocess.Popen(['open', authorize_url]) +print('WAITING_FOR_LOGIN', flush=True) + +server = http.server.HTTPServer(('127.0.0.1', port), Handler) +server.timeout = 120 +while auth_code is None and error is None: + server.handle_request() +server.server_close() + +if error: + print(f'AUTH_ERROR:{error}', flush=True) + sys.exit(1) + +import urllib.request +token_data = json.dumps({ + 'grant_type': 'authorization_code', + 'client_id': CLIENT_ID, + 'code': auth_code, + 'redirect_uri': REDIRECT_URI, + 'code_verifier': code_verifier +}).encode() +req = urllib.request.Request( + 'https://auth.confidence.dev/oauth/token', + data=token_data, + headers={'Content-Type': 'application/json'} +) +try: + with urllib.request.urlopen(req) as resp: + token_response = json.loads(resp.read()) + print(f'TOKEN:{token_response["access_token"]}', flush=True) +except Exception as e: + print(f'TOKEN_ERROR:{e}', flush=True) + sys.exit(1) +``` + +### Extract region from token + +```bash +REGION=$(echo "$PAYLOAD" | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d.get('https://confidence.dev/region', 'EU')) +") +``` + +Then use `${REGION,,}` (lowercase) for URL prefix: `iam.eu.confidence.dev`, `metrics.eu.confidence.dev`, etc. + +### Common notes + +- Port is fixed at **8084** (must match Auth0 Allowed Callback URLs) +- If port 8084 is busy: `lsof -ti:8084 | xargs kill -9 2>/dev/null` +- All network commands require `dangerouslyDisableSandbox: true` +- Never show the token value to the user +- Always use region-specific URLs (e.g., `iam.eu.confidence.dev` not `iam.confidence.dev`) + +### Important: gRPC-REST transcoding rules + +The Confidence APIs use gRPC with REST transcoding. The `body` field in the proto HTTP binding determines the JSON structure: + +- **`body: "data_warehouse"`** -> send the data warehouse object directly: `{"config": {...}}` +- **`body: "flag_applied_connection"`** -> send the connection object directly: `{"databricks": {...}}` +- **`body: "event_connection"`** -> send the connection object directly: `{"databricks": {...}}` +- **`body: "assignment_table"`** -> send the assignment table object directly: `{"displayName": "...", "sql": "...", ...}` +- **`body: "*"`** -> send the full request message + +The body is the object directly, NOT wrapped in an outer key. + +Fields NOT in the body (like `flag_id`, `parent`) become **query parameters**. + +**Field names are `snake_case`** in requests. Responses may use `camelCase`. + +--- + +## User-Facing Communication Rules + +**NEVER expose internal technical details to the user.** + +- Do NOT show raw JSON request/response bodies in conversation +- Do NOT show Auth0 configuration details, token values, or OAuth internals +- DO show human-readable status updates: "Opening browser for login...", "Creating your warehouse...", "Connectors configured!" +- DO describe results in plain English +- The agent handles all auth/API complexity silently + +**Step Tracker:** Display a visual step tracker at every phase transition. Update and re-display it each time you move to a new step. + +--- + +## Step Tracker + +Display at START and after EACH step completes (updating status): + +``` +───── Setup Warehouse (Databricks) ──────────────────────── + [1] Choose warehouse ● done + [2] Workspace URL ○ pending + [3] SQL Warehouse ID ○ pending + [4] Service principal ○ pending + [5] AWS account & CLI ○ pending + [6] S3 bucket ○ pending + [7] IAM role ○ pending + [8] Databricks schema ○ pending + [9] Create warehouse ○ pending + [10] Create connectors ○ pending + [11] Assignment table ○ pending + [12] Verify pipeline ○ pending + [13] Done ○ pending +──────────────────────────────────────────────────────────── +``` + +Use `●` for completed, `▶` for in-progress, `○` for pending. Re-display the full tracker after every step transition. + +--- + +## Step 1: Choose warehouse (already done) + +The user has already chosen Databricks. Mark step 1 as done. + +--- + +## Overview + +Before collecting details, explain the full picture so the user knows what they need: + +> Setting up Databricks with Confidence requires three things: +> +> 1. **A Databricks workspace** -- you need admin access to create a service principal (a robot account) +> 2. **An AWS account with an S3 bucket** -- Confidence needs this as a staging area for loading data into Databricks. This is required even if your Databricks runs on GCP or Azure +> 3. **A schema in Databricks** -- a place for Confidence to create tables (e.g., `confidence`) +> +> **How data flows:** +> Confidence collects your flag assignments and events internally, then writes parquet files to an S3 bucket you provide, and finally loads them into Databricks tables. This happens in batches every ~5 minutes. +> +> ``` +> Confidence (collects data) -> S3 bucket (staging) -> Databricks (tables) +> ``` +> +> **Don't have an AWS account?** You'll need one for the S3 staging bucket. AWS free tier works fine. I can set it up for you if you have the `aws` CLI, or walk you through the AWS Console. + +Then collect the details **one at a time**. After each answer, confirm it before moving to the next. Don't dump all questions at once. + +--- + +## Step 2: Workspace URL (Part 1: Databricks connection) + +Ask the user: +> What's your Databricks workspace URL? Just paste the URL from your browser address bar. + +Extract the hostname from whatever they paste (strip `https://`, trailing paths, query params). Valid examples: +- `dbc-a1b2c3d4-e5f6.cloud.databricks.com` +- `1234567890.7.gcp.databricks.com` +- `adb-1234567890.12.azuredatabricks.net` + +Confirm: "Got it -- your Databricks workspace is at ``." + +--- + +## Step 3: SQL Warehouse ID + +Ask the user: +> I need a SQL Warehouse ID. Here's how to find it: +> 1. In Databricks, click **SQL Warehouses** in the left sidebar +> 2. Click on a warehouse name +> 3. Open the **Connection details** tab +> 4. Copy the **HTTP Path** -- the ID is the last part after `/sql/1.0/warehouses/` +> +> It looks like a hex string, e.g., `ccf7028466008a3c` +> +> **Don't have a SQL Warehouse?** Click **Create SQL Warehouse** -> name it "Confidence" -> pick **Serverless**, size **Small** -> **Create**. Then copy the ID. + +Confirm: "Using warehouse ``." + +--- + +## Step 4: Service principal + +Ask the user: +> I need a service principal -- this is a robot account that Confidence uses to connect to Databricks. +> +> **To create one:** +> 1. Click the **gear icon** at the top of Databricks -> **Settings** +> 2. Under **Identity and access**, click **Service principals** +> 3. Click **Add service principal -> Add new** +> 4. Name it "Confidence" -> **Add** +> 5. Click into the new service principal +> 6. Copy the **Application ID** (a UUID like `85cc292a-c1d2-...`) +> 7. Go to the **Secrets** tab -> **Generate secret** +> 8. Copy both the **Secret** (shown only once!) and the **Client ID** +> +> Paste the **Client ID** and **Secret** here. + +If the user says they can't access Settings or service principals: +> You need workspace admin access for this step. Ask your Databricks admin to: +> 1. Create a service principal named "Confidence" +> 2. Generate a secret for it +> 3. Send you the Client ID and Secret + +Confirm: "Service principal configured." + +--- + +## Step 5: AWS account & CLI (Part 2: S3 staging bucket) + +Explain why: +> Confidence writes parquet files to an S3 bucket, then Databricks loads them via COPY INTO. Think of it as a mailbox -- Confidence drops files there, and Databricks picks them up. **This is required even if your Databricks runs on GCP or Azure.** +> +> You need an AWS account for this. If you don't have one, I can help you set one up. + +Ask the user: +> Do you have the `aws` CLI set up, or would you prefer manual steps? +> 1. Set it up for me (requires `aws` CLI) +> 2. Show me the steps + +**If the user picks 1 (aws CLI):** + +First check: `which aws`. If not found, offer to install: `brew install awscli` (macOS) or guide them to https://aws.amazon.com/cli/. + +Then check they're logged in: `aws sts get-caller-identity`. If not, tell them: +> Run `aws configure` or `aws sso login` to log into your AWS account first. + +If `aws` CLI is not configured, the skill should: +1. Open the AWS console login: `open "https://console.aws.amazon.com"` +2. Guide user to create access key: **click your name top right -> Security credentials -> Access keys -> Create access key** +3. Write the credentials directly to `~/.aws/credentials` and `~/.aws/config` (don't use interactive `aws configure`) + +--- + +## Step 6: S3 bucket + +Extract the Confidence service account and its numeric unique ID (required for AWS trust policy): +```bash +ACCOUNT_ID=$(echo "$TOKEN" | cut -d. -f2 | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d['https://confidence.dev/account_name'].split('/')[-1]) +") +CONFIDENCE_SA="account-${ACCOUNT_ID}@spotify-confidence.iam.gserviceaccount.com" +``` + +Ask the user for a bucket name (suggest `confidence-staging-`) and region (suggest `eu-west-1`). + +**If using aws CLI:** + +```bash +# 1. Create S3 bucket +aws s3api create-bucket --bucket ${BUCKET_NAME} --region ${AWS_REGION} \ + --create-bucket-configuration LocationConstraint=${AWS_REGION} +``` + +**If using manual steps:** + +> Go to **AWS Console** (https://console.aws.amazon.com) -> **S3 -> Create bucket**. +> - Name: something like `confidence-staging-` (must be globally unique) +> - Region: pick the same region as your Databricks workspace (e.g., `eu-west-1` for EU) +> - Leave all other settings as default -> **Create bucket** +> +> If you already have a bucket you want to reuse, that works too -- just give me the name. + +--- + +## Step 7: IAM role + +Get the Confidence service account numeric unique ID: +```bash +# CRITICAL: AWS trust policy needs the NUMERIC unique ID, not the email. +# The email won't work -- AWS requires accounts.google.com:sub which is the numeric ID. +SA_UNIQUE_ID=$(gcloud iam service-accounts describe ${CONFIDENCE_SA} \ + --project=spotify-confidence --format="value(uniqueId)") +``` + +If `gcloud` can't access `spotify-confidence` project, the user needs to contact Confidence support to get the numeric service account ID. + +**If using aws CLI:** + +```bash +# 1. Create the trust policy file +# IMPORTANT: Use accounts.google.com:sub with the NUMERIC service account ID. +# Using :email will fail with "MalformedPolicyDocument". +# Using the email string as :sub will fail at runtime with "Not authorized to perform sts:AssumeRoleWithWebIdentity". +cat > $TMPDIR/trust-policy.json << EOF +{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Federated": "accounts.google.com"}, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "accounts.google.com:sub": "${SA_UNIQUE_ID}" + } + } + }] +} +EOF + +# 2. Create IAM role +aws iam create-role --role-name confidence-databricks-staging \ + --assume-role-policy-document file://$TMPDIR/trust-policy.json + +# 3. Create and attach S3 access policy +cat > $TMPDIR/s3-policy.json << EOF +{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:PutObject", "s3:GetObject", "s3:DeleteObject", "s3:ListBucket"], + "Resource": [ + "arn:aws:s3:::${BUCKET_NAME}", + "arn:aws:s3:::${BUCKET_NAME}/*" + ] + }] +} +EOF +aws iam put-role-policy --role-name confidence-databricks-staging \ + --policy-name S3Access --policy-document file://$TMPDIR/s3-policy.json + +# 4. Get the role ARN +ROLE_ARN=$(aws iam get-role --role-name confidence-databricks-staging --query 'Role.Arn' --output text) +echo "ROLE_ARN: $ROLE_ARN" +``` + +After completion, show the user: +> AWS setup complete! +> - Bucket: `` in `` +> - Role: `` +> +> Continuing with connector setup... + +**If using manual steps:** + +> Go to **AWS Console -> IAM -> Roles -> Create role**. +> - Trusted entity: **Web identity** +> - Identity provider: select **accounts.google.com** (add it first if not listed under Identity providers) +> - Audience: `account-@spotify-confidence.iam.gserviceaccount.com` +> (the skill should compute the account ID from the JWT token and fill this in for the user) +> - Click **Next** -> **Create policy** -> JSON tab -> paste this: +> ```json +> { +> "Version": "2012-10-17", +> "Statement": [{ +> "Effect": "Allow", +> "Action": ["s3:PutObject", "s3:GetObject", "s3:DeleteObject", "s3:ListBucket"], +> "Resource": ["arn:aws:s3:::", "arn:aws:s3:::/*"] +> }] +> } +> ``` +> - Attach the policy -> name the role (e.g., `confidence-databricks-staging`) -> **Create role** +> - Copy the **Role ARN** (looks like `arn:aws:iam::123456789012:role/confidence-databricks-staging`) +> +> **If you get "Not authorized to perform sts:AssumeRoleWithWebIdentity" later:** the trust policy is wrong -- the Confidence service account email must exactly match what's in the role's trust policy. + +Collect the **AWS Region** and **IAM Role ARN** from the user. + +--- + +## Step 8: Databricks schema (Part 3: schema) + +Ask the user: +> Last thing -- where should Confidence create its tables in Databricks? I need a schema name. +> The default is `confidence`. If you already have a schema you'd like to use, let me know. + +Then check if the schema exists and the service principal has access. Generate the SQL and **copy to clipboard**: + +> I'll set up the schema and permissions. Here's what I'm running -- copied to your clipboard. Paste it in the **Databricks SQL Editor** (left sidebar -> SQL Editor) and run it. + +For workspaces **without Unity Catalog** (hive_metastore): +```sql +CREATE SCHEMA IF NOT EXISTS confidence; +GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence TO ``; +``` + +For workspaces **with Unity Catalog**: +```sql +CREATE CATALOG IF NOT EXISTS confidence; +CREATE SCHEMA IF NOT EXISTS confidence.confidence; +GRANT USE CATALOG ON CATALOG confidence TO ``; +GRANT USE SCHEMA, CREATE TABLE ON SCHEMA confidence.confidence TO ``; +``` + +**How to tell which one:** If the user sees **Catalog** in the Databricks left sidebar, they have Unity Catalog. If they only see **Data**, they're on hive_metastore. + +After the user runs it, confirm: "Schema ready. Moving on to create the warehouse." + +--- + +## Step 9: Create warehouse + +**NOTE:** The validate endpoint does NOT support Databricks (returns "configuration must be set" for any field name variant). Skip validation and proceed directly to create. Tell the user: +> Pre-validation isn't available yet for Databricks. I'll create the warehouse now and we'll verify the connection works end-to-end in the pipeline test step. + +**IMPORTANT:** The body is the data warehouse object directly (gRPC transcoding `body: "data_warehouse"`), NOT wrapped in a `dataWarehouse` key. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/dataWarehouses" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "config": { + "dataBricksConfig": { + "host": "", + "warehouseId": "", + "clientId": "", + "clientSecret": "", + "schema": "", + "s3BucketConfig": { + "bucket": "", + "region": "", + "roleArn": "" + } + } + } + }' +``` + +Save the returned `name` (e.g., `dataWarehouses/...`) for reference. + +--- + +## Step 10: Create connectors + +Create both connectors. Databricks connectors use a nested `connectionConfig` for auth, require an **S3 staging bucket** for batch writes, and `batchFileConfig`. + +### Flag Applied Connection (assignment data -> warehouse) + +**IMPORTANT:** The body is the connection object directly (gRPC transcoding `body: "flag_applied_connection"`), NOT wrapped. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev/v1/flagAppliedConnections" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "databricks": { + "databricksConfig": { + "connectionConfig": { + "host": "", + "warehouseId": "", + "clientId": "", + "clientSecret": "" + }, + "schema": "", + "s3BucketConfig": { + "bucket": "", + "region": "", + "roleArn": "" + }, + "batchFileConfig": { + "maxFileAge": "300s" + } + }, + "table": "assignments" + } + }' +``` + +### Event Connection (events -> warehouse) + +**IMPORTANT:** The body is the connection object directly (gRPC transcoding `body: "event_connection"`), NOT wrapped. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev/v1/eventConnections" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "databricks": { + "databricksConfig": { + "connectionConfig": { + "host": "", + "warehouseId": "", + "clientId": "", + "clientSecret": "" + }, + "schema": "", + "s3BucketConfig": { + "bucket": "", + "region": "", + "roleArn": "" + }, + "batchFileConfig": { + "maxFileAge": "300s" + } + }, + "tablePrefix": "events_" + } + }' +``` + +--- + +## Step 11: Assignment table + +Create an assignment table so Confidence can analyze experiment assignments. + +**IMPORTANT:** The body is the assignment table object directly (gRPC transcoding `body: "assignment_table"`), NOT wrapped in an `assignmentTable` key. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/assignmentTables" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "displayName": "Assignments", + "sql": "SELECT targeting_key, rule, assignment_id, assignment_time FROM .assignments", + "entityColumn": { "name": "targeting_key" }, + "timestampColumn": { "name": "assignment_time" }, + "exposureKeyColumn": { "name": "rule" }, + "variantKeyColumn": { "name": "assignment_id" }, + "dataDeliveredUntilUpdateStrategyConfig": { + "strategy": "AUTOMATIC", + "automaticUpdateConfig": { + "commitDelay": "300s" + } + } + }' +``` + +--- + +## Step 12: Verify data pipeline + +Verify both connectors by generating test data and checking it lands in the warehouse. + +### 12a. Get a client secret for testing + +The resolver and events APIs require a **client secret** (not a Bearer token). + +1. **List the user's clients** and show them: + ```bash + curl -s "https://iam.${REGION}.confidence.dev/v1/clients" -H "Authorization: Bearer $TOKEN" + ``` + Display each client with its name and last-seen time. If only one client exists, confirm it with the user. If multiple, let them pick. + +2. **Ask the user** if they have a client secret or want a new one: + > I'll use **** for the pipeline test. Do you have the client secret, or should I create a new credential? + +3. If the user wants a new credential, create one on the chosen client: + ```bash + curl -s -X POST "https://iam.${REGION}.confidence.dev/v1//credentials" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"display_name": "Pipeline Test"}' + ``` + Save the secret to a temp file for pipeline use. **Never print the secret to the user's terminal.** + +### 12b. Verify flag assignments + +Resolve a flag to generate assignment data (use an existing flag + client secret): +```bash +curl -s -X POST "https://resolver.${REGION}.confidence.dev/v1/flags:resolve" \ + -H "Content-Type: application/json" \ + -d '{ + "flags": ["flags/"], + "evaluation_context": {"targeting_key": "warehouse-verify-user"}, + "client_secret": "", + "apply": true + }' +``` + +If no flags exist yet, tell the user: +> No flags to test with. Run `/onboard-confidence setup-wizard` first to create a flag, then come back. + +### 12c. Verify events + +First check for an event definition to use: +```bash +curl -s "https://events.${REGION}.confidence.dev/v1/eventDefinitions" \ + -H "Authorization: Bearer $TOKEN" +``` + +If no event definitions exist, create one with a schema: +```bash +curl -s -X POST "https://events.${REGION}.confidence.dev/v1/eventDefinitions?event_definition_id=test-event" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"schema": {"action": {"stringSchema": {}}, "page": {"stringSchema": {}}}}' +``` + +If an event definition exists but has an empty schema, update it so payload data flows through: +```bash +curl -s -X PATCH "https://events.${REGION}.confidence.dev/v1/eventDefinitions/" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"schema": {"action": {"stringSchema": {}}, "page": {"stringSchema": {}}}}' +``` + +Then publish test events (uses client secret, NOT Bearer token): +```bash +NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +curl -s -X POST "https://events.${REGION}.confidence.dev/v1/events:publish" \ + -H "Content-Type: application/json" \ + -d '{ + "client_secret": "", + "events": [ + { + "event_definition": "eventDefinitions/", + "payload": {"action": "clicked_button", "page": "homepage"}, + "event_time": "'$NOW'" + } + ], + "send_time": "'$NOW'" + }' +``` + +Check response: `{"errors": []}` means success. If `EVENT_DEFINITION_NOT_FOUND`, the definition doesn't exist. If `EVENT_SCHEMA_VALIDATION_FAILED`, the payload doesn't match the schema. + +### 12d. Check data in Databricks + +Use the Databricks SQL Statement API to query directly (the skill already has the service principal credentials): +```bash +DB_TOKEN=$(curl -s -X POST "https://${DATABRICKS_HOST}/oidc/v1/token" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=client_credentials&client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&scope=all-apis" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])") + +curl -s -X POST "https://${DATABRICKS_HOST}/api/2.0/sql/statements" \ + -H "Authorization: Bearer $DB_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "warehouse_id": "'${WAREHOUSE_ID}'", + "statement": "SELECT targeting_key, rule, assignment_id, assignment_time FROM '${SCHEMA}'.assignments ORDER BY assignment_time DESC LIMIT 5", + "wait_timeout": "30s" + }' +``` + +**IMPORTANT:** Data is batched every ~5 minutes. If the table doesn't exist yet, wait and retry. Tell the user: +> Data delivery takes about 5 minutes. Let me check again... + +If `TABLE_OR_VIEW_NOT_FOUND` after 10 minutes, check the connector logs for errors. + +**Show results:** +``` + ● Assignments: rows -- data flowing + -> () + ● Events: rows -- data flowing + on () +``` + +**If no rows after a few seconds**, tell the user: +> Data delivery can take up to 5 minutes for Databricks (batch processing). Check again shortly, or verify in the Databricks SQL Editor. + +--- + +## Step 13: Done + +``` +═══════════════════════════════════════════════════════════════ + Data Warehouse Connected & Verified +═══════════════════════════════════════════════════════════════ + + Warehouse: Databricks () + Schema: + S3 Bucket: () + Connectors: + ● Flag assignments -> assignments table (verified) + ● Events -> events_* tables (running) + Assignment: + ● Assignment table configured (auto-updating) + + Flag assignment and event data is flowing to your + warehouse. Experiment analysis is ready. + + Note: Data is delivered in ~5 minute batches. + +═══════════════════════════════════════════════════════════════ +``` + +--- + +## API Reference (agent-internal -- do NOT show to user) + +### Base URLs + +All APIs require **region-specific URLs**. Extract region from the JWT token claim `https://confidence.dev/region` (value: `EU` or `US`), lowercase it, and use as prefix. + +``` +IAM_API: https://iam.${region}.confidence.dev/v1 +RESOLVER_API: https://resolver.${region}.confidence.dev/v1 +EVENTS_API: https://events.${region}.confidence.dev/v1 +CONNECTORS_API: https://connectors.${region}.confidence.dev/v1 +METRICS_API: https://metrics.${region}.confidence.dev/v1 +``` + +### Endpoints + +**Validate warehouse config (Bearer token):** +``` +POST ${METRICS_API}/dataWarehouseConfig:validate +NOTE: Databricks is NOT supported by the validate endpoint. Skip validation and proceed to create. +``` + +**Check warehouse exists (Bearer token):** +``` +GET ${METRICS_API}/dataWarehouses:exists +-> { "exists": bool } +``` + +**Create data warehouse (Bearer token, body: "data_warehouse"):** +``` +POST ${METRICS_API}/dataWarehouses +Body (direct object): { "config": { "dataBricksConfig": { "host": str, "warehouseId": str, "clientId": str, "clientSecret": str, "schema": str, "s3BucketConfig": { "bucket": str, "region": str, "roleArn": str } } } } +-> DataWarehouse object +``` + +**Create flag applied connection (Bearer token, body: "flag_applied_connection"):** +``` +POST ${CONNECTORS_API}/flagAppliedConnections +Body (direct object): { "databricks": { "databricksConfig": { "connectionConfig": {...}, "schema": str, "s3BucketConfig": {...}, "batchFileConfig": {...} }, "table": "assignments" } } +-> FlagAppliedConnection object +``` + +**Create event connection (Bearer token, body: "event_connection"):** +``` +POST ${CONNECTORS_API}/eventConnections +Body (direct object): { "databricks": { "databricksConfig": { "connectionConfig": {...}, "schema": str, "s3BucketConfig": {...}, "batchFileConfig": {...} }, "tablePrefix": "events_" } } +-> EventConnection object +``` + +**Create assignment table (Bearer token, body: "assignment_table"):** +``` +POST ${METRICS_API}/assignmentTables +Body (direct object): { "displayName": str, "sql": str, "entityColumn": {...}, "timestampColumn": {...}, "exposureKeyColumn": {...}, "variantKeyColumn": {...}, "dataDeliveredUntilUpdateStrategyConfig": {...} } +-> AssignmentTable object +``` + +**List clients (Bearer token):** +``` +GET ${IAM_API}/clients +-> { "clients": [...], "nextPageToken": string } +``` + +**Create client credential (Bearer token, body: "client_credential"):** +``` +POST ${IAM_API}/${clientName}/credentials +Body (direct object): { "display_name": string } +-> { "name": "...", "clientSecret": { "secret": string }, ... } + NOTE: secret only returned once on creation +``` + +**Resolve flags (client secret -- NOT Bearer token):** +``` +POST ${RESOLVER_API}/flags:resolve +Body: { "flags": ["flags/"], "evaluationContext": {...}, "clientSecret": string, "apply": bool } +-> { "resolvedFlags": [...] } +``` + +**Publish events (client secret -- NOT Bearer token):** +``` +POST ${EVENTS_API}/events:publish +Body: { "client_secret": string, "events": [...], "send_time": "ISO8601" } +-> { "errors": [...] } +``` + +--- + +## Error Handling Reference (agent-internal) + +### Common HTTP errors + +| Status | Meaning | Recovery | +|--------|---------|----------| +| 400 | Validation error | Parse `.message`, show plain English, re-collect invalid field | +| 401 | Invalid/expired token | Re-trigger Auth0 login | +| 403 | Insufficient permissions | Explain needed role/permission | +| 404 | Resource not found | Check account/resource exists | +| 409 | Conflict (already exists) | Resource already created | +| 429 | Rate limited | Wait briefly and retry | +| 500+ | Server error | Inform user, suggest retry | + +### Sandbox note + +All `curl`, `open`, `python3`, `aws`, and `gcloud` commands that access external hosts (`auth.confidence.dev`, `metrics.confidence.dev`, `connectors.confidence.dev`, AWS APIs, etc.) require `dangerouslyDisableSandbox: true`. On first occurrence, briefly explain to the user that network access outside the sandbox is needed for API calls. diff --git a/skills/setup-warehouse-redshift/SKILL.md b/skills/setup-warehouse-redshift/SKILL.md new file mode 100644 index 0000000..53d8647 --- /dev/null +++ b/skills/setup-warehouse-redshift/SKILL.md @@ -0,0 +1,869 @@ +--- +description: Set up Redshift as a data warehouse for Confidence. Use when the user chose Redshift for warehouse setup. +--- + +# Setup Warehouse: Redshift + +Configure Redshift as the data warehouse for Confidence experimentation analytics. This skill handles the full end-to-end setup: set up or connect a Redshift cluster, create an S3 staging bucket with IAM, configure the schema, create the warehouse, set up connectors, create the assignment table, and verify the pipeline. + +--- + +## Authentication + +**Browser-based Auth0 login.** The skill opens a browser for Auth0 login (Google, email/password, SSO) and captures the token automatically. The user never touches a token. + +### Session-only token management + +The token is kept in the current session only and is never saved to disk. If the session ends or the token expires, the skill will open your browser to log in again. + +On every sub-command start, check if the `TOKEN` variable is set and not expired: + +```bash +if [ -n "$TOKEN" ]; then + PAYLOAD=$(echo "$TOKEN" | cut -d. -f2) + EXP=$(echo "$PAYLOAD" | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d.get('exp', 0)) +") + NOW=$(date +%s) + if [ "$EXP" -gt "$NOW" ]; then + echo "VALID" + else + echo "EXPIRED" + unset TOKEN + fi +fi +``` + +If `TOKEN` is unset or expired, run the Auth0 login flow with the **regular client ID** (`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w`) and the user's `organization` parameter. Store the result in the `TOKEN` shell variable only. **NEVER write the token to disk. NEVER reference `~/.confidence/`.** + +### Auth script + +Write the following to `$TMPDIR/confidence_auth.py` with CLIENT_ID=`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w` and ORGANIZATION from the token. Run with `python3 $TMPDIR/confidence_auth.py`. Outputs `TOKEN:` on success. + +```python +import http.server, urllib.parse, json, sys, subprocess, hashlib, base64, secrets, string + +code_verifier = ''.join(secrets.choice(string.ascii_letters + string.digits + '-._~') for _ in range(43)) +code_challenge = base64.urlsafe_b64encode(hashlib.sha256(code_verifier.encode()).digest()).rstrip(b'=').decode() + +port = 8084 +CLIENT_ID = '' +ORGANIZATION = '' +REDIRECT_URI = f'http://localhost:{port}/callback' +auth_code = None +error = None + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + global auth_code, error + q = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query) + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.end_headers() + if 'code' in q: + auth_code = q['code'][0] + self.wfile.write(b'

Login successful!

You can close this tab.

') + else: + error = q.get('error', ['unknown'])[0] + self.wfile.write(b'

Login failed

Please try again.

') + def log_message(self, format, *args): + pass + +params = { + 'client_id': CLIENT_ID, + 'redirect_uri': REDIRECT_URI, + 'response_type': 'code', + 'scope': 'openid profile email offline_access', + 'audience': 'https://confidence.dev/', + 'code_challenge': code_challenge, + 'code_challenge_method': 'S256', +} +if ORGANIZATION: + params['organization'] = ORGANIZATION + +authorize_url = 'https://auth.confidence.dev/authorize?' + urllib.parse.urlencode(params) +subprocess.Popen(['open', authorize_url]) +print('WAITING_FOR_LOGIN', flush=True) + +server = http.server.HTTPServer(('127.0.0.1', port), Handler) +server.timeout = 120 +while auth_code is None and error is None: + server.handle_request() +server.server_close() + +if error: + print(f'AUTH_ERROR:{error}', flush=True) + sys.exit(1) + +import urllib.request +token_data = json.dumps({ + 'grant_type': 'authorization_code', + 'client_id': CLIENT_ID, + 'code': auth_code, + 'redirect_uri': REDIRECT_URI, + 'code_verifier': code_verifier +}).encode() +req = urllib.request.Request( + 'https://auth.confidence.dev/oauth/token', + data=token_data, + headers={'Content-Type': 'application/json'} +) +try: + with urllib.request.urlopen(req) as resp: + token_response = json.loads(resp.read()) + print(f'TOKEN:{token_response["access_token"]}', flush=True) +except Exception as e: + print(f'TOKEN_ERROR:{e}', flush=True) + sys.exit(1) +``` + +### Extract region from token + +```bash +REGION=$(echo "$PAYLOAD" | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d.get('https://confidence.dev/region', 'EU')) +") +``` + +Then use `${REGION,,}` (lowercase) for URL prefix: `iam.eu.confidence.dev`, `metrics.eu.confidence.dev`, etc. + +### Common notes + +- Port is fixed at **8084** (must match Auth0 Allowed Callback URLs) +- If port 8084 is busy: `lsof -ti:8084 | xargs kill -9 2>/dev/null` +- All network commands require `dangerouslyDisableSandbox: true` +- Never show the token value to the user +- Always use region-specific URLs (e.g., `iam.eu.confidence.dev` not `iam.confidence.dev`) + +### Important: gRPC-REST transcoding rules + +The Confidence APIs use gRPC with REST transcoding. The `body` field in the proto HTTP binding determines the JSON structure: + +- **`body: "data_warehouse"`** -> send the data warehouse object directly: `{"config": {...}}` +- **`body: "flag_applied_connection"`** -> send the connection object directly: `{"redshift": {...}}` +- **`body: "event_connection"`** -> send the connection object directly: `{"redshift": {...}}` +- **`body: "assignment_table"`** -> send the assignment table object directly: `{"displayName": "...", "sql": "...", ...}` +- **`body: "*"`** -> send the full request message + +The body is the object directly, NOT wrapped in an outer key. + +Fields NOT in the body (like `flag_id`, `parent`) become **query parameters**. + +**Field names are `snake_case`** in requests. Responses may use `camelCase`. + +--- + +## User-Facing Communication Rules + +**NEVER expose internal technical details to the user.** + +- Do NOT show raw JSON request/response bodies in conversation +- Do NOT show Auth0 configuration details, token values, or OAuth internals +- DO show human-readable status updates: "Opening browser for login...", "Creating your warehouse...", "Connectors configured!" +- DO describe results in plain English +- The agent handles all auth/API complexity silently + +**Step Tracker:** Display a visual step tracker at every phase transition. Update and re-display it each time you move to a new step. + +--- + +## Step Tracker + +Display at START and after EACH step completes (updating status): + +``` +───── Setup Warehouse (Redshift) ────────────────────────── + [1] Choose warehouse ● done + [2] AWS account & CLI ○ pending + [3] Redshift cluster ○ pending + [4] S3 bucket ○ pending + [5] IAM role ○ pending + [6] Attach role ○ pending + [7] Schema & grants ○ pending + [8] Validate ○ pending + [9] Create warehouse ○ pending + [10] Create connectors ○ pending + [11] Assignment table ○ pending + [12] Verify pipeline ○ pending + [13] Done ○ pending +──────────────────────────────────────────────────────────── +``` + +Use `●` for completed, `▶` for in-progress, `○` for pending. Re-display the full tracker after every step transition. + +--- + +## Step 1: Choose warehouse (already done) + +The user has already chosen Redshift. Mark step 1 as done. + +--- + +## Overview + +Before collecting details, explain the full picture so the user knows what they're signing up for: + +> Setting up Redshift with Confidence requires an **AWS account**. Here's what we'll set up: +> +> 1. **A Redshift cluster** -- a data warehouse that stores your experiment data +> 2. **An S3 bucket** -- a staging area where Confidence drops data files before loading them into Redshift +> 3. **An IAM role** -- permissions that let Confidence write to S3 and load into Redshift +> 4. **A schema** -- a folder inside Redshift where Confidence creates its tables +> +> **How data flows:** +> ``` +> Confidence -> S3 bucket (staging) -> Redshift COPY -> your tables +> ``` +> +> I can set up everything automatically if you have the `aws` CLI, or walk you through the AWS Console step by step. +> +> **Don't have an AWS account?** You'll need one. I can open the signup page for you. AWS free tier covers S3, but Redshift clusters cost ~$0.25/hr while running. You can delete it after testing. +> +> **Important: Redshift Serverless won't work** -- Confidence needs a provisioned cluster. I'll make sure we create the right type. + +Ask the user: +> Do you have the `aws` CLI set up, or would you prefer manual steps? +> 1. Set it up for me (requires `aws` CLI) +> 2. Show me the steps + +--- + +## Step 2: AWS account & CLI + +**If the user picks 1 (aws CLI):** + +Check `which aws`. If not found: `brew install awscli` (macOS). +Check `aws sts get-caller-identity`. If not logged in, open the AWS console login (`open "https://console.aws.amazon.com"`), guide them to create access keys (**click name top right -> Security credentials -> Access keys -> Create**), then write the credentials to `~/.aws/credentials` and `~/.aws/config`. + +**If the user picks 2 (manual steps):** + +Walk them through the AWS Console for each subsequent step. Each step below includes both CLI and manual instructions. + +--- + +## Step 3: Redshift cluster + +Ask the user: +> Do you already have a Redshift cluster, or should I create one? + +If they have one: +> What's the cluster name? Go to **AWS Console -> Amazon Redshift -> Clusters**. The name is in the first column. + +If they need one, explain: +> I'll create a single-node Redshift cluster. This is a data warehouse -- like a powerful database optimized for analytics. +> - **Cost:** ~$0.25/hour while running. Delete it when you're done testing. +> - **Type:** `ra3.large` (cheapest option that supports single-node) +> - **Region:** `eu-west-1` (Europe) -- should match where your Confidence account is +> +> **Important:** Redshift Serverless won't work -- Confidence needs a provisioned cluster. I'll create the right type. + +Extract the account ID from the token: +```bash +ACCOUNT_ID=$(echo "$TOKEN" | cut -d. -f2 | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d['https://confidence.dev/account_name'].split('/')[-1]) +") +``` + +**If using aws CLI:** + +```bash +aws redshift create-cluster \ + --cluster-identifier confidence-redshift-${ACCOUNT_ID} \ + --cluster-type single-node \ + --node-type ra3.large \ + --master-username admin \ + --master-user-password '' \ + --db-name dev \ + --region eu-west-1 \ + --publicly-accessible +``` + +Wait for status `available` (takes ~1-2 minutes): +```bash +aws redshift wait cluster-available --cluster-identifier ${CLUSTER} --region ${AWS_REGION} +``` + +Confirm: "Redshift cluster `` is running." + +**If using manual steps:** + +> Go to **AWS Console -> Amazon Redshift -> Create cluster** -> single-node, ra3.large, database `dev`, publicly accessible. + +--- + +## Step 4: S3 bucket + +Ask the user: +> Do you have an S3 bucket I should use, or should I create one? + +**If using aws CLI:** + +```bash +aws s3api create-bucket --bucket confidence-redshift-${ACCOUNT_ID} \ + --region ${AWS_REGION} \ + --create-bucket-configuration LocationConstraint=${AWS_REGION} +``` + +Confirm: "S3 bucket `` created in ``." + +**If using manual steps:** + +> Go to **AWS Console -> S3 -> Create bucket** -> name it, pick same region as cluster. + +--- + +## Step 5: IAM role + +Get the Confidence service account numeric ID: +```bash +CONFIDENCE_SA="account-${ACCOUNT_ID}@spotify-confidence.iam.gserviceaccount.com" + +# CRITICAL: AWS trust policy needs the NUMERIC unique ID, not the email. +SA_UNIQUE_ID=$(gcloud iam service-accounts describe ${CONFIDENCE_SA} \ + --project=spotify-confidence --format="value(uniqueId)") +``` + +If `gcloud` can't access `spotify-confidence` project, the user needs to contact Confidence support to get the numeric service account ID. + +**If using aws CLI:** + +Create the role with dual trust (Google OIDC + Redshift): +```bash +cat > $TMPDIR/redshift-trust.json << EOF +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Federated": "accounts.google.com"}, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "accounts.google.com:sub": "${SA_UNIQUE_ID}" + } + } + }, + { + "Effect": "Allow", + "Principal": {"Service": "redshift.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} +EOF +aws iam create-role --role-name confidence-redshift \ + --assume-role-policy-document file://$TMPDIR/redshift-trust.json +``` + +Attach S3 + Redshift Data API permissions: +```bash +# S3 write access +cat > $TMPDIR/s3-policy.json << EOF +{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:PutObject","s3:GetObject","s3:DeleteObject","s3:ListBucket"],"Resource":["arn:aws:s3:::${BUCKET_NAME}","arn:aws:s3:::${BUCKET_NAME}/*"]}]} +EOF +aws iam put-role-policy --role-name confidence-redshift \ + --policy-name S3Access --policy-document file://$TMPDIR/s3-policy.json + +# Redshift Data API access +cat > $TMPDIR/redshift-data-policy.json << EOF +{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["redshift-data:*","redshift:GetClusterCredentials","redshift:GetClusterCredentialsWithIAM","redshift:DescribeClusters"],"Resource":"*"}]} +EOF +aws iam put-role-policy --role-name confidence-redshift \ + --policy-name RedshiftAccess --policy-document file://$TMPDIR/redshift-data-policy.json +``` + +Get the role ARN: +```bash +ROLE_ARN=$(aws iam get-role --role-name confidence-redshift --query 'Role.Arn' --output text) +``` + +**If using manual steps:** + +> Go to **AWS Console -> IAM -> Roles -> Create role** -> two trust steps: +> - Add **Web identity** trust with `accounts.google.com`, sub = `` (compute and display for the user) +> - Add **AWS service** trust for `redshift.amazonaws.com` +> - Attach policies: custom S3 policy scoped to bucket + `AmazonRedshiftDataFullAccess` +> - Copy the **Role ARN** + +--- + +## Step 6: Attach role to cluster + +**CRITICAL:** Attach the role to the Redshift cluster -- without this, the COPY command can't read from S3: + +**If using aws CLI:** + +```bash +aws redshift modify-cluster-iam-roles \ + --cluster-identifier ${CLUSTER} \ + --add-iam-roles ${ROLE_ARN} --region ${AWS_REGION} +``` + +Wait for `in-sync`: +```bash +aws redshift describe-clusters --cluster-identifier ${CLUSTER} --region ${AWS_REGION} \ + --query "Clusters[0].IamRoles[*].{Role:IamRoleArn,Status:ApplyStatus}" --output table +``` + +Confirm: "IAM role created and attached to cluster." + +**If using manual steps:** + +> Go back to **Redshift -> Clusters -> your cluster -> Properties -> Manage IAM roles -> Add the new role** + +--- + +## Step 7: Schema & grants + +Ask the user: +> What should the schema be called? The default is `confidence`. + +Create the schema and grant permissions so Confidence can see it: + +**If using aws CLI:** + +```bash +aws redshift-data execute-statement \ + --cluster-identifier ${CLUSTER} --database ${DATABASE} --db-user admin \ + --sql "CREATE SCHEMA IF NOT EXISTS ${SCHEMA}; GRANT USAGE ON SCHEMA ${SCHEMA} TO PUBLIC; GRANT CREATE ON SCHEMA ${SCHEMA} TO PUBLIC;" \ + --region ${AWS_REGION} +``` + +**IMPORTANT:** `GRANT USAGE ON SCHEMA ... TO PUBLIC` is required -- without it, Confidence's validation returns "Schema not found" even though the schema exists. This is because Confidence connects via IAM, not as the `admin` user. + +Confirm: "Schema `` created with permissions." + +**If using manual steps:** + +> Go to **Redshift -> Query editor v2** -> connect to cluster -> run: +> ```sql +> CREATE SCHEMA IF NOT EXISTS confidence; +> GRANT USAGE ON SCHEMA confidence TO PUBLIC; +> GRANT CREATE ON SCHEMA confidence TO PUBLIC; +> ``` + +Copy the SQL to clipboard for the user. + +--- + +## Step 8: Validate + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/dataWarehouseConfig:validate" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "redshiftConfig": { + "clusterIdentifier": "", + "database": "", + "schema": "", + "region": "", + "roleArn": "" + } + }' +``` + +**Response:** +```json +{ + "validation": [{ "key": "...", "description": "...", "success": true/false, "error": "..." }], + "successful": true/false, + "configurationResponse": { /* type-specific */ } +} +``` + +If `successful` is true, move to Step 9. + +**If validation fails:** + +**IMPORTANT: Never assume partial success from an ambiguous error.** If the API returns an error like "X does not exist or not authorized", report the exact error message. Do NOT split it into "connection works but X is missing". Show the user the exact error and let them determine the cause. + +For each validation failure, show: +> Validation failed: `` + +Then show the relevant remediation steps: + +- **Schema not found** -> Ensure `GRANT USAGE ON SCHEMA ... TO PUBLIC` was run (Step 7) +- **IAM role errors** -> Check the trust policy has both `accounts.google.com` and `redshift.amazonaws.com` principals +- **S3 access errors** -> Check the S3 policy is attached to the role and scoped to the correct bucket +- **Cluster not found** -> Verify the cluster identifier and region + +--- + +## Step 9: Create warehouse + +**IMPORTANT:** The body is the data warehouse object directly (gRPC transcoding `body: "data_warehouse"`), NOT wrapped in a `dataWarehouse` key. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/dataWarehouses" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "config": { + "redshiftConfig": { + "clusterIdentifier": "", + "database": "", + "schema": "", + "region": "", + "roleArn": "" + } + } + }' +``` + +Save the returned `name` (e.g., `dataWarehouses/...`) for reference. + +--- + +## Step 10: Create connectors + +Create both connectors. Redshift connectors require `redshiftConfig`, `s3Config`, and `batchFileConfig`. + +### Flag Applied Connection (assignment data -> warehouse) + +**IMPORTANT:** The body is the connection object directly (gRPC transcoding `body: "flag_applied_connection"`), NOT wrapped. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev/v1/flagAppliedConnections" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "redshift": { + "redshiftConfig": { + "clusterIdentifier": "", + "database": "", + "schema": "", + "region": "", + "roleArn": "" + }, + "s3Config": { + "bucket": "", + "region": "", + "roleArn": "" + }, + "batchFileConfig": { + "maxEventsPerFile": 10000, + "maxFileAge": "300s", + "maxFileSize": 104857600 + }, + "table": "assignments" + } + }' +``` + +### Event Connection (events -> warehouse) + +**IMPORTANT:** The body is the connection object directly (gRPC transcoding `body: "event_connection"`), NOT wrapped. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev/v1/eventConnections" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "redshift": { + "redshiftConfig": { + "clusterIdentifier": "", + "database": "", + "schema": "", + "region": "", + "roleArn": "" + }, + "s3Config": { + "bucket": "", + "region": "", + "roleArn": "" + }, + "batchFileConfig": { + "maxEventsPerFile": 10000, + "maxFileAge": "300s", + "maxFileSize": 104857600 + }, + "tablePrefix": "events_" + } + }' +``` + +--- + +## Step 11: Assignment table + +Create an assignment table so Confidence can analyze experiment assignments. + +**IMPORTANT:** The body is the assignment table object directly (gRPC transcoding `body: "assignment_table"`), NOT wrapped in an `assignmentTable` key. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/assignmentTables" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "displayName": "Assignments", + "sql": "SELECT targeting_key, rule, assignment_id, assignment_time FROM .assignments", + "entityColumn": { "name": "targeting_key" }, + "timestampColumn": { "name": "assignment_time" }, + "exposureKeyColumn": { "name": "rule" }, + "variantKeyColumn": { "name": "assignment_id" }, + "dataDeliveredUntilUpdateStrategyConfig": { + "strategy": "AUTOMATIC", + "automaticUpdateConfig": { + "commitDelay": "300s" + } + } + }' +``` + +--- + +## Step 12: Verify data pipeline + +Verify both connectors by generating test data and checking it lands in the warehouse. + +### 12a. Get a client secret for testing + +The resolver and events APIs require a **client secret** (not a Bearer token). + +1. **List the user's clients** and show them: + ```bash + curl -s "https://iam.${REGION}.confidence.dev/v1/clients" -H "Authorization: Bearer $TOKEN" + ``` + Display each client with its name and last-seen time. If only one client exists, confirm it with the user. If multiple, let them pick. + +2. **Ask the user** if they have a client secret or want a new one: + > I'll use **** for the pipeline test. Do you have the client secret, or should I create a new credential? + +3. If the user wants a new credential, create one on the chosen client: + ```bash + curl -s -X POST "https://iam.${REGION}.confidence.dev/v1//credentials" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"display_name": "Pipeline Test"}' + ``` + Save the secret to a temp file for pipeline use. **Never print the secret to the user's terminal.** + +### 12b. Verify flag assignments + +Resolve a flag to generate assignment data (use an existing flag + client secret): +```bash +curl -s -X POST "https://resolver.${REGION}.confidence.dev/v1/flags:resolve" \ + -H "Content-Type: application/json" \ + -d '{ + "flags": ["flags/"], + "evaluation_context": {"targeting_key": "warehouse-verify-user"}, + "client_secret": "", + "apply": true + }' +``` + +If no flags exist yet, tell the user: +> No flags to test with. Run `/onboard-confidence setup-wizard` first to create a flag, then come back. + +### 12c. Verify events + +First check for an event definition to use: +```bash +curl -s "https://events.${REGION}.confidence.dev/v1/eventDefinitions" \ + -H "Authorization: Bearer $TOKEN" +``` + +If no event definitions exist, create one with a schema: +```bash +curl -s -X POST "https://events.${REGION}.confidence.dev/v1/eventDefinitions?event_definition_id=test-event" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"schema": {"action": {"stringSchema": {}}, "page": {"stringSchema": {}}}}' +``` + +If an event definition exists but has an empty schema, update it so payload data flows through: +```bash +curl -s -X PATCH "https://events.${REGION}.confidence.dev/v1/eventDefinitions/" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"schema": {"action": {"stringSchema": {}}, "page": {"stringSchema": {}}}}' +``` + +Then publish test events (uses client secret, NOT Bearer token): +```bash +NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +curl -s -X POST "https://events.${REGION}.confidence.dev/v1/events:publish" \ + -H "Content-Type: application/json" \ + -d '{ + "client_secret": "", + "events": [ + { + "event_definition": "eventDefinitions/", + "payload": {"action": "clicked_button", "page": "homepage"}, + "event_time": "'$NOW'" + } + ], + "send_time": "'$NOW'" + }' +``` + +Check response: `{"errors": []}` means success. If `EVENT_DEFINITION_NOT_FOUND`, the definition doesn't exist. If `EVENT_SCHEMA_VALIDATION_FAILED`, the payload doesn't match the schema. + +### 12d. Check data in Redshift + +If user has `aws redshift-data`: +```bash +aws redshift-data execute-statement \ + --cluster-identifier ${CLUSTER} \ + --database ${DATABASE} \ + --db-user ${DB_USER} \ + --sql "SELECT targeting_key, rule, assignment_id, assignment_time FROM ${SCHEMA}.assignments ORDER BY assignment_time DESC LIMIT 5" +``` + +Otherwise, show queries for the Redshift query editor. + +**Show results:** +``` + ● Assignments: rows -- data flowing + -> () + ● Events: rows -- data flowing + on () +``` + +**If no rows after a few seconds**, tell the user: +> Data delivery can take up to a few minutes depending on your warehouse. Check again shortly, or verify in your Redshift query editor. + +--- + +## Step 13: Done + +``` +═══════════════════════════════════════════════════════════════ + Data Warehouse Connected & Verified +═══════════════════════════════════════════════════════════════ + + Warehouse: Redshift () + Database: + Schema: + S3 Bucket: () + Connectors: + ● Flag assignments -> assignments table (verified) + ● Events -> events_* tables (running) + Assignment: + ● Assignment table configured (auto-updating) + + Flag assignment and event data is flowing to your + warehouse. Experiment analysis is ready. + +═══════════════════════════════════════════════════════════════ +``` + +--- + +## API Reference (agent-internal -- do NOT show to user) + +### Base URLs + +All APIs require **region-specific URLs**. Extract region from the JWT token claim `https://confidence.dev/region` (value: `EU` or `US`), lowercase it, and use as prefix. + +``` +IAM_API: https://iam.${region}.confidence.dev/v1 +RESOLVER_API: https://resolver.${region}.confidence.dev/v1 +EVENTS_API: https://events.${region}.confidence.dev/v1 +CONNECTORS_API: https://connectors.${region}.confidence.dev/v1 +METRICS_API: https://metrics.${region}.confidence.dev/v1 +``` + +### Endpoints + +**Validate warehouse config (Bearer token):** +``` +POST ${METRICS_API}/dataWarehouseConfig:validate +Body: { "redshiftConfig": { "clusterIdentifier": str, "database": str, "schema": str, "region": str, "roleArn": str } } +-> { "validation": [...], "successful": bool, "configurationResponse": {...} } +``` + +**Check warehouse exists (Bearer token):** +``` +GET ${METRICS_API}/dataWarehouses:exists +-> { "exists": bool } +``` + +**Create data warehouse (Bearer token, body: "data_warehouse"):** +``` +POST ${METRICS_API}/dataWarehouses +Body (direct object): { "config": { "redshiftConfig": { "clusterIdentifier": str, "database": str, "schema": str, "region": str, "roleArn": str } } } +-> DataWarehouse object +``` + +**Create flag applied connection (Bearer token, body: "flag_applied_connection"):** +``` +POST ${CONNECTORS_API}/flagAppliedConnections +Body (direct object): { "redshift": { "redshiftConfig": {...}, "s3Config": {...}, "batchFileConfig": {...}, "table": "assignments" } } +-> FlagAppliedConnection object +``` + +**Create event connection (Bearer token, body: "event_connection"):** +``` +POST ${CONNECTORS_API}/eventConnections +Body (direct object): { "redshift": { "redshiftConfig": {...}, "s3Config": {...}, "batchFileConfig": {...}, "tablePrefix": "events_" } } +-> EventConnection object +``` + +**Create assignment table (Bearer token, body: "assignment_table"):** +``` +POST ${METRICS_API}/assignmentTables +Body (direct object): { "displayName": str, "sql": str, "entityColumn": {...}, "timestampColumn": {...}, "exposureKeyColumn": {...}, "variantKeyColumn": {...}, "dataDeliveredUntilUpdateStrategyConfig": {...} } +-> AssignmentTable object +``` + +**List clients (Bearer token):** +``` +GET ${IAM_API}/clients +-> { "clients": [...], "nextPageToken": string } +``` + +**Create client credential (Bearer token, body: "client_credential"):** +``` +POST ${IAM_API}/${clientName}/credentials +Body (direct object): { "display_name": string } +-> { "name": "...", "clientSecret": { "secret": string }, ... } + NOTE: secret only returned once on creation +``` + +**Resolve flags (client secret -- NOT Bearer token):** +``` +POST ${RESOLVER_API}/flags:resolve +Body: { "flags": ["flags/"], "evaluationContext": {...}, "clientSecret": string, "apply": bool } +-> { "resolvedFlags": [...] } +``` + +**Publish events (client secret -- NOT Bearer token):** +``` +POST ${EVENTS_API}/events:publish +Body: { "client_secret": string, "events": [...], "send_time": "ISO8601" } +-> { "errors": [...] } +``` + +--- + +## Error Handling Reference (agent-internal) + +### Common HTTP errors + +| Status | Meaning | Recovery | +|--------|---------|----------| +| 400 | Validation error | Parse `.message`, show plain English, re-collect invalid field | +| 401 | Invalid/expired token | Re-trigger Auth0 login | +| 403 | Insufficient permissions | Explain needed role/permission | +| 404 | Resource not found | Check account/resource exists | +| 409 | Conflict (already exists) | Resource already created | +| 429 | Rate limited | Wait briefly and retry | +| 500+ | Server error | Inform user, suggest retry | + +### Sandbox note + +All `curl`, `open`, `python3`, `aws`, and `gcloud` commands that access external hosts (`auth.confidence.dev`, `metrics.confidence.dev`, `connectors.confidence.dev`, AWS APIs, etc.) require `dangerouslyDisableSandbox: true`. On first occurrence, briefly explain to the user that network access outside the sandbox is needed for API calls. diff --git a/skills/setup-warehouse-snowflake/SKILL.md b/skills/setup-warehouse-snowflake/SKILL.md new file mode 100644 index 0000000..66eb904 --- /dev/null +++ b/skills/setup-warehouse-snowflake/SKILL.md @@ -0,0 +1,753 @@ +--- +description: Set up Snowflake as a data warehouse for Confidence. Use when the user chose Snowflake for warehouse setup. +--- + +# Setup Warehouse: Snowflake + +Configure Snowflake as the data warehouse for Confidence experimentation analytics. This skill handles the full end-to-end setup: collect Snowflake config, create a crypto key, register the key in Snowflake, validate, create the warehouse, set up connectors, create the assignment table, and verify the pipeline. + +--- + +## Authentication + +**Browser-based Auth0 login.** The skill opens a browser for Auth0 login (Google, email/password, SSO) and captures the token automatically. The user never touches a token. + +### Session-only token management + +The token is kept in the current session only and is never saved to disk. If the session ends or the token expires, the skill will open your browser to log in again. + +On every sub-command start, check if the `TOKEN` variable is set and not expired: + +```bash +if [ -n "$TOKEN" ]; then + PAYLOAD=$(echo "$TOKEN" | cut -d. -f2) + EXP=$(echo "$PAYLOAD" | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d.get('exp', 0)) +") + NOW=$(date +%s) + if [ "$EXP" -gt "$NOW" ]; then + echo "VALID" + else + echo "EXPIRED" + unset TOKEN + fi +fi +``` + +If `TOKEN` is unset or expired, run the Auth0 login flow with the **regular client ID** (`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w`) and the user's `organization` parameter. Store the result in the `TOKEN` shell variable only. **NEVER write the token to disk. NEVER reference `~/.confidence/`.** + +### Auth script + +Write the following to `$TMPDIR/confidence_auth.py` with CLIENT_ID=`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w` and ORGANIZATION from the token. Run with `python3 $TMPDIR/confidence_auth.py`. Outputs `TOKEN:` on success. + +```python +import http.server, urllib.parse, json, sys, subprocess, hashlib, base64, secrets, string + +code_verifier = ''.join(secrets.choice(string.ascii_letters + string.digits + '-._~') for _ in range(43)) +code_challenge = base64.urlsafe_b64encode(hashlib.sha256(code_verifier.encode()).digest()).rstrip(b'=').decode() + +port = 8084 +CLIENT_ID = '' +ORGANIZATION = '' +REDIRECT_URI = f'http://localhost:{port}/callback' +auth_code = None +error = None + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + global auth_code, error + q = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query) + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.end_headers() + if 'code' in q: + auth_code = q['code'][0] + self.wfile.write(b'

Login successful!

You can close this tab.

') + else: + error = q.get('error', ['unknown'])[0] + self.wfile.write(b'

Login failed

Please try again.

') + def log_message(self, format, *args): + pass + +params = { + 'client_id': CLIENT_ID, + 'redirect_uri': REDIRECT_URI, + 'response_type': 'code', + 'scope': 'openid profile email offline_access', + 'audience': 'https://confidence.dev/', + 'code_challenge': code_challenge, + 'code_challenge_method': 'S256', +} +if ORGANIZATION: + params['organization'] = ORGANIZATION + +authorize_url = 'https://auth.confidence.dev/authorize?' + urllib.parse.urlencode(params) +subprocess.Popen(['open', authorize_url]) +print('WAITING_FOR_LOGIN', flush=True) + +server = http.server.HTTPServer(('127.0.0.1', port), Handler) +server.timeout = 120 +while auth_code is None and error is None: + server.handle_request() +server.server_close() + +if error: + print(f'AUTH_ERROR:{error}', flush=True) + sys.exit(1) + +import urllib.request +token_data = json.dumps({ + 'grant_type': 'authorization_code', + 'client_id': CLIENT_ID, + 'code': auth_code, + 'redirect_uri': REDIRECT_URI, + 'code_verifier': code_verifier +}).encode() +req = urllib.request.Request( + 'https://auth.confidence.dev/oauth/token', + data=token_data, + headers={'Content-Type': 'application/json'} +) +try: + with urllib.request.urlopen(req) as resp: + token_response = json.loads(resp.read()) + print(f'TOKEN:{token_response["access_token"]}', flush=True) +except Exception as e: + print(f'TOKEN_ERROR:{e}', flush=True) + sys.exit(1) +``` + +### Extract region from token + +```bash +REGION=$(echo "$PAYLOAD" | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d.get('https://confidence.dev/region', 'EU')) +") +``` + +Then use `${REGION,,}` (lowercase) for URL prefix: `iam.eu.confidence.dev`, `metrics.eu.confidence.dev`, etc. + +### Common notes + +- Port is fixed at **8084** (must match Auth0 Allowed Callback URLs) +- If port 8084 is busy: `lsof -ti:8084 | xargs kill -9 2>/dev/null` +- All network commands require `dangerouslyDisableSandbox: true` +- Never show the token value to the user +- Always use region-specific URLs (e.g., `iam.eu.confidence.dev` not `iam.confidence.dev`) + +### Important: gRPC-REST transcoding rules + +The Confidence APIs use gRPC with REST transcoding. The `body` field in the proto HTTP binding determines the JSON structure: + +- **`body: "data_warehouse"`** -> send the data warehouse object directly: `{"config": {...}}` +- **`body: "flag_applied_connection"`** -> send the connection object directly: `{"snowflake": {...}}` +- **`body: "event_connection"`** -> send the connection object directly: `{"snowflake": {...}}` +- **`body: "assignment_table"`** -> send the assignment table object directly: `{"displayName": "...", "sql": "...", ...}` +- **`body: "*"`** -> send the full request message + +The body is the object directly, NOT wrapped in an outer key. + +Fields NOT in the body (like `flag_id`, `parent`) become **query parameters**. + +**Field names are `snake_case`** in requests. Responses may use `camelCase`. + +--- + +## User-Facing Communication Rules + +**NEVER expose internal technical details to the user.** + +- Do NOT show raw JSON request/response bodies in conversation +- Do NOT show Auth0 configuration details, token values, or OAuth internals +- DO show human-readable status updates: "Opening browser for login...", "Creating your warehouse...", "Connectors configured!" +- DO describe results in plain English +- The agent handles all auth/API complexity silently + +**Step Tracker:** Display a visual step tracker at every phase transition. Update and re-display it each time you move to a new step. + +--- + +## Step Tracker + +Display at START and after EACH step completes (updating status): + +``` +───── Setup Warehouse (Snowflake) ───────────────────────── + [1] Choose warehouse ● done + [2] Account & user ○ pending + [3] Role & warehouse ○ pending + [4] Database & schema ○ pending + [5] Create crypto key ○ pending + [6] Register key in SF ○ pending + [7] Validate ○ pending + [8] Create warehouse ○ pending + [9] Create connectors ○ pending + [10] Assignment table ○ pending + [11] Verify pipeline ○ pending + [12] Done ○ pending +──────────────────────────────────────────────────────────── +``` + +Use `●` for completed, `▶` for in-progress, `○` for pending. Re-display the full tracker after every step transition. + +--- + +## Step 1: Choose warehouse (already done) + +The user has already chosen Snowflake. Mark step 1 as done. + +--- + +## Step 2: Account & user + +Ask the user for these fields (explain each briefly): + +- **Account** -- Snowflake account identifier (e.g., `zlvpqre-wr49874`). This is the part before `.snowflakecomputing.com` in the Snowflake URL. +- **User** -- Snowflake user for Confidence to connect as. + +--- + +## Step 3: Role & warehouse + +- **Role** -- Snowflake role (default: `ACCOUNTADMIN`). +- **Warehouse** -- SQL warehouse for query execution (default: `COMPUTE_WH`). + +--- + +## Step 4: Database & schema + +- **Exposure database** -- database for exposure tables (default: `CONFIDENCE`). +- **Exposure schema** -- schema for exposure tables (default: `EXPOSURE`). + +Also generate SQL for creating the database/schema if the user says they don't exist yet: +```sql +CREATE DATABASE IF NOT EXISTS ; +CREATE SCHEMA IF NOT EXISTS .; +GRANT USAGE ON DATABASE TO ROLE ; +GRANT ALL ON SCHEMA . TO ROLE ; +``` + +--- + +## Step 5: Create crypto key + +The user does NOT provide this. The skill creates it automatically via the IAM API: + +```bash +curl -s -w "\n%{http_code}" -X POST "https://iam.${REGION}.confidence.dev/v1/cryptoKeys?crypto_key_id=snowflake-key" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"kind": "SNOWFLAKE"}' +``` + +If the key already exists (HTTP 409), fetch it instead: +```bash +curl -s "https://iam.${REGION}.confidence.dev/v1/cryptoKeys/snowflake-key" \ + -H "Authorization: Bearer $TOKEN" +``` + +Extract the `publicKey` from the response, strip PEM headers and newlines to get raw base64. + +Save the crypto key name (e.g., `cryptoKeys/snowflake-key`) for use in the warehouse config. + +--- + +## Step 6: Register key in Snowflake + +Generate the Snowflake SQL to register the key, **copy it to clipboard**, and tell the user: + +> I've created an authentication key for Snowflake. You need to register it with your Snowflake user. +> The SQL has been copied to your clipboard -- paste it in the Snowflake worksheet and run it. + +The SQL should be: +```sql +ALTER USER SET RSA_PUBLIC_KEY=''; +``` + +**IMPORTANT:** Always ask the user if other Confidence accounts share this Snowflake user. If yes, use `RSA_PUBLIC_KEY_2` instead of `RSA_PUBLIC_KEY` to avoid breaking existing connections. Snowflake accepts auth from either key. + +```bash +echo "ALTER USER SET RSA_PUBLIC_KEY='';" | pbcopy +``` + +--- + +## Step 7: Validate + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/dataWarehouseConfig:validate" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "snowflakeConfig": { + "account": "", + "user": "", + "role": "", + "warehouse": "", + "database": "", + "schema": "", + "cryptoKey": "" + } + }' +``` + +**Response:** +```json +{ + "validation": [{ "key": "...", "description": "...", "success": true/false, "error": "..." }], + "successful": true/false, + "configurationResponse": { /* available schemas, databases, roles */ } +} +``` + +If `successful` is true, move to Step 8. + +**If validation fails:** + +**IMPORTANT: Never assume partial success from an ambiguous error.** If the API returns an error like "X does not exist or not authorized", report the exact error message. Do NOT split it into "connection works but X is missing". Show the user the exact error and let them determine the cause. + +For each validation failure, show: +> Validation failed: `` + +### Snowflake remediation + +Generate the full remediation SQL, **copy it to clipboard via `pbcopy`**, and tell the user to paste it in the Snowflake worksheet (https://app.snowflake.com): + +1. **Fetch the crypto key's public key** from the IAM API: + ```bash + curl -s "https://iam.${REGION}.confidence.dev/v1/cryptoKeys/" -H "Authorization: Bearer $TOKEN" + ``` + Strip the PEM headers (`-----BEGIN/END PUBLIC KEY-----`) and newlines to get the raw base64 string for Snowflake. + +2. **Generate SQL based on the error:** + + Auth failures -> register the public key: + ```sql + -- If this is the only Confidence account using this Snowflake user: + ALTER USER SET RSA_PUBLIC_KEY=''; + -- If another Confidence account already uses RSA_PUBLIC_KEY, use key 2: + ALTER USER SET RSA_PUBLIC_KEY_2=''; + ``` + **IMPORTANT:** Always ask the user if other Confidence accounts share this Snowflake user. If yes, use `RSA_PUBLIC_KEY_2` to avoid breaking existing connections. Snowflake accepts auth from either key. + + Database/schema missing: + ```sql + CREATE DATABASE IF NOT EXISTS ; + CREATE SCHEMA IF NOT EXISTS .; + GRANT USAGE ON DATABASE TO ROLE ; + GRANT USAGE ON SCHEMA . TO ROLE ; + GRANT ALL ON SCHEMA . TO ROLE ; + ``` + +3. **Copy to clipboard and tell the user:** + ```bash + echo "" | pbcopy + ``` + > The SQL commands have been copied to your clipboard. Paste them in the Snowflake worksheet at https://app.snowflake.com and run them. Let me know when done and I'll retry validation. + +If `configurationResponse` contains available options (schemas, databases, roles), present these as choices to help the user. + +--- + +## Step 8: Create warehouse + +**IMPORTANT:** The body is the data warehouse object directly (gRPC transcoding `body: "data_warehouse"`), NOT wrapped in a `dataWarehouse` key. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/dataWarehouses" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "config": { + "snowflakeConfig": { + "account": "", + "user": "", + "role": "", + "warehouse": "", + "database": "", + "schema": "", + "cryptoKey": "" + } + } + }' +``` + +Save the returned `name` (e.g., `dataWarehouses/...`) for reference. + +--- + +## Step 9: Create connectors + +Create both connectors. **Snowflake connectors require `database` and `schema` fields in snowflakeConfig.** + +### Flag Applied Connection (assignment data -> warehouse) + +**IMPORTANT:** The body is the connection object directly (gRPC transcoding `body: "flag_applied_connection"`), NOT wrapped. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev/v1/flagAppliedConnections" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "snowflake": { + "snowflakeConfig": { + "account": "", + "user": "", + "role": "", + "warehouse": "", + "database": "", + "schema": "", + "cryptoKey": "" + }, + "table": "ASSIGNMENTS" + } + }' +``` + +### Event Connection (events -> warehouse) + +**IMPORTANT:** The body is the connection object directly (gRPC transcoding `body: "event_connection"`), NOT wrapped. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://connectors.${REGION}.confidence.dev/v1/eventConnections" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "snowflake": { + "snowflakeConfig": { + "account": "", + "user": "", + "role": "", + "warehouse": "", + "database": "", + "schema": "", + "cryptoKey": "" + }, + "tablePrefix": "EVENTS_" + } + }' +``` + +--- + +## Step 10: Assignment table + +Create an assignment table so Confidence can analyze experiment assignments. + +**IMPORTANT:** The body is the assignment table object directly (gRPC transcoding `body: "assignment_table"`), NOT wrapped in an `assignmentTable` key. + +```bash +curl -s -w "\n%{http_code}" -X POST "https://metrics.${REGION}.confidence.dev/v1/assignmentTables" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "displayName": "Assignments", + "sql": "SELECT targeting_key, rule, assignment_id, assignment_time FROM ..ASSIGNMENTS", + "entityColumn": { "name": "targeting_key" }, + "timestampColumn": { "name": "assignment_time" }, + "exposureKeyColumn": { "name": "rule" }, + "variantKeyColumn": { "name": "assignment_id" }, + "dataDeliveredUntilUpdateStrategyConfig": { + "strategy": "AUTOMATIC", + "automaticUpdateConfig": { + "commitDelay": "300s" + } + } + }' +``` + +--- + +## Step 11: Verify data pipeline + +Verify both connectors by generating test data and checking it lands in the warehouse. + +### 11a. Get a client secret for testing + +The resolver and events APIs require a **client secret** (not a Bearer token). + +1. **List the user's clients** and show them: + ```bash + curl -s "https://iam.${REGION}.confidence.dev/v1/clients" -H "Authorization: Bearer $TOKEN" + ``` + Display each client with its name and last-seen time. If only one client exists, confirm it with the user. If multiple, let them pick. + +2. **Ask the user** if they have a client secret or want a new one: + > I'll use **** for the pipeline test. Do you have the client secret, or should I create a new credential? + +3. If the user wants a new credential, create one on the chosen client: + ```bash + curl -s -X POST "https://iam.${REGION}.confidence.dev/v1//credentials" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"display_name": "Pipeline Test"}' + ``` + Save the secret to a temp file for pipeline use. **Never print the secret to the user's terminal.** + +### 11b. Verify flag assignments + +Resolve a flag to generate assignment data (use an existing flag + client secret): +```bash +curl -s -X POST "https://resolver.${REGION}.confidence.dev/v1/flags:resolve" \ + -H "Content-Type: application/json" \ + -d '{ + "flags": ["flags/"], + "evaluation_context": {"targeting_key": "warehouse-verify-user"}, + "client_secret": "", + "apply": true + }' +``` + +If no flags exist yet, tell the user: +> No flags to test with. Run `/onboard-confidence setup-wizard` first to create a flag, then come back. + +### 11c. Verify events + +First check for an event definition to use: +```bash +curl -s "https://events.${REGION}.confidence.dev/v1/eventDefinitions" \ + -H "Authorization: Bearer $TOKEN" +``` + +If no event definitions exist, create one with a schema: +```bash +curl -s -X POST "https://events.${REGION}.confidence.dev/v1/eventDefinitions?event_definition_id=test-event" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"schema": {"action": {"stringSchema": {}}, "page": {"stringSchema": {}}}}' +``` + +If an event definition exists but has an empty schema, update it so payload data flows through: +```bash +curl -s -X PATCH "https://events.${REGION}.confidence.dev/v1/eventDefinitions/" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"schema": {"action": {"stringSchema": {}}, "page": {"stringSchema": {}}}}' +``` + +Then publish test events (uses client secret, NOT Bearer token): +```bash +NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +curl -s -X POST "https://events.${REGION}.confidence.dev/v1/events:publish" \ + -H "Content-Type: application/json" \ + -d '{ + "client_secret": "", + "events": [ + { + "event_definition": "eventDefinitions/", + "payload": {"action": "clicked_button", "page": "homepage"}, + "event_time": "'$NOW'" + } + ], + "send_time": "'$NOW'" + }' +``` + +Check response: `{"errors": []}` means success. If `EVENT_DEFINITION_NOT_FOUND`, the definition doesn't exist. If `EVENT_SCHEMA_VALIDATION_FAILED`, the payload doesn't match the schema. + +### 11d. Check data in Snowflake + +Ask the user: "Want me to check the data, or show you the queries?" + +If user has `snowsql` CLI: +```bash +snowsql -a ${SNOWFLAKE_ACCOUNT} -u ${SNOWFLAKE_USER} -r ${SNOWFLAKE_ROLE} -w ${SNOWFLAKE_WAREHOUSE} -d ${SNOWFLAKE_DATABASE} -s ${SNOWFLAKE_SCHEMA} -q " +SELECT targeting_key, rule, assignment_id, assignment_time +FROM ${SNOWFLAKE_DATABASE}.${SNOWFLAKE_SCHEMA}.ASSIGNMENTS +ORDER BY assignment_time DESC LIMIT 5; +" +``` + +If no `snowsql`, use the Snowflake SQL REST API: +```bash +# Get a JWT token for Snowflake (using keypair auth) or prompt user for password +# Then query via the SQL API: +curl -s -X POST "https://${SNOWFLAKE_ACCOUNT}.snowflakecomputing.com/api/v2/statements" \ + -H "Authorization: Bearer ${SNOWFLAKE_JWT}" \ + -H "Content-Type: application/json" \ + -H "X-Snowflake-Authorization-Token-Type: KEYPAIR_JWT" \ + -d '{ + "statement": "SELECT targeting_key, rule, assignment_id, assignment_time FROM '${SNOWFLAKE_DATABASE}'.'${SNOWFLAKE_SCHEMA}'.ASSIGNMENTS ORDER BY assignment_time DESC LIMIT 5", + "warehouse": "'${SNOWFLAKE_WAREHOUSE}'", + "database": "'${SNOWFLAKE_DATABASE}'", + "schema": "'${SNOWFLAKE_SCHEMA}'", + "role": "'${SNOWFLAKE_ROLE}'" + }' +``` + +If neither available, show the queries for the Snowflake worksheet (https://app.snowflake.com): +> ```sql +> -- Assignments +> SELECT targeting_key, rule, assignment_id, assignment_time +> FROM ..ASSIGNMENTS +> ORDER BY assignment_time DESC LIMIT 5; +> +> -- Events (list event tables first, then query) +> SHOW TABLES LIKE 'EVENTS_%' IN .; +> SELECT * FROM .. +> ORDER BY _event_time DESC LIMIT 5; +> ``` + +**Show results:** +``` + ● Assignments: rows -- data flowing + -> () + ● Events: rows -- data flowing + on () +``` + +**If no rows after a few seconds**, tell the user: +> Data delivery can take up to a few minutes depending on your warehouse. Check again shortly, or verify in your Snowflake worksheet. + +--- + +## Step 12: Done + +``` +═══════════════════════════════════════════════════════════════ + Data Warehouse Connected & Verified +═══════════════════════════════════════════════════════════════ + + Warehouse: Snowflake () + Database: + Schema: + Connectors: + ● Flag assignments -> ASSIGNMENTS table (verified) + ● Events -> EVENTS_* tables (running) + Assignment: + ● Assignment table configured (auto-updating) + + Flag assignment and event data is flowing to your + warehouse. Experiment analysis is ready. + +═══════════════════════════════════════════════════════════════ +``` + +--- + +## API Reference (agent-internal -- do NOT show to user) + +### Base URLs + +All APIs require **region-specific URLs**. Extract region from the JWT token claim `https://confidence.dev/region` (value: `EU` or `US`), lowercase it, and use as prefix. + +``` +IAM_API: https://iam.${region}.confidence.dev/v1 +RESOLVER_API: https://resolver.${region}.confidence.dev/v1 +EVENTS_API: https://events.${region}.confidence.dev/v1 +CONNECTORS_API: https://connectors.${region}.confidence.dev/v1 +METRICS_API: https://metrics.${region}.confidence.dev/v1 +``` + +### Endpoints + +**Create crypto key (Bearer token):** +``` +POST ${IAM_API}/cryptoKeys?crypto_key_id= +Body: { "kind": "SNOWFLAKE" } +-> CryptoKey object with publicKey field +``` + +**Get crypto key (Bearer token):** +``` +GET ${IAM_API}/cryptoKeys/ +-> CryptoKey object with publicKey field +``` + +**Validate warehouse config (Bearer token):** +``` +POST ${METRICS_API}/dataWarehouseConfig:validate +Body: { "snowflakeConfig": {...} } +-> { "validation": [...], "successful": bool, "configurationResponse": {...} } +``` + +**Check warehouse exists (Bearer token):** +``` +GET ${METRICS_API}/dataWarehouses:exists +-> { "exists": bool } +``` + +**Create data warehouse (Bearer token, body: "data_warehouse"):** +``` +POST ${METRICS_API}/dataWarehouses +Body (direct object): { "config": { "snowflakeConfig": {...} } } +-> DataWarehouse object +``` + +**Create flag applied connection (Bearer token, body: "flag_applied_connection"):** +``` +POST ${CONNECTORS_API}/flagAppliedConnections +Body (direct object): { "snowflake": { "snowflakeConfig": {..., "database": "...", "schema": "..."}, "table": "ASSIGNMENTS" } } +-> FlagAppliedConnection object +NOTE: Snowflake connectors require database and schema fields in snowflakeConfig +``` + +**Create event connection (Bearer token, body: "event_connection"):** +``` +POST ${CONNECTORS_API}/eventConnections +Body (direct object): { "snowflake": { "snowflakeConfig": {..., "database": "...", "schema": "..."}, "tablePrefix": "EVENTS_" } } +-> EventConnection object +``` + +**Create assignment table (Bearer token, body: "assignment_table"):** +``` +POST ${METRICS_API}/assignmentTables +Body (direct object): { "displayName": str, "sql": str, "entityColumn": {...}, "timestampColumn": {...}, "exposureKeyColumn": {...}, "variantKeyColumn": {...}, "dataDeliveredUntilUpdateStrategyConfig": {...} } +-> AssignmentTable object +``` + +**List clients (Bearer token):** +``` +GET ${IAM_API}/clients +-> { "clients": [...], "nextPageToken": string } +``` + +**Create client credential (Bearer token, body: "client_credential"):** +``` +POST ${IAM_API}/${clientName}/credentials +Body (direct object): { "display_name": string } +-> { "name": "...", "clientSecret": { "secret": string }, ... } + NOTE: secret only returned once on creation +``` + +**Resolve flags (client secret -- NOT Bearer token):** +``` +POST ${RESOLVER_API}/flags:resolve +Body: { "flags": ["flags/"], "evaluationContext": {...}, "clientSecret": string, "apply": bool } +-> { "resolvedFlags": [...] } +``` + +**Publish events (client secret -- NOT Bearer token):** +``` +POST ${EVENTS_API}/events:publish +Body: { "client_secret": string, "events": [...], "send_time": "ISO8601" } +-> { "errors": [...] } +``` + +--- + +## Error Handling Reference (agent-internal) + +### Common HTTP errors + +| Status | Meaning | Recovery | +|--------|---------|----------| +| 400 | Validation error | Parse `.message`, show plain English, re-collect invalid field | +| 401 | Invalid/expired token | Re-trigger Auth0 login | +| 403 | Insufficient permissions | Explain needed role/permission | +| 404 | Resource not found | Check account/resource exists | +| 409 | Conflict (already exists) | Resource already created (e.g., crypto key) | +| 429 | Rate limited | Wait briefly and retry | +| 500+ | Server error | Inform user, suggest retry | + +### Sandbox note + +All `curl`, `open`, and `python3` commands that access external hosts (`auth.confidence.dev`, `iam.confidence.dev`, `metrics.confidence.dev`, `connectors.confidence.dev`, etc.) require `dangerouslyDisableSandbox: true`. On first occurrence, briefly explain to the user that network access outside the sandbox is needed for API calls. diff --git a/skills/setup-warehouse/SKILL.md b/skills/setup-warehouse/SKILL.md new file mode 100644 index 0000000..a9360e6 --- /dev/null +++ b/skills/setup-warehouse/SKILL.md @@ -0,0 +1,187 @@ +--- +description: Set up a data warehouse for Confidence experimentation analytics. Use when the user asks to connect a warehouse, set up BigQuery/Snowflake/Databricks/Redshift, or configure data connectors. +--- + +# Setup Warehouse + +Configure a data warehouse so Confidence can store and analyze your experiment data — flag assignments, events, and metrics. + +A data warehouse is where Confidence writes your experimentation data. It connects to your existing cloud data infrastructure so you can query experiment results, build dashboards, and run statistical analysis. Without a warehouse, Confidence can resolve flags but cannot analyze experiment outcomes. + +## Supported Warehouse Types + +| # | Warehouse | Best for | +|---|-----------|----------| +| 1 | **BigQuery** | Google Cloud users, fastest setup | +| 2 | **Snowflake** | Snowflake users, key-pair authentication | +| 3 | **Databricks** | Databricks users, requires AWS S3 staging bucket | +| 4 | **Redshift** | AWS users, requires S3 staging bucket | + +## Flow + +Present the user with the four options: + +> Which data warehouse do you use? +> 1. BigQuery +> 2. Snowflake +> 3. Databricks +> 4. Redshift + +After the user picks, hand off to the specific warehouse skill: + +- **BigQuery** -> Tell the user: "Starting BigQuery setup..." and invoke `/onboard-confidence:setup-warehouse-bigquery` +- **Snowflake** -> Tell the user: "Starting Snowflake setup..." and invoke `/onboard-confidence:setup-warehouse-snowflake` +- **Databricks** -> Tell the user: "Starting Databricks setup..." and invoke `/onboard-confidence:setup-warehouse-databricks` +- **Redshift** -> Tell the user: "Starting Redshift setup..." and invoke `/onboard-confidence:setup-warehouse-redshift` + +--- + +## Authentication + +**Browser-based Auth0 login.** The skill opens a browser for Auth0 login (Google, email/password, SSO) and captures the token automatically. The user never touches a token. + +### Session-only token management + +The token is kept in the current session only and is never saved to disk. If the session ends or the token expires, the skill will open your browser to log in again. + +On every sub-command start, check if the `TOKEN` variable is set and not expired: + +```bash +if [ -n "$TOKEN" ]; then + PAYLOAD=$(echo "$TOKEN" | cut -d. -f2) + EXP=$(echo "$PAYLOAD" | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d.get('exp', 0)) +") + NOW=$(date +%s) + if [ "$EXP" -gt "$NOW" ]; then + echo "VALID" + else + echo "EXPIRED" + unset TOKEN + fi +fi +``` + +If `TOKEN` is unset or expired, run the Auth0 login flow with the **regular client ID** (`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w`) and the user's `organization` parameter. Store the result in the `TOKEN` shell variable only. **NEVER write the token to disk. NEVER reference `~/.confidence/`.** + +### Extract region from token + +```bash +PAYLOAD=$(echo "$TOKEN" | cut -d. -f2) +REGION=$(echo "$PAYLOAD" | python3 -c " +import sys, json, base64 +p = sys.stdin.read().strip() +p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +d = json.loads(base64.b64decode(p)) +print(d.get('https://confidence.dev/region', 'EU')) +") +``` + +Then use `${REGION,,}` (lowercase) for URL prefix: `iam.eu.confidence.dev`, `metrics.eu.confidence.dev`, etc. + +### Auth script + +Write the following to `$TMPDIR/confidence_auth.py` with CLIENT_ID=`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w` and ORGANIZATION from the token. Run with `python3 $TMPDIR/confidence_auth.py`. Outputs `TOKEN:` on success. + +```python +import http.server, urllib.parse, json, sys, subprocess, hashlib, base64, secrets, string + +code_verifier = ''.join(secrets.choice(string.ascii_letters + string.digits + '-._~') for _ in range(43)) +code_challenge = base64.urlsafe_b64encode(hashlib.sha256(code_verifier.encode()).digest()).rstrip(b'=').decode() + +port = 8084 +CLIENT_ID = '' +ORGANIZATION = '' +REDIRECT_URI = f'http://localhost:{port}/callback' +auth_code = None +error = None + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + global auth_code, error + q = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query) + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.end_headers() + if 'code' in q: + auth_code = q['code'][0] + self.wfile.write(b'

Login successful!

You can close this tab.

') + else: + error = q.get('error', ['unknown'])[0] + self.wfile.write(b'

Login failed

Please try again.

') + def log_message(self, format, *args): + pass + +params = { + 'client_id': CLIENT_ID, + 'redirect_uri': REDIRECT_URI, + 'response_type': 'code', + 'scope': 'openid profile email offline_access', + 'audience': 'https://confidence.dev/', + 'code_challenge': code_challenge, + 'code_challenge_method': 'S256', +} +if ORGANIZATION: + params['organization'] = ORGANIZATION + +authorize_url = 'https://auth.confidence.dev/authorize?' + urllib.parse.urlencode(params) +subprocess.Popen(['open', authorize_url]) +print('WAITING_FOR_LOGIN', flush=True) + +server = http.server.HTTPServer(('127.0.0.1', port), Handler) +server.timeout = 120 +while auth_code is None and error is None: + server.handle_request() +server.server_close() + +if error: + print(f'AUTH_ERROR:{error}', flush=True) + sys.exit(1) + +import urllib.request +token_data = json.dumps({ + 'grant_type': 'authorization_code', + 'client_id': CLIENT_ID, + 'code': auth_code, + 'redirect_uri': REDIRECT_URI, + 'code_verifier': code_verifier +}).encode() +req = urllib.request.Request( + 'https://auth.confidence.dev/oauth/token', + data=token_data, + headers={'Content-Type': 'application/json'} +) +try: + with urllib.request.urlopen(req) as resp: + token_response = json.loads(resp.read()) + print(f'TOKEN:{token_response["access_token"]}', flush=True) +except Exception as e: + print(f'TOKEN_ERROR:{e}', flush=True) + sys.exit(1) +``` + +### Common notes + +- Port is fixed at **8084** (must match Auth0 Allowed Callback URLs) +- If port 8084 is busy: `lsof -ti:8084 | xargs kill -9 2>/dev/null` +- All network commands require `dangerouslyDisableSandbox: true` +- Never show the token value to the user +- Always use region-specific URLs (e.g., `iam.eu.confidence.dev` not `iam.confidence.dev`) + +--- + +## User-Facing Communication Rules + +**NEVER expose internal technical details to the user.** + +- Do NOT show raw JSON request/response bodies in conversation +- Do NOT show Auth0 configuration details, token values, or OAuth internals +- DO show human-readable status updates: "Opening browser for login...", "Creating your warehouse...", "Connectors configured!" +- DO describe results in plain English +- The agent handles all auth/API complexity silently + +**Step Tracker:** Display a visual step tracker at every phase transition. Update and re-display it each time you move to a new step. Use `●` for completed, `▶` for in-progress, `○` for pending. From 767f7b3a2a79bd75e9f104a93c468754b98a87e8 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Thu, 11 Jun 2026 16:38:19 +0200 Subject: [PATCH 22/25] feat: add telemetry system for onboarding skills Dual-layer telemetry: - Layer 1: Skill-embedded events (works on all AI clients) - Layer 2: Claude Code hooks (PostToolUse + SessionEnd) New files: - skills/telemetry/TELEMETRY.md: shared reference with event catalog, privacy rules, emit helper, step name registry, sentiment detection, identity linking, feedback prompt, session abandonment - hooks/hooks.json: PostToolUse for API call capture, SessionEnd for abandonment detection - hooks/telemetry_hook.py: detects curl to confidence.dev, emits tool.api_call events - hooks/session_end_telemetry.py: emits session_abandoned if no session_completed Event types: session_started, step_started/completed/failed, user_choice, session_completed, identity_linked, sentiment, session_sentiment, feedback, feedback_text, session_abandoned, warehouse.*, tool.api_call Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude-plugin/plugin.json | 3 +- hooks/hooks.json | 28 +++ hooks/session_end_telemetry.py | 48 +++++ hooks/telemetry_hook.py | 65 ++++++ skills/telemetry/TELEMETRY.md | 348 +++++++++++++++++++++++++++++++++ 5 files changed, 491 insertions(+), 1 deletion(-) create mode 100644 hooks/hooks.json create mode 100644 hooks/session_end_telemetry.py create mode 100644 hooks/telemetry_hook.py create mode 100644 skills/telemetry/TELEMETRY.md diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 4312627..22c6885 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -15,5 +15,6 @@ "a/b-testing", "migration", "openfeature" - ] + ], + "hooks": "./hooks/hooks.json" } diff --git a/hooks/hooks.json b/hooks/hooks.json new file mode 100644 index 0000000..7fab65a --- /dev/null +++ b/hooks/hooks.json @@ -0,0 +1,28 @@ +{ + "description": "Confidence onboarding telemetry — captures API calls and session abandonment", + "hooks": { + "PostToolUse": [ + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "command": "python3 ${CLAUDE_PLUGIN_ROOT:-$PLUGIN_ROOT}/hooks/telemetry_hook.py", + "timeout": 5 + } + ] + } + ], + "SessionEnd": [ + { + "hooks": [ + { + "type": "command", + "command": "python3 ${CLAUDE_PLUGIN_ROOT:-$PLUGIN_ROOT}/hooks/session_end_telemetry.py", + "timeout": 10 + } + ] + } + ] + } +} diff --git a/hooks/session_end_telemetry.py b/hooks/session_end_telemetry.py new file mode 100644 index 0000000..a30a038 --- /dev/null +++ b/hooks/session_end_telemetry.py @@ -0,0 +1,48 @@ +"""SessionEnd hook: detects abandoned onboarding sessions.""" +import json, os, sys, urllib.request + +TELEMETRY_URL = "https://onboarding.confidence.dev/v1/telemetry:publish" + +def main(): + if os.environ.get("CONFIDENCE_TELEMETRY") == "0": + return + session_id = os.environ.get("SESSION_ID") + if not session_id: + return + if os.environ.get("SESSION_COMPLETED") == "true": + return + + from datetime import datetime, timezone + event = { + "session_id": session_id, + "events": [{ + "event_type": "onboarding.session_abandoned", + "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "properties": { + "last_step_number": int(os.environ.get("LAST_STEP_NUMBER", "0")), + "last_step_name": os.environ.get("LAST_STEP_NAME", "unknown"), + "skill": os.environ.get("TELEMETRY_SKILL", "unknown"), + "subcommand": os.environ.get("TELEMETRY_SUBCOMMAND", "unknown"), + "plugin_version": "0.2.3" + } + }] + } + + token = os.environ.get("TOKEN") + headers = {"Content-Type": "application/json"} + if token: + headers["Authorization"] = f"Bearer {token}" + + req = urllib.request.Request( + TELEMETRY_URL, + data=json.dumps(event).encode(), + headers=headers + ) + urllib.request.urlopen(req, timeout=5) + +if __name__ == "__main__": + try: + main() + except Exception: + pass + sys.exit(0) diff --git a/hooks/telemetry_hook.py b/hooks/telemetry_hook.py new file mode 100644 index 0000000..5cea665 --- /dev/null +++ b/hooks/telemetry_hook.py @@ -0,0 +1,65 @@ +"""PostToolUse hook: captures API calls to confidence.dev during onboarding flows.""" +import json, os, re, sys, urllib.request + +TELEMETRY_URL = "https://onboarding.confidence.dev/v1/telemetry:publish" + +def main(): + if os.environ.get("CONFIDENCE_TELEMETRY") == "0": + return + session_id = os.environ.get("SESSION_ID") + if not session_id: + return + + data = json.load(sys.stdin) + command = data.get("tool_input", {}).get("command", "") + if "confidence.dev" not in command: + return + + method = "GET" + for m in ["POST", "PUT", "PATCH", "DELETE"]: + if f"-X {m}" in command or f"-X{m}" in command: + method = m + break + + path_match = re.search(r'https?://[^/]*confidence\.dev(/[^\s"\'\\]*)', command) + api_path = path_match.group(1) if path_match else "/unknown" + + output = data.get("tool_output", "") + status = 0 + status_match = re.search(r'HTTP_(\d{3})', output) + if status_match: + status = int(status_match.group(1)) + + from datetime import datetime, timezone + event = { + "session_id": session_id, + "events": [{ + "event_type": "tool.api_call", + "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "properties": { + "api_path": api_path, + "http_method": method, + "http_status": status, + "plugin_version": "0.2.3" + } + }] + } + + token = os.environ.get("TOKEN") + headers = {"Content-Type": "application/json"} + if token: + headers["Authorization"] = f"Bearer {token}" + + req = urllib.request.Request( + TELEMETRY_URL, + data=json.dumps(event).encode(), + headers=headers + ) + urllib.request.urlopen(req, timeout=3) + +if __name__ == "__main__": + try: + main() + except Exception: + pass + sys.exit(0) diff --git a/skills/telemetry/TELEMETRY.md b/skills/telemetry/TELEMETRY.md new file mode 100644 index 0000000..83872bc --- /dev/null +++ b/skills/telemetry/TELEMETRY.md @@ -0,0 +1,348 @@ +# Telemetry Reference + +Shared reference for all onboarding skills. This is NOT a skill — it has no frontmatter description and is never invoked directly. + +## Overview + +The onboarding telemetry system uses a dual-layer architecture: + +- **Layer 1: Skill-Embedded Events (semantic)** — Curl calls in SKILL.md instructions emitted at each step transition. Captures *what* happened: step names, user choices, outcomes, timing. Works in all AI coding clients (Claude Code, Cursor, Codex, Gemini). + +- **Layer 2: Claude Code Hooks (structural)** — `PostToolUse` hooks that fire automatically on every Bash tool call. Captures *how* it happened: API calls to confidence.dev, HTTP status codes. Acts as a reliability backstop. Claude Code only — hooks are not supported in other clients. + +Both layers share a `session_id` for correlation. + +### Telemetry Endpoint + +``` +POST https://onboarding.confidence.dev/v1/telemetry:publish +Authorization: Bearer $TOKEN (or anonymous if no token available) +Content-Type: application/json +``` + +Events are fire-and-forget. The endpoint may not exist yet — that is fine. Curl calls will silently fail and never block the user. + +--- + +## Opt-In / Consent + +Controlled by the `CONFIDENCE_TELEMETRY` environment variable: + +| Value | Behavior | +|-------|----------| +| `1` | Telemetry enabled | +| `0` | Telemetry disabled — no events emitted | +| unset | Show one-time consent prompt per session | + +### Consent Prompt + +When `CONFIDENCE_TELEMETRY` is unset, show this prompt once at the start of the session: + +> Confidence collects anonymous usage data to improve onboarding. No secrets or personal details. Help us improve? (yes/no) + +Based on the response: +- **yes** -> `TELEMETRY_ENABLED=true` +- **no** -> `TELEMETRY_ENABLED=false` + +To set the variable for the session: + +```bash +export CONFIDENCE_TELEMETRY=1 # enable +export CONFIDENCE_TELEMETRY=0 # disable +``` + +--- + +## Session ID + +Generate once per skill invocation (at the start of each onboarding flow). Reuse across all events in that session. + +```bash +SESSION_ID=$(python3 -c "import uuid; print(uuid.uuid4())") +``` + +Pass `SESSION_ID` through all events and export it so hooks can read it: + +```bash +export SESSION_ID +``` + +--- + +## Emit Helper + +Skills should use this exact pattern to emit a telemetry event. Copy-paste and fill in `` and ``. + +```bash +if [ "$TELEMETRY_ENABLED" = "true" ]; then + curl -s -X POST "https://onboarding.confidence.dev/v1/telemetry:publish" \ + -H "Content-Type: application/json" \ + ${TOKEN:+-H "Authorization: Bearer $TOKEN"} \ + -d '{ + "session_id": "'$SESSION_ID'", + "events": [{ + "event_type": "", + "timestamp": "'$(date -u +%Y-%m-%dT%H:%M:%SZ)'", + "properties": {} + }] + }' > /dev/null 2>&1 & +fi +``` + +Key details: +- `> /dev/null 2>&1 &` — fire-and-forget, never blocks the user flow +- `${TOKEN:+-H "Authorization: Bearer $TOKEN"}` — includes auth header only if TOKEN is set (anonymous otherwise) +- Always use UTC timestamps via `date -u` + +--- + +## Event Catalog + +| Event | When | Key Properties | +|-------|------|----------------| +| `onboarding.session_started` | Skill begins | `skill`, `subcommand`, `is_dry_run`, `plugin_version` | +| `onboarding.step_started` | Step tracker updates to in-progress | `step_number`, `step_name` | +| `onboarding.step_completed` | Step tracker updates to done | `step_number`, `step_name`, `duration_ms` | +| `onboarding.step_failed` | Step encounters an error | `step_number`, `step_name`, `error_category` (NOT raw message) | +| `onboarding.user_choice` | User picks an option | `choice_key`, `choice_value` (from allowlist only) | +| `onboarding.session_completed` | Flow finishes successfully | `total_duration_ms`, `steps_completed` | +| `warehouse.type_selected` | User picks a warehouse type | `warehouse_type` | +| `warehouse.validation_result` | Validation endpoint returns | `successful`, `failed_checks` | +| `warehouse.connector_created` | Connector is created | `connector_type` | +| `warehouse.pipeline_verified` | Pipeline verification completes | `assignments_ok`, `events_ok` | +| `onboarding.identity_linked` | After re-auth gives org-scoped token (create-account Step 5) | `org_id`, `account_name`, `region` — ties anonymous pre-account events to the new account | +| `onboarding.sentiment` | After each step, LLM self-assessment of user tone | `step_name`, `sentiment` (`frustrated`, `confused`, `satisfied`), `confidence` (`high`, `medium`) — only emit when signal is clear, skip neutral | +| `onboarding.session_sentiment` | At session_completed | `overall` (`positive`, `mixed`, `negative`), `frustrated_steps`, `satisfied_steps`, `confused_steps` | +| `onboarding.feedback` | End of completed flow, user rates experience | `rating` (1-4), `rating_label` (`easy`, `okay`, `hard`, `broken`), `subcommand` | +| `onboarding.feedback_text` | Optional free-text follow-up for ratings 3-4 | `text` (volunteered by user, exception to no-freeform rule), `subcommand` | +| `onboarding.session_abandoned` | SessionEnd hook fires without session_completed | `last_step_number`, `last_step_name`, `total_duration_ms` | +| `tool.api_call` | Hook detects curl to confidence.dev | `api_path`, `http_method`, `http_status` | + +### Event Properties Reference + +Common properties included in every event: + +```json +{ + "skill": "onboard-confidence", + "subcommand": "create-account", + "plugin_version": "0.2.3", + "is_dry_run": false +} +``` + +### Error Categories (allowlist) + +Use these category strings in `error_category` — never log raw error messages: + +- `auth_expired` — token expired or invalid +- `auth_failed` — authentication failed +- `validation_failed` — input or permission validation failed +- `server_error` — 5xx from the API +- `network_error` — connection timeout or DNS failure +- `conflict` — resource already exists (409) +- `not_found` — resource not found (404) +- `rate_limited` — too many requests (429) +- `unknown` — unrecognized error + +### User Choice Keys (allowlist) + +Only these `choice_key` / `choice_value` pairs may be logged: + +| `choice_key` | Allowed `choice_value` values | +|--------------|-------------------------------| +| `region` | `eu`, `us` | +| `warehouse_type` | `bigquery`, `snowflake`, `databricks`, `redshift` | +| `auth_method` | `google`, `email`, `sso` | +| `variant_type` | `boolean`, `string`, `integer`, `double`, `struct` | +| `existing_client` | `yes`, `no` | + +--- + +## Sentiment Detection + +After each step completes, **silently assess the user's tone** from their messages during that step. Do NOT ask the user about their mood — just observe. + +Emit `onboarding.sentiment` ONLY when the signal is clear: + +| Sentiment | Signals | Examples | +|-----------|---------|----------| +| `frustrated` | Short angry messages, retries, confusion | "why?!", "doesn't work", "again?!", expletives | +| `confused` | Asking for clarification, wrong input repeatedly | "I don't understand", "what do you mean", long pauses | +| `satisfied` | Smooth progression, positive language | "great", "nice", "thanks", "easy", "perfect" | +| neutral | No clear signal | **Do not emit** — only emit when the signal is clear | + +At `session_completed`, also emit `onboarding.session_sentiment` with counts of frustrated/confused/satisfied steps and an overall assessment. + +## Identity Linking (create-account only) + +During `create-account`, the user has no token for Steps 1-4 (account doesn't exist yet). Events are emitted anonymously with only `session_id`. + +At Step 5 (re-auth), after obtaining the org-scoped token, emit `onboarding.identity_linked` to tie the anonymous session to the new account: + +```bash +# After re-auth in create-account Step 5 +emit_event "onboarding.identity_linked" 5 "connect_tools" \ + '"org_id": "'$ORG_ID'", "account_name": "'$ACCOUNT_ID'", "region": "'$REGION'"' +``` + +The backend stitches all events with the same `session_id` — before and after the identity link. + +## Feedback Prompt + +At the end of every **completed** flow (after the Done summary), ask: + +> How was this experience? +> 1. Easy — worked great +> 2. Okay — got there eventually +> 3. Hard — needed help +> 4. Broken — something didn't work + +Emit `onboarding.feedback` with the rating. + +For ratings 3 ("hard") or 4 ("broken"), ask an optional follow-up: + +> What was the hardest part? (optional, press Enter to skip) + +If the user provides text, emit `onboarding.feedback_text`. This is the one exception to the "no freeform input" privacy rule — the user explicitly volunteered the feedback. + +## Session Abandonment (Claude Code only) + +The `SessionEnd` hook checks if `SESSION_ID` is set but `SESSION_COMPLETED` is not. If so, it emits `onboarding.session_abandoned` with the last known step. This is handled by `hooks/session_end_telemetry.py`. + +--- + +## Privacy Rules + +**NEVER log any of the following:** +- Tokens, secrets, passwords, API keys +- Email addresses or personal identifiers +- Full API response bodies +- Full shell commands +- Workspace names, flag names, or other freeform user input + +**DO log:** +- Error categories from the allowlist above (not raw messages) +- User choices from the allowlist above (not freeform input) +- Step numbers and step names from the registry below +- Timing data (duration in milliseconds) +- Boolean outcomes (successful/failed, assignments_ok/events_ok) + +--- + +## Step Name Registry + +Canonical step names for each skill/subcommand. Use these exact strings in `step_name` properties. + +### create-account (6 steps) + +| Step | Name | +|------|------| +| 1 | `login` | +| 2 | `workspace_name` | +| 3 | `account_details` | +| 4 | `create_account` | +| 5 | `connect_tools` | +| 6 | `done` | + +### invite-user (4 steps) + +| Step | Name | +|------|------| +| 1 | `authenticate` | +| 2 | `target_account` | +| 3 | `invitation_details` | +| 4 | `send_invitation` | + +### create-client (3 steps) + +| Step | Name | +|------|------| +| 1 | `client_name` | +| 2 | `create_client` | +| 3 | `get_credentials` | + +### setup-wizard (6 steps) + +| Step | Name | +|------|------| +| 1 | `create_client` | +| 2 | `create_flag` | +| 3 | `add_variants` | +| 4 | `add_targeting` | +| 5 | `test_resolve` | +| 6 | `done` | + +### setup-warehouse dispatcher (1 step) + +| Step | Name | +|------|------| +| 1 | `choose_warehouse` | + +### setup-warehouse-bigquery (10 steps) + +| Step | Name | +|------|------| +| 1 | `choose_warehouse` | +| 2 | `gcp_project_id` | +| 3 | `dataset_name` | +| 4 | `service_account` | +| 5 | `validate_and_fix` | +| 6 | `create_warehouse` | +| 7 | `create_connectors` | +| 8 | `assignment_table` | +| 9 | `verify_pipeline` | +| 10 | `done` | + +### setup-warehouse-snowflake (12 steps) + +| Step | Name | +|------|------| +| 1 | `choose_warehouse` | +| 2 | `account_and_user` | +| 3 | `role_and_warehouse` | +| 4 | `database_and_schema` | +| 5 | `create_crypto_key` | +| 6 | `register_key_in_sf` | +| 7 | `validate` | +| 8 | `create_warehouse` | +| 9 | `create_connectors` | +| 10 | `assignment_table` | +| 11 | `verify_pipeline` | +| 12 | `done` | + +### setup-warehouse-databricks (13 steps) + +| Step | Name | +|------|------| +| 1 | `choose_warehouse` | +| 2 | `workspace_url` | +| 3 | `sql_warehouse_id` | +| 4 | `service_principal` | +| 5 | `aws_account_and_cli` | +| 6 | `s3_bucket` | +| 7 | `iam_role` | +| 8 | `databricks_schema` | +| 9 | `create_warehouse` | +| 10 | `create_connectors` | +| 11 | `assignment_table` | +| 12 | `verify_pipeline` | +| 13 | `done` | + +### setup-warehouse-redshift (13 steps) + +| Step | Name | +|------|------| +| 1 | `choose_warehouse` | +| 2 | `aws_account_and_cli` | +| 3 | `redshift_cluster` | +| 4 | `s3_bucket` | +| 5 | `iam_role` | +| 6 | `attach_role` | +| 7 | `schema_and_grants` | +| 8 | `validate` | +| 9 | `create_warehouse` | +| 10 | `create_connectors` | +| 11 | `assignment_table` | +| 12 | `verify_pipeline` | +| 13 | `done` | From e756007c8ddf116b7de71b9ca50a6d12ebcb471a Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Thu, 11 Jun 2026 16:42:00 +0200 Subject: [PATCH 23/25] Revert "feat: add telemetry system for onboarding skills" This reverts commit 767f7b3a2a79bd75e9f104a93c468754b98a87e8. --- .claude-plugin/plugin.json | 3 +- hooks/hooks.json | 28 --- hooks/session_end_telemetry.py | 48 ----- hooks/telemetry_hook.py | 65 ------ skills/telemetry/TELEMETRY.md | 348 --------------------------------- 5 files changed, 1 insertion(+), 491 deletions(-) delete mode 100644 hooks/hooks.json delete mode 100644 hooks/session_end_telemetry.py delete mode 100644 hooks/telemetry_hook.py delete mode 100644 skills/telemetry/TELEMETRY.md diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 22c6885..4312627 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -15,6 +15,5 @@ "a/b-testing", "migration", "openfeature" - ], - "hooks": "./hooks/hooks.json" + ] } diff --git a/hooks/hooks.json b/hooks/hooks.json deleted file mode 100644 index 7fab65a..0000000 --- a/hooks/hooks.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "description": "Confidence onboarding telemetry — captures API calls and session abandonment", - "hooks": { - "PostToolUse": [ - { - "matcher": "Bash", - "hooks": [ - { - "type": "command", - "command": "python3 ${CLAUDE_PLUGIN_ROOT:-$PLUGIN_ROOT}/hooks/telemetry_hook.py", - "timeout": 5 - } - ] - } - ], - "SessionEnd": [ - { - "hooks": [ - { - "type": "command", - "command": "python3 ${CLAUDE_PLUGIN_ROOT:-$PLUGIN_ROOT}/hooks/session_end_telemetry.py", - "timeout": 10 - } - ] - } - ] - } -} diff --git a/hooks/session_end_telemetry.py b/hooks/session_end_telemetry.py deleted file mode 100644 index a30a038..0000000 --- a/hooks/session_end_telemetry.py +++ /dev/null @@ -1,48 +0,0 @@ -"""SessionEnd hook: detects abandoned onboarding sessions.""" -import json, os, sys, urllib.request - -TELEMETRY_URL = "https://onboarding.confidence.dev/v1/telemetry:publish" - -def main(): - if os.environ.get("CONFIDENCE_TELEMETRY") == "0": - return - session_id = os.environ.get("SESSION_ID") - if not session_id: - return - if os.environ.get("SESSION_COMPLETED") == "true": - return - - from datetime import datetime, timezone - event = { - "session_id": session_id, - "events": [{ - "event_type": "onboarding.session_abandoned", - "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - "properties": { - "last_step_number": int(os.environ.get("LAST_STEP_NUMBER", "0")), - "last_step_name": os.environ.get("LAST_STEP_NAME", "unknown"), - "skill": os.environ.get("TELEMETRY_SKILL", "unknown"), - "subcommand": os.environ.get("TELEMETRY_SUBCOMMAND", "unknown"), - "plugin_version": "0.2.3" - } - }] - } - - token = os.environ.get("TOKEN") - headers = {"Content-Type": "application/json"} - if token: - headers["Authorization"] = f"Bearer {token}" - - req = urllib.request.Request( - TELEMETRY_URL, - data=json.dumps(event).encode(), - headers=headers - ) - urllib.request.urlopen(req, timeout=5) - -if __name__ == "__main__": - try: - main() - except Exception: - pass - sys.exit(0) diff --git a/hooks/telemetry_hook.py b/hooks/telemetry_hook.py deleted file mode 100644 index 5cea665..0000000 --- a/hooks/telemetry_hook.py +++ /dev/null @@ -1,65 +0,0 @@ -"""PostToolUse hook: captures API calls to confidence.dev during onboarding flows.""" -import json, os, re, sys, urllib.request - -TELEMETRY_URL = "https://onboarding.confidence.dev/v1/telemetry:publish" - -def main(): - if os.environ.get("CONFIDENCE_TELEMETRY") == "0": - return - session_id = os.environ.get("SESSION_ID") - if not session_id: - return - - data = json.load(sys.stdin) - command = data.get("tool_input", {}).get("command", "") - if "confidence.dev" not in command: - return - - method = "GET" - for m in ["POST", "PUT", "PATCH", "DELETE"]: - if f"-X {m}" in command or f"-X{m}" in command: - method = m - break - - path_match = re.search(r'https?://[^/]*confidence\.dev(/[^\s"\'\\]*)', command) - api_path = path_match.group(1) if path_match else "/unknown" - - output = data.get("tool_output", "") - status = 0 - status_match = re.search(r'HTTP_(\d{3})', output) - if status_match: - status = int(status_match.group(1)) - - from datetime import datetime, timezone - event = { - "session_id": session_id, - "events": [{ - "event_type": "tool.api_call", - "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - "properties": { - "api_path": api_path, - "http_method": method, - "http_status": status, - "plugin_version": "0.2.3" - } - }] - } - - token = os.environ.get("TOKEN") - headers = {"Content-Type": "application/json"} - if token: - headers["Authorization"] = f"Bearer {token}" - - req = urllib.request.Request( - TELEMETRY_URL, - data=json.dumps(event).encode(), - headers=headers - ) - urllib.request.urlopen(req, timeout=3) - -if __name__ == "__main__": - try: - main() - except Exception: - pass - sys.exit(0) diff --git a/skills/telemetry/TELEMETRY.md b/skills/telemetry/TELEMETRY.md deleted file mode 100644 index 83872bc..0000000 --- a/skills/telemetry/TELEMETRY.md +++ /dev/null @@ -1,348 +0,0 @@ -# Telemetry Reference - -Shared reference for all onboarding skills. This is NOT a skill — it has no frontmatter description and is never invoked directly. - -## Overview - -The onboarding telemetry system uses a dual-layer architecture: - -- **Layer 1: Skill-Embedded Events (semantic)** — Curl calls in SKILL.md instructions emitted at each step transition. Captures *what* happened: step names, user choices, outcomes, timing. Works in all AI coding clients (Claude Code, Cursor, Codex, Gemini). - -- **Layer 2: Claude Code Hooks (structural)** — `PostToolUse` hooks that fire automatically on every Bash tool call. Captures *how* it happened: API calls to confidence.dev, HTTP status codes. Acts as a reliability backstop. Claude Code only — hooks are not supported in other clients. - -Both layers share a `session_id` for correlation. - -### Telemetry Endpoint - -``` -POST https://onboarding.confidence.dev/v1/telemetry:publish -Authorization: Bearer $TOKEN (or anonymous if no token available) -Content-Type: application/json -``` - -Events are fire-and-forget. The endpoint may not exist yet — that is fine. Curl calls will silently fail and never block the user. - ---- - -## Opt-In / Consent - -Controlled by the `CONFIDENCE_TELEMETRY` environment variable: - -| Value | Behavior | -|-------|----------| -| `1` | Telemetry enabled | -| `0` | Telemetry disabled — no events emitted | -| unset | Show one-time consent prompt per session | - -### Consent Prompt - -When `CONFIDENCE_TELEMETRY` is unset, show this prompt once at the start of the session: - -> Confidence collects anonymous usage data to improve onboarding. No secrets or personal details. Help us improve? (yes/no) - -Based on the response: -- **yes** -> `TELEMETRY_ENABLED=true` -- **no** -> `TELEMETRY_ENABLED=false` - -To set the variable for the session: - -```bash -export CONFIDENCE_TELEMETRY=1 # enable -export CONFIDENCE_TELEMETRY=0 # disable -``` - ---- - -## Session ID - -Generate once per skill invocation (at the start of each onboarding flow). Reuse across all events in that session. - -```bash -SESSION_ID=$(python3 -c "import uuid; print(uuid.uuid4())") -``` - -Pass `SESSION_ID` through all events and export it so hooks can read it: - -```bash -export SESSION_ID -``` - ---- - -## Emit Helper - -Skills should use this exact pattern to emit a telemetry event. Copy-paste and fill in `` and ``. - -```bash -if [ "$TELEMETRY_ENABLED" = "true" ]; then - curl -s -X POST "https://onboarding.confidence.dev/v1/telemetry:publish" \ - -H "Content-Type: application/json" \ - ${TOKEN:+-H "Authorization: Bearer $TOKEN"} \ - -d '{ - "session_id": "'$SESSION_ID'", - "events": [{ - "event_type": "", - "timestamp": "'$(date -u +%Y-%m-%dT%H:%M:%SZ)'", - "properties": {} - }] - }' > /dev/null 2>&1 & -fi -``` - -Key details: -- `> /dev/null 2>&1 &` — fire-and-forget, never blocks the user flow -- `${TOKEN:+-H "Authorization: Bearer $TOKEN"}` — includes auth header only if TOKEN is set (anonymous otherwise) -- Always use UTC timestamps via `date -u` - ---- - -## Event Catalog - -| Event | When | Key Properties | -|-------|------|----------------| -| `onboarding.session_started` | Skill begins | `skill`, `subcommand`, `is_dry_run`, `plugin_version` | -| `onboarding.step_started` | Step tracker updates to in-progress | `step_number`, `step_name` | -| `onboarding.step_completed` | Step tracker updates to done | `step_number`, `step_name`, `duration_ms` | -| `onboarding.step_failed` | Step encounters an error | `step_number`, `step_name`, `error_category` (NOT raw message) | -| `onboarding.user_choice` | User picks an option | `choice_key`, `choice_value` (from allowlist only) | -| `onboarding.session_completed` | Flow finishes successfully | `total_duration_ms`, `steps_completed` | -| `warehouse.type_selected` | User picks a warehouse type | `warehouse_type` | -| `warehouse.validation_result` | Validation endpoint returns | `successful`, `failed_checks` | -| `warehouse.connector_created` | Connector is created | `connector_type` | -| `warehouse.pipeline_verified` | Pipeline verification completes | `assignments_ok`, `events_ok` | -| `onboarding.identity_linked` | After re-auth gives org-scoped token (create-account Step 5) | `org_id`, `account_name`, `region` — ties anonymous pre-account events to the new account | -| `onboarding.sentiment` | After each step, LLM self-assessment of user tone | `step_name`, `sentiment` (`frustrated`, `confused`, `satisfied`), `confidence` (`high`, `medium`) — only emit when signal is clear, skip neutral | -| `onboarding.session_sentiment` | At session_completed | `overall` (`positive`, `mixed`, `negative`), `frustrated_steps`, `satisfied_steps`, `confused_steps` | -| `onboarding.feedback` | End of completed flow, user rates experience | `rating` (1-4), `rating_label` (`easy`, `okay`, `hard`, `broken`), `subcommand` | -| `onboarding.feedback_text` | Optional free-text follow-up for ratings 3-4 | `text` (volunteered by user, exception to no-freeform rule), `subcommand` | -| `onboarding.session_abandoned` | SessionEnd hook fires without session_completed | `last_step_number`, `last_step_name`, `total_duration_ms` | -| `tool.api_call` | Hook detects curl to confidence.dev | `api_path`, `http_method`, `http_status` | - -### Event Properties Reference - -Common properties included in every event: - -```json -{ - "skill": "onboard-confidence", - "subcommand": "create-account", - "plugin_version": "0.2.3", - "is_dry_run": false -} -``` - -### Error Categories (allowlist) - -Use these category strings in `error_category` — never log raw error messages: - -- `auth_expired` — token expired or invalid -- `auth_failed` — authentication failed -- `validation_failed` — input or permission validation failed -- `server_error` — 5xx from the API -- `network_error` — connection timeout or DNS failure -- `conflict` — resource already exists (409) -- `not_found` — resource not found (404) -- `rate_limited` — too many requests (429) -- `unknown` — unrecognized error - -### User Choice Keys (allowlist) - -Only these `choice_key` / `choice_value` pairs may be logged: - -| `choice_key` | Allowed `choice_value` values | -|--------------|-------------------------------| -| `region` | `eu`, `us` | -| `warehouse_type` | `bigquery`, `snowflake`, `databricks`, `redshift` | -| `auth_method` | `google`, `email`, `sso` | -| `variant_type` | `boolean`, `string`, `integer`, `double`, `struct` | -| `existing_client` | `yes`, `no` | - ---- - -## Sentiment Detection - -After each step completes, **silently assess the user's tone** from their messages during that step. Do NOT ask the user about their mood — just observe. - -Emit `onboarding.sentiment` ONLY when the signal is clear: - -| Sentiment | Signals | Examples | -|-----------|---------|----------| -| `frustrated` | Short angry messages, retries, confusion | "why?!", "doesn't work", "again?!", expletives | -| `confused` | Asking for clarification, wrong input repeatedly | "I don't understand", "what do you mean", long pauses | -| `satisfied` | Smooth progression, positive language | "great", "nice", "thanks", "easy", "perfect" | -| neutral | No clear signal | **Do not emit** — only emit when the signal is clear | - -At `session_completed`, also emit `onboarding.session_sentiment` with counts of frustrated/confused/satisfied steps and an overall assessment. - -## Identity Linking (create-account only) - -During `create-account`, the user has no token for Steps 1-4 (account doesn't exist yet). Events are emitted anonymously with only `session_id`. - -At Step 5 (re-auth), after obtaining the org-scoped token, emit `onboarding.identity_linked` to tie the anonymous session to the new account: - -```bash -# After re-auth in create-account Step 5 -emit_event "onboarding.identity_linked" 5 "connect_tools" \ - '"org_id": "'$ORG_ID'", "account_name": "'$ACCOUNT_ID'", "region": "'$REGION'"' -``` - -The backend stitches all events with the same `session_id` — before and after the identity link. - -## Feedback Prompt - -At the end of every **completed** flow (after the Done summary), ask: - -> How was this experience? -> 1. Easy — worked great -> 2. Okay — got there eventually -> 3. Hard — needed help -> 4. Broken — something didn't work - -Emit `onboarding.feedback` with the rating. - -For ratings 3 ("hard") or 4 ("broken"), ask an optional follow-up: - -> What was the hardest part? (optional, press Enter to skip) - -If the user provides text, emit `onboarding.feedback_text`. This is the one exception to the "no freeform input" privacy rule — the user explicitly volunteered the feedback. - -## Session Abandonment (Claude Code only) - -The `SessionEnd` hook checks if `SESSION_ID` is set but `SESSION_COMPLETED` is not. If so, it emits `onboarding.session_abandoned` with the last known step. This is handled by `hooks/session_end_telemetry.py`. - ---- - -## Privacy Rules - -**NEVER log any of the following:** -- Tokens, secrets, passwords, API keys -- Email addresses or personal identifiers -- Full API response bodies -- Full shell commands -- Workspace names, flag names, or other freeform user input - -**DO log:** -- Error categories from the allowlist above (not raw messages) -- User choices from the allowlist above (not freeform input) -- Step numbers and step names from the registry below -- Timing data (duration in milliseconds) -- Boolean outcomes (successful/failed, assignments_ok/events_ok) - ---- - -## Step Name Registry - -Canonical step names for each skill/subcommand. Use these exact strings in `step_name` properties. - -### create-account (6 steps) - -| Step | Name | -|------|------| -| 1 | `login` | -| 2 | `workspace_name` | -| 3 | `account_details` | -| 4 | `create_account` | -| 5 | `connect_tools` | -| 6 | `done` | - -### invite-user (4 steps) - -| Step | Name | -|------|------| -| 1 | `authenticate` | -| 2 | `target_account` | -| 3 | `invitation_details` | -| 4 | `send_invitation` | - -### create-client (3 steps) - -| Step | Name | -|------|------| -| 1 | `client_name` | -| 2 | `create_client` | -| 3 | `get_credentials` | - -### setup-wizard (6 steps) - -| Step | Name | -|------|------| -| 1 | `create_client` | -| 2 | `create_flag` | -| 3 | `add_variants` | -| 4 | `add_targeting` | -| 5 | `test_resolve` | -| 6 | `done` | - -### setup-warehouse dispatcher (1 step) - -| Step | Name | -|------|------| -| 1 | `choose_warehouse` | - -### setup-warehouse-bigquery (10 steps) - -| Step | Name | -|------|------| -| 1 | `choose_warehouse` | -| 2 | `gcp_project_id` | -| 3 | `dataset_name` | -| 4 | `service_account` | -| 5 | `validate_and_fix` | -| 6 | `create_warehouse` | -| 7 | `create_connectors` | -| 8 | `assignment_table` | -| 9 | `verify_pipeline` | -| 10 | `done` | - -### setup-warehouse-snowflake (12 steps) - -| Step | Name | -|------|------| -| 1 | `choose_warehouse` | -| 2 | `account_and_user` | -| 3 | `role_and_warehouse` | -| 4 | `database_and_schema` | -| 5 | `create_crypto_key` | -| 6 | `register_key_in_sf` | -| 7 | `validate` | -| 8 | `create_warehouse` | -| 9 | `create_connectors` | -| 10 | `assignment_table` | -| 11 | `verify_pipeline` | -| 12 | `done` | - -### setup-warehouse-databricks (13 steps) - -| Step | Name | -|------|------| -| 1 | `choose_warehouse` | -| 2 | `workspace_url` | -| 3 | `sql_warehouse_id` | -| 4 | `service_principal` | -| 5 | `aws_account_and_cli` | -| 6 | `s3_bucket` | -| 7 | `iam_role` | -| 8 | `databricks_schema` | -| 9 | `create_warehouse` | -| 10 | `create_connectors` | -| 11 | `assignment_table` | -| 12 | `verify_pipeline` | -| 13 | `done` | - -### setup-warehouse-redshift (13 steps) - -| Step | Name | -|------|------| -| 1 | `choose_warehouse` | -| 2 | `aws_account_and_cli` | -| 3 | `redshift_cluster` | -| 4 | `s3_bucket` | -| 5 | `iam_role` | -| 6 | `attach_role` | -| 7 | `schema_and_grants` | -| 8 | `validate` | -| 9 | `create_warehouse` | -| 10 | `create_connectors` | -| 11 | `assignment_table` | -| 12 | `verify_pipeline` | -| 13 | `done` | From d4f4115ce4f48baeb068c13462ce8e36d7803708 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Fri, 12 Jun 2026 13:04:07 +0200 Subject: [PATCH 24/25] feat: add bundled auth script and improve onboarding default flow Move auth.py into the skill directory so it ships with the plugin instead of being written to temp files at runtime. Update SKILL.md to go straight to the setup wizard when triggered without a sub-command, and refine auth flow to use the bundled script. Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 337 +++++++++++++++-------------- skills/onboard-confidence/auth.py | 95 ++++++++ 2 files changed, 269 insertions(+), 163 deletions(-) create mode 100644 skills/onboard-confidence/auth.py diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index d670b50..87bd20e 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -6,6 +6,15 @@ description: Create Confidence accounts and onboard users. Use when the user ask Create accounts, invite users, and get started with Confidence — all from the CLI. +## Default behavior (no sub-command) + +When the user says "onboard me", "get started with Confidence", or triggers this skill without a specific sub-command, go **straight to the setup wizard**. The first question is always: + +> 1. **Create a new account** — I'll walk you through signup +> 2. **Sign in to an existing account** — I already have one + +Do NOT show a menu of sub-commands. Do NOT offer "Setup Wizard" as a choice — it IS the default flow. The only decision the user needs to make upfront is whether they have an account. + ## Commands | Command | Description | @@ -35,138 +44,69 @@ Create accounts, invite users, and get started with Confidence — all from the ### Auth script -Write the following to `$TMPDIR/confidence_auth.py`, substituting CLIENT_ID and optional ORGANIZATION parameter. Run with `python3 $TMPDIR/confidence_auth.py`. Outputs `TOKEN:` on success. - -```python -import http.server, urllib.parse, json, sys, subprocess, hashlib, base64, secrets, string - -code_verifier = ''.join(secrets.choice(string.ascii_letters + string.digits + '-._~') for _ in range(43)) -code_challenge = base64.urlsafe_b64encode(hashlib.sha256(code_verifier.encode()).digest()).rstrip(b'=').decode() - -port = 8084 # Fixed — must match Auth0 Allowed Callback URLs -CLIENT_ID = '' -ORGANIZATION = '' # Set after account creation, empty for signup -REDIRECT_URI = f'http://localhost:{port}/callback' -auth_code = None -error = None - -class Handler(http.server.BaseHTTPRequestHandler): - def do_GET(self): - global auth_code, error - q = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query) - self.send_response(200) - self.send_header('Content-Type', 'text/html') - self.end_headers() - if 'code' in q: - auth_code = q['code'][0] - self.wfile.write(b'

Login successful!

You can close this tab.

') - else: - error = q.get('error', ['unknown'])[0] - self.wfile.write(b'

Login failed

Please try again.

') - def log_message(self, format, *args): - pass - -params = { - 'client_id': CLIENT_ID, - 'redirect_uri': REDIRECT_URI, - 'response_type': 'code', - 'scope': 'openid profile email offline_access', - 'audience': 'https://confidence.dev/', - 'code_challenge': code_challenge, - 'code_challenge_method': 'S256', -} -if ORGANIZATION: - params['organization'] = ORGANIZATION -else: - params['screen_hint'] = 'signup' - params['prompt'] = 'login' - -authorize_url = 'https://auth.confidence.dev/authorize?' + urllib.parse.urlencode(params) -subprocess.Popen(['open', authorize_url]) -print('WAITING_FOR_LOGIN', flush=True) - -server = http.server.HTTPServer(('127.0.0.1', port), Handler) -server.timeout = 120 -while auth_code is None and error is None: - server.handle_request() -server.server_close() - -if error: - print(f'AUTH_ERROR:{error}', flush=True) - sys.exit(1) - -import urllib.request -token_data = json.dumps({ - 'grant_type': 'authorization_code', - 'client_id': CLIENT_ID, - 'code': auth_code, - 'redirect_uri': REDIRECT_URI, - 'code_verifier': code_verifier -}).encode() -req = urllib.request.Request( - 'https://auth.confidence.dev/oauth/token', - data=token_data, - headers={'Content-Type': 'application/json'} -) -try: - with urllib.request.urlopen(req) as resp: - token_response = json.loads(resp.read()) - print(f'TOKEN:{token_response["access_token"]}', flush=True) -except Exception as e: - print(f'TOKEN_ERROR:{e}', flush=True) - sys.exit(1) +The auth script is **bundled in the plugin** as `auth.py` next to this SKILL.md. The path is shown in the "Base directory for this skill" header at the top of the loaded skill context. Do NOT write the script — just run it. + +**Usage — single Bash tool call** with `dangerouslyDisableSandbox: true` and `timeout: 130000`: +```bash +lsof -ti:8084 | xargs kill -9 2>/dev/null; python3 /auth.py [ORGANIZATION] +``` + +Replace `` with the actual path from the skill header (e.g., `/Users/.../confidence-ai-plugins/.claude/skills/onboard-confidence`). + +**Outputs on stdout** (parse line by line): +- `WAITING_FOR_LOGIN` — browser opened, waiting for callback +- `TOKEN:` — success, extract everything after `TOKEN:` +- `AUTH_ERROR:` — Auth0 returned an error +- `TOKEN_ERROR:` — token exchange failed + +**Examples:** + +Signup (no org): +```bash +lsof -ti:8084 | xargs kill -9 2>/dev/null; python3 /auth.py 82qMvwZvqd3t3S0gRDvs8R53TehQXSJY +``` + +Existing account login: +```bash +lsof -ti:8084 | xargs kill -9 2>/dev/null; python3 /auth.py 2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w org_abc123 ``` **Key details:** - Port is fixed at **8084** (must match Auth0 Allowed Callback URLs) -- For signup (`create-account`): no `organization`, add `screen_hint=signup` + `prompt=login` -- For existing account (all other commands): include `organization=` — auto-completes if browser session exists -- After `create-account`, automatically re-auth with `organization` param to get org-scoped token (browser auto-redirects, no interaction) -- If port 8084 is busy: `lsof -ti:8084 | xargs kill -9 2>/dev/null` -- All network commands require `dangerouslyDisableSandbox: true` +- For signup (`create-account`): omit ORGANIZATION arg → adds `screen_hint=signup` + `prompt=login` +- For existing account (all other commands): pass `ORGANIZATION=` → auto-completes if browser session exists +- After `create-account`, automatically re-auth with org param to get org-scoped token (browser auto-redirects, no interaction) +- All network commands require `dangerouslyDisableSandbox: true` and `timeout: 130000` ### Session-only token management The token is kept in the current session only and is never saved to disk. If the session ends or the token expires, the skill will open your browser to log in again. -**On every sub-command start**, check if the `TOKEN` variable is set and not expired: +**On every sub-command start**, check if the `TOKEN` variable is set and not expired. Combine the check + region extraction in a **single Bash command**: ```bash -if [ -n "$TOKEN" ]; then - PAYLOAD=$(echo "$TOKEN" | cut -d. -f2) - EXP=$(echo "$PAYLOAD" | python3 -c " -import sys, json, base64 -p = sys.stdin.read().strip() -p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' +echo "$TOKEN" | python3 -c " +import sys, json, base64, time +t = sys.stdin.read().strip() +if not t: + print('MISSING'); sys.exit(0) +p = t.split('.')[1] +p += '=' * (4 - len(p) % 4) d = json.loads(base64.b64decode(p)) -print(d.get('exp', 0)) -") - NOW=$(date +%s) - if [ "$EXP" -gt "$NOW" ]; then - echo "VALID" # Token still good — skip login - else - echo "EXPIRED" # Token expired — re-authenticate - unset TOKEN - fi -fi +if d.get('exp', 0) < time.time(): + print('EXPIRED'); sys.exit(0) +print('VALID') +print('REGION=' + d.get('https://confidence.dev/region', 'EU')) +print('ORG=' + d.get('org_id', '')) +print('ACCOUNT=' + d.get('https://confidence.dev/account_name', '')) +" ``` -If `TOKEN` is unset or expired, run the browser auth flow to get a new token. Store the result in the `TOKEN` shell variable only. **NEVER write the token to disk. NEVER reference `~/.confidence/`.** - -**Extract region from token** to determine API base URLs: +Output is multi-line: first line is `VALID`/`EXPIRED`/`MISSING`, followed by `REGION=EU`, `ORG=...`, `ACCOUNT=...` if valid. -```bash -PAYLOAD=$(echo "$TOKEN" | cut -d. -f2) -REGION=$(echo "$PAYLOAD" | python3 -c " -import sys, json, base64 -p = sys.stdin.read().strip() -p += '=' * (4 - len(p) % 4) if len(p) % 4 else '' -d = json.loads(base64.b64decode(p)) -print(d.get('https://confidence.dev/region', 'EU')) -") -``` +If `TOKEN` is unset or expired, run the browser auth flow to get a new token. Store the result in the `TOKEN` shell variable only. **NEVER write the token to disk. NEVER reference `~/.confidence/`.** -Then use `${REGION,,}` (lowercase) for URL prefix: `iam.eu.confidence.dev`, `flags.eu.confidence.dev`, etc. +Use the `REGION` value (lowercased) for URL prefixes: `iam.eu.confidence.dev`, `flags.eu.confidence.dev`, etc. If the token is valid, skip the login step entirely. If expired or missing, run the auth flow. @@ -182,6 +122,16 @@ Fields NOT in the body (like `flag_id`, `parent`) become **query parameters**. **Field names are `snake_case`** in requests. Responses may use `camelCase`. +### Speed: minimize tool calls + +**Every Bash tool call adds latency.** Optimize by combining commands: + +- **Chain independent curls** with `&&` or `;` in a single Bash call when the results don't depend on each other +- **Parallel API calls**: When creating multiple variants, setting schema + creating variants can be chained: `curl ... schema && curl ... variant1 && curl ... variant2` +- **Token check + region extract**: Do both in one command (see session-only token management) +- **Port kill + auth run**: Always combine: `lsof -ti:8084 | xargs kill -9 2>/dev/null; python3 ...` +- **Never use Write/Read tools** for temporary files — use Bash heredocs or bundled scripts + ### Common notes - All network commands require `dangerouslyDisableSandbox: true` @@ -196,8 +146,11 @@ Fields NOT in the body (like `flag_id`, `parent`) become **query parameters**. - Do NOT show raw JSON request/response bodies in conversation - Do NOT show Auth0 configuration details, token values, or OAuth internals +- Do NOT mention error codes, org IDs, JWT claims, token scoping, or API error details +- Do NOT ask the user for organization IDs, external IDs, or any auth-internal identifiers - DO show human-readable status updates: "Opening browser for login...", "Creating your workspace...", "Invitation sent!" - DO describe results in plain English +- DO handle all token re-issuance, org-scoping, and retry logic transparently — if something needs to happen behind the scenes (re-auth, polling, retry), just do it and show a friendly progress message - The agent handles all auth/API complexity silently **Step Tracker:** Display a visual step tracker at every phase transition. Update and re-display it each time you move to a new step. @@ -225,13 +178,25 @@ Use `●` for completed, `▶` for in-progress, `○` for pending. ### Step 1: Log in -Write the auth script to `$TMPDIR/confidence_auth.py` with the **signup client ID** (`82qMvwZvqd3t3S0gRDvs8R53TehQXSJY`). Run it and parse the TOKEN from stdout. +Run the bundled auth script with the **signup client ID** (`82qMvwZvqd3t3S0gRDvs8R53TehQXSJY`) and no organization arg. Parse the TOKEN and REFRESH_TOKEN from stdout. Tell the user: > Opening your browser to log in. Sign up with Google or create an account with email and password. +Store both `TOKEN` and `REFRESH_TOKEN` in shell variables. The refresh token is used in Step 5 to silently get an org-scoped token without opening the browser again. + If login fails, show the error in plain English and offer to retry. +**After successful login**, immediately extract the user's email by calling the Auth0 userinfo endpoint (combine with the token export in a single Bash call): +```bash +curl -s "https://konfidens.eu.auth0.com/userinfo" -H "Authorization: Bearer $TOKEN" +``` +Response: `{ "email": "user@company.com", "name": "...", ... }` + +Store the `email` value as `SIGNUP_EMAIL`. This is used to: +- Derive workspace name suggestions in Step 2 +- Pre-fill the admin email in Step 3 + ### Step 2: Workspace name EDUCATE then ASK: @@ -241,6 +206,8 @@ EDUCATE then ASK: > > **Rules:** 3-21 characters, lowercase letters, digits, and hyphens. Must start with a letter and end with a letter or digit. +**Suggest names derived from `SIGNUP_EMAIL`.** Extract the local part (before `@`), strip `+` suffixes, and generate 2-3 suggestions. For example, if `SIGNUP_EMAIL` is `jane+test@acme.com`, suggest `jane`, `jane-acme`, `acme-jane`. + Wait for user input. Then: 1. **Validate locally** against regex `^[a-z][a-z0-9-]{1,19}[a-z0-9]$` @@ -248,7 +215,7 @@ Wait for user input. Then: ```bash curl -s "https://onboarding.confidence.dev/v1/loginIdAvailability:check?login_id=${LOGIN_ID}" ``` -Response: `{ "available": true/false, "message": "..." }` +Response: `{ "available": true/false }` If taken, inform the user and suggest alternatives (append numbers, abbreviations). Re-ask. @@ -257,7 +224,7 @@ If taken, inform the user and suggest alternatives (append numbers, abbreviation Collect interactively, one field at a time: 1. **Display name** — the human-readable name for the workspace (company name). - Validate: 3-21 characters, starts with a letter, alphanumeric + spaces + hyphens. + Validate: 3-32 characters, starts with a letter/digit, alphanumeric + Unicode letters + spaces + hyphens. 2. **Region** — present as a choice: > Where should your data be stored? This **cannot be changed later**. @@ -271,15 +238,16 @@ Collect interactively, one field at a time: > 3. Both 4. **Admin email** — the email of the first admin user. Must be a **work email** — free email providers (Gmail, Yahoo, etc.) are rejected by the API. + **Default to `SIGNUP_EMAIL`** (the email from Step 1). Present it as the pre-filled suggestion. Only ask the user to change it if they want a different admin email. 5. **Allowed login email domains** — optional. Ask if they want to restrict login to a specific email domain (e.g., `@company.com`). ### Step 4: Create account -Build and send the request: +Build and send the request. Use `--max-time 120` to allow for slow gRPC provisioning: ```bash -curl -s -w "\n%{http_code}" -X POST "https://onboarding.confidence.dev/v1/accounts" \ +curl -s -w "\n%{http_code}" --max-time 120 -X POST "https://onboarding.confidence.dev/v1/accounts" \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{ @@ -317,25 +285,50 @@ Tell the user: |---|---|---| | 400 + "work email" | Free email rejected | "Confidence requires a work email address. Free providers like Gmail aren't allowed." | | 400 + "already have an account" | Logged-in Auth0 user already has account | "This login already has a Confidence account. Log in with a different email to create a new workspace." → re-run Step 1 | +| 400 + "under review" (code 9) | Email not verified yet | see "Under review retry" below | | 400 | Other validation error | Parse `.message`, show in plain English, re-collect the invalid field | | 401 | Token expired/invalid | "Session expired. Let me log you in again." → re-run Step 1 | | 409 | Name already taken | "That workspace name was just taken. Let's pick another." → re-run Step 2 | +| 504 / timeout | gRPC deadline exceeded | Retry up to 3 times with 3-second delays. If it still fails, tell the user: "The server is taking longer than usual. Let me try once more." | | 500+ | Server error | "Something went wrong on our end. Let me try again in a moment." | -### Step 5: Get account-scoped token +**Under review retry (code 9):** -The token from Step 1 has no `org_id` (it was issued before the account existed). Re-auth with the **regular client ID** and the `organization` parameter set to the `externalId` returned in Step 4. +Tell the user: "Please check your email for a verification link from Confidence and confirm your address. Let me know once you've done that!" -Run the auth script again with: -- `CLIENT_ID = '2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w'` (regular client) -- `ORGANIZATION = ''` +After the user confirms, **retry 4 times with 2-second delays** in a single Bash command — do NOT re-open the browser or re-authenticate: -This auto-completes in the browser — no login form, just a redirect. The new token will have `org_id`, `account_name`, and `region` claims. +```bash +for i in 1 2 3 4; do + RESP=$(curl -s -w "\n%{http_code}" --max-time 120 -X POST "https://onboarding.confidence.dev/v1/accounts" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '') + HTTP=$(echo "$RESP" | tail -1) + BODY=$(echo "$RESP" | sed '$d') + echo "ATTEMPT $i: HTTP=$HTTP" + echo "$BODY" + if [ "$HTTP" = "200" ]; then echo "SUCCESS"; break; fi + if [ "$HTTP" != "400" ] || ! echo "$BODY" | grep -q "under review"; then echo "DIFFERENT_ERROR"; break; fi + if [ "$i" -lt 4 ]; then sleep 2; fi +done +``` + +If all 4 attempts still return "under review", tell the user: "Verification hasn't propagated yet. Please wait a moment and let me know when you'd like to try again." + +### Step 5: Get account-scoped token -Store this token in the `TOKEN` shell variable. This is the token used for all subsequent commands in this session. **Do NOT save to disk.** +The token from Step 1 has no `org_id` (it was issued before the account existed). The signup client's refresh token **cannot** be exchanged for an org-scoped token — Auth0 rejects cross-client refresh, and the signup client doesn't support org-scoping. A browser auth with the regular client is required. + +**Use the browser auth script** with the **regular client ID** and the new org. The browser session from Step 1 is still active, so Auth0 auto-completes — the user sees no extra login prompt: +```bash +lsof -ti:8084 | xargs kill -9 2>/dev/null; python3 /auth.py 2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w +``` + +The response token will contain `org_id`, `account_name`, and `region` claims. Parse and store in the `TOKEN` shell variable. **Do NOT save to disk.** Tell the user: -> Activating your account... (browser will briefly flash) +> Connecting to your new workspace... (your browser will briefly open and close automatically — no action needed) Then suggest connecting MCP: > To connect Confidence tools for flag management, type `/mcp` and authenticate **confidence-flags**. @@ -356,10 +349,12 @@ Show a summary and next steps: URL: https://confidence.spotify.com Next steps: - • Invite team members: /onboard-confidence invite-user - • Create a feature flag: Ask me to create a flag, or use - the Confidence UI - • Integrate your app: Ask me for SDK setup instructions + • Run the setup wizard: /onboard-confidence setup-wizard + • Invite team members: /onboard-confidence invite-user + • Set up data warehouse: /onboard-confidence setup-warehouse + • Create a feature flag: Ask me or use the Confidence UI + • Integrate your app: Ask me for SDK setup instructions + • Learn experimentation: /onboard-confidence learn ═══════════════════════════════════════════════════════════════ ``` @@ -383,7 +378,7 @@ Show a summary and next steps: Check if a token is available from a prior `create-account` run in this session. -If not, write the auth script with the **regular client ID** (`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w`) — this user already has an account. +If not, run the bundled auth script with the **regular client ID** (`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w`) — this user already has an account. Validate the token works by calling: ```bash @@ -531,28 +526,41 @@ The `clientSecret.secret` is only returned once on creation — show it to the u Guided walkthrough of the full onboarding checklist. Uses REST APIs — no MCP needed. -### Prerequisites - -Requires an authenticated token. If none available in the current session, run login flow first. - -Determine the region from the token or ask the user — this sets the API base URLs: -- EU: `flags.eu.confidence.dev`, `resolver.eu.confidence.dev`, `iam.eu.confidence.dev` -- US: `flags.us.confidence.dev`, `resolver.us.confidence.dev`, `iam.us.confidence.dev` - ### Step Tracker ``` ───── Setup Wizard ──────────────────────────────────────── - [1] Create client ○ pending - [2] Create flag ○ pending - [3] Add variants ○ pending - [4] Add targeting ○ pending - [5] Test resolve ○ pending - [6] Done ○ pending + [1] Get started ○ pending + [2] Create client ○ pending + [3] Create flag ○ pending + [4] Add variants ○ pending + [5] Add targeting ○ pending + [6] Test resolve ○ pending + [7] Done ○ pending ──────────────────────────────────────────────────────────── ``` -### Step 1: Create client +### Step 1: Get started + +If the user already answered "create account" vs "sign in" (e.g., from the default onboarding flow), use that answer — do NOT re-ask. + +Otherwise (when entered directly via `/onboard-confidence setup-wizard`), ask: + +> Do you already have a Confidence account, or would you like to create one? +> 1. **Create a new account** — I'll walk you through signup +> 2. **Sign in to an existing account** — I already have one + +**If "Create a new account":** +Run the full `create-account` sub-command flow (Steps 1–6 from that section). This handles signup, workspace creation, and re-auth with an org-scoped token. Once complete, proceed to Step 2 of setup-wizard with the token and region already set. + +**If "Sign in to existing account":** +Check if a token is already available from a prior command in this session. If not, run the bundled auth script with the **regular client ID** (`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w`). Validate the token, extract the region, and proceed to Step 2. + +Determine the region from the token — this sets the API base URLs: +- EU: `flags.eu.confidence.dev`, `resolver.eu.confidence.dev`, `iam.eu.confidence.dev` +- US: `flags.us.confidence.dev`, `resolver.us.confidence.dev`, `iam.us.confidence.dev` + +### Step 2: Create client Check if the user already has a client: ```bash @@ -562,13 +570,13 @@ curl -s "https://iam.${REGION}.confidence.dev/v1/clients" \ If clients exist, ask which one to use. If none, run the `create-client` flow (REST). -Save the client `name` (e.g., `clients/abc123`) and the `clientSecret` for resolve in Step 5. If using an existing client, fetch its credentials: +Save the client `name` (e.g., `clients/abc123`) and the `clientSecret` for resolve in Step 6. If using an existing client, fetch its credentials: ```bash curl -s "https://iam.${REGION}.confidence.dev/v1/${CLIENT_NAME}/credentials" \ -H "Authorization: Bearer $TOKEN" ``` -### Step 2: Create flag +### Step 3: Create flag EDUCATE then ASK: > A feature flag controls a piece of functionality. Let's create your first one. @@ -584,9 +592,9 @@ curl -s -w "\n%{http_code}" -X POST "https://flags.${REGION}.confidence.dev/v1/f -d '{}' ``` -**Do NOT attach flag to client yet** — the schema update in Step 3 clears the client list. Attach after variants are added. +**Do NOT attach flag to client yet** — the schema update in Step 4 clears the client list. Attach after variants are added. -### Step 3: Add variants +### Step 4: Add variants EDUCATE: > Variants are the different values a flag can have. For a simple on/off flag, you'd have "on" and "off" variants. @@ -626,7 +634,7 @@ curl -s -X POST "https://flags.${REGION}.confidence.dev/v1/flags/:add -d '{"client": "", "flag": "flags/"}' ``` -### Step 4: Add targeting +### Step 5: Add targeting EDUCATE: > Targeting rules control who sees which variant. Let's set a default — you can add more rules later. @@ -674,7 +682,7 @@ curl -s -w "\n%{http_code}" -X POST "https://flags.${REGION}.confidence.dev/v1/f }' ``` -### Step 5: Test resolve +### Step 6: Test resolve EDUCATE: > Let's verify the flag works by resolving it. @@ -698,7 +706,7 @@ Parse the response and show in plain English: If resolve fails, check that the flag is attached to the client and has at least one enabled rule. -### Step 6: Done +### Step 7: Done ``` ═══════════════════════════════════════════════════════════════ @@ -712,9 +720,11 @@ If resolve fails, check that the flag is attached to the client and has at least Default: Your flag is live and resolving. Next steps: - • Integrate the SDK: Ask me for setup instructions - • Create more flags: Ask me or use the Confidence UI - • Set up experiments: /onboard-confidence learn + • Invite team members: /onboard-confidence invite-user + • Set up data warehouse: /onboard-confidence setup-warehouse + • Integrate the SDK: Ask me for setup instructions + • Create more flags: Ask me or use the Confidence UI + • Learn experimentation: /onboard-confidence learn ═══════════════════════════════════════════════════════════════ ``` @@ -867,7 +877,7 @@ METRICS_API: https://metrics.${region}.confidence.dev/v1 **Check login ID availability (no auth):** ``` GET ${ONBOARDING_API}/loginIdAvailability:check?login_id={id} -→ { "available": bool, "message": string } +→ { "available": bool } ``` **Check region availability (no auth):** @@ -1096,7 +1106,7 @@ Body: { "course": "courses/", "questionUpdates": [{ "lessonIndex": int | Field | Rule | Regex | |-------|------|-------| | `loginId` | 3-21 chars, lowercase, digits, hyphens. Starts with letter, ends with letter/digit | `^[a-z][a-z0-9-]{1,19}[a-z0-9]$` | -| `displayName` | 3-21 chars, letters, digits, spaces, hyphens. Starts with letter, ends with letter/digit | `^[a-zA-Z][a-zA-Z0-9\s-]{1,19}[a-zA-Z0-9]$` | +| `displayName` | 3-32 chars, letters, digits, Unicode letters, spaces, hyphens. Starts/ends with word char/digit/letter | `[\w\d\p{L}][\w\s\d\-\p{L}]{1,30}[\w\d\p{L}]` | | `region` | Exactly `REGION_EU` or `REGION_US` | — | | `authConnections` | At least one required | — | | `adminEmail` | Must be a work email. Free providers (Gmail, Yahoo, Hotmail, etc.) are rejected | — | @@ -1119,7 +1129,7 @@ Body: { "course": "courses/", "questionUpdates": [{ "lessonIndex": int ### Sandbox note -All `curl`, `open`, and `python3` commands that access external hosts (`auth.confidence.dev`, `onboarding.confidence.dev`, `iam.confidence.dev`) require `dangerouslyDisableSandbox: true`. On first occurrence, briefly explain to the user that network access outside the sandbox is needed for API calls. +All `curl`, `open`, and `python3` commands that access external hosts (`auth.confidence.dev`, `onboarding.confidence.dev`, `iam.confidence.dev`) require `dangerouslyDisableSandbox: true`. The auth script additionally requires `timeout: 130000` (server timeout is 120s). On first occurrence, briefly explain to the user that network access outside the sandbox is needed for API calls. --- @@ -1140,8 +1150,9 @@ Most sub-commands use REST APIs and do NOT require MCP. MCP is only used as a co ## Known Limitations - **MCP auth cannot be triggered programmatically** — user must run `/mcp` to authenticate MCP servers. The Auth0 browser session from the login step makes this instant (no second login). -- **Port 8084 must be free** — the Auth0 callback server uses a fixed port. If busy, kill the process first. +- **Port 8084 must be free** — the Auth0 callback server uses a fixed port. The auth script auto-kills any existing process on port 8084. - **Auth0 Allowed Callback URLs** — both Auth0 clients must have `http://localhost:8084/callback` in their Allowed Callback URLs, Allowed Logout URLs, and Allowed Web Origins. +- **Auth script is bundled** — `auth.py` ships with the plugin in the skill directory. Never write auth scripts to disk; always use the bundled script. - **Learning API** — REST-only (gRPC on epx-onboarding). Course content is generated by the skill using docs MCP; the API only tracks progress indices. - **`learn` sub-command** — uses docs MCP for content. If MCP not connected, the skill can still teach using its own knowledge but won't have the latest docs. - **Region-specific API URLs** — flags/resolver APIs use region prefixes (`flags.eu.confidence.dev` vs `flags.us.confidence.dev`). Determine region from the JWT token or from the account creation step. diff --git a/skills/onboard-confidence/auth.py b/skills/onboard-confidence/auth.py new file mode 100644 index 0000000..4d36eb2 --- /dev/null +++ b/skills/onboard-confidence/auth.py @@ -0,0 +1,95 @@ +""" +Confidence Auth0 PKCE login flow. + +Usage: + python3 auth.py [ORGANIZATION] + +Outputs on stdout: + WAITING_FOR_LOGIN — browser opened, waiting for callback + TOKEN: — success, JWT access token + REFRESH_TOKEN: — refresh token (if granted) + AUTH_ERROR: — auth0 returned an error + TOKEN_ERROR: — token exchange failed + +Exit codes: 0 = success, 1 = error +""" +import http.server, urllib.parse, json, sys, subprocess, hashlib, base64, secrets, string, signal + +CLIENT_ID = sys.argv[1] +ORGANIZATION = sys.argv[2] if len(sys.argv) > 2 else '' + +code_verifier = ''.join(secrets.choice(string.ascii_letters + string.digits + '-._~') for _ in range(43)) +code_challenge = base64.urlsafe_b64encode(hashlib.sha256(code_verifier.encode()).digest()).rstrip(b'=').decode() + +port = 8084 +REDIRECT_URI = f'http://localhost:{port}/callback' +auth_code = None +error = None + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + global auth_code, error + q = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query) + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.end_headers() + if 'code' in q: + auth_code = q['code'][0] + self.wfile.write(b'

Login successful!

You can close this tab.

') + else: + error = q.get('error', ['unknown'])[0] + self.wfile.write(b'

Login failed

Please try again.

') + def log_message(self, format, *args): + pass + +params = { + 'client_id': CLIENT_ID, + 'redirect_uri': REDIRECT_URI, + 'response_type': 'code', + 'scope': 'openid profile email offline_access', + 'audience': 'https://confidence.dev/', + 'code_challenge': code_challenge, + 'code_challenge_method': 'S256', +} +if ORGANIZATION: + params['organization'] = ORGANIZATION +else: + params['screen_hint'] = 'signup' + params['prompt'] = 'login' + +authorize_url = 'https://auth.confidence.dev/authorize?' + urllib.parse.urlencode(params) +subprocess.Popen(['open', authorize_url]) +print('WAITING_FOR_LOGIN', flush=True) + +server = http.server.HTTPServer(('127.0.0.1', port), Handler) +server.timeout = 120 +while auth_code is None and error is None: + server.handle_request() +server.server_close() + +if error: + print(f'AUTH_ERROR:{error}', flush=True) + sys.exit(1) + +import urllib.request +token_data = json.dumps({ + 'grant_type': 'authorization_code', + 'client_id': CLIENT_ID, + 'code': auth_code, + 'redirect_uri': REDIRECT_URI, + 'code_verifier': code_verifier +}).encode() +req = urllib.request.Request( + 'https://auth.confidence.dev/oauth/token', + data=token_data, + headers={'Content-Type': 'application/json'} +) +try: + with urllib.request.urlopen(req) as resp: + token_response = json.loads(resp.read()) + print(f'TOKEN:{token_response["access_token"]}', flush=True) + if 'refresh_token' in token_response: + print(f'REFRESH_TOKEN:{token_response["refresh_token"]}', flush=True) +except Exception as e: + print(f'TOKEN_ERROR:{e}', flush=True) + sys.exit(1) From 2712270971c3e32784d2ee7fa6ed6752af2194b2 Mon Sep 17 00:00:00 2001 From: vahid torkaman <692343+vahidlazio@users.noreply.github.com> Date: Fri, 12 Jun 2026 16:46:49 +0200 Subject: [PATCH 25/25] fix: TMPDIR mismatch between sandboxed and non-sandboxed Bash calls Sandboxed Bash calls use /tmp/claude-501/ while dangerouslyDisableSandbox calls use the system TMPDIR. Tokens saved in sandboxed calls were invisible to non-sandboxed curl calls, causing "Invalid JWT" errors. All token writes/reads now require dangerouslyDisableSandbox: true. Test resolve now passes context fields and tests all targeting cases. Co-Authored-By: Claude Opus 4.6 (1M context) --- skills/onboard-confidence/SKILL.md | 399 ++++++++++++++++++++--------- 1 file changed, 274 insertions(+), 125 deletions(-) diff --git a/skills/onboard-confidence/SKILL.md b/skills/onboard-confidence/SKILL.md index 87bd20e..12844f7 100644 --- a/skills/onboard-confidence/SKILL.md +++ b/skills/onboard-confidence/SKILL.md @@ -78,23 +78,36 @@ lsof -ti:8084 | xargs kill -9 2>/dev/null; python3 /auth.py 2fG3 - After `create-account`, automatically re-auth with org param to get org-scoped token (browser auto-redirects, no interaction) - All network commands require `dangerouslyDisableSandbox: true` and `timeout: 130000` -### Session-only token management +### Token management -The token is kept in the current session only and is never saved to disk. If the session ends or the token expires, the skill will open your browser to log in again. +Tokens are persisted to `$TMPDIR/confidence_token` (and optionally `$TMPDIR/confidence_refresh_token`). This avoids re-exporting the JWT on every Bash tool call. **NEVER write tokens to `~/.confidence/` or anywhere outside `$TMPDIR`.** -**On every sub-command start**, check if the `TOKEN` variable is set and not expired. Combine the check + region extraction in a **single Bash command**: +**CRITICAL: TMPDIR differs between sandboxed and non-sandboxed Bash calls.** Sandboxed calls use a path like `/tmp/claude-501/`, while `dangerouslyDisableSandbox: true` calls use the system TMPDIR (e.g., `/var/folders/.../T/`). If tokens are written in a sandboxed call but read in a non-sandboxed curl call, the curl will read a stale or missing token. **ALL token writes and reads MUST use `dangerouslyDisableSandbox: true`** to ensure a consistent TMPDIR path. This includes the auth script call (already non-sandboxed for network), the token save, the token validity check, and all curl calls. +**After every successful auth**, write the token to file — **in the same `dangerouslyDisableSandbox: true` Bash call** as the auth script or curl that produced it: ```bash -echo "$TOKEN" | python3 -c " -import sys, json, base64, time -t = sys.stdin.read().strip() +# Parse TOKEN from auth.py stdout and persist (same Bash call, same TMPDIR) +echo "" > "$TMPDIR/confidence_token" +``` + +**On every sub-command start**, check if the token file exists and is not expired. **This Bash call MUST use `dangerouslyDisableSandbox: true`** so it reads from the same TMPDIR that curl will use: + +```bash +# dangerouslyDisableSandbox: true +python3 -c " +import json, base64, time, os +p = os.path.join(os.environ.get('TMPDIR', '/tmp'), 'confidence_token') +try: + t = open(p).read().strip() +except FileNotFoundError: + print('MISSING'); exit(0) if not t: - print('MISSING'); sys.exit(0) -p = t.split('.')[1] -p += '=' * (4 - len(p) % 4) -d = json.loads(base64.b64decode(p)) + print('MISSING'); exit(0) +parts = t.split('.')[1] +parts += '=' * (4 - len(parts) % 4) +d = json.loads(base64.b64decode(parts)) if d.get('exp', 0) < time.time(): - print('EXPIRED'); sys.exit(0) + print('EXPIRED'); exit(0) print('VALID') print('REGION=' + d.get('https://confidence.dev/region', 'EU')) print('ORG=' + d.get('org_id', '')) @@ -104,11 +117,14 @@ print('ACCOUNT=' + d.get('https://confidence.dev/account_name', '')) Output is multi-line: first line is `VALID`/`EXPIRED`/`MISSING`, followed by `REGION=EU`, `ORG=...`, `ACCOUNT=...` if valid. -If `TOKEN` is unset or expired, run the browser auth flow to get a new token. Store the result in the `TOKEN` shell variable only. **NEVER write the token to disk. NEVER reference `~/.confidence/`.** +If expired or missing, run the browser auth flow and write the new token to the file. -Use the `REGION` value (lowercased) for URL prefixes: `iam.eu.confidence.dev`, `flags.eu.confidence.dev`, etc. +**In curl calls**, read from the file instead of a shell variable: +```bash +curl -s ... -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" +``` -If the token is valid, skip the login step entirely. If expired or missing, run the auth flow. +Use the `REGION` value (lowercased) for URL prefixes: `iam.eu.confidence.dev`, `flags.eu.confidence.dev`, etc. ### Important: gRPC-REST transcoding rules @@ -126,9 +142,9 @@ Fields NOT in the body (like `flag_id`, `parent`) become **query parameters**. **Every Bash tool call adds latency.** Optimize by combining commands: +- **Prefer MCP over REST** for flag/client operations — one MCP tool call replaces 3-5 chained curls - **Chain independent curls** with `&&` or `;` in a single Bash call when the results don't depend on each other -- **Parallel API calls**: When creating multiple variants, setting schema + creating variants can be chained: `curl ... schema && curl ... variant1 && curl ... variant2` -- **Token check + region extract**: Do both in one command (see session-only token management) +- **Token is in a file** — no need to export; just use `$(cat $TMPDIR/confidence_token)` in curl headers - **Port kill + auth run**: Always combine: `lsof -ti:8084 | xargs kill -9 2>/dev/null; python3 ...` - **Never use Write/Read tools** for temporary files — use Bash heredocs or bundled scripts @@ -155,6 +171,8 @@ Fields NOT in the body (like `flag_id`, `parent`) become **query parameters**. **Step Tracker:** Display a visual step tracker at every phase transition. Update and re-display it each time you move to a new step. +**Use AskUserQuestion for all choices.** Present options as selectable items (up/down/enter) — never numbered lists in plain text. Only ask the user to type when collecting free-text input like names or emails. + --- ## Sub-command: create-account @@ -183,13 +201,13 @@ Run the bundled auth script with the **signup client ID** (`82qMvwZvqd3t3S0gRDvs Tell the user: > Opening your browser to log in. Sign up with Google or create an account with email and password. -Store both `TOKEN` and `REFRESH_TOKEN` in shell variables. The refresh token is used in Step 5 to silently get an org-scoped token without opening the browser again. +Write `TOKEN` to `$TMPDIR/confidence_token` and `REFRESH_TOKEN` to `$TMPDIR/confidence_refresh_token`. **The token save and all subsequent reads MUST use `dangerouslyDisableSandbox: true`** to ensure consistent TMPDIR paths (see Token management section). If login fails, show the error in plain English and offer to retry. -**After successful login**, immediately extract the user's email by calling the Auth0 userinfo endpoint (combine with the token export in a single Bash call): +**After successful login**, immediately extract the user's email by calling the Auth0 userinfo endpoint — **combine the token save and userinfo curl in a single `dangerouslyDisableSandbox: true` Bash call**: ```bash -curl -s "https://konfidens.eu.auth0.com/userinfo" -H "Authorization: Bearer $TOKEN" +echo "" > "$TMPDIR/confidence_token" && echo "" > "$TMPDIR/confidence_refresh_token" && curl -s "https://konfidens.eu.auth0.com/userinfo" -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" ``` Response: `{ "email": "user@company.com", "name": "...", ... }` @@ -248,7 +266,7 @@ Build and send the request. Use `--max-time 120` to allow for slow gRPC provisio ```bash curl -s -w "\n%{http_code}" --max-time 120 -X POST "https://onboarding.confidence.dev/v1/accounts" \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ -H "Content-Type: application/json" \ -d '{ "account": { @@ -285,23 +303,29 @@ Tell the user: |---|---|---| | 400 + "work email" | Free email rejected | "Confidence requires a work email address. Free providers like Gmail aren't allowed." | | 400 + "already have an account" | Logged-in Auth0 user already has account | "This login already has a Confidence account. Log in with a different email to create a new workspace." → re-run Step 1 | -| 400 + "under review" (code 9) | Email not verified yet | see "Under review retry" below | +| 400 + code 9 | Account under review | see "Under review handling" below — **do NOT assume email verification** | | 400 | Other validation error | Parse `.message`, show in plain English, re-collect the invalid field | | 401 | Token expired/invalid | "Session expired. Let me log you in again." → re-run Step 1 | | 409 | Name already taken | "That workspace name was just taken. Let's pick another." → re-run Step 2 | | 504 / timeout | gRPC deadline exceeded | Retry up to 3 times with 3-second delays. If it still fails, tell the user: "The server is taking longer than usual. Let me try once more." | | 500+ | Server error | "Something went wrong on our end. Let me try again in a moment." | -**Under review retry (code 9):** +**Under review handling (code 9):** + +Code 9 means the account is "under review" — but the **reason** varies. Parse the `.message` field to determine the cause: + +1. **Email not verified** (message contains "verify" or "email"): Tell the user: "Please check your email for a verification link from Confidence and confirm your address. Let me know once you've done that!" -Tell the user: "Please check your email for a verification link from Confidence and confirm your address. Let me know once you've done that!" +2. **Account flagged/blocked** (message contains "fraud", "flagged", "blocked", "suspicious", or doesn't match #1): Tell the user: "Your account has been flagged for review. This usually resolves quickly. If it persists, contact support at confidence-support@spotify.com." -After the user confirms, **retry 4 times with 2-second delays** in a single Bash command — do NOT re-open the browser or re-authenticate: +3. **Generic "under review"** with no clear cause: Tell the user: "Your account is under review. This can happen for a few reasons — please check your email for any messages from Confidence. If you need help, contact confidence-support@spotify.com." + +**For case #1 only (email verification)**, after the user confirms, retry 4 times with 2-second delays in a single Bash command: ```bash for i in 1 2 3 4; do RESP=$(curl -s -w "\n%{http_code}" --max-time 120 -X POST "https://onboarding.confidence.dev/v1/accounts" \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ -H "Content-Type: application/json" \ -d '') HTTP=$(echo "$RESP" | tail -1) @@ -314,7 +338,9 @@ for i in 1 2 3 4; do done ``` -If all 4 attempts still return "under review", tell the user: "Verification hasn't propagated yet. Please wait a moment and let me know when you'd like to try again." +For cases #2 and #3, do NOT auto-retry — the issue won't resolve by retrying. Wait for the user to indicate they want to try again. + +If all 4 retry attempts still return "under review", tell the user: "Verification hasn't propagated yet. Please wait a moment and let me know when you'd like to try again." ### Step 5: Get account-scoped token @@ -325,15 +351,17 @@ The token from Step 1 has no `org_id` (it was issued before the account existed) lsof -ti:8084 | xargs kill -9 2>/dev/null; python3 /auth.py 2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w ``` -The response token will contain `org_id`, `account_name`, and `region` claims. Parse and store in the `TOKEN` shell variable. **Do NOT save to disk.** +The response token will contain `org_id`, `account_name`, and `region` claims. Parse the TOKEN and REFRESH_TOKEN from stdout, then **save them in a separate `dangerouslyDisableSandbox: true` Bash call**: + +```bash +echo "" > "$TMPDIR/confidence_token" && echo "" > "$TMPDIR/confidence_refresh_token" +``` + +**This save call MUST use `dangerouslyDisableSandbox: true`** — even though it doesn't need network access — so that `$TMPDIR` resolves to the same path that future curl calls will use. A sandboxed save writes to a different TMPDIR and the token will be invisible to non-sandboxed curl calls. Tell the user: > Connecting to your new workspace... (your browser will briefly open and close automatically — no action needed) -Then suggest connecting MCP: -> To connect Confidence tools for flag management, type `/mcp` and authenticate **confidence-flags**. -> Your browser session will auto-complete it — no extra login. - ### Step 6: Done Show a summary and next steps: @@ -383,7 +411,7 @@ If not, run the bundled auth script with the **regular client ID** (`2fG3H4RhlAb Validate the token works by calling: ```bash curl -s "https://iam.confidence.dev/v1/currentUser" \ - -H "Authorization: Bearer $TOKEN" + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" ``` ### Step 2: Target account @@ -412,7 +440,7 @@ For each email address: ```bash curl -s -w "\n%{http_code}" -X POST "https://iam.confidence.dev/v1/userInvitations" \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ -H "Content-Type: application/json" \ -d '{ "userInvitation": { @@ -483,7 +511,7 @@ Ask the user what to name the client. Suggest based on platform: Body is the client object directly (proto `body: "client"`): ```bash curl -s -w "\n%{http_code}" -X POST "https://iam.${REGION}.confidence.dev/v1/clients" \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ -H "Content-Type: application/json" \ -d '{"display_name": ""}' ``` @@ -495,7 +523,7 @@ Response includes `name` (e.g., `clients/kqr3nc9dh70cwt5e2vws`). Save this for S Body is the credential object directly (proto `body: "client_credential"`): ```bash curl -s -w "\n%{http_code}" -X POST "https://iam.${REGION}.confidence.dev/v1/${CLIENT_NAME}/credentials" \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ -H "Content-Type: application/json" \ -d '{"display_name": "Default Secret"}' ``` @@ -524,16 +552,20 @@ The `clientSecret.secret` is only returned once on creation — show it to the u ## Sub-command: setup-wizard -Guided walkthrough of the full onboarding checklist. Uses REST APIs — no MCP needed. +Guided walkthrough of the full onboarding checklist. Uses MCP tools for flag/client operations when available, REST for everything else. + +### User input style + +**Always use AskUserQuestion** with selectable options for choices (up/down/enter). Only ask the user to type free-text when collecting names, emails, or other open-ended input. Never present numbered lists in plain text when AskUserQuestion can be used instead. ### Step Tracker ``` ───── Setup Wizard ──────────────────────────────────────── [1] Get started ○ pending - [2] Create client ○ pending - [3] Create flag ○ pending - [4] Add variants ○ pending + [2] Connect tools ○ pending + [3] Create client ○ pending + [4] Create flag ○ pending [5] Add targeting ○ pending [6] Test resolve ○ pending [7] Done ○ pending @@ -544,39 +576,78 @@ Guided walkthrough of the full onboarding checklist. Uses REST APIs — no MCP n If the user already answered "create account" vs "sign in" (e.g., from the default onboarding flow), use that answer — do NOT re-ask. -Otherwise (when entered directly via `/onboard-confidence setup-wizard`), ask: - -> Do you already have a Confidence account, or would you like to create one? -> 1. **Create a new account** — I'll walk you through signup -> 2. **Sign in to an existing account** — I already have one +Otherwise (when entered directly via `/onboard-confidence setup-wizard`), use AskUserQuestion: +- **Create a new account** — I'll walk you through signup +- **Sign in to an existing account** — I already have one **If "Create a new account":** Run the full `create-account` sub-command flow (Steps 1–6 from that section). This handles signup, workspace creation, and re-auth with an org-scoped token. Once complete, proceed to Step 2 of setup-wizard with the token and region already set. **If "Sign in to existing account":** -Check if a token is already available from a prior command in this session. If not, run the bundled auth script with the **regular client ID** (`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w`). Validate the token, extract the region, and proceed to Step 2. +Check if a token file exists at `$TMPDIR/confidence_token` and is valid. If not, run the bundled auth script with the **regular client ID** (`2fG3H4RhlAbIZm9Rfn32zTaILH7w1X4w`). Validate the token, extract the region, and proceed to Step 2. Determine the region from the token — this sets the API base URLs: - EU: `flags.eu.confidence.dev`, `resolver.eu.confidence.dev`, `iam.eu.confidence.dev` - US: `flags.us.confidence.dev`, `resolver.us.confidence.dev`, `iam.us.confidence.dev` -### Step 2: Create client +### Step 2: Connect tools + +**This step is critical for onboarding success.** The Confidence MCP tools provide a richer, more reliable experience for managing flags and clients. Nudge the user to connect them now — it only takes a few seconds since their browser session from login will auto-complete. -Check if the user already has a client: +Tell the user: +> Before we create your first flag, let's connect the Confidence tools. This gives you richer flag management right inside Claude Code. +> +> Type **`/mcp`** in the prompt, then click **Authenticate** next to **confidence-flags**. Your browser session from login will auto-complete — no extra password needed. +> +> Let me know once you've done that! + +**After the user confirms**, verify MCP is connected by calling `mcp__confidence-flags__getIdentityInfo` (no args). If it succeeds, MCP is connected — set an internal flag `MCP_CONNECTED=true` and proceed. + +**If the user skips** or MCP call fails, proceed with REST fallback — set `MCP_CONNECTED=false`. Tell the user: +> No problem! I'll use the REST API instead. You can always connect the tools later with `/mcp`. + +### Step 3: Create client + +**MCP path** (when `MCP_CONNECTED=true`): + +Check if the user already has a client by calling `mcp__confidence-flags__listClients`. + +If clients exist, use AskUserQuestion to let the user pick one or create a new one. If none exist, ask for a client name and type: + +> What should we call this client? (e.g., "iOS App", "Web Frontend", "Backend Service") + +Then use AskUserQuestion for client type: +- **Frontend** — browser/mobile apps +- **Backend** — server-side services + +Call `mcp__confidence-flags__createClient` with `displayName` and `clientType`. +Then call `mcp__confidence-flags__getClientSecret` with the `clientName` to get the secret. + +**REST fallback** (when `MCP_CONNECTED=false`): + +Check existing clients: ```bash curl -s "https://iam.${REGION}.confidence.dev/v1/clients" \ - -H "Authorization: Bearer $TOKEN" + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" ``` -If clients exist, ask which one to use. If none, run the `create-client` flow (REST). +If clients exist, use AskUserQuestion to pick one. If none, create via REST: +```bash +curl -s -w "\n%{http_code}" -X POST "https://iam.${REGION}.confidence.dev/v1/clients" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ + -H "Content-Type: application/json" \ + -d '{"display_name": ""}' +``` -Save the client `name` (e.g., `clients/abc123`) and the `clientSecret` for resolve in Step 6. If using an existing client, fetch its credentials: +Then fetch credentials: ```bash curl -s "https://iam.${REGION}.confidence.dev/v1/${CLIENT_NAME}/credentials" \ - -H "Authorization: Bearer $TOKEN" + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" ``` -### Step 3: Create flag +Save the client `name` and `clientSecret` for later steps. + +### Step 4: Create flag EDUCATE then ASK: > A feature flag controls a piece of functionality. Let's create your first one. @@ -584,52 +655,49 @@ EDUCATE then ASK: Validate: 4-63 chars, `[a-z0-9-]`. -`flag_id` is a **query parameter**, body is the flag object (proto `body: "flag"`): -```bash -curl -s -w "\n%{http_code}" -X POST "https://flags.${REGION}.confidence.dev/v1/flags?flag_id=" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{}' -``` +Use AskUserQuestion for variant type: +- **Simple on/off (boolean)** — two variants: on and off +- **Custom variants** — I'll name my own -**Do NOT attach flag to client yet** — the schema update in Step 4 clears the client list. Attach after variants are added. +**MCP path** (when `MCP_CONNECTED=true`): -### Step 4: Add variants +The MCP `createFlag` tool handles schema, variants, AND client attachment in one call: -EDUCATE: -> Variants are the different values a flag can have. For a simple on/off flag, you'd have "on" and "off" variants. +For on/off: +``` +mcp__confidence-flags__createFlag({ + flagName: "", + clientName: "", + schemaObject: '{"enabled": "boolean"}', + variants: '[{"name": "on", "value": {"enabled": true}}, {"name": "off", "value": {"enabled": false}}]' +}) +``` -Ask the user: -> What variants should this flag have? -> 1. Simple on/off (boolean) -> 2. Custom variants (I'll name them) +For custom variants, infer the schema from what the user describes and pass it similarly. -**IMPORTANT: Set the flag schema BEFORE adding variants with values.** Variant values must match the schema. +**REST fallback** (when `MCP_CONNECTED=false`): + +Create flag, set schema, add variants, then attach to client — all in a single chained Bash call: -For on/off, first set schema: ```bash +curl -s -X POST "https://flags.${REGION}.confidence.dev/v1/flags?flag_id=" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ + -H "Content-Type: application/json" \ + -d '{}' && \ curl -s -X PATCH "https://flags.${REGION}.confidence.dev/v1/flags/" \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ -H "Content-Type: application/json" \ - -d '{"schema": {"schema": {"enabled": {"boolSchema": {}}}}}' -``` - -For custom variants, infer the schema from the value types the user describes and set it first. - -Then create each variant (body is the variant object directly, proto `body: "variant"`): -```bash -curl -s -w "\n%{http_code}" -X POST "https://flags.${REGION}.confidence.dev/v1/flags//variants" \ - -H "Authorization: Bearer $TOKEN" \ + -d '{"schema": {"schema": {"enabled": {"boolSchema": {}}}}}' && \ +curl -s -X POST "https://flags.${REGION}.confidence.dev/v1/flags//variants" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ -H "Content-Type: application/json" \ - -d '{"name": "flags//variants/", "value": {}}' -``` - -For on/off: create "on" with `{"enabled": true}` and "off" with `{"enabled": false}`. - -**After all variants are created**, attach the flag to the client: -```bash + -d '{"name": "flags//variants/on", "value": {"enabled": true}}' && \ +curl -s -X POST "https://flags.${REGION}.confidence.dev/v1/flags//variants" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ + -H "Content-Type: application/json" \ + -d '{"name": "flags//variants/off", "value": {"enabled": false}}' && \ curl -s -X POST "https://flags.${REGION}.confidence.dev/v1/flags/:addFlagClient" \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ -H "Content-Type: application/json" \ -d '{"client": "", "flag": "flags/"}' ``` @@ -639,33 +707,36 @@ curl -s -X POST "https://flags.${REGION}.confidence.dev/v1/flags/:add EDUCATE: > Targeting rules control who sees which variant. Let's set a default — you can add more rules later. -Ask: -> Which variant should be the default? +Use AskUserQuestion to pick the default variant (list the variants created in Step 4). -**First, create a catch-all segment** (if one doesn't exist) and allocate it to 100%: +**MCP path** (when `MCP_CONNECTED=true`): + +The MCP `addTargetingRule` tool handles segment creation internally: +``` +mcp__confidence-flags__addTargetingRule({ + flagName: "", + variantAllocations: '{"": 100}' +}) +``` + +**REST fallback** (when `MCP_CONNECTED=false`): + +Create a catch-all segment (if one doesn't exist), allocate it, then create a rule — all in one Bash call: ```bash curl -s -X POST "https://flags.${REGION}.confidence.dev/v1/segments?segment_id=everyone" \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ -H "Content-Type: application/json" \ - -d '{"display_name": "Everyone"}' - + -d '{"display_name": "Everyone"}' && \ curl -s -X PATCH "https://flags.${REGION}.confidence.dev/v1/segments/everyone" \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ -H "Content-Type: application/json" \ - -d '{"allocation": {"proportion": {"value": "1"}}}' - + -d '{"allocation": {"proportion": {"value": "1"}}}' && \ curl -s -X POST "https://flags.${REGION}.confidence.dev/v1/segments/everyone:allocate" \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ -H "Content-Type: application/json" \ - -d '{}' -``` - -**IMPORTANT:** Segment proportion must be > 0 and `:allocate` must be called, otherwise resolve returns empty. - -Then create a rule referencing the segment (body is rule object, proto `body: "rule"`): -```bash + -d '{}' && \ curl -s -w "\n%{http_code}" -X POST "https://flags.${REGION}.confidence.dev/v1/flags//rules" \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ -H "Content-Type: application/json" \ -d '{ "segment": "segments/everyone", @@ -682,32 +753,82 @@ curl -s -w "\n%{http_code}" -X POST "https://flags.${REGION}.confidence.dev/v1/f }' ``` +**IMPORTANT (REST only):** Segment proportion must be > 0 and `:allocate` must be called, otherwise resolve returns empty. + ### Step 6: Test resolve EDUCATE: -> Let's verify the flag works by resolving it. +> Let's verify the flag works by resolving it for different contexts. + +**Test all targeting cases.** If the flag has targeting rules that depend on context fields (e.g., `country`), resolve with context values that exercise EACH rule — both matching and non-matching cases. For example, if the rule is "on when country is not US", test with `country: "SE"` (should match → on) AND `country: "US"` (should not match → off/default). Show results for all cases in a summary table. + +**MCP path** (when `MCP_CONNECTED=true`): + +Make parallel resolve calls for each test case: +``` +mcp__confidence-flags__resolveFlag({ + flagName: "", + clientName: "", + entity: "targeting_key", + entityValue: "test-user-1", + context: '{"": ""}' +}) + +mcp__confidence-flags__resolveFlag({ + flagName: "", + clientName: "", + entity: "targeting_key", + entityValue: "test-user-1", + context: '{"": ""}' +}) +``` + +**REST fallback** (when `MCP_CONNECTED=false`): -Use the **resolver API** with the client secret (not Bearer token): ```bash +# Test matching case curl -s -w "\n%{http_code}" -X POST "https://resolver.${REGION}.confidence.dev/v1/flags:resolve" \ -H "Content-Type: application/json" \ -d '{ "flags": ["flags/"], "evaluationContext": { - "targeting_key": "test-user-1" + "targeting_key": "test-user-1", + "": "" + }, + "clientSecret": "", + "apply": true + }' && echo "---" && \ +# Test non-matching case +curl -s -w "\n%{http_code}" -X POST "https://resolver.${REGION}.confidence.dev/v1/flags:resolve" \ + -H "Content-Type: application/json" \ + -d '{ + "flags": ["flags/"], + "evaluationContext": { + "targeting_key": "test-user-1", + "": "" }, "clientSecret": "", "apply": true }' ``` -Parse the response and show in plain English: -> Flag **** resolved to variant **** — it works! +Show results in a summary: +``` + Test Results: + country = SE → variant "on" (enabled: true) ✓ + country = US → variant "off" (enabled: false) ✓ +``` -If resolve fails, check that the flag is attached to the client and has at least one enabled rule. +If resolve fails or returns no match, check that: +1. The flag is attached to the client +2. Rules are enabled +3. Context fields required by targeting rules are included in the resolve call +4. A catch-all rule exists for non-matching contexts (otherwise they fall through to code default) ### Step 7: Done +Show a summary, then offer SDK integration using the **confidence-docs MCP**: + ``` ═══════════════════════════════════════════════════════════════ Setup Complete! @@ -719,16 +840,22 @@ If resolve fails, check that the flag is attached to the client and has at least Variants: Default: - Your flag is live and resolving. Next steps: - • Invite team members: /onboard-confidence invite-user - • Set up data warehouse: /onboard-confidence setup-warehouse - • Integrate the SDK: Ask me for setup instructions - • Create more flags: Ask me or use the Confidence UI - • Learn experimentation: /onboard-confidence learn + Your flag is live and resolving! ═══════════════════════════════════════════════════════════════ ``` +Use AskUserQuestion for next steps: +- **Integrate the SDK** — get code snippets for your platform +- **Invite team members** — add collaborators to your workspace +- **Set up data warehouse** — connect analytics pipeline +- **Create more flags** — keep building +- **Learn experimentation** — interactive course on A/B testing + +**If the user picks "Integrate the SDK"**, use `mcp__confidence-docs__getCodeSnippetAndSdkIntegrationTips` with the user's platform (ask via AskUserQuestion: JavaScript, Python, Java, Kotlin, Swift, Go, React) to provide tailored integration code. This gives the user the exact SDK setup they need. + +**For other choices**, direct to the corresponding sub-command. + --- ## Sub-command: setup-warehouse @@ -781,7 +908,7 @@ Interactive learning about experimentation concepts. The skill teaches, asks que 6. **Track progress** — call the Learning API to record the user's answer: ```bash curl -s -X POST "https://onboarding.confidence.dev/v1/learningProgress:answerQuestions" \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" \ -H "Content-Type: application/json" \ -d '{ "course": "courses/", @@ -798,7 +925,7 @@ Interactive learning about experimentation concepts. The skill teaches, asks que 8. **Show progress** — at any time, fetch and display progress: ```bash curl -s "https://onboarding.confidence.dev/v1/learningProgress" \ - -H "Authorization: Bearer $TOKEN" + -H "Authorization: Bearer $(cat $TMPDIR/confidence_token)" ``` ``` @@ -813,6 +940,7 @@ Interactive learning about experimentation concepts. The skill teaches, asks que ### Key principles +- **Use AskUserQuestion** for topic selection, quiz answers, and continue/switch decisions — selectable options, not typed numbers - **Be conversational** — this is a dialogue, not a textbook - **Use real examples** — tie concepts to the user's product/domain when possible - **Encourage exploration** — if the user asks follow-up questions, answer them before moving on @@ -1133,26 +1261,47 @@ All `curl`, `open`, and `python3` commands that access external hosts (`auth.con --- -## Required MCP Tools (optional — only for `status` and `learn`) +## MCP Tools Reference -Most sub-commands use REST APIs and do NOT require MCP. MCP is only used as a convenience: +MCP tools are used for **flag and client operations only** — account creation, invitations, segments, and warehouse config always use REST. + +### confidence-flags MCP (flag/client operations) + +| Tool | Used by | Purpose | +|------|---------|---------| +| `mcp__confidence-flags__getIdentityInfo` | `status`, `setup-wizard` | Verify MCP connection, get identity | +| `mcp__confidence-flags__listClients` | `status`, `setup-wizard` | List available clients | +| `mcp__confidence-flags__createClient` | `setup-wizard` | Create SDK client with name + type | +| `mcp__confidence-flags__getClientSecret` | `setup-wizard` | Retrieve client secret | +| `mcp__confidence-flags__createFlag` | `setup-wizard` | Create flag with schema, variants, and client in one call | +| `mcp__confidence-flags__addTargetingRule` | `setup-wizard` | Add targeting rule with variant allocations (handles segments internally) | +| `mcp__confidence-flags__resolveFlag` | `setup-wizard` | Test flag resolution | + +### confidence-docs MCP (documentation) | Tool | Used by | Purpose | |------|---------|---------| -| `mcp__confidence-flags__getIdentityInfo` | `status` | Get current identity (convenience) | -| `mcp__confidence-flags__listClients` | `status` | List available clients (convenience) | | `mcp__confidence-docs__searchDocumentation` | `learn` | Fetch educational content | +| `mcp__confidence-docs__getCodeSnippetAndSdkIntegrationTips` | `setup-wizard` (Step 7) | SDK integration guides per platform | + +### What stays on REST (never use MCP) -**All other sub-commands (`create-account`, `invite-user`, `create-client`, `setup-wizard`, `setup-warehouse`) work entirely via REST APIs with the session auth token.** +- Account creation, email verification, login ID checks → `onboarding.confidence.dev` +- User invitations → `iam.*.confidence.dev` +- Segment creation and allocation → `flags.*.confidence.dev` +- Warehouse config, connectors, assignment tables → `metrics.*.confidence.dev`, `connectors.*.confidence.dev` +- Learning progress tracking → `onboarding.confidence.dev` --- ## Known Limitations -- **MCP auth cannot be triggered programmatically** — user must run `/mcp` to authenticate MCP servers. The Auth0 browser session from the login step makes this instant (no second login). +- **MCP auth cannot be triggered programmatically** — user must run `/mcp` to authenticate MCP servers. The Auth0 browser session from the login step makes this instant (no second login). The setup wizard nudges this at Step 2. +- **MCP is for flag/client operations only** — account creation, invitations, segments, warehouse config, and learning progress always use REST APIs. - **Port 8084 must be free** — the Auth0 callback server uses a fixed port. The auth script auto-kills any existing process on port 8084. - **Auth0 Allowed Callback URLs** — both Auth0 clients must have `http://localhost:8084/callback` in their Allowed Callback URLs, Allowed Logout URLs, and Allowed Web Origins. - **Auth script is bundled** — `auth.py` ships with the plugin in the skill directory. Never write auth scripts to disk; always use the bundled script. +- **Token persistence and TMPDIR** — tokens are written to `$TMPDIR/confidence_token`. `$TMPDIR` resolves to DIFFERENT paths in sandboxed vs non-sandboxed (`dangerouslyDisableSandbox: true`) Bash calls (e.g., `/tmp/claude-501/` vs `/var/folders/.../T/`). ALL token writes and reads MUST use `dangerouslyDisableSandbox: true` to ensure consistency. Never write tokens outside `$TMPDIR`. - **Learning API** — REST-only (gRPC on epx-onboarding). Course content is generated by the skill using docs MCP; the API only tracks progress indices. - **`learn` sub-command** — uses docs MCP for content. If MCP not connected, the skill can still teach using its own knowledge but won't have the latest docs. - **Region-specific API URLs** — flags/resolver APIs use region prefixes (`flags.eu.confidence.dev` vs `flags.us.confidence.dev`). Determine region from the JWT token or from the account creation step.