diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index c01f0ac..54cce27 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -7,7 +7,7 @@ }, "metadata": { "description": "StackOne agent skills — integration infrastructure for AI agents with 200+ connectors, MCP, A2A, and SDKs", - "version": "2.0.0" + "version": "2.1.0" }, "plugins": [ { @@ -17,12 +17,14 @@ "repo": "StackOneHQ/agent-plugins-marketplace" }, "description": "Integration infrastructure for AI agents — account linking, 200+ connectors, 10,000+ actions, TypeScript/Python SDKs, MCP, A2A, and connector development CLI", - "version": "2.0.0", + "version": "2.1.0", "category": "integrations", "tags": [ "stackone", "integrations", "integration-infrastructure", + "unified-connectors", + "connector-development", "hris", "ats", "crm", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 4e44315..3aad116 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "stackone", - "version": "2.0.0", - "description": "Integration infrastructure for AI agents — account linking, 200+ connectors, 10,000+ actions, TypeScript/Python SDKs, MCP, A2A, and connector development CLI", + "version": "2.1.0", + "description": "Integration infrastructure for AI agents — account linking, 200+ connectors, 10,000+ actions, TypeScript/Python SDKs, MCP, A2A, unified connector development, and CLI", "author": { "name": "StackOne", "url": "https://stackone.com" @@ -13,6 +13,8 @@ "stackone", "integrations", "integration-infrastructure", + "unified-connectors", + "connector-development", "hris", "ats", "crm", diff --git a/README.md b/README.md index e2d7378..506392a 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Agent skills for [StackOne](https://stackone.com) — integration infrastructure # Add the marketplace /plugin marketplace add stackonehq/agent-plugins-marketplace -# Install the StackOne plugin (all 5 skills) +# Install the StackOne plugin (all 6 skills) /plugin install stackone@stackone-marketplace ``` @@ -33,6 +33,7 @@ npx skills add stackonehq/agent-plugins-marketplace@stackone-agents | [`stackone-agents`](skills/stackone-agents/) | Build AI agents with TypeScript/Python SDK, MCP, or A2A | "Add StackOne tools to my agent", "set up MCP" | | [`stackone-cli`](skills/stackone-cli/) | Custom connector development and deployment | "Build a custom connector", "deploy my connector" | | [`stackone-connectors`](skills/stackone-connectors/) | Discover connectors, actions, and integration capabilities | "Which providers does StackOne support?" | +| [`stackone-unified-connectors`](skills/stackone-unified-connectors/) | Build unified connectors that transform provider data into standardized schemas | "start unified build for [provider]", "map fields to schema" | Each skill includes step-by-step workflows, concrete examples, and troubleshooting for common errors. diff --git a/skills/stackone-cli/SKILL.md b/skills/stackone-cli/SKILL.md index d727286..1d363d0 100644 --- a/skills/stackone-cli/SKILL.md +++ b/skills/stackone-cli/SKILL.md @@ -107,3 +107,9 @@ Result: Working GitHub Actions pipeline for connector deployment. - Fetch the CLI reference for deployment troubleshooting - Common issues: missing required fields in connector config, network timeouts - For CI/CD failures, check that secrets are correctly configured in GitHub Actions + +## Related Skills + +- **stackone-unified-connectors**: For building schema-based connectors that transform provider data into standardized schemas with field mapping, enum translation, and unified pagination +- **stackone-connectors**: For discovering existing connector capabilities +- **stackone-agents**: For building AI agents that use connectors diff --git a/skills/stackone-unified-connectors/SKILL.md b/skills/stackone-unified-connectors/SKILL.md new file mode 100644 index 0000000..aae2a82 --- /dev/null +++ b/skills/stackone-unified-connectors/SKILL.md @@ -0,0 +1,458 @@ +--- +name: stackone-unified-connectors +description: Baseline skill for building unified/schema-based connectors that transform provider data into standardized schemas. Use alongside domain-specific schema skills (e.g., unified-hris-schema, unified-crm-schema) that define your organization's standard schemas. Use when user says "start unified build for [provider]", "build a schema-based connector", "map fields to schema", "test unified connector", or asks about field mapping, enum mapping, pagination configuration, or scope decisions. This skill provides implementation patterns; schema skills provide field definitions. Do NOT use for agentic/custom connectors (use stackone-cli), discovering existing connectors (use stackone-connectors), or building AI agents (use stackone-agents). +license: MIT +compatibility: Requires StackOne CLI (@stackone/cli). Requires access to provider API documentation. +metadata: + author: stackone + version: "2.1" +--- + +# StackOne Unified Connectors + +Build connectors that transform provider-specific data into standardized schemas with consistent field names, enum values, and pagination. + +## Skill Architecture + +This is a **baseline skill** that provides the core workflow and patterns for building unified connectors. It is designed to work alongside **domain-specific schema skills** that you create for your organization's specific use cases. + +**Recommended approach:** +1. Use this skill as your foundation for all unified connector work +2. Create domain-specific skills for each category you build connectors for (e.g., `unified-hris-schema`, `unified-messaging-schema`, `unified-crm-schema`) +3. Your domain-specific skills provide the schema definitions, field naming conventions, and enum value standards +4. This baseline skill provides the implementation patterns, CLI commands, and troubleshooting guidance + +This separation allows you to maintain consistent schemas across all providers within a category while leveraging the shared technical patterns from this baseline skill. + +## Important + +Before building unified connectors: +1. Read the CLI documentation: https://docs.stackone.com/guides/connector-engine/cli-reference +2. Use `stackone help ` for command-specific details +3. Always verify response structures with `--debug` before configuring mappings + +## Core Principles + +These principles apply to ALL unified connector work. Violations cause silent failures or broken mappings. + +### 1. All Config Field Names Use camelCase + +Every YAML configuration field uses camelCase, not snake_case: + +```yaml +# CORRECT +scopeDefinitions: fieldConfigs: targetFieldKey: +enumMapper: matchExpression: dataKey: +nextKey: pageSize: indexField: +stepFunction: functionName: dataSource: + +# WRONG - causes validation errors or silent failures +scope_definitions: field_configs: target_field_key: +``` + +### 2. Schema Field Names Use YOUR Naming Convention + +While config fields are camelCase, `targetFieldKey` values match YOUR schema (often snake_case): + +```yaml +fieldConfigs: + - targetFieldKey: first_name # YOUR schema field + expression: $.firstName # Provider's field +``` + +### 3. Always Use Version '2' for map_fields and typecast + +```yaml +stepFunction: + functionName: map_fields + version: '2' # REQUIRED - omitting causes empty results +``` + +### 4. Use Inline Fields in map_fields Parameters + +Pass `fields` directly in `map_fields` step parameters rather than action-level `fieldConfigs`. This avoids schema inference issues that cause build failures. + +```yaml +# RECOMMENDED - Inline fields +- stepId: map_data + stepFunction: + functionName: map_fields + version: '2' + parameters: + fields: + - targetFieldKey: email + expression: $.email # Direct reference, NO step prefix + type: string + dataSource: $.steps.get_data.output.data +``` + +### 5. Expression Context Depends on Location + +| Location | Expression Format | Example | +|----------|------------------|---------| +| Inline in `parameters.fields` | Direct field reference | `$.email`, `$.work.department` | +| Action-level `fieldConfigs` | Step ID prefix required | `$.get_employees.email` | + +### 6. Never Suggest User-Side Mapping + +The entire purpose of unified connectors is standardized output. Never suggest users handle mapping in application code. + +### 7. Verify Every Path Against Raw Response + +Never assume response structure. Always run with `--debug` first: + +```bash +stackone run --debug --connector --credentials --action-id +``` + +## Instructions + +### Step 1: Resolve Schema + +**Check for a domain-specific schema skill first.** + +Domain-specific schema skills (e.g., `unified-hris-schema`, `unified-crm-schema`) should define your organization's standard schema for that category. These skills complement this baseline skill by providing: +- Target field names and types +- Enum values and their meanings +- Required vs optional fields +- Nested object structures + +**If schema skill exists:** +- Use the schema definitions from that skill immediately +- Confirm which resource you're building (e.g., "employees", "contacts") +- Proceed to Step 2 + +**If no schema skill exists:** +- Ask user for schema in any format (YAML, JSON, markdown table, field list) +- Recommend creating a domain-specific schema skill for future builds in this category +- This ensures consistency across all providers you integrate + +**What a schema skill should contain:** +```yaml +# Example: unified-hris-schema skill structure +# - Field definitions with types +# - Enum values (e.g., employment_status: active, inactive, terminated) +# - Required fields marked +# - Nested structures documented +``` + +Creating domain-specific schema skills prevents drift between providers and reduces repeated schema discussions. + +### Step 2: Research Provider Endpoints (MANDATORY) + +**Do not skip this step.** Research ALL available endpoints before proceeding. + +For each endpoint, document: +- **Field Coverage**: Which schema fields does it return? +- **Performance**: Pagination support, rate limits +- **Permissions**: Required scopes (narrower is better) +- **Deprecation**: Never use deprecated endpoints + +### Step 3: Present Options to User (CHECKPOINT) + +Present a comparison table and get explicit user approval before implementing: + +```markdown +| Option | Endpoint | Field Coverage | Permissions | Status | +|--------|----------|----------------|-------------|--------| +| A | GET /v2/employees | 70% | Narrow | Active | +| B | POST /reports | 100% | Moderate | Active | +| C | POST /v1/data | 100% | Broad | Deprecated | + +Recommendation: Option B - Full coverage, not deprecated +``` + +**Do not proceed without user selection.** + +### Step 4: Configure Scopes + +Use `scopeDefinitions` (not `scope_definitions`): + +```yaml +scopeDefinitions: + employees:read: + description: Read employee data + employees:extended:read: + description: Extended employee data + includes: employees:read # Scope inheritance +``` + +**Principles:** +- Narrower scopes always preferred +- Never use deprecated endpoints +- Document trade-offs explicitly + +See `references/scope-patterns.md` for detailed patterns. + +### Step 5: Map Fields to Schema + +Use inline fields in map_fields parameters: + +```yaml +steps: + - stepId: map_data + stepFunction: + functionName: map_fields + version: '2' + parameters: + fields: + - targetFieldKey: id + expression: $.id + type: string + - targetFieldKey: email + expression: $.email + type: string + - targetFieldKey: department + expression: $.work.department # Nested field + type: string + - targetFieldKey: status + expression: $.status + type: enum + enumMapper: + matcher: + - matchExpression: '{{$.status == "Active"}}' + value: active + - matchExpression: '{{$.status == "Inactive"}}' + value: inactive + - matchExpression: '{{$.status == null}}' + value: unknown + dataSource: $.steps.get_data.output.data + + - stepId: typecast_data + stepFunction: + functionName: typecast + version: '2' + parameters: + fields: + - targetFieldKey: id + type: string + - targetFieldKey: email + type: string + - targetFieldKey: department + type: string + - targetFieldKey: status + type: enum + dataSource: $.steps.map_data.output.data + +result: + data: $.steps.typecast_data.output.data +``` + +See `references/field-mapping-patterns.md` for enum mapping, nested objects, and transformations. + +### Step 6: Configure Pagination + +For list endpoints, use cursor pagination with the `request` function: + +```yaml +cursor: + enabled: true + pageSize: 50 + +inputs: + - name: page_size + type: number + in: query + required: false + - name: cursor + type: string + in: query + required: false + +steps: + - stepId: get_data + stepFunction: + functionName: request + parameters: + url: /items + method: get + args: + # Dual-condition pattern for defaults + - name: limit + value: $.inputs.page_size + in: query + condition: "{{present(inputs.page_size)}}" + - name: limit + value: 50 + in: query + condition: "{{!present(inputs.page_size)}}" + - name: cursor + value: $.inputs.cursor + in: query + condition: "{{present(inputs.cursor)}}" + +result: + data: $.steps.get_data.output.data + next: $.steps.get_data.output.data.meta.nextCursor +``` + +**Important:** Use `request` function (not `paginated_request`) when you need dynamic inputs like `page_size`. The `paginated_request` function can have issues with `$.inputs.*` resolving to `undefined`. + +See `references/pagination-patterns.md` for detailed configuration. + +### Step 7: Validate Configuration + +```bash +stackone validate connectors//.connector.s1.yaml +``` + +### Step 8: Test Mappings + +**Phase 1: Raw Response** +```bash +stackone run --debug --connector --credentials --action-id +``` + +**Phase 2: Field Mapping** - Verify all fields use YOUR schema names, not provider names + +**Phase 3: Pagination** - Test first page, next page, last page, empty results + +**Phase 4: Schema Completeness** - All required fields present and populated + +### Step 9: Document Coverage + +Create a coverage document listing: +- Required fields: mapped status +- Optional fields: mapped or documented why not +- Scopes required +- Limitations + +## Examples + +### Example 1: Building a unified employee connector (with schema skill) + +User says: "start unified build for BambooHR" + +Actions: +1. Check for domain-specific schema skill (e.g., `unified-hris-schema`) +2. **If skill exists**: Load employee schema from skill, confirm fields, proceed +3. **If no skill**: Ask for schema, recommend creating `unified-hris-schema` skill for consistency across HRIS providers +4. Research BambooHR endpoints: `/v1/employees`, `/v1/employees/directory`, custom reports +5. Present options with trade-offs (field coverage, scopes, deprecation) +6. After user selects, implement map_fields with inline fields using schema from skill +7. Configure pagination with cursor support +8. Test with `--debug`, verify field names match schema +9. Document coverage + +Result: Working unified connector with standardized employee schema that matches other HRIS connectors. + +### Example 2: First connector in a new category + +User says: "build a unified messaging connector for Slack" + +Actions: +1. Check for `unified-messaging-schema` skill - none exists +2. Ask: "I don't see a messaging schema skill. What fields do you need for messages? I recommend we create a `unified-messaging-schema` skill so future messaging connectors (Teams, Discord) use the same schema." +3. Collaborate on schema definition +4. Suggest creating the schema skill before proceeding +5. Once schema is defined, proceed with standard workflow + +Result: New schema skill created, connector built, future messaging connectors will use same schema. + +### Example 3: Debugging field mapping issues + +User says: "My unified connector returns provider field names instead of my schema" + +Actions: +1. Check if `targetFieldKey` uses YOUR schema names (not provider names) +2. Verify `version: '2'` is specified on map_fields and typecast +3. Check expression context - inline fields should NOT have step prefix +4. Run with `--debug` to see raw response structure +5. Verify dataSource path is correct + +Result: Fields correctly mapped to user's schema. + +### Example 4: Pagination not working + +User says: "Pagination cursor isn't being passed correctly" + +Actions: +1. Run with `--debug` to see raw response structure +2. Verify `dataKey` path matches actual response (e.g., `data.employees` not just `employees`) +3. Verify `nextKey` path points to cursor value +4. Check if using `paginated_request` with dynamic inputs - switch to `request` with dual-condition pattern +5. Verify `result.next` returns the cursor value + +Result: Working pagination with correct cursor handling. + +## Troubleshooting + +### Fields return provider names instead of schema names +**Cause**: Missing or incorrect field mapping configuration. +**Fix**: Ensure `targetFieldKey` uses YOUR schema field names, not provider names. Verify map_fields step is present and dataSource is correct. + +### Mapping produces empty results +**Cause**: Missing `version: '2'` or wrong expression context. +**Fix**: Add `version: '2'` to map_fields and typecast. For inline fields, use direct references (`$.email`) without step prefix. + +### Enum values not translating +**Cause**: `matchExpression` doesn't match provider values (case-sensitive). +**Fix**: Check exact provider values with `--debug`. Use `.toLowerCase()` for case-insensitive matching. Always include null/unknown fallback. + +### Pagination returns same records +**Cause**: Cursor not being sent or extracted correctly. +**Fix**: Verify `iterator.key` matches API's expected parameter name. Check `nextKey` path against raw response. Verify `iterator.in` is correct (query/body/headers). + +### Build fails with schema inference errors +**Cause**: Action-level `fieldConfigs` triggering unwanted schema inference. +**Fix**: Use inline fields in `map_fields` parameters instead of action-level `fieldConfigs`. + +### Dynamic inputs resolve to undefined +**Cause**: Using `paginated_request` which doesn't handle `$.inputs.*` well. +**Fix**: Use standard `request` function with dual-condition pattern for defaults. + +## Key URLs + +| Resource | URL | +|----------|-----| +| CLI Package | https://www.npmjs.com/package/@stackone/cli | +| Connector Engine Docs | https://docs.stackone.com/guides/connector-engine | +| CLI Reference | https://docs.stackone.com/guides/connector-engine/cli-reference | + +## Creating Domain-Specific Schema Skills + +To maintain consistency across providers, create schema skills for each category you work with. A domain-specific schema skill should include: + +**Required content:** +- Field definitions with types (`string`, `number`, `boolean`, `datetime_string`, `enum`) +- Enum value definitions with descriptions +- Required vs optional field indicators +- Nested object structures + +**Example skill structure:** +```markdown +# Unified HRIS Schema + +## Employee Resource + +### Required Fields +| Field | Type | Description | +|-------|------|-------------| +| id | string | Unique identifier | +| email | string | Primary email address | +| first_name | string | Employee first name | +| last_name | string | Employee last name | +| employment_status | enum | Current employment status | + +### Enum: employment_status +| Value | Description | +|-------|-------------| +| active | Currently employed | +| inactive | On leave or suspended | +| terminated | No longer employed | + +### Optional Fields +| Field | Type | Description | +|-------|------|-------------| +| department | string | Department name | +| hire_date | datetime_string | Date of hire | +``` + +**Naming convention:** `unified-{category}-schema` (e.g., `unified-hris-schema`, `unified-crm-schema`, `unified-messaging-schema`) + +## Related Skills + +- **stackone-cli**: For deploying connectors and CLI commands +- **stackone-connectors**: For discovering existing connector capabilities +- **stackone-agents**: For building AI agents that use connectors +- **Your domain-specific schema skills**: For category-specific schemas (create as needed) diff --git a/skills/stackone-unified-connectors/references/field-mapping-patterns.md b/skills/stackone-unified-connectors/references/field-mapping-patterns.md new file mode 100644 index 0000000..cfe0fe4 --- /dev/null +++ b/skills/stackone-unified-connectors/references/field-mapping-patterns.md @@ -0,0 +1,297 @@ +# Field Mapping Patterns Reference + +**IMPORTANT**: This reference may become outdated. Always verify patterns against actual connector behavior with `--debug`. + +## Field Types + +| Type | Description | Example | +|------|-------------|---------| +| `string` | Text values | Names, IDs, emails | +| `number` | Numeric values | Counts, amounts | +| `boolean` | True/false | is_active | +| `datetime_string` | ISO date strings | hire_date | +| `enum` | Constrained values | status (requires enumMapper) | +| `object` | Nested structure | work_location | + +## Inline Fields (Recommended Approach) + +Define fields directly in `map_fields` step parameters: + +```yaml +- stepId: map_data + stepFunction: + functionName: map_fields + version: '2' + parameters: + fields: + - targetFieldKey: email + expression: $.email # Direct reference, NO step prefix + type: string + - targetFieldKey: department + expression: $.work.department # Nested field reference + type: string + dataSource: $.steps.get_data.output.data +``` + +**Why inline?** Action-level `fieldConfigs` can trigger schema inference that adds unwanted properties, causing build failures. + +## Enum Mapping + +### Basic Enum + +```yaml +fields: + - targetFieldKey: status + expression: $.status + type: enum + enumMapper: + matcher: + - matchExpression: '{{$.status == "Active"}}' + value: active + - matchExpression: '{{$.status == "Inactive"}}' + value: inactive + - matchExpression: '{{$.status == null}}' + value: unknown +``` + +### Case-Insensitive Matching + +```yaml +enumMapper: + matcher: + - matchExpression: '{{($.status || "").toLowerCase() == "active"}}' + value: active +``` + +### Multiple Source Values + +```yaml +enumMapper: + matcher: + - matchExpression: '{{$.type == "Full-Time" || $.type == "FT"}}' + value: full_time +``` + +### Built-in Mappers + +```yaml +- targetFieldKey: file_format + expression: '{{$.fullFileExtension || $.mimeType}}' + type: enum + enumMapper: + matcher: 'document_file_format_from_extension' +``` + +### Always Include Fallback + +```yaml +enumMapper: + matcher: + - matchExpression: '{{$.status == "Active"}}' + value: active + - matchExpression: '{{$.status == "Inactive"}}' + value: inactive + # ALWAYS include null/unknown fallback + - matchExpression: '{{$.status == null || $.status == ""}}' + value: unknown +``` + +## Nested Objects + +### Simple Nested Field + +```yaml +fields: + - targetFieldKey: city + expression: $.location.city + type: string + - targetFieldKey: country + expression: $.location.country + type: string +``` + +### Flattening Nested Data + +Provider returns: +```json +{ "work": { "department": "Sales", "title": "Manager" } } +``` + +Your schema is flat: +```yaml +fields: + - targetFieldKey: department + expression: $.work.department + type: string + - targetFieldKey: job_title + expression: $.work.title + type: string +``` + +## Array Fields + +### Simple Array + +```yaml +fields: + - targetFieldKey: email_addresses + expression: $.emails[*] + type: string + array: true +``` + +### JEXL Array Operations + +```yaml +fields: + - targetFieldKey: export_formats + expression: '{{keys(exportLinks)}}' + type: string + array: true +``` + +## Computed/Transformed Fields + +### Fallback Values + +```yaml +fields: + - targetFieldKey: file_format + expression: '{{$.fullFileExtension || $.mimeType}}' + type: string +``` + +### Conditional Logic + +```yaml +fields: + - targetFieldKey: default_format + expression: '{{exportLinks ? (keys(exportLinks)[0] || "application/pdf") : $.mimeType}}' + type: string +``` + +### Boolean Check + +```yaml +fields: + - targetFieldKey: is_exportable + expression: '{{$.exportLinks != null}}' + type: boolean +``` + +## Complete Working Example + +Mapping HiBob employee data: + +```yaml +steps: + - stepId: get_employees + stepFunction: + functionName: paginated_request + parameters: + url: /v1/people/search + method: post + args: + - name: fields + value: + - root.id + - root.email + - work.department + - work.title + in: body + response: + dataKey: employees + nextKey: nextCursor + iterator: + key: cursor + in: body + + - stepId: map_data + stepFunction: + functionName: map_fields + version: '2' + parameters: + fields: + - targetFieldKey: email + expression: $.email + type: string + - targetFieldKey: employee_id + expression: $.id + type: string + - targetFieldKey: department + expression: $.work.department + type: string + - targetFieldKey: job_title + expression: $.work.title + type: string + dataSource: $.steps.get_employees.output.data + + - stepId: typecast_data + stepFunction: + functionName: typecast + version: '2' + parameters: + fields: + - targetFieldKey: email + type: string + - targetFieldKey: employee_id + type: string + - targetFieldKey: department + type: string + - targetFieldKey: job_title + type: string + dataSource: $.steps.map_data.output.data + +result: + data: $.steps.typecast_data.output.data +``` + +## Common Mistakes + +### Wrong Expression Context + +```yaml +# WRONG - Using step prefix in inline fields +parameters: + fields: + - expression: $.get_employees.email # Don't use step prefix! + +# CORRECT - Direct field reference +parameters: + fields: + - expression: $.email +``` + +### Missing Version + +```yaml +# WRONG - No version +stepFunction: + functionName: map_fields + parameters: ... + +# CORRECT - Version 2 specified +stepFunction: + functionName: map_fields + version: '2' + parameters: ... +``` + +### Using Provider Field Names + +```yaml +# WRONG - Provider naming +- targetFieldKey: firstName + +# CORRECT - YOUR schema naming +- targetFieldKey: first_name +``` + +## Validation Checklist + +- [ ] Using inline `fields` in map_fields parameters +- [ ] Expressions use correct context (no step prefix for inline) +- [ ] `version: '2'` specified for map_fields and typecast +- [ ] All `targetFieldKey` values match YOUR schema +- [ ] All enum fields have `enumMapper` with null handler +- [ ] typecast step includes all mapped fields diff --git a/skills/stackone-unified-connectors/references/pagination-patterns.md b/skills/stackone-unified-connectors/references/pagination-patterns.md new file mode 100644 index 0000000..6d92c30 --- /dev/null +++ b/skills/stackone-unified-connectors/references/pagination-patterns.md @@ -0,0 +1,289 @@ +# Pagination Patterns Reference + +**IMPORTANT**: This reference may become outdated. Always verify pagination paths against actual API responses with `--debug`. + +## Action-Level Configuration + +```yaml +cursor: + enabled: true + pageSize: 50 # Must be within API's max limit +``` + +## Recommended: request Function with Manual Cursor + +Use `request` function when you need dynamic inputs like `page_size`: + +```yaml +inputs: + - name: page_size + description: Maximum items per page + type: number + in: query + required: false + - name: cursor + description: Pagination cursor + type: string + in: query + required: false + +steps: + - stepId: get_data + stepFunction: + functionName: request + parameters: + url: /items + method: get + args: + # Dual-condition pattern for defaults + - name: limit + value: $.inputs.page_size + in: query + condition: "{{present(inputs.page_size)}}" + - name: limit + value: 50 + in: query + condition: "{{!present(inputs.page_size)}}" + # Pass cursor when present + - name: cursor + value: $.inputs.cursor + in: query + condition: "{{present(inputs.cursor)}}" + +result: + data: $.steps.get_data.output.data + next: $.steps.get_data.output.data.meta.nextCursor +``` + +**Why this approach?** The `paginated_request` function can have issues with `$.inputs.*` resolving to `undefined`. + +## Alternative: paginated_request Function + +Use only when you don't need dynamic input parameters: + +```yaml +stepFunction: + functionName: paginated_request + parameters: + url: /v2/employees + method: get + response: + dataKey: data.employees # EXACT path to data array + nextKey: meta.pagination.next # EXACT path to cursor + indexField: id # Unique identifier field + iterator: + key: page_token # API's expected parameter NAME + in: query # WHERE to send cursor +``` + +## Configuration Fields + +### response.dataKey + +Path to the data array in API response. + +```yaml +# If response is: { "data": { "employees": [...] } } +response: + dataKey: data.employees # NOT just "employees" +``` + +### response.nextKey + +Path to the pagination cursor. + +```yaml +# If response is: { "meta": { "cursor": "abc123" } } +response: + nextKey: meta.cursor +``` + +### response.indexField + +Unique identifier field in each record. Usually `id`. + +```yaml +response: + indexField: id + # or if provider uses different name: + indexField: employee_id +``` + +### iterator.key + +The parameter name the API expects for the cursor. + +```yaml +# If API expects ?page_token=xxx +iterator: + key: page_token # Match API's expected param +``` + +### iterator.in + +Where to send the cursor: `query`, `body`, or `headers`. + +```yaml +iterator: + key: cursor + in: query # Most common + # in: body # Some APIs want cursor in request body +``` + +## Verification Process + +```bash +# 1. Get raw response to verify structure +stackone run --debug \ + --connector \ + --credentials \ + --action-id list_employees + +# 2. Examine response structure +# If response is: +# { +# "data": { +# "employees": [...], +# "meta": { "next_page": "abc123" } +# } +# } +# +# Then: +# dataKey: data.employees +# nextKey: data.meta.next_page +``` + +## Testing Checklist + +### Test 1: First Page + +```bash +stackone run --connector --credentials \ + --action-id list_items --params '{"page_size": 2}' +``` + +Verify: +- [ ] Data array returned +- [ ] Correct number of records +- [ ] Cursor present in response + +### Test 2: Next Page + +```bash +stackone run --connector --credentials \ + --action-id list_items --params '{"cursor": ""}' +``` + +Verify: +- [ ] Different records returned +- [ ] No duplicates from page 1 +- [ ] New cursor (or null if last page) + +### Test 3: Last Page + +Verify: +- [ ] Cursor is null/empty/absent +- [ ] No error on final page + +### Test 4: Empty Results + +```bash +stackone run --connector --credentials \ + --action-id list_items --params '{"filter": "nonexistent"}' +``` + +Verify: +- [ ] Empty array returned (not null/error) +- [ ] Cursor is null/absent + +## Common Issues + +### dataKey path incorrect + +```yaml +# Response: { "data": { "employees": [...] } } + +# WRONG +response: + dataKey: employees + +# CORRECT +response: + dataKey: data.employees +``` + +### nextKey path incorrect + +```yaml +# Response: { "pagination": { "next_cursor": "abc" } } + +# WRONG +response: + nextKey: cursor + +# CORRECT +response: + nextKey: pagination.next_cursor +``` + +### iterator.key doesn't match API + +```yaml +# API expects ?page_token=xxx + +# WRONG +iterator: + key: cursor + +# CORRECT +iterator: + key: page_token +``` + +### Dynamic inputs resolve to undefined + +```yaml +# WRONG - paginated_request with $.inputs +stepFunction: + functionName: paginated_request + parameters: + args: + - name: limit + value: $.inputs.page_size # May be undefined! + +# CORRECT - request with dual-condition +stepFunction: + functionName: request + parameters: + args: + - name: limit + value: $.inputs.page_size + condition: "{{present(inputs.page_size)}}" + - name: limit + value: 50 + condition: "{{!present(inputs.page_size)}}" +``` + +### Missing next in result block + +```yaml +# WRONG - cursor not returned +result: + data: $.steps.typecast_data.output.data + +# CORRECT - include next cursor +result: + data: $.steps.typecast_data.output.data + next: $.steps.get_data.output.data.meta.nextCursor +``` + +## Quick Validation + +| Field | What to Verify | How | +|-------|----------------|-----| +| `cursor.enabled` | Set to `true` | Check action config | +| `cursor.pageSize` | Within API limit | Check API docs | +| `dataKey` | Exact path to array | `--debug` output | +| `nextKey` | Exact path to cursor | `--debug` output | +| `iterator.key` | API's expected param | API documentation | +| `result.next` | Returns cursor | Check result block | diff --git a/skills/stackone-unified-connectors/references/scope-patterns.md b/skills/stackone-unified-connectors/references/scope-patterns.md new file mode 100644 index 0000000..5ddcb90 --- /dev/null +++ b/skills/stackone-unified-connectors/references/scope-patterns.md @@ -0,0 +1,291 @@ +# Scope Patterns Reference + +**IMPORTANT**: This reference may become outdated. Always verify scope requirements against provider API documentation. + +## Core Principles + +1. **Narrower scopes always preferred** - Request only what's needed +2. **Never use deprecated endpoints** - Even if they seem easier +3. **Document trade-offs explicitly** - Users should understand choices +4. **Required fields drive minimum scopes** - Optional fields may need additional scopes + +## Scope Definition Syntax + +```yaml +scopeDefinitions: + employees:read: + description: Read employee basic information +``` + +**Use `scopeDefinitions`** (camelCase), NOT `scope_definitions`. + +## Scope Hierarchy + +Use `includes` for scope inheritance: + +```yaml +scopeDefinitions: + employees:read: + description: Read basic employee data + + employees:extended:read: + description: Extended employee data including compensation + includes: employees:read # Inherits base scope + + employees:write: + description: Create and update employees + includes: employees:read # Write implies read +``` + +## Action-Level Scopes + +```yaml +- actionId: list_employees + requiredScopes: employees:read +``` + +## Field-Level Scopes + +For fields requiring additional permissions: + +```yaml +fieldConfigs: + - targetFieldKey: salary + expression: $.compensation.salary + type: number + requiredScopes: employees:compensation:read +``` + +## Decision Framework + +### Step 1: Categorize Required Fields + +| Category | Description | Example | +|----------|-------------|---------| +| **Critical** | Must have for core functionality | id, name, email | +| **Important** | High value but not blocking | department, hire_date | +| **Nice-to-have** | Additional context | office_location | + +### Step 2: Map Fields to Endpoints + +| Field | /v2/employees | /v2/employees/detailed | /v2/org/members | +|-------|---------------|------------------------|-----------------| +| id | Yes | Yes | Yes | +| first_name | Yes | Yes | Yes | +| department | No | Yes | Yes | +| salary | No | Yes | No | + +### Step 3: Map Endpoints to Scopes + +| Endpoint | Required Scopes | Notes | +|----------|-----------------|-------| +| /v2/employees | employees:read | Basic access | +| /v2/employees/detailed | employees:read, employees:compensation:read | Includes salary | +| /v2/org/members | employees:read, org:read | Cross-org data | + +### Step 4: Decision Tree + +``` +Can ALL critical fields be obtained with narrowest scope? +├─ YES → Use that endpoint +└─ NO → Continue + +Are there deprecated endpoints with the data? +├─ YES → DO NOT USE. Find alternative. +└─ NO → Continue + +Can critical fields be obtained with 2 endpoints? +├─ YES → Evaluate scope combination +│ ├─ Combined scopes still narrower? → Use both +│ └─ Single broader scope simpler? → Document trade-off, ask user +└─ NO → Continue + +Must request broader scope for critical field? +├─ YES → Request broader scope, document reason +└─ NO → Continue + +Important/nice-to-have fields need broader scope? +├─ YES → Make optional, document "requires additional scope" +└─ NO → Include in mapping +``` + +## Trade-off Analysis Template + +```markdown +## Scope Analysis: [Provider Name] + +### Minimum Viable Scopes +Required for critical fields only: +- `employees:read` - Basic data (id, name, email) + +### Recommended Scopes +Includes important fields: +- `employees:read` - Basic data +- `employees:extended:read` - Department, title, hire date + +### Full Feature Scopes +All available fields: +- `employees:read` +- `employees:extended:read` +- `employees:compensation:read` - Salary +- `org:read` - Organizational hierarchy + +### Trade-off Summary +| Level | Fields Available | Security Impact | +|-------|------------------|-----------------| +| Minimum | id, name, email | Lowest risk | +| Recommended | + department, title | Low risk | +| Full | + salary, reports_to | Higher risk | + +### Recommendation +Start with **Recommended**. Add compensation scope only if salary is critical. +``` + +## Performance vs Scope Trade-offs + +### Scenario: Multiple Endpoints vs Broader Scope + +```markdown +### Option A: Two Narrow-Scope Endpoints +- /v2/employees (employees:read) → basic data +- /v2/employees/extended (employees:extended:read) → additional data +- Total: 2 API calls, narrower scopes + +### Option B: One Broader-Scope Endpoint +- /v2/employees/full (employees:full:read) → all data +- Total: 1 API call, broader scope + +### Analysis +| Factor | Option A | Option B | +|--------|----------|----------| +| API calls | 2x | 1x | +| Rate limit impact | 2x | 1x | +| Scope breadth | Narrower | Broader | +| Data exposure | Less | More | + +### Decision +| Priority | Recommendation | +|----------|----------------| +| Security > Performance | Option A | +| Performance > Security | Option B | +``` + +## Deprecated Endpoint Handling + +### Never Use Deprecated Endpoints + +Even if they: +- Have better data +- Require fewer scopes +- Are "still working" + +### What To Do Instead + +1. **Find replacement endpoint:** +```yaml +# OLD (Deprecated): /v1/employees - Removal Q3 2024 +# NEW: /v2/employees +``` + +2. **If replacement has different scope requirements:** +```markdown +### Migration Impact +v2 API requires `employees:extended:read` for department data. +v1 only needed `employees:read`. + +Recommendation: Request additional scope rather than use deprecated v1. +``` + +3. **If replacement is missing fields:** +```markdown +### Field Gap +Deprecated /v1/employees included `legacy_id`. +New /v2/employees does not. + +Options: +1. Use /v2/employees/mappings for legacy_id (+1 API call) +2. Document that legacy_id is unavailable +3. Contact provider about alternative +``` + +## Common Scope Patterns by Category + +### HRIS Connectors + +```yaml +scopeDefinitions: + employees:read: + description: Read employee directory + employees:extended:read: + description: Extended employee data + includes: employees:read + employees:compensation:read: + description: Salary and compensation + employees:write: + description: Create and update employees + includes: employees:read + org:read: + description: Organization structure + time_off:read: + description: PTO and leave data +``` + +### CRM Connectors + +```yaml +scopeDefinitions: + contacts:read: + description: Read contact records + contacts:write: + description: Create and update contacts + includes: contacts:read + deals:read: + description: Read deal/opportunity data + deals:write: + description: Create and update deals + includes: deals:read + activities:read: + description: Read activities and tasks +``` + +### ATS Connectors + +```yaml +scopeDefinitions: + candidates:read: + description: Read candidate profiles + candidates:write: + description: Create and update candidates + includes: candidates:read + jobs:read: + description: Read job postings + jobs:write: + description: Create and update jobs + includes: jobs:read + applications:read: + description: Read job applications + assessments:read: + description: Read assessment results +``` + +## OAuth2 Scope Configuration + +```yaml +authentication: + - oauth2: + type: oauth2 + grantType: authorization_code + authorization: + scopes: employees:read employees:extended:read # Space-separated + scopeDelimiter: ' ' # Some APIs use comma +``` + +## Validation Checklist + +- [ ] Used `scopeDefinitions` (not `scope_definitions`) +- [ ] Documented minimum required scopes +- [ ] No deprecated endpoints used +- [ ] Trade-offs documented for broader scopes +- [ ] Field-level scopes noted where applicable +- [ ] Scope hierarchy defined with `includes` +- [ ] OAuth scope string uses correct delimiter