diff --git a/apps/scratch/.gitignore b/apps/scratch/.gitignore new file mode 100644 index 000000000..86df61026 --- /dev/null +++ b/apps/scratch/.gitignore @@ -0,0 +1,6 @@ +# Scratch apps live here — resolved as workspace packages (deps link to the +# monorepo) and ignored by knip (apps/** is in ignoreWorkspaces), but never +# committed. Keep this dir tracked via .gitkeep; ignore everything else. +* +!.gitignore +!.gitkeep diff --git a/apps/scratch/.gitkeep b/apps/scratch/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/packages/shared/package.json b/packages/shared/package.json index ef5a1c7b5..c98954700 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -53,6 +53,7 @@ "dotenv": "16.6.1", "js-yaml": "4.2.0", "picocolors": "1.1.1", + "yaml": "2.8.2", "zod": "4.3.6" } } diff --git a/packages/shared/src/cli/commands/doctor/bundle.ts b/packages/shared/src/cli/commands/doctor/bundle.ts index ff4d29d95..73918b9af 100644 --- a/packages/shared/src/cli/commands/doctor/bundle.ts +++ b/packages/shared/src/cli/commands/doctor/bundle.ts @@ -14,12 +14,14 @@ import fs from "node:fs"; import path from "node:path"; import yaml from "js-yaml"; +import { + APP_YAML_FILE, + bindingTypeOf, + DATABRICKS_YML_FILE, +} from "../../deploy-config"; import type { ResourceOrigin } from "./types"; import { errorMessage } from "./utils"; -export const DEFAULT_BUNDLE_FILE = "databricks.yml"; -export const DEFAULT_APP_YAML_FILE = "app.yaml"; - /** A `${resources...}` reference — a bundle-created resource. */ const RESOURCES_REF = /\$\{resources\.([^.]+)\.([^.}]+)\.[^}]+\}/; @@ -62,15 +64,6 @@ interface AppYamlDoc { env?: Array<{ name?: string; valueFrom?: string }>; } -/** The typed sub-key of a binding is its single non-`name` object property. */ -function bindingType(block: AppResourceBlock): string | undefined { - for (const [k, v] of Object.entries(block)) { - if (k === "name") continue; - if (v && typeof v === "object") return k; - } - return undefined; -} - /** Classifies a binding by its typed sub-key and origin: scanning its field * values for a `${resources.*}` reference (bundle-managed) vs anything else * (external). */ @@ -79,7 +72,7 @@ function classifyBinding(block: AppResourceBlock): { origin: ResourceOrigin; ref?: { type: string; key: string }; } { - const type = bindingType(block); + const type = bindingTypeOf(block); const typed = type ? block[type] : undefined; if (typed && typeof typed === "object") { for (const value of Object.values(typed as Record)) { @@ -126,8 +119,8 @@ function readYaml(filePath: string): T | null { */ export function readBundleInfo( cwd: string = process.cwd(), - bundleFile: string = DEFAULT_BUNDLE_FILE, - appYamlFile: string = DEFAULT_APP_YAML_FILE, + bundleFile: string = DATABRICKS_YML_FILE, + appYamlFile: string = APP_YAML_FILE, ): BundleInfo { const bindings = new Map(); const envToBinding = new Map(); diff --git a/packages/shared/src/cli/commands/registry/__fixtures__/analytics/app.yaml b/packages/shared/src/cli/commands/registry/__fixtures__/analytics/app.yaml new file mode 100644 index 000000000..860b549ae --- /dev/null +++ b/packages/shared/src/cli/commands/registry/__fixtures__/analytics/app.yaml @@ -0,0 +1,4 @@ +command: ['npm', 'run', 'start'] +env: + - name: DATABRICKS_WAREHOUSE_ID + valueFrom: sql-warehouse diff --git a/packages/shared/src/cli/commands/registry/__fixtures__/analytics/appkit.plugins.json b/packages/shared/src/cli/commands/registry/__fixtures__/analytics/appkit.plugins.json new file mode 100644 index 000000000..f27feddcc --- /dev/null +++ b/packages/shared/src/cli/commands/registry/__fixtures__/analytics/appkit.plugins.json @@ -0,0 +1,366 @@ +{ + "$schema": "https://databricks.github.io/appkit/schemas/template-plugins.schema.json", + "version": "2.0", + "plugins": { + "agents": { + "name": "agents", + "displayName": "Agents Plugin", + "description": "AI agents driven by markdown configs or code, with auto-tool-discovery from registered plugins", + "package": "@databricks/appkit", + "resources": { + "required": [], + "optional": [ + { + "type": "serving_endpoint", + "alias": "Default LLM for agents", + "resourceKey": "agents-serving-endpoint", + "description": "Default streaming-capable LLM endpoint for agents that don't pin their own model", + "permission": "CAN_QUERY", + "fields": { + "name": { + "env": "DATABRICKS_SERVING_ENDPOINT_NAME", + "description": "Default LLM serving endpoint name", + "origin": "user" + } + } + } + ] + }, + "stability": "beta" + }, + "analytics": { + "name": "analytics", + "displayName": "Analytics Plugin", + "description": "SQL query execution against Databricks SQL Warehouses", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "sql_warehouse", + "alias": "SQL Warehouse", + "resourceKey": "sql-warehouse", + "description": "SQL Warehouse for executing analytics queries", + "permission": "CAN_USE", + "fields": { + "id": { + "env": "DATABRICKS_WAREHOUSE_ID", + "description": "SQL Warehouse ID", + "discovery": { + "type": "kind", + "resourceKind": "warehouse" + }, + "origin": "user" + } + } + } + ], + "optional": [] + }, + "scaffolding": { + "rules": { + "must": [ + "Before init, ensure the SQL Warehouse passed via --set analytics.sql-warehouse.id is running" + ], + "should": [ + "After init, ensure config/queries/ has at least one .sql file before running npm run typegen" + ] + } + } + }, + "files": { + "name": "files", + "displayName": "Files Plugin", + "description": "File operations against Databricks Volumes and Unity Catalog", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "volume", + "alias": "Files", + "resourceKey": "files", + "description": "Permission to write to volumes", + "permission": "WRITE_VOLUME", + "fields": { + "path": { + "env": "DATABRICKS_VOLUME_FILES", + "description": "Volume path for file storage (e.g. /Volumes/catalog/schema/volume_name)", + "discovery": { + "type": "kind", + "resourceKind": "volume", + "select": "full_name" + }, + "origin": "user" + } + } + } + ], + "optional": [] + }, + "scaffolding": { + "rules": { + "must": [ + "Before init, verify your Unity Catalog volume exists and you have WRITE_VOLUME permission" + ] + } + } + }, + "genie": { + "name": "genie", + "displayName": "Genie Plugin", + "description": "AI/BI Genie space integration for natural language data queries", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "genie_space", + "alias": "Genie Space", + "resourceKey": "genie-space", + "description": "Genie Space for AI-powered data queries. Space IDs configured via plugin config.", + "permission": "CAN_RUN", + "fields": { + "id": { + "env": "DATABRICKS_GENIE_SPACE_ID", + "description": "Default Genie Space ID", + "discovery": { + "type": "kind", + "resourceKind": "genie_space" + }, + "origin": "user" + }, + "name": { + "description": "Genie Space display name", + "origin": "user" + } + } + } + ], + "optional": [] + }, + "scaffolding": { + "rules": { + "must": [ + "After init, configure the 'spaces' map in plugin config with alias-to-Space-ID mappings" + ] + } + } + }, + "jobs": { + "name": "jobs", + "displayName": "Jobs Plugin", + "description": "Manage Databricks Lakeflow Jobs.", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "job", + "alias": "Job", + "resourceKey": "job", + "description": "A Databricks job to trigger and monitor", + "permission": "CAN_MANAGE_RUN", + "fields": { + "id": { + "env": "DATABRICKS_JOB_ID", + "description": "Numeric Databricks job ID. Find it in the Jobs UI or via `databricks jobs list`.", + "origin": "user" + } + } + } + ], + "optional": [] + } + }, + "lakebase": { + "name": "lakebase", + "displayName": "Lakebase", + "description": "SQL query execution against Databricks Lakebase Autoscaling", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "postgres", + "alias": "Postgres", + "resourceKey": "postgres", + "description": "Lakebase Postgres database for persistent storage", + "permission": "CAN_CONNECT_AND_CREATE", + "fields": { + "project": { + "description": "Full Lakebase Postgres project resource name. Obtain by running `databricks postgres list-projects`, select the desired item from the output array and use its .name value.", + "examples": ["projects/{project-id}"], + "discovery": { + "type": "kind", + "resourceKind": "postgres_project", + "select": "name" + }, + "origin": "user" + }, + "branch": { + "description": "Full Lakebase Postgres branch resource name. Obtain by running `databricks postgres list-branches {project-name}`, select the desired item from the output array and use its .name value. Requires the project resource name.", + "examples": ["projects/{project-id}/branches/{branch-id}"], + "discovery": { + "type": "kind", + "resourceKind": "postgres_branch", + "select": "name", + "dependsOn": "project" + }, + "origin": "user" + }, + "database": { + "description": "Full Lakebase Postgres database resource name. Obtain by running `databricks postgres list-databases {branch-name}`, select the desired item from the output array and use its .name value. Requires the branch resource name.", + "examples": [ + "projects/{project-id}/branches/{branch-id}/databases/{database-id}" + ], + "discovery": { + "type": "kind", + "resourceKind": "postgres_database", + "select": "name", + "dependsOn": "branch" + }, + "origin": "user" + }, + "host": { + "env": "PGHOST", + "description": "Postgres host for local development. Auto-injected by the platform at deploy time.", + "localOnly": true, + "resolve": "postgres:host", + "origin": "platform" + }, + "databaseName": { + "env": "PGDATABASE", + "description": "Postgres database name for local development. Auto-injected by the platform at deploy time.", + "localOnly": true, + "resolve": "postgres:databaseName", + "origin": "platform" + }, + "endpointPath": { + "env": "LAKEBASE_ENDPOINT", + "description": "Lakebase endpoint resource name. Auto-injected at runtime via app.yaml valueFrom: postgres. For local development, obtain by running `databricks postgres list-endpoints {branch-name}`, select the desired item from the output array and use its .name value.", + "bundleIgnore": true, + "examples": [ + "projects/{project-id}/branches/{branch-id}/endpoints/{endpoint-id}" + ], + "resolve": "postgres:endpointPath", + "origin": "cli" + }, + "port": { + "env": "PGPORT", + "description": "Postgres port. Auto-injected by the platform at deploy time.", + "localOnly": true, + "value": "5432", + "origin": "platform" + }, + "sslmode": { + "env": "PGSSLMODE", + "description": "Postgres SSL mode. Auto-injected by the platform at deploy time.", + "localOnly": true, + "value": "require", + "origin": "platform" + } + } + } + ], + "optional": [] + }, + "scaffolding": { + "rules": { + "should": [ + "After init, run any database migrations for your chosen ORM before first request", + "After init, verify Lakebase connectivity with 'psql $PGHOST -c \"select 1\"'" + ] + } + } + }, + "server": { + "name": "server", + "displayName": "Server Plugin", + "description": "HTTP server with Express, static file serving, and Vite dev mode support", + "package": "@databricks/appkit", + "resources": { + "required": [], + "optional": [] + }, + "requiredByTemplate": true + }, + "serving": { + "name": "serving", + "displayName": "Model Serving Plugin", + "description": "Authenticated proxy to Databricks Model Serving endpoints", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "serving_endpoint", + "alias": "Serving Endpoint", + "resourceKey": "serving-endpoint", + "description": "Model Serving endpoint for inference", + "permission": "CAN_QUERY", + "fields": { + "name": { + "env": "DATABRICKS_SERVING_ENDPOINT_NAME", + "description": "Serving endpoint name", + "origin": "user" + } + } + } + ], + "optional": [] + } + } + }, + "scaffolding": { + "command": "databricks apps init", + "flags": { + "--name": { + "description": "Project name — sets fixture-analytics in package.json, databricks.yml, and .env. Required for non-interactive scaffolding.", + "required": true, + "pattern": "^[a-z][a-z0-9-]*$" + }, + "--template": { + "description": "Template path (local directory or GitHub URL)", + "required": false + }, + "--version": { + "description": "AppKit version to use; defaults to auto-detected", + "required": false + }, + "--features": { + "description": "Plugins to enable (comma-separated, no spaces; must match keys in this manifest's plugins map)", + "required": false, + "pattern": "^[a-zA-Z0-9_-]+(,[a-zA-Z0-9_-]+)*$" + }, + "--set": { + "description": "Set resource values (format: plugin.resourceKey.field=value, repeatable)", + "required": false + }, + "--output-dir": { + "description": "Directory to write the project to", + "required": false + }, + "--description": { + "description": "App description", + "required": false + }, + "--run": { + "description": "Run the app after creation (none, dev, dev-remote)", + "required": false + }, + "--auto-approve": { + "description": "Pass as a bare flag (no value) to skip prompts for optional resources. Not recommended for agent-driven init — conflicts with the 'ask user when in doubt' rule.", + "required": false + }, + "--profile": { + "description": "Databricks CLI profile to use for authentication (global flag)", + "required": false + } + }, + "rules": { + "must": [ + "Keep all secrets and credentials only in app.yaml, databricks.yml, and/or .env" + ], + "should": ["ask user when in doubt of resource to use for plugin"], + "never": [ + "guess resources when multiple or no options are available", + "embed secrets in files that will go to the client-bundle" + ] + } + } +} diff --git a/packages/shared/src/cli/commands/registry/__fixtures__/analytics/databricks.yml b/packages/shared/src/cli/commands/registry/__fixtures__/analytics/databricks.yml new file mode 100644 index 000000000..12107b56f --- /dev/null +++ b/packages/shared/src/cli/commands/registry/__fixtures__/analytics/databricks.yml @@ -0,0 +1,32 @@ +bundle: + name: fixture-analytics + +variables: + sql_warehouse_id: + description: SQL Warehouse ID + +resources: + apps: + app: + name: "fixture-analytics" + description: "fixture capture" + source_code_path: ./ + # Uncomment to enable on behalf of user API scopes. Available scopes: sql, dashboards.genie, files.files, serving.serving-endpoints + # user_api_scopes: + # - sql + + # The resources which this app has access to. + resources: + - name: sql-warehouse + sql_warehouse: + id: ${var.sql_warehouse_id} + permission: CAN_USE + +targets: + default: + default: true + workspace: + host: https://example.cloud.databricks.com + + variables: + sql_warehouse_id: abc123warehouse diff --git a/packages/shared/src/cli/commands/registry/__fixtures__/analytics/env.example.txt b/packages/shared/src/cli/commands/registry/__fixtures__/analytics/env.example.txt new file mode 100644 index 000000000..4ca5e82e9 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/__fixtures__/analytics/env.example.txt @@ -0,0 +1,5 @@ +DATABRICKS_HOST=https://... +DATABRICKS_WAREHOUSE_ID=your_sql_warehouse_id +DATABRICKS_APP_PORT=8000 +DATABRICKS_APP_NAME=fixture-analytics +FLASK_RUN_HOST=0.0.0.0 diff --git a/packages/shared/src/cli/commands/registry/__fixtures__/lakebase/app.yaml b/packages/shared/src/cli/commands/registry/__fixtures__/lakebase/app.yaml new file mode 100644 index 000000000..2d626e2f8 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/__fixtures__/lakebase/app.yaml @@ -0,0 +1,4 @@ +command: ['npm', 'run', 'start'] +env: + - name: LAKEBASE_ENDPOINT + valueFrom: postgres diff --git a/packages/shared/src/cli/commands/registry/__fixtures__/lakebase/appkit.plugins.json b/packages/shared/src/cli/commands/registry/__fixtures__/lakebase/appkit.plugins.json new file mode 100644 index 000000000..afd224672 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/__fixtures__/lakebase/appkit.plugins.json @@ -0,0 +1,366 @@ +{ + "$schema": "https://databricks.github.io/appkit/schemas/template-plugins.schema.json", + "version": "2.0", + "plugins": { + "agents": { + "name": "agents", + "displayName": "Agents Plugin", + "description": "AI agents driven by markdown configs or code, with auto-tool-discovery from registered plugins", + "package": "@databricks/appkit", + "resources": { + "required": [], + "optional": [ + { + "type": "serving_endpoint", + "alias": "Default LLM for agents", + "resourceKey": "agents-serving-endpoint", + "description": "Default streaming-capable LLM endpoint for agents that don't pin their own model", + "permission": "CAN_QUERY", + "fields": { + "name": { + "env": "DATABRICKS_SERVING_ENDPOINT_NAME", + "description": "Default LLM serving endpoint name", + "origin": "user" + } + } + } + ] + }, + "stability": "beta" + }, + "analytics": { + "name": "analytics", + "displayName": "Analytics Plugin", + "description": "SQL query execution against Databricks SQL Warehouses", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "sql_warehouse", + "alias": "SQL Warehouse", + "resourceKey": "sql-warehouse", + "description": "SQL Warehouse for executing analytics queries", + "permission": "CAN_USE", + "fields": { + "id": { + "env": "DATABRICKS_WAREHOUSE_ID", + "description": "SQL Warehouse ID", + "discovery": { + "type": "kind", + "resourceKind": "warehouse" + }, + "origin": "user" + } + } + } + ], + "optional": [] + }, + "scaffolding": { + "rules": { + "must": [ + "Before init, ensure the SQL Warehouse passed via --set analytics.sql-warehouse.id is running" + ], + "should": [ + "After init, ensure config/queries/ has at least one .sql file before running npm run typegen" + ] + } + } + }, + "files": { + "name": "files", + "displayName": "Files Plugin", + "description": "File operations against Databricks Volumes and Unity Catalog", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "volume", + "alias": "Files", + "resourceKey": "files", + "description": "Permission to write to volumes", + "permission": "WRITE_VOLUME", + "fields": { + "path": { + "env": "DATABRICKS_VOLUME_FILES", + "description": "Volume path for file storage (e.g. /Volumes/catalog/schema/volume_name)", + "discovery": { + "type": "kind", + "resourceKind": "volume", + "select": "full_name" + }, + "origin": "user" + } + } + } + ], + "optional": [] + }, + "scaffolding": { + "rules": { + "must": [ + "Before init, verify your Unity Catalog volume exists and you have WRITE_VOLUME permission" + ] + } + } + }, + "genie": { + "name": "genie", + "displayName": "Genie Plugin", + "description": "AI/BI Genie space integration for natural language data queries", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "genie_space", + "alias": "Genie Space", + "resourceKey": "genie-space", + "description": "Genie Space for AI-powered data queries. Space IDs configured via plugin config.", + "permission": "CAN_RUN", + "fields": { + "id": { + "env": "DATABRICKS_GENIE_SPACE_ID", + "description": "Default Genie Space ID", + "discovery": { + "type": "kind", + "resourceKind": "genie_space" + }, + "origin": "user" + }, + "name": { + "description": "Genie Space display name", + "origin": "user" + } + } + } + ], + "optional": [] + }, + "scaffolding": { + "rules": { + "must": [ + "After init, configure the 'spaces' map in plugin config with alias-to-Space-ID mappings" + ] + } + } + }, + "jobs": { + "name": "jobs", + "displayName": "Jobs Plugin", + "description": "Manage Databricks Lakeflow Jobs.", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "job", + "alias": "Job", + "resourceKey": "job", + "description": "A Databricks job to trigger and monitor", + "permission": "CAN_MANAGE_RUN", + "fields": { + "id": { + "env": "DATABRICKS_JOB_ID", + "description": "Numeric Databricks job ID. Find it in the Jobs UI or via `databricks jobs list`.", + "origin": "user" + } + } + } + ], + "optional": [] + } + }, + "lakebase": { + "name": "lakebase", + "displayName": "Lakebase", + "description": "SQL query execution against Databricks Lakebase Autoscaling", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "postgres", + "alias": "Postgres", + "resourceKey": "postgres", + "description": "Lakebase Postgres database for persistent storage", + "permission": "CAN_CONNECT_AND_CREATE", + "fields": { + "project": { + "description": "Full Lakebase Postgres project resource name. Obtain by running `databricks postgres list-projects`, select the desired item from the output array and use its .name value.", + "examples": ["projects/{project-id}"], + "discovery": { + "type": "kind", + "resourceKind": "postgres_project", + "select": "name" + }, + "origin": "user" + }, + "branch": { + "description": "Full Lakebase Postgres branch resource name. Obtain by running `databricks postgres list-branches {project-name}`, select the desired item from the output array and use its .name value. Requires the project resource name.", + "examples": ["projects/{project-id}/branches/{branch-id}"], + "discovery": { + "type": "kind", + "resourceKind": "postgres_branch", + "select": "name", + "dependsOn": "project" + }, + "origin": "user" + }, + "database": { + "description": "Full Lakebase Postgres database resource name. Obtain by running `databricks postgres list-databases {branch-name}`, select the desired item from the output array and use its .name value. Requires the branch resource name.", + "examples": [ + "projects/{project-id}/branches/{branch-id}/databases/{database-id}" + ], + "discovery": { + "type": "kind", + "resourceKind": "postgres_database", + "select": "name", + "dependsOn": "branch" + }, + "origin": "user" + }, + "host": { + "env": "PGHOST", + "description": "Postgres host for local development. Auto-injected by the platform at deploy time.", + "localOnly": true, + "resolve": "postgres:host", + "origin": "platform" + }, + "databaseName": { + "env": "PGDATABASE", + "description": "Postgres database name for local development. Auto-injected by the platform at deploy time.", + "localOnly": true, + "resolve": "postgres:databaseName", + "origin": "platform" + }, + "endpointPath": { + "env": "LAKEBASE_ENDPOINT", + "description": "Lakebase endpoint resource name. Auto-injected at runtime via app.yaml valueFrom: postgres. For local development, obtain by running `databricks postgres list-endpoints {branch-name}`, select the desired item from the output array and use its .name value.", + "bundleIgnore": true, + "examples": [ + "projects/{project-id}/branches/{branch-id}/endpoints/{endpoint-id}" + ], + "resolve": "postgres:endpointPath", + "origin": "cli" + }, + "port": { + "env": "PGPORT", + "description": "Postgres port. Auto-injected by the platform at deploy time.", + "localOnly": true, + "value": "5432", + "origin": "platform" + }, + "sslmode": { + "env": "PGSSLMODE", + "description": "Postgres SSL mode. Auto-injected by the platform at deploy time.", + "localOnly": true, + "value": "require", + "origin": "platform" + } + } + } + ], + "optional": [] + }, + "scaffolding": { + "rules": { + "should": [ + "After init, run any database migrations for your chosen ORM before first request", + "After init, verify Lakebase connectivity with 'psql $PGHOST -c \"select 1\"'" + ] + } + } + }, + "server": { + "name": "server", + "displayName": "Server Plugin", + "description": "HTTP server with Express, static file serving, and Vite dev mode support", + "package": "@databricks/appkit", + "resources": { + "required": [], + "optional": [] + }, + "requiredByTemplate": true + }, + "serving": { + "name": "serving", + "displayName": "Model Serving Plugin", + "description": "Authenticated proxy to Databricks Model Serving endpoints", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "serving_endpoint", + "alias": "Serving Endpoint", + "resourceKey": "serving-endpoint", + "description": "Model Serving endpoint for inference", + "permission": "CAN_QUERY", + "fields": { + "name": { + "env": "DATABRICKS_SERVING_ENDPOINT_NAME", + "description": "Serving endpoint name", + "origin": "user" + } + } + } + ], + "optional": [] + } + } + }, + "scaffolding": { + "command": "databricks apps init", + "flags": { + "--name": { + "description": "Project name — sets fixture-lb in package.json, databricks.yml, and .env. Required for non-interactive scaffolding.", + "required": true, + "pattern": "^[a-z][a-z0-9-]*$" + }, + "--template": { + "description": "Template path (local directory or GitHub URL)", + "required": false + }, + "--version": { + "description": "AppKit version to use; defaults to auto-detected", + "required": false + }, + "--features": { + "description": "Plugins to enable (comma-separated, no spaces; must match keys in this manifest's plugins map)", + "required": false, + "pattern": "^[a-zA-Z0-9_-]+(,[a-zA-Z0-9_-]+)*$" + }, + "--set": { + "description": "Set resource values (format: plugin.resourceKey.field=value, repeatable)", + "required": false + }, + "--output-dir": { + "description": "Directory to write the project to", + "required": false + }, + "--description": { + "description": "App description", + "required": false + }, + "--run": { + "description": "Run the app after creation (none, dev, dev-remote)", + "required": false + }, + "--auto-approve": { + "description": "Pass as a bare flag (no value) to skip prompts for optional resources. Not recommended for agent-driven init — conflicts with the 'ask user when in doubt' rule.", + "required": false + }, + "--profile": { + "description": "Databricks CLI profile to use for authentication (global flag)", + "required": false + } + }, + "rules": { + "must": [ + "Keep all secrets and credentials only in app.yaml, databricks.yml, and/or .env" + ], + "should": ["ask user when in doubt of resource to use for plugin"], + "never": [ + "guess resources when multiple or no options are available", + "embed secrets in files that will go to the client-bundle" + ] + } + } +} diff --git a/packages/shared/src/cli/commands/registry/__fixtures__/lakebase/databricks.yml b/packages/shared/src/cli/commands/registry/__fixtures__/lakebase/databricks.yml new file mode 100644 index 000000000..58d3f0cd6 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/__fixtures__/lakebase/databricks.yml @@ -0,0 +1,39 @@ +bundle: + name: fixture-lb + +variables: + postgres_branch: + description: Full Lakebase Postgres branch resource name. Obtain by running `databricks postgres list-branches {project-name}`, select the desired item from the output array and use its .name value. Requires the project resource name. + postgres_database: + description: Full Lakebase Postgres database resource name. Obtain by running `databricks postgres list-databases {branch-name}`, select the desired item from the output array and use its .name value. Requires the branch resource name. + postgres_project: + description: Full Lakebase Postgres project resource name. Obtain by running `databricks postgres list-projects`, select the desired item from the output array and use its .name value. + +resources: + apps: + app: + name: "fixture-lb" + description: "lb fixture" + source_code_path: ./ + # Uncomment to enable on behalf of user API scopes. Available scopes: sql, dashboards.genie, files.files, serving.serving-endpoints + # user_api_scopes: + # - sql + + # The resources which this app has access to. + resources: + - name: postgres + postgres: + branch: ${var.postgres_branch} + database: ${var.postgres_database} + permission: CAN_CONNECT_AND_CREATE + +targets: + default: + default: true + workspace: + host: https://example.cloud.databricks.com + + variables: + postgres_branch: projects/p1/branches/b1 + postgres_database: projects/p1/branches/b1/databases/db1 + postgres_project: projects/p1 diff --git a/packages/shared/src/cli/commands/registry/__fixtures__/lakebase/env.example.txt b/packages/shared/src/cli/commands/registry/__fixtures__/lakebase/env.example.txt new file mode 100644 index 000000000..3a50eb6c8 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/__fixtures__/lakebase/env.example.txt @@ -0,0 +1,9 @@ +DATABRICKS_HOST=https://... +PGDATABASE=your_postgres_databaseName +LAKEBASE_ENDPOINT=your_postgres_endpointPath +PGHOST=your_postgres_host +PGPORT=5432 +PGSSLMODE=require +DATABRICKS_APP_PORT=8000 +DATABRICKS_APP_NAME=fixture-lb +FLASK_RUN_HOST=0.0.0.0 diff --git a/packages/shared/src/cli/commands/registry/add.test.ts b/packages/shared/src/cli/commands/registry/add.test.ts new file mode 100644 index 000000000..538496e97 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/add.test.ts @@ -0,0 +1,338 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + partitionDeps, + partitionVerified, + pluginExportName, + profileFromEnv, + resolveItems, + resolveWithinBase, + scopesForResources, +} from "./add"; +import type { RegistryItem } from "./client"; +import type { ResourceRequirementRow } from "./requirements"; + +function item(name: string, extra: Partial = {}): RegistryItem { + return { name, ...extra }; +} + +const tempDirs: string[] = []; +function makeTempDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "add-profile-")); + tempDirs.push(dir); + return dir; +} +afterEach(() => { + for (const dir of tempDirs) { + try { + fs.rmSync(dir, { recursive: true, force: true }); + } catch { + // best effort + } + } + tempDirs.length = 0; +}); + +describe("profileFromEnv", () => { + it("reads DATABRICKS_CONFIG_PROFILE from cwd/.env", () => { + const dir = makeTempDir(); + fs.writeFileSync( + path.join(dir, ".env"), + "DATABRICKS_APP_PORT=8000\nDATABRICKS_CONFIG_PROFILE=dogfood\n", + ); + expect(profileFromEnv(dir)).toBe("dogfood"); + }); + + it("strips surrounding quotes from the value", () => { + const dir = makeTempDir(); + fs.writeFileSync( + path.join(dir, ".env"), + 'DATABRICKS_CONFIG_PROFILE="my prof"\n', + ); + expect(profileFromEnv(dir)).toBe("my prof"); + }); + + it("returns undefined when the key is absent", () => { + const dir = makeTempDir(); + fs.writeFileSync(path.join(dir, ".env"), "DATABRICKS_APP_PORT=8000\n"); + expect(profileFromEnv(dir)).toBeUndefined(); + }); + + it("returns undefined when there is no .env file", () => { + const dir = makeTempDir(); + expect(profileFromEnv(dir)).toBeUndefined(); + }); +}); + +/** A registry item shipping an index.ts with the given export block content. */ +function itemWithIndex(exportBlock: string): RegistryItem { + return { + name: "p", + files: [ + { + path: "index.ts", + target: "index.ts", + type: "registry:file", + content: `export { ${exportBlock} } from "./p";`, + }, + ], + }; +} + +function resourceRow(type: string): ResourceRequirementRow { + return { type, required: true, fields: [] }; +} + +describe("resolveItems", () => { + it("returns requested items in order", async () => { + const fetch = vi.fn(async (name: string) => item(name)); + const result = await resolveItems(["a", "b"], null, fetch); + expect(result.map((i) => i.name)).toEqual(["a", "b"]); + }); + + it("resolves transitive registryDependencies", async () => { + const graph: Record = { + a: item("a", { registryDependencies: ["b"] }), + b: item("b", { registryDependencies: ["c"] }), + c: item("c"), + }; + const fetch = vi.fn(async (name: string) => graph[name]); + const result = await resolveItems(["a"], null, fetch); + expect(result.map((i) => i.name)).toEqual(["a", "b", "c"]); + }); + + it("de-duplicates shared dependencies and fetches each once", async () => { + const graph: Record = { + a: item("a", { registryDependencies: ["shared"] }), + b: item("b", { registryDependencies: ["shared"] }), + shared: item("shared"), + }; + const fetch = vi.fn(async (name: string) => graph[name]); + const result = await resolveItems(["a", "b"], null, fetch); + expect(result.map((i) => i.name)).toEqual(["a", "b", "shared"]); + expect(fetch).toHaveBeenCalledTimes(3); + }); + + it("does not loop on circular dependencies", async () => { + const graph: Record = { + a: item("a", { registryDependencies: ["b"] }), + b: item("b", { registryDependencies: ["a"] }), + }; + const fetch = vi.fn(async (name: string) => graph[name]); + const result = await resolveItems(["a"], null, fetch); + expect(result.map((i) => i.name)).toEqual(["a", "b"]); + expect(fetch).toHaveBeenCalledTimes(2); + }); + + it("strips the namespace from dependency refs", async () => { + const graph: Record = { + a: item("a", { registryDependencies: ["@databricks-appkit/b"] }), + b: item("b"), + }; + const fetch = vi.fn(async (name: string) => graph[name]); + const result = await resolveItems(["a"], null, fetch); + expect(result.map((i) => i.name)).toEqual(["a", "b"]); + }); + + // The body `name` is untrusted (could claim a verified name to pass the gate), + // so resolveItems pins it to the fetch key. + it("pins item.name to the fetch key, ignoring a spoofed body name", async () => { + // Fetched under key "evil" but self-reports the verified name "analytics". + const fetch = vi.fn(async (_key: string) => ({ + ...item("analytics"), + files: [], + })); + const result = await resolveItems(["evil"], null, fetch); + expect(result.map((i) => i.name)).toEqual(["evil"]); + }); + + // A name is a fetch path and a plugins/ dir; `/` or `..` could redirect + // the fetch or escape the dest dir. + it("rejects a top-level ref that is not a plain slug (no fetch)", async () => { + const fetch = vi.fn(); + await expect( + resolveItems(["../../attacker/repo/payload"], null, fetch), + ).rejects.toThrow(/Invalid registry item name/); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("rejects a malicious transitive registryDependency ref", async () => { + const graph: Record = { + a: item("a", { registryDependencies: ["../../evil"] }), + }; + const fetch = vi.fn(async (name: string) => graph[name]); + await expect(resolveItems(["a"], null, fetch)).rejects.toThrow( + /Invalid registry item name/, + ); + }); + + it("fetches a level concurrently and preserves order", async () => { + let active = 0; + let maxActive = 0; + const fetch = vi.fn(async (name: string) => { + active++; + maxActive = Math.max(maxActive, active); + await Promise.resolve(); + active--; + return item(name); + }); + const result = await resolveItems(["a", "b", "c"], null, fetch); + expect(result.map((i) => i.name)).toEqual(["a", "b", "c"]); + expect(maxActive).toBeGreaterThan(1); // ran in parallel, not one-at-a-time + }); +}); + +describe("partitionVerified", () => { + it("splits requested names by the index's verified set", () => { + const res = partitionVerified( + ["metric-card", "hello"], + new Set(["metric-card"]), + ); + expect(res).toEqual({ verified: ["metric-card"], unverified: ["hello"] }); + }); + + it("strips the namespace before comparing", () => { + const res = partitionVerified( + ["@databricks-appkit/metric-card"], + new Set(["metric-card"]), + ); + expect(res).toEqual({ verified: ["metric-card"], unverified: [] }); + }); + + it("treats everything as unverified when the index is unreadable (null)", () => { + const res = partitionVerified(["a", "b"], null); + expect(res).toEqual({ verified: [], unverified: ["a", "b"] }); + }); + + // The gate runs over the resolved set, so a verified item pulling an + // unverified registryDependency is still caught. + it("flags an unverified transitive dep in the resolved set", async () => { + const graph: Record = { + "verified-a": item("verified-a", { registryDependencies: ["evil-dep"] }), + "evil-dep": item("evil-dep"), + }; + const fetch = vi.fn(async (name: string) => graph[name]); + const items = await resolveItems(["verified-a"], null, fetch); + const res = partitionVerified( + items.map((i) => i.name), + new Set(["verified-a"]), // only the top-level item is verified + ); + expect(res.unverified).toEqual(["evil-dep"]); + }); +}); + +describe("scopesForResources", () => { + it("maps scope-needing resource types to their user_api_scope", () => { + const scopes = scopesForResources([ + resourceRow("genie_space"), + resourceRow("serving_endpoint"), + resourceRow("volume"), + ]); + expect(Object.fromEntries(scopes)).toEqual({ + genie_space: "dashboards.genie", + serving_endpoint: "serving.serving-endpoints", + volume: "files.files", + }); + }); + + it("returns empty for resources that need no scope", () => { + expect(scopesForResources([resourceRow("sql_warehouse")]).size).toBe(0); + }); + + it("de-dupes repeated types", () => { + const scopes = scopesForResources([ + resourceRow("genie_space"), + resourceRow("genie_space"), + ]); + expect(scopes.size).toBe(1); + }); +}); + +describe("resolveWithinBase (path-traversal guard)", () => { + const base = "/app/server"; + + it("resolves a normal relative target under the base", () => { + expect(resolveWithinBase(base, "plugins/hello/index.ts")).toBe( + path.resolve(base, "plugins/hello/index.ts"), + ); + }); + + it("allows the base itself", () => { + expect(resolveWithinBase(base, ".")).toBe(path.resolve(base)); + }); + + it("rejects a `..` target that escapes the base", () => { + expect(() => resolveWithinBase(base, "../../../../../../tmp/evil")).toThrow( + /escapes/, + ); + }); + + it("rejects an absolute target", () => { + expect(() => resolveWithinBase(base, "/etc/passwd")).toThrow(/absolute/); + }); + + it("rejects a sneaky prefix sibling (base-adjacent dir)", () => { + // /app/server-evil must NOT be treated as inside /app/server + expect(() => resolveWithinBase(base, "../server-evil/x")).toThrow( + /escapes/, + ); + }); +}); + +describe("pluginExportName (code-injection guard)", () => { + it("returns a plain camelCase export name", () => { + expect(pluginExportName(itemWithIndex("helloPlugin"))).toBe("helloPlugin"); + }); + + it("prefers the lowercase factory over a PascalCase class", () => { + expect(pluginExportName(itemWithIndex("HelloPlugin, hello"))).toBe("hello"); + }); + + it("rejects an export token carrying an injected statement", () => { + // The chosen token is not a bare identifier → refuse (caller falls back) + expect( + pluginExportName( + itemWithIndex("evil()); require('child_process').exec('x'); (y"), + ), + ).toBeNull(); + }); + + it("returns null when there is no index.ts", () => { + expect(pluginExportName(item("p"))).toBeNull(); + }); +}); + +describe("partitionDeps (dependency-injection guard)", () => { + it("accepts plain names and scoped names with ranges", () => { + const { safe, rejected } = partitionDeps([ + "lodash", + "@databricks/appkit-ui@^0.41.0", + "react@19.2.0", + ]); + expect(safe).toEqual([ + "lodash", + "@databricks/appkit-ui@^0.41.0", + "react@19.2.0", + ]); + expect(rejected).toEqual([]); + }); + + it("rejects flag-like and URL/git specs (argument injection / RCE)", () => { + const { safe, rejected } = partitionDeps([ + "--registry=http://attacker", + "-g", + "evil@https://attacker/e.tgz", + "git+ssh://attacker/x", + "ok-pkg", + ]); + expect(safe).toEqual(["ok-pkg"]); + expect(rejected).toEqual([ + "--registry=http://attacker", + "-g", + "evil@https://attacker/e.tgz", + "git+ssh://attacker/x", + ]); + }); +}); diff --git a/packages/shared/src/cli/commands/registry/add.ts b/packages/shared/src/cli/commands/registry/add.ts new file mode 100644 index 000000000..dfab2bc12 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/add.ts @@ -0,0 +1,698 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { Command } from "commander"; +import pc from "picocolors"; +import { + fetchRegistryItem, + fetchVerifiedNames, + isValidItemName, + type RegistryItem, + type RegistryItemFile, + stripNamespace, +} from "./client"; +import { + buildConfigPlan, + collectBindingValueNeeds, + planHasContent, +} from "./config-plan"; +import { + reportConfigWrite, + validateBundle, + writeConfig, +} from "./config-writer"; +import { + JS_IDENTIFIER, + REGISTRY_REPO, + type RegistryToken, + resolveToken, +} from "./constants"; +import { parseEnv } from "./env-reconcile"; +import { + extractRequirements, + type ResourceRequirementRow, + renderRequirements, +} from "./requirements"; + +/** Subdirectories that commonly hold the frontend / server in an AppKit app. */ +const FRONTEND_SUBDIRS = ["client", "frontend", "web", "app"]; +const SERVER_SUBDIRS = ["server", "api", "backend"]; + +function isDir(p: string): boolean { + return fs.existsSync(p) && fs.statSync(p).isDirectory(); +} + +/** A registry item is a server plugin if it ships a manifest.json. */ +function isPluginItem(item: RegistryItem): boolean { + return (item.files ?? []).some( + (f) => path.basename(f.target ?? f.path) === "manifest.json", + ); +} + +/** + * Locates the frontend root for UI components. AppKit apps put the client in a + * client/ subdir (with its own components.json + src/); the CLI is typically + * run from the repo root. Prefer the dir with components.json, then a src/. + */ +function findFrontendRoot(cwd: string): string { + if (fs.existsSync(path.join(cwd, "components.json"))) return cwd; + for (const sub of FRONTEND_SUBDIRS) { + if (fs.existsSync(path.join(cwd, sub, "components.json"))) { + return path.join(cwd, sub); + } + } + if (isDir(path.join(cwd, "src"))) return cwd; + for (const sub of FRONTEND_SUBDIRS) { + if (isDir(path.join(cwd, sub, "src"))) return path.join(cwd, sub); + } + return cwd; +} + +/** Locates the server root for plugins (the server/ subdir, else cwd). */ +function findServerRoot(cwd: string): string { + for (const sub of SERVER_SUBDIRS) { + if (isDir(path.join(cwd, sub))) return path.join(cwd, sub); + } + return cwd; +} + +/** Nearest dir with a package.json, walking up from start (for dep install). */ +function findNearestPackageJson(start: string): string { + let dir = start; + for (;;) { + if (fs.existsSync(path.join(dir, "package.json"))) return dir; + const parent = path.dirname(dir); + if (parent === dir) return start; + dir = parent; + } +} + +/** + * The Databricks profile the app is configured with, read from + * `DATABRICKS_CONFIG_PROFILE` in `cwd/.env` via the same dotenv parser the app + * loads its env with. The CLI's top-level `dotenv/config` only loads the launch + * dir's `.env` into `process.env`; when `--cwd` points at a different app dir, + * that file isn't loaded, so the workspace picker and bundle validate would + * miss the app's profile — this reads it directly. Undefined if the file or key + * is absent, so the SDK's own default resolution still applies. + */ +export function profileFromEnv(cwd: string): string | undefined { + try { + const content = fs.readFileSync(path.join(cwd, ".env"), "utf-8"); + return parseEnv(content).DATABRICKS_CONFIG_PROFILE || undefined; + } catch { + return undefined; + } +} + +/** + * Resolves a registry item's `target` under `base`, enforcing that the result + * stays inside `base`. Registry items are untrusted remote data; a `target` + * like `../../../.zshrc` or an absolute path could otherwise write files + * anywhere on disk (arbitrary-write → RCE). Throws on any escape. + */ +export function resolveWithinBase(base: string, target: string): string { + if (path.isAbsolute(target)) { + throw new Error(`Refusing absolute file target from registry: ${target}`); + } + const baseResolved = path.resolve(base); + const resolved = path.resolve(baseResolved, target); + if ( + resolved !== baseResolved && + !resolved.startsWith(baseResolved + path.sep) + ) { + throw new Error( + `Refusing file target that escapes the destination directory: ${target}`, + ); + } + return resolved; +} + +/** UI file destination (relative to the frontend root): placed in src/ if present. */ +function uiTargetPath(base: string, file: RegistryItemFile): string { + let target = file.target ?? path.join("components", path.basename(file.path)); + if (!target.startsWith("src/") && isDir(path.join(base, "src"))) { + target = path.join("src", target); + } + return target; +} + +/** Best-effort: the `toPlugin` export name from the item's index.ts. Returns + * null (caller falls back to printed instructions) unless the name is a plain + * JS identifier — the item is untrusted remote content and the value is + * interpolated into the user's server.ts. */ +export function pluginExportName(item: RegistryItem): string | null { + const index = (item.files ?? []).find( + (f) => path.basename(f.target ?? f.path) === "index.ts", + ); + const match = index?.content.match(/export\s*\{([^}]*)\}/); + if (!match) return null; + const names = match[1].split(",").map((s) => s.trim()); + // Prefer the camelCase toPlugin instance over the PascalCase class. + const chosen = names.find((n) => /^[a-z]/.test(n)) ?? names[0]; + return chosen && JS_IDENTIFIER.test(chosen) ? chosen : null; +} + +function detectPackageManager(cwd: string): "pnpm" | "yarn" | "bun" | "npm" { + if (fs.existsSync(path.join(cwd, "pnpm-lock.yaml"))) return "pnpm"; + if (fs.existsSync(path.join(cwd, "yarn.lock"))) return "yarn"; + if (fs.existsSync(path.join(cwd, "bun.lockb"))) return "bun"; + return "npm"; +} + +/** + * A safe npm dependency spec: `[@scope/]name` with an optional `@version` + * range. Registry `dependencies` are untrusted remote data passed to the + * package manager, so we reject anything that isn't a plain name+range — + * blocks tarball/git URL specs (install-script RCE) and `-`-prefixed entries + * that the PM would parse as flags (argument injection). + */ +const SAFE_DEP_SPEC = + /^(@[a-z0-9][\w.-]*\/)?[a-z0-9][\w.-]*(@[\w.\-+~^><=|* ]+)?$/i; + +/** Splits deps into safe (installable) and rejected (surfaced to the user). */ +export function partitionDeps(deps: string[]): { + safe: string[]; + rejected: string[]; +} { + const safe: string[] = []; + const rejected: string[] = []; + for (const dep of deps) { + if (dep.startsWith("-") || !SAFE_DEP_SPEC.test(dep)) rejected.push(dep); + else safe.push(dep); + } + return { safe, rejected }; +} + +function installDependencies(deps: string[], cwd: string): void { + if (deps.length === 0) return; + + const { safe, rejected } = partitionDeps(deps); + if (rejected.length > 0) { + console.warn( + pc.yellow( + `Skipping suspicious dependenc${rejected.length === 1 ? "y" : "ies"} from the registry (not a plain name@version): ${rejected.join(", ")}. Install manually if you trust them.`, + ), + ); + } + if (safe.length === 0) return; + + if (!fs.existsSync(path.join(cwd, "package.json"))) { + console.warn( + pc.yellow( + `No package.json found — install these manually: ${safe.join(" ")}`, + ), + ); + return; + } + const pm = detectPackageManager(cwd); + const subcommand = pm === "npm" ? "install" : "add"; + console.log(`\nInstalling dependencies with ${pm}: ${safe.join(" ")}`); + // `--` stops the PM from parsing any dep as a flag (defense in depth). + const result = spawnSync(pm, [subcommand, "--", ...safe], { + stdio: "inherit", + cwd, + }); + if (result.status !== 0) { + console.warn( + pc.yellow( + `Dependency install exited with code ${result.status ?? "unknown"} — install manually if needed: ${safe.join(" ")}`, + ), + ); + } +} + +/** Runs `appkit plugin sync --write` via this same CLI binary. */ +function runPluginSync(cwd: string): void { + const result = spawnSync( + process.execPath, + [process.argv[1], "plugin", "sync", "--write"], + { stdio: "inherit", cwd }, + ); + if (result.status !== 0) { + console.warn( + pc.yellow( + " Plugin sync did not complete cleanly — run `appkit plugin sync --write` manually.", + ), + ); + } +} + +function writeItemFile( + base: string, + target: string, + content: string, + force: boolean, + cwd: string, +): void { + const dest = resolveWithinBase(base, target); + const existed = fs.existsSync(dest); + if (existed && !force) { + console.error( + pc.red( + `Refusing to overwrite ${path.relative(cwd, dest)} — pass --force to replace it.`, + ), + ); + process.exit(1); + } + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.writeFileSync(dest, content); + const label = existed ? pc.yellow("Updated") : pc.green("Created"); + console.log(`${label} ${path.relative(cwd, dest)}`); +} + +interface PluginSummary { + importPath: string; + exportName: string | null; +} + +/** + * Fetches the requested items plus their transitive registryDependencies. + * Dependencies are resolved breadth-first and de-duplicated by name, so a + * plugin that depends on another registry item pulls the whole graph in one + * `add`. Explicitly-requested items keep their request order and come first. + */ +export async function resolveItems( + names: string[], + token: RegistryToken | null, + fetchItem: ( + name: string, + token: RegistryToken | null, + ) => Promise = fetchRegistryItem, +): Promise { + const seen = new Set(); + const ordered: RegistryItem[] = []; + // A name is both the fetch path and the on-disk `plugins/` dir, so + // reject non-slug refs (`/`, `..`, control chars) before they reach either. + const enqueue = (ref: string): string => { + const name = stripNamespace(ref); + if (!isValidItemName(name)) { + throw new Error(`Invalid registry item name: ${JSON.stringify(ref)}`); + } + return name; + }; + // Breadth-first over the dependency graph, one level per iteration. Items in + // a level are fetched concurrently (fetch latency is additive otherwise), but + // levels stay ordered and dedup/cycle handling is unchanged: a name is marked + // seen before its level is fetched, so it's never fetched or queued twice. + let level = names.map(enqueue).filter((name) => { + if (seen.has(name)) return false; + seen.add(name); + return true; + }); + + while (level.length > 0) { + const items = await Promise.all( + level.map(async (key) => { + const item = await fetchItem(key, token); + // Pin to the fetch key: the body's self-reported `name` is untrusted + // and could claim a verified name to slip past the gate. The key is the + // trustworthy identity the index keys `verified` on. + item.name = key; + return item; + }), + ); + ordered.push(...items); + const next: string[] = []; + for (const item of items) { + for (const dep of item.registryDependencies ?? []) { + const depName = enqueue(dep); + if (seen.has(depName)) continue; + seen.add(depName); + next.push(depName); + } + } + level = next; + } + + return ordered; +} + +interface AddOptions { + force?: boolean; + cwd?: string; + register?: boolean; + /** false = don't reconcile resource env vars into .env. */ + resources?: boolean; + /** true = never prompt; use --env flags or leave unset (agent/CI). */ + yes?: boolean; + /** Pre-supplied env values from repeated --env KEY=VALUE flags. */ + env?: Record; + /** Databricks profile passed to `bundle validate` after writing config. */ + profile?: string; + /** true = install items the registry index doesn't mark verified. */ + allowUnverified?: boolean; +} + +/** + * Splits requested names into verified and unverified against the index's + * verified set. When `verified` is null the index couldn't be read — we can't + * prove anything is verified, so every name is treated as unverified (the gate + * then decides whether to warn-and-continue or block). Names are compared with + * the namespace stripped, matching how items are fetched. + */ +export function partitionVerified( + refs: string[], + verified: Set | null, +): { verified: string[]; unverified: string[] } { + const ok: string[] = []; + const bad: string[] = []; + for (const ref of refs) { + const name = stripNamespace(ref); + if (verified?.has(name)) ok.push(name); + else bad.push(name); + } + return { verified: ok, unverified: bad }; +} + +async function runAdd(refs: string[], opts: AddOptions): Promise { + const cwd = opts.cwd ? path.resolve(opts.cwd) : process.cwd(); + const token = resolveToken(); + if (token) { + console.log( + `Using ${token.envName} to fetch from ${REGISTRY_REPO} (private).`, + ); + } + + // Resolve the full graph and fetch the verified index concurrently (two + // independent round-trips). Item resolution is read-only — nothing is written + // or installed until after the gate below. + const verifiedP = opts.allowUnverified + ? Promise.resolve(null) + : fetchVerifiedNames(token); + const items = await resolveItems(refs, token); + + // Integrity gate over the *entire resolved set* (not just requested names, so + // an unverified transitive dep can't ride in on a verified item). Fails closed: + // a null verified set (unreadable index) makes every item count as unverified. + if (!opts.allowUnverified) { + const verified = await verifiedP; + const { unverified } = partitionVerified( + items.map((i) => i.name), + verified, + ); + if (unverified.length > 0) { + const reason = + verified === null + ? "could not read the registry index to verify these items" + : `not marked verified in ${REGISTRY_REPO}`; + console.error( + pc.red( + `Refusing to add unverified item(s) (${reason}): ${unverified.join(", ")}.`, + ), + ); + console.error( + pc.dim( + " Re-run with --allow-unverified if you trust the source; unverified items run code in your app.", + ), + ); + process.exit(1); + } + } + + const hasUi = items.some((i) => !isPluginItem(i)); + const hasPlugin = items.some(isPluginItem); + const frontendRoot = hasUi ? findFrontendRoot(cwd) : cwd; + const serverRoot = hasPlugin ? findServerRoot(cwd) : cwd; + if (hasUi && frontendRoot !== cwd) { + console.log(pc.dim(`UI components → ${path.relative(cwd, frontendRoot)}/`)); + } + if (hasPlugin && serverRoot !== cwd) { + console.log(pc.dim(`Plugins → ${path.relative(cwd, serverRoot)}/`)); + } + + const deps = new Set(); + let wroteUi = false; + const pluginSummaries: PluginSummary[] = []; + const allRequirements: ResourceRequirementRow[] = []; + + for (const item of items) { + for (const dep of item.dependencies ?? []) deps.add(dep); + + if (isPluginItem(item)) { + let pluginRel = path.join("plugins", item.name); + for (const file of item.files ?? []) { + const target = + file.target ?? + path.join("plugins", item.name, path.basename(file.path)); + writeItemFile( + serverRoot, + target, + file.content, + Boolean(opts.force), + cwd, + ); + if (path.basename(target) === "manifest.json") { + pluginRel = path.dirname(target); + } + } + const requirements = extractRequirements(item); + if (requirements.length > 0) { + console.log(`\n${renderRequirements(item, requirements)}`); + allRequirements.push(...requirements); + } + pluginSummaries.push({ + importPath: `./${pluginRel}`, + exportName: pluginExportName(item), + }); + } else { + for (const file of item.files ?? []) { + writeItemFile( + frontendRoot, + uiTargetPath(frontendRoot, file), + file.content, + Boolean(opts.force), + cwd, + ); + wroteUi = true; + } + } + } + + installDependencies([...deps], findNearestPackageJson(cwd)); + + if (hasPlugin) { + console.log(pc.dim("\nRegistering plugins (appkit plugin sync)...")); + runPluginSync(cwd); + } + + if (wroteUi) { + console.log( + pc.dim( + '\nReminder: import "@databricks/appkit-ui/styles.css" once at your app root so components are themed.', + ), + ); + } + // Lazy import: server-register pulls in @ast-grep/napi (a native addon), and + // this CLI is imported eagerly by index.ts, so a static import would make + // every unrelated command pay that cost. + const registerPluginInServer = + opts.register !== false && pluginSummaries.some((s) => s.exportName) + ? (await import("./server-register.js")).registerPluginInServer + : null; + for (const s of pluginSummaries) { + // Try to wire the plugin into the server's createApp call automatically; + // fall back to printing the snippet when the shape isn't the standard one. + let wired = false; + // Why auto-registration was skipped, so the fallback can say so (an empty + // reason means the user opted out via --no-register — no nag then). + let skipReason: string | undefined; + if (opts.register === false) { + // opted out — print the snippet without a "couldn't" message + } else if (!s.exportName) { + skipReason = + "couldn't read a plugin export name from the item's index.ts"; + } else if (registerPluginInServer) { + const result = registerPluginInServer( + serverRoot, + s.importPath, + s.exportName, + ); + // result.file is relative to serverRoot; show it from cwd for the user. + const shown = + result.file && path.relative(cwd, path.join(serverRoot, result.file)); + if (result.status === "wired") { + console.log(`\n${pc.green("Registered")} ${s.exportName} in ${shown}`); + wired = true; + } else if (result.status === "already") { + console.log( + pc.dim(`\n${s.exportName} is already registered in ${shown}`), + ); + wired = true; + } else { + skipReason = result.reason; + } + } + if (!wired) { + const imp = s.exportName ?? ""; + if (skipReason) { + console.log( + pc.yellow(`\nCouldn't auto-register ${imp} — ${skipReason}.`), + ); + } + console.log( + `${pc.bold("Add this to your server's createApp call:")}\n` + + pc.dim( + ` import { ${imp} } from "${s.importPath}";\n` + + ` const app = await createApp({ plugins: [${imp}(), /* ... */] });`, + ), + ); + } + } + + if (opts.resources !== false && allRequirements.length > 0) { + // Lazy import (same reason as server-register above): env-writer pulls in + // the workspace picker and, through it, the Databricks SDK. + const { collectBindingValues, reportEnvResolutions, syncEnv } = + await import("./env-writer.js"); + // Without --profile, fall back to the app's configured profile so the + // picker and bundle validate reach the workspace the app runs against. + const profile = opts.profile ?? profileFromEnv(cwd); + console.log(pc.dim("\nReconciling resource env vars into .env...")); + const resolutions = await syncEnv(allRequirements, { + cwd, + nonInteractive: Boolean(opts.yes), + values: opts.env, + profile, + }); + reportEnvResolutions(resolutions); + + // Deploy config (app.yaml + databricks.yml). Values come from what the + // user supplied for env fields (flags or prompts); other fields fall back + // to their manifest defaults inside buildConfigPlan. + const values: Record = { ...(opts.env ?? {}) }; + for (const r of resolutions) { + if (r.value !== undefined) values[r.env] = r.value; + } + // Binding fields with no env name (e.g. postgres project/branch/database) + // never flow through .env, so collect them separately — else their + // databricks.yml bundle variables stay unassigned and bundle validate fails. + const bindingNeeds = collectBindingValueNeeds(allRequirements); + if (bindingNeeds.length > 0) { + const bindingValues = await collectBindingValues(bindingNeeds, { + cwd, + nonInteractive: Boolean(opts.yes), + values: opts.env, + profile, + }); + Object.assign(values, bindingValues); + } + const plan = buildConfigPlan(allRequirements, values); + if (planHasContent(plan)) { + const result = writeConfig(cwd, plan); + reportConfigWrite(result); + if (result.databricksYmlChanged) validateBundle(cwd, profile); + } + warnScopeNeeding(allRequirements); + } +} + +/** + * v1 does not write `user_api_scopes` (deferred to the manifest scope + * extension). Warn when an added plugin's resource type is known to need one, + * so the user adds it before deploy. + */ +/** Resource types known to require a user_api_scope, and the scope each needs. */ +export const SCOPE_BY_RESOURCE_TYPE: Record = { + genie_space: "dashboards.genie", + serving_endpoint: "serving.serving-endpoints", + // volumes/files-backed access uses files.files + volume: "files.files", +}; + +/** Returns the user_api_scopes implied by a set of resource rows (deduped). */ +export function scopesForResources( + rows: ResourceRequirementRow[], +): Map { + const needed = new Map(); + for (const row of rows) { + const scope = SCOPE_BY_RESOURCE_TYPE[row.type]; + if (scope) needed.set(row.type, scope); + } + return needed; +} + +function warnScopeNeeding(rows: ResourceRequirementRow[]): void { + const needed = scopesForResources(rows); + if (needed.size === 0) return; + const list = [...needed.entries()] + .map(([type, scope]) => `${type} → ${scope}`) + .join(", "); + console.warn( + pc.yellow( + `\n Note: these resources may need a user_api_scope before deploy: ${list}.\n` + + " Add it under resources.apps.app.user_api_scopes in databricks.yml.", + ), + ); +} + +/** Commander reducer for repeatable `--env KEY=VALUE` flags. */ +function collectEnvFlag( + raw: string, + acc: Record, +): Record { + const eq = raw.indexOf("="); + if (eq === -1) { + console.error(`Ignoring --env "${raw}" (expected KEY=VALUE).`); + return acc; + } + const key = raw.slice(0, eq).trim(); + const value = raw.slice(eq + 1); + if (key) acc[key] = value; + return acc; +} + +export const addCommand = new Command("add") + .description("Add a UI component or server plugin from the AppKit registry") + .argument("", "Registry item name(s), e.g. metric-card or hello") + .option("-f, --force", "Overwrite existing files") + .option("-C, --cwd ", "Run as if started in ") + .option("--no-register", "Don't edit the server entry to register plugins") + .option("--no-resources", "Don't reconcile resource env vars into .env") + .option("-y, --yes", "Don't prompt; use --env values or leave vars unset") + .option( + "--env ", + "Pre-set a resource env var (repeatable)", + collectEnvFlag, + {}, + ) + .option( + "-p, --profile ", + "Databricks profile for the resource picker and bundle validate (defaults to the app's DATABRICKS_CONFIG_PROFILE)", + ) + .option( + "--allow-unverified", + "Add items the registry doesn't mark verified (runs untrusted code)", + ) + .addHelpText( + "after", + ` +No components.json is required. Item type is detected automatically: + • UI components → /src/components/appkit/ (client/ detected) + • Server plugins → /plugins//, runs plugin sync, and registers + them in your createApp call (use --no-register to skip the server edit) + +Server plugins declare Databricks resources. On add, their env vars are +reconciled into .env (and names into .env.example), and the deploy config +(app.yaml + databricks.yml resource bindings) is patched to match — existing +entries are never clobbered. Interactive by default; pass --yes for agents/CI +(uses --env values, leaves the rest unset) and --env KEY=VALUE to supply +values non-interactively. Pass --profile to validate the bundle after writing. + +The frontend/server roots are detected from common layouts, so you can run +this from the repo root. While the registry repo is private, a read token is +resolved from \`gh auth token\` or APPKIT_REGISTRY_TOKEN / GITHUB_TOKEN / GH_TOKEN. + +Examples: + $ appkit add metric-card # UI component + $ appkit add hello # server plugin + $ appkit add metric-card hello # mix in one call + $ appkit add analytics --yes --env DATABRICKS_WAREHOUSE_ID=abc123`, + ) + .action((items: string[], opts: AddOptions) => + runAdd(items, opts).catch((err) => { + console.error(err); + process.exit(1); + }), + ); diff --git a/packages/shared/src/cli/commands/registry/client.test.ts b/packages/shared/src/cli/commands/registry/client.test.ts new file mode 100644 index 000000000..b996fd481 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/client.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { isValidItemName, stripNamespace } from "./client"; + +describe("stripNamespace", () => { + it("removes the @databricks-appkit/ prefix", () => { + expect(stripNamespace("@databricks-appkit/metric-card")).toBe( + "metric-card", + ); + }); + + it("leaves an un-namespaced ref unchanged", () => { + expect(stripNamespace("hello")).toBe("hello"); + }); +}); + +describe("isValidItemName", () => { + it("accepts plain slugs", () => { + expect(isValidItemName("metric-card")).toBe(true); + expect(isValidItemName("hello")).toBe(true); + expect(isValidItemName("a.b_c-1")).toBe(true); + }); + + it("rejects path separators, dot-segments, and control chars", () => { + // path-traversal vectors from an untrusted registryDependency ref + expect(isValidItemName("../../attacker/repo/payload")).toBe(false); + expect(isValidItemName("a/b")).toBe(false); + expect(isValidItemName("a\\b")).toBe(false); + expect(isValidItemName(".")).toBe(false); + expect(isValidItemName("..")).toBe(false); + expect(isValidItemName("evil\x1b[31m")).toBe(false); + expect(isValidItemName("has space")).toBe(false); + expect(isValidItemName("")).toBe(false); + }); +}); diff --git a/packages/shared/src/cli/commands/registry/client.ts b/packages/shared/src/cli/commands/registry/client.ts new file mode 100644 index 000000000..9157477be --- /dev/null +++ b/packages/shared/src/cli/commands/registry/client.ts @@ -0,0 +1,147 @@ +import process from "node:process"; +import { + REGISTRY_INDEX_API_URL, + REGISTRY_INDEX_URL, + REGISTRY_ITEM_API_TEMPLATE, + REGISTRY_ITEM_URL_TEMPLATE, + REGISTRY_NAMESPACE, + REGISTRY_REPO, + type RegistryToken, +} from "./constants"; + +export interface RegistryItemFile { + path: string; + content: string; + type: string; + /** Destination path relative to the project root. */ + target?: string; +} + +export interface RegistryItem { + name: string; + type?: string; + dependencies?: string[]; + registryDependencies?: string[]; + files?: RegistryItemFile[]; +} + +/** Removes a leading `@databricks-appkit/` namespace from a component reference. */ +export function stripNamespace(component: string): string { + const prefix = `${REGISTRY_NAMESPACE}/`; + return component.startsWith(prefix) + ? component.slice(prefix.length) + : component; +} + +/** + * A registry item name is a slug: letters, digits, dot, underscore, hyphen — + * never a path separator or `.`/`..`. Names come from user refs and from an + * item's untrusted `registryDependencies`, and are used both as the fetch path + * (`public/r/.json`) and as the on-disk `plugins/` dir. Rejecting + * separators and dot-segments at the source stops a crafted ref like + * `../../attacker/repo/payload` from redirecting the fetch to another path in + * the registry repo or escaping the destination dir, and keeps control chars + * out of any printed name. + */ +const ITEM_NAME = /^[A-Za-z0-9._-]+$/; + +/** True when `name` is a safe registry item slug (post-namespace-strip). */ +export function isValidItemName(name: string): boolean { + return name !== "." && name !== ".." && ITEM_NAME.test(name); +} + +/** + * Auth headers for a registry request. With a token the GitHub Contents API is + * used and `Accept: raw` makes it return file bytes directly; without one the + * public raw URL needs no headers. Single source for the auth contract shared + * by every registry fetch. + */ +export function registryAuthHeaders( + token: RegistryToken | null, +): Record { + if (!token) return {}; + return { + Authorization: `Bearer ${token.value}`, + Accept: "application/vnd.github.raw", + }; +} + +/** + * Fetches and parses a single registry item. When a token is present the GitHub + * Contents API is used (works for the private/internal repo); otherwise the + * public raw URL is used. Exits the process with a helpful message on failure. + */ +export async function fetchRegistryItem( + name: string, + token: RegistryToken | null, +): Promise { + const template = token + ? REGISTRY_ITEM_API_TEMPLATE + : REGISTRY_ITEM_URL_TEMPLATE; + const url = template.replace("{name}", name); + const headers = registryAuthHeaders(token); + + let res: Awaited>; + try { + res = await fetch(url, { headers }); + } catch (err) { + console.error(`Failed to fetch "${name}" from ${url}`); + console.error(` ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + + if (res.status === 404) { + console.error(`"${name}" not found in ${REGISTRY_REPO}.`); + if (!token) { + console.error( + " If the registry repo is private, set APPKIT_REGISTRY_TOKEN (or GITHUB_TOKEN) to a token with read access.", + ); + } + process.exit(1); + } + if (res.status === 401 || res.status === 403) { + console.error( + `Access denied (HTTP ${res.status}) fetching "${name}" from ${REGISTRY_REPO}.`, + ); + console.error(" Check that your token has read access to the repository."); + process.exit(1); + } + if (!res.ok) { + console.error(`Registry returned HTTP ${res.status} for "${name}".`); + process.exit(1); + } + + return (await res.json()) as RegistryItem; +} + +/** One entry in the registry index (`registry.json`). */ +export interface RegistryIndexEntry { + name: string; + meta?: { verified?: boolean }; +} + +/** + * Fetches the registry index (`registry.json`) and returns the set of item + * names marked `meta.verified`. The `verified` flag lives only in the index — + * the per-item JSON at `public/r/.json` does not carry it — so the `add` + * integrity gate must consult this. Returns null (not an empty set) if the + * index can't be read, so the caller can tell "nothing verified" apart from + * "couldn't check". + */ +export async function fetchVerifiedNames( + token: RegistryToken | null, +): Promise | null> { + const url = token ? REGISTRY_INDEX_API_URL : REGISTRY_INDEX_URL; + try { + const res = await fetch(url, { headers: registryAuthHeaders(token) }); + if (!res.ok) return null; + const data = (await res.json()) as { items?: RegistryIndexEntry[] }; + const verified = new Set(); + for (const item of data.items ?? []) { + if (item.meta?.verified === true) verified.add(item.name); + } + return verified; + } catch { + return null; + } +} diff --git a/packages/shared/src/cli/commands/registry/config-plan.test.ts b/packages/shared/src/cli/commands/registry/config-plan.test.ts new file mode 100644 index 000000000..d34d392f5 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/config-plan.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from "vitest"; +import { buildConfigPlan, collectBindingValueNeeds } from "./config-plan"; +import type { ResourceRequirementRow } from "./requirements"; + +/** A DABs `${var.}` reference (literal bundle syntax, not JS interp). */ +function varRef(name: string): string { + // biome-ignore lint/style/useTemplate: template literal would trip noTemplateCurlyInString on literal DABs ${var.…} syntax + return "${var." + name + "}"; +} + +const WAREHOUSE: ResourceRequirementRow = { + type: "sql_warehouse", + resourceKey: "sql-warehouse", + permission: "CAN_USE", + required: true, + fields: [{ key: "id", env: "DATABRICKS_WAREHOUSE_ID", origin: "user" }], +}; + +// Mirrors the postgres resource from the lakebase fixture manifest. +const POSTGRES: ResourceRequirementRow = { + type: "postgres", + resourceKey: "postgres", + permission: "CAN_CONNECT_AND_CREATE", + required: true, + fields: [ + { key: "project", origin: "user" }, + { key: "branch", origin: "user" }, + { key: "database", origin: "user" }, + { key: "host", env: "PGHOST", origin: "platform", localOnly: true }, + { key: "endpointPath", env: "LAKEBASE_ENDPOINT", origin: "cli" }, + { key: "port", env: "PGPORT", origin: "platform", value: "5432" }, + ], +}; + +describe("buildConfigPlan — sql_warehouse", () => { + it("produces the app.yaml env entry (valueFrom = resourceKey)", () => { + const plan = buildConfigPlan([WAREHOUSE]); + expect(plan.appYamlEnv).toEqual([ + { name: "DATABRICKS_WAREHOUSE_ID", valueFrom: "sql-warehouse" }, + ]); + }); + + it("produces the sql_warehouse_id bundle variable and binding", () => { + const plan = buildConfigPlan([WAREHOUSE], { + DATABRICKS_WAREHOUSE_ID: "abc123warehouse", + }); + expect(plan.bundleVariables).toEqual([ + { + name: "sql_warehouse_id", + description: undefined, + value: "abc123warehouse", + }, + ]); + expect(plan.resourceBindings).toEqual([ + { + name: "sql-warehouse", + type: "sql_warehouse", + permission: "CAN_USE", + fields: { id: varRef("sql_warehouse_id") }, + }, + ]); + expect(plan.unverifiedTypes).toEqual([]); + }); +}); + +describe("buildConfigPlan — postgres", () => { + it("binds only branch+database, but declares all three variables", () => { + const plan = buildConfigPlan([POSTGRES], { + // user-provided values keyed by field key (no env for these) + project: "projects/p1", + branch: "projects/p1/branches/b1", + database: "projects/p1/branches/b1/databases/db1", + }); + expect(plan.bundleVariables.map((v) => v.name)).toEqual([ + "postgres_project", + "postgres_branch", + "postgres_database", + ]); + expect(plan.resourceBindings).toEqual([ + { + name: "postgres", + type: "postgres", + permission: "CAN_CONNECT_AND_CREATE", + fields: { + branch: varRef("postgres_branch"), + database: varRef("postgres_database"), + }, + }, + ]); + }); + + it("puts only cli-origin fields in app.yaml env (not platform)", () => { + const plan = buildConfigPlan([POSTGRES]); + expect(plan.appYamlEnv).toEqual([ + { name: "LAKEBASE_ENDPOINT", valueFrom: "postgres" }, + ]); + }); +}); + +describe("buildConfigPlan — malformed env names", () => { + it("drops a field whose env name is not a plain identifier", () => { + const plan = buildConfigPlan([ + { + type: "sql_warehouse", + resourceKey: "sql-warehouse", + permission: "CAN_USE", + required: true, + fields: [ + // untrusted manifest name with an injected line + { key: "id", env: "X\nINJECTED=1", origin: "user" }, + ], + }, + ]); + expect(plan.appYamlEnv).toEqual([]); + }); +}); + +describe("buildConfigPlan — unverified types", () => { + it("still emits env but flags the type and writes no binding", () => { + const genie: ResourceRequirementRow = { + type: "genie_space", + resourceKey: "genie-space", + required: true, + fields: [{ key: "id", env: "GENIE_SPACE_ID", origin: "user" }], + }; + const plan = buildConfigPlan([genie]); + expect(plan.appYamlEnv).toEqual([ + { name: "GENIE_SPACE_ID", valueFrom: "genie-space" }, + ]); + expect(plan.resourceBindings).toEqual([]); + expect(plan.bundleVariables).toEqual([]); + expect(plan.unverifiedTypes).toEqual(["genie_space"]); + }); +}); + +describe("collectBindingValueNeeds", () => { + it("reports postgres binding fields that have no env name", () => { + // project/branch/database carry bundle variables but no env → the .env + // flow never collects them; they must be gathered separately or the + // databricks.yml target variables stay unassigned. + const needs = collectBindingValueNeeds([POSTGRES]); + expect(needs.map((n) => n.fieldKey)).toEqual([ + "project", + "branch", + "database", + ]); + expect(needs.every((n) => n.resourceType === "postgres")).toBe(true); + }); + + it("does not report sql_warehouse (its binding field has an env name)", () => { + expect(collectBindingValueNeeds([WAREHOUSE])).toEqual([]); + }); + + it("ignores unverified types (no binding spec)", () => { + const genie: ResourceRequirementRow = { + type: "genie_space", + resourceKey: "genie-space", + required: true, + fields: [{ key: "id", env: "GENIE_SPACE_ID", origin: "user" }], + }; + expect(collectBindingValueNeeds([genie])).toEqual([]); + }); +}); diff --git a/packages/shared/src/cli/commands/registry/config-plan.ts b/packages/shared/src/cli/commands/registry/config-plan.ts new file mode 100644 index 000000000..01260cf8a --- /dev/null +++ b/packages/shared/src/cli/commands/registry/config-plan.ts @@ -0,0 +1,183 @@ +import type { AppYamlEnvEntry, ResourceBinding } from "../../deploy-config"; +import { + fieldOrigin, + isValidEnvName, + type ResourceRequirementRow, +} from "./requirements"; + +/** + * Deploy-config generation for a plugin's resources, reproducing what + * `databricks apps init` renders. Verified byte-for-byte against golden + * fixtures (see __fixtures__/) for the resource types listed in + * {@link BINDING_SPECS}. Unverified types degrade safely: their env entries + * are still produced (that shape is uniform), but the databricks.yml resource + * binding is skipped with a warning rather than guessed. + */ + +/** A `databricks.yml` top-level bundle variable. */ +export interface BundleVariable { + name: string; + description?: string; + /** The value placed under targets.default.variables. */ + value?: string; +} + +export interface ConfigPlan { + appYamlEnv: AppYamlEnvEntry[]; + bundleVariables: BundleVariable[]; + resourceBindings: ResourceBinding[]; + /** Resource types encountered that have no verified binding spec. */ + unverifiedTypes: string[]; +} + +/** + * Per-type rules for producing databricks.yml bundle variables and the app + * resource binding. Only types verified against golden fixtures appear here. + * + * - `variableFields`: field keys that become bundle variables (a superset of + * the binding fields; e.g. postgres declares project+branch+database). + * - `bindingFields`: field keys included in the resource binding (a subset; + * e.g. postgres binds branch+database but not project). + * - `variable(field)`: the bundle-variable name for a given field key. + */ +interface BindingSpec { + variableFields: string[]; + bindingFields: string[]; + variable: (fieldKey: string) => string; +} + +const BINDING_SPECS: Record = { + // Verified against __fixtures__/analytics. + sql_warehouse: { + variableFields: ["id"], + bindingFields: ["id"], + // fixture: variable is `sql_warehouse_id` + variable: (f) => `sql_warehouse_${f}`, + }, + // Verified against __fixtures__/lakebase. + postgres: { + variableFields: ["project", "branch", "database"], + bindingFields: ["branch", "database"], + // fixture: variables are `postgres_` (project/branch/database) + variable: (f) => `postgres_${f}`, + }, +}; + +/** + * Builds the deploy-config plan for a set of resource rows. `values` supplies + * the concrete values for the target-level bundle variables (keyed by the + * manifest field's env var name for env-bearing fields, else by field key); + * missing values leave the variable value undefined. + */ +export function buildConfigPlan( + rows: ResourceRequirementRow[], + values: Record = {}, +): ConfigPlan { + const appYamlEnv: AppYamlEnvEntry[] = []; + const bundleVariables: BundleVariable[] = []; + const resourceBindings: ResourceBinding[] = []; + const unverifiedTypes: string[] = []; + const seenEnv = new Set(); + const seenVar = new Set(); + + for (const row of rows) { + // app.yaml env: every env-bearing field maps to valueFrom = resourceKey, + // except platform-injected fields (the platform provides those directly). + const resourceKey = row.resourceKey ?? row.type; + for (const field of row.fields) { + if (!field.env || fieldOrigin(field) === "platform") continue; + // env names are untrusted manifest data emitted into app.yaml — drop + // anything that isn't a plain env identifier (mirrors the .env guard). + if (!isValidEnvName(field.env)) continue; + if (seenEnv.has(field.env)) continue; + seenEnv.add(field.env); + appYamlEnv.push({ name: field.env, valueFrom: resourceKey }); + } + + const spec = BINDING_SPECS[row.type]; + if (!spec) { + if (!unverifiedTypes.includes(row.type)) unverifiedTypes.push(row.type); + continue; + } + + // Bundle variables (superset of binding fields for this type). + for (const fieldKey of spec.variableFields) { + const varName = spec.variable(fieldKey); + if (seenVar.has(varName)) continue; + seenVar.add(varName); + const field = row.fields.find((f) => f.key === fieldKey); + const valueKey = field?.env ?? fieldKey; + bundleVariables.push({ + name: varName, + description: field?.description, + value: values[valueKey] ?? field?.value, + }); + } + + // Resource binding: only the spec's binding fields, referencing ${var.X}. + const fields: Record = {}; + for (const fieldKey of spec.bindingFields) { + fields[fieldKey] = `\${var.${spec.variable(fieldKey)}}`; + } + resourceBindings.push({ + name: resourceKey, + type: row.type, + permission: row.permission, + fields, + }); + } + + return { appYamlEnv, bundleVariables, resourceBindings, unverifiedTypes }; +} + +/** + * A bundle-variable value the user must supply that the .env reconciliation + * flow can't collect — a binding field with NO `env` name (e.g. postgres + * project/branch/database). Without collecting these, databricks.yml declares + * `${var.postgres_branch}` but never assigns it, and `bundle validate` fails. + * Keyed by `fieldKey` (matching how buildConfigPlan looks up `values`). + */ +export interface BindingValueNeed { + fieldKey: string; + resourceType: string; + description?: string; +} + +/** + * Binding fields that carry a bundle-variable value but have no `env` name and + * no static default — so they're invisible to collectEnvNeeds and must be + * collected separately (keyed by fieldKey) to produce a valid databricks.yml. + */ +export function collectBindingValueNeeds( + rows: ResourceRequirementRow[], +): BindingValueNeed[] { + const needs: BindingValueNeed[] = []; + const seen = new Set(); + for (const row of rows) { + const spec = BINDING_SPECS[row.type]; + if (!spec) continue; + for (const fieldKey of spec.variableFields) { + const field = row.fields.find((f) => f.key === fieldKey); + // Skip fields that already flow through .env (have an env name) or carry + // a static default — those get their value elsewhere. + if (field?.env || field?.value !== undefined) continue; + if (seen.has(fieldKey)) continue; + seen.add(fieldKey); + needs.push({ + fieldKey, + resourceType: row.type, + description: field?.description, + }); + } + } + return needs; +} + +/** True when the plan has any deploy-config content to write. */ +export function planHasContent(plan: ConfigPlan): boolean { + return ( + plan.appYamlEnv.length > 0 || + plan.bundleVariables.length > 0 || + plan.resourceBindings.length > 0 + ); +} diff --git a/packages/shared/src/cli/commands/registry/config-writer.test.ts b/packages/shared/src/cli/commands/registry/config-writer.test.ts new file mode 100644 index 000000000..81d2a278b --- /dev/null +++ b/packages/shared/src/cli/commands/registry/config-writer.test.ts @@ -0,0 +1,196 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { parseDocument } from "yaml"; +import { buildConfigPlan } from "./config-plan"; +import { writeConfig } from "./config-writer"; +import type { ResourceRequirementRow } from "./requirements"; + +const FIXTURES = path.join(__dirname, "__fixtures__"); +const tempDirs: string[] = []; + +function makeTempDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "config-writer-")); + tempDirs.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of tempDirs) { + try { + fs.rmSync(dir, { recursive: true, force: true }); + } catch { + // best effort + } + } + tempDirs.length = 0; +}); + +/** Compares two YAML strings by parsed value (ignores incidental formatting). */ +function sameYaml(a: string, b: string): boolean { + const pa = parseDocument(a).toJSON(); + const pb = parseDocument(b).toJSON(); + return JSON.stringify(pa) === JSON.stringify(pb); +} + +const WAREHOUSE: ResourceRequirementRow = { + type: "sql_warehouse", + resourceKey: "sql-warehouse", + permission: "CAN_USE", + required: true, + fields: [{ key: "id", env: "DATABRICKS_WAREHOUSE_ID", origin: "user" }], +}; + +describe("writeConfig — golden fixtures (analytics)", () => { + it("app.yaml env matches the databricks-rendered fixture", () => { + const cwd = makeTempDir(); + const plan = buildConfigPlan([WAREHOUSE], { + DATABRICKS_WAREHOUSE_ID: "abc123warehouse", + }); + writeConfig(cwd, plan); + + const generated = fs.readFileSync(path.join(cwd, "app.yaml"), "utf-8"); + const golden = fs.readFileSync( + path.join(FIXTURES, "analytics", "app.yaml"), + "utf-8", + ); + // The fixture also has `command:`; our additive writer only owns `env`. + const genEnv = parseDocument(generated).get("env"); + const goldEnv = parseDocument(golden).get("env"); + expect(JSON.stringify(genEnv)).toBe(JSON.stringify(goldEnv)); + }); + + it("databricks.yml variables + binding match the fixture's shapes", () => { + const cwd = makeTempDir(); + const plan = buildConfigPlan([WAREHOUSE], { + DATABRICKS_WAREHOUSE_ID: "abc123warehouse", + }); + writeConfig(cwd, plan); + + const generated = parseDocument( + fs.readFileSync(path.join(cwd, "databricks.yml"), "utf-8"), + ).toJSON(); + const golden = parseDocument( + fs.readFileSync( + path.join(FIXTURES, "analytics", "databricks.yml"), + "utf-8", + ), + ).toJSON(); + + // Variable definition + expect(generated.variables.sql_warehouse_id).toBeDefined(); + // Resource binding matches + expect(generated.resources.apps.app.resources).toEqual( + golden.resources.apps.app.resources, + ); + // Target value + expect(generated.targets.default.variables.sql_warehouse_id).toBe( + golden.targets.default.variables.sql_warehouse_id, + ); + }); +}); + +describe("writeConfig — additive patching", () => { + it("is idempotent: re-writing changes nothing", () => { + const cwd = makeTempDir(); + const plan = buildConfigPlan([WAREHOUSE], { + DATABRICKS_WAREHOUSE_ID: "abc123warehouse", + }); + const first = writeConfig(cwd, plan); + expect(first.appYamlChanged).toBe(true); + + const appAfterFirst = fs.readFileSync(path.join(cwd, "app.yaml"), "utf-8"); + const second = writeConfig(cwd, plan); + expect(second.appYamlChanged).toBe(false); + expect(second.databricksYmlChanged).toBe(false); + expect(fs.readFileSync(path.join(cwd, "app.yaml"), "utf-8")).toBe( + appAfterFirst, + ); + }); + + it("writes the target value even when binding + var already exist", () => { + // Scaffold scenario: databricks.yml already has the sql-warehouse binding + // and the top-level variable, but no target value assigned. `add` must + // still persist the resolved value under targets.default.variables. + const cwd = makeTempDir(); + fs.writeFileSync( + path.join(cwd, "databricks.yml"), + [ + "bundle:", + " name: app", + "resources:", + " apps:", + " app:", + " resources:", + " - name: sql-warehouse", + " sql_warehouse:", + // biome-ignore lint/suspicious/noTemplateCurlyInString: literal DABs ${var.…} bundle syntax, not a JS template placeholder + " id: ${var.sql_warehouse_id}", + " permission: CAN_USE", + "targets:", + " default:", + " default: true", + "variables:", + " sql_warehouse_id:", + " description: SQL Warehouse ID", + "", + ].join("\n"), + ); + + const plan = buildConfigPlan([WAREHOUSE], { + DATABRICKS_WAREHOUSE_ID: "resolved-wh", + }); + const result = writeConfig(cwd, plan); + + // No names added (binding + var pre-existed) but the file DID change. + expect(result.databricksYmlChanged).toBe(true); + const yml = parseDocument( + fs.readFileSync(path.join(cwd, "databricks.yml"), "utf-8"), + ).toJSON(); + expect(yml.targets.default.variables.sql_warehouse_id).toBe("resolved-wh"); + }); + + it("never clobbers an existing env entry or user comments", () => { + const cwd = makeTempDir(); + fs.writeFileSync( + path.join(cwd, "app.yaml"), + "command: ['npm', 'run', 'start']\n# my comment\nenv:\n - name: EXISTING\n valueFrom: other\n", + ); + const plan = buildConfigPlan([WAREHOUSE]); + writeConfig(cwd, plan); + + const out = fs.readFileSync(path.join(cwd, "app.yaml"), "utf-8"); + expect(out).toContain("# my comment"); + expect(out).toContain("EXISTING"); + expect(out).toContain("DATABRICKS_WAREHOUSE_ID"); + // command line preserved + expect(out).toContain("command:"); + }); + + it("skips databricks.yml binding for unverified types but keeps env", () => { + const cwd = makeTempDir(); + const genie: ResourceRequirementRow = { + type: "genie_space", + resourceKey: "genie-space", + required: true, + fields: [{ key: "id", env: "GENIE_SPACE_ID", origin: "user" }], + }; + const result = writeConfig(cwd, buildConfigPlan([genie])); + expect(result.unverifiedTypes).toEqual(["genie_space"]); + expect(fs.existsSync(path.join(cwd, "app.yaml"))).toBe(true); + // no binding written → databricks.yml not created + expect(fs.existsSync(path.join(cwd, "databricks.yml"))).toBe(false); + }); + + it("produces valid round-trippable YAML", () => { + const cwd = makeTempDir(); + writeConfig( + cwd, + buildConfigPlan([WAREHOUSE], { DATABRICKS_WAREHOUSE_ID: "w1" }), + ); + const db = fs.readFileSync(path.join(cwd, "databricks.yml"), "utf-8"); + expect(() => parseDocument(db).toJSON()).not.toThrow(); + expect(sameYaml(db, db)).toBe(true); + }); +}); diff --git a/packages/shared/src/cli/commands/registry/config-writer.ts b/packages/shared/src/cli/commands/registry/config-writer.ts new file mode 100644 index 000000000..4c58368b1 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/config-writer.ts @@ -0,0 +1,211 @@ +import { type SpawnSyncReturns, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import pc from "picocolors"; +import { parseDocument, type YAMLMap, type YAMLSeq } from "yaml"; +import { + APP_YAML_FILE, + type AppYamlEnvEntry, + bindingToNode, + DATABRICKS_YML_FILE, +} from "../../deploy-config"; +import { type ConfigPlan, planHasContent } from "./config-plan"; + +export interface ConfigWriteResult { + appYamlChanged: boolean; + databricksYmlChanged: boolean; + /** Env/binding names actually added (skipping ones already present). */ + added: string[]; + /** Resource types skipped for lack of a verified binding spec. */ + unverifiedTypes: string[]; +} + +/** Reads and parses a YAML file into a Document, or a fresh doc if absent. */ +function loadDoc(file: string): ReturnType { + if (fs.existsSync(file)) { + return parseDocument(fs.readFileSync(file, "utf-8")); + } + return parseDocument(""); +} + +/** + * Additively patches `app.yaml`'s `env:` list with entries not already present + * (matched by `name`). Returns the names added. + */ +function patchAppYaml(file: string, entries: AppYamlEnvEntry[]): string[] { + if (entries.length === 0) return []; + const doc = loadDoc(file); + let seq = doc.get("env") as YAMLSeq | undefined; + if (!seq || typeof (seq as YAMLSeq).add !== "function") { + doc.set("env", doc.createNode([])); + seq = doc.get("env") as YAMLSeq; + } + + const existingNames = new Set(); + for (const item of seq.items) { + const name = (item as YAMLMap)?.get?.("name"); + if (typeof name === "string") existingNames.add(name); + } + + const added: string[] = []; + for (const entry of entries) { + if (existingNames.has(entry.name)) continue; + seq.add(doc.createNode({ name: entry.name, valueFrom: entry.valueFrom })); + added.push(entry.name); + } + + if (added.length > 0) fs.writeFileSync(file, doc.toString()); + return added; +} + +/** Navigates/creates a nested map path, returning the leaf map. */ +function ensureMap( + doc: ReturnType, + pathKeys: string[], +): YAMLMap { + let node = doc.contents as unknown as YAMLMap; + const walked: string[] = []; + for (const key of pathKeys) { + walked.push(key); + let child = doc.getIn(walked) as YAMLMap | undefined; + if (!child || typeof (child as YAMLMap).set !== "function") { + doc.setIn(walked, doc.createNode({})); + child = doc.getIn(walked) as YAMLMap; + } + node = child; + } + return node; +} + +/** + * Additively patches `databricks.yml`: adds bundle `variables`, the app + * `resources` bindings, and the target-level variable values — each only if + * not already present. Returns the names added plus whether the file changed + * (a target-value-only write changes the file without adding any names). + */ +function patchDatabricksYml( + file: string, + plan: ConfigPlan, +): { added: string[]; changed: boolean } { + if (plan.bundleVariables.length === 0 && plan.resourceBindings.length === 0) { + return { added: [], changed: false }; + } + const doc = loadDoc(file); + const added: string[] = []; + + // Top-level bundle variables. + if (plan.bundleVariables.length > 0) { + const vars = ensureMap(doc, ["variables"]); + for (const v of plan.bundleVariables) { + if (vars.has(v.name)) continue; + const body: Record = {}; + if (v.description) body.description = v.description; + vars.set(v.name, doc.createNode(body)); + added.push(v.name); + } + } + + // App resource bindings. + if (plan.resourceBindings.length > 0) { + const app = ensureMap(doc, ["resources", "apps", "app"]); + let bindings = app.get("resources") as YAMLSeq | undefined; + if (!bindings || typeof (bindings as YAMLSeq).add !== "function") { + app.set("resources", doc.createNode([])); + bindings = app.get("resources") as YAMLSeq; + } + const existing = new Set(); + for (const item of bindings.items) { + const name = (item as YAMLMap)?.get?.("name"); + if (typeof name === "string") existing.add(name); + } + for (const binding of plan.resourceBindings) { + if (existing.has(binding.name)) continue; + bindings.add(doc.createNode(bindingToNode(binding))); + added.push(binding.name); + } + } + + // Target-level variable values. Tracked separately from `added` because the + // binding/top-level var may already exist (e.g. from scaffold) while the + // target VALUE is still missing — in that case nothing is in `added` yet the + // file still needs writing to persist the assigned value. + let wroteTargetValue = false; + const withValues = plan.bundleVariables.filter((v) => v.value !== undefined); + if (withValues.length > 0) { + const targetVars = ensureMap(doc, ["targets", "default", "variables"]); + for (const v of withValues) { + if (targetVars.has(v.name)) continue; + targetVars.set(v.name, v.value); + wroteTargetValue = true; + } + } + + const changed = added.length > 0 || wroteTargetValue; + if (changed) fs.writeFileSync(file, doc.toString()); + return { added, changed }; +} + +/** + * Applies a config plan to `app.yaml` and `databricks.yml` in `cwd` via + * comment-preserving additive patches. Never overwrites existing entries. + */ +export function writeConfig(cwd: string, plan: ConfigPlan): ConfigWriteResult { + const appAdded = patchAppYaml(path.join(cwd, APP_YAML_FILE), plan.appYamlEnv); + const db = patchDatabricksYml(path.join(cwd, DATABRICKS_YML_FILE), plan); + return { + appYamlChanged: appAdded.length > 0, + databricksYmlChanged: db.changed, + added: [...new Set([...appAdded, ...db.added])], + unverifiedTypes: plan.unverifiedTypes, + }; +} + +/** + * Runs `databricks bundle validate` as a post-write correctness gate. Returns + * true when the config validates (or when the CLI is unavailable — a missing + * CLI shouldn't fail an install). Surfaces validation errors to the user. + */ +export function validateBundle(cwd: string, profile?: string): boolean { + const args = ["bundle", "validate"]; + if (profile) args.push("-p", profile); + let result: SpawnSyncReturns; + try { + result = spawnSync("databricks", args, { cwd, encoding: "utf-8" }); + } catch { + console.warn( + pc.yellow(" Skipped bundle validate (databricks CLI not found)."), + ); + return true; + } + if (result.error) { + console.warn( + pc.yellow(" Skipped bundle validate (databricks CLI not found)."), + ); + return true; + } + if (result.status !== 0) { + console.warn(pc.yellow(" databricks bundle validate reported issues:")); + if (result.stderr) console.warn(result.stderr.trim()); + return false; + } + return true; +} + +/** Reports what the config write did, including any unverified-type warnings. */ +export function reportConfigWrite(result: ConfigWriteResult): void { + if (result.added.length > 0) { + console.log( + `${pc.green("Updated deploy config:")} ${result.added.join(", ")}`, + ); + } + if (result.unverifiedTypes.length > 0) { + console.warn( + pc.yellow( + ` No databricks.yml binding written for: ${result.unverifiedTypes.join(", ")}. ` + + "Add the resource binding manually before deploy.", + ), + ); + } +} + +export { planHasContent }; diff --git a/packages/shared/src/cli/commands/registry/constants.ts b/packages/shared/src/cli/commands/registry/constants.ts new file mode 100644 index 000000000..43cda3faf --- /dev/null +++ b/packages/shared/src/cli/commands/registry/constants.ts @@ -0,0 +1,66 @@ +import { spawnSync } from "node:child_process"; + +/** shadcn registry namespace consumers reference, e.g. `@databricks-appkit/metric-card`. */ +export const REGISTRY_NAMESPACE = "@databricks-appkit"; + +/** + * A plain JS identifier. A plugin's export name is interpolated into the user's + * server source, so it's validated against this before use — a registry item is + * untrusted, and anything else could inject code. + */ +export const JS_IDENTIFIER = /^[A-Za-z_$][\w$]*$/; + +/** GitHub repo hosting the registry, and the branch the built items live on. */ +export const REGISTRY_REPO = "databricks/appkit-registry"; +export const REGISTRY_REF = "main"; + +/** + * Public hosting: once the repo is public, items are fetchable directly from + * raw.githubusercontent.com with no auth. + */ +const PUBLIC_RAW_BASE = `https://raw.githubusercontent.com/${REGISTRY_REPO}/${REGISTRY_REF}`; +export const REGISTRY_ITEM_URL_TEMPLATE = `${PUBLIC_RAW_BASE}/public/r/{name}.json`; +export const REGISTRY_INDEX_URL = `${PUBLIC_RAW_BASE}/registry.json`; + +/** + * Private/internal hosting: while the repo is internal, files are fetched via + * the GitHub Contents API with a token. `Accept: application/vnd.github.raw` + * makes the API return the file bytes directly (the registry-item JSON). + */ +const GH_CONTENTS_API = `https://api.github.com/repos/${REGISTRY_REPO}/contents`; +export const REGISTRY_ITEM_API_TEMPLATE = `${GH_CONTENTS_API}/public/r/{name}.json?ref=${REGISTRY_REF}`; +export const REGISTRY_INDEX_API_URL = `${GH_CONTENTS_API}/registry.json?ref=${REGISTRY_REF}`; + +/** Env vars checked (in order) for a token granting read access to the repo. */ +export const TOKEN_ENV_VARS = [ + "APPKIT_REGISTRY_TOKEN", + "GITHUB_TOKEN", + "GH_TOKEN", +]; + +export interface RegistryToken { + envName: string; + value: string; +} + +/** + * Resolves a token granting read access to the registry repo: first the env + * vars in {@link TOKEN_ENV_VARS}, then the GitHub CLI (`gh auth token`) if the + * user is logged in. Returns null if none are available. + */ +export function resolveToken( + env: NodeJS.ProcessEnv = process.env, +): RegistryToken | null { + for (const envName of TOKEN_ENV_VARS) { + const value = env[envName]; + if (value) return { envName, value }; + } + try { + const res = spawnSync("gh", ["auth", "token"], { encoding: "utf-8" }); + const value = res.status === 0 ? res.stdout.trim() : ""; + if (value) return { envName: "gh auth token", value }; + } catch { + // gh not installed or not on PATH — fall through. + } + return null; +} diff --git a/packages/shared/src/cli/commands/registry/env-reconcile.test.ts b/packages/shared/src/cli/commands/registry/env-reconcile.test.ts new file mode 100644 index 000000000..4ccf4bd1e --- /dev/null +++ b/packages/shared/src/cli/commands/registry/env-reconcile.test.ts @@ -0,0 +1,246 @@ +import { describe, expect, it, vi } from "vitest"; +import { + collectEnvNeeds, + type EnvNeed, + isSafeEnvValue, + parseEnv, + reconcileEnv, + serializeEnvAppend, +} from "./env-reconcile"; +import type { ResourceRequirementRow } from "./requirements"; + +function row( + over: Partial = {}, +): ResourceRequirementRow { + return { + type: "sql_warehouse", + required: true, + fields: [{ key: "id", env: "DATABRICKS_WAREHOUSE_ID", origin: "user" }], + ...over, + }; +} + +describe("collectEnvNeeds", () => { + it("includes user-origin env fields", () => { + const needs = collectEnvNeeds([row()]); + expect(needs.map((n) => n.env)).toEqual(["DATABRICKS_WAREHOUSE_ID"]); + }); + + it("excludes platform-origin fields (deploy-injected)", () => { + const needs = collectEnvNeeds([ + row({ + type: "database", + fields: [ + { key: "host", env: "PGHOST", origin: "platform" }, + { key: "endpoint", env: "LAKEBASE_ENDPOINT", origin: "cli" }, + ], + }), + ]); + expect(needs.map((n) => n.env)).toEqual(["LAKEBASE_ENDPOINT"]); + }); + + it("excludes fields with no env name", () => { + const needs = collectEnvNeeds([ + row({ fields: [{ key: "name", origin: "user" }] }), + ]); + expect(needs).toEqual([]); + }); + + // The env name is untrusted and written as `NAME=value`; a newline in it + // would inject a second .env line. + it("excludes fields whose env name is not a plain identifier", () => { + const needs = collectEnvNeeds([ + row({ + fields: [ + { key: "a", env: "PORT=x\nDATABRICKS_HOST=evil", origin: "user" }, + { key: "b", env: "has space", origin: "user" }, + { key: "c", env: "OK_NAME", origin: "user" }, + ], + }), + ]); + expect(needs.map((n) => n.env)).toEqual(["OK_NAME"]); + }); + + it("orders required needs before optional and de-dupes shared vars", () => { + const needs = collectEnvNeeds([ + row({ + required: false, + type: "volume", + fields: [{ key: "name", env: "VOLUME_NAME", origin: "user" }], + }), + row(), + // duplicate env from another required resource + row({ + type: "other", + fields: [{ key: "id", env: "DATABRICKS_WAREHOUSE_ID", origin: "user" }], + }), + ]); + expect(needs.map((n) => n.env)).toEqual([ + "DATABRICKS_WAREHOUSE_ID", + "VOLUME_NAME", + ]); + }); + + it("carries the static default value", () => { + const needs = collectEnvNeeds([ + row({ + type: "database", + fields: [ + { key: "port", env: "PGPORT", origin: "static", value: "5432" }, + ], + }), + ]); + // static is not platform, so it's included with its default + expect(needs[0]).toMatchObject({ env: "PGPORT", defaultValue: "5432" }); + }); + + it("excludes localOnly platform fields even without a computed origin", () => { + // Registry-fetched authored manifest: no `origin`, classify from contract. + const needs = collectEnvNeeds([ + row({ + type: "database", + fields: [ + { key: "host", localOnly: true, env: "PGHOST" }, + { key: "port", localOnly: true, value: "5432", env: "PGPORT" }, + { + key: "endpoint", + resolve: "postgres:endpointPath", + env: "LAKEBASE_ENDPOINT", + }, + { key: "id", env: "DATABRICKS_WAREHOUSE_ID" }, + ], + }), + ]); + expect(needs.map((n) => n.env)).toEqual([ + "LAKEBASE_ENDPOINT", + "DATABRICKS_WAREHOUSE_ID", + ]); + }); +}); + +describe("parseEnv", () => { + it("parses KEY=VALUE lines, skipping comments and blanks", () => { + const parsed = parseEnv("# comment\nFOO=bar\n\nBAZ = qux \n"); + expect(parsed).toEqual({ FOO: "bar", BAZ: "qux" }); + }); + + it("strips surrounding quotes", () => { + expect(parseEnv("A=\"one\"\nB='two'")).toEqual({ A: "one", B: "two" }); + }); + + it("keeps '=' inside values", () => { + expect(parseEnv("URL=postgres://a=b")).toEqual({ URL: "postgres://a=b" }); + }); +}); + +describe("serializeEnvAppend", () => { + it("returns empty for no entries", () => { + expect(serializeEnvAppend([])).toBe(""); + }); + + it("emits KEY=VALUE lines with optional comment", () => { + expect( + serializeEnvAppend([{ env: "FOO", value: "bar", comment: "note" }]), + ).toBe("# note\nFOO=bar\n"); + }); +}); + +describe("reconcileEnv", () => { + const need: EnvNeed = { + env: "DATABRICKS_WAREHOUSE_ID", + resourceType: "sql_warehouse", + required: true, + origin: "user", + }; + + it("reports already-set vars with their value and never overwrites them", async () => { + const provide = vi.fn(); + const res = await reconcileEnv([need], { + existing: { DATABRICKS_WAREHOUSE_ID: "existing" }, + provide, + }); + // Value is carried so callers can assign it to databricks.yml target + // variables, but status stays "already-set" so .env isn't rewritten. + expect(res).toEqual([ + { + env: "DATABRICKS_WAREHOUSE_ID", + value: "existing", + status: "already-set", + }, + ]); + expect(provide).not.toHaveBeenCalled(); + }); + + it("uses static defaults without invoking provide", async () => { + const provide = vi.fn(); + const res = await reconcileEnv( + [{ ...need, defaultValue: "5432", env: "PGPORT" }], + { existing: {}, provide }, + ); + expect(res).toEqual([{ env: "PGPORT", value: "5432", status: "written" }]); + expect(provide).not.toHaveBeenCalled(); + }); + + it("writes a provided value", async () => { + const provide = vi.fn(async () => "wh-123"); + const res = await reconcileEnv([need], { existing: {}, provide }); + expect(res).toEqual([ + { env: "DATABRICKS_WAREHOUSE_ID", value: "wh-123", status: "written" }, + ]); + }); + + it("skips when provide returns undefined", async () => { + const provide = vi.fn(async () => undefined); + const res = await reconcileEnv([need], { existing: {}, provide }); + expect(res).toEqual([ + { env: "DATABRICKS_WAREHOUSE_ID", status: "skipped" }, + ]); + }); + + it("treats an empty existing value as unset", async () => { + const provide = vi.fn(async () => "filled"); + const res = await reconcileEnv([need], { + existing: { DATABRICKS_WAREHOUSE_ID: "" }, + provide, + }); + expect(res[0]).toEqual({ + env: "DATABRICKS_WAREHOUSE_ID", + value: "filled", + status: "written", + }); + }); + + // A value carrying a newline could inject a second .env line (e.g. override + // DATABRICKS_HOST → exfil). + it("skips a static default that would inject a newline", async () => { + const provide = vi.fn(); + const res = await reconcileEnv( + [{ ...need, defaultValue: "y\nDATABRICKS_HOST=attacker", env: "FLAG" }], + { existing: {}, provide }, + ); + expect(res).toEqual([{ env: "FLAG", status: "skipped" }]); + expect(provide).not.toHaveBeenCalled(); + }); + + it("skips a provided value that contains a CR/LF", async () => { + const provide = vi.fn(async () => "ok\r\nPGHOST=evil"); + const res = await reconcileEnv([need], { existing: {}, provide }); + expect(res).toEqual([ + { env: "DATABRICKS_WAREHOUSE_ID", status: "skipped" }, + ]); + }); +}); + +describe("isSafeEnvValue", () => { + it("accepts normal single-line values", () => { + expect(isSafeEnvValue("abc123")).toBe(true); + expect(isSafeEnvValue("main.sales.events")).toBe(true); + expect(isSafeEnvValue("")).toBe(true); + }); + + it("rejects values containing a newline or carriage return", () => { + expect(isSafeEnvValue("a\nb")).toBe(false); + expect(isSafeEnvValue("a\r\nb")).toBe(false); + expect(isSafeEnvValue("trailing\n")).toBe(false); + }); +}); diff --git a/packages/shared/src/cli/commands/registry/env-reconcile.ts b/packages/shared/src/cli/commands/registry/env-reconcile.ts new file mode 100644 index 000000000..6f2d3715b --- /dev/null +++ b/packages/shared/src/cli/commands/registry/env-reconcile.ts @@ -0,0 +1,167 @@ +import dotenv from "dotenv"; +import { + fieldOrigin, + isValidEnvName, + type RequirementField, + type ResourceRequirementRow, +} from "./requirements"; + +/** + * A single env var that an installed plugin needs in the local `.env`. + * `platform`-origin fields are excluded upstream — they are injected by + * Databricks Apps at deploy time and never belong in a hand-managed `.env`. + */ +export interface EnvNeed { + env: string; + resourceType: string; + required: boolean; + /** static-origin default value, pre-filled without prompting. */ + defaultValue?: string; + origin?: string; + description?: string; +} + +/** The resolved decision for one env var after reconciliation. */ +export interface EnvResolution { + env: string; + /** The value to write, or undefined when skipped / left unset. */ + value?: string; + status: "written" | "already-set" | "skipped"; +} + +/** + * A `.env` value is a single line: `KEY=VALUE`. A value carrying a CR/LF would + * write extra lines when serialized, so a malicious static default like + * `value: "y\nDATABRICKS_HOST=attacker"` could inject an unrelated key (host + * override → credential exfil). Registry manifests are untrusted, so any value + * with a line break is rejected rather than written. + */ +export function isSafeEnvValue(value: string): boolean { + return !/[\r\n]/.test(value); +} + +/** + * Flattens requirement rows into the env vars that belong in local `.env`. + * Excludes fields with no `env` name and `platform`-origin fields (deploy-time + * platform injection). Order: required resources first (as given), then optional. + */ +export function collectEnvNeeds(rows: ResourceRequirementRow[]): EnvNeed[] { + const needs: EnvNeed[] = []; + const seen = new Set(); + const ordered = [ + ...rows.filter((r) => r.required), + ...rows.filter((r) => !r.required), + ]; + for (const row of ordered) { + for (const field of row.fields) { + if (!includeInEnv(field)) continue; + const env = field.env as string; + if (seen.has(env)) continue; + seen.add(env); + needs.push({ + env, + resourceType: row.type, + required: row.required, + defaultValue: field.value, + origin: fieldOrigin(field), + description: field.description, + }); + } + } + return needs; +} + +/** A field belongs in `.env` iff it names a valid env var and isn't platform-injected. */ +function includeInEnv(field: RequirementField): boolean { + if (!field.env) return false; + // The env name comes from an untrusted manifest and is written as `NAME=value`; + // a malformed name (e.g. one containing a newline) could inject an extra .env + // line, so drop anything that isn't a plain env identifier. + if (!isValidEnvName(field.env)) return false; + // Origin is derived from the authored contract (localOnly/value/resolve) so + // registry-fetched manifests without a computed origin classify correctly. + return fieldOrigin(field) !== "platform"; +} + +/** + * Parses a `.env` file body into a KEY -> value map, using the same `dotenv` + * parser the app loads `.env` with at runtime so the CLI reads it identically. + */ +export function parseEnv(content: string): Record { + return dotenv.parse(content); +} + +/** + * Serializes new env entries for appending to a `.env` file. Only keys not + * already present are emitted; existing keys are never rewritten (we don't + * clobber user edits). Returns the text to append (empty if nothing new). + */ +export function serializeEnvAppend( + entries: Array<{ env: string; value: string; comment?: string }>, +): string { + if (entries.length === 0) return ""; + const lines: string[] = []; + for (const e of entries) { + if (e.comment) lines.push(`# ${e.comment}`); + lines.push(`${e.env}=${e.value}`); + } + return `${lines.join("\n")}\n`; +} + +/** Provides a value for an env need, or undefined to skip it. */ +export type ValueProvider = (need: EnvNeed) => Promise; + +export interface ReconcileOptions { + /** Existing parsed `.env` values (keys already present are left untouched). */ + existing: Record; + /** Resolves a value for each unset need (prompt in interactive, flag in CI). */ + provide: ValueProvider; +} + +/** + * Reconciles the needed env vars against what's already in `.env`. + * - Already-set keys are reported as "already-set" and never overwritten. + * - static-origin defaults are used without invoking `provide`. + * - Everything else defers to `provide`; a returned undefined means skip. + */ +export async function reconcileEnv( + needs: EnvNeed[], + opts: ReconcileOptions, +): Promise { + const resolutions: EnvResolution[] = []; + for (const need of needs) { + const current = opts.existing[need.env]; + if (current !== undefined && current !== "") { + // Carry the existing value so callers can still feed it into deploy + // config (databricks.yml target variables) — the var is set in .env but + // its bundle binding still needs the value assigned. + resolutions.push({ + env: need.env, + value: current, + status: "already-set", + }); + continue; + } + if (need.defaultValue !== undefined) { + // Static default from an untrusted manifest — refuse a value that would + // inject extra `.env` lines rather than silently writing it. + if (!isSafeEnvValue(need.defaultValue)) { + resolutions.push({ env: need.env, status: "skipped" }); + continue; + } + resolutions.push({ + env: need.env, + value: need.defaultValue, + status: "written", + }); + continue; + } + const value = await opts.provide(need); + if (value === undefined || value === "" || !isSafeEnvValue(value)) { + resolutions.push({ env: need.env, status: "skipped" }); + } else { + resolutions.push({ env: need.env, value, status: "written" }); + } + } + return resolutions; +} diff --git a/packages/shared/src/cli/commands/registry/env-writer.test.ts b/packages/shared/src/cli/commands/registry/env-writer.test.ts new file mode 100644 index 000000000..a3d9498fa --- /dev/null +++ b/packages/shared/src/cli/commands/registry/env-writer.test.ts @@ -0,0 +1,139 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { capChoices, syncEnv } from "./env-writer"; +import type { ResourceRequirementRow } from "./requirements"; + +const tempDirs: string[] = []; + +function makeTempDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "env-writer-")); + tempDirs.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of tempDirs) { + try { + fs.rmSync(dir, { recursive: true, force: true }); + } catch { + // best effort + } + } + tempDirs.length = 0; +}); + +const WAREHOUSE_ROW: ResourceRequirementRow = { + type: "sql_warehouse", + required: true, + fields: [{ key: "id", env: "DATABRICKS_WAREHOUSE_ID", origin: "user" }], +}; + +const PLATFORM_ROW: ResourceRequirementRow = { + type: "database", + required: true, + fields: [{ key: "host", env: "PGHOST", origin: "platform" }], +}; + +describe("syncEnv", () => { + it("writes provided values to .env and names to .env.example", async () => { + const cwd = makeTempDir(); + const res = await syncEnv([WAREHOUSE_ROW], { + cwd, + nonInteractive: true, + values: { DATABRICKS_WAREHOUSE_ID: "wh-123" }, + }); + + expect(res).toEqual([ + { env: "DATABRICKS_WAREHOUSE_ID", value: "wh-123", status: "written" }, + ]); + const env = fs.readFileSync(path.join(cwd, ".env"), "utf-8"); + expect(env).toContain("DATABRICKS_WAREHOUSE_ID=wh-123"); + const example = fs.readFileSync(path.join(cwd, ".env.example"), "utf-8"); + expect(example).toContain("DATABRICKS_WAREHOUSE_ID="); + expect(example).not.toContain("wh-123"); + }); + + it("never overwrites an already-set var", async () => { + const cwd = makeTempDir(); + fs.writeFileSync( + path.join(cwd, ".env"), + "DATABRICKS_WAREHOUSE_ID=preexisting\n", + ); + const res = await syncEnv([WAREHOUSE_ROW], { + cwd, + nonInteractive: true, + values: { DATABRICKS_WAREHOUSE_ID: "wh-123" }, + }); + + expect(res[0].status).toBe("already-set"); + // The existing value is carried through so it can be assigned to the + // databricks.yml target variable (else `bundle validate` fails on an + // unassigned ${var.…}). + expect(res[0].value).toBe("preexisting"); + const env = fs.readFileSync(path.join(cwd, ".env"), "utf-8"); + expect(env).toContain("DATABRICKS_WAREHOUSE_ID=preexisting"); + expect(env).not.toContain("wh-123"); + }); + + it("excludes platform-injected fields from .env entirely", async () => { + const cwd = makeTempDir(); + const res = await syncEnv([PLATFORM_ROW], { + cwd, + nonInteractive: true, + values: { PGHOST: "should-be-ignored" }, + }); + + expect(res).toEqual([]); + expect(fs.existsSync(path.join(cwd, ".env"))).toBe(false); + }); + + it("in non-interactive mode, leaves vars without a flag unset", async () => { + const cwd = makeTempDir(); + const res = await syncEnv([WAREHOUSE_ROW], { + cwd, + nonInteractive: true, + }); + expect(res[0].status).toBe("skipped"); + // .env not created since nothing was written + expect(fs.existsSync(path.join(cwd, ".env"))).toBe(false); + }); + + it("preserves existing .env content when appending", async () => { + const cwd = makeTempDir(); + fs.writeFileSync(path.join(cwd, ".env"), "EXISTING=1"); + await syncEnv([WAREHOUSE_ROW], { + cwd, + nonInteractive: true, + values: { DATABRICKS_WAREHOUSE_ID: "wh-123" }, + }); + const env = fs.readFileSync(path.join(cwd, ".env"), "utf-8"); + expect(env).toContain("EXISTING=1"); + expect(env).toContain("DATABRICKS_WAREHOUSE_ID=wh-123"); + }); +}); + +describe("capChoices", () => { + const many = Array.from({ length: 100 }, (_, i) => ({ + value: `w${i}`, + label: `Warehouse ${i}`, + })); + + it("returns the list unchanged when at or under the limit", () => { + const few = many.slice(0, 5); + expect(capChoices(few, "sql_warehouse", 25)).toBe(few); + }); + + it("truncates to the limit when over", () => { + const capped = capChoices(many, "sql_warehouse", 25); + expect(capped).toHaveLength(25); + expect(capped[0].value).toBe("w0"); + expect(capped[24].value).toBe("w24"); + }); + + it("keeps original order", () => { + const capped = capChoices(many, "sql_warehouse", 3); + expect(capped.map((c) => c.value)).toEqual(["w0", "w1", "w2"]); + }); +}); diff --git a/packages/shared/src/cli/commands/registry/env-writer.ts b/packages/shared/src/cli/commands/registry/env-writer.ts new file mode 100644 index 000000000..d6f2073eb --- /dev/null +++ b/packages/shared/src/cli/commands/registry/env-writer.ts @@ -0,0 +1,320 @@ +import fs from "node:fs"; +import path from "node:path"; +import { autocomplete, isCancel, select, text } from "@clack/prompts"; +import pc from "picocolors"; +import type { BindingValueNeed } from "./config-plan"; +import { + collectEnvNeeds, + type EnvNeed, + type EnvResolution, + parseEnv, + reconcileEnv, + serializeEnvAppend, + type ValueProvider, +} from "./env-reconcile"; +import type { ResourceRequirementRow } from "./requirements"; +import { + composeResourceId, + isFlatListable, + isParentContext, + listParentContextStep, + listWorkspaceResources, + parentContextDepth, +} from "./workspace-picker"; + +export interface EnvSyncOptions { + /** Directory holding `.env` / `.env.example` (the app root). */ + cwd: string; + /** true = never prompt (agent/CI). Uses flag values or leaves unset. */ + nonInteractive: boolean; + /** Pre-supplied env values from flags, e.g. { DATABRICKS_WAREHOUSE_ID: "abc" }. */ + values?: Record; + /** Databricks profile for the workspace picker (else the CLI default). */ + profile?: string; +} + +/** Sentinel select value meaning "let me type the id myself". */ +const MANUAL = "__manual__"; + +/** + * Max resources shown in a picker select. Real workspaces can have thousands + * (e.g. 5000+ SQL warehouses); an unbounded select is unusable. Beyond this we + * show the first N and log how many were hidden — never silently drop — and the + * "Enter manually" option always lets the user type an id the list omits. + */ +const PICKER_LIMIT = 25; + +/** Reads a `.env`-style file into a map; empty when the file is absent. */ +function readEnvFile(file: string): Record { + if (!fs.existsSync(file)) return {}; + return parseEnv(fs.readFileSync(file, "utf-8")); +} + +/** Appends text to a file, creating it (with a trailing newline) if needed. */ +function appendToFile(file: string, text: string): void { + if (text === "") return; + if (fs.existsSync(file)) { + const existing = fs.readFileSync(file, "utf-8"); + const sep = existing.length > 0 && !existing.endsWith("\n") ? "\n" : ""; + fs.writeFileSync(file, existing + sep + text); + } else { + fs.writeFileSync(file, text); + } +} + +/** + * Caps a choice list to {@link PICKER_LIMIT} for display, logging how many were + * hidden so the truncation is never silent. The caller always appends an + * "Enter manually" option, so an omitted resource is still reachable. + */ +export function capChoices( + choices: T[], + resourceType: string, + limit = PICKER_LIMIT, +): T[] { + if (choices.length <= limit) return choices; + console.log( + pc.dim( + ` ${choices.length} ${resourceType}s found; showing first ${limit}. ` + + 'Use "Enter manually" if yours is not listed.', + ), + ); + return choices.slice(0, limit); +} + +/** Free-text prompt for one env need; undefined to skip. */ +async function promptText(need: EnvNeed): Promise { + const tag = need.required ? "required" : "optional"; + const answer = await text({ + message: `${need.env} (${need.resourceType}, ${tag})`, + placeholder: need.description ?? "leave blank to skip", + }); + if (isCancel(answer)) return undefined; + const value = (answer ?? "").trim(); + return value === "" ? undefined : value; +} + +/** Presents one workspace list as a select; MANUAL/cancel handled by caller. */ +async function selectFrom( + message: string, + choices: { value: string; label: string }[], +): Promise { + const picked = await select({ + message, + options: [...choices, { value: MANUAL, label: "Enter manually / skip" }], + }); + if (isCancel(picked)) return null; + return String(picked) as string | typeof MANUAL; +} + +/** + * Type-to-filter picker over the full choice list (no cap): the user searches + * by name/id as they type. Appends "Enter manually" so an omitted value is + * still reachable. Returns MANUAL to fall through to free-text, or null on + * cancel. + */ +async function autocompleteFrom( + message: string, + choices: { value: string; label: string }[], +): Promise { + const picked = await autocomplete({ + message, + options: [...choices, { value: MANUAL, label: "Enter manually / skip" }], + placeholder: "type to search…", + }); + if (isCancel(picked)) return null; + return String(picked) as string | typeof MANUAL; +} + +/** + * Drill-down picker for parent-context types (volume→catalog/schema, + * secret→scope, vector_search_index→endpoint). Walks each step, listing the + * next level from the prior pick. Returns the final resource id, or undefined + * to fall back to free-text (on cancel, empty level, or MANUAL at any step). + */ +async function pickParentContext( + need: EnvNeed, + profile: string | undefined, +): Promise { + const depth = parentContextDepth(need.resourceType); + const picks: string[] = []; + for (let i = 0; i < depth; i++) { + const step = listParentContextStep(need.resourceType, i, picks, profile); + if (!step || step.choices.length === 0) { + console.log( + pc.dim( + ` No ${step?.key ?? need.resourceType} found — enter the id manually.`, + ), + ); + return undefined; + } + const picked = await selectFrom( + `${need.env} — pick a ${step.key}`, + capChoices(step.choices, step.key), + ); + if (picked === null || picked === MANUAL) return undefined; + picks.push(picked); + } + // Compose the id from the picks: most types end on a self-qualified id, but a + // secret needs both scope and key (scope/key). + return composeResourceId(need.resourceType, picks); +} + +/** + * Builds the value provider. Precedence: --env flag, then (interactive only) a + * workspace picker — flat select for flat-listable types, drill-down for + * parent-context types — else a free-text prompt. The picker degrades to + * free-text whenever the workspace can't be listed (no profile, offline, auth + * error, empty) so it never hard-fails. + */ +function makeProvider(opts: EnvSyncOptions): ValueProvider { + return async (need: EnvNeed) => { + const fromFlag = opts.values?.[need.env]; + if (fromFlag !== undefined) return fromFlag; + if (opts.nonInteractive) return undefined; + + if (isFlatListable(need.resourceType)) { + const { choices, truncated, error } = await listWorkspaceResources( + need.resourceType, + opts.profile, + ); + if (choices.length > 0) { + if (truncated) { + console.log( + pc.dim( + ` Showing the first ${choices.length} ${need.resourceType}s; use "Enter manually" if yours isn't listed.`, + ), + ); + } + const picked = await autocompleteFrom( + `${need.env} — search ${need.resourceType}s`, + choices, + ); + if (picked === null) return undefined; + if (picked !== MANUAL) return picked; + // fall through to free-text + } else if (error) { + // Listing failed (usually auth/profile) — say so, don't pretend the + // workspace is empty, and point at the fix. + console.log( + pc.yellow( + ` Couldn't list ${need.resourceType}s from the workspace (${error}).`, + ), + ); + console.log( + pc.dim( + " Enter an id manually, or re-run with --profile (or set DATABRICKS_CONFIG_PROFILE) so the picker can reach the workspace.", + ), + ); + } else { + console.log( + pc.dim( + ` No ${need.resourceType} found in the workspace — enter an id manually.`, + ), + ); + } + } else if (isParentContext(need.resourceType)) { + const picked = await pickParentContext(need, opts.profile); + if (picked !== undefined) return picked; + // fall through to free-text + } + + return promptText(need); + }; +} + +/** + * Reconciles a plugin's declared resource env vars into the app's local `.env` + * (and mirrors variable names into `.env.example`). Never overwrites keys the + * user already set; skips platform-injected fields. Returns the per-var + * resolutions so callers can report what happened. + */ +export async function syncEnv( + rows: ResourceRequirementRow[], + opts: EnvSyncOptions, +): Promise { + const needs = collectEnvNeeds(rows); + if (needs.length === 0) return []; + + const envPath = path.join(opts.cwd, ".env"); + const examplePath = path.join(opts.cwd, ".env.example"); + const existing = readEnvFile(envPath); + + const resolutions = await reconcileEnv(needs, { + existing, + provide: makeProvider(opts), + }); + + const written = resolutions.filter( + (r): r is EnvResolution & { value: string } => + r.status === "written" && r.value !== undefined, + ); + appendToFile( + envPath, + serializeEnvAppend(written.map((r) => ({ env: r.env, value: r.value }))), + ); + + // .env.example carries the variable names (no secret values), and only for + // vars not already documented there. + const exampleExisting = readEnvFile(examplePath); + const newExampleKeys = needs.filter((n) => !(n.env in exampleExisting)); + appendToFile( + examplePath, + serializeEnvAppend(newExampleKeys.map((n) => ({ env: n.env, value: "" }))), + ); + + return resolutions; +} + +/** + * Prompts for the {@link BindingValueNeed}s that `.env` reconciliation can't + * collect, returning a `fieldKey -> value` map for buildConfigPlan. Values are + * not written to `.env` (these fields have no env var). Non-interactive mode + * uses `values[fieldKey]` if provided, else leaves the field unset. + */ +export async function collectBindingValues( + needs: BindingValueNeed[], + opts: EnvSyncOptions, +): Promise> { + const out: Record = {}; + for (const need of needs) { + const fromFlag = opts.values?.[need.fieldKey]; + if (fromFlag !== undefined) { + out[need.fieldKey] = fromFlag; + continue; + } + if (opts.nonInteractive) continue; + const answer = await text({ + message: `${need.fieldKey} (${need.resourceType}) — required for databricks.yml`, + placeholder: need.description ?? "leave blank to set before deploy", + }); + if (isCancel(answer)) continue; + const value = (answer ?? "").trim(); + if (value !== "") out[need.fieldKey] = value; + } + return out; +} + +/** Prints a concise summary of what env reconciliation did. */ +export function reportEnvResolutions(resolutions: EnvResolution[]): void { + if (resolutions.length === 0) return; + const written = resolutions.filter((r) => r.status === "written"); + const already = resolutions.filter((r) => r.status === "already-set"); + const skipped = resolutions.filter((r) => r.status === "skipped"); + + if (written.length > 0) { + console.log( + `${pc.green("Wrote to .env:")} ${written.map((r) => r.env).join(", ")}`, + ); + } + if (already.length > 0) { + console.log(pc.dim(`Already set: ${already.map((r) => r.env).join(", ")}`)); + } + if (skipped.length > 0) { + console.log( + `${pc.yellow("Left unset (set before deploy):")} ${skipped + .map((r) => r.env) + .join(", ")}`, + ); + } +} diff --git a/packages/shared/src/cli/commands/registry/index.ts b/packages/shared/src/cli/commands/registry/index.ts new file mode 100644 index 000000000..72b69bd02 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/index.ts @@ -0,0 +1,28 @@ +import { Command } from "commander"; +import { registryInfoCommand } from "./info"; +import { registryListCommand, registrySearchCommand } from "./list"; + +/** + * Parent command for AppKit component registry operations. + * Subcommands: + * - list: Enumerate items available in the registry + * - search: Find items by name, description, type, or keyword + * - info: Show an item's resource requirements and dependencies + * + * Note: `appkit add ` is exposed as a top-level command (see add.ts) + * since it is the primary entry point for consumers. + */ +export const registryCommand = new Command("registry") + .description("AppKit component registry commands") + .addCommand(registryListCommand) + .addCommand(registrySearchCommand) + .addCommand(registryInfoCommand) + .addHelpText( + "after", + ` +Examples: + $ appkit registry list + $ appkit registry search kpi dashboard + $ appkit registry info analytics + $ appkit add metric-card`, + ); diff --git a/packages/shared/src/cli/commands/registry/info.ts b/packages/shared/src/cli/commands/registry/info.ts new file mode 100644 index 000000000..a809d368f --- /dev/null +++ b/packages/shared/src/cli/commands/registry/info.ts @@ -0,0 +1,66 @@ +import process from "node:process"; +import { Command } from "commander"; +import pc from "picocolors"; +import { fetchRegistryItem, isValidItemName, stripNamespace } from "./client"; +import { resolveToken } from "./constants"; +import { extractRequirements, renderRequirements } from "./requirements"; + +async function runInfo(ref: string, opts: { json?: boolean }): Promise { + const token = resolveToken(); + // Validate before fetching: the name is interpolated into the fetch path, so + // reject non-slug refs (matches the `add` guard) rather than let `/` or `..` + // redirect the request to another path in the repo. + const name = stripNamespace(ref); + if (!isValidItemName(name)) { + console.error(`Invalid registry item name: ${JSON.stringify(ref)}`); + process.exit(1); + } + const item = await fetchRegistryItem(name, token); + const rows = extractRequirements(item); + + if (opts.json) { + console.log( + JSON.stringify( + { + name: item.name, + type: item.type, + dependencies: item.dependencies ?? [], + registryDependencies: item.registryDependencies ?? [], + resources: rows, + }, + null, + 2, + ), + ); + return; + } + + console.log(pc.bold(item.name)); + const deps = item.dependencies ?? []; + const registryDeps = item.registryDependencies ?? []; + if (deps.length > 0) { + console.log(pc.dim(` npm dependencies: ${deps.join(", ")}`)); + } + if (registryDeps.length > 0) { + console.log(pc.dim(` registry dependencies: ${registryDeps.join(", ")}`)); + } + console.log(`\n${renderRequirements(item, rows)}`); +} + +export const registryInfoCommand = new Command("info") + .description("Show an item's resource requirements and dependencies") + .argument("", "Registry item name, e.g. analytics") + .option("--json", "Output as JSON") + .addHelpText( + "after", + ` +Examples: + $ appkit registry info analytics + $ appkit registry info @databricks-appkit/analytics --json`, + ) + .action((ref: string, opts: { json?: boolean }) => + runInfo(ref, opts).catch((err) => { + console.error(err); + process.exit(1); + }), + ); diff --git a/packages/shared/src/cli/commands/registry/list.ts b/packages/shared/src/cli/commands/registry/list.ts new file mode 100644 index 000000000..c9aeef0c0 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/list.ts @@ -0,0 +1,218 @@ +import process from "node:process"; +import { Command } from "commander"; +import pc from "picocolors"; +import { registryAuthHeaders } from "./client"; +import { + REGISTRY_INDEX_API_URL, + REGISTRY_INDEX_URL, + REGISTRY_REPO, + resolveToken, +} from "./constants"; + +interface RegistryIndexItem { + name: string; + type?: string; + title?: string; + description?: string; + categories?: string[]; + meta?: { verified?: boolean }; + files?: Array<{ path?: string; target?: string }>; +} + +function isVerified(item: RegistryIndexItem): boolean { + return item.meta?.verified === true; +} + +/** Free-text haystack for search matching. */ +function searchHaystack(item: RegistryIndexItem): string { + return [ + item.name, + item.title ?? "", + item.description ?? "", + itemKind(item), + ...(item.categories ?? []), + ] + .join(" ") + .toLowerCase(); +} + +/** True if every whitespace-separated term in `query` appears in the item. */ +function matchesQuery(item: RegistryIndexItem, query: string): boolean { + const haystack = searchHaystack(item); + return query + .toLowerCase() + .split(/\s+/) + .filter(Boolean) + .every((term) => haystack.includes(term)); +} + +/** A friendly kind for the TYPE column: plugin, component, hook, theme, … */ +function itemKind(item: RegistryIndexItem): string { + const hasManifest = (item.files ?? []).some( + (f) => + f.target?.endsWith("manifest.json") || f.path?.endsWith("manifest.json"), + ); + if (hasManifest) return "plugin"; + switch (item.type) { + case "registry:component": + case "registry:block": + return "component"; + case "registry:hook": + return "hook"; + case "registry:lib": + return "lib"; + case "registry:theme": + return "theme"; + case "registry:ui": + return "ui"; + case "registry:page": + return "page"; + default: + return item.type?.replace(/^registry:/, "") || "item"; + } +} + +const KIND_COLOR: Record string> = { + plugin: pc.magenta, + component: pc.blue, + hook: pc.cyan, + theme: pc.yellow, + lib: pc.green, +}; + +function printTable(items: RegistryIndexItem[]): void { + if (items.length === 0) { + console.log(pc.dim("No items found in the registry.")); + return; + } + const kinds = items.map(itemKind); + const maxName = Math.max(4, ...items.map((i) => i.name.length)); + const maxKind = Math.max(4, ...kinds.map((k) => k.length)); + const verifiedCol = "VERIFIED"; + // Pad plain text before coloring so ANSI codes don't break alignment. + const header = `${"NAME".padEnd(maxName)} ${"TYPE".padEnd(maxKind)} ${verifiedCol} DESCRIPTION`; + console.log(pc.bold(header)); + console.log(pc.dim("─".repeat(header.length))); + for (const [i, item] of items.entries()) { + const verified = isVerified(item); + const kind = kinds[i]; + const colorKind = KIND_COLOR[kind] ?? pc.white; + const name = pc.cyan(item.name.padEnd(maxName)); + const kindCell = colorKind(kind.padEnd(maxKind)); + const mark = verified + ? pc.green("✓".padEnd(verifiedCol.length)) + : " ".repeat(verifiedCol.length); + const desc = item.description ?? item.title ?? ""; + console.log( + `${name} ${kindCell} ${mark} ${verified ? desc : pc.dim(desc)}`, + ); + } +} + +/** Fetches the registry index (token-aware), or exits with a helpful message. */ +async function fetchIndex(): Promise { + const token = resolveToken(); + const url = token ? REGISTRY_INDEX_API_URL : REGISTRY_INDEX_URL; + + let res: Awaited>; + try { + res = await fetch(url, { headers: registryAuthHeaders(token) }); + } catch (err) { + console.error(pc.red(`Failed to reach the registry at ${url}`)); + console.error(` ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + if (res.status === 404 || res.status === 401 || res.status === 403) { + console.error( + pc.red( + `Could not read the registry index from ${REGISTRY_REPO} (HTTP ${res.status}).`, + ), + ); + if (!token) { + console.error( + " If the repo is private, set APPKIT_REGISTRY_TOKEN (or GITHUB_TOKEN) to a token with read access.", + ); + } + process.exit(1); + } + if (!res.ok) { + console.error(pc.red(`Registry returned HTTP ${res.status} for ${url}`)); + process.exit(1); + } + + const data = (await res.json()) as { items?: RegistryIndexItem[] }; + return data.items ?? []; +} + +function output(items: RegistryIndexItem[], opts: { json?: boolean }): void { + if (opts.json) { + // Surface `kind` + `verified` as top-level fields for easy scripting. + console.log( + JSON.stringify( + items.map((i) => ({ + ...i, + kind: itemKind(i), + verified: isVerified(i), + })), + null, + 2, + ), + ); + } else { + printTable(items); + } +} + +async function runList(opts: { + json?: boolean; + verified?: boolean; +}): Promise { + let items = await fetchIndex(); + if (opts.verified) items = items.filter(isVerified); + output(items, opts); +} + +async function runSearch( + query: string, + opts: { json?: boolean; verified?: boolean }, +): Promise { + let items = await fetchIndex(); + items = items.filter((i) => matchesQuery(i, query)); + if (opts.verified) items = items.filter(isVerified); + if (items.length === 0 && !opts.json) { + console.log(pc.dim(`No items match "${query}".`)); + return; + } + output(items, opts); +} + +export const registryListCommand = new Command("list") + .description("List items available in the AppKit registry") + .option("--json", "Output as JSON") + .option("--verified", "Show only verified items") + .action((opts: { json?: boolean; verified?: boolean }) => + runList(opts).catch((err) => { + console.error(err); + process.exit(1); + }), + ); + +export const registrySearchCommand = new Command("search") + .description("Search registry items by name, description, type, or keyword") + .argument("", "Search terms (all must match)") + .option("--json", "Output as JSON") + .option("--verified", "Show only verified items") + .addHelpText( + "after", + ` +Examples: + $ appkit registry search chart + $ appkit registry search kpi dashboard + $ appkit registry search plugin --json`, + ) + .action((query: string[], opts: { json?: boolean; verified?: boolean }) => + runSearch(query.join(" "), opts).catch((err) => { + console.error(err); + process.exit(1); + }), + ); diff --git a/packages/shared/src/cli/commands/registry/requirements.test.ts b/packages/shared/src/cli/commands/registry/requirements.test.ts new file mode 100644 index 000000000..bce47a0e2 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/requirements.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, it } from "vitest"; +import type { RegistryItem } from "./client"; +import { + extractRequirements, + fieldOrigin, + isValidEnvName, + renderRequirements, +} from "./requirements"; + +function pluginItem(manifest: unknown): RegistryItem { + return { + name: "analytics", + files: [ + { + path: "manifest.json", + target: "manifest.json", + type: "registry:file", + content: JSON.stringify(manifest), + }, + ], + }; +} + +const ANALYTICS_MANIFEST = { + name: "analytics", + resources: { + required: [ + { + type: "sql_warehouse", + resourceKey: "sql-warehouse", + permission: "CAN_USE", + description: "SQL warehouse for queries", + fields: { + id: { env: "DATABRICKS_WAREHOUSE_ID", origin: "user" }, + }, + }, + ], + optional: [ + { + type: "volume", + fields: { name: { env: "VOLUME_NAME", origin: "user" } }, + }, + ], + }, +}; + +describe("extractRequirements", () => { + it("returns required rows first, then optional", () => { + const rows = extractRequirements(pluginItem(ANALYTICS_MANIFEST)); + expect(rows.map((r) => [r.type, r.required])).toEqual([ + ["sql_warehouse", true], + ["volume", false], + ]); + }); + + it("captures permission, fields, env and origin", () => { + const [warehouse] = extractRequirements(pluginItem(ANALYTICS_MANIFEST)); + expect(warehouse.permission).toBe("CAN_USE"); + expect(warehouse.fields).toEqual([ + { + key: "id", + env: "DATABRICKS_WAREHOUSE_ID", + origin: "user", + description: undefined, + }, + ]); + }); + + it("returns empty for a UI item with no manifest", () => { + const ui: RegistryItem = { + name: "metric-card", + files: [ + { + path: "metric-card.tsx", + target: "components/metric-card.tsx", + type: "registry:component", + content: "export const MetricCard = () => null;", + }, + ], + }; + expect(extractRequirements(ui)).toEqual([]); + }); + + it("returns empty for a plugin with no declared resources", () => { + const rows = extractRequirements( + pluginItem({ name: "hello", resources: { required: [], optional: [] } }), + ); + expect(rows).toEqual([]); + }); + + it("tolerates malformed manifest json", () => { + const item: RegistryItem = { + name: "broken", + files: [ + { + path: "manifest.json", + target: "manifest.json", + type: "registry:file", + content: "{ not json", + }, + ], + }; + expect(extractRequirements(item)).toEqual([]); + }); +}); + +describe("renderRequirements", () => { + it("renders a no-requirements line for items without resources", () => { + const out = renderRequirements(pluginItem({ name: "hello" })); + expect(out).toContain("no resource requirements"); + }); + + it("lists each resource with its env vars and origin", () => { + const out = renderRequirements(pluginItem(ANALYTICS_MANIFEST)); + expect(out).toContain("sql_warehouse"); + expect(out).toContain("required"); + expect(out).toContain("CAN_USE"); + expect(out).toContain("DATABRICKS_WAREHOUSE_ID"); + expect(out).toContain("volume"); + expect(out).toContain("optional"); + expect(out).toContain("VOLUME_NAME"); + }); +}); + +describe("fieldOrigin", () => { + it("trusts an explicit computed origin (synced manifest)", () => { + expect(fieldOrigin({ key: "id", origin: "platform" })).toBe("platform"); + expect(fieldOrigin({ key: "id", origin: "user" })).toBe("user"); + }); + + it("derives platform from localOnly when origin is absent", () => { + expect(fieldOrigin({ key: "host", localOnly: true })).toBe("platform"); + }); + + it("derives static from a default value when origin is absent", () => { + expect(fieldOrigin({ key: "port", value: "5432" })).toBe("static"); + }); + + it("derives cli from a resolve key when origin is absent", () => { + expect(fieldOrigin({ key: "endpoint", resolve: "postgres:host" })).toBe( + "cli", + ); + }); + + it("defaults a bare env field to user", () => { + expect(fieldOrigin({ key: "id", env: "DATABRICKS_WAREHOUSE_ID" })).toBe( + "user", + ); + }); + + it("gives localOnly precedence over a default value", () => { + expect(fieldOrigin({ key: "port", localOnly: true, value: "5432" })).toBe( + "platform", + ); + }); +}); + +describe("isValidEnvName", () => { + it("accepts plain env identifiers", () => { + expect(isValidEnvName("DATABRICKS_WAREHOUSE_ID")).toBe(true); + expect(isValidEnvName("_private")).toBe(true); + expect(isValidEnvName("PORT2")).toBe(true); + }); + + it("rejects names with a newline, space, or leading digit", () => { + expect(isValidEnvName("PORT=x\nDATABRICKS_HOST=evil")).toBe(false); + expect(isValidEnvName("has space")).toBe(false); + expect(isValidEnvName("2FOO")).toBe(false); + expect(isValidEnvName("")).toBe(false); + expect(isValidEnvName("FOO=BAR")).toBe(false); + }); +}); diff --git a/packages/shared/src/cli/commands/registry/requirements.ts b/packages/shared/src/cli/commands/registry/requirements.ts new file mode 100644 index 000000000..371cdecbb --- /dev/null +++ b/packages/shared/src/cli/commands/registry/requirements.ts @@ -0,0 +1,171 @@ +import path from "node:path"; +import pc from "picocolors"; +import { computeOriginFromField } from "../../../schemas/manifest"; +import type { RegistryItem } from "./client"; + +/** + * A single resource field as declared in a plugin manifest. `origin` is the + * computed classifier written by `plugin sync` (platform/static/cli/user) that + * says how the value reaches the running app. + */ +export interface RequirementField { + key: string; + env?: string; + /** + * Computed classifier written by `plugin sync` — only present in a synced + * manifest. Authored manifests omit it, so consumers must derive the + * effective origin via {@link fieldOrigin} rather than reading this directly. + */ + origin?: string; + description?: string; + /** Default literal value (static origin); pre-filled without prompting. */ + value?: string; + /** Resolver key (cli origin), e.g. "postgres:host". */ + resolve?: string; + /** Local-dev-only field; platform-injected at deploy time. */ + localOnly?: boolean; +} + +/** A resource requirement flattened for display. */ +export interface ResourceRequirementRow { + type: string; + resourceKey?: string; + permission?: string; + required: boolean; + description?: string; + fields: RequirementField[]; +} + +interface ManifestFieldShape { + env?: string; + origin?: string; + description?: string; + value?: string; + resolve?: string; + localOnly?: boolean; +} +interface ManifestResourceShape { + type?: string; + resourceKey?: string; + permission?: string; + description?: string; + fields?: Record; +} +interface ManifestShape { + resources?: { + required?: ManifestResourceShape[]; + optional?: ManifestResourceShape[]; + }; +} + +function toFields( + fields: Record | undefined, +): RequirementField[] { + return Object.entries(fields ?? {}).map(([key, f]) => ({ + key, + env: f.env, + origin: f.origin, + description: f.description, + value: f.value, + resolve: f.resolve, + localOnly: f.localOnly, + })); +} + +function toRows( + resources: ManifestResourceShape[] | undefined, + required: boolean, +): ResourceRequirementRow[] { + return (resources ?? []).map((r) => ({ + type: r.type ?? "unknown", + resourceKey: r.resourceKey, + permission: r.permission, + required, + description: r.description, + fields: toFields(r.fields), + })); +} + +/** The manifest.json file shipped by a plugin item, or null for UI items. */ +function findManifest(item: RegistryItem): ManifestShape | null { + const file = (item.files ?? []).find( + (f) => path.basename(f.target ?? f.path) === "manifest.json", + ); + if (!file) return null; + try { + return JSON.parse(file.content) as ManifestShape; + } catch { + return null; + } +} + +/** + * Extracts a plugin item's declared resource requirements (required first, + * then optional). Returns an empty array for UI items or plugins that declare + * no resources. + */ +export function extractRequirements( + item: RegistryItem, +): ResourceRequirementRow[] { + const manifest = findManifest(item); + if (!manifest) return []; + return [ + ...toRows(manifest.resources?.required, true), + ...toRows(manifest.resources?.optional, false), + ]; +} + +/** + * Renders the resource requirements for an item as human-readable lines. + * Returns a single "no resources" line when there are none, so callers can + * print unconditionally. + */ +export function renderRequirements( + item: RegistryItem, + rows: ResourceRequirementRow[] = extractRequirements(item), +): string { + if (rows.length === 0) { + return pc.dim(`${item.name}: no resource requirements.`); + } + + const lines: string[] = [pc.bold(`Resources required by ${item.name}:`)]; + for (const row of rows) { + const tag = row.required ? pc.yellow("required") : pc.dim("optional"); + const perm = row.permission ? pc.dim(` [${row.permission}]`) : ""; + lines.push(` ${pc.cyan(row.type)} (${tag})${perm}`); + if (row.description) lines.push(` ${pc.dim(row.description)}`); + for (const field of row.fields) { + if (!field.env) continue; + const origin = field.origin ? pc.dim(` (${field.origin})`) : ""; + lines.push(` - ${field.env}${origin}`); + } + } + return lines.join("\n"); +} + +/** + * A syntactically valid environment variable name. Field `env` names come from + * an untrusted manifest and are written into `.env` as `NAME=value` (a newline + * in the name would inject a second line — e.g. `PORT=x\nDATABRICKS_HOST=…` → + * credential exfil) and emitted into `app.yaml`. Anything that isn't a plain + * env identifier is dropped before it reaches those sinks. + */ +const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; + +/** True when `name` is a safe, well-formed environment variable name. */ +export function isValidEnvName(name: string): boolean { + return ENV_NAME.test(name); +} + +/** + * Effective origin of a field: trust a synced manifest's stamped `origin`, else + * derive it from the field shape via the same rule `plugin sync` uses, so an + * authored manifest (no `origin`) still classifies correctly. + */ +export function fieldOrigin( + field: RequirementField, +): "platform" | "static" | "cli" | "user" { + if (field.origin) + return field.origin as "platform" | "static" | "cli" | "user"; + return computeOriginFromField(field); +} diff --git a/packages/shared/src/cli/commands/registry/server-register.test.ts b/packages/shared/src/cli/commands/registry/server-register.test.ts new file mode 100644 index 000000000..48db1eb4d --- /dev/null +++ b/packages/shared/src/cli/commands/registry/server-register.test.ts @@ -0,0 +1,148 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { registerPluginInServer } from "./server-register"; + +const tempDirs: string[] = []; + +function makeTempDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "server-register-")); + tempDirs.push(dir); + return dir; +} + +/** Writes a server entry with a createApp({ plugins: [...] }) call. */ +function writeServer(dir: string, rel: string, plugins = ""): string { + const file = path.join(dir, rel); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync( + file, + `import { createApp } from "@databricks/appkit";\n\n` + + `const app = await createApp({ plugins: [${plugins}] });\n`, + ); + return file; +} + +afterEach(() => { + for (const dir of tempDirs) { + try { + fs.rmSync(dir, { recursive: true, force: true }); + } catch { + // best effort + } + } + tempDirs.length = 0; +}); + +describe("registerPluginInServer", () => { + it("wires a plugin into a server root that is a subdir (e.g. api/)", () => { + // The server root is the resolved subdir, not the repo root — entry files + // are looked up within it, so an app laid out under api/ still gets wired. + const dir = makeTempDir(); + const serverRoot = path.join(dir, "api"); + writeServer(serverRoot, "index.ts"); + + const result = registerPluginInServer( + serverRoot, + "./plugins/hello", + "hello", + ); + + expect(result.status).toBe("wired"); + expect(result.file).toBe("index.ts"); + const written = fs.readFileSync(path.join(serverRoot, "index.ts"), "utf-8"); + expect(written).toContain('import { hello } from "./plugins/hello";'); + expect(written).toContain("hello()"); + }); + + it("finds an entry under src/ within the server root", () => { + const dir = makeTempDir(); + writeServer(dir, "src/server.ts"); + + const result = registerPluginInServer(dir, "./plugins/hello", "hello"); + + expect(result.status).toBe("wired"); + expect(result.file).toBe(path.join("src", "server.ts")); + }); + + it("is idempotent — reports already-registered without duplicating", () => { + const dir = makeTempDir(); + writeServer(dir, "index.ts", "hello()"); + + const result = registerPluginInServer(dir, "./plugins/hello", "hello"); + + expect(result.status).toBe("already"); + const written = fs.readFileSync(path.join(dir, "index.ts"), "utf-8"); + expect(written.match(/hello\(\)/g)).toHaveLength(1); + }); + + it("adds the import when the same path is imported under a different binding", () => { + // Regression: import de-dup must key on the binding, not the module path. + // An existing `import { HelloPlugin } from "./plugins/hello"` (different + // local name, not in the plugins array) must not suppress the `hello` + // import, or the added `hello()` element references an unimported symbol. + const dir = makeTempDir(); + const file = path.join(dir, "index.ts"); + fs.writeFileSync( + file, + `import { createApp } from "@databricks/appkit";\n` + + `import { HelloPlugin } from "./plugins/hello";\n\n` + + `const app = await createApp({ plugins: [] });\n`, + ); + + const result = registerPluginInServer(dir, "./plugins/hello", "hello"); + + expect(result.status).toBe("wired"); + const written = fs.readFileSync(file, "utf-8"); + expect(written).toContain('import { hello } from "./plugins/hello";'); + expect(written).toContain("hello()"); + // The pre-existing, differently-named import is left intact. + expect(written).toContain('import { HelloPlugin } from "./plugins/hello";'); + }); + + it("does not add a conflicting import when the binding already exists", () => { + // The binding `hello` is already imported (from another module) but not yet + // in the array. De-dup keys on the binding, so no second `hello` import is + // added — that would be a duplicate declaration. Path-based de-dup would + // miss this and emit a conflicting `import { hello } from "./plugins/hello"`. + const dir = makeTempDir(); + const file = path.join(dir, "index.ts"); + fs.writeFileSync( + file, + `import { createApp } from "@databricks/appkit";\n` + + `import { hello } from "./elsewhere";\n\n` + + `const app = await createApp({ plugins: [] });\n`, + ); + + const result = registerPluginInServer(dir, "./plugins/hello", "hello"); + + expect(result.status).toBe("wired"); + const written = fs.readFileSync(file, "utf-8"); + // Exactly one `hello` binding, still the original — no duplicate added. + expect(written.match(/import \{ hello \} from/g)).toHaveLength(1); + expect(written).toContain('import { hello } from "./elsewhere";'); + expect(written).not.toContain('import { hello } from "./plugins/hello";'); + expect(written).toContain("hello()"); + }); + + it("skips (for a printed fallback) when there is no server entry", () => { + const dir = makeTempDir(); + const result = registerPluginInServer(dir, "./plugins/hello", "hello"); + expect(result.status).toBe("skipped"); + }); + + it("skips when the entry has no createApp plugins array", () => { + const dir = makeTempDir(); + fs.writeFileSync(path.join(dir, "index.ts"), "const x = 1;\n"); + const result = registerPluginInServer(dir, "./plugins/hello", "hello"); + expect(result.status).toBe("skipped"); + }); + + it("refuses an export name that is not a plain identifier", () => { + const dir = makeTempDir(); + writeServer(dir, "index.ts"); + const result = registerPluginInServer(dir, "./plugins/x", "evil()); x("); + expect(result.status).toBe("skipped"); + }); +}); diff --git a/packages/shared/src/cli/commands/registry/server-register.ts b/packages/shared/src/cli/commands/registry/server-register.ts new file mode 100644 index 000000000..0dc82a3f6 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/server-register.ts @@ -0,0 +1,166 @@ +import fs from "node:fs"; +import path from "node:path"; +import { Lang, parse, type SgNode } from "@ast-grep/napi"; +import { JS_IDENTIFIER } from "./constants"; + +/** Server entry candidates within the server root, in priority order. */ +const SERVER_FILE_CANDIDATES = [ + "server.ts", + "index.ts", + "src/server.ts", + "src/index.ts", +]; + +export interface RegisterResult { + /** wired = edited; already = plugin was present; skipped = couldn't safely edit. */ + status: "wired" | "already" | "skipped"; + file?: string; + reason?: string; +} + +function findServerFile(serverRoot: string): string | null { + for (const candidate of SERVER_FILE_CANDIDATES) { + const p = path.join(serverRoot, candidate); + if (fs.existsSync(p)) return p; + } + return null; +} + +/** The `plugins: [...]` array node inside a createApp call, if present. */ +function findPluginsArray(root: SgNode): SgNode | null { + for (const pair of root.findAll({ rule: { kind: "pair" } })) { + const key = pair.find({ rule: { kind: "property_identifier" } }); + if (key?.text() !== "plugins") continue; + const arr = pair.find({ rule: { kind: "array" } }); + if (arr) return arr; + } + return null; +} + +function arrayElementNames(arr: SgNode): Set { + const names = new Set(); + for (const child of arr.children()) { + if (child.kind() === "identifier") { + names.add(child.text()); + } else if (child.kind() === "call_expression") { + const callee = child.children()[0]; + if (callee?.kind() === "identifier") names.add(callee.text()); + } + } + return names; +} + +/** Local binding names introduced by an import statement — default, namespace, + * and named specifiers, with aliases resolved to the local name. */ +function importBindingNames(stmt: SgNode): Set { + const names = new Set(); + const clause = stmt.find({ rule: { kind: "import_clause" } }); + if (!clause) return names; + for (const child of clause.children()) { + const kind = child.kind(); + if (kind === "identifier") { + names.add(child.text()); + } else if (kind === "namespace_import") { + const id = child.find({ rule: { kind: "identifier" } }); + if (id) names.add(id.text()); + } else if (kind === "named_imports") { + for (const spec of child.findAll({ + rule: { kind: "import_specifier" }, + })) { + const local = spec.field("alias")?.text() ?? spec.field("name")?.text(); + if (local) names.add(local); + } + } + } + return names; +} + +/** + * Best-effort: register a plugin in the server entry's `createApp({ plugins })` + * call by inserting the import and adding it to the array. Only edits the + * standard shape (a `plugins: [...]` array literal); returns `skipped` otherwise + * so the caller can fall back to printing manual instructions. Idempotent. + */ +export function registerPluginInServer( + serverRoot: string, + importPath: string, + exportName: string, +): RegisterResult { + // exportName and importPath are interpolated into the user's server source. + // Registry items are untrusted, so refuse anything that isn't a plain JS + // identifier / clean relative module path — prevents code injection via a + // crafted export name or import path. + if (!JS_IDENTIFIER.test(exportName)) { + return { status: "skipped", reason: "invalid plugin export name" }; + } + if (!/^[.][./A-Za-z0-9_-]*$/.test(importPath)) { + return { status: "skipped", reason: "invalid plugin import path" }; + } + + const serverFile = findServerFile(serverRoot); + if (!serverFile) { + return { status: "skipped", reason: "no server entry file found" }; + } + + const content = fs.readFileSync(serverFile, "utf-8"); + const lang = serverFile.endsWith(".tsx") ? Lang.Tsx : Lang.TypeScript; + const root = parse(lang, content).root(); + + const arr = findPluginsArray(root); + if (!arr) { + return { + status: "skipped", + reason: "no createApp({ plugins: [...] }) array found", + }; + } + + const file = path.relative(serverRoot, serverFile); + if (arrayElementNames(arr).has(exportName)) { + return { status: "already", file }; + } + + const edits = []; + + // toPlugin exports are factories, registered as a call: `hello()`. + const newElem = `${exportName}()`; + + // Insert before the first element, matching its indentation so the array + // formatting is preserved (or inline for a single-line array). + const elementKinds = ["identifier", "call_expression", "spread_element"]; + const firstEl = arr + .children() + .find((c) => elementKinds.includes(c.kind() as string)); + if (!firstEl) { + edits.push(arr.replace(`[${newElem}]`)); + } else { + const startIdx = firstEl.range().start.index; + const lineStart = content.lastIndexOf("\n", startIdx - 1); + const indent = content.slice(lineStart + 1, startIdx); + const multiline = lineStart !== -1 && /^[ \t]*$/.test(indent); + const sep = multiline ? `,\n${indent}` : ", "; + edits.push(firstEl.replace(`${newElem}${sep}${firstEl.text()}`)); + } + + // Add the import unless `exportName` is already bound by some import. De-dup + // on the binding, not the module path: an existing import from the same path + // under a different name (e.g. `import { HelloPlugin } from "./plugins/hello"`) + // must not suppress this one, or the `${exportName}()` element just added to + // the array would reference an unimported symbol and the server won't compile. + const importStmts = root.findAll({ rule: { kind: "import_statement" } }); + const hasBinding = importStmts.some((s) => + importBindingNames(s).has(exportName), + ); + const importLine = `import { ${exportName} } from "${importPath}";`; + if (!hasBinding && importStmts.length > 0) { + const last = importStmts[importStmts.length - 1]; + edits.push(last.replace(`${last.text()}\n${importLine}`)); + } + + let output = root.commitEdits(edits); + if (!hasBinding && importStmts.length === 0) { + output = `${importLine}\n${output}`; + } + fs.writeFileSync(serverFile, output); + + return { status: "wired", file }; +} diff --git a/packages/shared/src/cli/commands/registry/workspace-picker.test.ts b/packages/shared/src/cli/commands/registry/workspace-picker.test.ts new file mode 100644 index 000000000..b3546e175 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/workspace-picker.test.ts @@ -0,0 +1,369 @@ +import { describe, expect, it, vi } from "vitest"; +import { + composeResourceId, + isFlatListable, + isParentContext, + listParentContextStep, + listWorkspaceResources, + MAX_PICKER_RESULTS, + parentContextDepth, + toChoices, +} from "./workspace-picker"; + +describe("isFlatListable", () => { + it("recognizes flat types and rejects parent-context/unknown ones", () => { + expect(isFlatListable("sql_warehouse")).toBe(true); + expect(isFlatListable("genie_space")).toBe(true); + // parent-context types are handled elsewhere + expect(isFlatListable("volume")).toBe(false); + expect(isFlatListable("secret")).toBe(false); + expect(isFlatListable("nonsense")).toBe(false); + }); +}); + +describe("toChoices", () => { + it("reads id and label from a bare array", () => { + const choices = toChoices( + [{ id: "w1", name: "Warehouse One" }], + "id", + "name", + ); + expect(choices).toEqual([{ value: "w1", label: "Warehouse One (w1)" }]); + }); + + it("unwraps a single wrapper key holding the array", () => { + const choices = toChoices({ warehouses: [{ id: "w2" }] }, "id", "name"); + expect(choices).toEqual([{ value: "w2", label: "w2" }]); + }); + + it("skips items missing the id field", () => { + const choices = toChoices([{ name: "no id" }, { id: "ok" }], "id", "name"); + expect(choices).toEqual([{ value: "ok", label: "ok" }]); + }); + + it("coerces non-string ids (e.g. numeric job_id)", () => { + const choices = toChoices([{ job_id: 42, name: "ETL" }], "job_id", "name"); + expect(choices).toEqual([{ value: "42", label: "ETL (42)" }]); + }); +}); + +describe("listWorkspaceResources", () => { + // The client factory param is typed WorkspaceClient; we can't import that + // type here (SDK import is restricted to appkit's wrapper), so the fake is + // built as a plain object and passed through the factory's inferred type. + type ClientFactory = Parameters[2]; + type FakeClient = ReturnType>; + /** Builds a fake client whose services yield the given items. */ + function fakeClient(services: Record): FakeClient { + return services as unknown as FakeClient; + } + + /** An async-iterable service.list() that yields the provided items. */ + function asyncList(items: unknown[]) { + return () => + (async function* () { + for (const i of items) yield i; + })(); + } + + it("returns choices from a successful warehouse list (SDK)", async () => { + const factory = () => + fakeClient({ + warehouses: { list: asyncList([{ id: "w1", name: "One" }]) }, + }); + const res = await listWorkspaceResources( + "sql_warehouse", + undefined, + factory, + ); + expect(res).toEqual({ + choices: [{ value: "w1", label: "One (w1)" }], + truncated: false, + }); + }); + + it("maps job_id + settings.name for jobs", async () => { + const factory = () => + fakeClient({ + jobs: { list: asyncList([{ job_id: 42, settings: { name: "ETL" } }]) }, + }); + const res = await listWorkspaceResources("job", undefined, factory); + expect(res.choices).toEqual([{ value: "42", label: "ETL (42)" }]); + }); + + it("adapts genie listSpaces (Promise-wrapped .spaces)", async () => { + const factory = () => + fakeClient({ + genie: { + listSpaces: async () => ({ + spaces: [{ space_id: "s1", title: "Sales" }], + }), + }, + }); + const res = await listWorkspaceResources("genie_space", undefined, factory); + expect(res.choices).toEqual([{ value: "s1", label: "Sales (s1)" }]); + }); + + // genie listSpaces is single-page; the adapter must follow next_page_token + // so large workspaces aren't capped at one page. + it("follows genie next_page_token across pages", async () => { + const pages: Record< + string, + { spaces: unknown[]; next_page_token?: string } + > = { + "": { spaces: [{ space_id: "s1" }], next_page_token: "p2" }, + p2: { spaces: [{ space_id: "s2" }] }, + }; + const seen: (string | undefined)[] = []; + const factory = () => + fakeClient({ + genie: { + listSpaces: async (req: { page_token?: string }) => { + seen.push(req.page_token); + return pages[req.page_token ?? ""]; + }, + }, + }); + const res = await listWorkspaceResources("genie_space", undefined, factory); + expect(res.choices.map((c) => c.value)).toEqual(["s1", "s2"]); + expect(seen).toEqual([undefined, "p2"]); + }); + + it("stops genie pagination if the same token is echoed back", async () => { + const factory = () => + fakeClient({ + genie: { + listSpaces: async () => ({ + spaces: [{ space_id: "s1" }], + next_page_token: "same", + }), + }, + }); + // Would loop forever if the repeated-token guard weren't present. + const res = await listWorkspaceResources("genie_space", undefined, factory); + expect(res.choices.length).toBeGreaterThan(0); + }); + + it("passes the profile to the client factory", async () => { + const factory = vi.fn(() => + fakeClient({ warehouses: { list: asyncList([]) } }), + ); + await listWorkspaceResources("sql_warehouse", "my-profile", factory); + expect(factory).toHaveBeenCalledWith("my-profile"); + }); + + it("caps results and reports truncation, stopping pagination early", async () => { + // Yield far more than the cap; the iterator must be abandoned at the cap. + let yielded = 0; + const factory = () => + fakeClient({ + warehouses: { + list: () => + (async function* () { + for (let i = 0; i < 10_000; i++) { + yielded++; + yield { id: `w${i}`, name: `W${i}` }; + } + })(), + }, + }); + const res = await listWorkspaceResources( + "sql_warehouse", + undefined, + factory, + ); + expect(res.truncated).toBe(true); + expect(res.choices).toHaveLength(MAX_PICKER_RESULTS); + // Pagination stopped: we consumed only up to the cap, not all 10k. + expect(yielded).toBe(MAX_PICKER_RESULTS); + }); + + it("returns empty listing for an unknown type", async () => { + const factory = () => fakeClient({}); + expect( + await listWorkspaceResources("nonsense", undefined, factory), + ).toEqual({ choices: [], truncated: false }); + }); + + it("returns empty listing when the SDK call throws (auth/network error)", async () => { + const factory = () => + fakeClient({ + warehouses: { + list: () => { + throw new Error("auth failed"); + }, + }, + }); + expect( + (await listWorkspaceResources("sql_warehouse", undefined, factory)) + .choices, + ).toEqual([]); + }); + + it("reports the failure reason instead of a silent empty listing", async () => { + const factory = () => + fakeClient({ + warehouses: { + list: () => { + throw new Error( + "default auth: cannot configure default credentials", + ); + }, + }, + }); + const res = await listWorkspaceResources( + "sql_warehouse", + undefined, + factory, + ); + expect(res.choices).toEqual([]); + expect(res.error).toContain("cannot configure default credentials"); + }); + + it("has no error field for a genuinely empty (unknown-type) listing", async () => { + const factory = () => fakeClient({}); + const res = await listWorkspaceResources("nonsense", undefined, factory); + expect(res.error).toBeUndefined(); + }); + + it("returns empty listing when the client factory throws", async () => { + const factory = () => { + throw new Error("no config"); + }; + expect( + (await listWorkspaceResources("sql_warehouse", undefined, factory)) + .choices, + ).toEqual([]); + }); +}); + +describe("isParentContext / parentContextDepth", () => { + it("identifies the four parent-context types and their depth", () => { + expect(isParentContext("volume")).toBe(true); + expect(isParentContext("uc_function")).toBe(true); + expect(isParentContext("secret")).toBe(true); + expect(isParentContext("vector_search_index")).toBe(true); + // flat types are not parent-context + expect(isParentContext("sql_warehouse")).toBe(false); + + expect(parentContextDepth("volume")).toBe(3); // catalog → schema → volume + expect(parentContextDepth("secret")).toBe(2); // scope → key + expect(parentContextDepth("vector_search_index")).toBe(2); + expect(parentContextDepth("sql_warehouse")).toBe(0); + }); +}); + +describe("listParentContextStep", () => { + it("lists catalogs at step 0 for volume", () => { + const run = vi.fn(() => ({ + status: 0, + stdout: JSON.stringify([{ name: "main" }]), + })); + const step = listParentContextStep("volume", 0, [], "my-profile", run); + expect(step?.key).toBe("catalog"); + expect(step?.choices).toEqual([{ value: "main", label: "main (main)" }]); + expect(run).toHaveBeenCalledWith([ + "catalogs", + "list", + "-o", + "json", + "-p", + "my-profile", + ]); + }); + + it("passes the picked catalog+schema as positional args at step 2", () => { + const run = vi.fn(() => ({ + status: 0, + stdout: JSON.stringify([ + { full_name: "main.sales.events", name: "events" }, + ]), + })); + const step = listParentContextStep( + "volume", + 2, + ["main", "sales"], + undefined, + run, + ); + expect(step?.key).toBe("volume"); + // positional args, not flags + expect(run).toHaveBeenCalledWith([ + "volumes", + "list", + "main", + "sales", + "-o", + "json", + ]); + expect(step?.choices).toEqual([ + { value: "main.sales.events", label: "events (main.sales.events)" }, + ]); + }); + + it("drills scope → key for secret", () => { + const run = vi.fn(() => ({ + status: 0, + stdout: JSON.stringify([{ key: "api-token" }]), + })); + const step = listParentContextStep( + "secret", + 1, + ["my-scope"], + undefined, + run, + ); + expect(step?.key).toBe("key"); + expect(run).toHaveBeenCalledWith([ + "secrets", + "list-secrets", + "my-scope", + "-o", + "json", + ]); + expect(step?.choices).toEqual([ + { value: "api-token", label: "api-token (api-token)" }, + ]); + }); + + it("returns null past the end of the chain", () => { + const run = vi.fn(() => ({ status: 0, stdout: "[]" })); + expect(listParentContextStep("secret", 5, [], undefined, run)).toBeNull(); + }); + + it("returns empty choices (not null) when a level lists nothing", () => { + const run = vi.fn(() => ({ status: 0, stdout: "[]" })); + const step = listParentContextStep("volume", 0, [], undefined, run); + expect(step?.choices).toEqual([]); + }); + + // A prior pick starting with `-` would be parsed as a flag when passed as a + // positional arg, so refuse it rather than shell out with it. + it("refuses a `-`-prefixed parent pick without running the CLI", () => { + const run = vi.fn(() => ({ status: 0, stdout: "[]" })); + const step = listParentContextStep( + "volume", + 1, + ["--profile"], + undefined, + run, + ); + expect(step?.choices).toEqual([]); + expect(run).not.toHaveBeenCalled(); + }); +}); + +describe("composeResourceId", () => { + it("joins scope and key for a secret", () => { + expect(composeResourceId("secret", ["my-scope", "api-token"])).toBe( + "my-scope/api-token", + ); + }); + + it("returns the last (self-qualified) pick for other types", () => { + expect( + composeResourceId("volume", ["main", "sales", "main.sales.events"]), + ).toBe("main.sales.events"); + expect(composeResourceId("vector_search_index", ["ep", "idx"])).toBe("idx"); + }); +}); diff --git a/packages/shared/src/cli/commands/registry/workspace-picker.ts b/packages/shared/src/cli/commands/registry/workspace-picker.ts new file mode 100644 index 000000000..6228290d1 --- /dev/null +++ b/packages/shared/src/cli/commands/registry/workspace-picker.ts @@ -0,0 +1,452 @@ +import { spawnSync } from "node:child_process"; +import { + createWorkspaceClient, + type LegacyWorkspaceClient, +} from "../../../workspace-client"; + +/** + * Lists a user's real Databricks workspace resources so `appkit add` can offer + * a picker instead of blind free-text entry. + * + * Flat resource types are listed via the Databricks SDK client (typed, + * auto-paginating) obtained through the sanctioned `workspace-client` facade. + * Parent-context types (volume, uc_function, secret, vector_search_index) still + * shell out to the `databricks` CLI for their drill-down. Every path fails + * soft: any error returns an empty list and the caller drops to free-text entry. + */ + +/** A workspace resource choice surfaced in the picker. */ +export interface WorkspaceChoice { + /** Value written to the env var (the resource id/name). */ + value: string; + /** Human label shown in the picker (name, falling back to value). */ + label: string; +} + +/** + * Per-type SDK lister: streams a resource type off the WorkspaceClient into + * `{value,label}` choices. `list()` returns an async iterable of raw SDK + * objects; `toChoice` maps each to a picker choice. The SDK auto-paginates, so + * we simply iterate to completion. + */ +interface SdkLister { + list: (client: LegacyWorkspaceClient) => AsyncIterable; + toChoice: (item: Record) => WorkspaceChoice | null; +} + +/** Builds a `{value,label}` from an id field and optional label field. */ +function choiceFrom( + item: Record, + idField: string, + labelField?: string, +): WorkspaceChoice | null { + const id = item[idField]; + if (id === undefined || id === null) return null; + const value = String(id); + const rawLabel = labelField ? item[labelField] : undefined; + const label = + typeof rawLabel === "string" && rawLabel.length > 0 + ? `${rawLabel} (${value})` + : value; + return { value, label }; +} + +/** + * Genie listSpaces returns a single page, not an auto-paginating iterable like + * the other services. Adapt it to one that follows `next_page_token`; the + * repeated-token guard avoids an infinite loop if the API echoes a token back. + */ +async function* iterateGenieSpaces( + client: LegacyWorkspaceClient, +): AsyncIterable { + let pageToken: string | undefined; + do { + const res = await client.genie.listSpaces( + pageToken ? { page_token: pageToken } : {}, + ); + for (const space of res.spaces ?? []) yield space; + const next = res.next_page_token; + if (next && next === pageToken) break; + pageToken = next; + } while (pageToken); +} + +/** Flat, top-level listable resource types, backed by SDK services. */ +export const SDK_LISTERS: Record = { + sql_warehouse: { + list: (c) => c.warehouses.list({}), + toChoice: (i) => choiceFrom(i, "id", "name"), + }, + job: { + list: (c) => c.jobs.list({}), + // job name lives under settings.name; id is top-level job_id + toChoice: (i) => { + const settings = i.settings as { name?: string } | undefined; + return choiceFrom({ ...i, name: settings?.name }, "job_id", "name"); + }, + }, + serving_endpoint: { + list: (c) => c.servingEndpoints.list(), + toChoice: (i) => choiceFrom(i, "name", "name"), + }, + uc_connection: { + list: (c) => c.connections.list({}), + toChoice: (i) => choiceFrom(i, "name", "full_name"), + }, + database: { + list: (c) => c.database.listDatabaseInstances({}), + toChoice: (i) => choiceFrom(i, "name", "name"), + }, + genie_space: { + list: iterateGenieSpaces, + toChoice: (i) => choiceFrom(i, "space_id", "title"), + }, + experiment: { + list: (c) => c.experiments.listExperiments({}), + toChoice: (i) => choiceFrom(i, "experiment_id", "name"), + }, + app: { + list: (c) => c.apps.list({}), + toChoice: (i) => choiceFrom(i, "name", "name"), + }, +}; + +/** True when a resource type can be listed flat (no parent context). */ +export function isFlatListable(resourceType: string): boolean { + return resourceType in SDK_LISTERS; +} + +/** + * Constructs a raw SDK workspace client for the given profile (or default + * resolution), via the sanctioned `workspace-client` facade. Uses the legacy + * escape hatch because the picker needs services (connections, database, + * experiments, apps) the facade doesn't yet proxy directly. + */ +export function makeWorkspaceClient(profile?: string): LegacyWorkspaceClient { + return createWorkspaceClient( + profile ? { profile } : {}, + ).toLegacyWorkspaceClient(); +} + +/** + * Max resources fetched for the picker. `list()` auto-paginates, so on a large + * workspace (5000+ warehouses) breaking out at the cap stops pagination early; + * the picker's "Enter manually" option covers anything beyond it. + */ +export const MAX_PICKER_RESULTS = 200; + +/** A listing result plus whether it was truncated at the fetch cap. */ +export interface WorkspaceListing { + choices: WorkspaceChoice[]; + truncated: boolean; + /** Short reason the listing failed (auth/config/network), if it did — lets + * the caller distinguish a real error from a genuinely empty workspace. */ + error?: string; +} + +/** First line of an error, capped, for a user-facing one-liner. */ +function shortErrorMessage(err: unknown): string { + const msg = err instanceof Error ? err.message : String(err); + return msg.split("\n")[0].trim().slice(0, 200); +} + +/** + * Lists workspace resources of a flat-listable type via the SDK, stopping at + * MAX_PICKER_RESULTS so pagination doesn't drain a huge workspace. Returns an + * empty listing on any failure (unknown type, auth/config error, network) so + * the caller can fall back to free-text entry. `clientFactory` is injectable + * for tests. + */ +export async function listWorkspaceResources( + resourceType: string, + profile?: string, + clientFactory: ( + profile?: string, + ) => LegacyWorkspaceClient = makeWorkspaceClient, +): Promise { + const lister = SDK_LISTERS[resourceType]; + if (!lister) return { choices: [], truncated: false }; + try { + const client = clientFactory(profile); + const choices: WorkspaceChoice[] = []; + let truncated = false; + for await (const item of lister.list(client)) { + if (typeof item !== "object" || item === null) continue; + const choice = lister.toChoice(item as Record); + if (!choice) continue; + choices.push(choice); + if (choices.length >= MAX_PICKER_RESULTS) { + // Stop iterating — halts the async iterator, so no further pages fetch. + truncated = true; + break; + } + } + return { choices, truncated }; + } catch (err) { + // Don't swallow silently: return the reason so the caller can tell the user + // the picker fell back because listing failed (typically auth/profile), not + // because the workspace has none of this resource. + return { choices: [], truncated: false, error: shortErrorMessage(err) }; + } +} + +/** Runs a databricks CLI subcommand returning JSON; injectable for tests. */ +export type CliRunner = (args: string[]) => { + status: number | null; + stdout: string; +}; + +const defaultRunner: CliRunner = (args) => { + // Parent-context lists can be large; raise maxBuffer well above the 1MB + // default so a big JSON response isn't truncated into "no resources found". + const res = spawnSync("databricks", args, { + encoding: "utf-8", + maxBuffer: 64 * 1024 * 1024, + }); + return { status: res.status, stdout: res.stdout ?? "" }; +}; + +/** + * Extracts `{value,label}` choices from a parsed CLI list response. The CLI + * returns either a bare array or an object wrapping one; we scan for the first + * array of objects. Items missing the id field are skipped. + */ +export function toChoices( + parsed: unknown, + idField: string, + labelField?: string, +): WorkspaceChoice[] { + const arr = firstArray(parsed); + const choices: WorkspaceChoice[] = []; + for (const item of arr) { + if (typeof item !== "object" || item === null) continue; + const choice = choiceFrom( + item as Record, + idField, + labelField, + ); + if (choice) choices.push(choice); + } + return choices; +} + +/** Finds the first array in a CLI response (bare array or single wrapper key). */ +function firstArray(parsed: unknown): unknown[] { + if (Array.isArray(parsed)) return parsed; + if (parsed && typeof parsed === "object") { + for (const v of Object.values(parsed)) { + if (Array.isArray(v)) return v; + } + } + return []; +} + +/** + * Runs a `databricks … list -o json` command and returns parsed choices, or + * [] on any failure (CLI missing/errored, empty, non-JSON). `command` is the + * argv after `databricks`; `-o json` and `-p ` are appended. + * Used for the parent-context drill-down (catalogs/schemas/scopes/endpoints). + */ +export function runList( + command: string[], + idField: string, + labelField: string | undefined, + profile: string | undefined, + runner: CliRunner = defaultRunner, +): WorkspaceChoice[] { + const args = [...command, "-o", "json"]; + if (profile) args.push("-p", profile); + + let result: { status: number | null; stdout: string }; + try { + result = runner(args); + } catch { + return []; + } + if (result.status !== 0 || !result.stdout.trim()) return []; + + let parsed: unknown; + try { + parsed = JSON.parse(result.stdout); + } catch { + return []; + } + return toChoices(parsed, idField, labelField); +} + +/** + * A drill-down step for a parent-context resource type. `list(parents)` builds + * the CLI argv given the values picked in prior steps (e.g. [catalog] → schema + * list command). `key` labels the step for prompts. + */ +export interface ParentContextStep { + key: string; + list: (parents: string[]) => { + command: string[]; + idField: string; + labelField?: string; + }; +} + +/** + * Drill-down chains for parent-context resource types. Each ends by listing + * the resource itself; earlier steps list the parents to pick first. + * Positional-arg CLI gotcha: `databricks schemas list ` etc. take the + * parent as a positional, not a flag. + */ +export const PARENT_CONTEXT_CHAINS: Record = { + volume: [ + { + key: "catalog", + list: () => ({ + command: ["catalogs", "list"], + idField: "name", + labelField: "name", + }), + }, + { + key: "schema", + list: ([catalog]) => ({ + command: ["schemas", "list", catalog], + idField: "name", + labelField: "name", + }), + }, + { + key: "volume", + list: ([catalog, schema]) => ({ + command: ["volumes", "list", catalog, schema], + idField: "full_name", + labelField: "name", + }), + }, + ], + uc_function: [ + { + key: "catalog", + list: () => ({ + command: ["catalogs", "list"], + idField: "name", + labelField: "name", + }), + }, + { + key: "schema", + list: ([catalog]) => ({ + command: ["schemas", "list", catalog], + idField: "name", + labelField: "name", + }), + }, + { + key: "function", + list: ([catalog, schema]) => ({ + command: ["functions", "list", catalog, schema], + idField: "full_name", + labelField: "name", + }), + }, + ], + secret: [ + { + key: "scope", + list: () => ({ + command: ["secrets", "list-scopes"], + idField: "name", + labelField: "name", + }), + }, + { + key: "key", + list: ([scope]) => ({ + command: ["secrets", "list-secrets", scope], + idField: "key", + labelField: "key", + }), + }, + ], + vector_search_index: [ + { + key: "endpoint", + list: () => ({ + command: ["vector-search-endpoints", "list-endpoints"], + idField: "name", + labelField: "name", + }), + }, + { + key: "index", + list: ([endpoint]) => ({ + command: ["vector-search-indexes", "list-indexes", endpoint], + idField: "name", + labelField: "name", + }), + }, + ], +}; + +/** True when a resource type needs a parent-context drill-down to list. */ +export function isParentContext(resourceType: string): boolean { + return resourceType in PARENT_CONTEXT_CHAINS; +} + +/** One resolved step of a drill-down: the choices to present at this level. */ +export interface DrillStep { + key: string; + choices: WorkspaceChoice[]; +} + +/** + * Lists the choices for a single drill-down step given the values picked so + * far. Returns [] on failure. The caller drives the interaction (present + * `choices`, collect a pick, call again with it appended to `parents`). + */ +export function listParentContextStep( + resourceType: string, + stepIndex: number, + parents: string[], + profile?: string, + runner: CliRunner = defaultRunner, +): DrillStep | null { + const chain = PARENT_CONTEXT_CHAINS[resourceType]; + if (!chain || stepIndex >= chain.length) return null; + const step = chain[stepIndex]; + // Prior picks become positional CLI args; a value starting with `-` would be + // parsed as a flag, so refuse it (empty step → free-text fallback). Real + // catalog/schema/scope/endpoint names never start with `-`. + if (parents.some((p) => p.startsWith("-"))) { + return { key: step.key, choices: [] }; + } + const spec = step.list(parents); + return { + key: step.key, + choices: runList( + spec.command, + spec.idField, + spec.labelField, + profile, + runner, + ), + }; +} + +/** Number of drill-down steps for a parent-context type (0 if not one). */ +export function parentContextDepth(resourceType: string): number { + return PARENT_CONTEXT_CHAINS[resourceType]?.length ?? 0; +} + +/** + * Builds the final resource identifier from the values picked across a + * drill-down. Most types end on a self-qualified id (volume/uc_function list + * `full_name`; a vector-search index name is already catalog.schema-qualified), + * so the last pick is the whole answer. A `secret` is addressed by both its + * scope and key (`scope/key`) — returning only the key drops the scope and + * yields a value that can't locate the secret — so its picks are joined. + */ +export function composeResourceId( + resourceType: string, + picks: string[], +): string { + if (resourceType === "secret") return picks.join("/"); + return picks[picks.length - 1]; +} diff --git a/packages/shared/src/cli/deploy-config.ts b/packages/shared/src/cli/deploy-config.ts new file mode 100644 index 000000000..9da8c0df4 --- /dev/null +++ b/packages/shared/src/cli/deploy-config.ts @@ -0,0 +1,57 @@ +/** + * Single source for the `app.yaml` + `databricks.yml` app-resource binding + * contract, so the registry writer (`config-writer.ts`) and the doctor reader + * (`bundle.ts`/`checks-wiring.ts`) on opposite ends of it can't drift apart. + */ + +/** Canonical Databricks bundle config file name. */ +export const DATABRICKS_YML_FILE = "databricks.yml"; + +/** Canonical Databricks Apps runtime config file name. */ +export const APP_YAML_FILE = "app.yaml"; + +/** An `app.yaml` env entry: `- name: `, `valueFrom: `. */ +export interface AppYamlEnvEntry { + name: string; + valueFrom: string; +} + +/** + * A `databricks.yml` app-resource binding under `resources.apps..resources[]`, + * serialized as `{ name, : { ...fields, permission? } }` (see {@link bindingToNode}). + */ +export interface ResourceBinding { + /** Binding name — the `valueFrom` join key (equals the resourceKey). */ + name: string; + /** Resource type key, e.g. `sql_warehouse` / `postgres`. */ + type: string; + permission?: string; + /** Binding fields, typically `${var.}` references. */ + fields: Record; +} + +/** + * Encodes a binding into its `databricks.yml` node: the type is the single + * non-`name` object key. Inverse of {@link bindingTypeOf}. + */ +export function bindingToNode( + binding: ResourceBinding, +): Record { + const inner: Record = { ...binding.fields }; + if (binding.permission) inner.permission = binding.permission; + return { name: binding.name, [binding.type]: inner }; +} + +/** + * Reads the type back out of a `databricks.yml` binding node — the single + * non-`name` object-valued property; undefined if absent. See {@link bindingToNode}. + */ +export function bindingTypeOf( + block: Record, +): string | undefined { + for (const [k, v] of Object.entries(block)) { + if (k === "name") continue; + if (v && typeof v === "object") return k; + } + return undefined; +} diff --git a/packages/shared/src/cli/index.ts b/packages/shared/src/cli/index.ts index 53398b0e1..86db0219d 100644 --- a/packages/shared/src/cli/index.ts +++ b/packages/shared/src/cli/index.ts @@ -10,6 +10,8 @@ import { doctorCommand } from "./commands/doctor/index.js"; import { generateTypesCommand } from "./commands/generate-types.js"; import { lintCommand } from "./commands/lint.js"; import { pluginCommand } from "./commands/plugin/index.js"; +import { addCommand } from "./commands/registry/add.js"; +import { registryCommand } from "./commands/registry/index.js"; import { setupCommand } from "./commands/setup.js"; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -30,5 +32,9 @@ cmd.addCommand(docsCommand); cmd.addCommand(pluginCommand); cmd.addCommand(codemodCommand); cmd.addCommand(doctorCommand); +// Registry commands are executable but hidden from --help while the feature +// is still in development (registry + add work end-to-end but aren't announced). +cmd.addCommand(registryCommand, { hidden: true }); +cmd.addCommand(addCommand, { hidden: true }); await cmd.parseAsync(); diff --git a/packages/shared/src/schemas/manifest.ts b/packages/shared/src/schemas/manifest.ts index bf41293f7..b3d886609 100644 --- a/packages/shared/src/schemas/manifest.ts +++ b/packages/shared/src/schemas/manifest.ts @@ -784,11 +784,11 @@ export const originSchema = z * - `resolve !== undefined` → `"cli"` (resolved by the CLI during init). * - else → `"user"` (user must provide the value at init time). * - * Co-located with `templateFieldEntrySchema` because the transform is the - * only consumer. Kept private so any other "origin computation" goes - * through the schema rather than re-implementing the rules. + * The single source for this rule: `plugin sync`'s transform stamps `origin` + * with it, and consumers that read an authored manifest (no stamped `origin`) + * derive it through this rather than re-implementing the cascade. */ -function computeOriginFromField(field: { +export function computeOriginFromField(field: { localOnly?: boolean; value?: string; resolve?: string; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 86f24f334..61b579dd9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -576,6 +576,9 @@ importers: picocolors: specifier: 1.1.1 version: 1.1.1 + yaml: + specifier: 2.8.2 + version: 2.8.2 zod: specifier: 4.3.6 version: 4.3.6 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 3b88e3501..4dddd5993 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,7 @@ packages: - "packages/*" - "apps/*" + # Local scratch apps (gitignored contents). Workspace-linked so their appkit + # deps resolve to the monorepo; knip skips them via the apps/** ignore. + - "apps/scratch/*" - "docs"