From 29a255e3f75c0f9016b573528ad0205c1a0d11bd Mon Sep 17 00:00:00 2001 From: marcopiraccini Date: Sun, 19 Jul 2026 17:34:33 +0200 Subject: [PATCH 1/6] ecs support Signed-off-by: marcopiraccini --- packages/workflow/lib/auth/index.ts | 57 +++++++++---- packages/workflow/lib/platform.ts | 27 ++++++ packages/workflow/plugins/db.ts | 25 +++--- packages/world/src/index.ts | 24 ++---- packages/world/src/lib/client.ts | 6 +- packages/world/src/lib/platform.ts | 30 +++++++ packages/world/src/lib/sa-path.ts | 5 -- packages/world/test/platform.test.ts | 122 +++++++++++++++++++++++++++ 8 files changed, 244 insertions(+), 52 deletions(-) create mode 100644 packages/workflow/lib/platform.ts create mode 100644 packages/world/src/lib/platform.ts delete mode 100644 packages/world/src/lib/sa-path.ts create mode 100644 packages/world/test/platform.test.ts diff --git a/packages/workflow/lib/auth/index.ts b/packages/workflow/lib/auth/index.ts index cc6ff9d..7f3d691 100644 --- a/packages/workflow/lib/auth/index.ts +++ b/packages/workflow/lib/auth/index.ts @@ -1,15 +1,21 @@ import type { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify' import { createK8sTokenValidator } from './k8s-token.ts' -import { Unauthorized, Forbidden } from '../errors.ts' +import { Unauthorized, Forbidden, AppNotFound } from '../errors.ts' export interface AuthConfig { - mode: 'k8s-token' | 'api-key' | 'both' | 'none' - defaultAppId?: number + // Present when the platform supplies an identity to verify. Authentication is + // enabled exactly when the means to perform it is supplied, so "authenticate + // but without the config to do so" is unrepresentable. k8s?: { apiServer: string caCert?: string adminServiceAccount?: string + saTokenPath?: string } + // Resolve the tenant from the URL rather than pinning one application. + multiTenant: boolean + // Used when multiTenant is false. + defaultAppId?: number } declare module 'fastify' { @@ -30,23 +36,46 @@ function isAdminPath (url: string): boolean { )) || url.startsWith('/api/v1/versions/') } +// Resolve an application named in the URL. Throws rather than leaving appId at +// its default, which would scope queries to application_id = 0 and make an +// unknown application look like an empty one. +async function resolveApp (app: FastifyInstance, appLabel: string): Promise { + const result = await app.pg.query( + 'SELECT id FROM workflow_applications WHERE app_id = $1', + [appLabel] + ) + if (result.rows.length === 0) throw new AppNotFound(appLabel) + return result.rows[0].id +} + async function authPlugin (app: FastifyInstance, config: AuthConfig): Promise { app.decorateRequest('appId', 0) app.decorateRequest('isAdmin', false) - // No-auth mode: set appId from config and skip all token parsing - if (config.mode === 'none') { + const validateK8s = config.k8s + ? createK8sTokenValidator(app.pg, config.k8s, app.log) + : null + + // Unauthenticated: every caller is admin. Tenancy still applies on managed + // platforms, where the client names its application in the URL and ICC is the + // one that registered it. + if (!validateK8s) { app.addHook('onRequest', async (request: FastifyRequest) => { - request.appId = config.defaultAppId || 0 + const url = request.url.split('?')[0] + if (PUBLIC_PATHS.has(url)) return + request.isAdmin = true + + const appIdMatch = config.multiTenant + ? url.match(/^\/api\/v1\/apps\/([^/]+)/) + : null + request.appId = appIdMatch + ? await resolveApp(app, appIdMatch[1]) + : config.defaultAppId || 0 }) return } - const validateK8s = config.k8s - ? createK8sTokenValidator(app.pg, config.k8s, app.log) - : null - app.addHook('onRequest', async (request: FastifyRequest, reply: FastifyReply) => { const url = request.url.split('?')[0] @@ -82,13 +111,7 @@ async function authPlugin (app: FastifyInstance, config: AuthConfig): Promise 0) { - request.appId = result.rows[0].id - } + request.appId = await resolveApp(app, appIdMatch[1]) } return } diff --git a/packages/workflow/lib/platform.ts b/packages/workflow/lib/platform.ts new file mode 100644 index 0000000..6005284 --- /dev/null +++ b/packages/workflow/lib/platform.ts @@ -0,0 +1,27 @@ +import { existsSync } from 'node:fs' + +// Mirrors packages/world/src/lib/platform.ts. Duplicated deliberately: the +// service must not take a dependency on the client package. + +// Location of the mounted Kubernetes service account, overridable for testing. +export function saPath (file: string): string { + const base = process.env.PLT_WORLD_SA_PATH || '/var/run/secrets/kubernetes.io/serviceaccount' + return `${base}/${file}` +} + +// A service account token is what marks the process as running in K8s. +export function isRunningInK8s (): boolean { + return existsSync(saPath('token')) +} + +// ECS injects a task-scoped metadata endpoint into every container. Note that +// AWS_EXECUTION_ENV is not used: Lambda sets it too, with a different prefix. +export function isRunningInEcs (): boolean { + return Boolean(process.env.ECS_CONTAINER_METADATA_URI_V4 || process.env.ECS_CONTAINER_METADATA_URI) +} + +// A managed platform is one where ICC provisions applications, so tenancy +// applies even when there is no identity to authenticate. +export function isManagedPlatform (): boolean { + return isRunningInK8s() || isRunningInEcs() +} diff --git a/packages/workflow/plugins/db.ts b/packages/workflow/plugins/db.ts index 77b3ed7..195b32a 100644 --- a/packages/workflow/plugins/db.ts +++ b/packages/workflow/plugins/db.ts @@ -1,7 +1,7 @@ import fp from 'fastify-plugin' -import { existsSync } from 'node:fs' import type { FastifyInstance } from 'fastify' import { initDb, decorateDb } from '../lib/db.ts' +import { saPath, isRunningInK8s, isManagedPlatform } from '../lib/platform.ts' import type { AuthConfig } from '../lib/auth/index.ts' declare module 'fastify' { @@ -19,24 +19,27 @@ async function dbPlugin (app: FastifyInstance): Promise { const pool = await initDb({ connectionString }) decorateDb(app, pool, connectionString) - // Detect mode and build auth config - const isK8s = existsSync('/var/run/secrets/kubernetes.io/serviceaccount/token') + // Two independent axes, both derived from platform-injected facts. K8s + // supplies an identity to verify; K8s and ECS both mean ICC provisions + // applications, so tenancy applies with or without authentication. + const isK8s = isRunningInK8s() + const multiTenant = isManagedPlatform() let authConfig: AuthConfig - if (isK8s) { - const authMode = (process.env.WF_AUTH_MODE || 'k8s-token') as 'api-key' | 'k8s-token' | 'both' + if (multiTenant) { authConfig = { - mode: authMode, - k8s: authMode !== 'api-key' + multiTenant: true, + k8s: isK8s ? { apiServer: process.env.K8S_API_SERVER || 'https://kubernetes.default.svc', - caCert: process.env.K8S_CA_CERT || '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', + caCert: process.env.K8S_CA_CERT || saPath('ca.crt'), adminServiceAccount: process.env.K8S_ADMIN_SERVICE_ACCOUNT, + saTokenPath: saPath('token'), } : undefined, } - app.log.info('Starting in multi-tenant mode (K8s detected)') + app.log.info({ authenticated: isK8s }, 'Starting in multi-tenant mode') } else { const appIdStr = process.env.PLT_WORLD_APP_ID || 'default' const result = await pool.query( @@ -46,8 +49,8 @@ async function dbPlugin (app: FastifyInstance): Promise { RETURNING id`, [appIdStr] ) - authConfig = { mode: 'none', defaultAppId: result.rows[0].id } - app.log.info('Starting in single-tenant mode (no K8s detected)') + authConfig = { multiTenant: false, defaultAppId: result.rows[0].id } + app.log.info('Starting in single-tenant mode (unmanaged)') } app.decorate('authConfig', authConfig) diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index f664b55..aa48f76 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -2,7 +2,7 @@ import { readFileSync } from 'node:fs' import { join } from 'node:path' import { getSharedContext } from '@platformatic/globals' import type { World } from '@workflow/world' -import { saPath } from './lib/sa-path.ts' +import { isManagedPlatform } from './lib/platform.ts' import { HttpClient } from './lib/client.ts' import type { ClientConfig } from './lib/client.ts' import { createStorage } from './lib/storage.ts' @@ -36,7 +36,7 @@ export function createPlatformaticWorld (config: PlatformaticWorldConfig): World // (http://..svc.cluster.local:/...) so the // workflow service can dispatch cross-namespace. Registering here with // localhost would create a duplicate handler that fails when picked. - if (isRunningInK8s()) return + if (isManagedPlatform()) return // Local dev (no ICC) — register with localhost so the workflow service // running on the same machine can reach us. @@ -80,15 +80,6 @@ async function versionFromSharedContext (): Promise { } } -function isRunningInK8s (): boolean { - try { - readFileSync(saPath('token')) - return true - } catch { - return false - } -} - function readAppName (): string { try { const pkg = JSON.parse(readFileSync(join(process.cwd(), 'package.json'), 'utf8')) @@ -104,14 +95,14 @@ export function createWorld (options?: Partial): World { throw new Error('PLT_WORLD_SERVICE_URL environment variable is required') } - const runningInK8s = isRunningInK8s() + const managed = isManagedPlatform() // PLT_APP_NAME is the platform's own name for the application (watt-extra // resolves it the same way), so it is preferred over the package name. const explicitAppId = options?.appId || process.env.PLT_WORLD_APP_ID || process.env.PLT_APP_NAME const appId = explicitAppId || readAppName() - if (runningInK8s && !explicitAppId) { + if (managed && !explicitAppId) { // The package name is not guaranteed unique -- a Next.js app is often just // "next" -- and where apps share a service account the binding check cannot // catch a wrong claim. Say which ID was assumed rather than failing. @@ -129,9 +120,10 @@ export function createWorld (options?: Partial): World { serviceUrl, appId, deploymentVersion: explicitVersion || 'local', - // In K8s ICC assigns the version, so a 'local' stamp means "not resolved yet" and - // must not be used to enqueue (see queue.ts). Standalone/local dev keeps 'local'. - requireResolvedVersion: runningInK8s, + // On a managed platform ICC assigns the version, so a 'local' stamp means "not + // resolved yet" and must not be used to enqueue (see queue.ts). Standalone + // keeps 'local'. + requireResolvedVersion: managed, } // No explicit version: start at 'local'. When running inside a watt runtime, the diff --git a/packages/world/src/lib/client.ts b/packages/world/src/lib/client.ts index 4fbfd13..30ab0cf 100644 --- a/packages/world/src/lib/client.ts +++ b/packages/world/src/lib/client.ts @@ -1,8 +1,8 @@ -import { existsSync, readFileSync } from 'node:fs' +import { readFileSync } from 'node:fs' import { Readable } from 'node:stream' import { Pool } from 'undici' import { encode } from 'cbor-x' -import { saPath } from './sa-path.ts' +import { saPath, isRunningInK8s } from './platform.ts' export interface ClientConfig { serviceUrl: string @@ -61,7 +61,7 @@ export class HttpClient { this.#pool = new Pool(config.serviceUrl) this.#baseUrl = `/api/v1/apps/${config.appId}` // No token file means single-tenant mode, where no auth is sent. - this.#inK8s = existsSync(saPath('token')) + this.#inK8s = isRunningInK8s() } #authHeaders (): Record { diff --git a/packages/world/src/lib/platform.ts b/packages/world/src/lib/platform.ts new file mode 100644 index 0000000..3834c40 --- /dev/null +++ b/packages/world/src/lib/platform.ts @@ -0,0 +1,30 @@ +import { readFileSync } from 'node:fs' + +// Location of the mounted Kubernetes service account, overridable for testing. +export function saPath (file: string): string { + const base = process.env.PLT_WORLD_SA_PATH || '/var/run/secrets/kubernetes.io/serviceaccount' + return `${base}/${file}` +} + +// A readable service account token is what marks the pod as running in K8s. +export function isRunningInK8s (): boolean { + try { + readFileSync(saPath('token')) + return true + } catch { + return false + } +} + +// ECS injects a task-scoped metadata endpoint into every container. Note that +// AWS_EXECUTION_ENV is not used: Lambda sets it too, with a different prefix. +export function isRunningInEcs (): boolean { + return Boolean(process.env.ECS_CONTAINER_METADATA_URI_V4 || process.env.ECS_CONTAINER_METADATA_URI) +} + +// A managed platform is one where ICC assigns the application and version and +// registers handlers at reachable URLs. Distinct from having an identity to +// authenticate with, which only K8s provides. +export function isManagedPlatform (): boolean { + return isRunningInK8s() || isRunningInEcs() +} diff --git a/packages/world/src/lib/sa-path.ts b/packages/world/src/lib/sa-path.ts deleted file mode 100644 index 3fb64f4..0000000 --- a/packages/world/src/lib/sa-path.ts +++ /dev/null @@ -1,5 +0,0 @@ -// Location of the mounted Kubernetes service account, overridable for testing. -export function saPath (file: string): string { - const base = process.env.PLT_WORLD_SA_PATH || '/var/run/secrets/kubernetes.io/serviceaccount' - return `${base}/${file}` -} diff --git a/packages/world/test/platform.test.ts b/packages/world/test/platform.test.ts new file mode 100644 index 0000000..146e86a --- /dev/null +++ b/packages/world/test/platform.test.ts @@ -0,0 +1,122 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { createServer } from 'node:http' +import { mkdirSync, writeFileSync, rmSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { isRunningInK8s, isRunningInEcs, isManagedPlatform } from '../src/lib/platform.ts' +import { createWorld } from '../src/index.ts' + +const ECS_VARS = ['ECS_CONTAINER_METADATA_URI_V4', 'ECS_CONTAINER_METADATA_URI'] + +function withEnv (vars: Record, fn: () => void | Promise) { + const saved: Record = {} + for (const [k, v] of Object.entries(vars)) { + saved[k] = process.env[k] + if (v === undefined) delete process.env[k] + else process.env[k] = v + } + const restore = () => { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k] + else process.env[k] = v + } + } + return Promise.resolve() + .then(fn) + .finally(restore) +} + +// Nothing is mounted or set in the test process, so this is the standalone case. +const NOT_ECS = Object.fromEntries(ECS_VARS.map(v => [v, undefined])) + +test('standalone is neither K8s nor ECS, and is not managed', async () => { + await withEnv({ PLT_WORLD_SA_PATH: join(tmpdir(), 'plt-world-absent'), ...NOT_ECS }, () => { + assert.equal(isRunningInK8s(), false) + assert.equal(isRunningInEcs(), false) + assert.equal(isManagedPlatform(), false) + }) +}) + +test('ECS is detected from the task metadata endpoint', async () => { + await withEnv({ + PLT_WORLD_SA_PATH: join(tmpdir(), 'plt-world-absent'), + ECS_CONTAINER_METADATA_URI_V4: 'http://169.254.170.2/v4/abc', + ECS_CONTAINER_METADATA_URI: undefined, + }, () => { + assert.equal(isRunningInEcs(), true) + assert.equal(isRunningInK8s(), false, 'ECS supplies no service account identity') + assert.equal(isManagedPlatform(), true) + }) +}) + +test('the older v3 metadata variable is also honoured', async () => { + await withEnv({ + PLT_WORLD_SA_PATH: join(tmpdir(), 'plt-world-absent'), + ECS_CONTAINER_METADATA_URI_V4: undefined, + ECS_CONTAINER_METADATA_URI: 'http://169.254.170.2/v3/abc', + }, () => { + assert.equal(isRunningInEcs(), true) + assert.equal(isManagedPlatform(), true) + }) +}) + +test('K8s is managed and additionally supplies an identity', async () => { + const saDir = join(tmpdir(), `plt-world-platform-k8s-${process.pid}`) + mkdirSync(saDir, { recursive: true }) + writeFileSync(join(saDir, 'token'), 'sa-token') + try { + await withEnv({ PLT_WORLD_SA_PATH: saDir, ...NOT_ECS }, () => { + assert.equal(isRunningInK8s(), true) + assert.equal(isManagedPlatform(), true) + }) + } finally { + rmSync(saDir, { recursive: true, force: true }) + } +}) + +test('on ECS an explicit application ID is required', async () => { + await withEnv({ + PLT_WORLD_SA_PATH: join(tmpdir(), 'plt-world-absent'), + ECS_CONTAINER_METADATA_URI_V4: 'http://169.254.170.2/v4/abc', + ECS_CONTAINER_METADATA_URI: undefined, + PLT_WORLD_SERVICE_URL: 'http://localhost:9999', + PLT_WORLD_APP_ID: undefined, + }, async () => { + assert.throws( + () => createWorld(), + { message: 'World application ID is required on a managed platform; set options.appId or PLT_WORLD_APP_ID' } + ) + const world = createWorld({ appId: 'explicit-app' }) + await world.close() + }) +}) + +test('on ECS start() does not self-register handlers', async () => { + let handlerCalled = false + const server = createServer((req, res) => { + if (req.url?.includes('/handlers')) handlerCalled = true + res.writeHead(201, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ registered: true })) + }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as { port: number } + + try { + await withEnv({ + PLT_WORLD_SA_PATH: join(tmpdir(), 'plt-world-absent'), + ECS_CONTAINER_METADATA_URI_V4: 'http://169.254.170.2/v4/abc', + ECS_CONTAINER_METADATA_URI: undefined, + PLT_WORLD_SERVICE_URL: `http://127.0.0.1:${port}`, + PLT_WORLD_APP_ID: 'ecs-app', + PORT: String(port), + }, async () => { + const world = createWorld() + await world.start() + await world.close() + assert.equal(handlerCalled, false, 'ICC registers handlers on a managed platform') + }) + } finally { + await new Promise(resolve => server.close(() => resolve())) + } +}) From 89696a3fe914e7c15a9a495c724fc2507a56dfe8 Mon Sep 17 00:00:00 2001 From: marcopiraccini Date: Sun, 19 Jul 2026 21:05:12 +0200 Subject: [PATCH 2/6] test: cover tenant isolation and unknown application rejection Two applications on one unauthenticated service, asserting a cross-tenant read returns only the caller's data and that an application ICC never registered fails closed with a 404 rather than reading as an empty tenant. --- .../workflow/test/ecs-multitenancy.test.ts | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 packages/workflow/test/ecs-multitenancy.test.ts diff --git a/packages/workflow/test/ecs-multitenancy.test.ts b/packages/workflow/test/ecs-multitenancy.test.ts new file mode 100644 index 0000000..83d75c8 --- /dev/null +++ b/packages/workflow/test/ecs-multitenancy.test.ts @@ -0,0 +1,101 @@ +import { describe, it, before, after } from 'node:test' +import assert from 'node:assert/strict' +import { randomBytes } from 'node:crypto' +import { join, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' +import Fastify from 'fastify' +import autoload from '@fastify/autoload' +import type { FastifyInstance } from 'fastify' + +const __dirname = dirname(fileURLToPath(import.meta.url)) + +// ECS supplies no service account token, so the service runs unauthenticated. +// Tenancy still applies, resolved from the application named in the URL. +describe('multi-tenancy without authentication (ECS)', () => { + let app: FastifyInstance + const appA = `ecs-a-${randomBytes(4).toString('hex')}` + const appB = `ecs-b-${randomBytes(4).toString('hex')}` + const ids: Record = {} + let savedEcs: string | undefined + let savedSaPath: string | undefined + + before(async () => { + process.env.DATABASE_URL = process.env.DATABASE_URL || 'postgresql://wf:wf@localhost:5434/workflow' + process.env.WF_ENABLE_POLLER = 'false' + savedEcs = process.env.ECS_CONTAINER_METADATA_URI_V4 + savedSaPath = process.env.PLT_WORLD_SA_PATH + process.env.ECS_CONTAINER_METADATA_URI_V4 = 'http://169.254.170.2/v4/test' + // Point service account discovery at a path that does not exist, so the + // run looks like ECS even if the suite executes inside a cluster. + process.env.PLT_WORLD_SA_PATH = join(__dirname, 'no-such-serviceaccount') + + app = Fastify({ logger: false }) + await app.register(autoload, { dir: join(__dirname, '..', 'plugins') }) + await app.ready() + + for (const appId of [appA, appB]) { + const res = await app.inject({ method: 'POST', url: '/api/v1/apps', payload: { appId } }) + assert.ok(res.statusCode === 201 || res.statusCode === 200, `registering ${appId}: ${res.statusCode}`) + const row = await app.pg.query('SELECT id FROM workflow_applications WHERE app_id = $1', [appId]) + ids[appId] = row.rows[0].id + await app.pg.query( + `INSERT INTO workflow_runs (id, application_id, workflow_name, deployment_id, status) + VALUES ($1, $2, $3, $4, $5)`, + [`run-${appId}`, ids[appId], `wf-${appId}`, 'd1', 'completed'] + ) + } + }) + + after(async () => { + for (const appId of [appA, appB]) { + if (ids[appId]) { + await app.pg.query('DELETE FROM workflow_runs WHERE application_id = $1', [ids[appId]]) + await app.pg.query('DELETE FROM workflow_applications WHERE id = $1', [ids[appId]]) + } + } + await app.close() + if (savedEcs === undefined) delete process.env.ECS_CONTAINER_METADATA_URI_V4 + else process.env.ECS_CONTAINER_METADATA_URI_V4 = savedEcs + if (savedSaPath === undefined) delete process.env.PLT_WORLD_SA_PATH + else process.env.PLT_WORLD_SA_PATH = savedSaPath + }) + + it('starts multi-tenant with no authentication configured', () => { + assert.equal(app.authConfig.multiTenant, true) + assert.equal(app.authConfig.k8s, undefined, 'ECS supplies no identity to verify') + assert.equal(app.authConfig.defaultAppId, undefined, 'no single implicit tenant') + }) + + it('scopes a request to the application named in the URL', async () => { + const res = await app.inject({ method: 'GET', url: `/api/v1/apps/${appA}/runs` }) + assert.equal(res.statusCode, 200) + const runs = res.json().data + assert.deepEqual(runs.map((r: { runId: string }) => r.runId), [`run-${appA}`]) + }) + + it('does not leak runs across tenants', async () => { + const res = await app.inject({ method: 'GET', url: `/api/v1/apps/${appB}/runs` }) + assert.equal(res.statusCode, 200) + const runs = res.json().data + assert.deepEqual(runs.map((r: { runId: string }) => r.runId), [`run-${appB}`]) + assert.ok(!runs.some((r: { runId: string }) => r.runId === `run-${appA}`)) + }) + + it('reads a run from its own tenant but not from another', async () => { + const own = await app.inject({ method: 'GET', url: `/api/v1/apps/${appA}/runs/run-${appA}` }) + assert.equal(own.statusCode, 200) + + const other = await app.inject({ method: 'GET', url: `/api/v1/apps/${appB}/runs/run-${appA}` }) + assert.equal(other.statusCode, 404, "another tenant's run must not be readable") + }) + + it('rejects an application ICC never registered', async () => { + const res = await app.inject({ method: 'GET', url: '/api/v1/apps/never-registered/runs' }) + // Must fail closed. Previously an unresolved application left appId at 0, + // so this returned 200 with an empty list and a typo looked like an empty + // tenant. The shared error handler in events.ts drops `code`, so match on + // the message instead. + assert.equal(res.statusCode, 404) + assert.match(res.json().message, /never-registered/) + }) +}) From dae397f6138d3982a23b4bb6d1545f467a2153dc Mon Sep 17 00:00:00 2001 From: marcopiraccini Date: Sun, 19 Jul 2026 21:29:09 +0200 Subject: [PATCH 3/6] docs: add ECS workflow support plan Records why tenancy is resolved from the URL, why AuthConfig.mode was removed, and the outstanding ICC and machinist work: the ECS provider is missing applyDeployment/applyService and stubs the skew-protection gateway methods, and ICC's registerWorkflowApp returns early without a service account token. --- ECS-WORKFLOW-SUPPORT.md | 228 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 ECS-WORKFLOW-SUPPORT.md diff --git a/ECS-WORKFLOW-SUPPORT.md b/ECS-WORKFLOW-SUPPORT.md new file mode 100644 index 0000000..0f384dd --- /dev/null +++ b/ECS-WORKFLOW-SUPPORT.md @@ -0,0 +1,228 @@ +# ECS Workflow Support + +Status: the Workflow Service and World client changes described in sections 5.1 to 5.3 are implemented. The ICC and machinist work in section 5.4 is not, and nothing runs end to end on ECS until it is. + +## 1. Problem Statement + +The Workflow Service and the World client both need to run on ECS, where there is no Kubernetes service account token. Authentication is explicitly out of scope for the first iteration (see section 7), but per-application isolation is not: several apps must be able to share one Workflow Service on ECS without reading each other's runs. + +Before this change an ECS deployment silently collapsed to a single tenant, and two further behaviours regressed in ways that were not obvious from the logs. + +## 2. What Breaks on ECS Without This Change + +Three independent questions are currently answered by one filesystem check: + +1. **Authentication** - do I have a platform identity to present and verify? +2. **Tenancy** - is one Workflow Service serving several applications? +3. **Management** - is ICC provisioning me, assigning appId and version, and registering my handlers? + +On Kubernetes all three answers are "yes", so `existsSync('/var/run/secrets/kubernetes.io/serviceaccount/token')` works as a proxy for all of them. On ECS the answers are no, yes, yes, and the proxy breaks. + +### Server side + +`packages/workflow/plugins/db.ts:23` uses that check to pick the operating mode. With no token present the service starts with `{ mode: 'none', defaultAppId }`, and `lib/auth/index.ts` then pins every request to that one application and sets `isAdmin = true`. Tenancy is gone, not just authentication. + +### Client side + +`packages/world/src/index.ts` calls `isRunningInK8s()` in three places, and only the first is really about Kubernetes: + +| Site | Gate | Result on ECS today | +|---|---|---| +| `index.ts:93` | require an explicit appId | falls back to `readAppName()`, so an app whose package is named `next` claims tenant `next` | +| `index.ts:32` | skip self-registering handlers, ICC does it with reachable URLs | self-registers `http://localhost:$PORT`, unreachable from another task | +| `index.ts:~108` | `requireResolvedVersion` | enqueues as version `local` instead of waiting for the assigned one | + +Only the first is a tenancy problem. The other two are deployment topology, and they break even though authentication is intentionally off. Handler registration is the most dangerous: a localhost URL means runs dispatch into a black hole rather than failing loudly. + +## 3. Findings from the Current Code + +These were verified against the codebase and shape the design. + +**Tenancy already has exactly one chokepoint.** Handlers read `request.appId` in 29 places versus `params.appId` in 2 (both of those are the admin `k8s-binding` routes, which operate on app records rather than querying app data). Every data-plane query is already scoped `WHERE application_id = $n`. Whatever the auth hook puts in `request.appId` is the isolation boundary. + +**The tenant is already named in the URL.** Every data-plane route is `/api/v1/apps/:appId/...`. Only `/api/v1/apps` and `/api/v1/versions/notify` are not, and both are registry/admin routes. The client hardcodes `/api/v1/apps/${appId}` as its base path, so the tenant is present on every request it makes. + +**The multi-tenant resolution already exists.** The admin branch of `lib/auth/index.ts` already resolves an application from the URL when the caller has no binding. The unauthenticated path needs the same lookup, not a new mechanism. + +**Authentication is not gated by what appears to gate it.** `AuthConfig.mode` is read in exactly one place, and the field that actually decides whether tokens are verified is `config.k8s` (`lib/auth/index.ts:46`). The enum and the behaviour have drifted apart. + +**There is a latent bug in that existing branch.** When the app is not found, `request.appId` stays at the decorator default of `0`, so queries scope to `application_id = 0` and return empty results. A typo'd tenant looks like an empty tenant rather than an error. + +**`db.ts:23` hardcodes the service account path** and ignores `PLT_WORLD_SA_PATH`, so the service cannot be driven into multi-tenant mode for testing. + +**There is no authentication anywhere else in the internal control plane.** machinist has zero request hooks in its entire source, no auth plugin, and ICC's machinist client sends only `Content-Type` on all six call sites. machinist is a `ClusterIP` service on 4444. It is also the most privileged component in the system, since it creates and deletes workloads and applies image pull secrets. The trust boundary sits at ICC's external edge; everything behind it is unauthenticated and mutually trusting. World's K8s-token auth is the one exception. + +## 4. Decisions + +**Detect the platform, do not configure it.** ECS injects `ECS_CONTAINER_METADATA_URI_V4` into every container, exactly as the kubelet mounts the service account token. This keeps the "no configuration flag needed" property the design document already claims for mode selection. + +**Separate identity from management.** `isRunningInK8s()` keeps one job, deciding whether an SA token is sent. A new `isManagedPlatform()` covers Kubernetes or ECS and drives appId, version, and handler registration. + +**Resolve tenancy from the URL on managed platforms.** The client already sends the appId on every request, so nothing new needs conveying; the server only has to stop discarding it. Applications are registered by ICC and unknown ones are rejected. Unmanaged deployments keep today's single implicit tenant. + +**No authentication on ECS in this iteration.** Consistent with the rest of the internal control plane (section 3). Revisited in section 7. + +## 5. Design + +### 5.1 Platform detection + +Add to `packages/world/src/lib/k8s.ts` (or rename it to `platform.ts`): + +```ts +export function isRunningInEcs (): boolean { + return Boolean(process.env.ECS_CONTAINER_METADATA_URI_V4 || process.env.ECS_CONTAINER_METADATA_URI) +} + +export function isManagedPlatform (): boolean { + return isRunningInK8s() || isRunningInEcs() +} +``` + +`AWS_EXECUTION_ENV` is deliberately not used as the primary signal, because Lambda sets it too with a different prefix. + +There is no override flag. `isManagedPlatform()` treats "on a managed platform" as "ICC is managing me", which is a proxy rather than a fact, and it can be wrong in one case: running the client in a pod or task with no ICC present. That deployment would skip handler registration and wait for a deployment version that never arrives. This is a known limitation rather than a regression, since `isRunningInK8s()` gates exactly those behaviours today and produces the same outcome. An override should be added when something actually needs it, designed against the real case, rather than shipped as a speculative knob. + +### 5.2 Server side (`packages/workflow`) + +The client already passes the application explicitly: `HttpClient` hardcodes `/api/v1/apps/${appId}` as its base path, so every request carries the tenant. Nothing new needs to be conveyed. The server simply has to stop discarding it. + +There is no ECS-side alternative to this. `ECS_CONTAINER_METADATA_URI_V4` is a link-local endpoint scoped to the calling task, so it describes a container to itself and cannot tell a server anything about its caller. Unlike Kubernetes, where the caller presents a token the server verifies against an authority, nothing identifying arrives inbound on ECS. Mapping the source IP to a task via `DescribeTasks` was considered and rejected: it requires cluster-wide describe permissions, it breaks behind a load balancer, NAT, or bridge-mode networking, and it is authentication in disguise. If verified identity on ECS is wanted, SigV4 with the task role is the correct form of it (section 7). + +**`AuthConfig.mode` goes away.** It is a four-value enum read in exactly one place (`lib/auth/index.ts:38`), and only `'none'` is distinguished there; `'k8s-token'`, `'api-key'`, and `'both'` all fall through to the same branch, where behaviour is actually decided by whether `config.k8s` is set. The enum is therefore one boolean's worth of information, and it is not even the field that does the work. Its unimplemented values are an active trap: `WF_AUTH_MODE=api-key` leaves `config.k8s` undefined, so the validator is null and every request is rejected as unauthenticated. + +Replace it with the two axes this document has been separating throughout, and let authentication be enabled exactly when the configuration needed to perform it is supplied: + +```ts +interface AuthConfig { + k8s?: K8sConfig // present: authenticate via TokenReview + multiTenant: boolean // resolve the tenant from the URL + defaultAppId?: number // used when multiTenant is false +} +``` + +That makes "authenticate but without the means to" unrepresentable, rather than a runtime surprise. `db.ts` computes both from platform detection: `k8s` when a service account token is present, `multiTenant` from `isManagedPlatform()`. + +The hook then reads as three plain branches: + +```ts +app.addHook('onRequest', async (request) => { + const url = request.url.split('?')[0] + if (PUBLIC_PATHS.has(url)) return + + if (validateK8s) { /* existing TokenReview path, unchanged */ return } + + request.isAdmin = true + if (!config.multiTenant) { + request.appId = config.defaultAppId || 0 + return + } + const match = url.match(/^\/api\/v1\/apps\/([^/]+)/) + if (!match) { + request.appId = config.defaultAppId || 0 // /api/v1/apps, /versions/notify + return + } + const result = await app.pg.query('SELECT id FROM workflow_applications WHERE app_id = $1', [match[1]]) + if (result.rows.length === 0) throw new NotFound(`unknown application ${match[1]}`) + request.appId = result.rows[0].id +}) +``` + +Applications are registered by ICC, never auto-created. ICC already calls `POST /api/v1/apps` when it discovers a workflow pod on Kubernetes, and on ECS it is the component deploying the task and injecting `PLT_WORLD_APP_ID`, so it knows the identifier at deploy time and can register it the same way. Auto-creation was considered and dropped: it would silently mint a tenant from a typo, and since a wrong appId then reads as an empty application rather than an error, it would turn a misconfiguration into a debugging exercise. + +Unmanaged deployments are untouched. With no Kubernetes token and no ECS metadata, the service keeps today's single-tenant behaviour, pinning `defaultAppId` from the startup upsert of `PLT_WORLD_APP_ID || 'default'`. That avoids the one case where URL resolution would regress local development, namely the client's `readAppName()` fallback disagreeing with the server's startup upsert, a mismatch currently hidden by the single-tenant collapse. + +Fix the latent bug from section 3: an unresolved application in the authenticated admin branch should raise a not-found rather than leaving `appId` at `0`. The unauthenticated path above does this from the start. + +Stop hardcoding the SA path at `db.ts:23`. Note that this check now selects tenancy as well as authentication, so it becomes `isManagedPlatform()` on the server too, with the token presence deciding only whether requests are authenticated. + +### 5.3 Client side (`packages/world`) + +Switch the three sites in section 2 from `isRunningInK8s()` to `isManagedPlatform()`. `#authHeaders()` in `lib/client.ts` keeps using `isRunningInK8s()`, since only Kubernetes supplies a token to send. + +With no ECS metadata present, behaviour is bit for bit unchanged on Kubernetes and on a laptop. + +### 5.4 ICC and machinist + +Both repositories need work before anything runs end to end on ECS. The specifics below were read off the current code rather than estimated. + +**ICC.** `registerWorkflowApp` (`services/control-plane/plugins/instances.js:45`) has four Kubernetes couplings, and the first makes the other three unreachable. + +Lines 48-49 return early when there is no service account token. On ECS there is none, so the function does nothing at all: no application registered, no handlers registered. This has to become "authenticate when a token exists" rather than "abort when it does not". + +Line 51 builds Authorization headers from that token. Unnecessary on ECS, where the service is unauthenticated. + +Lines 71-79 POST a `k8s-binding` of `{namespace, serviceAccount: 'default'}`, which is meaningless without a service account, and return on any non-201 status, so a failure there also blocks handler registration. Skip it on ECS. + +Line 86 builds the handler base URL as `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}`. ECS needs the equivalent from Cloud Map service discovery or an internal load balancer. This is the piece with genuine unknowns. + +Beyond that, `PLT_WORLD_APP_ID` and `PLT_WORLD_DEPLOYMENT_VERSION` must be injected into ECS tasks as they already are for Kubernetes workflow apps. + +**machinist.** The ECS provider (`services/main/plugins/providers/ecs.js`) implements reading and managing existing workloads: `getMachine`, `getMachines`, `setMachineLabels`, `getControllers`, `getController`, `updateControllerReplicas`, `deleteController`, `getServicesByLabels`, and `deleteService`. Three gaps remain, and only one of them is about skew protection. + +`applyDeployment` and `applyService` are absent entirely. They create the workload, so the deploy path cannot run on ECS at all. This is the basic deploy path, unrelated to skew protection. + +`listGateways`, `applyHTTPRoute`, `getHTTPRoute`, and `deleteHTTPRoute` exist as stubs that throw `MCHNST_NOT_IMPLEMENTED_BY_PROVIDER` (501), each labelled "Skew protection". Version-routed traffic therefore has no ECS implementation. That matters here because workflow apps deploy with `expirePolicy: 'workflow'` and depend on the version registry for draining, so workflow apps on ECS would have no version-safe drain even once registration works. + +`applySecret` also throws 501. Kubernetes image-pull secrets have no ECS analogue: private images are pulled via a task-definition `repositoryCredentials` pointing at Secrets Manager, set when the task is registered rather than as a standalone resource. + +### 5.5 Configuration Surface + +"Detect the platform, do not configure it" applies to behaviour, not to identity. Three distinct categories survive, and it is worth being precise about which is which. + +**Detected, never configured.** Whether to authenticate, whether to be multi-tenant, whether to self-register handlers, and whether to require a resolved deployment version. No mode flag and no platform switch is set by anyone, on any of the three environments. + +**Injected by ICC, not set by a user.** On managed platforms the app receives `PLT_WORLD_SERVICE_URL`, `PLT_WORLD_APP_ID`, and `PLT_WORLD_DEPLOYMENT_VERSION`, and ICC registers the application. The identifier cannot be detected: that is exactly the `readAppName()` collision this plan removes. Detection establishes which platform a process is on, not which application it is. + +**Operator configuration, set once per deployment.** `DATABASE_URL` for the service, plus `K8S_ADMIN_SERVICE_ACCOUNT` on Kubernetes. The latter has no default, and without it `adminServiceAccount` is undefined, so `isAdmin` never becomes true and ICC is not recognised as the control plane. It is not derivable, since the service cannot know which service account belongs to ICC. + +Standalone remains the near-zero-config case it is today: point `PLT_WORLD_SERVICE_URL` at the service and every other value defaults. + +## 6. Operating Modes, Revised + +This supersedes the two-mode model in `PLATFORMATIC-WORLD-DESIGN.md` section 6.1. Authentication and tenancy become independent axes rather than one toggle: + +| Environment | Authentication | Tenancy | +|---|---|---| +| Kubernetes with ICC | SA token via TokenReview | multi-tenant, binding-derived | +| ECS with ICC | none | multi-tenant, URL-derived, ICC-registered | +| Local development | none | single implicit tenant, unchanged | + +Local development is explicitly left alone. Multi-tenancy is a property of managed platforms, where ICC is present to register applications; without it the service keeps the single implicit application it has today. + +## 7. Security Posture + +On ECS, anything that can reach the port can name any tenant and is treated as admin. Tenancy is an isolation boundary, not a security boundary, and security groups plus private subnets do the actual protecting. + +This is a deliberate decision, recorded here rather than left implicit. It matches the existing posture of the internal control plane described in section 3, and is a strictly smaller concession than the unauthenticated machinist path that already exists. + +Two caveats. Consistency with existing practice is not the same as being correct, and the machinist posture deserves its own review, particularly because a VPC security group is a coarser instrument than an in-cluster service with NetworkPolicy available. Separately, dropping authentication on ECS removes the tenancy enforcement that TokenReview provides on Kubernetes, which is why URL-derived tenancy must be real isolation in the queries rather than advisory. Section 3 confirms it already is. + +The natural future direction is IAM. ECS tasks carry task roles, which are platform-issued, rotating, and verifiable, the same properties that make SA tokens the right choice on Kubernetes. Callers would sign with SigV4, the service would verify the caller identity, and bindings would key on role ARN instead of `namespace:serviceaccount`. That work should cover machinist and World together rather than World alone. + +## 8. Sequencing + +1. Server-side tenancy resolution. Done. +2. Client-side platform split and ECS detection. Done. +3. ICC and machinist. Not started. Largest and least certain, and in other repositories. + +Steps 1 and 2 give correct isolation but leave handlers registered at localhost and versions stamped `local`, so ECS is not functional until step 3. + +Note that steps 1 and 2 do not leave ECS neutral in the meantime. Unregistered applications are now rejected, and ICC does not register them on ECS (section 5.4), so an ECS deployment moves from silently collapsing into one tenant to returning not-found on every request. That is the intended direction, since failing closed beats silent cross-tenant reads, but it means step 3 is not optional follow-up: the two must land together before an ECS deployment is pointed at this. + +## 9. Testing + +Covered by `packages/workflow/test/ecs-multitenancy.test.ts`: two applications share one unauthenticated service and a cross-tenant read returns only the caller's data, a run is unreadable from another tenant, and an application that was never registered fails closed with a 404 rather than reading as an empty one. + +Covered by `packages/world/test/platform.test.ts`: ECS detected from both metadata variables, Kubernetes both managed and authenticated, standalone neither, an explicit appId required on a managed platform, and no handler self-registration there. + +Both suites simulate a platform purely through environment: `PLT_WORLD_SA_PATH` points service account discovery at a path that does or does not exist, and ECS is simulated by setting the metadata variable. No cluster or AWS account is needed. + +One note for whoever extends these. The shared error handler in `plugins/events.ts` rebuilds every error response as `{statusCode, error, message}` and drops `code`, so assertions must match on status and message rather than on the `WF_*` code, even though `lib/errors.ts` defines one for every error type. + +## 10. Open Questions + +Confirm `ECS_CONTAINER_METADATA_URI_V4` is present on the launch type and platform version actually in use. This was taken from the AWS contract, not observed on a live task. + +Decide how ICC addresses app tasks on ECS for handler registration. Cloud Map service discovery and an internal load balancer are the candidates; this is the largest unknown in section 5.4. + +Decide whether skew protection is in scope for ECS. The four gateway and HTTPRoute methods throw 501 today, so version-routed traffic has no ECS implementation. Workflow apps deploy with `expirePolicy: 'workflow'` and depend on the version registry for draining, so without it they would run on ECS but with no version-safe drain. That is a larger question than the rest of section 5.4 and may warrant its own plan. From b80361c4c76610f36c06f5f1857d2fdf4b8c9b14 Mon Sep 17 00:00:00 2001 From: marcopiraccini Date: Mon, 20 Jul 2026 14:13:09 +0200 Subject: [PATCH 4/6] fix(world): default the application ID rather than requiring it Reconciles this branch with the product decision that a user should set as little as possible: requiring an explicit ID would force every deployment ICC does not template (observe mode, desk) to hand-set an identity the deployer already knows. Falls back through PLT_APP_NAME to the package name, warning on a managed platform which ID was assumed. --- ECS-WORKFLOW-SUPPORT.md | 8 +++++--- packages/world/test/platform.test.ts | 26 +++++++++++++++++++------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/ECS-WORKFLOW-SUPPORT.md b/ECS-WORKFLOW-SUPPORT.md index 0f384dd..e59d56b 100644 --- a/ECS-WORKFLOW-SUPPORT.md +++ b/ECS-WORKFLOW-SUPPORT.md @@ -28,7 +28,7 @@ On Kubernetes all three answers are "yes", so `existsSync('/var/run/secrets/kube | Site | Gate | Result on ECS today | |---|---|---| -| `index.ts:93` | require an explicit appId | falls back to `readAppName()`, so an app whose package is named `next` claims tenant `next` | +| `index.ts:93` | warn when the appId was not configured | silent fallback to `readAppName()`, so an app whose package is named `next` claims tenant `next` with no signal | | `index.ts:32` | skip self-registering handlers, ICC does it with reachable URLs | self-registers `http://localhost:$PORT`, unreachable from another task | | `index.ts:~108` | `requireResolvedVersion` | enqueues as version `local` instead of waiting for the assigned one | @@ -139,6 +139,8 @@ Stop hardcoding the SA path at `db.ts:23`. Note that this check now selects tena Switch the three sites in section 2 from `isRunningInK8s()` to `isManagedPlatform()`. `#authHeaders()` in `lib/client.ts` keeps using `isRunningInK8s()`, since only Kubernetes supplies a token to send. +The application ID is a warning rather than a requirement. Requiring it would mean every deployment that ICC does not template, which is observe mode and desk alike, has to hand-set an identity the deployer already knows, and minimising what a user must configure is a product requirement. So the client resolves `options.appId`, then `PLT_WORLD_APP_ID`, then `PLT_APP_NAME` (the name watt-extra already resolves), and finally the package name. On a managed platform, falling through to the package name logs which ID was assumed, because that name is not guaranteed unique and a wrong claim is not always caught: where apps share a service account the binding check authorises any application bound to it. + With no ECS metadata present, behaviour is bit for bit unchanged on Kubernetes and on a laptop. ### 5.4 ICC and machinist @@ -171,7 +173,7 @@ Beyond that, `PLT_WORLD_APP_ID` and `PLT_WORLD_DEPLOYMENT_VERSION` must be injec **Detected, never configured.** Whether to authenticate, whether to be multi-tenant, whether to self-register handlers, and whether to require a resolved deployment version. No mode flag and no platform switch is set by anyone, on any of the three environments. -**Injected by ICC, not set by a user.** On managed platforms the app receives `PLT_WORLD_SERVICE_URL`, `PLT_WORLD_APP_ID`, and `PLT_WORLD_DEPLOYMENT_VERSION`, and ICC registers the application. The identifier cannot be detected: that is exactly the `readAppName()` collision this plan removes. Detection establishes which platform a process is on, not which application it is. +**Injected by ICC, not set by a user.** On managed platforms the app receives `PLT_WORLD_SERVICE_URL`, `PLT_WORLD_APP_ID`, and `PLT_WORLD_DEPLOYMENT_VERSION`, and ICC registers the application. The identifier cannot be reliably detected, so when it is absent the client falls back to the package name and says so (section 5.3). Detection establishes which platform a process is on, not which application it is. **Operator configuration, set once per deployment.** `DATABASE_URL` for the service, plus `K8S_ADMIN_SERVICE_ACCOUNT` on Kubernetes. The latter has no default, and without it `adminServiceAccount` is undefined, so `isAdmin` never becomes true and ICC is not recognised as the control plane. It is not derivable, since the service cannot know which service account belongs to ICC. @@ -213,7 +215,7 @@ Note that steps 1 and 2 do not leave ECS neutral in the meantime. Unregistered a Covered by `packages/workflow/test/ecs-multitenancy.test.ts`: two applications share one unauthenticated service and a cross-tenant read returns only the caller's data, a run is unreadable from another tenant, and an application that was never registered fails closed with a 404 rather than reading as an empty one. -Covered by `packages/world/test/platform.test.ts`: ECS detected from both metadata variables, Kubernetes both managed and authenticated, standalone neither, an explicit appId required on a managed platform, and no handler self-registration there. +Covered by `packages/world/test/platform.test.ts`: ECS detected from both metadata variables, Kubernetes both managed and authenticated, standalone neither, the appId falling back with a warning on a managed platform, and no handler self-registration there. Both suites simulate a platform purely through environment: `PLT_WORLD_SA_PATH` points service account discovery at a path that does or does not exist, and ECS is simulated by setting the metadata variable. No cluster or AWS account is needed. diff --git a/packages/world/test/platform.test.ts b/packages/world/test/platform.test.ts index 146e86a..c64a3e9 100644 --- a/packages/world/test/platform.test.ts +++ b/packages/world/test/platform.test.ts @@ -75,20 +75,32 @@ test('K8s is managed and additionally supplies an identity', async () => { } }) -test('on ECS an explicit application ID is required', async () => { +test('on ECS the application ID falls back, warning which one it assumed', async () => { await withEnv({ PLT_WORLD_SA_PATH: join(tmpdir(), 'plt-world-absent'), ECS_CONTAINER_METADATA_URI_V4: 'http://169.254.170.2/v4/abc', ECS_CONTAINER_METADATA_URI: undefined, PLT_WORLD_SERVICE_URL: 'http://localhost:9999', PLT_WORLD_APP_ID: undefined, + PLT_APP_NAME: undefined, }, async () => { - assert.throws( - () => createWorld(), - { message: 'World application ID is required on a managed platform; set options.appId or PLT_WORLD_APP_ID' } - ) - const world = createWorld({ appId: 'explicit-app' }) - await world.close() + const warnings: string[] = [] + const originalWarn = console.warn + console.warn = (msg: string) => { warnings.push(String(msg)) } + + try { + const fallback = createWorld() + await fallback.close() + assert.equal(warnings.length, 1) + assert.match(warnings[0], /no application ID configured/) + + warnings.length = 0 + const explicit = createWorld({ appId: 'explicit-app' }) + await explicit.close() + assert.deepEqual(warnings, []) + } finally { + console.warn = originalWarn + } }) }) From 2f1ef5994c7538ac5df461bb9a186ff5e3bd4c58 Mon Sep 17 00:00:00 2001 From: marcopiraccini Date: Wed, 12 Aug 2026 16:45:28 +0200 Subject: [PATCH 5/6] fixup Signed-off-by: marcopiraccini --- ECS-WORKFLOW-SUPPORT.md | 230 ------------------------ README-ECS.md | 162 +++++++++++++++++ packages/workflow/plugins/handlers.ts | 45 +++-- packages/workflow/test/handlers.test.ts | 44 +++++ 4 files changed, 239 insertions(+), 242 deletions(-) delete mode 100644 ECS-WORKFLOW-SUPPORT.md create mode 100644 README-ECS.md diff --git a/ECS-WORKFLOW-SUPPORT.md b/ECS-WORKFLOW-SUPPORT.md deleted file mode 100644 index e59d56b..0000000 --- a/ECS-WORKFLOW-SUPPORT.md +++ /dev/null @@ -1,230 +0,0 @@ -# ECS Workflow Support - -Status: the Workflow Service and World client changes described in sections 5.1 to 5.3 are implemented. The ICC and machinist work in section 5.4 is not, and nothing runs end to end on ECS until it is. - -## 1. Problem Statement - -The Workflow Service and the World client both need to run on ECS, where there is no Kubernetes service account token. Authentication is explicitly out of scope for the first iteration (see section 7), but per-application isolation is not: several apps must be able to share one Workflow Service on ECS without reading each other's runs. - -Before this change an ECS deployment silently collapsed to a single tenant, and two further behaviours regressed in ways that were not obvious from the logs. - -## 2. What Breaks on ECS Without This Change - -Three independent questions are currently answered by one filesystem check: - -1. **Authentication** - do I have a platform identity to present and verify? -2. **Tenancy** - is one Workflow Service serving several applications? -3. **Management** - is ICC provisioning me, assigning appId and version, and registering my handlers? - -On Kubernetes all three answers are "yes", so `existsSync('/var/run/secrets/kubernetes.io/serviceaccount/token')` works as a proxy for all of them. On ECS the answers are no, yes, yes, and the proxy breaks. - -### Server side - -`packages/workflow/plugins/db.ts:23` uses that check to pick the operating mode. With no token present the service starts with `{ mode: 'none', defaultAppId }`, and `lib/auth/index.ts` then pins every request to that one application and sets `isAdmin = true`. Tenancy is gone, not just authentication. - -### Client side - -`packages/world/src/index.ts` calls `isRunningInK8s()` in three places, and only the first is really about Kubernetes: - -| Site | Gate | Result on ECS today | -|---|---|---| -| `index.ts:93` | warn when the appId was not configured | silent fallback to `readAppName()`, so an app whose package is named `next` claims tenant `next` with no signal | -| `index.ts:32` | skip self-registering handlers, ICC does it with reachable URLs | self-registers `http://localhost:$PORT`, unreachable from another task | -| `index.ts:~108` | `requireResolvedVersion` | enqueues as version `local` instead of waiting for the assigned one | - -Only the first is a tenancy problem. The other two are deployment topology, and they break even though authentication is intentionally off. Handler registration is the most dangerous: a localhost URL means runs dispatch into a black hole rather than failing loudly. - -## 3. Findings from the Current Code - -These were verified against the codebase and shape the design. - -**Tenancy already has exactly one chokepoint.** Handlers read `request.appId` in 29 places versus `params.appId` in 2 (both of those are the admin `k8s-binding` routes, which operate on app records rather than querying app data). Every data-plane query is already scoped `WHERE application_id = $n`. Whatever the auth hook puts in `request.appId` is the isolation boundary. - -**The tenant is already named in the URL.** Every data-plane route is `/api/v1/apps/:appId/...`. Only `/api/v1/apps` and `/api/v1/versions/notify` are not, and both are registry/admin routes. The client hardcodes `/api/v1/apps/${appId}` as its base path, so the tenant is present on every request it makes. - -**The multi-tenant resolution already exists.** The admin branch of `lib/auth/index.ts` already resolves an application from the URL when the caller has no binding. The unauthenticated path needs the same lookup, not a new mechanism. - -**Authentication is not gated by what appears to gate it.** `AuthConfig.mode` is read in exactly one place, and the field that actually decides whether tokens are verified is `config.k8s` (`lib/auth/index.ts:46`). The enum and the behaviour have drifted apart. - -**There is a latent bug in that existing branch.** When the app is not found, `request.appId` stays at the decorator default of `0`, so queries scope to `application_id = 0` and return empty results. A typo'd tenant looks like an empty tenant rather than an error. - -**`db.ts:23` hardcodes the service account path** and ignores `PLT_WORLD_SA_PATH`, so the service cannot be driven into multi-tenant mode for testing. - -**There is no authentication anywhere else in the internal control plane.** machinist has zero request hooks in its entire source, no auth plugin, and ICC's machinist client sends only `Content-Type` on all six call sites. machinist is a `ClusterIP` service on 4444. It is also the most privileged component in the system, since it creates and deletes workloads and applies image pull secrets. The trust boundary sits at ICC's external edge; everything behind it is unauthenticated and mutually trusting. World's K8s-token auth is the one exception. - -## 4. Decisions - -**Detect the platform, do not configure it.** ECS injects `ECS_CONTAINER_METADATA_URI_V4` into every container, exactly as the kubelet mounts the service account token. This keeps the "no configuration flag needed" property the design document already claims for mode selection. - -**Separate identity from management.** `isRunningInK8s()` keeps one job, deciding whether an SA token is sent. A new `isManagedPlatform()` covers Kubernetes or ECS and drives appId, version, and handler registration. - -**Resolve tenancy from the URL on managed platforms.** The client already sends the appId on every request, so nothing new needs conveying; the server only has to stop discarding it. Applications are registered by ICC and unknown ones are rejected. Unmanaged deployments keep today's single implicit tenant. - -**No authentication on ECS in this iteration.** Consistent with the rest of the internal control plane (section 3). Revisited in section 7. - -## 5. Design - -### 5.1 Platform detection - -Add to `packages/world/src/lib/k8s.ts` (or rename it to `platform.ts`): - -```ts -export function isRunningInEcs (): boolean { - return Boolean(process.env.ECS_CONTAINER_METADATA_URI_V4 || process.env.ECS_CONTAINER_METADATA_URI) -} - -export function isManagedPlatform (): boolean { - return isRunningInK8s() || isRunningInEcs() -} -``` - -`AWS_EXECUTION_ENV` is deliberately not used as the primary signal, because Lambda sets it too with a different prefix. - -There is no override flag. `isManagedPlatform()` treats "on a managed platform" as "ICC is managing me", which is a proxy rather than a fact, and it can be wrong in one case: running the client in a pod or task with no ICC present. That deployment would skip handler registration and wait for a deployment version that never arrives. This is a known limitation rather than a regression, since `isRunningInK8s()` gates exactly those behaviours today and produces the same outcome. An override should be added when something actually needs it, designed against the real case, rather than shipped as a speculative knob. - -### 5.2 Server side (`packages/workflow`) - -The client already passes the application explicitly: `HttpClient` hardcodes `/api/v1/apps/${appId}` as its base path, so every request carries the tenant. Nothing new needs to be conveyed. The server simply has to stop discarding it. - -There is no ECS-side alternative to this. `ECS_CONTAINER_METADATA_URI_V4` is a link-local endpoint scoped to the calling task, so it describes a container to itself and cannot tell a server anything about its caller. Unlike Kubernetes, where the caller presents a token the server verifies against an authority, nothing identifying arrives inbound on ECS. Mapping the source IP to a task via `DescribeTasks` was considered and rejected: it requires cluster-wide describe permissions, it breaks behind a load balancer, NAT, or bridge-mode networking, and it is authentication in disguise. If verified identity on ECS is wanted, SigV4 with the task role is the correct form of it (section 7). - -**`AuthConfig.mode` goes away.** It is a four-value enum read in exactly one place (`lib/auth/index.ts:38`), and only `'none'` is distinguished there; `'k8s-token'`, `'api-key'`, and `'both'` all fall through to the same branch, where behaviour is actually decided by whether `config.k8s` is set. The enum is therefore one boolean's worth of information, and it is not even the field that does the work. Its unimplemented values are an active trap: `WF_AUTH_MODE=api-key` leaves `config.k8s` undefined, so the validator is null and every request is rejected as unauthenticated. - -Replace it with the two axes this document has been separating throughout, and let authentication be enabled exactly when the configuration needed to perform it is supplied: - -```ts -interface AuthConfig { - k8s?: K8sConfig // present: authenticate via TokenReview - multiTenant: boolean // resolve the tenant from the URL - defaultAppId?: number // used when multiTenant is false -} -``` - -That makes "authenticate but without the means to" unrepresentable, rather than a runtime surprise. `db.ts` computes both from platform detection: `k8s` when a service account token is present, `multiTenant` from `isManagedPlatform()`. - -The hook then reads as three plain branches: - -```ts -app.addHook('onRequest', async (request) => { - const url = request.url.split('?')[0] - if (PUBLIC_PATHS.has(url)) return - - if (validateK8s) { /* existing TokenReview path, unchanged */ return } - - request.isAdmin = true - if (!config.multiTenant) { - request.appId = config.defaultAppId || 0 - return - } - const match = url.match(/^\/api\/v1\/apps\/([^/]+)/) - if (!match) { - request.appId = config.defaultAppId || 0 // /api/v1/apps, /versions/notify - return - } - const result = await app.pg.query('SELECT id FROM workflow_applications WHERE app_id = $1', [match[1]]) - if (result.rows.length === 0) throw new NotFound(`unknown application ${match[1]}`) - request.appId = result.rows[0].id -}) -``` - -Applications are registered by ICC, never auto-created. ICC already calls `POST /api/v1/apps` when it discovers a workflow pod on Kubernetes, and on ECS it is the component deploying the task and injecting `PLT_WORLD_APP_ID`, so it knows the identifier at deploy time and can register it the same way. Auto-creation was considered and dropped: it would silently mint a tenant from a typo, and since a wrong appId then reads as an empty application rather than an error, it would turn a misconfiguration into a debugging exercise. - -Unmanaged deployments are untouched. With no Kubernetes token and no ECS metadata, the service keeps today's single-tenant behaviour, pinning `defaultAppId` from the startup upsert of `PLT_WORLD_APP_ID || 'default'`. That avoids the one case where URL resolution would regress local development, namely the client's `readAppName()` fallback disagreeing with the server's startup upsert, a mismatch currently hidden by the single-tenant collapse. - -Fix the latent bug from section 3: an unresolved application in the authenticated admin branch should raise a not-found rather than leaving `appId` at `0`. The unauthenticated path above does this from the start. - -Stop hardcoding the SA path at `db.ts:23`. Note that this check now selects tenancy as well as authentication, so it becomes `isManagedPlatform()` on the server too, with the token presence deciding only whether requests are authenticated. - -### 5.3 Client side (`packages/world`) - -Switch the three sites in section 2 from `isRunningInK8s()` to `isManagedPlatform()`. `#authHeaders()` in `lib/client.ts` keeps using `isRunningInK8s()`, since only Kubernetes supplies a token to send. - -The application ID is a warning rather than a requirement. Requiring it would mean every deployment that ICC does not template, which is observe mode and desk alike, has to hand-set an identity the deployer already knows, and minimising what a user must configure is a product requirement. So the client resolves `options.appId`, then `PLT_WORLD_APP_ID`, then `PLT_APP_NAME` (the name watt-extra already resolves), and finally the package name. On a managed platform, falling through to the package name logs which ID was assumed, because that name is not guaranteed unique and a wrong claim is not always caught: where apps share a service account the binding check authorises any application bound to it. - -With no ECS metadata present, behaviour is bit for bit unchanged on Kubernetes and on a laptop. - -### 5.4 ICC and machinist - -Both repositories need work before anything runs end to end on ECS. The specifics below were read off the current code rather than estimated. - -**ICC.** `registerWorkflowApp` (`services/control-plane/plugins/instances.js:45`) has four Kubernetes couplings, and the first makes the other three unreachable. - -Lines 48-49 return early when there is no service account token. On ECS there is none, so the function does nothing at all: no application registered, no handlers registered. This has to become "authenticate when a token exists" rather than "abort when it does not". - -Line 51 builds Authorization headers from that token. Unnecessary on ECS, where the service is unauthenticated. - -Lines 71-79 POST a `k8s-binding` of `{namespace, serviceAccount: 'default'}`, which is meaningless without a service account, and return on any non-201 status, so a failure there also blocks handler registration. Skip it on ECS. - -Line 86 builds the handler base URL as `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}`. ECS needs the equivalent from Cloud Map service discovery or an internal load balancer. This is the piece with genuine unknowns. - -Beyond that, `PLT_WORLD_APP_ID` and `PLT_WORLD_DEPLOYMENT_VERSION` must be injected into ECS tasks as they already are for Kubernetes workflow apps. - -**machinist.** The ECS provider (`services/main/plugins/providers/ecs.js`) implements reading and managing existing workloads: `getMachine`, `getMachines`, `setMachineLabels`, `getControllers`, `getController`, `updateControllerReplicas`, `deleteController`, `getServicesByLabels`, and `deleteService`. Three gaps remain, and only one of them is about skew protection. - -`applyDeployment` and `applyService` are absent entirely. They create the workload, so the deploy path cannot run on ECS at all. This is the basic deploy path, unrelated to skew protection. - -`listGateways`, `applyHTTPRoute`, `getHTTPRoute`, and `deleteHTTPRoute` exist as stubs that throw `MCHNST_NOT_IMPLEMENTED_BY_PROVIDER` (501), each labelled "Skew protection". Version-routed traffic therefore has no ECS implementation. That matters here because workflow apps deploy with `expirePolicy: 'workflow'` and depend on the version registry for draining, so workflow apps on ECS would have no version-safe drain even once registration works. - -`applySecret` also throws 501. Kubernetes image-pull secrets have no ECS analogue: private images are pulled via a task-definition `repositoryCredentials` pointing at Secrets Manager, set when the task is registered rather than as a standalone resource. - -### 5.5 Configuration Surface - -"Detect the platform, do not configure it" applies to behaviour, not to identity. Three distinct categories survive, and it is worth being precise about which is which. - -**Detected, never configured.** Whether to authenticate, whether to be multi-tenant, whether to self-register handlers, and whether to require a resolved deployment version. No mode flag and no platform switch is set by anyone, on any of the three environments. - -**Injected by ICC, not set by a user.** On managed platforms the app receives `PLT_WORLD_SERVICE_URL`, `PLT_WORLD_APP_ID`, and `PLT_WORLD_DEPLOYMENT_VERSION`, and ICC registers the application. The identifier cannot be reliably detected, so when it is absent the client falls back to the package name and says so (section 5.3). Detection establishes which platform a process is on, not which application it is. - -**Operator configuration, set once per deployment.** `DATABASE_URL` for the service, plus `K8S_ADMIN_SERVICE_ACCOUNT` on Kubernetes. The latter has no default, and without it `adminServiceAccount` is undefined, so `isAdmin` never becomes true and ICC is not recognised as the control plane. It is not derivable, since the service cannot know which service account belongs to ICC. - -Standalone remains the near-zero-config case it is today: point `PLT_WORLD_SERVICE_URL` at the service and every other value defaults. - -## 6. Operating Modes, Revised - -This supersedes the two-mode model in `PLATFORMATIC-WORLD-DESIGN.md` section 6.1. Authentication and tenancy become independent axes rather than one toggle: - -| Environment | Authentication | Tenancy | -|---|---|---| -| Kubernetes with ICC | SA token via TokenReview | multi-tenant, binding-derived | -| ECS with ICC | none | multi-tenant, URL-derived, ICC-registered | -| Local development | none | single implicit tenant, unchanged | - -Local development is explicitly left alone. Multi-tenancy is a property of managed platforms, where ICC is present to register applications; without it the service keeps the single implicit application it has today. - -## 7. Security Posture - -On ECS, anything that can reach the port can name any tenant and is treated as admin. Tenancy is an isolation boundary, not a security boundary, and security groups plus private subnets do the actual protecting. - -This is a deliberate decision, recorded here rather than left implicit. It matches the existing posture of the internal control plane described in section 3, and is a strictly smaller concession than the unauthenticated machinist path that already exists. - -Two caveats. Consistency with existing practice is not the same as being correct, and the machinist posture deserves its own review, particularly because a VPC security group is a coarser instrument than an in-cluster service with NetworkPolicy available. Separately, dropping authentication on ECS removes the tenancy enforcement that TokenReview provides on Kubernetes, which is why URL-derived tenancy must be real isolation in the queries rather than advisory. Section 3 confirms it already is. - -The natural future direction is IAM. ECS tasks carry task roles, which are platform-issued, rotating, and verifiable, the same properties that make SA tokens the right choice on Kubernetes. Callers would sign with SigV4, the service would verify the caller identity, and bindings would key on role ARN instead of `namespace:serviceaccount`. That work should cover machinist and World together rather than World alone. - -## 8. Sequencing - -1. Server-side tenancy resolution. Done. -2. Client-side platform split and ECS detection. Done. -3. ICC and machinist. Not started. Largest and least certain, and in other repositories. - -Steps 1 and 2 give correct isolation but leave handlers registered at localhost and versions stamped `local`, so ECS is not functional until step 3. - -Note that steps 1 and 2 do not leave ECS neutral in the meantime. Unregistered applications are now rejected, and ICC does not register them on ECS (section 5.4), so an ECS deployment moves from silently collapsing into one tenant to returning not-found on every request. That is the intended direction, since failing closed beats silent cross-tenant reads, but it means step 3 is not optional follow-up: the two must land together before an ECS deployment is pointed at this. - -## 9. Testing - -Covered by `packages/workflow/test/ecs-multitenancy.test.ts`: two applications share one unauthenticated service and a cross-tenant read returns only the caller's data, a run is unreadable from another tenant, and an application that was never registered fails closed with a 404 rather than reading as an empty one. - -Covered by `packages/world/test/platform.test.ts`: ECS detected from both metadata variables, Kubernetes both managed and authenticated, standalone neither, the appId falling back with a warning on a managed platform, and no handler self-registration there. - -Both suites simulate a platform purely through environment: `PLT_WORLD_SA_PATH` points service account discovery at a path that does or does not exist, and ECS is simulated by setting the metadata variable. No cluster or AWS account is needed. - -One note for whoever extends these. The shared error handler in `plugins/events.ts` rebuilds every error response as `{statusCode, error, message}` and drops `code`, so assertions must match on status and message rather than on the `WF_*` code, even though `lib/errors.ts` defines one for every error type. - -## 10. Open Questions - -Confirm `ECS_CONTAINER_METADATA_URI_V4` is present on the launch type and platform version actually in use. This was taken from the AWS contract, not observed on a live task. - -Decide how ICC addresses app tasks on ECS for handler registration. Cloud Map service discovery and an internal load balancer are the candidates; this is the largest unknown in section 5.4. - -Decide whether skew protection is in scope for ECS. The four gateway and HTTPRoute methods throw 501 today, so version-routed traffic has no ECS implementation. Workflow apps deploy with `expirePolicy: 'workflow'` and depend on the version registry for draining, so without it they would run on ECS but with no version-safe drain. That is a larger question than the rest of section 5.4 and may warrant its own plan. diff --git a/README-ECS.md b/README-ECS.md new file mode 100644 index 0000000..b64bb21 --- /dev/null +++ b/README-ECS.md @@ -0,0 +1,162 @@ +# Platformatic World on ECS + +Running workflow applications on AWS ECS instead of Kubernetes. + +Everything in the [main README](README.md) still applies: runs are pinned to the deployment version that started them, and the workflow service is still the thing that pins them. What changes is what the platform can tell the application about itself. + +## What is different on ECS + +One filesystem check answered three questions on Kubernetes -- whether there is an identity to authenticate with, whether the service is multi-tenant, and whether ICC provisions the application. ECS answers them differently, so they are separate: + +| | Kubernetes | ECS | +|---|---|---| +| Authentication | service account token, verified by the workflow service | **none** | +| Tenancy | several applications per workflow service | same | +| Provisioning | ICC assigns the application ID and version, and registers handlers | same | + +**There is no authentication on ECS in this release.** ECS has no service account token, so the workflow service accepts requests from anything that can reach it -- which is what the rest of the internal control plane already does, machinist included. Tenancy is unaffected: applications still cannot read each other's runs, because every data-plane route names its application in the URL and the workflow service rejects one it does not know. + +Treat the workflow service as an internal service. Put it in a security group that only application tasks and ICC can reach. + +## Prerequisites + +- An ECS cluster running Fargate tasks, with ICC and machinist deployed against it (`PLT_PROVIDER=ecs`). +- **A Cloud Map private DNS namespace**, and machinist configured with its id. This is not optional for workflow applications: it is how ICC learns the address to send workflow runs to. Without it, applications deploy and register, and no run ever reaches them. +- The workflow service itself, reachable from application tasks, with `DATABASE_URL` pointing at its PostgreSQL database. + +## Configuration + +### machinist + +``` +PLT_PROVIDER=ecs +PLT_ECS_REGION=us-east-1 +PLT_ECS_CLUSTER=my-cluster +PLT_ECS_SUBNETS=subnet-a,subnet-b +PLT_ECS_SECURITY_GROUPS=sg-app +PLT_ECS_EXECUTION_ROLE_ARN=arn:aws:iam::123456789012:role/exec +PLT_ECS_TASK_ROLE_ARN=arn:aws:iam::123456789012:role/task +PLT_ECS_CLOUD_MAP_NAMESPACE_ID=ns-abc123 # required for workflow apps +PLT_ECS_LOG_GROUP=/plt/apps # optional +PLT_ECS_LISTENER_ARN=arn:aws:...:listener/.. # only for skew protection +``` + +machinist's own IAM permissions are listed in its README. Cloud Map addressing +depends on `servicediscovery:ListServices`, `CreateService`, `GetNamespace`, +`GetService`, `DeleteService`, `ListInstances`, and `DeregisterInstance`. It +also requires `ecs:DescribeTaskDefinition`. + +machinist uses `GetService` to resolve the actual Cloud Map service name from +the registry ARN. It uses `DescribeTaskDefinition` to discover the application +port when an A-record registry and a service without a load balancer do not +expose one directly. + +### ICC + +``` +PLT_WORKFLOW_URL=http://workflow.plt.local:3042 +``` + +**This URL is handed to every workflow application**, as `PLT_WORLD_SERVICE_URL`. On Kubernetes the address ICC uses and the address a pod uses are the same, so this never came up; on ECS it has to be resolvable *from application tasks*, not only from ICC. A Cloud Map name in the same VPC is the straightforward choice. + +If it is set to something only ICC's own network can resolve, every workflow application will start and fail on a URL it cannot reach, and it will look like an application bug. + +### The application + +Nothing. ICC injects all three variables the World client needs: + +| Variable | Value | +|---|---| +| `PLT_WORLD_SERVICE_URL` | from ICC's `PLT_WORKFLOW_URL` | +| `PLT_WORLD_APP_ID` | the application name ICC registered | +| `PLT_WORLD_DEPLOYMENT_VERSION` | the version ICC assigned | + +Setting `PLT_WORLD_SERVICE_URL` yourself in the deploy environment overrides the injected one, which is the escape hatch for an external workflow service. + +`K8S_ADMIN_SERVICE_ACCOUNT` has no meaning on ECS and can be left unset. + +## What happens when you deploy + +1. ICC builds a provider-neutral workload spec and sends it to machinist. +2. machinist registers a Fargate task definition and creates one ECS service per version, tagged with the application name, the version, and `plt.dev/workflow`. It registers the service in Cloud Map, and -- if skew protection is on -- creates the version's target group and attaches it in the same call. +3. The task starts. The World client sees `ECS_CONTAINER_METADATA_URI_V4`, which ECS injects into every container, and knows it is on a managed platform: it does not self-register its handlers, and it waits for the assigned version rather than stamping runs `local`. +4. The task registers with ICC, which registers the application with the workflow service and then its queue handlers at the Cloud Map address: + + ``` + http://.:3042/.well-known/workflow/v1/flow + /.well-known/workflow/v1/step + /.well-known/workflow/v1/webhook + ``` + + The handler identity is stable for the version: + + ```text + / + ``` + + It does not identify an ECS task. A task replacement or a scale event leaves + the handler unchanged, while Cloud Map sends each request to a currently + healthy task belonging to that version's service. + +5. Runs dispatch to that address, pinned to the version that started them. Each + active or expiring version retains its own handler and therefore executes + using its own code. The workflow service removes that handler only when ICC + explicitly expires the version. + +## Checking it worked + +```sh +# The version's Cloud Map service exists +aws servicediscovery list-services \ + --filters Name=NAMESPACE_ID,Values=$PLT_ECS_CLOUD_MAP_NAMESPACE_ID \ + --query 'Services[].Name' + +# The ECS service carries the tags ICC identifies it by +aws ecs describe-services --cluster my-cluster --services my-app-v1 --include TAGS \ + --query 'services[0].tags' + +# The workflow service has handlers for the version, at a resolvable address +psql "$DATABASE_URL" -c \ + "select deployment_version, workflow_url from workflow_queue_handlers + order by last_heartbeat desc limit 5" +``` + +If the handler endpoints read `*.svc.cluster.local`, ICC did not receive an address from machinist -- check `PLT_ECS_CLOUD_MAP_NAMESPACE_ID`. + +## Known limitations + +**No authentication.** As above. The workflow service trusts its network on ECS. + +**Version labels are normalised.** ECS service names take letters, numbers, underscores and hyphens; a semantic version produces `my-app-v1.2.3`, which ECS rejects. machinist rewrites it and appends a short digest of the original, so `my-app-v1.2.3` becomes `my-app-v1-2-3-4f878d`. The version label itself is unchanged -- it is what runs are pinned to, and what `?dpl=` carries. + +**Skew protection is query-only.** An ALB cannot set a response cookie, so cookie pinning is unavailable on ECS. See the skew protection documentation for what that means for your applications. + +**One ECS service per version.** Target groups per load balancer is 100 and cannot be raised, which caps a single load balancer at roughly 33 applications with three live versions each. + +**Cleanup is configurable.** With `PLT_SKEW_AUTO_CLEANUP=true`, ICC asks +machinist to delete an expired version's ECS service and the resources created +with it, including its Cloud Map service, target group, and private-image pull +secret. With the setting disabled, ICC only scales the ECS service to zero. A +zero-task service has no Fargate compute charge, but retained resources still +consume ECS, Cloud Map, and especially target-group quotas. Changing the setting +affects future expirations; it does not retroactively delete versions that are +already expired. + +## Validation status + +The complete path has been exercised on a real Fargate cluster with query-based +skew protection: ICC deployed a workflow application, machinist created its +versioned ECS service and Cloud Map registration, ICC registered a +version-scoped handler, and a 12-step workflow completed through that handler. +This repository supplies the Workflow service and World client parts of that +path; the matching ICC and machinist ECS support must be deployed as well. + +## Troubleshooting + +**The application logs `no application ID configured; assuming "next" from package.json`.** `PLT_WORLD_APP_ID` did not reach the task. The application is claiming a tenant named after its package, which is very unlikely to be the one ICC registered. Check that the deploy went through ICC rather than being created directly in ECS. + +**Runs stay queued and never execute.** No handler is registered at a reachable address. Check the Cloud Map namespace is configured, then that the workflow service's security group allows it to reach application tasks on the application port. + +**The application never appears as a workflow application in ICC.** ICC identifies one by the `plt.dev/workflow` tag on the ECS service. A service created outside ICC will not have it; ECS also does not propagate tags to tasks unless the service asks it to, which machinist sets when it creates one. + +**`PLT_WORLD_SERVICE_URL environment variable is required` at startup.** ICC injects it only for applications it knows are workflow applications. Same cause as above. diff --git a/packages/workflow/plugins/handlers.ts b/packages/workflow/plugins/handlers.ts index 62310ff..99dd48b 100644 --- a/packages/workflow/plugins/handlers.ts +++ b/packages/workflow/plugins/handlers.ts @@ -22,18 +22,39 @@ async function handlersPlugin (app: FastifyInstance): Promise { throw new BadRequest('podId (or machineId), deploymentVersion, and endpoints are required') } - await app.pg.query( - `INSERT INTO workflow_queue_handlers (application_id, pod_id, deployment_version, workflow_url, step_url, webhook_url) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (application_id, pod_id) DO UPDATE SET - deployment_version = $3, - workflow_url = $4, - step_url = $5, - webhook_url = $6, - last_heartbeat = NOW()`, - [appId, machineId, body.deploymentVersion, - body.endpoints.workflow, body.endpoints.step, body.endpoints.webhook] - ) + const client = await app.pg.connect() + try { + await client.query('BEGIN') + await client.query( + `INSERT INTO workflow_queue_handlers (application_id, pod_id, deployment_version, workflow_url, step_url, webhook_url) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (application_id, pod_id) DO UPDATE SET + deployment_version = $3, + workflow_url = $4, + step_url = $5, + webhook_url = $6, + last_heartbeat = NOW()`, + [appId, machineId, body.deploymentVersion, + body.endpoints.workflow, body.endpoints.step, body.endpoints.webhook] + ) + + // ICC registers a version-scoped Service endpoint, not a pod/task + // endpoint. Keep exactly that logical handler when an installation moves + // from the old machine-scoped identity to the stable service identity. + // Other deployment versions remain independently routable, including + // while expiring; the expire endpoint is still what removes their row. + await client.query( + `DELETE FROM workflow_queue_handlers + WHERE application_id = $1 AND deployment_version = $2 AND pod_id != $3`, + [appId, body.deploymentVersion, machineId] + ) + await client.query('COMMIT') + } catch (err) { + await client.query('ROLLBACK') + throw err + } finally { + client.release() + } reply.code(201) return { registered: true } diff --git a/packages/workflow/test/handlers.test.ts b/packages/workflow/test/handlers.test.ts index fbef06c..e7e017a 100644 --- a/packages/workflow/test/handlers.test.ts +++ b/packages/workflow/test/handlers.test.ts @@ -64,6 +64,50 @@ describe('handlers', () => { assert.equal(result.rows[0].workflow_url, 'http://pod-1:3000/workflow-v2') }) + it('should replace obsolete machine-scoped rows for the same version only', async () => { + const register = async (podId: string, deploymentVersion: string, host: string) => { + return ctx.app.inject({ + method: 'POST', + url: `/api/v1/apps/${ctx.appId}/handlers`, + headers: { authorization: `Bearer ${ctx.apiKey}` }, + payload: { + podId, + deploymentVersion, + endpoints: { + workflow: `http://${host}/workflow`, + step: `http://${host}/step`, + webhook: `http://${host}/webhook`, + }, + }, + }) + } + + assert.equal((await register('old-task-id', 'v3.0.0', 'invalid-task-host')).statusCode, 201) + assert.equal((await register('platformatic/v3.0.0', 'v3.0.0', 'version-service')).statusCode, 201) + assert.equal((await register('platformatic/v4.0.0', 'v4.0.0', 'next-version-service')).statusCode, 201) + + const result = await ctx.app.pg.query( + `SELECT pod_id, deployment_version, workflow_url FROM workflow_queue_handlers + WHERE application_id = (SELECT id FROM workflow_applications WHERE app_id = $1) + AND deployment_version IN ('v3.0.0', 'v4.0.0') + ORDER BY deployment_version`, + [ctx.appId] + ) + + assert.deepEqual(result.rows, [ + { + pod_id: 'platformatic/v3.0.0', + deployment_version: 'v3.0.0', + workflow_url: 'http://version-service/workflow', + }, + { + pod_id: 'platformatic/v4.0.0', + deployment_version: 'v4.0.0', + workflow_url: 'http://next-version-service/workflow', + }, + ]) + }) + it('should reject handler without required fields', async () => { const res = await ctx.app.inject({ method: 'POST', From 2b3de1beacb78b5e841c184fbf44d1f9c83e268e Mon Sep 17 00:00:00 2001 From: marcopiraccini Date: Thu, 13 Aug 2026 14:40:26 +0200 Subject: [PATCH 6/6] fixups Signed-off-by: marcopiraccini --- README-ECS.md | 6 ++- README.md | 41 ++++++++-------- packages/workflow/README.md | 6 ++- packages/workflow/migrations/008.do.sql | 2 + packages/workflow/migrations/008.undo.sql | 2 + packages/workflow/plugins/handlers.ts | 57 ++++++++++++++++------- packages/workflow/test/handlers.test.ts | 53 ++++++++++++++++++--- packages/workflow/test/router.test.ts | 27 +++++++++++ packages/world/README.md | 8 ++-- 9 files changed, 152 insertions(+), 50 deletions(-) create mode 100644 packages/workflow/migrations/008.do.sql create mode 100644 packages/workflow/migrations/008.undo.sql diff --git a/README-ECS.md b/README-ECS.md index b64bb21..e059e4f 100644 --- a/README-ECS.md +++ b/README-ECS.md @@ -14,7 +14,7 @@ One filesystem check answered three questions on Kubernetes -- whether there is | Tenancy | several applications per workflow service | same | | Provisioning | ICC assigns the application ID and version, and registers handlers | same | -**There is no authentication on ECS in this release.** ECS has no service account token, so the workflow service accepts requests from anything that can reach it -- which is what the rest of the internal control plane already does, machinist included. Tenancy is unaffected: applications still cannot read each other's runs, because every data-plane route names its application in the URL and the workflow service rejects one it does not know. +**There is no authentication on ECS in this release.** ECS has no service account token, so the workflow service accepts requests from anything that can reach it -- which is what the rest of the internal control plane already does, machinist included. Data remains logically scoped by application in SQL, preventing accidental mixing, but this is not access isolation: a caller that can reach the service and knows another application ID can name it in the URL. Treat the workflow service as an internal service. Put it in a security group that only application tasks and ICC can reach. @@ -98,6 +98,10 @@ Setting `PLT_WORLD_SERVICE_URL` yourself in the deploy environment overrides the the handler unchanged, while Cloud Map sends each request to a currently healthy task belonging to that version's service. + ICC marks this registration as `serviceScoped`. The workflow service then + replaces obsolete machine-scoped rows for that version while leaving every + other active or expiring version independently routable. + 5. Runs dispatch to that address, pinned to the version that started them. Each active or expiring version retains its own handler and therefore executes using its own code. The workflow service removes that handler only when ICC diff --git a/README.md b/README.md index 84813a6..e67899e 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ # Platformatic World -Deployment-aware workflow orchestration for self-hosted Kubernetes environments. +Deployment-aware workflow orchestration for self-hosted Kubernetes and AWS ECS environments. Platformatic World solves the version-pinning problem for [Workflow DevKit](https://docs.platformatic.dev/): when new code deploys, in-flight workflow runs must continue executing on the code version that started them. The Vercel world handles this via Vercel's infrastructure. Platformatic World provides the same guarantees for self-hosted environments by routing queue messages through a central service that pins each run to its originating deployment version. +For ECS-specific deployment, discovery, and security details, see [Platformatic World on ECS](README-ECS.md). + ## Architecture ```mermaid @@ -35,32 +37,33 @@ graph LR ## Operating Modes -`@platformatic/world` and the Workflow Service run in **two distinct modes**. -The service auto-detects which one based on the presence of a Kubernetes -service-account token. Apps just point `PLT_WORLD_SERVICE_URL` at the -service URL and use the same SDK in both modes. +`@platformatic/world` and the Workflow Service run in three distinct modes. +They distinguish Kubernetes through its service-account token and ECS through +the task metadata endpoint injected into containers. Applications use the same +SDK in every mode. -| Aspect | Local mode (single-tenant) | Kubernetes mode (with ICC) | -|---|---|---| -| Triggered by | No K8s service-account token detected | K8s service-account token present at runtime | -| Authentication | None | K8s `TokenReview` per request | -| Apps | One implicit app (`default`) auto-provisioned | One app per K8s ServiceAccount binding, provisioned by ICC | -| Pod-to-handler registration | App calls `world.start()` on boot | ICC registers handlers via the admin API; `world.start()` is a no-op | -| Deployment version | Defaults to `local` (or `PLT_WORLD_DEPLOYMENT_VERSION`) | Auto-detected from the pod's `plt.dev/version` label | -| Admin API | Open (no auth) | Restricted to the configured admin ServiceAccount (e.g. `platformatic:icc`) | -| Run-pinning across deploys | Yes (every run records the version that started it) | Yes (same mechanism; ICC drives version lifecycle) | +| Aspect | Local mode | Kubernetes with ICC | ECS with ICC | +|---|---|---|---| +| Triggered by | No managed-platform signal | K8s service-account token | ECS task metadata endpoint | +| Authentication | None | K8s `TokenReview` per request | None; network-trusted | +| Apps | One implicit app (`default`) | Provisioned by ICC and bound to K8s ServiceAccounts | Provisioned by ICC and selected by URL | +| Handler registration | App calls `world.start()` | ICC registers the version's K8s Service | ICC registers the version's Cloud Map service | +| Deployment version | `local` or configured explicitly | Assigned by ICC | Assigned by ICC | +| Admin API | Open | Restricted to the configured admin ServiceAccount | Open inside the trusted network | +| Run-pinning across deploys | Yes | Yes; active and expiring versions retain their handlers | Yes; active and expiring versions retain their handlers | **Local mode** is what you use for development, CI, and the e2e tests in this repo. It runs the same code paths as production -- only the auth and handler-registration entry points differ. -**Kubernetes mode** is the production deployment under +**Managed modes** are production deployments under [ICC](https://github.com/platformatic/intelligent-command-center). ICC is -the control plane: it provisions apps, binds K8s ServiceAccounts to apps, -registers pod handler endpoints, and drives version lifecycle (drain / -expire). The service itself is identical between the two modes. +the control plane: it provisions applications, registers version-level service +endpoints, and drives version lifecycle (drain / expire). On Kubernetes it also +binds ServiceAccounts to applications for authentication. See the +[ECS guide](README-ECS.md) for the unauthenticated, network-trusted ECS model. -The diagram at the top shows the K8s-with-ICC mode. In local mode, replace +The diagram at the top shows a managed ICC mode. In local mode, replace the ICC box with nothing -- the service runs standalone against PostgreSQL and accepts unauthenticated traffic from apps on the same machine. diff --git a/packages/workflow/README.md b/packages/workflow/README.md index aa9d5a1..9f01ba3 100644 --- a/packages/workflow/README.md +++ b/packages/workflow/README.md @@ -1,6 +1,6 @@ # @platformatic/workflow -Workflow orchestration service for [Vercel Workflow DevKit](https://useworkflow.dev) on self-hosted Kubernetes. Manages all workflow state (runs, steps, events, hooks, streams) and routes queue messages to the correct deployment version. +Workflow orchestration service for [Vercel Workflow DevKit](https://useworkflow.dev) on self-hosted Kubernetes and AWS ECS. Manages all workflow state (runs, steps, events, hooks, streams) and routes queue messages to the correct deployment version. ## Quick Start @@ -33,6 +33,8 @@ Options: **Multi-tenant** (Kubernetes) — K8s service account token present. All requests authenticated via K8s TokenReview API. Per-application isolation enforced at the SQL level. +**Multi-tenant** (ECS) — ECS task metadata endpoint present. Applications are scoped in SQL, but requests are unauthenticated and callers select the application in the URL. Keep the service reachable only from trusted security groups. See the repository's [ECS guide](../../README-ECS.md). + ## API All app-scoped endpoints are prefixed with `/api/v1/apps/:appId`. @@ -63,7 +65,7 @@ All app-scoped endpoints are prefixed with `/api/v1/apps/:appId`. | Method | Path | Description | |---|---|---| | `POST` | `/queue` | Enqueue a message (accepts `application/json` or `application/cbor`) | -| `POST` | `/handlers` | Register queue handler endpoints | +| `POST` | `/handlers` | Register queue handler endpoints (`serviceScoped: true` for an ICC-managed version Service) | | `PUT` | `/runs/:runId/streams/:name` | Write stream chunks | | `GET` | `/runs/:runId/streams` | List stream names | | `GET` | `/runs/:runId/streams/:name/chunks` | Paginated stream chunks (`?limit`, `?cursor`) | diff --git a/packages/workflow/migrations/008.do.sql b/packages/workflow/migrations/008.do.sql new file mode 100644 index 0000000..a18e41f --- /dev/null +++ b/packages/workflow/migrations/008.do.sql @@ -0,0 +1,2 @@ +ALTER TABLE workflow_queue_handlers + ADD COLUMN service_scoped BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/packages/workflow/migrations/008.undo.sql b/packages/workflow/migrations/008.undo.sql new file mode 100644 index 0000000..af826a5 --- /dev/null +++ b/packages/workflow/migrations/008.undo.sql @@ -0,0 +1,2 @@ +ALTER TABLE workflow_queue_handlers + DROP COLUMN service_scoped; diff --git a/packages/workflow/plugins/handlers.ts b/packages/workflow/plugins/handlers.ts index 99dd48b..5158f06 100644 --- a/packages/workflow/plugins/handlers.ts +++ b/packages/workflow/plugins/handlers.ts @@ -3,13 +3,14 @@ import type { FastifyInstance } from 'fastify' import { BadRequest } from '../lib/errors.ts' async function handlersPlugin (app: FastifyInstance): Promise { - // Register pod queue handler endpoints + // Register machine- or version-service-scoped queue handler endpoints app.post('/api/v1/apps/:appId/handlers', async (request, reply) => { const appId = request.appId const body = request.body as { podId?: string machineId?: string deploymentVersion: string + serviceScoped?: boolean endpoints: { workflow: string step: string @@ -25,29 +26,49 @@ async function handlersPlugin (app: FastifyInstance): Promise { const client = await app.pg.connect() try { await client.query('BEGIN') + + // Registrations for one application are serialized so an old + // machine-scoped caller cannot race a service-scoped registration and + // recreate a row after the latter has consolidated the version. + await client.query( + 'SELECT id FROM workflow_applications WHERE id = $1 FOR UPDATE', + [appId] + ) + await client.query( - `INSERT INTO workflow_queue_handlers (application_id, pod_id, deployment_version, workflow_url, step_url, webhook_url) - VALUES ($1, $2, $3, $4, $5, $6) + `INSERT INTO workflow_queue_handlers + (application_id, pod_id, deployment_version, workflow_url, step_url, webhook_url, service_scoped) + SELECT $1::integer, $2::varchar, $3::varchar, $4::varchar, + $5::varchar, $6::varchar, $7::boolean + WHERE $7::boolean OR NOT EXISTS ( + SELECT 1 FROM workflow_queue_handlers + WHERE application_id = $1::integer + AND deployment_version = $3::varchar + AND service_scoped + ) ON CONFLICT (application_id, pod_id) DO UPDATE SET - deployment_version = $3, - workflow_url = $4, - step_url = $5, - webhook_url = $6, + deployment_version = $3::varchar, + workflow_url = $4::varchar, + step_url = $5::varchar, + webhook_url = $6::varchar, + service_scoped = $7::boolean, last_heartbeat = NOW()`, [appId, machineId, body.deploymentVersion, - body.endpoints.workflow, body.endpoints.step, body.endpoints.webhook] + body.endpoints.workflow, body.endpoints.step, body.endpoints.webhook, + body.serviceScoped === true] ) - // ICC registers a version-scoped Service endpoint, not a pod/task - // endpoint. Keep exactly that logical handler when an installation moves - // from the old machine-scoped identity to the stable service identity. - // Other deployment versions remain independently routable, including - // while expiring; the expire endpoint is still what removes their row. - await client.query( - `DELETE FROM workflow_queue_handlers - WHERE application_id = $1 AND deployment_version = $2 AND pod_id != $3`, - [appId, body.deploymentVersion, machineId] - ) + if (body.serviceScoped === true) { + // ICC registers one version-scoped Service endpoint, not a pod/task + // endpoint. Replace obsolete machine-scoped rows for this version only. + // Active and expiring versions retain their independent handlers; the + // expire endpoint is what removes a version's final row. + await client.query( + `DELETE FROM workflow_queue_handlers + WHERE application_id = $1 AND deployment_version = $2 AND pod_id != $3`, + [appId, body.deploymentVersion, machineId] + ) + } await client.query('COMMIT') } catch (err) { await client.query('ROLLBACK') diff --git a/packages/workflow/test/handlers.test.ts b/packages/workflow/test/handlers.test.ts index e7e017a..04b87c6 100644 --- a/packages/workflow/test/handlers.test.ts +++ b/packages/workflow/test/handlers.test.ts @@ -30,7 +30,7 @@ describe('handlers', () => { }, }) - assert.equal(res.statusCode, 201) + assert.equal(res.statusCode, 201, res.body) assert.deepEqual(JSON.parse(res.body), { registered: true }) }) @@ -64,8 +64,42 @@ describe('handlers', () => { assert.equal(result.rows[0].workflow_url, 'http://pod-1:3000/workflow-v2') }) - it('should replace obsolete machine-scoped rows for the same version only', async () => { - const register = async (podId: string, deploymentVersion: string, host: string) => { + it('should preserve multiple machine-scoped handlers for the same version', async () => { + const register = async (podId: string, host: string) => ctx.app.inject({ + method: 'POST', + url: `/api/v1/apps/${ctx.appId}/handlers`, + headers: { authorization: `Bearer ${ctx.apiKey}` }, + payload: { + podId, + deploymentVersion: 'v-multi', + endpoints: { + workflow: `http://${host}/workflow`, + step: `http://${host}/step`, + webhook: `http://${host}/webhook`, + }, + }, + }) + + assert.equal((await register('pod-a', 'pod-a')).statusCode, 201) + assert.equal((await register('pod-b', 'pod-b')).statusCode, 201) + + const result = await ctx.app.pg.query( + `SELECT pod_id FROM workflow_queue_handlers + WHERE application_id = (SELECT id FROM workflow_applications WHERE app_id = $1) + AND deployment_version = 'v-multi' + ORDER BY pod_id`, + [ctx.appId] + ) + assert.deepEqual(result.rows, [{ pod_id: 'pod-a' }, { pod_id: 'pod-b' }]) + }) + + it('should keep one service-scoped handler per active or expiring version', async () => { + const register = async ( + podId: string, + deploymentVersion: string, + host: string, + serviceScoped = false + ) => { return ctx.app.inject({ method: 'POST', url: `/api/v1/apps/${ctx.appId}/handlers`, @@ -73,6 +107,7 @@ describe('handlers', () => { payload: { podId, deploymentVersion, + serviceScoped, endpoints: { workflow: `http://${host}/workflow`, step: `http://${host}/step`, @@ -83,11 +118,15 @@ describe('handlers', () => { } assert.equal((await register('old-task-id', 'v3.0.0', 'invalid-task-host')).statusCode, 201) - assert.equal((await register('platformatic/v3.0.0', 'v3.0.0', 'version-service')).statusCode, 201) - assert.equal((await register('platformatic/v4.0.0', 'v4.0.0', 'next-version-service')).statusCode, 201) + assert.equal((await register('platformatic/v3.0.0', 'v3.0.0', 'version-service', true)).statusCode, 201) + assert.equal((await register('platformatic/v4.0.0', 'v4.0.0', 'next-version-service', true)).statusCode, 201) + + // A caller from before service-scoped registrations were introduced must + // not displace or compete with the stable version Service. + assert.equal((await register('late-old-task-id', 'v3.0.0', 'late-invalid-host')).statusCode, 201) const result = await ctx.app.pg.query( - `SELECT pod_id, deployment_version, workflow_url FROM workflow_queue_handlers + `SELECT pod_id, deployment_version, workflow_url, service_scoped FROM workflow_queue_handlers WHERE application_id = (SELECT id FROM workflow_applications WHERE app_id = $1) AND deployment_version IN ('v3.0.0', 'v4.0.0') ORDER BY deployment_version`, @@ -99,11 +138,13 @@ describe('handlers', () => { pod_id: 'platformatic/v3.0.0', deployment_version: 'v3.0.0', workflow_url: 'http://version-service/workflow', + service_scoped: true, }, { pod_id: 'platformatic/v4.0.0', deployment_version: 'v4.0.0', workflow_url: 'http://next-version-service/workflow', + service_scoped: true, }, ]) }) diff --git a/packages/workflow/test/router.test.ts b/packages/workflow/test/router.test.ts index ac9b73c..f19e734 100644 --- a/packages/workflow/test/router.test.ts +++ b/packages/workflow/test/router.test.ts @@ -4,6 +4,33 @@ import { routeMessage } from '../queue/router.ts' import type pg from 'pg' describe('queue router', () => { + it('routes both active and expiring versions, but not expired versions', async () => { + for (const status of ['active', 'expiring']) { + const pool = { + query: async (sql: string) => { + if (sql.includes('workflow_deployment_versions')) return { rows: [{ status }] } + return { + rows: [{ + workflow_url: `http://${status}/flow`, + step_url: `http://${status}/step`, + webhook_url: `http://${status}/webhook`, + }], + } + }, + } as unknown as pg.Pool + + assert.deepEqual( + await routeMessage(pool, 1, `v-${status}`, '__wkf_workflow_test'), + { url: `http://${status}/flow` } + ) + } + + const expiredPool = { + query: async () => ({ rows: [{ status: 'expired' }] }), + } as unknown as pg.Pool + assert.equal(await routeMessage(expiredPool, 1, 'v-expired', '__wkf_workflow_test'), null) + }) + it('deduplicates the selected endpoint URLs before random selection', async () => { const pool = { query: async (sql: string) => { diff --git a/packages/world/README.md b/packages/world/README.md index ea87283..3c0528b 100644 --- a/packages/world/README.md +++ b/packages/world/README.md @@ -1,6 +1,6 @@ # @platformatic/world -Drop-in [World](https://useworkflow.dev/docs/deploying) implementation for [Vercel Workflow DevKit](https://useworkflow.dev) on self-hosted Kubernetes. Routes workflow state through a central [Workflow Service](https://github.com/platformatic/platformatic-world/tree/main/packages/workflow) that pins each run to the deployment version that started it. +Drop-in [World](https://useworkflow.dev/docs/deploying) implementation for [Vercel Workflow DevKit](https://useworkflow.dev) on self-hosted Kubernetes and AWS ECS. Routes workflow state through a central [Workflow Service](https://github.com/platformatic/platformatic-world/tree/main/packages/workflow) that pins each run to the deployment version that started it. ## Installation @@ -34,7 +34,7 @@ export async function register() { For other frameworks, call `world.start()` during your server's startup. -In Kubernetes with [ICC](https://icc.platformatic.dev/), handler registration is automatic — `world.start()` is a no-op. +On Kubernetes or ECS with [ICC](https://icc.platformatic.dev/), handler registration is automatic — `world.start()` is a no-op. See the repository's [ECS guide](../../README-ECS.md) for ECS configuration and its network-trusted security model. ### Direct usage @@ -61,9 +61,9 @@ High-level factory with automatic config resolution from environment variables. |---|---|---|---| | `serviceUrl` | `PLT_WORLD_SERVICE_URL` | *required* | Workflow Service URL | | `appId` | `PLT_WORLD_APP_ID` | `package.json` name | Application identifier | -| `deploymentVersion` | `PLT_WORLD_DEPLOYMENT_VERSION` | K8s label or `'local'` | Deployment version | +| `deploymentVersion` | `PLT_WORLD_DEPLOYMENT_VERSION` | `'local'` | Deployment version assigned by ICC on managed platforms | -In Kubernetes, the deployment version is auto-detected from the pod's `plt.dev/version` label via the K8s API. +On Kubernetes and ECS, ICC supplies the deployment version through the application environment/runtime context. ### `createPlatformaticWorld(config)`