diff --git a/integration-tests/cli/test/sync.test.ts b/integration-tests/cli/test/sync.test.ts index 13e407b5a..38179a93e 100644 --- a/integration-tests/cli/test/sync.test.ts +++ b/integration-tests/cli/test/sync.test.ts @@ -30,7 +30,10 @@ const initWorkspace = (t: any) => { }; }; -const gen = (name = 'patients', workflows = ['trigger-job(body="fn()")']) => { +const gen = ( + name = 'patients', + workflows = ['trigger-job(expression="fn()")'] +) => { // generate a project const project = generateProject(name, workflows, { openfnUuid: true, @@ -44,7 +47,7 @@ test('fetch a new project', async (t) => { const { workspace, read } = initWorkspace(t); const project = gen(); - await run( + const { stdout } = await run( `openfn project fetch \ --workspace ${workspace} \ --endpoint ${endpoint} \ @@ -239,7 +242,7 @@ test('pull an update to project', async (t) => { test('checkout by alias', async (t) => { const { workspace, read } = initWorkspace(t); const main = gen(); - const staging = gen('patients-staging', ['trigger-job(body="fn(x)")']); + const staging = gen('patients-staging', ['trigger-job(expression="fn(x)")']); await run( `openfn project fetch \ diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 451eefcab..ad44a9741 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -4,7 +4,7 @@ ### Minor Changes -- 8b9f402: fetch: allow state files to be writtem to JSON with --format +- 8b9f402: fetch: allow state files to be written to JSON with --format ### Patch Changes diff --git a/packages/cli/src/projects/deploy.ts b/packages/cli/src/projects/deploy.ts index 42bc34e8f..9c47afbf1 100644 --- a/packages/cli/src/projects/deploy.ts +++ b/packages/cli/src/projects/deploy.ts @@ -264,7 +264,7 @@ Pass --force to override this error and deploy anyway.`); updateForkedFrom(finalProject); const configData = finalProject.generateConfig(); await writeFile( - path.resolve(options.workspace, configData.path), + path.resolve(options.workspace!, configData.path), configData.content ); diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index d1d618c79..8ca84ab40 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -335,7 +335,13 @@ To ignore this error and override the local file, pass --force (-f) // TODO canMergeInto needs to return a reason if (!skipVersionCheck && !remoteProject.canMergeInto(localProject!)) { // TODO allow rename - throw new Error('Error! An incompatible project exists at this location'); + const e = new Error( + `Error! An incompatible project exists at this location.` + ); + + delete e.stack; + + throw e; } } } diff --git a/packages/project/src/Workflow.ts b/packages/project/src/Workflow.ts index 07b976a0e..b96e31783 100644 --- a/packages/project/src/Workflow.ts +++ b/packages/project/src/Workflow.ts @@ -113,7 +113,14 @@ class Workflow { // Get properties on any step or edge by id or uuid get(id: string): WithMeta { - const item = this.index.edges[id] || this.index.steps[id]; + // first check if we're passed a UUID - in which case we map it to an id + if (id in this.index.id) { + id = this.index.id[id]; + } + + // now look up the item proper + let item = this.index.edges[id] || this.index.steps[id]; + if (!item) { throw new Error(`step/edge with id "${id}" does not exist in workflow`); } diff --git a/packages/project/src/gen/generator.ts b/packages/project/src/gen/generator.ts index cf41384a2..0a3579fa3 100644 --- a/packages/project/src/gen/generator.ts +++ b/packages/project/src/gen/generator.ts @@ -111,7 +111,7 @@ const initOperations = (options: any = {}) => { n1.next ??= {}; - n1.next[n2.name] = e; + n1.next[n2.id ?? slugify(n2.name)] = e; return [n1, n2]; }, diff --git a/packages/project/src/gen/workflow.ohm b/packages/project/src/gen/workflow.ohm index fe2ee2502..a8ab5efa0 100644 --- a/packages/project/src/gen/workflow.ohm +++ b/packages/project/src/gen/workflow.ohm @@ -29,7 +29,7 @@ Workflow { prop = (alnum | "-" | "_")+ "=" propValue - propValue = quoted_prop | bool | int | alnum+ + propValue = quoted_prop | bool | int | alnum+ // TODO we only parse numbers as positive ints right now // fine for tests diff --git a/packages/project/src/serialize/to-app-state.ts b/packages/project/src/serialize/to-app-state.ts index 2eb4cb8f4..b6504de58 100644 --- a/packages/project/src/serialize/to-app-state.ts +++ b/packages/project/src/serialize/to-app-state.ts @@ -57,7 +57,7 @@ export default function ( return state; } -const mapWorkflow = (workflow: Workflow) => { +export const mapWorkflow = (workflow: Workflow) => { if (workflow instanceof Workflow) { // @ts-ignore workflow = workflow.toJSON(); @@ -96,10 +96,10 @@ const mapWorkflow = (workflow: Workflow) => { let isTrigger = false; let node: Provisioner.Job | Provisioner.Trigger; - if (s.type && !s.expression) { + if (!s.expression && !s.adaptor) { isTrigger = true; node = { - type: s.type, + type: s.type ?? 'webhook', // this is mostly for tests ...renameKeys(s.openfn, { uuid: 'id' }), } as Provisioner.Trigger; wfState.triggers[node.type] = node; @@ -147,6 +147,11 @@ const mapWorkflow = (workflow: Workflow) => { e.source_job_id = node.id; } + if (rules.label) { + // TODO needs unit test + e.condition_label = rules.label; + } + if (rules.condition) { if (typeof rules.condition === 'boolean') { e.condition_type = rules.condition ? 'always' : 'never'; diff --git a/packages/project/src/util/version.ts b/packages/project/src/util/version.ts index ee73a560d..97b101bd0 100644 --- a/packages/project/src/util/version.ts +++ b/packages/project/src/util/version.ts @@ -1,73 +1,126 @@ -import { ConditionalStepEdge, Job, Trigger, Workflow } from '@openfn/lexicon'; +import * as l from '@openfn/lexicon'; import crypto from 'node:crypto'; +import { get } from 'lodash-es'; +import { mapWorkflow } from '../serialize/to-app-state'; +import Workflow from '../Workflow'; const SHORT_HASH_LENGTH = 12; -export const project = () => {}; - function isDefined(v: any) { return v !== undefined && v !== null; } -export const generateHash = (workflow: Workflow, source = 'cli') => { +export const parse = (version: string) => { + const [source, hash] = version.split(':'); + return { source, hash }; +}; + +export const generateHash = ( + wfJson: l.Workflow, + { source = 'cli', sha = true } = {} +) => { + const workflow = new Workflow(wfJson); + const parts: string[] = []; + // convert the workflow into a v1 state object + // this means we can match keys with lightning + // and everything gets cleaner + const wfState = mapWorkflow(workflow); + // These are the keys we hash against - const wfKeys = ['name', 'credentials'].sort() as Array; + const wfKeys = ['name', 'positions'].sort(); + + // These keys are manually sorted to match lightning equivalents const stepKeys = [ 'name', - 'adaptors', - 'adaptor', // there's both adaptor & adaptors key in steps somehow - 'expression', - 'configuration', // assumes a string credential id - 'expression', - - // TODO need to model trigger types in this, which I think are currently ignored - ].sort() as Array; + 'adaptor', + 'keychain_credential_id', + 'project_credential_id', + 'body', + ].sort(); + + const triggerKeys = ['type', 'cron_expression', 'enabled'].sort(); + const edgeKeys = [ - 'condition', + 'name', // generated 'label', - 'disabled', // This feels more like an option - should be excluded? + 'condition_type', + 'condition_label', + 'condition_expression', + 'enabled', ].sort(); wfKeys.forEach((key) => { - if (isDefined(workflow[key])) { - parts.push(key, serializeValue(workflow[key])); + const value = get(workflow, key); + if (isDefined(value)) { + parts.push(serializeValue(value)); } }); - const steps = (workflow.steps || []).slice().sort((a, b) => { - const aName = a.name ?? ''; - const bName = b.name ?? ''; - return aName.localeCompare(bName); + // do the trigger first + for (const triggerId in wfState.triggers) { + const trigger = wfState.triggers[triggerId]; + triggerKeys.forEach((key) => { + const value = get(trigger, key); + if (isDefined(value)) { + parts.push(serializeValue(value)); + } + }); + } + + // Now do all steps + const steps = Object.values(wfState.jobs).sort((a, b) => { + const aName = a.name ?? a.id ?? ''; + const bName = b.name ?? b.id ?? ''; + return aName.toLowerCase().localeCompare(bName.toLowerCase()); }); + for (const step of steps) { stepKeys.forEach((key) => { - if (isDefined((step as any)[key])) { - parts.push(key, serializeValue((step as any)[key])); + const value = get(step, key); + if (isDefined(value)) { + parts.push(serializeValue(value)); } }); + } + + const edges = Object.values(wfState.edges) + .map((edge) => { + const sourceId = (edge.source_trigger_id ?? edge.source_job_id) as string; + const source: any = workflow.get(sourceId); + const target: any = workflow.get(edge.target_job_id); + (edge as any).name = `${source.name ?? source.id}-${ + target.name ?? target.id + }`; + return edge; + }) + .sort((a: any, b: any) => { + // sort edges by name + // where name is sourcename-target name + const aName = a.name ?? ''; + const bName = b.name ?? ''; + return aName.localeCompare(bName); + }); - if (step.next && Array.isArray(step.next)) { - const steps = step.next.slice() as Array; - steps.slice().sort((a: ConditionalStepEdge, b: ConditionalStepEdge) => { - const aLabel = a.label || ''; - const bLabel = b.label || ''; - return aLabel.localeCompare(bLabel); - }); - for (const edge of step.next) { - edgeKeys.forEach((key) => { - if (isDefined(edge[key])) { - parts.push(key, serializeValue(edge[key])); - } - }); + // now do edges + for (const edge of edges) { + edgeKeys.forEach((key) => { + const value = get(edge, key); + if (isDefined(value)) { + parts.push(serializeValue(value)); } - } + }); } const str = parts.join(''); - const hash = crypto.createHash('sha256').update(str).digest('hex'); - return `${source}:${hash.substring(0, SHORT_HASH_LENGTH)}`; + // console.log(str); + if (sha) { + const hash = crypto.createHash('sha256').update(str).digest('hex'); + return `${source}:${hash.substring(0, SHORT_HASH_LENGTH)}`; + } else { + return `${source}:${str}`; + } }; function serializeValue(val: unknown) { diff --git a/packages/project/test/gen/generator.test.ts b/packages/project/test/gen/generator.test.ts index 1ad5c2319..e0677b750 100644 --- a/packages/project/test/gen/generator.test.ts +++ b/packages/project/test/gen/generator.test.ts @@ -13,7 +13,7 @@ const gen = (src: string, t: ExecutionContext, options = {}) => { ...options, }); if (t) { - t.log(JSON.stringify(result.toJSON(), null, 2)); + // t.log(JSON.stringify(result.toJSON(), null, 2)); } return result.toJSON(); }; diff --git a/packages/project/test/parse/from-app-state.test.ts b/packages/project/test/parse/from-app-state.test.ts index 8b5f138e2..e72600672 100644 --- a/packages/project/test/parse/from-app-state.test.ts +++ b/packages/project/test/parse/from-app-state.test.ts @@ -138,7 +138,37 @@ test('should create a Project from prov state with a workflow', (t) => { }); }); -test('mapWorkflow: map a simple trigger', (t) => { +test('mapWorkflow: map a cron trigger', (t) => { + const mapped = mapWorkflow({ + id: 'cron', + name: 'w', + deleted_at: null, + triggers: { + cron: { + id: '1234', + type: 'cron', + cron_expression: '0 1 0 0', + enabled: true, + }, + }, + jobs: {}, + edges: {}, + }); + + const [trigger] = mapped.steps; + t.deepEqual(trigger, { + id: 'cron', + type: 'cron', + next: {}, + openfn: { + enabled: true, + uuid: '1234', + cron_expression: '0 1 0 0', + }, + }); +}); + +test('mapWorkflow: map a webhook trigger', (t) => { const mapped = mapWorkflow(state.workflows['my-workflow']); const [trigger] = mapped.steps; diff --git a/packages/project/test/util/version-workflow.test.ts b/packages/project/test/util/version-workflow.test.ts index fc9b23dae..d9352c33c 100644 --- a/packages/project/test/util/version-workflow.test.ts +++ b/packages/project/test/util/version-workflow.test.ts @@ -1,10 +1,90 @@ import test from 'ava'; -import { generateHash } from '../../src/util/version'; -import { generateWorkflow } from '../../src'; +import { generateHash, parse } from '../../src/util/version'; +import Project, { generateWorkflow } from '../../src'; -// TODO just caught a bug with both of these - needs to add tests around this -test.todo('include edge label in hash'); -test.todo('include edge expression in hash'); +// this is an actual lightning workflow state, copied verbatim +// todo already out of data as the version will change soon +// next, update this +const example = { + id: '320157d2-260d-4e32-91c0-db935547c263', + name: 'Turtle Power', + edges: [ + { + enabled: true, + id: 'ed3ebfbf-6fa3-4438-b21d-06f7eec216c1', + condition_type: 'always', + source_trigger_id: 'bf10f31a-cf51-45a2-95a4-756d0a25af53', + target_job_id: '4d18c46b-3bb4-4af1-81e2-07f9aee527fc', + }, + { + enabled: true, + id: '253bf2d7-1a01-44c8-8e2e-ccf50de92dff', + condition_type: 'js_expression', + condition_label: 'always tbh', + condition_expression: 'state.data', + source_job_id: '4d18c46b-3bb4-4af1-81e2-07f9aee527fc', + target_job_id: '40b839bd-5ade-414e-8dde-ed3ae77239ea', + }, + ], + version_history: ['app:91105e0d0600'], + inserted_at: '2025-12-19T15:26:49Z', + jobs: [ + { + id: '4d18c46b-3bb4-4af1-81e2-07f9aee527fc', + name: 'Transform data', + body: 'fri1', + adaptor: '@openfn/language-http@7.2.6', + project_credential_id: 'dd409089-5569-4157-8cf6-528ace283348', + }, + { + id: '40b839bd-5ade-414e-8dde-ed3ae77239ea', + name: 'do something', + body: '// Check out the Job Writing Guide for help getting started:\n// https://docs.openfn.org/documentation/jobs/job-writing-guide\n', + adaptor: '@openfn/language-http@7.2.6', + project_credential_id: null, + }, + ], + triggers: [ + { + enabled: false, + id: 'bf10f31a-cf51-45a2-95a4-756d0a25af53', + type: 'webhook', + }, + ], + updated_at: '2026-01-23T12:08:47Z', + lock_version: 34, + deleted_at: null, + concurrency: null, +}; + +// TODO I need more control over ordering +// so I want to generate a bunch of decoded strings which test the order + +test.skip('match lightning version', async (t) => { + const [expected] = example.version_history; + + // load the project from v1 state + const proj = await Project.from('state', { + workflows: [example], + }); + + /** + * why difference? + * + * the order of stuff is quite different + * the app version seems to have the node name 3 times? + * + * step/node order is different + * + * ok, cli doesnt include structure, the edge targets + */ + + const wf = proj.workflows[0]; + const hash = wf.getVersionHash(); + t.log(expected); + t.log(hash); + t.is(parse(hash).hash, parse(expected).hash); +}); test('generate an 12 character version hash for a basic workflow', (t) => { const workflow = generateWorkflow( @@ -15,7 +95,7 @@ test('generate an 12 character version hash for a basic workflow', (t) => { ` ); const hash = workflow.getVersionHash(); - t.is(hash, 'cli:518f491717a7'); + t.is(hash, 'cli:72aed7c5f224'); }); test('unique hash but different steps order', (t) => { @@ -24,22 +104,26 @@ test('unique hash but different steps order', (t) => { @name same-workflow @id id-one a-b - b-c + a-c + a-d ` ); + + // different order of nodes but should generate the same hash const workflow2 = generateWorkflow( ` @name same-workflow @id id-two + a-d a-c - c-b + a-b ` ); - // different order of nodes (b & c changed position) but should generate the same hash // validate second step is actually different t.is(workflow1.steps[1].name, 'b'); - t.is(workflow2.steps[1].name, 'c'); + t.is(workflow2.steps[1].name, 'd'); + // assert that hashes are the same t.is(generateHash(workflow1), generateHash(workflow2)); }); @@ -74,6 +158,81 @@ test('hash changes when workflow name changes', (t) => { t.not(generateHash(wf1), generateHash(wf2)); }); +test('hash a trigger', (t) => { + // check that various changes on a trigger update the hash + const webhook = generateWorkflow( + `@name wf-1 + @id workflow-id + t(type=webhook)-x(expression=x) + ` + ); + const cron = generateWorkflow( + `@name wf-1 + @id workflow-id + t(type=cron)-x(expression=x) + ` + ); + + t.not(generateHash(webhook), generateHash(cron)); + + const cronEnabled = generateWorkflow( + `@name wf-1 + @id workflow-id + t(enabled=false)-x + ` + ); + t.not(generateHash(webhook), generateHash(cronEnabled)); + + const cronExpression = generateWorkflow( + `@name wf-1 + @id workflow-id + t(cron_expression="1")-x + ` + ); + t.not(generateHash(webhook), generateHash(cronExpression)); +}); + +test('hash changes across an edge', (t) => { + const basicEdge = generateWorkflow( + ` + @name wf-1 + @id workflow-id + a-b + ` + ); + + const withLabel = generateWorkflow( + ` + @name wf-1 + @id workflow-id + a-(label=x)-b + ` + ); + + t.not(generateHash(basicEdge), generateHash(withLabel)); + + const withCondition = generateWorkflow( + ` + @name wf-1 + @id workflow-id + a-(condition=always)-b + ` + ); + + t.not(generateHash(basicEdge), generateHash(withCondition)); + + const withDisabled = generateWorkflow( + ` + @name wf-1 + @id workflow-id + a-(disabled=true)-b + ` + ); + + t.not(generateHash(basicEdge), generateHash(withDisabled)); +}); + +// TODO joe to think more about credential mapping (keychain and project cred keys) // can't get credentials to work in the generator, need to fix that test.skip('hash changes when credentials field changes', (t) => { const wf1 = generateWorkflow(