From dcc535216b802ef63f3276145d2d80828cd33772 Mon Sep 17 00:00:00 2001 From: "Feng(Travis) Yuan" Date: Mon, 8 Sep 2025 09:53:16 +1000 Subject: [PATCH 01/12] feat: KSBP-58641 extend-k8s-deployer-with-component-dependencies --- .gitignore | 1 + k8s-deployer/.gitignore | 2 +- k8s-deployer/src/dependency-resolver.ts | 212 ++++++++++ k8s-deployer/src/errors.ts | 80 +++- k8s-deployer/src/index.ts | 40 ++ k8s-deployer/src/pitfile/schema-v1.ts | 1 + k8s-deployer/src/test-suite-handler.ts | 34 +- .../test/component-dependency.spec.ts | 125 ++++++ k8s-deployer/test/dependency-resolver.spec.ts | 363 ++++++++++++++++++ k8s-deployer/test/errors.spec.ts | 71 ++++ .../test/pitfile/pitfile-loader.spec.ts | 50 ++- ...tfile-invalid-with-cyclic-dependencies.yml | 40 ++ ...itfile-valid-with-complex-dependencies.yml | 83 ++++ .../test-pitfile-valid-with-dependencies.yml | 57 +++ ...est-pitfile-valid-without-dependencies.yml | 43 +++ 15 files changed, 1184 insertions(+), 18 deletions(-) create mode 100644 k8s-deployer/src/dependency-resolver.ts create mode 100644 k8s-deployer/test/component-dependency.spec.ts create mode 100644 k8s-deployer/test/dependency-resolver.spec.ts create mode 100644 k8s-deployer/test/errors.spec.ts create mode 100644 k8s-deployer/test/pitfile/test-pitfile-invalid-with-cyclic-dependencies.yml create mode 100644 k8s-deployer/test/pitfile/test-pitfile-valid-with-complex-dependencies.yml create mode 100644 k8s-deployer/test/pitfile/test-pitfile-valid-with-dependencies.yml create mode 100644 k8s-deployer/test/pitfile/test-pitfile-valid-without-dependencies.yml diff --git a/.gitignore b/.gitignore index be713ce..91025a3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ tmp* .env .idea .DS_Store +.tool-versions \ No newline at end of file diff --git a/k8s-deployer/.gitignore b/k8s-deployer/.gitignore index f6eecb5..ec70ed0 100644 --- a/k8s-deployer/.gitignore +++ b/k8s-deployer/.gitignore @@ -1,4 +1,4 @@ node_modules/ dist/ tmp/ -coverage/ +coverage/ \ No newline at end of file diff --git a/k8s-deployer/src/dependency-resolver.ts b/k8s-deployer/src/dependency-resolver.ts new file mode 100644 index 0000000..c666ac1 --- /dev/null +++ b/k8s-deployer/src/dependency-resolver.ts @@ -0,0 +1,212 @@ +import { Schema } from "./model.js" +import { + CyclicDependencyError, + InvalidDependencyError, + SelfDependencyError, + DuplicateComponentIdError, + DependencyValidationError +} from "./errors.js" + +export interface TopologicalSortResult { + sortedComponents: Array + levels: Array> // Components grouped by dependency level +} + +/** + * Validate all dependencies for a set of components and throws appropriate errors + * Validation checks the following: + * 1. Cyclic dependencies + * 2. Self-dependencies + * 3. Invalid references + * 4. Duplicate component IDs + */ +export const validateDependencies = ( + components: Array, + testSuiteName: string +): void => { + const errors: Array = [] + + // Check for duplicate component IDs + const componentIdCount = new Map() + components.forEach(({ id }) => { + componentIdCount.set(id, (componentIdCount.get(id) ?? 0) + 1) + if (componentIdCount.get(id) > 1) { + errors.push(new DuplicateComponentIdError(id)) + } + }) + + // Check for invalid component ID references and self-dependencies + components.forEach(component => { + const { id, dependsOn } = component + if (dependsOn) { + dependsOn.forEach(depId => { + if (depId === id) { + errors.push(new SelfDependencyError(id)) + } + if (!componentIdCount.has(depId)) { + errors.push(new InvalidDependencyError(id, depId)) + } + }) + } + }) + + // Check for cyclic dependencies + const cyclicError = detectCyclicDependencies(components) + if (cyclicError) { + errors.push(cyclicError) + } + + // Throw all errors found + if (errors.length > 0) { + throw new DependencyValidationError(testSuiteName, errors) + } +} + +/** + * Detect cyclic dependencies using DFS + */ +export const detectCyclicDependencies = (components: Array): CyclicDependencyError | undefined => { + const unvisited = 0, visiting = 1, visited = 2 + const state = new Map() + const parent = new Map() + const componentMap = new Map() + + // Initialize + components.forEach(comp => { + componentMap.set(comp.id, comp) + state.set(comp.id, unvisited) + }) + + const dfs = (componentId: string): CyclicDependencyError | undefined => { + if (state.get(componentId) === visited) { + return undefined + } + + // Found a cycle. Include the cycle path in the error + if (state.get(componentId) === visiting) { + return new CyclicDependencyError(reconstructCyclePath(componentId, parent), componentId) + } + + state.set(componentId, visiting) + + const component = componentMap.get(componentId) + if (!component) { + return undefined + } + if (component.dependsOn) { + for (const depId of component.dependsOn) { + parent.set(depId, componentId) + const result = dfs(depId) + if (result) { + return result + } + } + } + + state.set(componentId, visited) + } + + // Check all components + for (const component of components) { + if (state.get(component.id) === unvisited) { + const result = dfs(component.id) + if (result) { + return result + } + } + } +} + +/** + * Perform topological sort + * Preserve original order for components at the same dependency level + */ +export const topologicalSort = (components: Array): TopologicalSortResult => { + // Build adjacency list and in-degree count + const graph = new Map() + const inDegreeMap = new Map() + const componentMap = new Map() + + // Initialize + components.forEach(comp => { + componentMap.set(comp.id, comp) + graph.set(comp.id, []) + inDegreeMap.set(comp.id, 0) + }) + + // Build graph and calculate in-degrees + components.forEach(({ id, dependsOn }) => + dependsOn?.forEach(depId => { + graph.get(depId)?.push(id) + inDegreeMap.set(id, (inDegreeMap.get(id) ?? 0) + 1) + }) + ) + + // Find components with in-degree = 0 + const queue: string[] = [] + inDegreeMap.forEach((degree, id) => { if (degree === 0) queue.push(id) }) + + // Sort components using BFS + const result: Schema.DeployableComponent[] = [] + const levels: Schema.DeployableComponent[][] = [] + + while (queue.length > 0) { + const currentLevel: Schema.DeployableComponent[] = [] + const currentLevelSize = queue.length + + // Process all components at current level + for (let i = 0; i < currentLevelSize; i++) { + const currentId = queue.shift()! + const component = componentMap.get(currentId)! + currentLevel.push(component) + + // Update in-degrees of dependent components + graph.get(currentId)?.forEach(depId => { + const newDegree = (inDegreeMap.get(depId) ?? 0) - 1 + inDegreeMap.set(depId, newDegree) + if (newDegree === 0) { + queue.push(depId) + } + }) + } + + // Sort current level by original order defined in the pitfile + // The deployment and undeployment order on the same dependency level follows the component definition order + // This behaviour is consistent with the existing behaviour, thus maintaining backward compatibility + currentLevel.sort((a, b) => { + const aIndex = components.findIndex(c => c.id === a.id) + const bIndex = components.findIndex(c => c.id === b.id) + return aIndex - bIndex + }) + + // Add sorted components to result + currentLevel.forEach(component => result.push(component)) + levels.push([...currentLevel]) + } + + return { sortedComponents: result, levels } +} + +/** + * Return components in reverse order for undeployment + * Only reverse dependency levels but maintain original component definition order within each level + * The deployment and undeployment order on the same dependency level follows the component definition order + * This behaviour is consistent with the existing behaviour, thus maintaining backward compatibility + */ +export const reverseTopologicalSort = (sortResult: TopologicalSortResult): Array => [...sortResult.levels].reverse().flat() + +/** + * Traceback the dependency path when a cycle is detected + * This is for troubleshooting convenience + */ +const reconstructCyclePath = (startId: string, parent: Map): Array => { + const path: Array = [] + let current = startId + + do { + path.push(current) + current = parent.get(current)! + } while (current !== startId) + + return path +} diff --git a/k8s-deployer/src/errors.ts b/k8s-deployer/src/errors.ts index ffa1fea..ee2d318 100644 --- a/k8s-deployer/src/errors.ts +++ b/k8s-deployer/src/errors.ts @@ -1,15 +1,75 @@ -export class SchemaValidationError extends Error { - constructor(message: string) { - super(`SchemaValidationError: ${message}`); +export class CyclicDependencyError extends Error { + public readonly cyclePath: Array + public readonly componentId: string + + constructor(cyclePath: Array, componentId: string) { + const cycleString = cyclePath.join(' → ') + super(`Cyclic dependency detected: ${cycleString} → ${componentId}`) + this.name = "CyclicDependencyError" + this.cyclePath = cyclePath + this.componentId = componentId + } +} + +export class InvalidDependencyError extends Error { + public readonly componentId: string + public readonly invalidDependency: string + + constructor(componentId: string, invalidDependency: string) { + super(`Component ${componentId} references non-existent component ${invalidDependency}`) + this.name = "InvalidDependencyError" + this.componentId = componentId + this.invalidDependency = invalidDependency + } +} + +export class SelfDependencyError extends Error { + public readonly componentId: string + + constructor(componentId: string) { + super(`Component ${componentId} cannot depend on itself`) + this.name = "SelfDependencyError" + this.componentId = componentId } } -export class ApiSchemaValidationError extends SchemaValidationError { - data?: string - url: string - constructor(message: string, url: string, data?: any) { - super(message); - this.url = url - this.data = data +export class DuplicateComponentIdError extends Error { + public readonly componentId: string + + constructor(componentId: string) { + super(`Duplicate component ID ${componentId} found`) + this.name = "DuplicateComponentIdError" + this.componentId = componentId + } +} + +export class DependencyValidationError extends Error { + public readonly errors: Array + public readonly testSuiteName: string + + constructor(testSuiteName: string, errors: Array) { + const errorMessages = errors.map(e => e.message).join('; ') + super(`Dependency validation failed for test suite ${testSuiteName}: ${errorMessages}`) + this.name = "DependencyValidationError" + this.errors = errors + this.testSuiteName = testSuiteName + } +} + +export class ApiSchemaValidationError extends Error { + public readonly validationErrors: string + public readonly endpoint: string + public readonly response: string + public readonly url: string + public readonly data?: string + + constructor(validationErrors: string, endpoint: string, response: string) { + super(`API schema validation failed for endpoint ${endpoint}: ${validationErrors}`) + this.name = "ApiSchemaValidationError" + this.validationErrors = validationErrors + this.endpoint = endpoint + this.response = response + this.url = endpoint + this.data = response } } \ No newline at end of file diff --git a/k8s-deployer/src/index.ts b/k8s-deployer/src/index.ts index 3de895d..fd91add 100644 --- a/k8s-deployer/src/index.ts +++ b/k8s-deployer/src/index.ts @@ -4,6 +4,8 @@ import { Config } from "./config.js" import * as PifFileLoader from "./pitfile/pitfile-loader.js" import * as SuiteHandler from "./test-suite-handler.js" import { DeployedTestSuite } from "./model.js" +import { validateDependencies } from "./dependency-resolver.js" +import { DependencyValidationError, CyclicDependencyError } from "./errors.js" const main = async () => { logger.info("main()...") @@ -13,6 +15,44 @@ const main = async () => { const file = await PifFileLoader.loadFromFile(config.pitfile) + // EARLY VALIDATION: Check all test suites for dependency issues + logger.info("") + logger.info("--------------------- Validating Component Dependencies ---------------------") + logger.info("") + + for (let i = 0; i < file.testSuites.length; i++) { + const testSuite = file.testSuites[i] + + try { + validateDependencies(testSuite.deployment.graph.components, testSuite.name) + logger.info("Test suite '%s' dependencies validated successfully", testSuite.name) + } catch (error) { + if (error instanceof DependencyValidationError) { + // Log dependency validation errors + logger.error("") + logger.error("DEPENDENCY VALIDATION FAILED for test suite '%s'", testSuite.name) + logger.error("") + + error.errors.forEach(err => { + if (err instanceof CyclicDependencyError) { + logger.error("CYCLIC DEPENDENCY DETECTED:") + logger.error("Cycle: %s", err.cyclePath.join(' → ')) + logger.error("This creates an infinite loop and cannot be resolved.") + logger.error("Please fix the dependency chain in your pitfile.yml") + } else { + logger.error("%s", err.message) + } + }) + + logger.error("") + logger.error("DEPLOYMENT ABORTED: Fix dependency issues before proceeding") + logger.error("") + } + + throw error + } + } + const artefacts = new Array>() for (let i = 0; i < file.testSuites.length; i++) { const testSuite = file.testSuites[i] diff --git a/k8s-deployer/src/pitfile/schema-v1.ts b/k8s-deployer/src/pitfile/schema-v1.ts index 97265f3..d040ea4 100644 --- a/k8s-deployer/src/pitfile/schema-v1.ts +++ b/k8s-deployer/src/pitfile/schema-v1.ts @@ -60,6 +60,7 @@ export class DeployableComponent { deploy: DeployInstructions undeploy: DeployInstructions logTailing?: LogTailing + dependsOn?: Array // Optional array of component IDs this component depends on } export class Graph { diff --git a/k8s-deployer/src/test-suite-handler.ts b/k8s-deployer/src/test-suite-handler.ts index 444c49e..75c2ed8 100644 --- a/k8s-deployer/src/test-suite-handler.ts +++ b/k8s-deployer/src/test-suite-handler.ts @@ -9,6 +9,7 @@ import * as PifFileLoader from "./pitfile/pitfile-loader.js" import { PodLogTail } from "./pod-log-tail.js" import * as Shell from "./shell-facade.js" import * as TestRunner from "./test-app-client/test-runner.js" +import { topologicalSort, reverseTopologicalSort } from "./dependency-resolver.js" export const generatePrefix = (env: string): Prefix => { return generatePrefixByDate(new Date(), env) @@ -36,15 +37,24 @@ export const generatePrefixByDate = (date: Date, env: string): Prefix => { /** * Deploying: - * 1. all components in the graph, + * 1. all components in the graph in the topological order * 2. test app for the graph. */ const deployGraph = async (config: Config, workspace: string, testSuiteId: string, graph: Schema.Graph, namespace: Namespace, testAppDirForRemoteTestSuite?: string): Promise => { + // Dependencies are already validated in main(), so it's safe to directly sort here. + const { sortedComponents } = topologicalSort(graph.components) + + logger.info("") + logger.info("Dependency Resolution for %s:", testSuiteId) + logger.info("Deployment order: %s", sortedComponents.map(c => c.id).join(" → ")) + logger.info("") + + // Deploy components in topological order const deployments: Array = new Array() - for (let i = 0; i < graph.components.length; i++) { - const componentSpec = graph.components[i] + for (let i = 0; i < sortedComponents.length; i++) { + const componentSpec = sortedComponents[i] logger.info("") - logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", i + 1, graph.components.length, componentSpec.name, testSuiteId) + logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", i + 1, sortedComponents.length, componentSpec.name, testSuiteId) logger.info("") const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) deployments.push(new DeployedComponent(commitSha, componentSpec)) @@ -217,8 +227,20 @@ export const undeployAll = async (config: Config, pitfile: Schema.PitFile, suite } await Deployer.undeployComponent(item.workspace, item.namespace, item.graphDeployment.testApp) - for (let deploymentInfo of item.graphDeployment.components) { - await Deployer.undeployComponent(item.workspace, item.namespace, deploymentInfo) + + // Undeploy components in reverse topological order + const componentSpecs = item.graphDeployment.components.map(dep => dep.component) + const reverseSortedComponents = reverseTopologicalSort(topologicalSort(componentSpecs)) + + logger.info("") + logger.info("Undeployment order: %s", reverseSortedComponents.map(c => c.id).join(" → ")) + logger.info("") + + for (let componentSpec of reverseSortedComponents) { + const deployedComponent = item.graphDeployment.components.find(dep => dep.component.id === componentSpec.id) + if (deployedComponent) { + await Deployer.undeployComponent(item.workspace, item.namespace, deployedComponent) + } } await K8s.deleteNamespace(config.parentNamespace, item.namespace, config.namespaceTimeoutSeconds, item.workspace) diff --git a/k8s-deployer/test/component-dependency.spec.ts b/k8s-deployer/test/component-dependency.spec.ts new file mode 100644 index 0000000..2768417 --- /dev/null +++ b/k8s-deployer/test/component-dependency.spec.ts @@ -0,0 +1,125 @@ +import { describe, it } from "mocha" +import { expect } from "chai" +import * as path from "path" +import { fileURLToPath } from "url" +import * as PifFileLoader from "../src/pitfile/pitfile-loader.js" +import { validateDependencies, topologicalSort, reverseTopologicalSort } from "../src/dependency-resolver.js" +import { DependencyValidationError, CyclicDependencyError } from "../src/errors.js" +import { Schema } from "../src/model.js" + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) + +describe("Component Dependency Tests", () => { + describe("Valid Dependencies", () => { + it("should validate and sort components with valid dependencies", async () => { + const pitfilePath = path.join(__dirname, "pitfile", "test-pitfile-valid-with-dependencies.yml") + const pitfile = await PifFileLoader.loadFromFile(pitfilePath) + const testSuite = pitfile.testSuites[0] + const components = testSuite.deployment.graph.components + + // Should not throw validation error + expect(() => validateDependencies(components, testSuite.name)).not.to.throw() + + // Test topological sorting + const sortResult = topologicalSort(components) + const sortedIds = sortResult.sortedComponents.map(c => c.id) + + // Expected order: database -> api-service, cache -> frontend + expect(sortedIds).to.deep.equal(["database", "api-service", "cache", "frontend"]) + + // Test levels + expect(sortResult.levels).to.have.length(3) + expect(sortResult.levels[0].map(c => c.id)).to.deep.equal(["database"]) + expect(sortResult.levels[1].map(c => c.id)).to.deep.equal(["api-service", "cache"]) + expect(sortResult.levels[2].map(c => c.id)).to.deep.equal(["frontend"]) + + // Test reverse sorting for undeployment + const reverseSorted = reverseTopologicalSort(sortResult) + const reverseIds = reverseSorted.map(c => c.id) + + // Expected undeployment order: frontend -> api-service, cache -> database + expect(reverseIds).to.deep.equal(["frontend", "api-service", "cache", "database"]) + }) + + it("should handle components without dependsOn field (backward compatibility)", async () => { + const pitfilePath = path.join(__dirname, "pitfile", "test-pitfile-valid-without-dependencies.yml") + const pitfile = await PifFileLoader.loadFromFile(pitfilePath) + const testSuite = pitfile.testSuites[0] + const components = testSuite.deployment.graph.components + + // Should not throw validation error + expect(() => validateDependencies(components, testSuite.name)).not.to.throw() + + // Should maintain original order + const sortResult = topologicalSort(components) + expect(sortResult.sortedComponents.map(c => c.id)).to.deep.equal(["component-a", "component-b", "component-c"]) + }) + }) + + describe("Cyclic Dependencies", () => { + it("should detect and report cyclic dependencies", async () => { + const pitfilePath = path.join(__dirname, "pitfile", "test-pitfile-invalid-with-cyclic-dependencies.yml") + const pitfile = await PifFileLoader.loadFromFile(pitfilePath) + const testSuite = pitfile.testSuites[0] + const components = testSuite.deployment.graph.components + + // Should throw validation error + expect(() => validateDependencies(components, testSuite.name)).to.throw(DependencyValidationError) + + try { + validateDependencies(components, testSuite.name) + } catch (error) { + expect(error).to.be.instanceOf(DependencyValidationError) + expect(error.errors).to.have.length(1) + expect(error.errors[0]).to.be.instanceOf(CyclicDependencyError) + + const cyclicError = error.errors[0] as CyclicDependencyError + expect(cyclicError.cyclePath).to.deep.equal(["component-a", "component-b"]) + } + }) + }) + + describe("Complex Dependency Scenarios", () => { + it("should handle complex dependency graphs correctly", async () => { + const pitfilePath = path.join(__dirname, "pitfile", "test-pitfile-valid-with-complex-dependencies.yml") + const pitfile = await PifFileLoader.loadFromFile(pitfilePath) + const testSuite = pitfile.testSuites[0] + const components = testSuite.deployment.graph.components + + // Should validate successfully + expect(() => validateDependencies(components, testSuite.name)).not.to.throw() + + // Test deployment order + const sortResult = topologicalSort(components) + const deploymentOrder = sortResult.sortedComponents.map(c => c.id) + + // Expected deployment order: + // database, message-queue -> api-service, cache-service, task-worker -> backend for frontend -> frontend + expect(deploymentOrder).to.deep.equal([ + "database", "message-queue", + "api-service", "cache-service", "task-worker", + "backend-for-frontend", + "frontend" + ]) + + // Test undeployment order + const undeploymentOrder = reverseTopologicalSort(sortResult).map(c => c.id) + + // Expected undeployment order: + // frontend -> backend for frontend -> api-service, cache-service, task-worker -> database, message-queue + expect(undeploymentOrder).to.deep.equal([ + "frontend", + "backend-for-frontend", + "api-service", "cache-service", "task-worker", + "database", "message-queue" + ]) + + // Verify levels + expect(sortResult.levels).to.have.length(4) + expect(sortResult.levels[0].map(c => c.id)).to.deep.equal(["database", "message-queue"]) + expect(sortResult.levels[1].map(c => c.id)).to.deep.equal(["api-service", "cache-service", "task-worker"]) + expect(sortResult.levels[2].map(c => c.id)).to.deep.equal(["backend-for-frontend"]) + expect(sortResult.levels[3].map(c => c.id)).to.deep.equal(["frontend"]) + }) + }) +}) diff --git a/k8s-deployer/test/dependency-resolver.spec.ts b/k8s-deployer/test/dependency-resolver.spec.ts new file mode 100644 index 0000000..552789e --- /dev/null +++ b/k8s-deployer/test/dependency-resolver.spec.ts @@ -0,0 +1,363 @@ +import { describe, it } from "mocha" +import { expect } from "chai" +import { + validateDependencies, + detectCyclicDependencies, + topologicalSort, + reverseTopologicalSort +} from "../src/dependency-resolver.js" +import { Schema } from "../src/model.js" +import { + CyclicDependencyError, + DependencyValidationError +} from "../src/errors.js" + +describe("Dependency Resolver", () => { + describe("validateDependencies", () => { + it("should pass validation for components without dependencies", () => { + const components: Array = [ + { + name: "Component A", + id: "component-a", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" } + }, + { + name: "Component B", + id: "component-b", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" } + } + ] + + expect(() => validateDependencies(components, "test-suite")).not.to.throw() + }) + + it("should pass validation for valid dependencies", () => { + const components: Array = [ + { + name: "Database", + id: "database", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: [] + }, + { + name: "API Service", + id: "api-service", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["database"] + } + ] + + expect(() => validateDependencies(components, "test-suite")).not.to.throw() + }) + + it("should throw error for duplicate component IDs", () => { + const components: Array = [ + { + name: "Component A", + id: "duplicate-id", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" } + }, + { + name: "Component B", + id: "duplicate-id", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" } + } + ] + + expect(() => validateDependencies(components, "test-suite")).to.throw(DependencyValidationError) + }) + + it("should throw error for self-dependency", () => { + const components: Array = [ + { + name: "Component A", + id: "component-a", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["component-a"] + } + ] + + expect(() => validateDependencies(components, "test-suite")).to.throw(DependencyValidationError) + }) + + it("should throw error for invalid dependency reference", () => { + const components: Array = [ + { + name: "Component A", + id: "component-a", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["non-existent-component"] + } + ] + + expect(() => validateDependencies(components, "test-suite")).to.throw(DependencyValidationError) + }) + + it("should throw error for cyclic dependencies", () => { + const components: Array = [ + { + name: "Component A", + id: "component-a", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["component-b"] + }, + { + name: "Component B", + id: "component-b", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["component-a"] + } + ] + + expect(() => validateDependencies(components, "test-suite")).to.throw(DependencyValidationError) + }) + }) + + describe("detectCyclicDependencies", () => { + it("should return undefined for no cycles", () => { + const components: Array = [ + { + name: "Database", + id: "database", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: [] + }, + { + name: "API Service", + id: "api-service", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["database"] + } + ] + + expect(detectCyclicDependencies(components)).to.be.undefined + }) + + it("should detect simple cycle", () => { + const components: Array = [ + { + name: "Component A", + id: "component-a", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["component-b"] + }, + { + name: "Component B", + id: "component-b", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["component-a"] + } + ] + + const result = detectCyclicDependencies(components) + expect(result).to.be.instanceOf(CyclicDependencyError) + expect(result.cyclePath).to.deep.equal(["component-a", "component-b"]) + }) + + it("should detect complex cycle", () => { + const components: Array = [ + { + name: "Component A", + id: "component-a", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["component-b"] + }, + { + name: "Component B", + id: "component-b", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["component-c"] + }, + { + name: "Component C", + id: "component-c", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["component-a"] + } + ] + + const result = detectCyclicDependencies(components) + expect(result).to.be.instanceOf(CyclicDependencyError) + expect(result.cyclePath).to.have.length(3) + expect(result.cyclePath).to.include("component-a") + expect(result.cyclePath).to.include("component-b") + expect(result.cyclePath).to.include("component-c") + }) + }) + + describe("topologicalSort", () => { + it("should sort components without dependencies in original order", () => { + const components: Array = [ + { + name: "Component A", + id: "component-a", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" } + }, + { + name: "Component B", + id: "component-b", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" } + }, + { + name: "Component C", + id: "component-c", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" } + } + ] + + const result = topologicalSort(components) + expect(result.sortedComponents.map(c => c.id)).to.deep.equal(["component-a", "component-b", "component-c"]) + }) + + it("should sort components with dependencies correctly", () => { + const components: Array = [ + { + name: "API Service", + id: "api-service", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["database"] + }, + { + name: "Frontend", + id: "frontend", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["api-service"] + }, + { + name: "Database", + id: "database", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: [] + } + ] + + const result = topologicalSort(components) + expect(result.sortedComponents.map(c => c.id)).to.deep.equal(["database", "api-service", "frontend"]) + }) + + it("should maintain original order for same-level components", () => { + const components: Array = [ + { + name: "Database", + id: "database", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: [] + }, + { + name: "API Service", + id: "api-service", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["database"] + }, + { + name: "Cache", + id: "cache", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["database"] + } + ] + + const result = topologicalSort(components) + expect(result.sortedComponents.map(c => c.id)).to.deep.equal(["database", "api-service", "cache"]) + expect(result.levels).to.have.length(2) + expect(result.levels[0].map(c => c.id)).to.deep.equal(["database"]) + expect(result.levels[1].map(c => c.id)).to.deep.equal(["api-service", "cache"]) + }) + }) + + describe("reverseTopologicalSort", () => { + it("should reverse dependency levels but maintain order within levels", () => { + const components: Array = [ + { + name: "Database", + id: "database", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: [] + }, + { + name: "API Service", + id: "api-service", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["database"] + }, + { + name: "Frontend", + id: "frontend", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["api-service"] + }, + { + name: "Cache", + id: "cache", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" }, + dependsOn: ["database"] + } + ] + + const reverseResult = reverseTopologicalSort(topologicalSort(components)) + + // Reverse levels only, but same order within level: frontend, api-service, cache, database + expect(reverseResult.map(c => c.id)).to.deep.equal(["frontend", "api-service", "cache", "database"]) + }) + }) +}) diff --git a/k8s-deployer/test/errors.spec.ts b/k8s-deployer/test/errors.spec.ts new file mode 100644 index 0000000..3c2be70 --- /dev/null +++ b/k8s-deployer/test/errors.spec.ts @@ -0,0 +1,71 @@ +import { describe, it } from "mocha" +import { expect } from "chai" +import { + CyclicDependencyError, + InvalidDependencyError, + SelfDependencyError, + DuplicateComponentIdError, + DependencyValidationError +} from "../src/errors.js" + +describe("Error Classes", () => { + describe("CyclicDependencyError", () => { + it("should create error with cycle path", () => { + const cyclePath = ["component-a", "component-b", "component-c"] + const error = new CyclicDependencyError(cyclePath, "component-a") + + expect(error.name).to.equal("CyclicDependencyError") + expect(error.message).to.equal("Cyclic dependency detected: component-a → component-b → component-c → component-a") + expect(error.cyclePath).to.deep.equal(cyclePath) + expect(error.componentId).to.equal("component-a") + }) + }) + + describe("InvalidDependencyError", () => { + it("should create error with component and invalid dependency", () => { + const error = new InvalidDependencyError("component-a", "non-existent") + + expect(error.name).to.equal("InvalidDependencyError") + expect(error.message).to.equal("Component component-a references non-existent component non-existent") + expect(error.componentId).to.equal("component-a") + expect(error.invalidDependency).to.equal("non-existent") + }) + }) + + describe("SelfDependencyError", () => { + it("should create error with component ID", () => { + const error = new SelfDependencyError("component-a") + + expect(error.name).to.equal("SelfDependencyError") + expect(error.message).to.equal("Component component-a cannot depend on itself") + expect(error.componentId).to.equal("component-a") + }) + }) + + describe("DuplicateComponentIdError", () => { + it("should create error with duplicate component ID", () => { + const error = new DuplicateComponentIdError("duplicate-id") + + expect(error.name).to.equal("DuplicateComponentIdError") + expect(error.message).to.equal("Duplicate component ID duplicate-id found") + expect(error.componentId).to.equal("duplicate-id") + }) + }) + + describe("DependencyValidationError", () => { + it("should create error with test suite name and errors", () => { + const errors = [ + new InvalidDependencyError("component-a", "non-existent"), + new SelfDependencyError("component-b") + ] + const error = new DependencyValidationError("test-suite", errors) + + expect(error.name).to.equal("DependencyValidationError") + expect(error.message).to.include("Dependency validation failed for test suite test-suite") + expect(error.message).to.include("Component component-a references non-existent component non-existent") + expect(error.message).to.include("Component component-b cannot depend on itself") + expect(error.testSuiteName).to.equal("test-suite") + expect(error.errors).to.deep.equal(errors) + }) + }) +}) diff --git a/k8s-deployer/test/pitfile/pitfile-loader.spec.ts b/k8s-deployer/test/pitfile/pitfile-loader.spec.ts index 2cf7272..13402d3 100644 --- a/k8s-deployer/test/pitfile/pitfile-loader.spec.ts +++ b/k8s-deployer/test/pitfile/pitfile-loader.spec.ts @@ -1,5 +1,5 @@ import * as chai from "chai" -import chaiAsPromised from 'chai-as-promised' +import chaiAsPromised from "chai-as-promised" import * as sinon from "sinon" chai.use(chaiAsPromised) @@ -51,6 +51,54 @@ describe("Loads pitfile from disk", () => { await chai.expect(PifFileLoader.loadFromFile("dist/test/pitfile/test-pitfile-valid-3-invalid.yml")).eventually.rejectedWith(errorMessage) }) + it("should load pitfile with dependsOn field", async () => { + const file = await PifFileLoader.loadFromFile("dist/test/pitfile/test-pitfile-valid-with-dependencies.yml") + chai.expect(file.projectName).eq("Dependency Test Example") + chai.expect(file.testSuites).lengthOf(1) + + const testSuite = file.testSuites[0] + chai.expect(testSuite.name).eq("Dependency Test Suite") + chai.expect(testSuite.deployment.graph.components).lengthOf(4) + + const components = testSuite.deployment.graph.components + const database = components.find(c => c.id === "database") + const apiService = components.find(c => c.id === "api-service") + const cache = components.find(c => c.id === "cache") + const frontend = components.find(c => c.id === "frontend") + + chai.expect(database).not.undefined + chai.expect(database!.dependsOn).undefined + + chai.expect(apiService).not.undefined + chai.expect(apiService!.dependsOn).deep.equal(["database"]) + + chai.expect(cache).not.undefined + chai.expect(cache!.dependsOn).deep.equal(["database"]) + + chai.expect(frontend).not.undefined + chai.expect(frontend!.dependsOn).deep.equal(["api-service", "cache"]) + }) + + it("should load pitfile with cyclic dependencies", async () => { + const file = await PifFileLoader.loadFromFile("dist/test/pitfile/test-pitfile-invalid-with-cyclic-dependencies.yml") + chai.expect(file.projectName).eq("Cyclic Dependency Test Example") + chai.expect(file.testSuites).lengthOf(1) + + const testSuite = file.testSuites[0] + chai.expect(testSuite.name).eq("Cyclic Dependency Test Suite") + chai.expect(testSuite.deployment.graph.components).lengthOf(2) + + const components = testSuite.deployment.graph.components + const componentA = components.find(c => c.id === "component-a") + const componentB = components.find(c => c.id === "component-b") + + chai.expect(componentA).not.undefined + chai.expect(componentA!.dependsOn).deep.equal(["component-b"]) + + chai.expect(componentB).not.undefined + chai.expect(componentB!.dependsOn).deep.equal(["component-a"]) + }) + after(() => { sandbox.restore() }) diff --git a/k8s-deployer/test/pitfile/test-pitfile-invalid-with-cyclic-dependencies.yml b/k8s-deployer/test/pitfile/test-pitfile-invalid-with-cyclic-dependencies.yml new file mode 100644 index 0000000..d132018 --- /dev/null +++ b/k8s-deployer/test/pitfile/test-pitfile-invalid-with-cyclic-dependencies.yml @@ -0,0 +1,40 @@ +version: 1.0 +projectName: Cyclic Dependency Test Example + +lockManager: + enabled: false + +testSuites: + - name: Cyclic Dependency Test Suite + id: cyclic-test + + deployment: + graph: + testApp: + name: Test App + id: test-app + location: + type: LOCAL + deploy: + command: "echo 'test app deployed'" + undeploy: + command: "echo 'test app undeployed'" + + components: + - name: Component A + id: component-a + dependsOn: + - component-b + deploy: + command: "echo 'component-a deployed'" + undeploy: + command: "echo 'component-a undeployed'" + + - name: Component B + id: component-b + dependsOn: + - component-a + deploy: + command: "echo 'component-b deployed'" + undeploy: + command: "echo 'component-b undeployed'" diff --git a/k8s-deployer/test/pitfile/test-pitfile-valid-with-complex-dependencies.yml b/k8s-deployer/test/pitfile/test-pitfile-valid-with-complex-dependencies.yml new file mode 100644 index 0000000..b4d2615 --- /dev/null +++ b/k8s-deployer/test/pitfile/test-pitfile-valid-with-complex-dependencies.yml @@ -0,0 +1,83 @@ +version: 1.0 +projectName: Complex Dependency Test Example + +lockManager: + enabled: false + +testSuites: + - name: Complex Dependency Test Suite + id: complex-dependency-test + + deployment: + graph: + testApp: + name: Test App + id: test-app + location: + type: LOCAL + deploy: + command: "echo 'test app deployed'" + undeploy: + command: "echo 'test app undeployed'" + + components: + - name: Database + id: database + deploy: + command: "echo 'database deployed'" + undeploy: + command: "echo 'database undeployed'" + + - name: Message Queue + id: message-queue + deploy: + command: "echo 'message queue deployed'" + undeploy: + command: "echo 'message queue undeployed'" + + - name: API Service + id: api-service + dependsOn: + - database + - message-queue + deploy: + command: "echo 'api-service deployed'" + undeploy: + command: "echo 'api-service undeployed'" + + - name: Cache Service + id: cache-service + dependsOn: + - database + deploy: + command: "echo 'cache service deployed'" + undeploy: + command: "echo 'cache service undeployed'" + + - name: Backend For Frontend + id: backend-for-frontend + dependsOn: + - api-service + - cache-service + deploy: + command: "echo 'backend for frontend deployed'" + undeploy: + command: "echo 'backend for frontend undeployed'" + + - name: Frontend + id: frontend + dependsOn: + - backend-for-frontend + deploy: + command: "echo 'frontend deployed'" + undeploy: + command: "echo 'frontend undeployed'" + + - name: Task Worker + id: task-worker + dependsOn: + - message-queue + deploy: + command: "echo 'task worker deployed'" + undeploy: + command: "echo 'task worker undeployed'" \ No newline at end of file diff --git a/k8s-deployer/test/pitfile/test-pitfile-valid-with-dependencies.yml b/k8s-deployer/test/pitfile/test-pitfile-valid-with-dependencies.yml new file mode 100644 index 0000000..d0c58eb --- /dev/null +++ b/k8s-deployer/test/pitfile/test-pitfile-valid-with-dependencies.yml @@ -0,0 +1,57 @@ +version: 1.0 +projectName: Dependency Test Example + +lockManager: + enabled: false + +testSuites: + - name: Dependency Test Suite + id: dependency-test + + deployment: + graph: + testApp: + name: Test App + id: test-app + location: + type: LOCAL + deploy: + command: "echo 'test app deployed'" + undeploy: + command: "echo 'test app undeployed'" + + components: + - name: Database + id: database + deploy: + command: "echo 'database deployed'" + undeploy: + command: "echo 'database undeployed'" + + - name: API Service + id: api-service + dependsOn: + - database + deploy: + command: "echo 'api-service deployed'" + undeploy: + command: "echo 'api-service undeployed'" + + - name: Cache + id: cache + dependsOn: + - database + deploy: + command: "echo 'cache deployed'" + undeploy: + command: "echo 'cache undeployed'" + + - name: Frontend + id: frontend + dependsOn: + - api-service + - cache + deploy: + command: "echo 'frontend deployed'" + undeploy: + command: "echo 'frontend undeployed'" diff --git a/k8s-deployer/test/pitfile/test-pitfile-valid-without-dependencies.yml b/k8s-deployer/test/pitfile/test-pitfile-valid-without-dependencies.yml new file mode 100644 index 0000000..51661d5 --- /dev/null +++ b/k8s-deployer/test/pitfile/test-pitfile-valid-without-dependencies.yml @@ -0,0 +1,43 @@ +version: 1.0 +projectName: Backward Compatibility Test Example (no dependsOn field) + +lockManager: + enabled: false + +testSuites: + - name: Backward Compatibility Test Suite (no dependsOn field) + id: backward-compatibility-test + + deployment: + graph: + testApp: + name: Test App + id: test-app + location: + type: LOCAL + deploy: + command: "echo 'test app deployed'" + undeploy: + command: "echo 'test app undeployed'" + + components: + - name: Component A + id: component-a + deploy: + command: "echo 'component-a deployed'" + undeploy: + command: "echo 'component-a undeployed'" + + - name: Component B + id: component-b + deploy: + command: "echo 'component-b deployed'" + undeploy: + command: "echo 'component-b undeployed'" + + - name: Component C + id: component-c + deploy: + command: "echo 'component-c deployed'" + undeploy: + command: "echo 'component-c undeployed'" From 013055fa5adc804eacf4b7b4ccbc14756a3c3383 Mon Sep 17 00:00:00 2001 From: "Feng(Travis) Yuan" Date: Mon, 8 Sep 2025 13:47:49 +1000 Subject: [PATCH 02/12] chore: clean-up-comments --- k8s-deployer/src/dependency-resolver.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/k8s-deployer/src/dependency-resolver.ts b/k8s-deployer/src/dependency-resolver.ts index c666ac1..50d3989 100644 --- a/k8s-deployer/src/dependency-resolver.ts +++ b/k8s-deployer/src/dependency-resolver.ts @@ -171,8 +171,7 @@ export const topologicalSort = (components: Array): } // Sort current level by original order defined in the pitfile - // The deployment and undeployment order on the same dependency level follows the component definition order - // This behaviour is consistent with the existing behaviour, thus maintaining backward compatibility + // The deployment order on the same dependency level follows the component definition order currentLevel.sort((a, b) => { const aIndex = components.findIndex(c => c.id === a.id) const bIndex = components.findIndex(c => c.id === b.id) @@ -190,8 +189,7 @@ export const topologicalSort = (components: Array): /** * Return components in reverse order for undeployment * Only reverse dependency levels but maintain original component definition order within each level - * The deployment and undeployment order on the same dependency level follows the component definition order - * This behaviour is consistent with the existing behaviour, thus maintaining backward compatibility + * The undeployment order on the same dependency level follows the component definition order */ export const reverseTopologicalSort = (sortResult: TopologicalSortResult): Array => [...sortResult.levels].reverse().flat() From d3711c7309495f6e66fd91f5db070ce1079ae276 Mon Sep 17 00:00:00 2001 From: "James Charters james.charters@kindredgroup.com" Date: Tue, 10 Mar 2026 10:32:56 +1100 Subject: [PATCH 03/12] feat: [KSBP-101776] - Unify component dependency graph and parallel deployment instruction --- k8s-deployer/src/dependency-resolver.ts | 25 +++++ k8s-deployer/src/pitfile/schema-v1.ts | 1 + k8s-deployer/src/test-suite-handler.ts | 52 ++++++--- k8s-deployer/test/dependency-resolver.spec.ts | 101 +++++++++++++++++- k8s-deployer/test/test-suite-handler.spec.ts | 3 +- 5 files changed, 168 insertions(+), 14 deletions(-) diff --git a/k8s-deployer/src/dependency-resolver.ts b/k8s-deployer/src/dependency-resolver.ts index 50d3989..78990cb 100644 --- a/k8s-deployer/src/dependency-resolver.ts +++ b/k8s-deployer/src/dependency-resolver.ts @@ -208,3 +208,28 @@ const reconstructCyclePath = (startId: string, parent: Map): Arr return path } + +/** + * Print the dependency graph in a visually grouped format. + * Components that can be deployed in parallel are shown together and annotated. + */ +export const printDependencyGraph = (components: Array): void => { + const { levels } = topologicalSort(components) + const sep = "─".repeat(40) + const edges = components.flatMap(c => (c.dependsOn ?? []).map(dep => ` ${dep} ──▶ ${c.id}`)) + + console.log("Dependency Graph") + console.log(sep) + levels.forEach((level, idx) => + console.log(` Stage ${idx + 1} │ ${level.map(c => c.parallel ? `${c.id} ⚡` : c.id).join(" ")}`) + ) + if (edges.length > 0) { + console.log(sep) + edges.forEach(e => console.log(e)) + } + if (components.some(c => c.parallel)) { + console.log(sep) + console.log(" ⚡ = deployed concurrently within stage") + } + console.log(sep) +} diff --git a/k8s-deployer/src/pitfile/schema-v1.ts b/k8s-deployer/src/pitfile/schema-v1.ts index d040ea4..b313cea 100644 --- a/k8s-deployer/src/pitfile/schema-v1.ts +++ b/k8s-deployer/src/pitfile/schema-v1.ts @@ -61,6 +61,7 @@ export class DeployableComponent { undeploy: DeployInstructions logTailing?: LogTailing dependsOn?: Array // Optional array of component IDs this component depends on + parallel?: boolean // If true, this component may be deployed concurrently with other parallel components at the same dependency level } export class Graph { diff --git a/k8s-deployer/src/test-suite-handler.ts b/k8s-deployer/src/test-suite-handler.ts index 75c2ed8..8f8a7f2 100644 --- a/k8s-deployer/src/test-suite-handler.ts +++ b/k8s-deployer/src/test-suite-handler.ts @@ -9,7 +9,7 @@ import * as PifFileLoader from "./pitfile/pitfile-loader.js" import { PodLogTail } from "./pod-log-tail.js" import * as Shell from "./shell-facade.js" import * as TestRunner from "./test-app-client/test-runner.js" -import { topologicalSort, reverseTopologicalSort } from "./dependency-resolver.js" +import { topologicalSort, reverseTopologicalSort, printDependencyGraph } from "./dependency-resolver.js" export const generatePrefix = (env: string): Prefix => { return generatePrefixByDate(new Date(), env) @@ -37,27 +37,55 @@ export const generatePrefixByDate = (date: Date, env: string): Prefix => { /** * Deploying: - * 1. all components in the graph in the topological order + * 1. all components in the graph in topological order, deploying parallel-flagged components concurrently within each level * 2. test app for the graph. */ const deployGraph = async (config: Config, workspace: string, testSuiteId: string, graph: Schema.Graph, namespace: Namespace, testAppDirForRemoteTestSuite?: string): Promise => { // Dependencies are already validated in main(), so it's safe to directly sort here. - const { sortedComponents } = topologicalSort(graph.components) + const { sortedComponents, levels } = topologicalSort(graph.components) logger.info("") logger.info("Dependency Resolution for %s:", testSuiteId) logger.info("Deployment order: %s", sortedComponents.map(c => c.id).join(" → ")) logger.info("") - // Deploy components in topological order - const deployments: Array = new Array() - for (let i = 0; i < sortedComponents.length; i++) { - const componentSpec = sortedComponents[i] - logger.info("") - logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", i + 1, sortedComponents.length, componentSpec.name, testSuiteId) - logger.info("") - const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) - deployments.push(new DeployedComponent(commitSha, componentSpec)) + logger.info("") + logger.info("Dependency Graph for %s:", testSuiteId) + printDependencyGraph(graph.components) + logger.info("") + + // Deploy components level by level. Within each level, components with parallel:true are deployed concurrently. + const deployments: Array = [] + let componentIndex = 0 + for (const level of levels) { + const parallelGroup = level.filter(c => c.parallel === true) + const sequentialGroup = level.filter(c => c.parallel !== true) + + // Deploy all parallel-flagged components in this level concurrently + if (parallelGroup.length > 0) { + logger.info("") + logger.info("Deploying %d component(s) in parallel for suite \"%s\": %s", parallelGroup.length, testSuiteId, parallelGroup.map(c => c.id).join(", ")) + const parallelResults = await Promise.all( + parallelGroup.map(async componentSpec => { + const idx = ++componentIndex + logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) + const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) + logger.info("Graph component \"%s\" for suite \"%s\" deployed.", componentSpec.name, testSuiteId) + return new DeployedComponent(commitSha, componentSpec) + }) + ) + deployments.push(...parallelResults) + } + + // Deploy sequential components one by one + for (const componentSpec of sequentialGroup) { + const idx = ++componentIndex + logger.info("") + logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) + logger.info("") + const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) + deployments.push(new DeployedComponent(commitSha, componentSpec)) + } } logger.info("") diff --git a/k8s-deployer/test/dependency-resolver.spec.ts b/k8s-deployer/test/dependency-resolver.spec.ts index 552789e..86574a8 100644 --- a/k8s-deployer/test/dependency-resolver.spec.ts +++ b/k8s-deployer/test/dependency-resolver.spec.ts @@ -4,7 +4,8 @@ import { validateDependencies, detectCyclicDependencies, topologicalSort, - reverseTopologicalSort + reverseTopologicalSort, + printDependencyGraph } from "../src/dependency-resolver.js" import { Schema } from "../src/model.js" import { @@ -220,6 +221,7 @@ describe("Dependency Resolver", () => { }) describe("topologicalSort", () => { + it("should sort components without dependencies in original order", () => { const components: Array = [ { @@ -249,6 +251,35 @@ describe("Dependency Resolver", () => { expect(result.sortedComponents.map(c => c.id)).to.deep.equal(["component-a", "component-b", "component-c"]) }) + it("should preserve original order if no dependsOn fields are present", () => { + const components: Array = [ + { + name: "Component X", + id: "component-x", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" } + }, + { + name: "Component Y", + id: "component-y", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" } + }, + { + name: "Component Z", + id: "component-z", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, + undeploy: { command: "undeploy.sh" } + } + ] + + const result = topologicalSort(components) + expect(result.sortedComponents.map(c => c.id)).to.deep.equal(["component-x", "component-y", "component-z"]) + }) + it("should sort components with dependencies correctly", () => { const components: Array = [ { @@ -360,4 +391,72 @@ describe("Dependency Resolver", () => { expect(reverseResult.map(c => c.id)).to.deep.equal(["frontend", "api-service", "cache", "database"]) }) }) + + describe("printDependencyGraph", () => { + let logOutput: string[] + let originalLog: typeof console.log + beforeEach(() => { + logOutput = [] + originalLog = console.log + console.log = (msg?: any) => logOutput.push(String(msg)) + }) + afterEach(() => { + console.log = originalLog + }) + + it("prints graph for components without dependencies", () => { + const components: Array = [ + { name: "A", id: "a", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } }, + { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } }, + { name: "C", id: "c", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } } + ] + printDependencyGraph(components) + expect(logOutput[0]).to.equal("Dependency Graph") + expect(logOutput).to.include(" Stage 1 │ a b c") + }) + + it("prints graph for components with dependencies at multiple stages", () => { + const components: Array = [ + { name: "DB", id: "db", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: [] }, + { name: "API", id: "api", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["db"] }, + { name: "Cache", id: "cache", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["db"] } + ] + printDependencyGraph(components) + expect(logOutput[0]).to.equal("Dependency Graph") + expect(logOutput).to.include(" Stage 1 │ db") + expect(logOutput).to.include(" Stage 2 │ api cache") + expect(logOutput).to.include(" db ──▶ api") + expect(logOutput).to.include(" db ──▶ cache") + }) + + it("prints graph for chained dependencies", () => { + const components: Array = [ + { name: "A", id: "a", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: [] }, + { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"] }, + { name: "C", id: "c", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["b"] } + ] + printDependencyGraph(components) + expect(logOutput[0]).to.equal("Dependency Graph") + expect(logOutput).to.include(" Stage 1 │ a") + expect(logOutput).to.include(" Stage 2 │ b") + expect(logOutput).to.include(" Stage 3 │ c") + expect(logOutput).to.include(" a ──▶ b") + expect(logOutput).to.include(" b ──▶ c") + }) + + it("annotates parallel components with ⚡ and prints a legend", () => { + const components: Array = [ + { name: "A", id: "a", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } }, + { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"], parallel: true }, + { name: "C", id: "c", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"], parallel: true } + ] + printDependencyGraph(components) + expect(logOutput[0]).to.equal("Dependency Graph") + expect(logOutput).to.include(" Stage 1 │ a") + expect(logOutput).to.include(" Stage 2 │ b ⚡ c ⚡") + expect(logOutput).to.include(" a ──▶ b") + expect(logOutput).to.include(" a ──▶ c") + expect(logOutput).to.include(" ⚡ = deployed concurrently within stage") + }) + }) }) diff --git a/k8s-deployer/test/test-suite-handler.spec.ts b/k8s-deployer/test/test-suite-handler.spec.ts index 134e5fc..3ca9d36 100644 --- a/k8s-deployer/test/test-suite-handler.spec.ts +++ b/k8s-deployer/test/test-suite-handler.spec.ts @@ -359,7 +359,8 @@ describe("Deployment happy path", async () => { .returns( { ok: true, - json: async () => { return new webapi.ReportResponse("session-2", testSuiteId, webapi.TestStatus.COMPLETED, report) } + // Serialize then parse to simulate real HTTP JSON: Date objects become ISO strings + json: async () => JSON.parse(JSON.stringify(new webapi.ReportResponse("session-2", testSuiteId, webapi.TestStatus.COMPLETED, report))) } ) From de78384a4b05824a938666733ebc987bfbfaeb35 Mon Sep 17 00:00:00 2001 From: "James Charters james.charters@kindredgroup.com" Date: Tue, 10 Mar 2026 11:20:17 +1100 Subject: [PATCH 04/12] feat: [KSBP-101776] - Parallel deployment of test app --- k8s-deployer/src/test-suite-handler.ts | 90 +++++++++++++------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/k8s-deployer/src/test-suite-handler.ts b/k8s-deployer/src/test-suite-handler.ts index 8f8a7f2..1716ecc 100644 --- a/k8s-deployer/src/test-suite-handler.ts +++ b/k8s-deployer/src/test-suite-handler.ts @@ -44,54 +44,11 @@ const deployGraph = async (config: Config, workspace: string, testSuiteId: strin // Dependencies are already validated in main(), so it's safe to directly sort here. const { sortedComponents, levels } = topologicalSort(graph.components) - logger.info("") - logger.info("Dependency Resolution for %s:", testSuiteId) - logger.info("Deployment order: %s", sortedComponents.map(c => c.id).join(" → ")) - logger.info("") - logger.info("") logger.info("Dependency Graph for %s:", testSuiteId) printDependencyGraph(graph.components) logger.info("") - // Deploy components level by level. Within each level, components with parallel:true are deployed concurrently. - const deployments: Array = [] - let componentIndex = 0 - for (const level of levels) { - const parallelGroup = level.filter(c => c.parallel === true) - const sequentialGroup = level.filter(c => c.parallel !== true) - - // Deploy all parallel-flagged components in this level concurrently - if (parallelGroup.length > 0) { - logger.info("") - logger.info("Deploying %d component(s) in parallel for suite \"%s\": %s", parallelGroup.length, testSuiteId, parallelGroup.map(c => c.id).join(", ")) - const parallelResults = await Promise.all( - parallelGroup.map(async componentSpec => { - const idx = ++componentIndex - logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) - const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) - logger.info("Graph component \"%s\" for suite \"%s\" deployed.", componentSpec.name, testSuiteId) - return new DeployedComponent(commitSha, componentSpec) - }) - ) - deployments.push(...parallelResults) - } - - // Deploy sequential components one by one - for (const componentSpec of sequentialGroup) { - const idx = ++componentIndex - logger.info("") - logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) - logger.info("") - const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) - deployments.push(new DeployedComponent(commitSha, componentSpec)) - } - } - logger.info("") - - logger.info("%s Deploying test app \"%s\" for suite \"%s\" %s", LOG_SEPARATOR_LINE, graph.testApp.name, testSuiteId, LOG_SEPARATOR_LINE) - logger.info("") - if (testAppDirForRemoteTestSuite) { // When suite is remote its pitfile is sitting within test app itself. // We just downloaded pitfile from remote location into workspace @@ -101,10 +58,53 @@ const deployGraph = async (config: Config, workspace: string, testSuiteId: strin ) graph.testApp.location.path = testAppDirForRemoteTestSuite } + + // Deploy components level by level. Within each level, components with parallel:true are deployed concurrently. + const deployments: Array = [] + let componentIndex = 0 + const deployComponentsPromise = (async () => { + for (const level of levels) { + const parallelGroup = level.filter(c => c.parallel === true) + const sequentialGroup = level.filter(c => c.parallel !== true) + + // Deploy all parallel-flagged components in this level concurrently + if (parallelGroup.length > 0) { + logger.info("") + logger.info("Deploying %d component(s) in parallel for suite \"%s\": %s", parallelGroup.length, testSuiteId, parallelGroup.map(c => c.id).join(", ")) + const parallelResults = await Promise.all( + parallelGroup.map(async componentSpec => { + const idx = ++componentIndex + logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) + const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) + logger.info("Graph component \"%s\" for suite \"%s\" deployed.", componentSpec.name, testSuiteId) + return new DeployedComponent(commitSha, componentSpec) + }) + ) + deployments.push(...parallelResults) + } + + // Deploy sequential components one by one + for (const componentSpec of sequentialGroup) { + const idx = ++componentIndex + logger.info("") + logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) + logger.info("") + const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) + deployments.push(new DeployedComponent(commitSha, componentSpec)) + } + } + })() + + // Deploy test app concurrently with the component levels; deployGraph does not return until both are done. + logger.info("%s Deploying test app \"%s\" for suite \"%s\" %s", LOG_SEPARATOR_LINE, graph.testApp.name, testSuiteId, LOG_SEPARATOR_LINE) const params = [ testSuiteId ] - const testAppCommitSha = await Deployer.deployComponent(config, workspace, graph.testApp, namespace, params) + const deployTestAppPromise = Deployer.deployComponent(config, workspace, graph.testApp, namespace, params) + .then(commitSha => new DeployedComponent(commitSha, graph.testApp)) + + const [ , testAppDeployedComponent] = await Promise.all([deployComponentsPromise, deployTestAppPromise]) + logger.info("") - return new GraphDeploymentResult(deployments, new DeployedComponent(testAppCommitSha, graph.testApp)) + return new GraphDeploymentResult(deployments, testAppDeployedComponent) } const downloadPitFile = async (testSuite: Schema.TestSuite, destination: string): Promise => { From b57d1ebba8fd991e1044d3210ab4339d4461203d Mon Sep 17 00:00:00 2001 From: "James Charters james.charters@kindredgroup.com" Date: Tue, 10 Mar 2026 14:07:31 +1100 Subject: [PATCH 05/12] feat: [KSBP-101776] - Enhance parallel deployment logic and testing --- k8s-deployer/src/test-suite-handler.ts | 62 ++--- k8s-deployer/test/test-suite-handler.spec.ts | 228 +++++++++++++++++-- 2 files changed, 246 insertions(+), 44 deletions(-) diff --git a/k8s-deployer/src/test-suite-handler.ts b/k8s-deployer/src/test-suite-handler.ts index 1716ecc..382ffa4 100644 --- a/k8s-deployer/src/test-suite-handler.ts +++ b/k8s-deployer/src/test-suite-handler.ts @@ -40,7 +40,7 @@ export const generatePrefixByDate = (date: Date, env: string): Prefix => { * 1. all components in the graph in topological order, deploying parallel-flagged components concurrently within each level * 2. test app for the graph. */ -const deployGraph = async (config: Config, workspace: string, testSuiteId: string, graph: Schema.Graph, namespace: Namespace, testAppDirForRemoteTestSuite?: string): Promise => { +export const deployGraph = async (config: Config, workspace: string, testSuiteId: string, graph: Schema.Graph, namespace: Namespace, testAppDirForRemoteTestSuite?: string): Promise => { // Dependencies are already validated in main(), so it's safe to directly sort here. const { sortedComponents, levels } = topologicalSort(graph.components) @@ -59,7 +59,10 @@ const deployGraph = async (config: Config, workspace: string, testSuiteId: strin graph.testApp.location.path = testAppDirForRemoteTestSuite } - // Deploy components level by level. Within each level, components with parallel:true are deployed concurrently. + // Deploy components level by level. + // Within each level, parallel-flagged components run concurrently with each other AND with the + // sequential chain — both groups start at the same time and the level only advances once both + // are done. const deployments: Array = [] let componentIndex = 0 const deployComponentsPromise = (async () => { @@ -67,31 +70,36 @@ const deployGraph = async (config: Config, workspace: string, testSuiteId: strin const parallelGroup = level.filter(c => c.parallel === true) const sequentialGroup = level.filter(c => c.parallel !== true) - // Deploy all parallel-flagged components in this level concurrently - if (parallelGroup.length > 0) { - logger.info("") - logger.info("Deploying %d component(s) in parallel for suite \"%s\": %s", parallelGroup.length, testSuiteId, parallelGroup.map(c => c.id).join(", ")) - const parallelResults = await Promise.all( - parallelGroup.map(async componentSpec => { - const idx = ++componentIndex - logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) - const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) - logger.info("Graph component \"%s\" for suite \"%s\" deployed.", componentSpec.name, testSuiteId) - return new DeployedComponent(commitSha, componentSpec) - }) - ) - deployments.push(...parallelResults) - } - - // Deploy sequential components one by one - for (const componentSpec of sequentialGroup) { - const idx = ++componentIndex - logger.info("") - logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) - logger.info("") - const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) - deployments.push(new DeployedComponent(commitSha, componentSpec)) - } + // Fire both groups concurrently; await both before moving to the next level. + const parallelChain = parallelGroup.length > 0 + ? (async () => { + logger.info("") + logger.info("Deploying %d component(s) in parallel for suite \"%s\": %s", parallelGroup.length, testSuiteId, parallelGroup.map(c => c.id).join(", ")) + const results = await Promise.all( + parallelGroup.map(async componentSpec => { + const idx = ++componentIndex + logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) + const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) + logger.info("Graph component \"%s\" for suite \"%s\" deployed.", componentSpec.name, testSuiteId) + return new DeployedComponent(commitSha, componentSpec) + }) + ) + deployments.push(...results) + })() + : Promise.resolve() + + const sequentialChain = (async () => { + for (const componentSpec of sequentialGroup) { + const idx = ++componentIndex + logger.info("") + logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) + logger.info("") + const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) + deployments.push(new DeployedComponent(commitSha, componentSpec)) + } + })() + + await Promise.all([parallelChain, sequentialChain]) } })() diff --git a/k8s-deployer/test/test-suite-handler.spec.ts b/k8s-deployer/test/test-suite-handler.spec.ts index 3ca9d36..d1acc5f 100644 --- a/k8s-deployer/test/test-suite-handler.spec.ts +++ b/k8s-deployer/test/test-suite-handler.spec.ts @@ -220,10 +220,11 @@ describe("Deployment happy path", async () => { chai.expect(fsAccessStubs.getCall(2).calledWith("lock-manager/deployment/pit/is-deployment-ready.sh")).be.true chai.expect(fsAccessStubs.getCall(3).calledWith("23456_t1/comp-1")).be.false chai.expect(fsAccessStubs.getCall(3).calledWith("./comp-1")).be.true - chai.expect(fsAccessStubs.getCall(4).calledWith("comp-1/deployment/pit/deploy.sh")).be.true - chai.expect(fsAccessStubs.getCall(5).calledWith("comp-1/deployment/pit/is-deployment-ready.sh")).be.true - chai.expect(fsAccessStubs.getCall(6).calledWith("./comp-1-test-app")).be.true - chai.expect(fsAccessStubs.getCall(7).calledWith("comp-1-test-app/deployment/pit/deploy.sh")).be.true + // testApp deploys concurrently with comp-1, so its directory check interleaves + chai.expect(fsAccessStubs.getCall(4).calledWith("./comp-1-test-app")).be.true + chai.expect(fsAccessStubs.getCall(5).calledWith("comp-1/deployment/pit/deploy.sh")).be.true + chai.expect(fsAccessStubs.getCall(6).calledWith("comp-1-test-app/deployment/pit/deploy.sh")).be.true + chai.expect(fsAccessStubs.getCall(7).calledWith("comp-1/deployment/pit/is-deployment-ready.sh")).be.true chai.expect(fsAccessStubs.getCall(8).calledWith("comp-1-test-app/deployment/pit/is-deployment-ready.sh")).be.true // check shell calls @@ -258,25 +259,26 @@ describe("Deployment happy path", async () => { { homeDir: "lock-manager" }) ).be.true + // testApp deploys concurrently with comp-1, so git log calls interleave chai.expect(execStub.getCall(5).calledWith(`cd comp-1 && git log --pretty=format:"%h" -1`)).be.true - chai.expect(execStub.getCall(6).calledWith( + chai.expect(execStub.getCall(6).calledWith(`cd comp-1-test-app && git log --pretty=format:"%h" -1`)).be.true + + chai.expect(execStub.getCall(7).calledWith( "deployment/pit/deploy.sh nsChild", { homeDir: "comp-1", logFileName: "12345_t1/logs/deploy-nsChild-comp-1.log", tailTarget: sinon.match.any }) ).be.true - chai.expect(execStub.getCall(7).calledWith( - "deployment/pit/is-deployment-ready.sh nsChild", - { homeDir: "comp-1" } - )).be.true - - chai.expect(execStub.getCall(8).calledWith(`cd comp-1-test-app && git log --pretty=format:"%h" -1`)).be.true - - chai.expect(execStub.getCall(9).calledWith( + chai.expect(execStub.getCall(8).calledWith( "deployment/pit/deploy.sh nsChild t1", { homeDir: "comp-1-test-app", logFileName: `12345_t1/logs/deploy-nsChild-comp-1-test-app.log`, tailTarget: sinon.match.any }) ).be.true + chai.expect(execStub.getCall(9).calledWith( + "deployment/pit/is-deployment-ready.sh nsChild", + { homeDir: "comp-1" }) + ).be.true + chai.expect(execStub.getCall(10).calledWith( "deployment/pit/is-deployment-ready.sh nsChild", { homeDir: "comp-1-test-app" }) @@ -435,10 +437,202 @@ describe("Deployment happy path", async () => { chai.expect(fsAccessStubs.getCall(2).calledWith("lock-manager/deployment/pit/is-deployment-ready.sh")).be.true chai.expect(fsAccessStubs.getCall(3).calledWith("23456_t2/comp-1")).be.false chai.expect(fsAccessStubs.getCall(3).calledWith("./comp-1")).be.true - chai.expect(fsAccessStubs.getCall(4).calledWith("./deployment/pit/deploy.sh")).be.true - chai.expect(fsAccessStubs.getCall(5).calledWith("./deployment/pit/is-deployment-ready.sh")).be.true - chai.expect(fsAccessStubs.getCall(6).calledWith("./comp-1-test-app")).be.true - chai.expect(fsAccessStubs.getCall(7).calledWith("comp-1-test-app/deployment/pit/deploy.sh")).be.true + // testApp deploys concurrently with comp-1, so its directory check interleaves + chai.expect(fsAccessStubs.getCall(4).calledWith("./comp-1-test-app")).be.true + chai.expect(fsAccessStubs.getCall(5).calledWith("./deployment/pit/deploy.sh")).be.true + chai.expect(fsAccessStubs.getCall(6).calledWith("comp-1-test-app/deployment/pit/deploy.sh")).be.true + chai.expect(fsAccessStubs.getCall(7).calledWith("./deployment/pit/is-deployment-ready.sh")).be.true chai.expect(fsAccessStubs.getCall(8).calledWith("comp-1-test-app/deployment/pit/is-deployment-ready.sh")).be.true }) }) + +describe("deployGraph - deployment ordering and concurrency", async () => { + const config: Config = { + commitSha: "test-sha", + workspace: "/tmp", + clusterUrl: "http://localhost", + parentNamespace: "ns", + subNamespacePrefix: DEFAULT_SUB_NAMESPACE_PREFIX, + subNamespaceGeneratorType: SUB_NAMESPACE_GENERATOR_TYPE_DATE, + pitfile: "pitfile.yml", + namespaceTimeoutSeconds: 2, + report: {}, + targetEnv: "test", + testStatusPollFrequencyMs: 100, + testTimeoutMs: 1000, + deployCheckFrequencyMs: 100, + params: new Map(), + useMockLockManager: false, + servicesAreExposedViaProxy: false, + lockManagerApiRetries: 1, + enableCleanups: false, + testRunnerAppPort: 80 + } + const namespace = "test-ns" + const testSuiteId = "suite-1" + const workspace = "test-workspace" + + type Gate = { resolve: () => void; promise: Promise } + const makeGate = (): Gate => { + let resolve!: () => void + const promise = new Promise(r => { resolve = r }) + return { resolve, promise } + } + + const makeSpec = (id: string, opts: { parallel?: boolean; dependsOn?: string[] } = {}) => ({ + name: `${id}-name`, id, + location: { type: LocationType.Local }, + deploy: { command: `${id}/deploy.sh`, statusCheck: { command: `${id}/ready.sh` } }, + undeploy: { command: `${id}/undeploy.sh` }, + ...opts + }) + + const loadWithStub = async (deployStub: sinon.SinonStub) => + esmock( + "../src/test-suite-handler.js", + { "../src/deployer.js": { deployComponent: deployStub } }, + { "../src/logger.js": { logger: { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} } } } + ) + + it("returns GraphDeploymentResult with all deployed components", async () => { + const deployStub = sinon.stub().callsFake(async (_cfg, _ws, spec) => `sha-${spec.id}`) + const SuiteHandler = await loadWithStub(deployStub) + const graph = { + testApp: makeSpec("test-app"), + components: [makeSpec("comp-a"), makeSpec("comp-b")] + } + const result = await SuiteHandler.deployGraph(config, workspace, testSuiteId, graph, namespace) + chai.expect(result.components).to.have.length(2) + chai.expect(result.components.map((d: any) => d.component.id)).to.include.members(["comp-a", "comp-b"]) + chai.expect(result.testApp.component.id).to.equal("test-app") + chai.expect(result.testApp.commitSha).to.equal("sha-test-app") + }) + + it("deploys parallel-flagged components concurrently", async () => { + const gates: Record = { B: makeGate(), C: makeGate(), testApp: makeGate() } + const started: string[] = [] + const completed: string[] = [] + const deployStub = sinon.stub().callsFake(async (_cfg, _ws, spec) => { + started.push(spec.id) + await gates[spec.id].promise + completed.push(spec.id) + return `sha-${spec.id}` + }) + const SuiteHandler = await loadWithStub(deployStub) + const graph = { + testApp: makeSpec("testApp"), + components: [makeSpec("B", { parallel: true }), makeSpec("C", { parallel: true })] + } + const deployPromise = SuiteHandler.deployGraph(config, workspace, testSuiteId, graph, namespace) + // Both B and C should have started before either completes + chai.expect(started).to.include("B") + chai.expect(started).to.include("C") + chai.expect(completed).to.not.include("B") + chai.expect(completed).to.not.include("C") + gates["B"].resolve() + gates["C"].resolve() + gates["testApp"].resolve() + await deployPromise + chai.expect(completed).to.include.members(["B", "C"]) + }) + + it("deploys mixed parallel/sequential at same level concurrently", async () => { + const gates: Record = { B: makeGate(), C: makeGate(), testApp: makeGate() } + const started: string[] = [] + const completed: string[] = [] + const deployStub = sinon.stub().callsFake(async (_cfg, _ws, spec) => { + started.push(spec.id) + await gates[spec.id].promise + completed.push(spec.id) + return `sha-${spec.id}` + }) + const SuiteHandler = await loadWithStub(deployStub) + // B is parallel:true, C has no parallel flag — same dependency level, so neither waits on the other + const graph = { + testApp: makeSpec("testApp"), + components: [makeSpec("B", { parallel: true }), makeSpec("C")] + } + const deployPromise = SuiteHandler.deployGraph(config, workspace, testSuiteId, graph, namespace) + // B (parallelGroup) and C (sequentialGroup) fire their respective chains simultaneously — + // both should be in-flight before either completes + chai.expect(started).to.include("B") + chai.expect(started).to.include("C") + chai.expect(completed).to.not.include("B") + chai.expect(completed).to.not.include("C") + gates["B"].resolve() + gates["C"].resolve() + gates["testApp"].resolve() + await deployPromise + chai.expect(completed).to.include.members(["B", "C"]) + }) + + it("second dependency level does not start until first level completes", async () => { + // Topology: A → B(parallel), A → C, B → D, C → D + const gates: Record = { + A: makeGate(), B: makeGate(), C: makeGate(), D: makeGate(), testApp: makeGate() + } + const started: string[] = [] + const deployStub = sinon.stub().callsFake(async (_cfg, _ws, spec) => { + started.push(spec.id) + await gates[spec.id].promise + return `sha-${spec.id}` + }) + const SuiteHandler = await loadWithStub(deployStub) + const graph = { + testApp: makeSpec("testApp"), + components: [ + makeSpec("A"), + makeSpec("B", { parallel: true, dependsOn: ["A"] }), + makeSpec("C", { dependsOn: ["A"] }), + makeSpec("D", { dependsOn: ["B", "C"] }) + ] + } + const deployPromise = SuiteHandler.deployGraph(config, workspace, testSuiteId, graph, namespace) + + // Level 1: only A started; testApp is already in-flight concurrently + chai.expect(started).to.include("A") + chai.expect(started).to.include("testApp") + chai.expect(started).to.not.include("B") + chai.expect(started).to.not.include("C") + chai.expect(started).to.not.include("D") + + // Resolve A — level 2 (B and C) should start + gates["A"].resolve() + await new Promise(r => setTimeout(r, 0)) + chai.expect(started).to.include("B") + chai.expect(started).to.include("C") + chai.expect(started).to.not.include("D") + + // Resolve B and C — level 3 (D) should start + gates["B"].resolve() + gates["C"].resolve() + await new Promise(r => setTimeout(r, 0)) + chai.expect(started).to.include("D") + + gates["D"].resolve() + gates["testApp"].resolve() + await deployPromise + }) + + it("deploys test app concurrently with the component chain", async () => { + const gates: Record = { A: makeGate(), testApp: makeGate() } + const started: string[] = [] + const deployStub = sinon.stub().callsFake(async (_cfg, _ws, spec) => { + started.push(spec.id) + await gates[spec.id].promise + return `sha-${spec.id}` + }) + const SuiteHandler = await loadWithStub(deployStub) + const graph = { + testApp: makeSpec("testApp"), + components: [makeSpec("A")] + } + const deployPromise = SuiteHandler.deployGraph(config, workspace, testSuiteId, graph, namespace) + // A and testApp should both be in-flight before either completes + chai.expect(started).to.include("A") + chai.expect(started).to.include("testApp") + gates["A"].resolve() + gates["testApp"].resolve() + await deployPromise + }) +}) From ddbab055a914b30db05712112823495c508b3abb Mon Sep 17 00:00:00 2001 From: "James Charters james.charters@kindredgroup.com" Date: Tue, 10 Mar 2026 15:06:40 +1100 Subject: [PATCH 06/12] feat: [KSBP-101776] - Add flag for parallelTestApp --- k8s-deployer/src/pitfile/schema-v1.ts | 6 ++++++ k8s-deployer/src/test-suite-handler.ts | 18 ++++++++++++++---- k8s-deployer/test/test-suite-handler.spec.ts | 6 ++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/k8s-deployer/src/pitfile/schema-v1.ts b/k8s-deployer/src/pitfile/schema-v1.ts index b313cea..294de72 100644 --- a/k8s-deployer/src/pitfile/schema-v1.ts +++ b/k8s-deployer/src/pitfile/schema-v1.ts @@ -67,6 +67,12 @@ export class DeployableComponent { export class Graph { testApp: DeployableComponent components: Array + /** + * When true, the test app is deployed concurrently with the component chain. + * Only enable this if the test app can start independently of the components. + * Defaults to false (sequential: test app starts after all components are ready). + */ + parallelTestApp?: boolean } export class Deployment { diff --git a/k8s-deployer/src/test-suite-handler.ts b/k8s-deployer/src/test-suite-handler.ts index 382ffa4..4e2dc23 100644 --- a/k8s-deployer/src/test-suite-handler.ts +++ b/k8s-deployer/src/test-suite-handler.ts @@ -103,13 +103,23 @@ export const deployGraph = async (config: Config, workspace: string, testSuiteId } })() - // Deploy test app concurrently with the component levels; deployGraph does not return until both are done. logger.info("%s Deploying test app \"%s\" for suite \"%s\" %s", LOG_SEPARATOR_LINE, graph.testApp.name, testSuiteId, LOG_SEPARATOR_LINE) const params = [ testSuiteId ] - const deployTestAppPromise = Deployer.deployComponent(config, workspace, graph.testApp, namespace, params) - .then(commitSha => new DeployedComponent(commitSha, graph.testApp)) - const [ , testAppDeployedComponent] = await Promise.all([deployComponentsPromise, deployTestAppPromise]) + let testAppDeployedComponent: DeployedComponent + if (graph.parallelTestApp === true) { + // Deploy test app concurrently with the component levels (opt-in). + // Only use this when the test app can start independently of the components. + const deployTestAppPromise = Deployer.deployComponent(config, workspace, graph.testApp, namespace, params) + .then(commitSha => new DeployedComponent(commitSha, graph.testApp)) + const [ , resolved] = await Promise.all([deployComponentsPromise, deployTestAppPromise]) + testAppDeployedComponent = resolved + } else { + // Default: wait for all components to be ready before deploying the test app. + await deployComponentsPromise + const commitSha = await Deployer.deployComponent(config, workspace, graph.testApp, namespace, params) + testAppDeployedComponent = new DeployedComponent(commitSha, graph.testApp) + } logger.info("") return new GraphDeploymentResult(deployments, testAppDeployedComponent) diff --git a/k8s-deployer/test/test-suite-handler.spec.ts b/k8s-deployer/test/test-suite-handler.spec.ts index d1acc5f..bab2d8c 100644 --- a/k8s-deployer/test/test-suite-handler.spec.ts +++ b/k8s-deployer/test/test-suite-handler.spec.ts @@ -57,6 +57,7 @@ describe("Deployment happy path", async () => { location: { type: LocationType.Local }, deployment: { graph: { + parallelTestApp: true, testApp: { name: "comp-1-test-app-name", id: "comp-1-test-app", @@ -498,6 +499,7 @@ describe("deployGraph - deployment ordering and concurrency", async () => { const deployStub = sinon.stub().callsFake(async (_cfg, _ws, spec) => `sha-${spec.id}`) const SuiteHandler = await loadWithStub(deployStub) const graph = { + parallelTestApp: true, testApp: makeSpec("test-app"), components: [makeSpec("comp-a"), makeSpec("comp-b")] } @@ -520,6 +522,7 @@ describe("deployGraph - deployment ordering and concurrency", async () => { }) const SuiteHandler = await loadWithStub(deployStub) const graph = { + parallelTestApp: true, testApp: makeSpec("testApp"), components: [makeSpec("B", { parallel: true }), makeSpec("C", { parallel: true })] } @@ -549,6 +552,7 @@ describe("deployGraph - deployment ordering and concurrency", async () => { const SuiteHandler = await loadWithStub(deployStub) // B is parallel:true, C has no parallel flag — same dependency level, so neither waits on the other const graph = { + parallelTestApp: true, testApp: makeSpec("testApp"), components: [makeSpec("B", { parallel: true }), makeSpec("C")] } @@ -579,6 +583,7 @@ describe("deployGraph - deployment ordering and concurrency", async () => { }) const SuiteHandler = await loadWithStub(deployStub) const graph = { + parallelTestApp: true, testApp: makeSpec("testApp"), components: [ makeSpec("A"), @@ -624,6 +629,7 @@ describe("deployGraph - deployment ordering and concurrency", async () => { }) const SuiteHandler = await loadWithStub(deployStub) const graph = { + parallelTestApp: true, testApp: makeSpec("testApp"), components: [makeSpec("A")] } From f720f65c09ad91e5d5d4fec5274d79092747bd01 Mon Sep 17 00:00:00 2001 From: "James Charters james.charters@kindredgroup.com" Date: Tue, 10 Mar 2026 15:17:27 +1100 Subject: [PATCH 07/12] feat: [KSBP-101776] - Update emoji --- k8s-deployer/src/dependency-resolver.ts | 4 ++-- k8s-deployer/src/pitfile/schema-v1.ts | 6 ------ k8s-deployer/src/test-suite-handler.ts | 2 +- k8s-deployer/test/dependency-resolver.spec.ts | 6 +++--- k8s-deployer/test/test-suite-handler.spec.ts | 19 +++++++------------ 5 files changed, 13 insertions(+), 24 deletions(-) diff --git a/k8s-deployer/src/dependency-resolver.ts b/k8s-deployer/src/dependency-resolver.ts index 78990cb..a142423 100644 --- a/k8s-deployer/src/dependency-resolver.ts +++ b/k8s-deployer/src/dependency-resolver.ts @@ -221,7 +221,7 @@ export const printDependencyGraph = (components: Array - console.log(` Stage ${idx + 1} │ ${level.map(c => c.parallel ? `${c.id} ⚡` : c.id).join(" ")}`) + console.log(` Stage ${idx + 1} │ ${level.map(c => c.parallel ? `${c.id} 🔀` : c.id).join(" ")}`) ) if (edges.length > 0) { console.log(sep) @@ -229,7 +229,7 @@ export const printDependencyGraph = (components: Array c.parallel)) { console.log(sep) - console.log(" ⚡ = deployed concurrently within stage") + console.log(" 🔀 = concurrent deployment") } console.log(sep) } diff --git a/k8s-deployer/src/pitfile/schema-v1.ts b/k8s-deployer/src/pitfile/schema-v1.ts index 294de72..b313cea 100644 --- a/k8s-deployer/src/pitfile/schema-v1.ts +++ b/k8s-deployer/src/pitfile/schema-v1.ts @@ -67,12 +67,6 @@ export class DeployableComponent { export class Graph { testApp: DeployableComponent components: Array - /** - * When true, the test app is deployed concurrently with the component chain. - * Only enable this if the test app can start independently of the components. - * Defaults to false (sequential: test app starts after all components are ready). - */ - parallelTestApp?: boolean } export class Deployment { diff --git a/k8s-deployer/src/test-suite-handler.ts b/k8s-deployer/src/test-suite-handler.ts index 4e2dc23..624cb36 100644 --- a/k8s-deployer/src/test-suite-handler.ts +++ b/k8s-deployer/src/test-suite-handler.ts @@ -107,7 +107,7 @@ export const deployGraph = async (config: Config, workspace: string, testSuiteId const params = [ testSuiteId ] let testAppDeployedComponent: DeployedComponent - if (graph.parallelTestApp === true) { + if (graph.testApp.parallel === true) { // Deploy test app concurrently with the component levels (opt-in). // Only use this when the test app can start independently of the components. const deployTestAppPromise = Deployer.deployComponent(config, workspace, graph.testApp, namespace, params) diff --git a/k8s-deployer/test/dependency-resolver.spec.ts b/k8s-deployer/test/dependency-resolver.spec.ts index 86574a8..0216765 100644 --- a/k8s-deployer/test/dependency-resolver.spec.ts +++ b/k8s-deployer/test/dependency-resolver.spec.ts @@ -444,7 +444,7 @@ describe("Dependency Resolver", () => { expect(logOutput).to.include(" b ──▶ c") }) - it("annotates parallel components with ⚡ and prints a legend", () => { + it("annotates parallel components with 🔀 and prints a legend", () => { const components: Array = [ { name: "A", id: "a", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } }, { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"], parallel: true }, @@ -453,10 +453,10 @@ describe("Dependency Resolver", () => { printDependencyGraph(components) expect(logOutput[0]).to.equal("Dependency Graph") expect(logOutput).to.include(" Stage 1 │ a") - expect(logOutput).to.include(" Stage 2 │ b ⚡ c ⚡") + expect(logOutput).to.include(" Stage 2 │ b 🔀 c 🔀") expect(logOutput).to.include(" a ──▶ b") expect(logOutput).to.include(" a ──▶ c") - expect(logOutput).to.include(" ⚡ = deployed concurrently within stage") + expect(logOutput).to.include(" 🔀 = concurrent deployment") }) }) }) diff --git a/k8s-deployer/test/test-suite-handler.spec.ts b/k8s-deployer/test/test-suite-handler.spec.ts index bab2d8c..92472db 100644 --- a/k8s-deployer/test/test-suite-handler.spec.ts +++ b/k8s-deployer/test/test-suite-handler.spec.ts @@ -57,7 +57,6 @@ describe("Deployment happy path", async () => { location: { type: LocationType.Local }, deployment: { graph: { - parallelTestApp: true, testApp: { name: "comp-1-test-app-name", id: "comp-1-test-app", @@ -68,7 +67,8 @@ describe("Deployment happy path", async () => { }, logTailing: { enabled: true - } + }, + parallel: true }, components: [ { @@ -499,8 +499,7 @@ describe("deployGraph - deployment ordering and concurrency", async () => { const deployStub = sinon.stub().callsFake(async (_cfg, _ws, spec) => `sha-${spec.id}`) const SuiteHandler = await loadWithStub(deployStub) const graph = { - parallelTestApp: true, - testApp: makeSpec("test-app"), + testApp: makeSpec("test-app", { parallel: true }), components: [makeSpec("comp-a"), makeSpec("comp-b")] } const result = await SuiteHandler.deployGraph(config, workspace, testSuiteId, graph, namespace) @@ -522,8 +521,7 @@ describe("deployGraph - deployment ordering and concurrency", async () => { }) const SuiteHandler = await loadWithStub(deployStub) const graph = { - parallelTestApp: true, - testApp: makeSpec("testApp"), + testApp: makeSpec("testApp", { parallel: true }), components: [makeSpec("B", { parallel: true }), makeSpec("C", { parallel: true })] } const deployPromise = SuiteHandler.deployGraph(config, workspace, testSuiteId, graph, namespace) @@ -552,8 +550,7 @@ describe("deployGraph - deployment ordering and concurrency", async () => { const SuiteHandler = await loadWithStub(deployStub) // B is parallel:true, C has no parallel flag — same dependency level, so neither waits on the other const graph = { - parallelTestApp: true, - testApp: makeSpec("testApp"), + testApp: makeSpec("testApp", { parallel: true }), components: [makeSpec("B", { parallel: true }), makeSpec("C")] } const deployPromise = SuiteHandler.deployGraph(config, workspace, testSuiteId, graph, namespace) @@ -583,8 +580,7 @@ describe("deployGraph - deployment ordering and concurrency", async () => { }) const SuiteHandler = await loadWithStub(deployStub) const graph = { - parallelTestApp: true, - testApp: makeSpec("testApp"), + testApp: makeSpec("testApp", { parallel: true }), components: [ makeSpec("A"), makeSpec("B", { parallel: true, dependsOn: ["A"] }), @@ -629,8 +625,7 @@ describe("deployGraph - deployment ordering and concurrency", async () => { }) const SuiteHandler = await loadWithStub(deployStub) const graph = { - parallelTestApp: true, - testApp: makeSpec("testApp"), + testApp: makeSpec("testApp", { parallel: true }), components: [makeSpec("A")] } const deployPromise = SuiteHandler.deployGraph(config, workspace, testSuiteId, graph, namespace) From 98701e2e5e1df39390b12eaaa487498899dc1b31 Mon Sep 17 00:00:00 2001 From: "James Charters james.charters@kindredgroup.com" Date: Tue, 10 Mar 2026 15:29:26 +1100 Subject: [PATCH 08/12] feat: [KSBP-101776] - Additional work on parallelisation of test app with other components --- k8s-deployer/src/dependency-resolver.ts | 34 +++++++++++++++---- k8s-deployer/src/test-suite-handler.ts | 2 +- k8s-deployer/test/dependency-resolver.spec.ts | 32 ++++++++++++++--- 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/k8s-deployer/src/dependency-resolver.ts b/k8s-deployer/src/dependency-resolver.ts index a142423..15a2c89 100644 --- a/k8s-deployer/src/dependency-resolver.ts +++ b/k8s-deployer/src/dependency-resolver.ts @@ -210,24 +210,44 @@ const reconstructCyclePath = (startId: string, parent: Map): Arr } /** - * Print the dependency graph in a visually grouped format. - * Components that can be deployed in parallel are shown together and annotated. + * Print the full deployment graph including testApp placement. + * + * - Components are shown in topological stages. + * - If testApp.parallel === true it is shown in a separate concurrent section + * (it runs alongside all component stages). + * - Otherwise testApp is shown as the final sequential stage after all components. */ -export const printDependencyGraph = (components: Array): void => { +export const printDependencyGraph = (graph: Schema.Graph): void => { + const { components, testApp } = graph const { levels } = topologicalSort(components) const sep = "─".repeat(40) + const label = (c: Schema.DeployableComponent) => c.parallel ? `${c.id} 🔀` : c.id const edges = components.flatMap(c => (c.dependsOn ?? []).map(dep => ` ${dep} ──▶ ${c.id}`)) + const anyConcurrent = components.some(c => c.parallel) || testApp.parallel === true console.log("Dependency Graph") console.log(sep) - levels.forEach((level, idx) => - console.log(` Stage ${idx + 1} │ ${level.map(c => c.parallel ? `${c.id} 🔀` : c.id).join(" ")}`) - ) + + if (testApp.parallel === true) { + // testApp runs concurrently with the entire component chain — show it in a separate section + levels.forEach((level, idx) => + console.log(` Stage ${idx + 1} │ ${level.map(label).join(" ")}`) + ) + console.log(sep) + console.log(` ${label(testApp)} (concurrent with component stages)`) + } else { + // testApp runs after all components — append it as the final stage + levels.forEach((level, idx) => + console.log(` Stage ${idx + 1} │ ${level.map(label).join(" ")}`) + ) + console.log(` Stage ${levels.length + 1} │ ${testApp.id}`) + } + if (edges.length > 0) { console.log(sep) edges.forEach(e => console.log(e)) } - if (components.some(c => c.parallel)) { + if (anyConcurrent) { console.log(sep) console.log(" 🔀 = concurrent deployment") } diff --git a/k8s-deployer/src/test-suite-handler.ts b/k8s-deployer/src/test-suite-handler.ts index 624cb36..05733b4 100644 --- a/k8s-deployer/src/test-suite-handler.ts +++ b/k8s-deployer/src/test-suite-handler.ts @@ -46,7 +46,7 @@ export const deployGraph = async (config: Config, workspace: string, testSuiteId logger.info("") logger.info("Dependency Graph for %s:", testSuiteId) - printDependencyGraph(graph.components) + printDependencyGraph(graph) logger.info("") if (testAppDirForRemoteTestSuite) { diff --git a/k8s-deployer/test/dependency-resolver.spec.ts b/k8s-deployer/test/dependency-resolver.spec.ts index 0216765..c4d0088 100644 --- a/k8s-deployer/test/dependency-resolver.spec.ts +++ b/k8s-deployer/test/dependency-resolver.spec.ts @@ -395,6 +395,11 @@ describe("Dependency Resolver", () => { describe("printDependencyGraph", () => { let logOutput: string[] let originalLog: typeof console.log + const testApp: Schema.DeployableComponent = { + name: "test-app", id: "test-app", + location: { type: Schema.LocationType.Local }, + deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } + } beforeEach(() => { logOutput = [] originalLog = console.log @@ -410,9 +415,10 @@ describe("Dependency Resolver", () => { { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } }, { name: "C", id: "c", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } } ] - printDependencyGraph(components) + printDependencyGraph({ testApp, components }) expect(logOutput[0]).to.equal("Dependency Graph") expect(logOutput).to.include(" Stage 1 │ a b c") + expect(logOutput).to.include(" Stage 2 │ test-app") }) it("prints graph for components with dependencies at multiple stages", () => { @@ -421,10 +427,11 @@ describe("Dependency Resolver", () => { { name: "API", id: "api", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["db"] }, { name: "Cache", id: "cache", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["db"] } ] - printDependencyGraph(components) + printDependencyGraph({ testApp, components }) expect(logOutput[0]).to.equal("Dependency Graph") expect(logOutput).to.include(" Stage 1 │ db") expect(logOutput).to.include(" Stage 2 │ api cache") + expect(logOutput).to.include(" Stage 3 │ test-app") expect(logOutput).to.include(" db ──▶ api") expect(logOutput).to.include(" db ──▶ cache") }) @@ -435,11 +442,12 @@ describe("Dependency Resolver", () => { { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"] }, { name: "C", id: "c", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["b"] } ] - printDependencyGraph(components) + printDependencyGraph({ testApp, components }) expect(logOutput[0]).to.equal("Dependency Graph") expect(logOutput).to.include(" Stage 1 │ a") expect(logOutput).to.include(" Stage 2 │ b") expect(logOutput).to.include(" Stage 3 │ c") + expect(logOutput).to.include(" Stage 4 │ test-app") expect(logOutput).to.include(" a ──▶ b") expect(logOutput).to.include(" b ──▶ c") }) @@ -450,13 +458,29 @@ describe("Dependency Resolver", () => { { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"], parallel: true }, { name: "C", id: "c", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"], parallel: true } ] - printDependencyGraph(components) + printDependencyGraph({ testApp, components }) expect(logOutput[0]).to.equal("Dependency Graph") expect(logOutput).to.include(" Stage 1 │ a") expect(logOutput).to.include(" Stage 2 │ b 🔀 c 🔀") + expect(logOutput).to.include(" Stage 3 │ test-app") expect(logOutput).to.include(" a ──▶ b") expect(logOutput).to.include(" a ──▶ c") expect(logOutput).to.include(" 🔀 = concurrent deployment") }) + + it("shows testApp in a concurrent section when testApp.parallel is true", () => { + const parallelTestApp: Schema.DeployableComponent = { ...testApp, id: "my-test-app", parallel: true } + const components: Array = [ + { name: "A", id: "a", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } }, + { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"] } + ] + printDependencyGraph({ testApp: parallelTestApp, components }) + expect(logOutput[0]).to.equal("Dependency Graph") + expect(logOutput).to.include(" Stage 1 │ a") + expect(logOutput).to.include(" Stage 2 │ b") + expect(logOutput).to.include(" my-test-app 🔀 (concurrent with component stages)") + expect(logOutput).to.not.include(" Stage 3 │ my-test-app") + expect(logOutput).to.include(" 🔀 = concurrent deployment") + }) }) }) From b9f22549e371df0fc6beec91a0dfe1ce1d0c56f5 Mon Sep 17 00:00:00 2001 From: "James Charters james.charters@kindredgroup.com" Date: Tue, 10 Mar 2026 16:26:42 +1100 Subject: [PATCH 09/12] feat: [KSBP-101776] - Updates to treatment of combined parallel and sequential at same level --- k8s-deployer/src/dependency-resolver.ts | 19 +++- k8s-deployer/src/test-suite-handler.ts | 55 +++++------ k8s-deployer/test/dependency-resolver.spec.ts | 16 +++- k8s-deployer/test/test-suite-handler.spec.ts | 91 ++++++++++--------- 4 files changed, 105 insertions(+), 76 deletions(-) diff --git a/k8s-deployer/src/dependency-resolver.ts b/k8s-deployer/src/dependency-resolver.ts index 15a2c89..5ad7eca 100644 --- a/k8s-deployer/src/dependency-resolver.ts +++ b/k8s-deployer/src/dependency-resolver.ts @@ -221,24 +221,35 @@ export const printDependencyGraph = (graph: Schema.Graph): void => { const { components, testApp } = graph const { levels } = topologicalSort(components) const sep = "─".repeat(40) - const label = (c: Schema.DeployableComponent) => c.parallel ? `${c.id} 🔀` : c.id const edges = components.flatMap(c => (c.dependsOn ?? []).map(dep => ` ${dep} ──▶ ${c.id}`)) const anyConcurrent = components.some(c => c.parallel) || testApp.parallel === true + // Format a single dependency level into a display string. + // Parallel components are grouped in brackets; sequential follow after an arrow. + // e.g. "[B 🔀 C 🔀] → D E" or "[B 🔀] → C" or "A B" (all sequential) + const formatLevel = (level: Array): string => { + const parallel = level.filter(c => c.parallel === true) + const sequential = level.filter(c => c.parallel !== true) + const parallelPart = parallel.length > 0 ? `[${parallel.map(c => `${c.id} 🔀`).join(" ")}]` : "" + const sequentialPart = sequential.map(c => c.id).join(" ") + if (parallelPart && sequentialPart) return `${parallelPart} → ${sequentialPart}` + return parallelPart || sequentialPart + } + console.log("Dependency Graph") console.log(sep) if (testApp.parallel === true) { // testApp runs concurrently with the entire component chain — show it in a separate section levels.forEach((level, idx) => - console.log(` Stage ${idx + 1} │ ${level.map(label).join(" ")}`) + console.log(` Stage ${idx + 1} │ ${formatLevel(level)}`) ) console.log(sep) - console.log(` ${label(testApp)} (concurrent with component stages)`) + console.log(` ${testApp.id} 🔀 (concurrent with component stages)`) } else { // testApp runs after all components — append it as the final stage levels.forEach((level, idx) => - console.log(` Stage ${idx + 1} │ ${level.map(label).join(" ")}`) + console.log(` Stage ${idx + 1} │ ${formatLevel(level)}`) ) console.log(` Stage ${levels.length + 1} │ ${testApp.id}`) } diff --git a/k8s-deployer/src/test-suite-handler.ts b/k8s-deployer/src/test-suite-handler.ts index 05733b4..ff350a0 100644 --- a/k8s-deployer/src/test-suite-handler.ts +++ b/k8s-deployer/src/test-suite-handler.ts @@ -70,36 +70,31 @@ export const deployGraph = async (config: Config, workspace: string, testSuiteId const parallelGroup = level.filter(c => c.parallel === true) const sequentialGroup = level.filter(c => c.parallel !== true) - // Fire both groups concurrently; await both before moving to the next level. - const parallelChain = parallelGroup.length > 0 - ? (async () => { - logger.info("") - logger.info("Deploying %d component(s) in parallel for suite \"%s\": %s", parallelGroup.length, testSuiteId, parallelGroup.map(c => c.id).join(", ")) - const results = await Promise.all( - parallelGroup.map(async componentSpec => { - const idx = ++componentIndex - logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) - const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) - logger.info("Graph component \"%s\" for suite \"%s\" deployed.", componentSpec.name, testSuiteId) - return new DeployedComponent(commitSha, componentSpec) - }) - ) - deployments.push(...results) - })() - : Promise.resolve() - - const sequentialChain = (async () => { - for (const componentSpec of sequentialGroup) { - const idx = ++componentIndex - logger.info("") - logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) - logger.info("") - const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) - deployments.push(new DeployedComponent(commitSha, componentSpec)) - } - })() - - await Promise.all([parallelChain, sequentialChain]) + // First, deploy parallel-flagged components concurrently. + // Then, once all parallel components at this level are done, deploy sequential ones in order. + if (parallelGroup.length > 0) { + logger.info("") + logger.info("Deploying %d component(s) in parallel for suite \"%s\": %s", parallelGroup.length, testSuiteId, parallelGroup.map(c => c.id).join(", ")) + const results = await Promise.all( + parallelGroup.map(async componentSpec => { + const idx = ++componentIndex + logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) + const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) + logger.info("Graph component \"%s\" for suite \"%s\" deployed.", componentSpec.name, testSuiteId) + return new DeployedComponent(commitSha, componentSpec) + }) + ) + deployments.push(...results) + } + + for (const componentSpec of sequentialGroup) { + const idx = ++componentIndex + logger.info("") + logger.info("Deploying graph component (%s of %s) \"%s\" for suite \"%s\"...", idx, sortedComponents.length, componentSpec.name, testSuiteId) + logger.info("") + const commitSha = await Deployer.deployComponent(config, workspace, componentSpec, namespace) + deployments.push(new DeployedComponent(commitSha, componentSpec)) + } } })() diff --git a/k8s-deployer/test/dependency-resolver.spec.ts b/k8s-deployer/test/dependency-resolver.spec.ts index c4d0088..9ed14a4 100644 --- a/k8s-deployer/test/dependency-resolver.spec.ts +++ b/k8s-deployer/test/dependency-resolver.spec.ts @@ -461,13 +461,27 @@ describe("Dependency Resolver", () => { printDependencyGraph({ testApp, components }) expect(logOutput[0]).to.equal("Dependency Graph") expect(logOutput).to.include(" Stage 1 │ a") - expect(logOutput).to.include(" Stage 2 │ b 🔀 c 🔀") + expect(logOutput).to.include(" Stage 2 │ [b 🔀 c 🔀]") expect(logOutput).to.include(" Stage 3 │ test-app") expect(logOutput).to.include(" a ──▶ b") expect(logOutput).to.include(" a ──▶ c") expect(logOutput).to.include(" 🔀 = concurrent deployment") }) + it("shows mixed parallel/sequential at same level as [parallel] → sequential", () => { + const components: Array = [ + { name: "A", id: "a", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } }, + { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"], parallel: true }, + { name: "C", id: "c", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"] } + ] + printDependencyGraph({ testApp, components }) + expect(logOutput[0]).to.equal("Dependency Graph") + expect(logOutput).to.include(" Stage 1 │ a") + expect(logOutput).to.include(" Stage 2 │ [b 🔀] → c") + expect(logOutput).to.include(" Stage 3 │ test-app") + expect(logOutput).to.include(" 🔀 = concurrent deployment") + }) + it("shows testApp in a concurrent section when testApp.parallel is true", () => { const parallelTestApp: Schema.DeployableComponent = { ...testApp, id: "my-test-app", parallel: true } const components: Array = [ diff --git a/k8s-deployer/test/test-suite-handler.spec.ts b/k8s-deployer/test/test-suite-handler.spec.ts index 92472db..2d7bece 100644 --- a/k8s-deployer/test/test-suite-handler.spec.ts +++ b/k8s-deployer/test/test-suite-handler.spec.ts @@ -15,7 +15,7 @@ import * as webapi from "../src/test-app-client/web-api/schema-v1.js" import { generatePrefixByDate } from "../src/test-suite-handler.js" describe("Helper functions", () => { - it ("should generate readable date prefix", () => { + it("should generate readable date prefix", () => { const date = new Date('March 1, 2024 00:00:00.123 UTC') const prefix = generatePrefixByDate(date, "desktop") @@ -27,7 +27,7 @@ describe("Deployment happy path", async () => { let prefix = "12345" let testSuiteId = "t1" const namespace = "nsChild" - const workspace = `${ prefix }_${ testSuiteId }` + const workspace = `${prefix}_${testSuiteId}` const k8DeployerConfig: Config = { commitSha: "sha4567", workspace: "/tmp/some/dir", @@ -75,7 +75,7 @@ describe("Deployment happy path", async () => { name: "comp-1-name", id: "comp-1", location: { type: LocationType.Local }, - deploy: { command: "deployment/pit/deploy.sh", statusCheck: { timeoutSeconds: 1, command: "deployment/pit/is-deployment-ready.sh" }}, + deploy: { command: "deployment/pit/deploy.sh", statusCheck: { timeoutSeconds: 1, command: "deployment/pit/is-deployment-ready.sh" } }, logTailing: { enabled: true, containerName: "comp-1-specific-container" @@ -86,7 +86,7 @@ describe("Deployment happy path", async () => { } } - it ("processTestSuite", async () => { + it("processTestSuite", async () => { const report = { executedScenarios: [ @@ -94,7 +94,7 @@ describe("Deployment happy path", async () => { "t1-sc1", new Date(), new Date(new Date().getTime() + 20_000), - [ new TestStream("t1-sc1-stream1", [ new ScalarMetric("tps", 100) ], [ new ScalarMetric("tps", 101) ], TestOutcomeType.PASS) ], + [new TestStream("t1-sc1-stream1", [new ScalarMetric("tps", 100)], [new ScalarMetric("tps", 101)], TestOutcomeType.PASS)], ["comp-1"] ) ] @@ -126,7 +126,7 @@ describe("Deployment happy path", async () => { // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - const httpClientStub = sinon.stub() - httpClientStub.withArgs(sinon.match(`\/${ namespace }\.${ testSuiteId }\/start`), sinon.match.any).returns({ + httpClientStub.withArgs(sinon.match(`\/${namespace}\.${testSuiteId}\/start`), sinon.match.any).returns({ ok: true, json: async () => new webapi.StartResponse("session-1", testSuiteId) }) @@ -134,13 +134,13 @@ describe("Deployment happy path", async () => { ok: true, json: async () => new webapi.StatusResponse("session-1", testSuiteId, webapi.TestStatus.RUNNING) } - const respStatusCompleted = { ok: true, json: async () => new webapi.StatusResponse("session-1", testSuiteId, webapi.TestStatus.COMPLETED)} - httpClientStub.withArgs(sinon.match(`\/${ namespace }\.${ testSuiteId }\/status\?sessionId=session-1`), sinon.match.any) + const respStatusCompleted = { ok: true, json: async () => new webapi.StatusResponse("session-1", testSuiteId, webapi.TestStatus.COMPLETED) } + httpClientStub.withArgs(sinon.match(`\/${namespace}\.${testSuiteId}\/status\?sessionId=session-1`), sinon.match.any) .onFirstCall().returns(respStatusRunning) .onSecondCall().returns(respStatusRunning) .onThirdCall().returns(respStatusCompleted) - httpClientStub.withArgs(sinon.match(`\/${ namespace }\.${ testSuiteId }\/reports\?sessionId=session-1`), sinon.match.any) + httpClientStub.withArgs(sinon.match(`\/${namespace}\.${testSuiteId}\/reports\?sessionId=session-1`), sinon.match.any) .returns( { ok: true, @@ -177,7 +177,8 @@ describe("Deployment happy path", async () => { "child_process": { spawn: (script: string, args: string[], opts: SpawnOptions[]) => { // delegate spawining calls to stub and record them for later assertion - return nodeShellSpawnStub(script, args, opts) } + return nodeShellSpawnStub(script, args, opts) + } } } ) @@ -185,14 +186,14 @@ describe("Deployment happy path", async () => { const SuiteHandler = await esmock( "../src/test-suite-handler.js", { - "../src/k8s.js": { ...K8s, generateNamespaceName:() => namespace } , - "../src/pod-log-tail.js": { ...PodLogTail } + "../src/k8s.js": { ...K8s, generateNamespaceName: () => namespace }, + "../src/pod-log-tail.js": { ...PodLogTail } }, { - "../src/logger.js": { logger: { debug: () => {}, info: () => {}, warn: (s: string, a: any) => { logger.warn(s, a) }, error: (s: string, a: any) => { logger.error(s, a) } } }, + "../src/logger.js": { logger: { debug: () => { }, info: () => { }, warn: (s: string, a: any) => { logger.warn(s, a) }, error: (s: string, a: any) => { logger.error(s, a) } } }, "node-fetch": httpImportMock, "../src/shell-facade.js": shellImportMock, - "fs": { promises: { access: async (path: string, mode: number) => await fsAccessStubs(path, mode) }} + "fs": { promises: { access: async (path: string, mode: number) => await fsAccessStubs(path, mode) } } }, ) @@ -204,7 +205,7 @@ describe("Deployment happy path", async () => { projectName: "TestPitFile", version: SchemaVersion.VERSION_1_0, lockManager: { enabled: true }, - testSuites: [ testSuite ] + testSuites: [testSuite] } await SuiteHandler.processTestSuite(prefix, k8DeployerConfig, pitfile, testSuiteNumber, testSuite) @@ -290,16 +291,16 @@ describe("Deployment happy path", async () => { chai.expect(nodeShellSpawnStub.getCall(0).calledWith( "k8s-deployer/scripts/tail-container-log.sh", - [ namespace, "comp-1", "comp-1-specific-container" ] + [namespace, "comp-1", "comp-1-specific-container"] )).be.true chai.expect(nodeShellSpawnStub.getCall(1).calledWith( - "k8s-deployer/scripts/tail-container-log.sh", - [ namespace, "comp-1-test-app", "" ] + "k8s-deployer/scripts/tail-container-log.sh", + [namespace, "comp-1-test-app", ""] )).be.true }) - it ("processTestSuite with a different workspace", async () => { + it("processTestSuite with a different workspace", async () => { prefix = '23456' testSuiteId = 't2' const testSuite2 = JSON.parse(JSON.stringify(testSuite)) @@ -312,7 +313,7 @@ describe("Deployment happy path", async () => { "t2-sc2", new Date(), new Date(new Date().getTime() + 20_000), - [ new TestStream("t2-sc2-stream1", [ new ScalarMetric("tps", 100) ], [ new ScalarMetric("tps", 101) ], TestOutcomeType.PASS) ], + [new TestStream("t2-sc2-stream1", [new ScalarMetric("tps", 100)], [new ScalarMetric("tps", 101)], TestOutcomeType.PASS)], ["comp-2"] ) ] @@ -344,7 +345,7 @@ describe("Deployment happy path", async () => { // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - const httpClientStub = sinon.stub() - httpClientStub.withArgs(sinon.match(`\/${ namespace }\.${ testSuiteId }\/start`), sinon.match.any).returns({ + httpClientStub.withArgs(sinon.match(`\/${namespace}\.${testSuiteId}\/start`), sinon.match.any).returns({ ok: true, json: async () => new webapi.StartResponse("session-2", testSuiteId) }) @@ -352,13 +353,13 @@ describe("Deployment happy path", async () => { ok: true, json: async () => new webapi.StatusResponse("session-2", testSuiteId, webapi.TestStatus.RUNNING) } - const respStatusCompleted = { ok: true, json: async () => new webapi.StatusResponse("session-2", testSuiteId, webapi.TestStatus.COMPLETED)} - httpClientStub.withArgs(sinon.match(`\/${ namespace }\.${ testSuiteId }\/status\?sessionId=session-2`), sinon.match.any) + const respStatusCompleted = { ok: true, json: async () => new webapi.StatusResponse("session-2", testSuiteId, webapi.TestStatus.COMPLETED) } + httpClientStub.withArgs(sinon.match(`\/${namespace}\.${testSuiteId}\/status\?sessionId=session-2`), sinon.match.any) .onFirstCall().returns(respStatusRunning) .onSecondCall().returns(respStatusRunning) .onThirdCall().returns(respStatusCompleted) - httpClientStub.withArgs(sinon.match(`\/${ namespace }\.${ testSuiteId }\/reports\?sessionId=session-2`), sinon.match.any) + httpClientStub.withArgs(sinon.match(`\/${namespace}\.${testSuiteId}\/reports\?sessionId=session-2`), sinon.match.any) .returns( { ok: true, @@ -396,7 +397,8 @@ describe("Deployment happy path", async () => { "child_process": { spawn: (script: string, args: string[], opts: SpawnOptions[]) => { // delegate spawining calls to stub and record them for later assertion - return nodeShellSpawnStub(script, args, opts) } + return nodeShellSpawnStub(script, args, opts) + } } } ) @@ -404,14 +406,14 @@ describe("Deployment happy path", async () => { const SuiteHandler = await esmock( "../src/test-suite-handler.js", { - "../src/k8s.js": { ...K8s, generateNamespaceName:() => namespace } , - "../src/pod-log-tail.js": { ...PodLogTail } + "../src/k8s.js": { ...K8s, generateNamespaceName: () => namespace }, + "../src/pod-log-tail.js": { ...PodLogTail } }, { - "../src/logger.js": { logger: { debug: () => {}, info: () => {}, warn: (s: string, a: any) => { logger.warn(s, a) }, error: (s: string, a: any) => { logger.error(s, a) } } }, + "../src/logger.js": { logger: { debug: () => { }, info: () => { }, warn: (s: string, a: any) => { logger.warn(s, a) }, error: (s: string, a: any) => { logger.error(s, a) } } }, "node-fetch": httpImportMock, "../src/shell-facade.js": shellImportMock, - "fs": { promises: { access: async (path: string, mode: number) => await fsAccessStubs(path, mode) }} + "fs": { promises: { access: async (path: string, mode: number) => await fsAccessStubs(path, mode) } } }, ) @@ -423,7 +425,7 @@ describe("Deployment happy path", async () => { projectName: "TestPitFile", version: SchemaVersion.VERSION_1_0, lockManager: { enabled: true }, - testSuites: [ testSuite2 ] + testSuites: [testSuite2] } await SuiteHandler.processTestSuite(prefix, k8DeployerConfig, pitfile, testSuiteNumber, testSuite2) @@ -492,7 +494,7 @@ describe("deployGraph - deployment ordering and concurrency", async () => { esmock( "../src/test-suite-handler.js", { "../src/deployer.js": { deployComponent: deployStub } }, - { "../src/logger.js": { logger: { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} } } } + { "../src/logger.js": { logger: { debug: () => { }, info: () => { }, warn: () => { }, error: () => { } } } } ) it("returns GraphDeploymentResult with all deployed components", async () => { @@ -537,7 +539,7 @@ describe("deployGraph - deployment ordering and concurrency", async () => { chai.expect(completed).to.include.members(["B", "C"]) }) - it("deploys mixed parallel/sequential at same level concurrently", async () => { + it("sequential components at same level wait for parallel ones to complete first", async () => { const gates: Record = { B: makeGate(), C: makeGate(), testApp: makeGate() } const started: string[] = [] const completed: string[] = [] @@ -547,20 +549,22 @@ describe("deployGraph - deployment ordering and concurrency", async () => { completed.push(spec.id) return `sha-${spec.id}` }) + const SuiteHandler = await loadWithStub(deployStub) - // B is parallel:true, C has no parallel flag — same dependency level, so neither waits on the other + // B is parallel:true, C has no parallel flag — same dependency level const graph = { testApp: makeSpec("testApp", { parallel: true }), components: [makeSpec("B", { parallel: true }), makeSpec("C")] } const deployPromise = SuiteHandler.deployGraph(config, workspace, testSuiteId, graph, namespace) - // B (parallelGroup) and C (sequentialGroup) fire their respective chains simultaneously — - // both should be in-flight before either completes + // B (parallelGroup) starts immediately; C (sequentialGroup) must wait for B to finish chai.expect(started).to.include("B") - chai.expect(started).to.include("C") - chai.expect(completed).to.not.include("B") - chai.expect(completed).to.not.include("C") + chai.expect(started).to.not.include("C") + // Release B — C should now be able to start gates["B"].resolve() + await new Promise(r => setTimeout(r, 0)) // flush async chains + chai.expect(completed).to.include("B") + chai.expect(started).to.include("C") gates["C"].resolve() gates["testApp"].resolve() await deployPromise @@ -597,15 +601,20 @@ describe("deployGraph - deployment ordering and concurrency", async () => { chai.expect(started).to.not.include("C") chai.expect(started).to.not.include("D") - // Resolve A — level 2 (B and C) should start + // Resolve A — level 2: B (parallel) starts first; C (sequential) waits for B gates["A"].resolve() await new Promise(r => setTimeout(r, 0)) chai.expect(started).to.include("B") - chai.expect(started).to.include("C") + chai.expect(started).to.not.include("C") chai.expect(started).to.not.include("D") - // Resolve B and C — level 3 (D) should start + // Resolve B — C (sequential) can now start gates["B"].resolve() + await new Promise(r => setTimeout(r, 0)) + chai.expect(started).to.include("C") + chai.expect(started).to.not.include("D") + + // Resolve C — level 3 (D) should start gates["C"].resolve() await new Promise(r => setTimeout(r, 0)) chai.expect(started).to.include("D") From 7e1c479553792af18a4373734423f54af2830af3 Mon Sep 17 00:00:00 2001 From: "James Charters james.charters@kindredgroup.com" Date: Thu, 12 Mar 2026 09:53:01 +1100 Subject: [PATCH 10/12] feat: [KSBP-10176] - Move parallel flag to DeployableComponent type --- k8s-deployer/src/dependency-resolver.ts | 10 +++++----- k8s-deployer/src/pitfile/schema-v1.ts | 2 +- k8s-deployer/src/test-suite-handler.ts | 6 +++--- k8s-deployer/test/dependency-resolver.spec.ts | 8 ++++---- k8s-deployer/test/test-suite-handler.spec.ts | 10 +++++----- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/k8s-deployer/src/dependency-resolver.ts b/k8s-deployer/src/dependency-resolver.ts index 5ad7eca..09e7708 100644 --- a/k8s-deployer/src/dependency-resolver.ts +++ b/k8s-deployer/src/dependency-resolver.ts @@ -213,7 +213,7 @@ const reconstructCyclePath = (startId: string, parent: Map): Arr * Print the full deployment graph including testApp placement. * * - Components are shown in topological stages. - * - If testApp.parallel === true it is shown in a separate concurrent section + * - If testApp.deploy.parallel === true it is shown in a separate concurrent section * (it runs alongside all component stages). * - Otherwise testApp is shown as the final sequential stage after all components. */ @@ -222,14 +222,14 @@ export const printDependencyGraph = (graph: Schema.Graph): void => { const { levels } = topologicalSort(components) const sep = "─".repeat(40) const edges = components.flatMap(c => (c.dependsOn ?? []).map(dep => ` ${dep} ──▶ ${c.id}`)) - const anyConcurrent = components.some(c => c.parallel) || testApp.parallel === true + const anyConcurrent = components.some(c => c.deploy.parallel) || testApp.deploy.parallel === true // Format a single dependency level into a display string. // Parallel components are grouped in brackets; sequential follow after an arrow. // e.g. "[B 🔀 C 🔀] → D E" or "[B 🔀] → C" or "A B" (all sequential) const formatLevel = (level: Array): string => { - const parallel = level.filter(c => c.parallel === true) - const sequential = level.filter(c => c.parallel !== true) + const parallel = level.filter(c => c.deploy.parallel === true) + const sequential = level.filter(c => c.deploy.parallel !== true) const parallelPart = parallel.length > 0 ? `[${parallel.map(c => `${c.id} 🔀`).join(" ")}]` : "" const sequentialPart = sequential.map(c => c.id).join(" ") if (parallelPart && sequentialPart) return `${parallelPart} → ${sequentialPart}` @@ -239,7 +239,7 @@ export const printDependencyGraph = (graph: Schema.Graph): void => { console.log("Dependency Graph") console.log(sep) - if (testApp.parallel === true) { + if (testApp.deploy.parallel === true) { // testApp runs concurrently with the entire component chain — show it in a separate section levels.forEach((level, idx) => console.log(` Stage ${idx + 1} │ ${formatLevel(level)}`) diff --git a/k8s-deployer/src/pitfile/schema-v1.ts b/k8s-deployer/src/pitfile/schema-v1.ts index b313cea..5cb9f09 100644 --- a/k8s-deployer/src/pitfile/schema-v1.ts +++ b/k8s-deployer/src/pitfile/schema-v1.ts @@ -29,6 +29,7 @@ export class DeployInstructions { command: string params?: Array statusCheck?: StatusCheck + parallel?: boolean // If true, this component may be deployed concurrently with other parallel components at the same dependency level } export class LockManager { @@ -61,7 +62,6 @@ export class DeployableComponent { undeploy: DeployInstructions logTailing?: LogTailing dependsOn?: Array // Optional array of component IDs this component depends on - parallel?: boolean // If true, this component may be deployed concurrently with other parallel components at the same dependency level } export class Graph { diff --git a/k8s-deployer/src/test-suite-handler.ts b/k8s-deployer/src/test-suite-handler.ts index ff350a0..3eca793 100644 --- a/k8s-deployer/src/test-suite-handler.ts +++ b/k8s-deployer/src/test-suite-handler.ts @@ -67,8 +67,8 @@ export const deployGraph = async (config: Config, workspace: string, testSuiteId let componentIndex = 0 const deployComponentsPromise = (async () => { for (const level of levels) { - const parallelGroup = level.filter(c => c.parallel === true) - const sequentialGroup = level.filter(c => c.parallel !== true) + const parallelGroup = level.filter(c => c.deploy.parallel === true) + const sequentialGroup = level.filter(c => c.deploy.parallel !== true) // First, deploy parallel-flagged components concurrently. // Then, once all parallel components at this level are done, deploy sequential ones in order. @@ -102,7 +102,7 @@ export const deployGraph = async (config: Config, workspace: string, testSuiteId const params = [ testSuiteId ] let testAppDeployedComponent: DeployedComponent - if (graph.testApp.parallel === true) { + if (graph.testApp.deploy.parallel === true) { // Deploy test app concurrently with the component levels (opt-in). // Only use this when the test app can start independently of the components. const deployTestAppPromise = Deployer.deployComponent(config, workspace, graph.testApp, namespace, params) diff --git a/k8s-deployer/test/dependency-resolver.spec.ts b/k8s-deployer/test/dependency-resolver.spec.ts index 9ed14a4..2388d7a 100644 --- a/k8s-deployer/test/dependency-resolver.spec.ts +++ b/k8s-deployer/test/dependency-resolver.spec.ts @@ -455,8 +455,8 @@ describe("Dependency Resolver", () => { it("annotates parallel components with 🔀 and prints a legend", () => { const components: Array = [ { name: "A", id: "a", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } }, - { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"], parallel: true }, - { name: "C", id: "c", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"], parallel: true } + { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"] }, + { name: "C", id: "c", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"] } ] printDependencyGraph({ testApp, components }) expect(logOutput[0]).to.equal("Dependency Graph") @@ -471,7 +471,7 @@ describe("Dependency Resolver", () => { it("shows mixed parallel/sequential at same level as [parallel] → sequential", () => { const components: Array = [ { name: "A", id: "a", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } }, - { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"], parallel: true }, + { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"] }, { name: "C", id: "c", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"] } ] printDependencyGraph({ testApp, components }) @@ -483,7 +483,7 @@ describe("Dependency Resolver", () => { }) it("shows testApp in a concurrent section when testApp.parallel is true", () => { - const parallelTestApp: Schema.DeployableComponent = { ...testApp, id: "my-test-app", parallel: true } + const parallelTestApp: Schema.DeployableComponent = { ...testApp, id: "my-test-app", deploy: { ...testApp.deploy, parallel: true } } const components: Array = [ { name: "A", id: "a", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } }, { name: "B", id: "b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["a"] } diff --git a/k8s-deployer/test/test-suite-handler.spec.ts b/k8s-deployer/test/test-suite-handler.spec.ts index 2d7bece..36dce9f 100644 --- a/k8s-deployer/test/test-suite-handler.spec.ts +++ b/k8s-deployer/test/test-suite-handler.spec.ts @@ -63,12 +63,12 @@ describe("Deployment happy path", async () => { location: { type: LocationType.Local }, deploy: { command: "deployment/pit/deploy.sh", - statusCheck: { timeoutSeconds: 1, command: "deployment/pit/is-deployment-ready.sh" } + statusCheck: { timeoutSeconds: 1, command: "deployment/pit/is-deployment-ready.sh" }, + parallel: true }, logTailing: { enabled: true - }, - parallel: true + } }, components: [ { @@ -485,9 +485,9 @@ describe("deployGraph - deployment ordering and concurrency", async () => { const makeSpec = (id: string, opts: { parallel?: boolean; dependsOn?: string[] } = {}) => ({ name: `${id}-name`, id, location: { type: LocationType.Local }, - deploy: { command: `${id}/deploy.sh`, statusCheck: { command: `${id}/ready.sh` } }, + deploy: { command: `${id}/deploy.sh`, statusCheck: { command: `${id}/ready.sh` }, parallel: opts.parallel }, undeploy: { command: `${id}/undeploy.sh` }, - ...opts + ...(opts.dependsOn !== undefined ? { dependsOn: opts.dependsOn } : {}) }) const loadWithStub = async (deployStub: sinon.SinonStub) => From 9f51480aafabd1b8311880cb3efba084dcb3e261 Mon Sep 17 00:00:00 2001 From: "James Charters james.charters@kindredgroup.com" Date: Thu, 12 Mar 2026 16:18:08 +1100 Subject: [PATCH 11/12] feat: [KSBP-10176] - Pretty mermaid diagrams --- k8s-deployer/package-lock.json | 33 ++ k8s-deployer/package.json | 1 + k8s-deployer/src/dependency-resolver.ts | 99 ++++-- .../test/component-dependency.spec.ts | 162 ++++++++++ k8s-deployer/test/dependency-resolver.spec.ts | 121 +++++++- .../test/pitfile/pitfile-loader.spec.ts | 35 +++ ...est-pitfile-valid-with-parallel-deploy.yml | 102 +++++++ ...-pitfile-valid-with-parallel-subgroups.yml | 287 ++++++++++++++++++ 8 files changed, 803 insertions(+), 37 deletions(-) create mode 100644 k8s-deployer/test/pitfile/test-pitfile-valid-with-parallel-deploy.yml create mode 100644 k8s-deployer/test/pitfile/test-pitfile-valid-with-parallel-subgroups.yml diff --git a/k8s-deployer/package-lock.json b/k8s-deployer/package-lock.json index 26f93c2..a2bcfda 100644 --- a/k8s-deployer/package-lock.json +++ b/k8s-deployer/package-lock.json @@ -12,6 +12,7 @@ "ajv": "^8.12.0", "express": "^4.19.2", "express-openapi-validator": "^5.1.6", + "mermaid-ascii": "^1.0.0", "node-fetch": "^3.3.2", "swagger-ui-express": "^5.0.0", "uuid": "^9.0.1", @@ -796,6 +797,14 @@ "text-hex": "1.0.x" } }, + "node_modules/commander": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", + "integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==", + "engines": { + "node": ">=16" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1918,6 +1927,17 @@ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" }, + "node_modules/mermaid-ascii": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mermaid-ascii/-/mermaid-ascii-1.0.0.tgz", + "integrity": "sha512-vlJWXpQijkfgindiKSqiX/lrnDx1L65m1dgWJ7ioTo+mW4o1MHTz/VVfXM2SrFixVB4/6LK6v88C3rgvT4wuhw==", + "dependencies": { + "commander": "^11.1.0" + }, + "bin": { + "mermaid-ascii": "dist/cli.js" + } + }, "node_modules/methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", @@ -3827,6 +3847,11 @@ "text-hex": "1.0.x" } }, + "commander": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", + "integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==" + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -4689,6 +4714,14 @@ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" }, + "mermaid-ascii": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mermaid-ascii/-/mermaid-ascii-1.0.0.tgz", + "integrity": "sha512-vlJWXpQijkfgindiKSqiX/lrnDx1L65m1dgWJ7ioTo+mW4o1MHTz/VVfXM2SrFixVB4/6LK6v88C3rgvT4wuhw==", + "requires": { + "commander": "^11.1.0" + } + }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", diff --git a/k8s-deployer/package.json b/k8s-deployer/package.json index 429a9f5..e2500cc 100644 --- a/k8s-deployer/package.json +++ b/k8s-deployer/package.json @@ -40,6 +40,7 @@ "ajv": "^8.12.0", "express": "^4.19.2", "express-openapi-validator": "^5.1.6", + "mermaid-ascii": "^1.0.0", "node-fetch": "^3.3.2", "swagger-ui-express": "^5.0.0", "uuid": "^9.0.1", diff --git a/k8s-deployer/src/dependency-resolver.ts b/k8s-deployer/src/dependency-resolver.ts index 09e7708..de10b6b 100644 --- a/k8s-deployer/src/dependency-resolver.ts +++ b/k8s-deployer/src/dependency-resolver.ts @@ -1,4 +1,5 @@ import { Schema } from "./model.js" +import { mermaidToAscii } from "mermaid-ascii" import { CyclicDependencyError, InvalidDependencyError, @@ -209,58 +210,100 @@ const reconstructCyclePath = (startId: string, parent: Map): Arr return path } +// Returns the mermaid node identifier for a component. +// Parallel components get a _🔀 suffix so the box label shows the flag. +const nodeId = (c: Schema.DeployableComponent): string => + c.deploy.parallel === true ? `${c.id}_🔀` : c.id + +// Build a `graph TD` mermaid source string from the sorted levels + testApp. +const buildMermaidSrc = ( + levels: Array>, + components: Array, + testApp: Schema.DeployableComponent +): string => { + const componentMap = new Map(components.map(c => [c.id, c])) + const edgeLines: string[] = [] + + // Edges from dependsOn relationships + components.forEach(c => { + (c.dependsOn ?? []).forEach(depId => { + const dep = componentMap.get(depId)! + edgeLines.push(` ${nodeId(dep)} --> ${nodeId(c)}`) + }) + }) + + // Sequential testApp: connect every last-level component to the testApp node + if (testApp.deploy.parallel !== true && levels.length > 0) { + levels[levels.length - 1].forEach(c => { + edgeLines.push(` ${nodeId(c)} --> ${testApp.id}`) + }) + } + + // Nodes that appear in no edge must be declared explicitly or they won't render + const edgeText = edgeLines.join("\n") + const allNodes = [ + ...components, + ...(testApp.deploy.parallel !== true ? [testApp] : []) + ] + const isolatedDeclarations = allNodes + .filter(c => !edgeText.includes(nodeId(c))) + .map(c => ` ${nodeId(c)}`) + + return ["graph TD", ...isolatedDeclarations, ...edgeLines].join("\n") +} + +// Format a single dependency level into a display string. +// Parallel components are grouped in brackets; sequential follow after an arrow. +// e.g. "[B 🔀 C 🔀] → D E" or "[B 🔀] → C" or "A B" (all sequential) +const formatLevel = (level: Array): string => { + const parallel = level.filter(c => c.deploy.parallel === true) + const sequential = level.filter(c => c.deploy.parallel !== true) + const parallelPart = parallel.length > 0 ? `[${parallel.map(c => `${c.id} 🔀`).join(" ")}]` : "" + const sequentialPart = sequential.map(c => c.id).join(" ") + if (parallelPart && sequentialPart) return `${parallelPart} → ${sequentialPart}` + return parallelPart || sequentialPart +} + /** * Print the full deployment graph including testApp placement. * - * - Components are shown in topological stages. - * - If testApp.deploy.parallel === true it is shown in a separate concurrent section - * (it runs alongside all component stages). - * - Otherwise testApp is shown as the final sequential stage after all components. + * Outputs two representations: + * 1. Stage list — "Stage N │ [parallel 🔀] → sequential" text summary + * 2. ASCII art diagram via mermaid-ascii (parallel components labelled with _🔀) + * + * If testApp.deploy.parallel === true it is shown in a concurrent banner rather + * than as a stage/diagram node. */ export const printDependencyGraph = (graph: Schema.Graph): void => { const { components, testApp } = graph const { levels } = topologicalSort(components) const sep = "─".repeat(40) - const edges = components.flatMap(c => (c.dependsOn ?? []).map(dep => ` ${dep} ──▶ ${c.id}`)) - const anyConcurrent = components.some(c => c.deploy.parallel) || testApp.deploy.parallel === true - - // Format a single dependency level into a display string. - // Parallel components are grouped in brackets; sequential follow after an arrow. - // e.g. "[B 🔀 C 🔀] → D E" or "[B 🔀] → C" or "A B" (all sequential) - const formatLevel = (level: Array): string => { - const parallel = level.filter(c => c.deploy.parallel === true) - const sequential = level.filter(c => c.deploy.parallel !== true) - const parallelPart = parallel.length > 0 ? `[${parallel.map(c => `${c.id} 🔀`).join(" ")}]` : "" - const sequentialPart = sequential.map(c => c.id).join(" ") - if (parallelPart && sequentialPart) return `${parallelPart} → ${sequentialPart}` - return parallelPart || sequentialPart - } console.log("Dependency Graph") console.log(sep) + // ── Stage list ──────────────────────────────────────────────────────────── if (testApp.deploy.parallel === true) { - // testApp runs concurrently with the entire component chain — show it in a separate section levels.forEach((level, idx) => console.log(` Stage ${idx + 1} │ ${formatLevel(level)}`) ) console.log(sep) console.log(` ${testApp.id} 🔀 (concurrent with component stages)`) } else { - // testApp runs after all components — append it as the final stage levels.forEach((level, idx) => console.log(` Stage ${idx + 1} │ ${formatLevel(level)}`) ) console.log(` Stage ${levels.length + 1} │ ${testApp.id}`) } - if (edges.length > 0) { - console.log(sep) - edges.forEach(e => console.log(e)) - } - if (anyConcurrent) { - console.log(sep) - console.log(" 🔀 = concurrent deployment") - } + // ── ASCII art diagram ───────────────────────────────────────────────────── + console.log(sep) + const mermaidSrc = buildMermaidSrc(levels, components, testApp) + const originalDebug = console.debug + console.debug = () => {} + const asciiArt = mermaidToAscii(mermaidSrc) + console.debug = originalDebug + asciiArt.split("\n").forEach(line => console.log(line)) + console.log(sep) } diff --git a/k8s-deployer/test/component-dependency.spec.ts b/k8s-deployer/test/component-dependency.spec.ts index 2768417..989e3be 100644 --- a/k8s-deployer/test/component-dependency.spec.ts +++ b/k8s-deployer/test/component-dependency.spec.ts @@ -79,6 +79,168 @@ describe("Component Dependency Tests", () => { }) }) + describe("Parallel Deploy Flag", () => { + it("should load and sort components with parallel deploy flag set", async () => { + const pitfilePath = path.join(__dirname, "pitfile", "test-pitfile-valid-with-parallel-deploy.yml") + const pitfile = await PifFileLoader.loadFromFile(pitfilePath) + const testSuite = pitfile.testSuites[0] + const components = testSuite.deployment.graph.components + + // Should validate successfully + expect(() => validateDependencies(components, testSuite.name)).not.to.throw() + + const sortResult = topologicalSort(components) + + // Level 0: no dependencies — database, message-queue, config-service, metrics-collector + expect(sortResult.levels[0].map(c => c.id)).to.deep.equal([ + "database", "message-queue", "config-service", "metrics-collector" + ]) + + // Level 1: api-service (depends on database + message-queue), cache-service (depends on database) + expect(sortResult.levels[1].map(c => c.id)).to.deep.equal(["api-service", "cache-service"]) + + // Level 2: backend-for-frontend (depends on api-service + cache-service) + expect(sortResult.levels[2].map(c => c.id)).to.deep.equal(["backend-for-frontend"]) + + // Level 3: frontend (depends on backend-for-frontend) + expect(sortResult.levels[3].map(c => c.id)).to.deep.equal(["frontend"]) + + // Parallel flags are preserved through sorting + const db = components.find(c => c.id === "database")! + const configService = components.find(c => c.id === "config-service")! + const apiService = components.find(c => c.id === "api-service")! + const cacheService = components.find(c => c.id === "cache-service")! + + expect(db.deploy.parallel).to.be.true + expect(apiService.deploy.parallel).to.be.true + expect(configService.deploy.parallel).to.be.undefined // sequential + expect(cacheService.deploy.parallel).to.be.undefined // sequential, despite depending on a parallel component + }) + + it("testApp parallel flag is independent of component parallel flags", async () => { + const pitfilePath = path.join(__dirname, "pitfile", "test-pitfile-valid-with-parallel-deploy.yml") + const pitfile = await PifFileLoader.loadFromFile(pitfilePath) + const testSuite = pitfile.testSuites[0] + + expect(testSuite.deployment.graph.testApp.deploy.parallel).to.be.true + }) + + it("multi-stage graph: mixed parallel/sequential at every level, cross-stage ancestry, fan-in convergence", async () => { + const pitfilePath = path.join(__dirname, "pitfile", "test-pitfile-valid-with-parallel-subgroups.yml") + const pitfile = await PifFileLoader.loadFromFile(pitfilePath) + // Suite index 1: multi-stage-mixed + const testSuite = pitfile.testSuites[1] + const components = testSuite.deployment.graph.components + + expect(() => validateDependencies(components, testSuite.name)).not.to.throw() + + const sortResult = topologicalSort(components) + expect(sortResult.levels).to.have.length(5) + + // Stage 0: two independent roots — Infra (parallel) and Config (sequential) + expect(sortResult.levels[0].map(c => c.id)).to.deep.equal(["infra", "config"]) + expect(components.find(c => c.id === "infra")!.deploy.parallel).to.be.true + expect(components.find(c => c.id === "config")!.deploy.parallel).to.be.undefined + + // Stage 1: Auth🔀, Cache🔀 depend only on Infra; Registry depends on both roots (sequential) + expect(sortResult.levels[1].map(c => c.id)).to.deep.equal(["auth", "cache", "registry"]) + expect(components.find(c => c.id === "auth")!.deploy.parallel).to.be.true + expect(components.find(c => c.id === "cache")!.deploy.parallel).to.be.true + expect(components.find(c => c.id === "registry")!.deploy.parallel).to.be.undefined + // Registry's cross-stage ancestry: depends on both stage-0 nodes + expect(components.find(c => c.id === "registry")!.dependsOn).to.deep.equal(["infra", "config"]) + + // Stage 2: API🔀 (Auth+Cache), Worker🔀 (Cache+Registry — one parallel, one sequential parent), Scheduler (Registry) + expect(sortResult.levels[2].map(c => c.id)).to.deep.equal(["api", "worker", "scheduler"]) + expect(components.find(c => c.id === "api")!.deploy.parallel).to.be.true + expect(components.find(c => c.id === "worker")!.deploy.parallel).to.be.true + expect(components.find(c => c.id === "scheduler")!.deploy.parallel).to.be.undefined + expect(components.find(c => c.id === "worker")!.dependsOn).to.deep.equal(["cache", "registry"]) + + // Stage 3: Gateway — sequential fan-in from all three stage-2 nodes + expect(sortResult.levels[3].map(c => c.id)).to.deep.equal(["gateway"]) + expect(components.find(c => c.id === "gateway")!.deploy.parallel).to.be.undefined + expect(components.find(c => c.id === "gateway")!.dependsOn).to.deep.equal(["api", "worker", "scheduler"]) + + // Stage 4: Frontend🔀 and Admin🔀 both depend on Gateway + expect(sortResult.levels[4].map(c => c.id)).to.deep.equal(["frontend", "admin"]) + expect(components.find(c => c.id === "frontend")!.deploy.parallel).to.be.true + expect(components.find(c => c.id === "admin")!.deploy.parallel).to.be.true + + // Undeployment: reverse stages, within each stage order is preserved + const undeployOrder = reverseTopologicalSort(sortResult).map(c => c.id) + expect(undeployOrder).to.deep.equal([ + "frontend", "admin", + "gateway", + "api", "worker", "scheduler", + "auth", "cache", "registry", + "infra", "config" + ]) + + // testApp is marked parallel (runs concurrently with component stages) + expect(testSuite.deployment.graph.testApp.deploy.parallel).to.be.true + }) + + it("A -> [B🔀, C🔀, D🔀] -> E: all middle components parallel, flanked by sequential nodes", async () => { + const pitfilePath = path.join(__dirname, "pitfile", "test-pitfile-valid-with-parallel-subgroups.yml") + const pitfile = await PifFileLoader.loadFromFile(pitfilePath) + // Suite index 0: all-parallel-middle + const testSuite = pitfile.testSuites[0] + const components = testSuite.deployment.graph.components + + expect(() => validateDependencies(components, testSuite.name)).not.to.throw() + + const sortResult = topologicalSort(components) + + // Level 0: A (no dependencies, sequential) + expect(sortResult.levels[0].map(c => c.id)).to.deep.equal(["node-a"]) + expect(components.find(c => c.id === "node-a")!.deploy.parallel).to.be.undefined + + // Level 1: B, C, D (all depend on A, all parallel) + expect(sortResult.levels[1].map(c => c.id)).to.deep.equal(["node-b", "node-c", "node-d"]) + expect(components.find(c => c.id === "node-b")!.deploy.parallel).to.be.true + expect(components.find(c => c.id === "node-c")!.deploy.parallel).to.be.true + expect(components.find(c => c.id === "node-d")!.deploy.parallel).to.be.true + + // Level 2: E (depends on B, C, D — sequential) + expect(sortResult.levels[2].map(c => c.id)).to.deep.equal(["node-e"]) + expect(components.find(c => c.id === "node-e")!.deploy.parallel).to.be.undefined + + // Undeployment reverses: E -> B,C,D -> A + const undeployOrder = reverseTopologicalSort(sortResult).map(c => c.id) + expect(undeployOrder).to.deep.equal(["node-e", "node-b", "node-c", "node-d", "node-a"]) + }) + + it("A -> [B🔀, C🔀, D] -> E: mixed parallel/sequential middle layer, E depends on all three", async () => { + const pitfilePath = path.join(__dirname, "pitfile", "test-pitfile-valid-with-parallel-subgroups.yml") + const pitfile = await PifFileLoader.loadFromFile(pitfilePath) + // Suite index 2: mixed-parallel-middle + const testSuite = pitfile.testSuites[2] + const components = testSuite.deployment.graph.components + + expect(() => validateDependencies(components, testSuite.name)).not.to.throw() + + const sortResult = topologicalSort(components) + + // Level 0: A + expect(sortResult.levels[0].map(c => c.id)).to.deep.equal(["node-a"]) + + // Level 1: B, C, D — same dependency level; B and C parallel, D sequential + expect(sortResult.levels[1].map(c => c.id)).to.deep.equal(["node-b", "node-c", "node-d"]) + expect(components.find(c => c.id === "node-b")!.deploy.parallel).to.be.true + expect(components.find(c => c.id === "node-c")!.deploy.parallel).to.be.true + expect(components.find(c => c.id === "node-d")!.deploy.parallel).to.be.undefined // sequential + + // Level 2: E (waits for all of B, C and D) + expect(sortResult.levels[2].map(c => c.id)).to.deep.equal(["node-e"]) + expect(components.find(c => c.id === "node-e")!.deploy.parallel).to.be.undefined + + // Undeployment reverses: E -> B,C,D -> A + const undeployOrder = reverseTopologicalSort(sortResult).map(c => c.id) + expect(undeployOrder).to.deep.equal(["node-e", "node-b", "node-c", "node-d", "node-a"]) + }) + }) + describe("Complex Dependency Scenarios", () => { it("should handle complex dependency graphs correctly", async () => { const pitfilePath = path.join(__dirname, "pitfile", "test-pitfile-valid-with-complex-dependencies.yml") diff --git a/k8s-deployer/test/dependency-resolver.spec.ts b/k8s-deployer/test/dependency-resolver.spec.ts index 2388d7a..64ee5b8 100644 --- a/k8s-deployer/test/dependency-resolver.spec.ts +++ b/k8s-deployer/test/dependency-resolver.spec.ts @@ -395,6 +395,7 @@ describe("Dependency Resolver", () => { describe("printDependencyGraph", () => { let logOutput: string[] let originalLog: typeof console.log + let originalDebug: typeof console.debug const testApp: Schema.DeployableComponent = { name: "test-app", id: "test-app", location: { type: Schema.LocationType.Local }, @@ -403,10 +404,13 @@ describe("Dependency Resolver", () => { beforeEach(() => { logOutput = [] originalLog = console.log + originalDebug = console.debug console.log = (msg?: any) => logOutput.push(String(msg)) + console.debug = () => {} }) afterEach(() => { console.log = originalLog + console.debug = originalDebug }) it("prints graph for components without dependencies", () => { @@ -417,8 +421,14 @@ describe("Dependency Resolver", () => { ] printDependencyGraph({ testApp, components }) expect(logOutput[0]).to.equal("Dependency Graph") + // Stage list expect(logOutput).to.include(" Stage 1 │ a b c") expect(logOutput).to.include(" Stage 2 │ test-app") + // ASCII art: box lines contain each node id + expect(logOutput.some(l => l.includes("│") && l.includes(" a "))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes(" b "))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes(" c "))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("test-app"))).to.be.true }) it("prints graph for components with dependencies at multiple stages", () => { @@ -429,11 +439,15 @@ describe("Dependency Resolver", () => { ] printDependencyGraph({ testApp, components }) expect(logOutput[0]).to.equal("Dependency Graph") + // Stage list expect(logOutput).to.include(" Stage 1 │ db") expect(logOutput).to.include(" Stage 2 │ api cache") expect(logOutput).to.include(" Stage 3 │ test-app") - expect(logOutput).to.include(" db ──▶ api") - expect(logOutput).to.include(" db ──▶ cache") + // ASCII art + expect(logOutput.some(l => l.includes("│") && l.includes("db"))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("api"))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("cache"))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("test-app"))).to.be.true }) it("prints graph for chained dependencies", () => { @@ -444,12 +458,16 @@ describe("Dependency Resolver", () => { ] printDependencyGraph({ testApp, components }) expect(logOutput[0]).to.equal("Dependency Graph") + // Stage list expect(logOutput).to.include(" Stage 1 │ a") expect(logOutput).to.include(" Stage 2 │ b") expect(logOutput).to.include(" Stage 3 │ c") expect(logOutput).to.include(" Stage 4 │ test-app") - expect(logOutput).to.include(" a ──▶ b") - expect(logOutput).to.include(" b ──▶ c") + // ASCII art + expect(logOutput.some(l => l.includes("│") && l.includes(" a "))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes(" b "))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes(" c "))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("test-app"))).to.be.true }) it("annotates parallel components with 🔀 and prints a legend", () => { @@ -460,12 +478,14 @@ describe("Dependency Resolver", () => { ] printDependencyGraph({ testApp, components }) expect(logOutput[0]).to.equal("Dependency Graph") + // Stage list expect(logOutput).to.include(" Stage 1 │ a") expect(logOutput).to.include(" Stage 2 │ [b 🔀 c 🔀]") expect(logOutput).to.include(" Stage 3 │ test-app") - expect(logOutput).to.include(" a ──▶ b") - expect(logOutput).to.include(" a ──▶ c") - expect(logOutput).to.include(" 🔀 = concurrent deployment") + // ASCII art: parallel nodes have _🔀 suffix in their box label + expect(logOutput.some(l => l.includes("│") && l.includes(" a "))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("b_🔀"))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("c_🔀"))).to.be.true }) it("shows mixed parallel/sequential at same level as [parallel] → sequential", () => { @@ -476,10 +496,14 @@ describe("Dependency Resolver", () => { ] printDependencyGraph({ testApp, components }) expect(logOutput[0]).to.equal("Dependency Graph") + // Stage list expect(logOutput).to.include(" Stage 1 │ a") expect(logOutput).to.include(" Stage 2 │ [b 🔀] → c") expect(logOutput).to.include(" Stage 3 │ test-app") - expect(logOutput).to.include(" 🔀 = concurrent deployment") + // ASCII art + expect(logOutput.some(l => l.includes("│") && l.includes(" a "))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("b_🔀"))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes(" c "))).to.be.true }) it("shows testApp in a concurrent section when testApp.parallel is true", () => { @@ -490,11 +514,90 @@ describe("Dependency Resolver", () => { ] printDependencyGraph({ testApp: parallelTestApp, components }) expect(logOutput[0]).to.equal("Dependency Graph") + // Stage list expect(logOutput).to.include(" Stage 1 │ a") expect(logOutput).to.include(" Stage 2 │ b") expect(logOutput).to.include(" my-test-app 🔀 (concurrent with component stages)") expect(logOutput).to.not.include(" Stage 3 │ my-test-app") - expect(logOutput).to.include(" 🔀 = concurrent deployment") + // ASCII art: testApp does NOT appear as a box (it's concurrent) + expect(logOutput.some(l => l.includes("│") && l.includes("my-test-app"))).to.be.false + expect(logOutput.some(l => l.includes("│") && l.includes(" a "))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes(" b "))).to.be.true + }) + + it("A -> [B🔀, C🔀, D🔀] -> E: parallel middle layer shown in stage list brackets, ASCII art has _🔀 labels", () => { + const parallelTestApp: Schema.DeployableComponent = { ...testApp, deploy: { ...testApp.deploy, parallel: true } } + const components: Array = [ + { name: "A", id: "node-a", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } }, + { name: "B", id: "node-b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["node-a"] }, + { name: "C", id: "node-c", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["node-a"] }, + { name: "D", id: "node-d", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["node-a"] }, + { name: "E", id: "node-e", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["node-b", "node-c", "node-d"] } + ] + printDependencyGraph({ testApp: parallelTestApp, components }) + // Stage list + expect(logOutput).to.include(" Stage 1 │ node-a") + expect(logOutput).to.include(" Stage 2 │ [node-b 🔀 node-c 🔀 node-d 🔀]") + expect(logOutput).to.include(" Stage 3 │ node-e") + // testApp concurrent + expect(logOutput).to.include(" test-app 🔀 (concurrent with component stages)") + expect(logOutput).to.not.include(" Stage 4 │ test-app") + // ASCII art: parallel nodes use _🔀 suffix; sequential node-a and node-e are plain + expect(logOutput.some(l => l.includes("│") && l.includes("node-a"))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("node-b_🔀"))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("node-c_🔀"))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("node-d_🔀"))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("node-e"))).to.be.true + }) + + it("A -> [B🔀, C🔀, D] -> E: mixed middle layer shown as [parallel] → sequential in stage list", () => { + const components: Array = [ + { name: "A", id: "node-a", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } }, + { name: "B", id: "node-b", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["node-a"] }, + { name: "C", id: "node-c", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["node-a"] }, + { name: "D", id: "node-d", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["node-a"] }, + { name: "E", id: "node-e", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["node-b", "node-c", "node-d"] } + ] + printDependencyGraph({ testApp, components }) + // Stage list + expect(logOutput).to.include(" Stage 1 │ node-a") + expect(logOutput).to.include(" Stage 2 │ [node-b 🔀 node-c 🔀] → node-d") + expect(logOutput).to.include(" Stage 3 │ node-e") + expect(logOutput).to.include(" Stage 4 │ test-app") + // ASCII art + expect(logOutput.some(l => l.includes("│") && l.includes("node-b_🔀"))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("node-c_🔀"))).to.be.true + expect(logOutput.some(l => l.includes("│") && l.includes("node-d"))).to.be.true + }) + + it("multi-stage graph: stage list and ASCII art correct, concurrent testApp banner, cross-ancestry edges", () => { + const parallelTestApp: Schema.DeployableComponent = { ...testApp, id: "test-app", deploy: { ...testApp.deploy, parallel: true } } + const components: Array = [ + { name: "Infra", id: "infra", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" } }, + { name: "Config", id: "config", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" } }, + { name: "Auth", id: "auth", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["infra"] }, + { name: "Cache", id: "cache", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["infra"] }, + { name: "Registry", id: "registry", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["infra", "config"] }, + { name: "API", id: "api", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["auth", "cache"] }, + { name: "Worker", id: "worker", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["cache", "registry"] }, + { name: "Scheduler", id: "scheduler", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["registry"] }, + { name: "Gateway", id: "gateway", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh" }, undeploy: { command: "undeploy.sh" }, dependsOn: ["api", "worker", "scheduler"] }, + { name: "Frontend", id: "frontend", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["gateway"] }, + { name: "Admin", id: "admin", location: { type: Schema.LocationType.Local }, deploy: { command: "deploy.sh", parallel: true }, undeploy: { command: "undeploy.sh" }, dependsOn: ["gateway"] } + ] + printDependencyGraph({ testApp: parallelTestApp, components }) + // Stage list + expect(logOutput).to.include(" Stage 1 │ [infra 🔀] → config") + expect(logOutput).to.include(" Stage 2 │ [auth 🔀 cache 🔀] → registry") + expect(logOutput).to.include(" Stage 3 │ [api 🔀 worker 🔀] → scheduler") + expect(logOutput).to.include(" Stage 4 │ gateway") + expect(logOutput).to.include(" Stage 5 │ [frontend 🔀 admin 🔀]") + // testApp concurrent + expect(logOutput).to.include(" test-app 🔀 (concurrent with component stages)") + // ASCII art: all node ids appear in box lines + for (const id of ["infra_🔀", "config", "auth_🔀", "cache_🔀", "registry", "api_🔀", "worker_🔀", "scheduler", "gateway", "frontend_🔀", "admin_🔀"]) { + expect(logOutput.some(l => l.includes("│") && l.includes(id)), `expected box for ${id}`).to.be.true + } }) }) }) diff --git a/k8s-deployer/test/pitfile/pitfile-loader.spec.ts b/k8s-deployer/test/pitfile/pitfile-loader.spec.ts index 13402d3..2cf8314 100644 --- a/k8s-deployer/test/pitfile/pitfile-loader.spec.ts +++ b/k8s-deployer/test/pitfile/pitfile-loader.spec.ts @@ -79,6 +79,41 @@ describe("Loads pitfile from disk", () => { chai.expect(frontend!.dependsOn).deep.equal(["api-service", "cache"]) }) + it("should load pitfile with parallel deploy flag", async () => { + const file = await PifFileLoader.loadFromFile("dist/test/pitfile/test-pitfile-valid-with-parallel-deploy.yml") + chai.expect(file.projectName).eq("Parallel Deploy Test Example") + chai.expect(file.testSuites).lengthOf(1) + + const testSuite = file.testSuites[0] + chai.expect(testSuite.deployment.graph.components).lengthOf(8) + + const components = testSuite.deployment.graph.components + const database = components.find(c => c.id === "database") + const messageQueue = components.find(c => c.id === "message-queue") + const configService = components.find(c => c.id === "config-service") + const apiService = components.find(c => c.id === "api-service") + const cacheService = components.find(c => c.id === "cache-service") + const bff = components.find(c => c.id === "backend-for-frontend") + + // Components with parallel: true + chai.expect(database!.deploy.parallel).to.be.true + chai.expect(messageQueue!.deploy.parallel).to.be.true + chai.expect(apiService!.deploy.parallel).to.be.true + chai.expect(bff!.deploy.parallel).to.be.true + + // Component without parallel flag defaults to undefined (sequential) + chai.expect(configService!.deploy.parallel).to.be.undefined + chai.expect(cacheService!.deploy.parallel).to.be.undefined + + // dependsOn combined with parallel + chai.expect(apiService!.dependsOn).to.deep.equal(["database", "message-queue"]) + chai.expect(cacheService!.dependsOn).to.deep.equal(["database"]) + chai.expect(bff!.dependsOn).to.deep.equal(["api-service", "cache-service"]) + + // testApp also has parallel: true + chai.expect(testSuite.deployment.graph.testApp.deploy.parallel).to.be.true + }) + it("should load pitfile with cyclic dependencies", async () => { const file = await PifFileLoader.loadFromFile("dist/test/pitfile/test-pitfile-invalid-with-cyclic-dependencies.yml") chai.expect(file.projectName).eq("Cyclic Dependency Test Example") diff --git a/k8s-deployer/test/pitfile/test-pitfile-valid-with-parallel-deploy.yml b/k8s-deployer/test/pitfile/test-pitfile-valid-with-parallel-deploy.yml new file mode 100644 index 0000000..cc2bc9c --- /dev/null +++ b/k8s-deployer/test/pitfile/test-pitfile-valid-with-parallel-deploy.yml @@ -0,0 +1,102 @@ +version: 1.0 +projectName: Parallel Deploy Test Example + +lockManager: + enabled: false + +testSuites: + - name: Parallel Deploy Test Suite + id: parallel-deploy-test + + deployment: + graph: + testApp: + name: Test App + id: test-app + location: + type: LOCAL + deploy: + command: "echo 'test app deployed'" + parallel: true + undeploy: + command: "echo 'test app undeployed'" + + components: + # Two independent components that deploy in parallel with each other + - name: Database + id: database + deploy: + command: "echo 'database deployed'" + parallel: true + undeploy: + command: "echo 'database undeployed'" + + - name: Message Queue + id: message-queue + deploy: + command: "echo 'message queue deployed'" + parallel: true + undeploy: + command: "echo 'message queue undeployed'" + + # Sequential component with no dependencies (no parallel flag) + - name: Config Service + id: config-service + deploy: + command: "echo 'config service deployed'" + undeploy: + command: "echo 'config service undeployed'" + + # Component that depends on two parallel components, itself runs in parallel at its level + - name: API Service + id: api-service + dependsOn: + - database + - message-queue + deploy: + command: "echo 'api-service deployed'" + parallel: true + undeploy: + command: "echo 'api-service undeployed'" + + # Component that depends on one parallel component, itself sequential at its level + - name: Cache Service + id: cache-service + dependsOn: + - database + deploy: + command: "echo 'cache service deployed'" + undeploy: + command: "echo 'cache service undeployed'" + + # Component that depends on a mix of parallel and sequential upstream components + - name: Backend For Frontend + id: backend-for-frontend + dependsOn: + - api-service + - cache-service + deploy: + command: "echo 'backend for frontend deployed'" + parallel: true + undeploy: + command: "echo 'backend for frontend undeployed'" + + # Leaf component with a single dependency, parallel at its level + - name: Frontend + id: frontend + dependsOn: + - backend-for-frontend + deploy: + command: "echo 'frontend deployed'" + parallel: true + undeploy: + command: "echo 'frontend undeployed'" + + # Independent component that is parallel and shares its level with database/message-queue + - name: Metrics Collector + id: metrics-collector + deploy: + command: "echo 'metrics collector deployed'" + parallel: true + undeploy: + command: "echo 'metrics collector undeployed'" diff --git a/k8s-deployer/test/pitfile/test-pitfile-valid-with-parallel-subgroups.yml b/k8s-deployer/test/pitfile/test-pitfile-valid-with-parallel-subgroups.yml new file mode 100644 index 0000000..f47e236 --- /dev/null +++ b/k8s-deployer/test/pitfile/test-pitfile-valid-with-parallel-subgroups.yml @@ -0,0 +1,287 @@ +version: 1.0 +projectName: Parallel Subgroup Test Example + +lockManager: + enabled: false + +testSuites: + # Scenario 1: A -> [B, C, D] -> E where B, C and D are all parallel + # A has no dependencies and is sequential. + # B, C, D each depend on A and are all marked parallel (they run concurrently). + # E depends on B, C and D and is sequential. + - name: All-Parallel Middle Layer + id: all-parallel-middle + + deployment: + graph: + testApp: + name: Test App + id: test-app + location: + type: LOCAL + deploy: + command: "echo 'test app deployed'" + undeploy: + command: "echo 'test app undeployed'" + + components: + - name: A + id: node-a + deploy: + command: "echo 'node-a deployed'" + undeploy: + command: "echo 'node-a undeployed'" + + - name: B + id: node-b + dependsOn: + - node-a + deploy: + command: "echo 'node-b deployed'" + parallel: true + undeploy: + command: "echo 'node-b undeployed'" + + - name: C + id: node-c + dependsOn: + - node-a + deploy: + command: "echo 'node-c deployed'" + parallel: true + undeploy: + command: "echo 'node-c undeployed'" + + - name: D + id: node-d + dependsOn: + - node-a + deploy: + command: "echo 'node-d deployed'" + parallel: true + undeploy: + command: "echo 'node-d undeployed'" + + - name: E + id: node-e + dependsOn: + - node-b + - node-c + - node-d + deploy: + command: "echo 'node-e deployed'" + undeploy: + command: "echo 'node-e undeployed'" + + # Scenario 3: Multi-stage graph with mixed parallel/sequential at every level and + # components depending on ancestors from more than one upstream stage. + # + # Stage 0: [Infra🔀, Config] + # Stage 1: [Auth🔀, Cache🔀, Registry] + # - Auth depends on Infra (parallel) + # - Cache depends on Infra (parallel) + # - Registry depends on Infra + Config (sequential — spans both stage-0 roots) + # Stage 2: [API🔀, Worker🔀, Scheduler] + # - API depends on Auth + Cache (parallel) + # - Worker depends on Cache + Registry (parallel — cross-ancestry: one parallel, one sequential parent) + # - Scheduler depends on Registry (sequential) + # Stage 3: [Gateway] + # - depends on API + Worker + Scheduler (sequential — fan-in from all stage-2 nodes) + # Stage 4: [Frontend🔀, Admin🔀] + # - both depend on Gateway (parallel) + - name: Multi-Stage Mixed Parallel Graph + id: multi-stage-mixed + + deployment: + graph: + testApp: + name: Test App + id: test-app + location: + type: LOCAL + deploy: + command: "echo 'test app deployed'" + parallel: true + undeploy: + command: "echo 'test app undeployed'" + + components: + # Stage 0 — two independent roots + - name: Infra + id: infra + deploy: + command: "echo 'infra deployed'" + parallel: true + undeploy: + command: "echo 'infra undeployed'" + + - name: Config + id: config + deploy: + command: "echo 'config deployed'" + undeploy: + command: "echo 'config undeployed'" + + # Stage 1 — Auth and Cache depend only on Infra (parallel); + # Registry depends on both roots (sequential) + - name: Auth + id: auth + dependsOn: + - infra + deploy: + command: "echo 'auth deployed'" + parallel: true + undeploy: + command: "echo 'auth undeployed'" + + - name: Cache + id: cache + dependsOn: + - infra + deploy: + command: "echo 'cache deployed'" + parallel: true + undeploy: + command: "echo 'cache undeployed'" + + - name: Registry + id: registry + dependsOn: + - infra + - config + deploy: + command: "echo 'registry deployed'" + undeploy: + command: "echo 'registry undeployed'" + + # Stage 2 — API and Worker are parallel; Scheduler is sequential. + # Worker's parents span a parallel (Cache) and a sequential (Registry) node. + - name: API + id: api + dependsOn: + - auth + - cache + deploy: + command: "echo 'api deployed'" + parallel: true + undeploy: + command: "echo 'api undeployed'" + + - name: Worker + id: worker + dependsOn: + - cache + - registry + deploy: + command: "echo 'worker deployed'" + parallel: true + undeploy: + command: "echo 'worker undeployed'" + + - name: Scheduler + id: scheduler + dependsOn: + - registry + deploy: + command: "echo 'scheduler deployed'" + undeploy: + command: "echo 'scheduler undeployed'" + + # Stage 3 — Gateway fans in from all three stage-2 nodes; sequential + - name: Gateway + id: gateway + dependsOn: + - api + - worker + - scheduler + deploy: + command: "echo 'gateway deployed'" + undeploy: + command: "echo 'gateway undeployed'" + + # Stage 4 — Frontend and Admin both depend on Gateway; both parallel + - name: Frontend + id: frontend + dependsOn: + - gateway + deploy: + command: "echo 'frontend deployed'" + parallel: true + undeploy: + command: "echo 'frontend undeployed'" + + - name: Admin + id: admin + dependsOn: + - gateway + deploy: + command: "echo 'admin deployed'" + parallel: true + undeploy: + command: "echo 'admin undeployed'" + + # Scenario 2: A -> [B(parallel), C(parallel), D(sequential)] -> E + # E depends on B, C and D, but only B and C are marked parallel. + # D is sequential at the same dependency level — it must complete before E starts. + - name: Mixed Parallel and Sequential Middle Layer + id: mixed-parallel-middle + + deployment: + graph: + testApp: + name: Test App + id: test-app + location: + type: LOCAL + deploy: + command: "echo 'test app deployed'" + undeploy: + command: "echo 'test app undeployed'" + + components: + - name: A + id: node-a + deploy: + command: "echo 'node-a deployed'" + undeploy: + command: "echo 'node-a undeployed'" + + - name: B + id: node-b + dependsOn: + - node-a + deploy: + command: "echo 'node-b deployed'" + parallel: true + undeploy: + command: "echo 'node-b undeployed'" + + - name: C + id: node-c + dependsOn: + - node-a + deploy: + command: "echo 'node-c deployed'" + parallel: true + undeploy: + command: "echo 'node-c undeployed'" + + - name: D + id: node-d + dependsOn: + - node-a + deploy: + command: "echo 'node-d deployed'" + undeploy: + command: "echo 'node-d undeployed'" + + - name: E + id: node-e + dependsOn: + - node-b + - node-c + - node-d + deploy: + command: "echo 'node-e deployed'" + undeploy: + command: "echo 'node-e undeployed'" From cd92ade2fff4073d94d930b06dadaef22b406aa6 Mon Sep 17 00:00:00 2001 From: "James Charters james.charters@kindredgroup.com" Date: Fri, 13 Mar 2026 09:44:23 +1100 Subject: [PATCH 12/12] feat: [KSBP-10176] - Update README --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4daf3be..170050f 100644 --- a/README.md +++ b/README.md @@ -141,8 +141,9 @@ testSuites: name: Talos Certifier Test App id: talos-certifier-test-app location: - type: LOCAL # optional, defautls to 'LOCAL' + type: LOCAL # optional, defaults to 'LOCAL' deploy: + parallel: true # optional, default is false timeoutSeconds: 120 command: deployment/pit/deploy.sh params: # Optional command line parameters @@ -161,6 +162,7 @@ testSuites: location: type: LOCAL deploy: + parallel: true # optional, default is false command: deployment/pit/deploy.sh statusCheck: command: deployment/pit/is-deployment-ready.sh @@ -173,7 +175,10 @@ testSuites: # Lets assume Talos Certifier and Replicator (made for testing Talos Certifier) are in the same repository location: type: LOCAL + dependsOn: + - talos-certifier # optional, deployment of component will not be attempted until these dependencies are up and healthy deploy: + parallel: true # optional, default is false command: deployment/pit/deploy.sh statusCheck: command: deployment/pit/is-deployment-ready.sh @@ -189,6 +194,7 @@ testSuites: gitRepository: git://127.0.0.1/some-other-component.git gitRef: # Optional, defaults to "refs/remotes/origin/master" deploy: + parallel: false # optional command: deployment/pit/deploy.sh statusCheck: command: deployment/pit/is-deployment-ready.sh