From af1a7e285831c0fd7195b284ce06385abf52f2ed Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Wed, 24 Jun 2026 10:09:10 +0100 Subject: [PATCH 01/33] feat: enforce using actor.json as source of truth and fallback to generic builder token --- bin/build.ts | 9 ++++----- bin/utils.ts | 51 ++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/bin/build.ts b/bin/build.ts index a5784c7..6bbad1d 100644 --- a/bin/build.ts +++ b/bin/build.ts @@ -115,19 +115,18 @@ class ApifyBuilder { */ static fromActorName = (actorName: string): ApifyBuilder => { const username = actorName.split('/')[0]; - // GitHib secrets only allow word characters (alphanum + underscore) + // GitHub secrets only allow word characters (alphanum + underscore) const usernameInGitHubSecretsFormat = username.replaceAll(/\W/g, '_').toUpperCase(); const usernameEnvVar = `APIFY_TOKEN_${usernameInGitHubSecretsFormat}`; - const token = process.env[usernameEnvVar]; + const token = process.env[usernameEnvVar] ?? process.env.BUILDER_APIFY_TOKEN; if (!token) { throw new Error( `Cannot find Apify API token for username: ${username}. ` + - `Have you set secret env var to this GitHub repo with key: ${usernameEnvVar}?`, + `Have you set secret env var ${usernameEnvVar} or BUILDER_APIFY_TOKEN as a fallback?`, ); } const apifyClient = new ApifyClient({ token }); - const builder = new ApifyBuilder(apifyClient, actorName); - return builder; + return new ApifyBuilder(apifyClient, actorName); }; /** diff --git a/bin/utils.ts b/bin/utils.ts index 477a6b5..78ce6c5 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -1,6 +1,8 @@ import { spawnSync } from 'node:child_process'; import fs from 'node:fs/promises'; +import { ApifyClient } from 'apify-client'; + import type { ActorConfig } from './types.js'; export const spawnCommandInGhWorkspace = (command: string, args: string[] = []) => { @@ -29,6 +31,23 @@ export const getEnvVar = (varName: string, defaultValue?: string): string => { return value; }; +let cachedBuilderUsername: string | undefined; + +export const resolveBuilderTokenUsername = async (): Promise => { + if (cachedBuilderUsername) return cachedBuilderUsername; + const token = process.env.BUILDER_APIFY_TOKEN; + if (!token) { + throw new Error( + 'BUILDER_APIFY_TOKEN is not set. Either use the "owner_actor-name" folder convention ' + + 'or set the BUILDER_APIFY_TOKEN secret.', + ); + } + const client = new ApifyClient({ token }); + const user = await client.user().get(); + cachedBuilderUsername = user.username!; + return cachedBuilderUsername; +}; + /** * Reads and parses all directories in `actors` directory * This works locally if checkoutRepoLocally is called first @@ -50,13 +69,35 @@ export const getRepoActors = async (): Promise => { } const actorConfigs: ActorConfig[] = []; for (const actorDir of [...actorDirs, ...standaloneActorDirs]) { - const match = actorDir.match(/^([^/]+)\/(.+)_([^_]+)$/); - if (!match) { - throw new Error(`Invalid actor directory name. Got "${actorDir}", expected "actor.owner-name_actor-name"`); + const actorJsonPath = `./${actorDir}/.actor/actor.json`; + let actorJson: { name?: string }; + try { + actorJson = JSON.parse(await fs.readFile(actorJsonPath, 'utf-8')); + } catch { + throw new Error( + `Missing or unreadable .actor/actor.json in "${actorDir}". ` + + `Every actor folder must contain .actor/actor.json with a "name" field.`, + ); } - const [, folderType, owner, actorName] = match; + if (!actorJson.name) { + throw new Error( + `Missing "name" field in "${actorJsonPath}". ` + + `Every actor folder must have .actor/actor.json with a "name" field.`, + ); + } + + const folderName = actorDir.split('/')[1]; + const folderType = actorDir.split('/')[0]; + const ownerMatch = folderName.match(/^(.+)_[^_]+$/); + let owner: string; + if (ownerMatch) { + owner = ownerMatch[1]; + } else { + owner = await resolveBuilderTokenUsername(); + } + actorConfigs.push({ - actorName: `${owner}/${actorName}`, + actorName: `${owner}/${actorJson.name}`, folder: actorDir, isStandalone: folderType === 'standalone-actors', }); From e4bb717a7351726abfae58e0742606f76b0759b0 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Wed, 24 Jun 2026 10:18:56 +0100 Subject: [PATCH 02/33] feat: support building from single actor repo --- bin/diff-changes.ts | 24 +++++++++++++++--------- bin/utils.ts | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index 7af55e0..4d9180c 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -22,8 +22,7 @@ export const maybeParseActorFolder = ( /** * Also works for folders */ -const isIgnoredTopLevelFile = (lowercaseFilePath: string) => { - // On top level, we should only have dev-only readme and .actor/ is just for apify push CLI (real Actor configs are in /actors) +const isIgnoredTopLevelFile = (lowercaseFilePath: string, isSingleActorRepo: boolean) => { const IGNORED_TOP_LEVEL_FILES = [ '.vscode/', '.gitignore', @@ -33,7 +32,8 @@ const isIgnoredTopLevelFile = (lowercaseFilePath: string) => { 'eslint.config.mjs', '.prettierrc', '.editorconfig', - '.actor/', + // In root .actor/ mode, .actor/ changes must trigger builds + ...(isSingleActorRepo ? [] : ['.actor/']), ]; // Strip out deprecated /code and /shared folders, treat them as top-level code const sanitizedLowercaseFilePath = lowercaseFilePath.replace(/^code\//, '').replace(/^shared\//, ''); @@ -51,10 +51,15 @@ type FileChange = includes: 'all-actors' | ActorConfig; }; -const classifyFileChange = (originalFilePath: string, actorConfigs: ActorConfig[], commits: Commit[]): FileChange => { +const classifyFileChange = ( + originalFilePath: string, + actorConfigs: ActorConfig[], + commits: Commit[], + isSingleActorRepo: boolean, +): FileChange => { // Lowercase for case-insensitive matching; keep original for git show (case-sensitive on Linux) const lowercaseFilePath = originalFilePath.toLowerCase(); - if (isIgnoredTopLevelFile(lowercaseFilePath)) { + if (isIgnoredTopLevelFile(lowercaseFilePath, isSingleActorRepo)) { return { impact: 'ignored' }; } @@ -101,11 +106,12 @@ export const getChangedActors = ({ }: ShouldBuildAndTestOptions): ActorConfig[] => { // folder -> ActorConfig const actorsChangedMap = new Map(); + const isSingleActorRepo = actorConfigs.some(({ folder }) => folder === ''); const actorConfigsWithoutStandalone = actorConfigs.filter(({ isStandalone }) => !isStandalone); for (const originalFilePath of filepathsChanged) { - const fileChange = classifyFileChange(originalFilePath, actorConfigs, commits); + const fileChange = classifyFileChange(originalFilePath, actorConfigs, commits, isSingleActorRepo); if (fileChange.impact === 'ignored') { continue; } @@ -130,12 +136,12 @@ export const getChangedActors = ({ const formatFiles = (files: string[]) => (files.length > 0 ? files.join(', ') : ''); const ignoredFilesChanged = filepathsChanged.filter( - (file) => classifyFileChange(file, actorConfigs, commits).impact === 'ignored', + (file) => classifyFileChange(file, actorConfigs, commits, isSingleActorRepo).impact === 'ignored', ); console.error(`[DIFF]: Ignored files (don't trigger test or build): ${formatFiles(ignoredFilesChanged)}`); const cosmeticChanges = filepathsChanged - .map((file) => ({ file, change: classifyFileChange(file, actorConfigs, commits) })) + .map((file) => ({ file, change: classifyFileChange(file, actorConfigs, commits, isSingleActorRepo) })) .filter(({ change }) => change.impact === 'cosmetic') as { file: string; change: Extract; @@ -154,7 +160,7 @@ export const getChangedActors = ({ ); const functionalFilesChanged = filepathsChanged.filter( - (file) => classifyFileChange(file, actorConfigs, commits).impact === 'functional', + (file) => classifyFileChange(file, actorConfigs, commits, isSingleActorRepo).impact === 'functional', ); console.error(`[DIFF]: Functional files (trigger test & release build): ${formatFiles(functionalFilesChanged)}`); diff --git a/bin/utils.ts b/bin/utils.ts index 78ce6c5..c07b26b 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -67,6 +67,25 @@ export const getRepoActors = async (): Promise => { console.warn(`No /standalone-actors directory found in repo`); standaloneActorDirs = []; } + if (actorDirs.length === 0 && standaloneActorDirs.length === 0) { + const rootActorJsonPath = './.actor/actor.json'; + let actorJson: { name?: string }; + try { + actorJson = JSON.parse(await fs.readFile(rootActorJsonPath, 'utf-8')); + } catch { + return []; + } + if (!actorJson.name) { + throw new Error( + `Missing "name" field in "${rootActorJsonPath}". ` + + `The .actor/actor.json must have a "name" field.`, + ); + } + const owner = await resolveBuilderTokenUsername(); + console.error(`Root .actor/ mode: single actor ${owner}/${actorJson.name}`); + return [{ actorName: `${owner}/${actorJson.name}`, folder: '', isStandalone: false }]; + } + const actorConfigs: ActorConfig[] = []; for (const actorDir of [...actorDirs, ...standaloneActorDirs]) { const actorJsonPath = `./${actorDir}/.actor/actor.json`; From aefdb558d52b3490dcda70c4f93ed435cc679f85 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Wed, 24 Jun 2026 10:34:36 +0100 Subject: [PATCH 03/33] feat: match changed files by folder path instead of reconstructed actor name --- bin/diff-changes.ts | 11 +++++------ test/unit/bin/diff-changes.test.ts | 10 +++++----- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index 4d9180c..9b87e8b 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -10,11 +10,10 @@ interface ShouldBuildAndTestOptions { export const maybeParseActorFolder = ( lowercaseFilePath: string, -): { isActorFolder: true; actorName: string } | { isActorFolder: false } => { - const match = lowercaseFilePath.match(/^(?:standalone-)?actors\/([^/]+)\/.+/); +): { isActorFolder: true; folder: string } | { isActorFolder: false } => { + const match = lowercaseFilePath.match(/^((?:standalone-)?actors\/[^/]+)\/.+/); if (match) { - // Some usernames weirdly use underscores, e.g. google_maps_email_extractor_standby-contact-details-scraper so we only need replace the last one - return { isActorFolder: true, actorName: match[1].replace(/_(?=[^_]*$)/, '/') }; + return { isActorFolder: true, folder: match[1] }; } return { isActorFolder: false }; }; @@ -70,14 +69,14 @@ const classifyFileChange = ( const actorFolderInfo = maybeParseActorFolder(lowercaseFilePath); if (actorFolderInfo.isActorFolder) { const actorConfigChanged = actorConfigs.find( - ({ actorName }) => actorName.toLowerCase() === actorFolderInfo.actorName, + ({ folder }) => folder.toLowerCase() === actorFolderInfo.folder, ); // This is some super weird case that happened once in the past but I don't remember the context anymore if (actorConfigChanged === undefined) { console.error( 'SHOULD NEVER HAPPEN: changes was found in an actor folder which no longer exists in the current commit, skipping this file', { - actorName: actorFolderInfo.actorName, + folder: actorFolderInfo.folder, lowercaseFilePath, }, ); diff --git a/test/unit/bin/diff-changes.test.ts b/test/unit/bin/diff-changes.test.ts index 727c366..592ec60 100644 --- a/test/unit/bin/diff-changes.test.ts +++ b/test/unit/bin/diff-changes.test.ts @@ -6,7 +6,7 @@ import type { ActorConfig } from '../../../bin/types.js'; const miniActor: ActorConfig = { actorName: 'foo/bar', folder: 'actors/foo_bar', isStandalone: false }; const standaloneActor: ActorConfig = { - actorName: 'standalone', + actorName: 'owner/standalone', folder: 'standalone-actors/standalone', isStandalone: true, }; @@ -15,17 +15,17 @@ const actorConfigs = [miniActor, standaloneActor]; const commits = [{ sha: 'Commit1', author: '', date: '', message: '' }]; describe('maybeParseActorFolder', () => { - it('returns actorName for actors/ path', () => { + it('returns folder for actors/ path', () => { expect(maybeParseActorFolder('actors/foo_bar/actor.json')).toEqual({ isActorFolder: true, - actorName: 'foo/bar', + folder: 'actors/foo_bar', }); }); - it('returns actorName for standalone-actors/ path', () => { + it('returns folder for standalone-actors/ path', () => { expect(maybeParseActorFolder('standalone-actors/my_actor/main.ts')).toEqual({ isActorFolder: true, - actorName: 'my/actor', + folder: 'standalone-actors/my_actor', }); }); From bfba4d362eeb2410d00d6d17c01013480adf5f69 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Wed, 24 Jun 2026 10:37:11 +0100 Subject: [PATCH 04/33] add relevant tests --- test/unit/bin/diff-changes.test.ts | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/unit/bin/diff-changes.test.ts b/test/unit/bin/diff-changes.test.ts index 592ec60..6d63469 100644 --- a/test/unit/bin/diff-changes.test.ts +++ b/test/unit/bin/diff-changes.test.ts @@ -37,6 +37,13 @@ describe('maybeParseActorFolder', () => { expect(maybeParseActorFolder('actors/foo_bar')).toEqual({ isActorFolder: false }); }); + it('returns folder for ownerless actors/ path', () => { + expect(maybeParseActorFolder('actors/shopify/src/main.ts')).toEqual({ + isActorFolder: true, + folder: 'actors/shopify', + }); + }); + it('returns false for unrelated folder', () => { expect(maybeParseActorFolder('src/utils.ts')).toEqual({ isActorFolder: false }); }); @@ -171,6 +178,39 @@ describe('getChangedActors', () => { expect(result).toContainEqual(standaloneActor); }); + it('matches ownerless folder where folder name differs from actor name', () => { + const ownerlessActor: ActorConfig = { + actorName: 'myteam/shopify-scraper', + folder: 'actors/shopify', + isStandalone: false, + }; + const result = getChangedActors({ + filepathsChanged: ['actors/shopify/src/main.ts'], + actorConfigs: [ownerlessActor], + commits, + }); + expect(result).toEqual([ownerlessActor]); + }); + + it('in single-actor repo, .actor/ changes trigger builds', () => { + const rootActor: ActorConfig = { actorName: 'myteam/my-actor', folder: '', isStandalone: false }; + const result = getChangedActors({ + filepathsChanged: ['.actor/actor.json'], + actorConfigs: [rootActor], + commits, + }); + expect(result).toEqual([rootActor]); + }); + + it('in multi-actor repo, .actor/ changes are ignored', () => { + const result = getChangedActors({ + filepathsChanged: ['.actor/actor.json'], + actorConfigs, + commits, + }); + expect(result).toEqual([]); + }); + it('file paths are matched case-insensitively', () => { const result = getChangedActors({ filepathsChanged: ['Actors/FOO_BAR/Main.ts'], From 278ae9c2565c7fab41b439c19d39211fdc5baf4d Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Wed, 24 Jun 2026 11:11:33 +0100 Subject: [PATCH 05/33] small refactor --- bin/utils.ts | 53 +++++++++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/bin/utils.ts b/bin/utils.ts index c07b26b..5b0b2eb 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -48,6 +48,23 @@ export const resolveBuilderTokenUsername = async (): Promise => { return cachedBuilderUsername; }; +const readActorName = async (actorJsonPath: string): Promise => { + const actorJson: { name?: string } = JSON.parse(await fs.readFile(actorJsonPath, 'utf-8')); + if (!actorJson.name) { + throw new Error( + `Missing "name" field in "${actorJsonPath}". ` + + `Every actor folder must have .actor/actor.json with a "name" field.`, + ); + } + return actorJson.name; +}; + +const resolveOwner = async (folderName: string): Promise => { + const ownerMatch = folderName.match(/^(.+)_[^_]+$/); + if (ownerMatch) return ownerMatch[1]; + return resolveBuilderTokenUsername(); +}; + /** * Reads and parses all directories in `actors` directory * This works locally if checkoutRepoLocally is called first @@ -68,55 +85,35 @@ export const getRepoActors = async (): Promise => { standaloneActorDirs = []; } if (actorDirs.length === 0 && standaloneActorDirs.length === 0) { - const rootActorJsonPath = './.actor/actor.json'; - let actorJson: { name?: string }; + let actorName: string; try { - actorJson = JSON.parse(await fs.readFile(rootActorJsonPath, 'utf-8')); + actorName = await readActorName('./.actor/actor.json'); } catch { return []; } - if (!actorJson.name) { - throw new Error( - `Missing "name" field in "${rootActorJsonPath}". ` + - `The .actor/actor.json must have a "name" field.`, - ); - } const owner = await resolveBuilderTokenUsername(); - console.error(`Root .actor/ mode: single actor ${owner}/${actorJson.name}`); - return [{ actorName: `${owner}/${actorJson.name}`, folder: '', isStandalone: false }]; + console.error(`Root .actor/ mode: single actor ${owner}/${actorName}`); + return [{ actorName: `${owner}/${actorName}`, folder: '', isStandalone: false }]; } const actorConfigs: ActorConfig[] = []; for (const actorDir of [...actorDirs, ...standaloneActorDirs]) { - const actorJsonPath = `./${actorDir}/.actor/actor.json`; - let actorJson: { name?: string }; + let actorName: string; try { - actorJson = JSON.parse(await fs.readFile(actorJsonPath, 'utf-8')); + actorName = await readActorName(`./${actorDir}/.actor/actor.json`); } catch { throw new Error( `Missing or unreadable .actor/actor.json in "${actorDir}". ` + `Every actor folder must contain .actor/actor.json with a "name" field.`, ); } - if (!actorJson.name) { - throw new Error( - `Missing "name" field in "${actorJsonPath}". ` + - `Every actor folder must have .actor/actor.json with a "name" field.`, - ); - } const folderName = actorDir.split('/')[1]; const folderType = actorDir.split('/')[0]; - const ownerMatch = folderName.match(/^(.+)_[^_]+$/); - let owner: string; - if (ownerMatch) { - owner = ownerMatch[1]; - } else { - owner = await resolveBuilderTokenUsername(); - } + const owner = await resolveOwner(folderName); actorConfigs.push({ - actorName: `${owner}/${actorJson.name}`, + actorName: `${owner}/${actorName}`, folder: actorDir, isStandalone: folderType === 'standalone-actors', }); From 48cd2155d54a7523103972dd809b82070c3b4baf Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Wed, 24 Jun 2026 13:41:12 +0100 Subject: [PATCH 06/33] validate actor on user account --- bin/build.ts | 44 ++++++++++--------------- bin/types.ts | 2 ++ bin/utils.ts | 36 ++++++++++++++++++-- test/unit/bin/diff-changes.test.ts | 6 ++-- test/unit/should-built-and-test.test.ts | 12 +++++++ 5 files changed, 68 insertions(+), 32 deletions(-) diff --git a/bin/build.ts b/bin/build.ts index 6bbad1d..c3f1167 100644 --- a/bin/build.ts +++ b/bin/build.ts @@ -10,7 +10,7 @@ type BuildPrActorOptions = { buildTag?: string; versionNumber: string; gitRepoUrl: string; - actorName: string; + actorConfig: ActorConfig; useDockerCache: boolean; }; class ApifyBuilder { @@ -56,6 +56,7 @@ class ApifyBuilder { buildTag, versionNumber, gitRepoUrl, + actorConfig, useDockerCache, }: BuildPrActorOptions): Promise => { const actorClient = this.apifyClient.actor(this.actorName); @@ -93,7 +94,7 @@ class ApifyBuilder { const { id, actId, buildNumber } = await actorClient.build(versionNumber, { useCache: useDockerCache }); console.error(`[${this.actorName}]: ${id} (${buildNumber})`); - return { buildId: id, actorId: actId, buildNumber, actorName: this.actorName }; + return { buildId: id, actorId: actId, buildNumber, actorName: this.actorName, actorConfig }; }; waitForBuildToFinish = async (buildId: string, actorName: string): Promise => { @@ -110,20 +111,10 @@ class ApifyBuilder { return build; }; - /** - * Create ApifyBuilder with actor owner's token - */ - static fromActorName = (actorName: string): ApifyBuilder => { - const username = actorName.split('/')[0]; - // GitHub secrets only allow word characters (alphanum + underscore) - const usernameInGitHubSecretsFormat = username.replaceAll(/\W/g, '_').toUpperCase(); - const usernameEnvVar = `APIFY_TOKEN_${usernameInGitHubSecretsFormat}`; - const token = process.env[usernameEnvVar] ?? process.env.BUILDER_APIFY_TOKEN; + static fromActorConfig = ({ actorName, tokenEnvVar }: ActorConfig): ApifyBuilder => { + const token = process.env[tokenEnvVar]; if (!token) { - throw new Error( - `Cannot find Apify API token for username: ${username}. ` + - `Have you set secret env var ${usernameEnvVar} or BUILDER_APIFY_TOKEN as a fallback?`, - ); + throw new Error(`Env var ${tokenEnvVar} is not set (needed for actor "${actorName}").`); } const apifyClient = new ApifyClient({ token }); return new ApifyBuilder(apifyClient, actorName); @@ -173,9 +164,7 @@ class ApifyBuilder { const { items } = await this.apifyClient.actor(this.actorName).builds().list(); // Deleting default build throws an error, so we skip it - const { defaultBuildNumber, defaultBuildTag } = await ApifyBuilder.fromActorName( - this.actorName, - ).getDefaultVersionAndTag(); + const { defaultBuildNumber, defaultBuildTag } = await this.getDefaultVersionAndTag(); const daysAgoUnixProd = Date.now() - DEFAULT_DAYS_BACK_PROD_VERSIONS * 24 * 60 * 60 * 1000; const daysAgoUnixDevel = Date.now() - DEFAULT_DAYS_BACK_DEVEL * 24 * 60 * 60 * 1000; @@ -244,13 +233,13 @@ export const runBuilds = async ({ const circleActors = isLatest ? await findCircleApifyManaged(actorConfigs) : []; - for (const { actorName, folder } of actorConfigs.concat(circleActors)) { + for (const actorConfig of actorConfigs.concat(circleActors)) { let versionNumber: string; let buildTag: string | undefined; if (isLatest) { const { defaultVersionNumber, defaultBuildTag } = - await ApifyBuilder.fromActorName(actorName).getDefaultVersionAndTag(); + await ApifyBuilder.fromActorConfig(actorConfig).getDefaultVersionAndTag(); versionNumber = defaultVersionNumber; buildTag = defaultBuildTag; } else { @@ -259,10 +248,10 @@ export const runBuilds = async ({ // Depending on if these are miniactors or standaloneActors let gitRepoUrl = `${repoUrl}#${branch}`; - if (folder) { - gitRepoUrl = `${gitRepoUrl}:${folder}`; + if (actorConfig.folder) { + gitRepoUrl = `${gitRepoUrl}:${actorConfig.folder}`; } - buildConfigs.push({ actorName, gitRepoUrl, versionNumber, buildTag, useDockerCache }); + buildConfigs.push({ actorConfig, gitRepoUrl, versionNumber, buildTag, useDockerCache }); } if (dryRun) { @@ -272,7 +261,7 @@ export const runBuilds = async ({ console.error('STARTED BUILDS:'); const startedBuilds = await Promise.all( buildConfigs.map(async (buildConfig) => { - const builder = ApifyBuilder.fromActorName(buildConfig.actorName); + const builder = ApifyBuilder.fromActorConfig(buildConfig.actorConfig); const buildData = await builder.startActorBuild(buildConfig); return buildData; }), @@ -281,7 +270,7 @@ export const runBuilds = async ({ console.error('FINISHED BUILDS:'); await Promise.all( startedBuilds.map(async (buildData) => { - const builder = ApifyBuilder.fromActorName(buildData.actorName); + const builder = ApifyBuilder.fromActorConfig(buildData.actorConfig); await builder.waitForBuildToFinish(buildData.buildId, buildData.actorName); }), ); @@ -296,8 +285,8 @@ export const runBuilds = async ({ }; export const deleteOldBuilds = async (actorConfigs: ActorConfig[]) => { - for (const { actorName } of actorConfigs) { - await ApifyBuilder.fromActorName(actorName).deleteOldBuilds(); + for (const actorConfig of actorConfigs) { + await ApifyBuilder.fromActorConfig(actorConfig).deleteOldBuilds(); } }; @@ -335,6 +324,7 @@ const findCircleApifyManaged = async (actorConfigs: ActorConfig[]): Promise => { return resolveBuilderTokenUsername(); }; +const resolveTokenEnvVar = (owner: string): string => { + const usernameInGitHubSecretsFormat = owner.replaceAll(/\W/g, '_').toUpperCase(); + const usernameEnvVar = `APIFY_TOKEN_${usernameInGitHubSecretsFormat}`; + if (process.env[usernameEnvVar]) return usernameEnvVar; + if (process.env.BUILDER_APIFY_TOKEN) return 'BUILDER_APIFY_TOKEN'; + throw new Error( + `Cannot find Apify API token for owner "${owner}". ` + + `Set either ${usernameEnvVar} or BUILDER_APIFY_TOKEN.`, + ); +}; + +const validateActorExists = async (actorName: string, tokenEnvVar: string): Promise => { + const client = new ApifyClient({ token: process.env[tokenEnvVar]! }); + const actor = await client.actor(actorName).get(); + if (!actor) { + throw new Error( + `Actor "${actorName}" not found using token from ${tokenEnvVar}. ` + + `If this is a new actor, create it on the Apify platform first. ` + + `Otherwise, check that the folder name matches the actual actor owner.`, + ); + } +}; + /** * Reads and parses all directories in `actors` directory * This works locally if checkoutRepoLocally is called first @@ -92,8 +115,11 @@ export const getRepoActors = async (): Promise => { return []; } const owner = await resolveBuilderTokenUsername(); - console.error(`Root .actor/ mode: single actor ${owner}/${actorName}`); - return [{ actorName: `${owner}/${actorName}`, folder: '', isStandalone: false }]; + const fullName = `${owner}/${actorName}`; + const tokenEnvVar = resolveTokenEnvVar(owner); + await validateActorExists(fullName, tokenEnvVar); + console.error(`Root .actor/ mode: single actor ${fullName}`); + return [{ actorName: fullName, folder: '', isStandalone: false, tokenEnvVar }]; } const actorConfigs: ActorConfig[] = []; @@ -111,11 +137,15 @@ export const getRepoActors = async (): Promise => { const folderName = actorDir.split('/')[1]; const folderType = actorDir.split('/')[0]; const owner = await resolveOwner(folderName); + const fullName = `${owner}/${actorName}`; + const tokenEnvVar = resolveTokenEnvVar(owner); + await validateActorExists(fullName, tokenEnvVar); actorConfigs.push({ - actorName: `${owner}/${actorName}`, + actorName: fullName, folder: actorDir, isStandalone: folderType === 'standalone-actors', + tokenEnvVar, }); } console.error( diff --git a/test/unit/bin/diff-changes.test.ts b/test/unit/bin/diff-changes.test.ts index 6d63469..b30770e 100644 --- a/test/unit/bin/diff-changes.test.ts +++ b/test/unit/bin/diff-changes.test.ts @@ -4,11 +4,12 @@ import { getChangedActors, maybeParseActorFolder } from '../../../bin/diff-chang import * as DiffJsonSchema from '../../../bin/diff-json-schema.js'; import type { ActorConfig } from '../../../bin/types.js'; -const miniActor: ActorConfig = { actorName: 'foo/bar', folder: 'actors/foo_bar', isStandalone: false }; +const miniActor: ActorConfig = { actorName: 'foo/bar', folder: 'actors/foo_bar', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_FOO' }; const standaloneActor: ActorConfig = { actorName: 'owner/standalone', folder: 'standalone-actors/standalone', isStandalone: true, + tokenEnvVar: 'APIFY_TOKEN_OWNER', }; const actorConfigs = [miniActor, standaloneActor]; @@ -183,6 +184,7 @@ describe('getChangedActors', () => { actorName: 'myteam/shopify-scraper', folder: 'actors/shopify', isStandalone: false, + tokenEnvVar: 'APIFY_TOKEN_MYTEAM', }; const result = getChangedActors({ filepathsChanged: ['actors/shopify/src/main.ts'], @@ -193,7 +195,7 @@ describe('getChangedActors', () => { }); it('in single-actor repo, .actor/ changes trigger builds', () => { - const rootActor: ActorConfig = { actorName: 'myteam/my-actor', folder: '', isStandalone: false }; + const rootActor: ActorConfig = { actorName: 'myteam/my-actor', folder: '', isStandalone: false, tokenEnvVar: 'BUILDER_APIFY_TOKEN' }; const result = getChangedActors({ filepathsChanged: ['.actor/actor.json'], actorConfigs: [rootActor], diff --git a/test/unit/should-built-and-test.test.ts b/test/unit/should-built-and-test.test.ts index 3dfb131..9d02f32 100644 --- a/test/unit/should-built-and-test.test.ts +++ b/test/unit/should-built-and-test.test.ts @@ -12,16 +12,19 @@ describe('Should build and test parser', () => { actorName: 'lukaskrivka/testing-github-integration-1', folder: 'actors/lukaskrivka_testing-github-integration-1', isStandalone: false, + tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', }, { actorName: 'lukaskrivka/testing-github-integration-2', folder: 'actors/lukaskrivka_testing-github-integration-2', isStandalone: false, + tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', }, { actorName: 'lukaskrivka/test-standalone', folder: 'standalone-actors/lukaskrivka_test-standalone', isStandalone: true, + tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', }, ]; @@ -252,46 +255,55 @@ describe('Should build and test parser', () => { actorName: 'compass/Google-Maps-Reviews-Scraper', folder: 'actors/compass_Google-Maps-Reviews-Scraper', isStandalone: false, + tokenEnvVar: 'APIFY_TOKEN_COMPASS', }, { actorName: 'compass/crawler-google-places', folder: 'actors/compass_crawler-google-places', isStandalone: false, + tokenEnvVar: 'APIFY_TOKEN_COMPASS', }, { actorName: 'compass/easy-google-maps', folder: 'actors/compass_easy-google-maps', isStandalone: false, + tokenEnvVar: 'APIFY_TOKEN_COMPASS', }, { actorName: 'compass/google-maps-extractor', folder: 'actors/compass_google-maps-extractor', isStandalone: false, + tokenEnvVar: 'APIFY_TOKEN_COMPASS', }, { actorName: 'compass/google-places-api', folder: 'actors/compass_google-places-api', isStandalone: false, + tokenEnvVar: 'APIFY_TOKEN_COMPASS', }, { actorName: 'lukaskrivka/google-maps-with-contact-details', folder: 'actors/lukaskrivka_google-maps-with-contact-details', isStandalone: false, + tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', }, { actorName: 'natasha.lekh/gas-prices-scraper', folder: 'actors/natasha.lekh_gas-prices-scraper', isStandalone: false, + tokenEnvVar: 'APIFY_TOKEN_NATASHA_LEKH', }, { actorName: 'natasha.lekh/vegan-places-finder', folder: 'actors/natasha.lekh_vegan-places-finder', isStandalone: false, + tokenEnvVar: 'APIFY_TOKEN_NATASHA_LEKH', }, { actorName: 'lukaskrivka/google-maps-scraper-orchestrator', folder: 'standalone-actors/lukaskrivka_google-maps-scraper-orchestrator', isStandalone: true, + tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', }, ]; From 78c095979fdf4ac0d037d1bbde875ddc7fd832dd Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Thu, 25 Jun 2026 14:47:20 +0100 Subject: [PATCH 07/33] use config file as source of truth for actors in repo --- bin/types.ts | 11 ++++ bin/utils.ts | 165 +++++++++++++++++---------------------------------- 2 files changed, 67 insertions(+), 109 deletions(-) diff --git a/bin/types.ts b/bin/types.ts index c2f0d42..697ec54 100644 --- a/bin/types.ts +++ b/bin/types.ts @@ -85,6 +85,17 @@ export interface GithubCommit { modified: string[]; } +export interface ActorConfigFileEntry { + folder: string; + owner: string; + tokenEnvVar: string; + isStandalone?: boolean; +} + +export interface ActorConfigFile { + actors: ActorConfigFileEntry[]; +} + export interface BuildData { buildId: string; actorId: string; diff --git a/bin/utils.ts b/bin/utils.ts index 5c1921e..c6909d1 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -1,9 +1,7 @@ import { spawnSync } from 'node:child_process'; import fs from 'node:fs/promises'; -import { ApifyClient } from 'apify-client'; - -import type { ActorConfig } from './types.js'; +import type { ActorConfig, ActorConfigFile } from './types.js'; export const spawnCommandInGhWorkspace = (command: string, args: string[] = []) => { console.error(command, args.join(' ')); @@ -31,135 +29,84 @@ export const getEnvVar = (varName: string, defaultValue?: string): string => { return value; }; -let cachedBuilderUsername: string | undefined; - -export const resolveBuilderTokenUsername = async (): Promise => { - if (cachedBuilderUsername) return cachedBuilderUsername; - const token = process.env.BUILDER_APIFY_TOKEN; - if (!token) { - throw new Error( - 'BUILDER_APIFY_TOKEN is not set. Either use the "owner_actor-name" folder convention ' + - 'or set the BUILDER_APIFY_TOKEN secret.', - ); - } - const client = new ApifyClient({ token }); - const user = await client.user().get(); - cachedBuilderUsername = user.username!; - return cachedBuilderUsername; -}; - -const readActorName = async (actorJsonPath: string): Promise => { - const actorJson: { name?: string } = JSON.parse(await fs.readFile(actorJsonPath, 'utf-8')); - if (!actorJson.name) { - throw new Error( - `Missing "name" field in "${actorJsonPath}". ` + - `Every actor folder must have .actor/actor.json with a "name" field.`, - ); - } - return actorJson.name; -}; - -const resolveOwner = async (folderName: string): Promise => { - const ownerMatch = folderName.match(/^(.+)_[^_]+$/); - if (ownerMatch) return ownerMatch[1]; - return resolveBuilderTokenUsername(); +export const getRepoActors = async (): Promise => { + return readConfigFile(); }; -const resolveTokenEnvVar = (owner: string): string => { - const usernameInGitHubSecretsFormat = owner.replaceAll(/\W/g, '_').toUpperCase(); - const usernameEnvVar = `APIFY_TOKEN_${usernameInGitHubSecretsFormat}`; - if (process.env[usernameEnvVar]) return usernameEnvVar; - if (process.env.BUILDER_APIFY_TOKEN) return 'BUILDER_APIFY_TOKEN'; - throw new Error( - `Cannot find Apify API token for owner "${owner}". ` + - `Set either ${usernameEnvVar} or BUILDER_APIFY_TOKEN.`, - ); -}; +const CONFIG_FILE_NAME = '.test-tools-actors-config.json'; -const validateActorExists = async (actorName: string, tokenEnvVar: string): Promise => { - const client = new ApifyClient({ token: process.env[tokenEnvVar]! }); - const actor = await client.actor(actorName).get(); - if (!actor) { +export const readConfigFile = async (): Promise => { + let raw: string; + try { + raw = await fs.readFile(CONFIG_FILE_NAME, 'utf-8'); + } catch { throw new Error( - `Actor "${actorName}" not found using token from ${tokenEnvVar}. ` + - `If this is a new actor, create it on the Apify platform first. ` + - `Otherwise, check that the folder name matches the actual actor owner.`, + `Config file "${CONFIG_FILE_NAME}" not found in the current directory. ` + + `Run "init-config" to generate one, then fill in the owner and tokenEnvVar fields.`, ); } -}; -/** - * Reads and parses all directories in `actors` directory - * This works locally if checkoutRepoLocally is called first - */ -export const getRepoActors = async (): Promise => { - let actorDirs: string[]; + let config: ActorConfigFile; try { - actorDirs = (await fs.readdir(`./actors`)).map((dir) => `actors/${dir}`); + config = JSON.parse(raw); } catch { - console.warn(`No /actors directory found in repo`); - actorDirs = []; + throw new Error(`Config file "${CONFIG_FILE_NAME}" contains invalid JSON.`); } - let standaloneActorDirs: string[]; - try { - standaloneActorDirs = (await fs.readdir(`./standalone-actors`)).map((dir) => `standalone-actors/${dir}`); - } catch { - console.warn(`No /standalone-actors directory found in repo`); - standaloneActorDirs = []; - } - if (actorDirs.length === 0 && standaloneActorDirs.length === 0) { - let actorName: string; - try { - actorName = await readActorName('./.actor/actor.json'); - } catch { - return []; - } - const owner = await resolveBuilderTokenUsername(); - const fullName = `${owner}/${actorName}`; - const tokenEnvVar = resolveTokenEnvVar(owner); - await validateActorExists(fullName, tokenEnvVar); - console.error(`Root .actor/ mode: single actor ${fullName}`); - return [{ actorName: fullName, folder: '', isStandalone: false, tokenEnvVar }]; + + if (!Array.isArray(config.actors)) { + throw new Error(`Config file "${CONFIG_FILE_NAME}" must have an "actors" array at the top level.`); } + const seenFolders = new Set(); const actorConfigs: ActorConfig[] = []; - for (const actorDir of [...actorDirs, ...standaloneActorDirs]) { - let actorName: string; + + for (const entry of config.actors) { + const folder = entry.folder === '.' ? '' : entry.folder; + + if (seenFolders.has(folder)) { + throw new Error( + `Duplicate folder "${entry.folder}" in "${CONFIG_FILE_NAME}". Each actor must have a unique folder.`, + ); + } + seenFolders.add(folder); + + const actorJsonPath = folder ? `${folder}/.actor/actor.json` : '.actor/actor.json'; + + if (folder) { + try { + await fs.access(folder); + } catch { + throw new Error( + `Folder "${folder}" declared in "${CONFIG_FILE_NAME}" does not exist on disk.`, + ); + } + } + + let actorJson: { name?: string }; try { - actorName = await readActorName(`./${actorDir}/.actor/actor.json`); + actorJson = JSON.parse(await fs.readFile(actorJsonPath, 'utf-8')); } catch { throw new Error( - `Missing or unreadable .actor/actor.json in "${actorDir}". ` + - `Every actor folder must contain .actor/actor.json with a "name" field.`, + `Cannot read "${actorJsonPath}". Every actor entry in "${CONFIG_FILE_NAME}" ` + + `must have a corresponding .actor/actor.json file.`, ); } - const folderName = actorDir.split('/')[1]; - const folderType = actorDir.split('/')[0]; - const owner = await resolveOwner(folderName); - const fullName = `${owner}/${actorName}`; - const tokenEnvVar = resolveTokenEnvVar(owner); - await validateActorExists(fullName, tokenEnvVar); + if (!actorJson.name) { + throw new Error( + `Missing "name" field in "${actorJsonPath}". ` + + `Every actor must have a "name" in its .actor/actor.json.`, + ); + } actorConfigs.push({ - actorName: fullName, - folder: actorDir, - isStandalone: folderType === 'standalone-actors', - tokenEnvVar, + actorName: `${entry.owner}/${actorJson.name}`, + folder, + isStandalone: entry.isStandalone ?? false, + tokenEnvVar: entry.tokenEnvVar, }); } - console.error( - `Actors in repo: ${actorConfigs - .filter(({ isStandalone }) => !isStandalone) - .map(({ actorName }) => actorName) - .join(', ')}`, - ); - console.error( - `Standalone actors in repo: ${actorConfigs - .filter(({ isStandalone }) => !!isStandalone) - .map(({ actorName }) => actorName) - .join(', ')}`, - ); + return actorConfigs; }; From 9a6c9f64e94b9d40cac7e08450a9105267ea974c Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Thu, 25 Jun 2026 14:50:21 +0100 Subject: [PATCH 08/33] treat .actor in root always as functional --- bin/diff-changes.ts | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index 9b87e8b..7796f2b 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -18,10 +18,7 @@ export const maybeParseActorFolder = ( return { isActorFolder: false }; }; -/** - * Also works for folders - */ -const isIgnoredTopLevelFile = (lowercaseFilePath: string, isSingleActorRepo: boolean) => { +const isIgnoredTopLevelFile = (lowercaseFilePath: string) => { const IGNORED_TOP_LEVEL_FILES = [ '.vscode/', '.gitignore', @@ -31,8 +28,6 @@ const isIgnoredTopLevelFile = (lowercaseFilePath: string, isSingleActorRepo: boo 'eslint.config.mjs', '.prettierrc', '.editorconfig', - // In root .actor/ mode, .actor/ changes must trigger builds - ...(isSingleActorRepo ? [] : ['.actor/']), ]; // Strip out deprecated /code and /shared folders, treat them as top-level code const sanitizedLowercaseFilePath = lowercaseFilePath.replace(/^code\//, '').replace(/^shared\//, ''); @@ -54,11 +49,10 @@ const classifyFileChange = ( originalFilePath: string, actorConfigs: ActorConfig[], commits: Commit[], - isSingleActorRepo: boolean, ): FileChange => { // Lowercase for case-insensitive matching; keep original for git show (case-sensitive on Linux) const lowercaseFilePath = originalFilePath.toLowerCase(); - if (isIgnoredTopLevelFile(lowercaseFilePath, isSingleActorRepo)) { + if (isIgnoredTopLevelFile(lowercaseFilePath)) { return { impact: 'ignored' }; } @@ -105,12 +99,11 @@ export const getChangedActors = ({ }: ShouldBuildAndTestOptions): ActorConfig[] => { // folder -> ActorConfig const actorsChangedMap = new Map(); - const isSingleActorRepo = actorConfigs.some(({ folder }) => folder === ''); const actorConfigsWithoutStandalone = actorConfigs.filter(({ isStandalone }) => !isStandalone); for (const originalFilePath of filepathsChanged) { - const fileChange = classifyFileChange(originalFilePath, actorConfigs, commits, isSingleActorRepo); + const fileChange = classifyFileChange(originalFilePath, actorConfigs, commits); if (fileChange.impact === 'ignored') { continue; } @@ -135,12 +128,12 @@ export const getChangedActors = ({ const formatFiles = (files: string[]) => (files.length > 0 ? files.join(', ') : ''); const ignoredFilesChanged = filepathsChanged.filter( - (file) => classifyFileChange(file, actorConfigs, commits, isSingleActorRepo).impact === 'ignored', + (file) => classifyFileChange(file, actorConfigs, commits).impact === 'ignored', ); console.error(`[DIFF]: Ignored files (don't trigger test or build): ${formatFiles(ignoredFilesChanged)}`); const cosmeticChanges = filepathsChanged - .map((file) => ({ file, change: classifyFileChange(file, actorConfigs, commits, isSingleActorRepo) })) + .map((file) => ({ file, change: classifyFileChange(file, actorConfigs, commits) })) .filter(({ change }) => change.impact === 'cosmetic') as { file: string; change: Extract; @@ -159,7 +152,7 @@ export const getChangedActors = ({ ); const functionalFilesChanged = filepathsChanged.filter( - (file) => classifyFileChange(file, actorConfigs, commits, isSingleActorRepo).impact === 'functional', + (file) => classifyFileChange(file, actorConfigs, commits).impact === 'functional', ); console.error(`[DIFF]: Functional files (trigger test & release build): ${formatFiles(functionalFilesChanged)}`); From c3f06042393528d72d9a9fb797a794a67c21e653 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Thu, 25 Jun 2026 15:09:15 +0100 Subject: [PATCH 09/33] add init-config command to jump start configuration --- bin/main.ts | 19 +++++++++++++++++- bin/utils.ts | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/bin/main.ts b/bin/main.ts index 120ea1b..b07df38 100644 --- a/bin/main.ts +++ b/bin/main.ts @@ -13,7 +13,7 @@ import { getPushData } from './github.js'; import { notifyToSlack } from './slack.js'; import { reportTestResults } from './test-report.js'; import type { Config } from './types.js'; -import { getRepoActors, setCwd, spawnCommandInGhWorkspace } from './utils.js'; +import { generateConfigFile, getRepoActors, setCwd, spawnCommandInGhWorkspace } from './utils.js'; /** * Middlewares to be run before every command execution @@ -202,6 +202,23 @@ await yargs() }); }, ) + .command( + 'init-config', + 'Generate a starter config file by scanning the repo for .actor/actor.json files', + (args) => + args + .option('default-owner', { + type: 'string', + description: 'Apify username to use as owner for all actors', + }) + .option('default-token-var', { + type: 'string', + description: 'Env var name holding the Apify token for all actors', + }), + async ({ defaultOwner, defaultTokenVar }) => { + await generateConfigFile({ defaultOwner, defaultTokenVar }); + }, + ) .command( 'delete-old-builds', '', diff --git a/bin/utils.ts b/bin/utils.ts index c6909d1..ab57985 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -1,7 +1,8 @@ -import { spawnSync } from 'node:child_process'; +import { execSync, spawnSync } from 'node:child_process'; import fs from 'node:fs/promises'; +import path from 'node:path'; -import type { ActorConfig, ActorConfigFile } from './types.js'; +import type { ActorConfig, ActorConfigFile, ActorConfigFileEntry } from './types.js'; export const spawnCommandInGhWorkspace = (command: string, args: string[] = []) => { console.error(command, args.join(' ')); @@ -110,6 +111,57 @@ export const readConfigFile = async (): Promise => { return actorConfigs; }; +interface GenerateConfigOptions { + defaultOwner?: string; + defaultTokenVar?: string; +} + +export const generateConfigFile = async (options: GenerateConfigOptions = {}): Promise => { + try { + await fs.access(CONFIG_FILE_NAME); + throw new Error( + `Config file "${CONFIG_FILE_NAME}" already exists. ` + + `Remove it first if you want to regenerate.`, + ); + } catch (err) { + if (err instanceof Error && err.message.includes('already exists')) throw err; + } + + const trackedFiles = execSync('git ls-files', { encoding: 'utf-8' }).trim().split('\n'); + const actorJsonFiles = trackedFiles.filter((f) => f.endsWith('.actor/actor.json')); + + if (actorJsonFiles.length === 0) { + throw new Error('No .actor/actor.json files found in the repository.'); + } + + const hasRoot = actorJsonFiles.includes('.actor/actor.json'); + const subfolderActorJsonFiles = actorJsonFiles.filter((f) => f !== '.actor/actor.json'); + + if (hasRoot && subfolderActorJsonFiles.length > 0) { + console.error( + `\nNote: Found a root-level .actor/actor.json alongside ${subfolderActorJsonFiles.length} subfolder actor(s).` + + `\nIf the root .actor/actor.json only exists to satisfy the Apify CLI and is not a real actor,` + + `\nconsider removing its entry from the generated config file.\n`, + ); + } + + const entries: ActorConfigFileEntry[] = []; + + for (const actorJsonPath of actorJsonFiles) { + const folder = actorJsonPath === '.actor/actor.json' ? '' : path.dirname(path.dirname(actorJsonPath)); + + entries.push({ + folder: folder || '.', + owner: options.defaultOwner ?? '', + tokenEnvVar: options.defaultTokenVar ?? '', + }); + } + + const config: ActorConfigFile = { actors: entries }; + await fs.writeFile(CONFIG_FILE_NAME, `$${JSON.stringify(config, null, 4)}`); + console.error(`Created "${CONFIG_FILE_NAME}" with ${entries.length} actor(s). Fill in the owner and tokenEnvVar fields.`); +}; + export const setCwd = ({ workspace }: { workspace: string | undefined }) => { if (workspace) { process.chdir(workspace); From 92316c0bfd8f46ac3ff77039df348d3941498182 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Thu, 25 Jun 2026 15:50:19 +0100 Subject: [PATCH 10/33] remove access check readConfigFile --- bin/utils.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/bin/utils.ts b/bin/utils.ts index ab57985..6e7070e 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -73,16 +73,6 @@ export const readConfigFile = async (): Promise => { const actorJsonPath = folder ? `${folder}/.actor/actor.json` : '.actor/actor.json'; - if (folder) { - try { - await fs.access(folder); - } catch { - throw new Error( - `Folder "${folder}" declared in "${CONFIG_FILE_NAME}" does not exist on disk.`, - ); - } - } - let actorJson: { name?: string }; try { actorJson = JSON.parse(await fs.readFile(actorJsonPath, 'utf-8')); @@ -158,7 +148,7 @@ export const generateConfigFile = async (options: GenerateConfigOptions = {}): P } const config: ActorConfigFile = { actors: entries }; - await fs.writeFile(CONFIG_FILE_NAME, `$${JSON.stringify(config, null, 4)}`); + await fs.writeFile(CONFIG_FILE_NAME, JSON.stringify(config, null, 4)); console.error(`Created "${CONFIG_FILE_NAME}" with ${entries.length} actor(s). Fill in the owner and tokenEnvVar fields.`); }; From 5a50e39f56cbbd8ad7133395192c1963d19c9f90 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Thu, 25 Jun 2026 15:51:32 +0100 Subject: [PATCH 11/33] update relevant tests --- test/unit/bin/diff-changes.test.ts | 4 +- test/unit/bin/utils.test.ts | 244 ++++++++++++++++++++++++ test/unit/should-built-and-test.test.ts | 15 +- 3 files changed, 260 insertions(+), 3 deletions(-) create mode 100644 test/unit/bin/utils.test.ts diff --git a/test/unit/bin/diff-changes.test.ts b/test/unit/bin/diff-changes.test.ts index b30770e..4e7f9f5 100644 --- a/test/unit/bin/diff-changes.test.ts +++ b/test/unit/bin/diff-changes.test.ts @@ -204,13 +204,13 @@ describe('getChangedActors', () => { expect(result).toEqual([rootActor]); }); - it('in multi-actor repo, .actor/ changes are ignored', () => { + it('in multi-actor repo, .actor/ changes trigger builds for all non-standalone actors', () => { const result = getChangedActors({ filepathsChanged: ['.actor/actor.json'], actorConfigs, commits, }); - expect(result).toEqual([]); + expect(result).toEqual([miniActor]); }); it('file paths are matched case-insensitively', () => { diff --git a/test/unit/bin/utils.test.ts b/test/unit/bin/utils.test.ts new file mode 100644 index 0000000..1564e85 --- /dev/null +++ b/test/unit/bin/utils.test.ts @@ -0,0 +1,244 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { generateConfigFile, readConfigFile } from '../../../bin/utils.js'; + +const { fsMock, execSyncMock } = vi.hoisted(() => ({ + fsMock: { + readFile: vi.fn(), + writeFile: vi.fn(), + access: vi.fn(), + }, + execSyncMock: vi.fn(), +})); + +vi.mock('node:fs/promises', () => ({ default: fsMock })); + +vi.mock('node:child_process', () => ({ + execSync: execSyncMock, + spawnSync: vi.fn(), +})); + +afterEach(() => vi.restoreAllMocks()); + +const validConfig = (actors: object[]) => JSON.stringify({ actors }); +const actorJson = (name: string) => JSON.stringify({ name }); + +const expectFileRead = (filePath: string) => { + expect(fsMock.readFile).toHaveBeenCalledWith(filePath, expect.anything()); +}; + +const mockFiles = (files: Record) => { + fsMock.readFile.mockImplementation(async (filePath: string) => { + if (filePath in files) return Promise.resolve(files[filePath]); + return Promise.reject(new Error(`ENOENT: ${filePath}`)); + }); +}; + +describe('readConfigFile', () => { + it('returns correct ActorConfig[] for a valid config', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'actors/shopify', owner: 'myteam', tokenEnvVar: 'APIFY_TOKEN_MYTEAM' }, + ]), + 'actors/shopify/.actor/actor.json': actorJson('shopify-scraper'), + }); + + const result = await readConfigFile(); + expectFileRead('actors/shopify/.actor/actor.json'); + expect(result).toEqual([ + { + actorName: 'myteam/shopify-scraper', + folder: 'actors/shopify', + isStandalone: false, + tokenEnvVar: 'APIFY_TOKEN_MYTEAM', + }, + ]); + }); + + it('normalizes folder "." to ""', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: '.', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + ]), + '.actor/actor.json': actorJson('my-actor'), + }); + + const result = await readConfigFile(); + expect(result[0].folder).toBe(''); + expectFileRead('.actor/actor.json'); + }); + + it('defaults isStandalone to false when omitted', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'actors/web-scraper', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + ]), + 'actors/web-scraper/.actor/actor.json': actorJson('web-scraper'), + }); + + const result = await readConfigFile(); + expectFileRead('actors/web-scraper/.actor/actor.json'); + expect(result[0].isStandalone).toBe(false); + }); + + it('respects isStandalone: true', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'standalone/orchestrator', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY', isStandalone: true }, + ]), + 'standalone/orchestrator/.actor/actor.json': actorJson('orchestrator'), + }); + + const result = await readConfigFile(); + expectFileRead('standalone/orchestrator/.actor/actor.json'); + expect(result[0].isStandalone).toBe(true); + }); + + it('handles multiple actors', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'actors/web-scraper', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 'actors/email-sender', owner: 'other-team', tokenEnvVar: 'APIFY_TOKEN_OTHER_TEAM', isStandalone: true }, + ]), + 'actors/web-scraper/.actor/actor.json': actorJson('web-scraper'), + 'actors/email-sender/.actor/actor.json': actorJson('email-sender'), + }); + + const result = await readConfigFile(); + expectFileRead('actors/web-scraper/.actor/actor.json'); + expectFileRead('actors/email-sender/.actor/actor.json'); + expect(result).toHaveLength(2); + expect(result[0].actorName).toBe('apify/web-scraper'); + expect(result[1].actorName).toBe('other-team/email-sender'); + expect(result[1].isStandalone).toBe(true); + }); + + it('throws when config file is missing', async () => { + fsMock.readFile.mockRejectedValue(new Error('ENOENT')); + await expect(readConfigFile()).rejects.toThrow('not found'); + }); + + it('throws when config file contains invalid JSON', async () => { + fsMock.readFile.mockResolvedValue('{bad json'); + await expect(readConfigFile()).rejects.toThrow('invalid JSON'); + }); + + it('throws when actors array is missing', async () => { + fsMock.readFile.mockResolvedValue(JSON.stringify({ notActors: [] })); + await expect(readConfigFile()).rejects.toThrow('"actors" array'); + }); + + it('throws on duplicate folders', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'actors/shopify', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 'actors/shopify', owner: 'other-team', tokenEnvVar: 'APIFY_TOKEN_OTHER' }, + ]), + 'actors/shopify/.actor/actor.json': actorJson('shopify-scraper'), + }); + + await expect(readConfigFile()).rejects.toThrow('Duplicate folder'); + }); + + it('throws on duplicate folders after normalization ("." and "")', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: '.', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: '', owner: 'other-team', tokenEnvVar: 'APIFY_TOKEN_OTHER' }, + ]), + '.actor/actor.json': actorJson('my-actor'), + }); + + await expect(readConfigFile()).rejects.toThrow('Duplicate folder'); + }); + + it('throws when actor.json is missing', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'actors/shopify', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + ]), + }); + + await expect(readConfigFile()).rejects.toThrow('Cannot read'); + }); + + it('throws when actor.json has no name field', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'actors/shopify', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + ]), + 'actors/shopify/.actor/actor.json': JSON.stringify({ description: 'no name here' }), + }); + + await expect(readConfigFile()).rejects.toThrow('Missing "name"'); + }); +}); + +describe('generateConfigFile', () => { + it('generates config from discovered actor.json files', async () => { + fsMock.access.mockRejectedValue(new Error('ENOENT')); + execSyncMock.mockReturnValue( + 'actors/shopify/.actor/actor.json\nactors/amazon/.actor/actor.json\n', + ); + + await generateConfigFile(); + + expect(fsMock.access).toHaveBeenCalledWith('.test-tools-actors-config.json'); + expect(fsMock.writeFile).toHaveBeenCalledOnce(); + const written = JSON.parse(fsMock.writeFile.mock.calls[0][1]); + expect(written.actors).toHaveLength(2); + expect(written.actors[0]).toEqual({ + folder: 'actors/shopify', + owner: '', + tokenEnvVar: '', + }); + expect(written.actors[1]).toEqual({ + folder: 'actors/amazon', + owner: '', + tokenEnvVar: '', + }); + }); + + it('uses "." for root-level actor', async () => { + fsMock.access.mockRejectedValue(new Error('ENOENT')); + execSyncMock.mockReturnValue('.actor/actor.json\n'); + + await generateConfigFile(); + + const written = JSON.parse(fsMock.writeFile.mock.calls[0][1]); + expect(written.actors[0].folder).toBe('.'); + }); + + it('applies --default-owner and --default-token-var flags', async () => { + fsMock.access.mockRejectedValue(new Error('ENOENT')); + execSyncMock.mockReturnValue('actors/shopify/.actor/actor.json\n'); + + await generateConfigFile({ defaultOwner: 'myteam', defaultTokenVar: 'MY_TOKEN' }); + + const written = JSON.parse(fsMock.writeFile.mock.calls[0][1]); + expect(written.actors[0].owner).toBe('myteam'); + expect(written.actors[0].tokenEnvVar).toBe('MY_TOKEN'); + }); + + it('throws if config file already exists', async () => { + await expect(generateConfigFile()).rejects.toThrow('already exists'); + expect(fsMock.access).toHaveBeenCalledWith('.test-tools-actors-config.json'); + }); + + it('throws if no actor.json files found', async () => { + fsMock.access.mockRejectedValue(new Error('ENOENT')); + execSyncMock.mockReturnValue('src/main.ts\npackage.json\n'); + + await expect(generateConfigFile()).rejects.toThrow('No .actor/actor.json files found'); + }); + + it('warns when root actor.json exists alongside subfolder actors', async () => { + fsMock.access.mockRejectedValue(new Error('ENOENT')); + execSyncMock.mockReturnValue('.actor/actor.json\nactors/shopify/.actor/actor.json\n'); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => { /**/ }); + + await generateConfigFile(); + + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('root-level .actor/actor.json')); + }); +}); diff --git a/test/unit/should-built-and-test.test.ts b/test/unit/should-built-and-test.test.ts index 9d02f32..8ab49bc 100644 --- a/test/unit/should-built-and-test.test.ts +++ b/test/unit/should-built-and-test.test.ts @@ -54,7 +54,7 @@ describe('Should build and test parser', () => { }); test('Ignores other ignored files and folders', () => { - const FILES = ['.vscode/', '.gitignore', '.husky/', '.eslintrc', '.editorconfig', '.actor/']; + const FILES = ['.vscode/', '.gitignore', '.husky/', '.eslintrc', '.editorconfig']; const actorsChanged = getChangedActors({ actorConfigs: ACTOR_CONFIGS, @@ -66,6 +66,19 @@ describe('Should build and test parser', () => { expect(actorsChanged).toEqual([]); }); + test('.actor/ changes always trigger builds for all non-standalone actors', () => { + const FILES = ['.actor/actor.json']; + + const actorsChanged = getChangedActors({ + actorConfigs: ACTOR_CONFIGS, + isLatest: false, + filepathsChanged: FILES, + commits, + }); + + expect(actorsChanged).toEqual(ACTOR_CONFIGS.filter(({ isStandalone }) => !isStandalone)); + }); + test('Only builds latest for all Actors', () => { const FILES = ['shared/CHANGELOG.md', 'CHANGELOG.md']; From 0a8487c1ab2e56100a33536d715d836a4b880ca9 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Thu, 25 Jun 2026 15:54:54 +0100 Subject: [PATCH 12/33] update readme --- README.md | 97 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 85 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8eb6b25..df818c3 100644 --- a/README.md +++ b/README.md @@ -4,29 +4,102 @@ ## Getting Started -1. Install the package `npm i -D apify-test-tools` - - because it uses [annotate](https://vitest.dev/guide/test-context.html#annotate), `vitest` version to be at least `3.2.0` - - make sure that `target` and `module` in your `tsconfig.json`'s `compilerOptions` are set to `ES2022` -2. create test directories: `mkdir -p test/platform/core` - - core (hourly) tests should go to `test/platform/core` - - daily tests should go to `test/platform` -3. setup github worklows TODO +### 1. Install the package -File structure: +```bash +npm i -D apify-test-tools +``` + +- Requires `vitest` version `3.2.0` or later (uses [annotate](https://vitest.dev/guide/test-context.html#annotate)) +- Make sure `target` and `module` in your `tsconfig.json`'s `compilerOptions` are set to `ES2022` + +### 2. Create the config file + +Every repo that uses `apify-test-tools` must have a `.test-tools-actors-config.json` file at the root. This file tells the tool which actors live in the repo, who owns them, and which token to use. + +You can generate a starter config automatically: + +```bash +npx apify-test-tools init-config +``` + +This scans the repo for `.actor/actor.json` files and creates `.test-tools-actors-config.json` with placeholder values. You can also pass defaults: + +```bash +npx apify-test-tools init-config --default-owner myteam --default-token-var APIFY_TOKEN_MYTEAM +``` + +The generated file looks like this: +```json +{ + "actors": [ + { + "folder": "actors/web-scraper", + "owner": "", + "tokenEnvVar": "" + }, + { + "folder": "actors/email-sender", + "owner": "", + "tokenEnvVar": "", + "isStandalone": true + } + ] +} ``` -google-maps + +Each entry has: + +| Field | Required | Description | +|-------|----------|-------------| +| `folder` | yes | Relative path from repo root to the actor directory. Use `"."` for a single-actor repo where `.actor/` is at the root. | +| `owner` | yes | Apify username that owns the actor. Combined with the `name` from `/.actor/actor.json` to form the full actor name (`owner/name`). | +| `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. No fallback — if the env var is not set at build time, the build fails. | +| `isStandalone` | no | Defaults to `false`. Standalone actors are only built when their own folder changes, not when shared code changes. | + +The actor's `name` is always read from `/.actor/actor.json` — it is **not** duplicated in the config. + +### 3. Set up actor folders + +Each actor in the config must have a `.actor/actor.json` file with at least a `name` field: + +``` +my-repo +├── .test-tools-actors-config.json ├── actors -└── src +│ ├── web-scraper +│ │ ├── .actor +│ │ │ └── actor.json <- { "name": "web-scraper" } +│ │ └── src/ +│ └── email-sender +│ ├── .actor +│ │ └── actor.json <- { "name": "email-sender" } +│ └── src/ └── test ├── unit └── platform - ├── core <- Core tests need to be inside core directory + ├── core <- Core (hourly) tests │ └── core.test.ts - ├── some.test.ts <- Other tests can be defined anywhere inside platform directory + ├── some.test.ts <- Daily tests can be anywhere inside platform/ └── some-other.test.ts ``` +For a single-actor repo, set `"folder": "."` in the config and place `.actor/actor.json` at the repo root. + +### 4. Create test directories + +```bash +mkdir -p test/platform/core +``` + +- Core (hourly) tests go in `test/platform/core` +- Daily tests go anywhere in `test/platform` + +### 5. Set up GitHub workflows + +See the [GitHub workflows](#github-worklows) section below. + ## Github worklows There should be 4 GH workflow files in `.github/workflows`. From f097d2eba3cb23936c8250fec4f6a66c036554a7 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Fri, 26 Jun 2026 14:38:56 +0100 Subject: [PATCH 13/33] Replace owner with config-level actorName, add dockerContextDir resolution, remove init-config --- bin/main.ts | 19 +-- bin/types.ts | 6 +- bin/utils.ts | 92 +++++------- test/unit/bin/diff-changes.test.ts | 6 +- test/unit/bin/utils.test.ts | 189 ++++++++++++------------ test/unit/should-built-and-test.test.ts | 12 ++ 6 files changed, 150 insertions(+), 174 deletions(-) diff --git a/bin/main.ts b/bin/main.ts index b07df38..120ea1b 100644 --- a/bin/main.ts +++ b/bin/main.ts @@ -13,7 +13,7 @@ import { getPushData } from './github.js'; import { notifyToSlack } from './slack.js'; import { reportTestResults } from './test-report.js'; import type { Config } from './types.js'; -import { generateConfigFile, getRepoActors, setCwd, spawnCommandInGhWorkspace } from './utils.js'; +import { getRepoActors, setCwd, spawnCommandInGhWorkspace } from './utils.js'; /** * Middlewares to be run before every command execution @@ -202,23 +202,6 @@ await yargs() }); }, ) - .command( - 'init-config', - 'Generate a starter config file by scanning the repo for .actor/actor.json files', - (args) => - args - .option('default-owner', { - type: 'string', - description: 'Apify username to use as owner for all actors', - }) - .option('default-token-var', { - type: 'string', - description: 'Env var name holding the Apify token for all actors', - }), - async ({ defaultOwner, defaultTokenVar }) => { - await generateConfigFile({ defaultOwner, defaultTokenVar }); - }, - ) .command( 'delete-old-builds', '', diff --git a/bin/types.ts b/bin/types.ts index 697ec54..a823b2e 100644 --- a/bin/types.ts +++ b/bin/types.ts @@ -87,9 +87,10 @@ export interface GithubCommit { export interface ActorConfigFileEntry { folder: string; - owner: string; + actorName: string; tokenEnvVar: string; isStandalone?: boolean; + overrideActorContext?: string[]; } export interface ActorConfigFile { @@ -101,7 +102,6 @@ export interface BuildData { actorId: string; actorName: string; actorConfig: ActorConfig; - // folder: string | undefined; buildNumber: string; } @@ -110,4 +110,6 @@ export interface ActorConfig { folder: string; isStandalone: boolean; tokenEnvVar: string; + dockerContextDir: string; + overrideActorContext?: string[]; } diff --git a/bin/utils.ts b/bin/utils.ts index 6e7070e..86543a9 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -1,8 +1,8 @@ -import { execSync, spawnSync } from 'node:child_process'; +import { spawnSync } from 'node:child_process'; import fs from 'node:fs/promises'; import path from 'node:path'; -import type { ActorConfig, ActorConfigFile, ActorConfigFileEntry } from './types.js'; +import type { ActorConfig, ActorConfigFile } from './types.js'; export const spawnCommandInGhWorkspace = (command: string, args: string[] = []) => { console.error(command, args.join(' ')); @@ -43,7 +43,7 @@ export const readConfigFile = async (): Promise => { } catch { throw new Error( `Config file "${CONFIG_FILE_NAME}" not found in the current directory. ` + - `Run "init-config" to generate one, then fill in the owner and tokenEnvVar fields.`, + `Please create one with the required actor entries.`, ); } @@ -71,9 +71,26 @@ export const readConfigFile = async (): Promise => { } seenFolders.add(folder); + const nameParts = entry.actorName?.split('/'); + if (!nameParts || nameParts.length !== 2 || !nameParts[0] || !nameParts[1]) { + throw new Error( + `Invalid "actorName" for folder "${entry.folder}" in "${CONFIG_FILE_NAME}". ` + + `Must be in "owner/name" format (e.g. "apify/web-scraper").`, + ); + } + + if (entry.overrideActorContext !== undefined) { + if (!Array.isArray(entry.overrideActorContext) || !entry.overrideActorContext.every((p) => typeof p === 'string')) { + throw new Error( + `Invalid "overrideActorContext" for folder "${entry.folder}" in "${CONFIG_FILE_NAME}". ` + + `Must be an array of strings.`, + ); + } + } + const actorJsonPath = folder ? `${folder}/.actor/actor.json` : '.actor/actor.json'; - let actorJson: { name?: string }; + let actorJson: { dockerContextDir?: string }; try { actorJson = JSON.parse(await fs.readFile(actorJsonPath, 'utf-8')); } catch { @@ -83,74 +100,33 @@ export const readConfigFile = async (): Promise => { ); } - if (!actorJson.name) { + const actorDotDir = folder ? `${folder}/.actor` : '.actor'; + const rawDockerContextDir = actorJson.dockerContextDir ?? '..'; + const resolved = path.resolve(process.cwd(), actorDotDir, rawDockerContextDir); + const dockerContextDir = path.relative(process.cwd(), resolved); + + if (dockerContextDir.startsWith('..')) { throw new Error( - `Missing "name" field in "${actorJsonPath}". ` + - `Every actor must have a "name" in its .actor/actor.json.`, + `"dockerContextDir" for folder "${entry.folder}" resolves outside the repository root. ` + + `Resolved path: "${dockerContextDir}".`, ); } + const normalizedDockerContextDir = dockerContextDir === '.' ? '' : dockerContextDir; + actorConfigs.push({ - actorName: `${entry.owner}/${actorJson.name}`, + actorName: entry.actorName, folder, isStandalone: entry.isStandalone ?? false, tokenEnvVar: entry.tokenEnvVar, + dockerContextDir: normalizedDockerContextDir, + overrideActorContext: entry.overrideActorContext, }); } return actorConfigs; }; -interface GenerateConfigOptions { - defaultOwner?: string; - defaultTokenVar?: string; -} - -export const generateConfigFile = async (options: GenerateConfigOptions = {}): Promise => { - try { - await fs.access(CONFIG_FILE_NAME); - throw new Error( - `Config file "${CONFIG_FILE_NAME}" already exists. ` + - `Remove it first if you want to regenerate.`, - ); - } catch (err) { - if (err instanceof Error && err.message.includes('already exists')) throw err; - } - - const trackedFiles = execSync('git ls-files', { encoding: 'utf-8' }).trim().split('\n'); - const actorJsonFiles = trackedFiles.filter((f) => f.endsWith('.actor/actor.json')); - - if (actorJsonFiles.length === 0) { - throw new Error('No .actor/actor.json files found in the repository.'); - } - - const hasRoot = actorJsonFiles.includes('.actor/actor.json'); - const subfolderActorJsonFiles = actorJsonFiles.filter((f) => f !== '.actor/actor.json'); - - if (hasRoot && subfolderActorJsonFiles.length > 0) { - console.error( - `\nNote: Found a root-level .actor/actor.json alongside ${subfolderActorJsonFiles.length} subfolder actor(s).` + - `\nIf the root .actor/actor.json only exists to satisfy the Apify CLI and is not a real actor,` + - `\nconsider removing its entry from the generated config file.\n`, - ); - } - - const entries: ActorConfigFileEntry[] = []; - - for (const actorJsonPath of actorJsonFiles) { - const folder = actorJsonPath === '.actor/actor.json' ? '' : path.dirname(path.dirname(actorJsonPath)); - - entries.push({ - folder: folder || '.', - owner: options.defaultOwner ?? '', - tokenEnvVar: options.defaultTokenVar ?? '', - }); - } - - const config: ActorConfigFile = { actors: entries }; - await fs.writeFile(CONFIG_FILE_NAME, JSON.stringify(config, null, 4)); - console.error(`Created "${CONFIG_FILE_NAME}" with ${entries.length} actor(s). Fill in the owner and tokenEnvVar fields.`); -}; export const setCwd = ({ workspace }: { workspace: string | undefined }) => { if (workspace) { diff --git a/test/unit/bin/diff-changes.test.ts b/test/unit/bin/diff-changes.test.ts index 4e7f9f5..267487a 100644 --- a/test/unit/bin/diff-changes.test.ts +++ b/test/unit/bin/diff-changes.test.ts @@ -4,12 +4,13 @@ import { getChangedActors, maybeParseActorFolder } from '../../../bin/diff-chang import * as DiffJsonSchema from '../../../bin/diff-json-schema.js'; import type { ActorConfig } from '../../../bin/types.js'; -const miniActor: ActorConfig = { actorName: 'foo/bar', folder: 'actors/foo_bar', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_FOO' }; +const miniActor: ActorConfig = { actorName: 'foo/bar', folder: 'actors/foo_bar', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_FOO', dockerContextDir: '' }; const standaloneActor: ActorConfig = { actorName: 'owner/standalone', folder: 'standalone-actors/standalone', isStandalone: true, tokenEnvVar: 'APIFY_TOKEN_OWNER', + dockerContextDir: 'standalone-actors/standalone', }; const actorConfigs = [miniActor, standaloneActor]; @@ -185,6 +186,7 @@ describe('getChangedActors', () => { folder: 'actors/shopify', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_MYTEAM', + dockerContextDir: '', }; const result = getChangedActors({ filepathsChanged: ['actors/shopify/src/main.ts'], @@ -195,7 +197,7 @@ describe('getChangedActors', () => { }); it('in single-actor repo, .actor/ changes trigger builds', () => { - const rootActor: ActorConfig = { actorName: 'myteam/my-actor', folder: '', isStandalone: false, tokenEnvVar: 'BUILDER_APIFY_TOKEN' }; + const rootActor: ActorConfig = { actorName: 'myteam/my-actor', folder: '', isStandalone: false, tokenEnvVar: 'BUILDER_APIFY_TOKEN', dockerContextDir: '' }; const result = getChangedActors({ filepathsChanged: ['.actor/actor.json'], actorConfigs: [rootActor], diff --git a/test/unit/bin/utils.test.ts b/test/unit/bin/utils.test.ts index 1564e85..475734b 100644 --- a/test/unit/bin/utils.test.ts +++ b/test/unit/bin/utils.test.ts @@ -1,27 +1,19 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { generateConfigFile, readConfigFile } from '../../../bin/utils.js'; +import { readConfigFile } from '../../../bin/utils.js'; -const { fsMock, execSyncMock } = vi.hoisted(() => ({ +const { fsMock } = vi.hoisted(() => ({ fsMock: { readFile: vi.fn(), - writeFile: vi.fn(), - access: vi.fn(), }, - execSyncMock: vi.fn(), })); vi.mock('node:fs/promises', () => ({ default: fsMock })); -vi.mock('node:child_process', () => ({ - execSync: execSyncMock, - spawnSync: vi.fn(), -})); - afterEach(() => vi.restoreAllMocks()); const validConfig = (actors: object[]) => JSON.stringify({ actors }); -const actorJson = (name: string) => JSON.stringify({ name }); +const actorJson = (fields: Record = {}) => JSON.stringify(fields); const expectFileRead = (filePath: string) => { expect(fsMock.readFile).toHaveBeenCalledWith(filePath, expect.anything()); @@ -38,9 +30,9 @@ describe('readConfigFile', () => { it('returns correct ActorConfig[] for a valid config', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ - { folder: 'actors/shopify', owner: 'myteam', tokenEnvVar: 'APIFY_TOKEN_MYTEAM' }, + { folder: 'actors/shopify', actorName: 'myteam/shopify-scraper', tokenEnvVar: 'APIFY_TOKEN_MYTEAM' }, ]), - 'actors/shopify/.actor/actor.json': actorJson('shopify-scraper'), + 'actors/shopify/.actor/actor.json': actorJson({ dockerContextDir: '../../..' }), }); const result = await readConfigFile(); @@ -51,6 +43,8 @@ describe('readConfigFile', () => { folder: 'actors/shopify', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_MYTEAM', + dockerContextDir: '', + overrideActorContext: undefined, }, ]); }); @@ -58,9 +52,9 @@ describe('readConfigFile', () => { it('normalizes folder "." to ""', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ - { folder: '.', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: '.', actorName: 'apify/my-actor', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), - '.actor/actor.json': actorJson('my-actor'), + '.actor/actor.json': actorJson({}), }); const result = await readConfigFile(); @@ -68,45 +62,77 @@ describe('readConfigFile', () => { expectFileRead('.actor/actor.json'); }); + it('defaults dockerContextDir to actor folder when absent from actor.json', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'actors/web-scraper', actorName: 'apify/web-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + ]), + 'actors/web-scraper/.actor/actor.json': actorJson({}), + }); + + const result = await readConfigFile(); + expect(result[0].dockerContextDir).toBe('actors/web-scraper'); + }); + + it('resolves dockerContextDir relative to .actor/ folder', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'actors/shopify', actorName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN' }, + ]), + 'actors/shopify/.actor/actor.json': actorJson({ dockerContextDir: '../../..' }), + }); + + const result = await readConfigFile(); + expect(result[0].dockerContextDir).toBe(''); + }); + it('defaults isStandalone to false when omitted', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ - { folder: 'actors/web-scraper', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 'actors/web-scraper', actorName: 'apify/web-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), - 'actors/web-scraper/.actor/actor.json': actorJson('web-scraper'), + 'actors/web-scraper/.actor/actor.json': actorJson({}), }); const result = await readConfigFile(); - expectFileRead('actors/web-scraper/.actor/actor.json'); expect(result[0].isStandalone).toBe(false); }); it('respects isStandalone: true', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ - { folder: 'standalone/orchestrator', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY', isStandalone: true }, + { folder: 'standalone/orchestrator', actorName: 'apify/orchestrator', tokenEnvVar: 'APIFY_TOKEN_APIFY', isStandalone: true }, ]), - 'standalone/orchestrator/.actor/actor.json': actorJson('orchestrator'), + 'standalone/orchestrator/.actor/actor.json': actorJson({}), }); const result = await readConfigFile(); - expectFileRead('standalone/orchestrator/.actor/actor.json'); expect(result[0].isStandalone).toBe(true); }); + it('passes through overrideActorContext', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'actors/shopify', actorName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: ['actors/shopify', 'packages'] }, + ]), + 'actors/shopify/.actor/actor.json': actorJson({ dockerContextDir: '../../..' }), + }); + + const result = await readConfigFile(); + expect(result[0].overrideActorContext).toEqual(['actors/shopify', 'packages']); + }); + it('handles multiple actors', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ - { folder: 'actors/web-scraper', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, - { folder: 'actors/email-sender', owner: 'other-team', tokenEnvVar: 'APIFY_TOKEN_OTHER_TEAM', isStandalone: true }, + { folder: 'actors/web-scraper', actorName: 'apify/web-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 'actors/email-sender', actorName: 'other-team/email-sender', tokenEnvVar: 'APIFY_TOKEN_OTHER_TEAM', isStandalone: true }, ]), - 'actors/web-scraper/.actor/actor.json': actorJson('web-scraper'), - 'actors/email-sender/.actor/actor.json': actorJson('email-sender'), + 'actors/web-scraper/.actor/actor.json': actorJson({}), + 'actors/email-sender/.actor/actor.json': actorJson({}), }); const result = await readConfigFile(); - expectFileRead('actors/web-scraper/.actor/actor.json'); - expectFileRead('actors/email-sender/.actor/actor.json'); expect(result).toHaveLength(2); expect(result[0].actorName).toBe('apify/web-scraper'); expect(result[1].actorName).toBe('other-team/email-sender'); @@ -131,10 +157,10 @@ describe('readConfigFile', () => { it('throws on duplicate folders', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ - { folder: 'actors/shopify', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, - { folder: 'actors/shopify', owner: 'other-team', tokenEnvVar: 'APIFY_TOKEN_OTHER' }, + { folder: 'actors/shopify', actorName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 'actors/shopify', actorName: 'other/shopify', tokenEnvVar: 'APIFY_TOKEN_OTHER' }, ]), - 'actors/shopify/.actor/actor.json': actorJson('shopify-scraper'), + 'actors/shopify/.actor/actor.json': actorJson({}), }); await expect(readConfigFile()).rejects.toThrow('Duplicate folder'); @@ -143,10 +169,10 @@ describe('readConfigFile', () => { it('throws on duplicate folders after normalization ("." and "")', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ - { folder: '.', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, - { folder: '', owner: 'other-team', tokenEnvVar: 'APIFY_TOKEN_OTHER' }, + { folder: '.', actorName: 'apify/actor-a', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: '', actorName: 'other/actor-b', tokenEnvVar: 'APIFY_TOKEN_OTHER' }, ]), - '.actor/actor.json': actorJson('my-actor'), + '.actor/actor.json': actorJson({}), }); await expect(readConfigFile()).rejects.toThrow('Duplicate folder'); @@ -155,90 +181,65 @@ describe('readConfigFile', () => { it('throws when actor.json is missing', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ - { folder: 'actors/shopify', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 'actors/shopify', actorName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), }); await expect(readConfigFile()).rejects.toThrow('Cannot read'); }); - it('throws when actor.json has no name field', async () => { + it('throws when actorName is missing', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ - { folder: 'actors/shopify', owner: 'apify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 'actors/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), - 'actors/shopify/.actor/actor.json': JSON.stringify({ description: 'no name here' }), + 'actors/shopify/.actor/actor.json': actorJson({}), }); - await expect(readConfigFile()).rejects.toThrow('Missing "name"'); + await expect(readConfigFile()).rejects.toThrow('Invalid "actorName"'); }); -}); - -describe('generateConfigFile', () => { - it('generates config from discovered actor.json files', async () => { - fsMock.access.mockRejectedValue(new Error('ENOENT')); - execSyncMock.mockReturnValue( - 'actors/shopify/.actor/actor.json\nactors/amazon/.actor/actor.json\n', - ); - await generateConfigFile(); - - expect(fsMock.access).toHaveBeenCalledWith('.test-tools-actors-config.json'); - expect(fsMock.writeFile).toHaveBeenCalledOnce(); - const written = JSON.parse(fsMock.writeFile.mock.calls[0][1]); - expect(written.actors).toHaveLength(2); - expect(written.actors[0]).toEqual({ - folder: 'actors/shopify', - owner: '', - tokenEnvVar: '', - }); - expect(written.actors[1]).toEqual({ - folder: 'actors/amazon', - owner: '', - tokenEnvVar: '', + it('throws when actorName has no slash', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'actors/shopify', actorName: 'shopify-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + ]), + 'actors/shopify/.actor/actor.json': actorJson({}), }); - }); - - it('uses "." for root-level actor', async () => { - fsMock.access.mockRejectedValue(new Error('ENOENT')); - execSyncMock.mockReturnValue('.actor/actor.json\n'); - - await generateConfigFile(); - const written = JSON.parse(fsMock.writeFile.mock.calls[0][1]); - expect(written.actors[0].folder).toBe('.'); + await expect(readConfigFile()).rejects.toThrow('Invalid "actorName"'); }); - it('applies --default-owner and --default-token-var flags', async () => { - fsMock.access.mockRejectedValue(new Error('ENOENT')); - execSyncMock.mockReturnValue('actors/shopify/.actor/actor.json\n'); - - await generateConfigFile({ defaultOwner: 'myteam', defaultTokenVar: 'MY_TOKEN' }); - - const written = JSON.parse(fsMock.writeFile.mock.calls[0][1]); - expect(written.actors[0].owner).toBe('myteam'); - expect(written.actors[0].tokenEnvVar).toBe('MY_TOKEN'); - }); + it('throws when actorName has empty parts', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'actors/shopify', actorName: '/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + ]), + 'actors/shopify/.actor/actor.json': actorJson({}), + }); - it('throws if config file already exists', async () => { - await expect(generateConfigFile()).rejects.toThrow('already exists'); - expect(fsMock.access).toHaveBeenCalledWith('.test-tools-actors-config.json'); + await expect(readConfigFile()).rejects.toThrow('Invalid "actorName"'); }); - it('throws if no actor.json files found', async () => { - fsMock.access.mockRejectedValue(new Error('ENOENT')); - execSyncMock.mockReturnValue('src/main.ts\npackage.json\n'); + it('throws when overrideActorContext is not an array', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'actors/shopify', actorName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: 'packages' }, + ]), + 'actors/shopify/.actor/actor.json': actorJson({}), + }); - await expect(generateConfigFile()).rejects.toThrow('No .actor/actor.json files found'); + await expect(readConfigFile()).rejects.toThrow('Invalid "overrideActorContext"'); }); - it('warns when root actor.json exists alongside subfolder actors', async () => { - fsMock.access.mockRejectedValue(new Error('ENOENT')); - execSyncMock.mockReturnValue('.actor/actor.json\nactors/shopify/.actor/actor.json\n'); - const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => { /**/ }); - - await generateConfigFile(); + it('throws when overrideActorContext contains non-strings', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 'actors/shopify', actorName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: [123] }, + ]), + 'actors/shopify/.actor/actor.json': actorJson({}), + }); - expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('root-level .actor/actor.json')); + await expect(readConfigFile()).rejects.toThrow('Invalid "overrideActorContext"'); }); }); diff --git a/test/unit/should-built-and-test.test.ts b/test/unit/should-built-and-test.test.ts index 8ab49bc..9578fc2 100644 --- a/test/unit/should-built-and-test.test.ts +++ b/test/unit/should-built-and-test.test.ts @@ -13,18 +13,21 @@ describe('Should build and test parser', () => { folder: 'actors/lukaskrivka_testing-github-integration-1', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', + dockerContextDir: '', }, { actorName: 'lukaskrivka/testing-github-integration-2', folder: 'actors/lukaskrivka_testing-github-integration-2', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', + dockerContextDir: '', }, { actorName: 'lukaskrivka/test-standalone', folder: 'standalone-actors/lukaskrivka_test-standalone', isStandalone: true, tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', + dockerContextDir: 'standalone-actors/lukaskrivka_test-standalone', }, ]; @@ -269,54 +272,63 @@ describe('Should build and test parser', () => { folder: 'actors/compass_Google-Maps-Reviews-Scraper', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_COMPASS', + dockerContextDir: '', }, { actorName: 'compass/crawler-google-places', folder: 'actors/compass_crawler-google-places', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_COMPASS', + dockerContextDir: '', }, { actorName: 'compass/easy-google-maps', folder: 'actors/compass_easy-google-maps', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_COMPASS', + dockerContextDir: '', }, { actorName: 'compass/google-maps-extractor', folder: 'actors/compass_google-maps-extractor', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_COMPASS', + dockerContextDir: '', }, { actorName: 'compass/google-places-api', folder: 'actors/compass_google-places-api', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_COMPASS', + dockerContextDir: '', }, { actorName: 'lukaskrivka/google-maps-with-contact-details', folder: 'actors/lukaskrivka_google-maps-with-contact-details', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', + dockerContextDir: '', }, { actorName: 'natasha.lekh/gas-prices-scraper', folder: 'actors/natasha.lekh_gas-prices-scraper', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_NATASHA_LEKH', + dockerContextDir: '', }, { actorName: 'natasha.lekh/vegan-places-finder', folder: 'actors/natasha.lekh_vegan-places-finder', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_NATASHA_LEKH', + dockerContextDir: '', }, { actorName: 'lukaskrivka/google-maps-scraper-orchestrator', folder: 'standalone-actors/lukaskrivka_google-maps-scraper-orchestrator', isStandalone: true, tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', + dockerContextDir: 'standalone-actors/lukaskrivka_google-maps-scraper-orchestrator', }, ]; From 151e200d4053764f0851217a71c576594455d075 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Fri, 26 Jun 2026 15:23:48 +0100 Subject: [PATCH 14/33] Rewrite change detection to use dockerContextDir-based actor scoping --- bin/diff-changes.ts | 214 ++++++++++++------------ bin/types.ts | 2 - bin/utils.ts | 7 +- test/unit/bin/diff-changes.test.ts | 65 +++---- test/unit/bin/utils.test.ts | 53 +++--- test/unit/should-built-and-test.test.ts | 28 +--- 6 files changed, 165 insertions(+), 204 deletions(-) diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index 7796f2b..5bfe5bc 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -8,16 +8,6 @@ interface ShouldBuildAndTestOptions { commits: Commit[]; } -export const maybeParseActorFolder = ( - lowercaseFilePath: string, -): { isActorFolder: true; folder: string } | { isActorFolder: false } => { - const match = lowercaseFilePath.match(/^((?:standalone-)?actors\/[^/]+)\/.+/); - if (match) { - return { isActorFolder: true, folder: match[1] }; - } - return { isActorFolder: false }; -}; - const isIgnoredTopLevelFile = (lowercaseFilePath: string) => { const IGNORED_TOP_LEVEL_FILES = [ '.vscode/', @@ -29,66 +19,109 @@ const isIgnoredTopLevelFile = (lowercaseFilePath: string) => { '.prettierrc', '.editorconfig', ]; - // Strip out deprecated /code and /shared folders, treat them as top-level code - const sanitizedLowercaseFilePath = lowercaseFilePath.replace(/^code\//, '').replace(/^shared\//, ''); - - return IGNORED_TOP_LEVEL_FILES.some((ignoredFile) => sanitizedLowercaseFilePath.startsWith(ignoredFile)); + // Strip deprecated code/ and shared/ prefixes — repos like apify-store/amazon use these + const sanitized = lowercaseFilePath.replace(/^code\//, '').replace(/^shared\//, ''); + return IGNORED_TOP_LEVEL_FILES.some((pattern) => sanitized.startsWith(pattern)); }; -type FileChange = +type FileChangeForActor = | { impact: 'ignored' } - // Only things that influence how the Actor looks - e.g. README and CHANGELOG files, schema titles, descriptions, reordering, etc. We only need to rebuild on release - | { impact: 'cosmetic'; semanticallyVerified: boolean; includes: 'all-actors' | ActorConfig } - // Influences how the Actor works - we need to run tests - | { - impact: 'functional'; - includes: 'all-actors' | ActorConfig; - }; + | { impact: 'outside-context' } + | { impact: 'cosmetic'; semanticallyVerified: boolean } + | { impact: 'functional' }; + +const isFileInContext = (lowercaseFilePath: string, actor: ActorConfig): boolean => { + if (actor.overrideActorContext) { + return actor.overrideActorContext.some((contextPath) => { + const lowerContextPath = contextPath.toLowerCase(); + return lowerContextPath === '' || lowercaseFilePath.startsWith(`${lowerContextPath}/`); + }); + } + const lowerDockerContext = actor.dockerContextDir.toLowerCase(); + return lowerDockerContext === '' || lowercaseFilePath.startsWith(`${lowerDockerContext}/`); +}; +/** + * Classify a single file change for a single actor. + * + * Steps (in order): + * 1. Hardcoded ignore list (repo-level dev files) → ignored + * 2. Context matching (dockerContextDir or overrideActorContext) → outside-context if no match + * 3. README/CHANGELOG by filename → cosmetic (not semantically verified) + * 4. .json inside the actor's own folder with only cosmetic schema diffs → cosmetic (semantically verified) + * 5. Everything else → functional + */ const classifyFileChange = ( originalFilePath: string, - actorConfigs: ActorConfig[], + actor: ActorConfig, commits: Commit[], -): FileChange => { - // Lowercase for case-insensitive matching; keep original for git show (case-sensitive on Linux) + cosmeticCache: Map, +): FileChangeForActor => { const lowercaseFilePath = originalFilePath.toLowerCase(); + if (isIgnoredTopLevelFile(lowercaseFilePath)) { return { impact: 'ignored' }; } - if (lowercaseFilePath.endsWith('changelog.md')) { - return { impact: 'cosmetic', semanticallyVerified: false, includes: 'all-actors' }; + if (!isFileInContext(lowercaseFilePath, actor)) { + return { impact: 'outside-context' }; } - const actorFolderInfo = maybeParseActorFolder(lowercaseFilePath); - if (actorFolderInfo.isActorFolder) { - const actorConfigChanged = actorConfigs.find( - ({ folder }) => folder.toLowerCase() === actorFolderInfo.folder, - ); - // This is some super weird case that happened once in the past but I don't remember the context anymore - if (actorConfigChanged === undefined) { - console.error( - 'SHOULD NEVER HAPPEN: changes was found in an actor folder which no longer exists in the current commit, skipping this file', - { - folder: actorFolderInfo.folder, - lowercaseFilePath, - }, - ); - return { impact: 'ignored' }; - } - if (lowercaseFilePath.endsWith('readme.md')) { - return { impact: 'cosmetic', semanticallyVerified: false, includes: actorConfigChanged }; + if (lowercaseFilePath.endsWith('readme.md') || lowercaseFilePath.endsWith('changelog.md')) { + return { impact: 'cosmetic', semanticallyVerified: false }; + } + + const lowerFolder = actor.folder.toLowerCase(); + const isInActorFolder = lowerFolder === '' || lowercaseFilePath.startsWith(`${lowerFolder}/`); + if (lowercaseFilePath.endsWith('.json') && isInActorFolder) { + let isCosmetic = cosmeticCache.get(originalFilePath); + if (isCosmetic === undefined) { + isCosmetic = isCosmeticOnlyJsonSchemaChange(commits, originalFilePath); + cosmeticCache.set(originalFilePath, isCosmetic); } - // originalFilePath must be used here (not lowercaseFilePath) — git show is case-sensitive on Linux - if (lowercaseFilePath.endsWith('.json') && isCosmeticOnlyJsonSchemaChange(commits, originalFilePath)) { - return { impact: 'cosmetic', semanticallyVerified: true, includes: actorConfigChanged }; + if (isCosmetic) { + return { impact: 'cosmetic', semanticallyVerified: true }; } - - return { impact: 'functional', includes: actorConfigChanged }; } - // For any other files, we assume they can interact with the code - return { impact: 'functional', includes: 'all-actors' }; + return { impact: 'functional' }; +}; + +/** + * Check if a file falls inside another actor's folder. + * Root actors (folder === "") never exclude files from siblings. + */ +const isExcludedBySibling = (lowercaseFilePath: string, actor: ActorConfig, allActors: ActorConfig[]): boolean => { + return allActors.some( + (other) => + other.folder !== actor.folder && + other.folder !== '' && + lowercaseFilePath.startsWith(`${other.folder.toLowerCase()}/`), + ); +}; + +type LoggableImpact = 'ignored' | 'cosmetic' | 'functional'; + +const IMPACT_PRIORITY: Record = { functional: 3, cosmetic: 2, ignored: 1 }; + +/** + * A file can be classified differently by different actors (e.g. functional for a broad-context + * actor, outside-context for a narrow one). For logging we keep the most significant classification + * across all actors: functional > cosmetic > ignored. Files that are outside-context for every + * actor are treated as ignored. + */ +const updateFileImpact = ( + fileImpacts: Map, + filePath: string, + impact: FileChangeForActor['impact'], +): void => { + if (impact === 'outside-context') return; + + const loggable = impact as LoggableImpact; + const current = fileImpacts.get(filePath); + if (!current || IMPACT_PRIORITY[loggable] > IMPACT_PRIORITY[current]) { + fileImpacts.set(filePath, loggable); + } }; export const getChangedActors = ({ @@ -97,72 +130,47 @@ export const getChangedActors = ({ isLatest = false, commits, }: ShouldBuildAndTestOptions): ActorConfig[] => { - // folder -> ActorConfig const actorsChangedMap = new Map(); + const cosmeticCache = new Map(); + const fileImpacts = new Map(); - const actorConfigsWithoutStandalone = actorConfigs.filter(({ isStandalone }) => !isStandalone); + for (const actor of actorConfigs) { + for (const originalFilePath of filepathsChanged) { + const lowercaseFilePath = originalFilePath.toLowerCase(); - for (const originalFilePath of filepathsChanged) { - const fileChange = classifyFileChange(originalFilePath, actorConfigs, commits); - if (fileChange.impact === 'ignored') { - continue; - } + if (isExcludedBySibling(lowercaseFilePath, actor, actorConfigs)) { + continue; + } - if (fileChange.impact === 'cosmetic' && !isLatest) { - continue; - } + const change = classifyFileChange(originalFilePath, actor, commits, cosmeticCache); + updateFileImpact(fileImpacts, originalFilePath, change.impact); - if (fileChange.includes !== 'all-actors') { - actorsChangedMap.set(fileChange.includes.folder, fileChange.includes); - } else if (fileChange.includes === 'all-actors') { - // Standalone Actors are handled always via specific actors change, not all-actors - for (const actorConfig of actorConfigsWithoutStandalone) { - actorsChangedMap.set(actorConfig.folder, actorConfig); - } + if (change.impact === 'ignored' || change.impact === 'outside-context') continue; + if (change.impact === 'cosmetic' && !isLatest) continue; + + actorsChangedMap.set(actor.folder, actor); } } const actorsChanged = Array.from(actorsChangedMap.values()); - // All below here is just for logging + // Logging const formatFiles = (files: string[]) => (files.length > 0 ? files.join(', ') : ''); - const ignoredFilesChanged = filepathsChanged.filter( - (file) => classifyFileChange(file, actorConfigs, commits).impact === 'ignored', - ); - console.error(`[DIFF]: Ignored files (don't trigger test or build): ${formatFiles(ignoredFilesChanged)}`); - - const cosmeticChanges = filepathsChanged - .map((file) => ({ file, change: classifyFileChange(file, actorConfigs, commits) })) - .filter(({ change }) => change.impact === 'cosmetic') as { - file: string; - change: Extract; - }[]; - const semanticallyVerifiedFiles = cosmeticChanges - .filter(({ change }) => change.semanticallyVerified) - .map(({ file }) => file); - const inherentlyCosmeticFiles = cosmeticChanges - .filter(({ change }) => !change.semanticallyVerified) - .map(({ file }) => file); - console.error( - `[DIFF]: Cosmetic-only JSON schema changes (semantically verified, only trigger release build): ${formatFiles(semanticallyVerifiedFiles)}`, - ); - console.error( - `[DIFF]: Inherently cosmetic files (README, CHANGELOG — only trigger release build): ${formatFiles(inherentlyCosmeticFiles)}`, - ); + const ignoredFiles = filepathsChanged.filter((file) => { + const impact = fileImpacts.get(file); + return impact === 'ignored' || !impact; + }); + const cosmeticFiles = filepathsChanged.filter((file) => fileImpacts.get(file) === 'cosmetic'); + const functionalFiles = filepathsChanged.filter((file) => fileImpacts.get(file) === 'functional'); - const functionalFilesChanged = filepathsChanged.filter( - (file) => classifyFileChange(file, actorConfigs, commits).impact === 'functional', - ); - console.error(`[DIFF]: Functional files (trigger test & release build): ${formatFiles(functionalFilesChanged)}`); + console.error(`[DIFF]: Ignored files (don't trigger test or build): ${formatFiles(ignoredFiles)}`); + console.error(`[DIFF]: Cosmetic files (only trigger release build): ${formatFiles(cosmeticFiles)}`); + console.error(`[DIFF]: Functional files (trigger test & release build): ${formatFiles(functionalFiles)}`); if (actorsChanged.length > 0) { - const miniactors = actorsChanged.filter((config) => !config.isStandalone).map((config) => config.actorName); - const standaloneActors = actorsChanged - .filter((config) => config.isStandalone) - .map((config) => config.actorName); - console.error(`[DIFF]: MiniActors to be built and tested: ${miniactors.join(', ')}`); - console.error(`[DIFF]: Standalone Actors to be built and tested: ${standaloneActors.join(', ')}`); + const actorNames = actorsChanged.map((config) => config.actorName); + console.error(`[DIFF]: Actors to be built and tested: ${actorNames.join(', ')}`); } else { console.error(`[DIFF]: No relevant files changed, skipping builds and tests`); } diff --git a/bin/types.ts b/bin/types.ts index a823b2e..06d4bd5 100644 --- a/bin/types.ts +++ b/bin/types.ts @@ -89,7 +89,6 @@ export interface ActorConfigFileEntry { folder: string; actorName: string; tokenEnvVar: string; - isStandalone?: boolean; overrideActorContext?: string[]; } @@ -108,7 +107,6 @@ export interface BuildData { export interface ActorConfig { actorName: string; folder: string; - isStandalone: boolean; tokenEnvVar: string; dockerContextDir: string; overrideActorContext?: string[]; diff --git a/bin/utils.ts b/bin/utils.ts index 86543a9..15089af 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -80,7 +80,10 @@ export const readConfigFile = async (): Promise => { } if (entry.overrideActorContext !== undefined) { - if (!Array.isArray(entry.overrideActorContext) || !entry.overrideActorContext.every((p) => typeof p === 'string')) { + if ( + !Array.isArray(entry.overrideActorContext) || + !entry.overrideActorContext.every((p) => typeof p === 'string') + ) { throw new Error( `Invalid "overrideActorContext" for folder "${entry.folder}" in "${CONFIG_FILE_NAME}". ` + `Must be an array of strings.`, @@ -117,7 +120,6 @@ export const readConfigFile = async (): Promise => { actorConfigs.push({ actorName: entry.actorName, folder, - isStandalone: entry.isStandalone ?? false, tokenEnvVar: entry.tokenEnvVar, dockerContextDir: normalizedDockerContextDir, overrideActorContext: entry.overrideActorContext, @@ -127,7 +129,6 @@ export const readConfigFile = async (): Promise => { return actorConfigs; }; - export const setCwd = ({ workspace }: { workspace: string | undefined }) => { if (workspace) { process.chdir(workspace); diff --git a/test/unit/bin/diff-changes.test.ts b/test/unit/bin/diff-changes.test.ts index 267487a..f90cf89 100644 --- a/test/unit/bin/diff-changes.test.ts +++ b/test/unit/bin/diff-changes.test.ts @@ -1,14 +1,18 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { getChangedActors, maybeParseActorFolder } from '../../../bin/diff-changes.js'; +import { getChangedActors } from '../../../bin/diff-changes.js'; import * as DiffJsonSchema from '../../../bin/diff-json-schema.js'; import type { ActorConfig } from '../../../bin/types.js'; -const miniActor: ActorConfig = { actorName: 'foo/bar', folder: 'actors/foo_bar', isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_FOO', dockerContextDir: '' }; +const miniActor: ActorConfig = { + actorName: 'foo/bar', + folder: 'actors/foo_bar', + tokenEnvVar: 'APIFY_TOKEN_FOO', + dockerContextDir: '', +}; const standaloneActor: ActorConfig = { actorName: 'owner/standalone', folder: 'standalone-actors/standalone', - isStandalone: true, tokenEnvVar: 'APIFY_TOKEN_OWNER', dockerContextDir: 'standalone-actors/standalone', }; @@ -16,41 +20,6 @@ const actorConfigs = [miniActor, standaloneActor]; const commits = [{ sha: 'Commit1', author: '', date: '', message: '' }]; -describe('maybeParseActorFolder', () => { - it('returns folder for actors/ path', () => { - expect(maybeParseActorFolder('actors/foo_bar/actor.json')).toEqual({ - isActorFolder: true, - folder: 'actors/foo_bar', - }); - }); - - it('returns folder for standalone-actors/ path', () => { - expect(maybeParseActorFolder('standalone-actors/my_actor/main.ts')).toEqual({ - isActorFolder: true, - folder: 'standalone-actors/my_actor', - }); - }); - - it('returns false for top-level file', () => { - expect(maybeParseActorFolder('package.json')).toEqual({ isActorFolder: false }); - }); - - it('returns false for path with no file inside actor folder', () => { - expect(maybeParseActorFolder('actors/foo_bar')).toEqual({ isActorFolder: false }); - }); - - it('returns folder for ownerless actors/ path', () => { - expect(maybeParseActorFolder('actors/shopify/src/main.ts')).toEqual({ - isActorFolder: true, - folder: 'actors/shopify', - }); - }); - - it('returns false for unrelated folder', () => { - expect(maybeParseActorFolder('src/utils.ts')).toEqual({ isActorFolder: false }); - }); -}); - describe('getChangedActors', () => { beforeEach(() => { vi.spyOn(DiffJsonSchema, 'isCosmeticOnlyJsonSchemaChange').mockReturnValue(false); @@ -130,7 +99,7 @@ describe('getChangedActors', () => { expect(result).toEqual([miniActor]); }); - it('returns all non-standalone actors when a non-actor-folder functional file changes', () => { + it('does not trigger narrow-context actor when shared file changes', () => { const result = getChangedActors({ filepathsChanged: ['shared/utils.ts'], actorConfigs, @@ -140,7 +109,7 @@ describe('getChangedActors', () => { expect(result).not.toContainEqual(standaloneActor); }); - it('does not include standalone actor in all-actors expansion from changelog', () => { + it('does not trigger narrow-context actor from root changelog', () => { const result = getChangedActors({ filepathsChanged: ['CHANGELOG.md'], actorConfigs, @@ -151,7 +120,7 @@ describe('getChangedActors', () => { expect(result).not.toContainEqual(standaloneActor); }); - it('includes standalone actor when its own folder changes', () => { + it('triggers narrow-context actor when its own folder changes', () => { const result = getChangedActors({ filepathsChanged: ['standalone-actors/standalone/src/main.ts'], actorConfigs, @@ -170,7 +139,7 @@ describe('getChangedActors', () => { expect(result).toContainEqual(miniActor); }); - it('handles mixed changes: returns both mini and standalone actors', () => { + it('handles mixed changes: returns both broad and narrow-context actors', () => { const result = getChangedActors({ filepathsChanged: ['actors/foo_bar/src/main.ts', 'standalone-actors/standalone/Dockerfile'], actorConfigs, @@ -180,11 +149,10 @@ describe('getChangedActors', () => { expect(result).toContainEqual(standaloneActor); }); - it('matches ownerless folder where folder name differs from actor name', () => { + it('matches folder where folder name differs from actor name', () => { const ownerlessActor: ActorConfig = { actorName: 'myteam/shopify-scraper', folder: 'actors/shopify', - isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_MYTEAM', dockerContextDir: '', }; @@ -197,7 +165,12 @@ describe('getChangedActors', () => { }); it('in single-actor repo, .actor/ changes trigger builds', () => { - const rootActor: ActorConfig = { actorName: 'myteam/my-actor', folder: '', isStandalone: false, tokenEnvVar: 'BUILDER_APIFY_TOKEN', dockerContextDir: '' }; + const rootActor: ActorConfig = { + actorName: 'myteam/my-actor', + folder: '', + tokenEnvVar: 'BUILDER_APIFY_TOKEN', + dockerContextDir: '', + }; const result = getChangedActors({ filepathsChanged: ['.actor/actor.json'], actorConfigs: [rootActor], @@ -206,7 +179,7 @@ describe('getChangedActors', () => { expect(result).toEqual([rootActor]); }); - it('in multi-actor repo, .actor/ changes trigger builds for all non-standalone actors', () => { + it('in multi-actor repo, .actor/ changes only trigger broad-context actors', () => { const result = getChangedActors({ filepathsChanged: ['.actor/actor.json'], actorConfigs, diff --git a/test/unit/bin/utils.test.ts b/test/unit/bin/utils.test.ts index 475734b..8da1d35 100644 --- a/test/unit/bin/utils.test.ts +++ b/test/unit/bin/utils.test.ts @@ -41,7 +41,6 @@ describe('readConfigFile', () => { { actorName: 'myteam/shopify-scraper', folder: 'actors/shopify', - isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_MYTEAM', dockerContextDir: '', overrideActorContext: undefined, @@ -86,34 +85,15 @@ describe('readConfigFile', () => { expect(result[0].dockerContextDir).toBe(''); }); - it('defaults isStandalone to false when omitted', async () => { - mockFiles({ - '.test-tools-actors-config.json': validConfig([ - { folder: 'actors/web-scraper', actorName: 'apify/web-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, - ]), - 'actors/web-scraper/.actor/actor.json': actorJson({}), - }); - - const result = await readConfigFile(); - expect(result[0].isStandalone).toBe(false); - }); - - it('respects isStandalone: true', async () => { - mockFiles({ - '.test-tools-actors-config.json': validConfig([ - { folder: 'standalone/orchestrator', actorName: 'apify/orchestrator', tokenEnvVar: 'APIFY_TOKEN_APIFY', isStandalone: true }, - ]), - 'standalone/orchestrator/.actor/actor.json': actorJson({}), - }); - - const result = await readConfigFile(); - expect(result[0].isStandalone).toBe(true); - }); - it('passes through overrideActorContext', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ - { folder: 'actors/shopify', actorName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: ['actors/shopify', 'packages'] }, + { + folder: 'actors/shopify', + actorName: 'myteam/shopify', + tokenEnvVar: 'APIFY_TOKEN', + overrideActorContext: ['actors/shopify', 'packages'], + }, ]), 'actors/shopify/.actor/actor.json': actorJson({ dockerContextDir: '../../..' }), }); @@ -126,7 +106,11 @@ describe('readConfigFile', () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ { folder: 'actors/web-scraper', actorName: 'apify/web-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, - { folder: 'actors/email-sender', actorName: 'other-team/email-sender', tokenEnvVar: 'APIFY_TOKEN_OTHER_TEAM', isStandalone: true }, + { + folder: 'actors/email-sender', + actorName: 'other-team/email-sender', + tokenEnvVar: 'APIFY_TOKEN_OTHER_TEAM', + }, ]), 'actors/web-scraper/.actor/actor.json': actorJson({}), 'actors/email-sender/.actor/actor.json': actorJson({}), @@ -136,7 +120,6 @@ describe('readConfigFile', () => { expect(result).toHaveLength(2); expect(result[0].actorName).toBe('apify/web-scraper'); expect(result[1].actorName).toBe('other-team/email-sender'); - expect(result[1].isStandalone).toBe(true); }); it('throws when config file is missing', async () => { @@ -224,7 +207,12 @@ describe('readConfigFile', () => { it('throws when overrideActorContext is not an array', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ - { folder: 'actors/shopify', actorName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: 'packages' }, + { + folder: 'actors/shopify', + actorName: 'myteam/shopify', + tokenEnvVar: 'APIFY_TOKEN', + overrideActorContext: 'packages', + }, ]), 'actors/shopify/.actor/actor.json': actorJson({}), }); @@ -235,7 +223,12 @@ describe('readConfigFile', () => { it('throws when overrideActorContext contains non-strings', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ - { folder: 'actors/shopify', actorName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: [123] }, + { + folder: 'actors/shopify', + actorName: 'myteam/shopify', + tokenEnvVar: 'APIFY_TOKEN', + overrideActorContext: [123], + }, ]), 'actors/shopify/.actor/actor.json': actorJson({}), }); diff --git a/test/unit/should-built-and-test.test.ts b/test/unit/should-built-and-test.test.ts index 9578fc2..32b9bea 100644 --- a/test/unit/should-built-and-test.test.ts +++ b/test/unit/should-built-and-test.test.ts @@ -11,21 +11,18 @@ describe('Should build and test parser', () => { { actorName: 'lukaskrivka/testing-github-integration-1', folder: 'actors/lukaskrivka_testing-github-integration-1', - isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: '', }, { actorName: 'lukaskrivka/testing-github-integration-2', folder: 'actors/lukaskrivka_testing-github-integration-2', - isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: '', }, { actorName: 'lukaskrivka/test-standalone', folder: 'standalone-actors/lukaskrivka_test-standalone', - isStandalone: true, tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: 'standalone-actors/lukaskrivka_test-standalone', }, @@ -69,7 +66,7 @@ describe('Should build and test parser', () => { expect(actorsChanged).toEqual([]); }); - test('.actor/ changes always trigger builds for all non-standalone actors', () => { + test('.actor/ changes trigger builds for broad-context actors', () => { const FILES = ['.actor/actor.json']; const actorsChanged = getChangedActors({ @@ -79,7 +76,7 @@ describe('Should build and test parser', () => { commits, }); - expect(actorsChanged).toEqual(ACTOR_CONFIGS.filter(({ isStandalone }) => !isStandalone)); + expect(actorsChanged).toEqual(ACTOR_CONFIGS.slice(0, 2)); }); test('Only builds latest for all Actors', () => { @@ -92,10 +89,10 @@ describe('Should build and test parser', () => { commits, }); - expect(actorsChanged).toEqual(ACTOR_CONFIGS.filter(({ isStandalone }) => !isStandalone)); + expect(actorsChanged).toEqual(ACTOR_CONFIGS.slice(0, 2)); }); - test('Code updated, tests miniactors', () => { + test('Code updated, tests broad-context actors', () => { const FILES = ['code/src/main.ts', 'package.json']; const actorsChanged = getChangedActors({ @@ -105,7 +102,7 @@ describe('Should build and test parser', () => { commits, }); - expect(actorsChanged).toEqual(ACTOR_CONFIGS.filter(({ isStandalone }) => !isStandalone)); + expect(actorsChanged).toEqual(ACTOR_CONFIGS.slice(0, 2)); }); test('Specific Actor functionality configs updated', () => { @@ -138,7 +135,7 @@ describe('Should build and test parser', () => { expect(actorsChanged).toEqual(ACTOR_CONFIGS.slice(0, 2)); }); - test('Miniactor, Code and standalone actor updated,', () => { + test('Actor folder, shared code and narrow-context actor updated', () => { const FILES = [ 'actors/lukaskrivka_testing-github-integration-1/.actor/actor.json', 'code/src/main.ts', @@ -235,7 +232,7 @@ describe('Should build and test parser', () => { expect(actorsChanged).toEqual([ACTOR_CONFIGS[1]]); }); - test('Standalone actor with cosmetic-only JSON change in PR context skips tests', () => { + test('Narrow-context actor with cosmetic-only JSON change in PR context skips tests', () => { const FILES = ['standalone-actors/lukaskrivka_test-standalone/.actor/actor.json']; isCosmeticOnlyJsonSchemaSpy.mockReturnValue(true); @@ -270,63 +267,54 @@ describe('Should build and test parser', () => { // Edge case of capitals in actor name :) actorName: 'compass/Google-Maps-Reviews-Scraper', folder: 'actors/compass_Google-Maps-Reviews-Scraper', - isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', }, { actorName: 'compass/crawler-google-places', folder: 'actors/compass_crawler-google-places', - isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', }, { actorName: 'compass/easy-google-maps', folder: 'actors/compass_easy-google-maps', - isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', }, { actorName: 'compass/google-maps-extractor', folder: 'actors/compass_google-maps-extractor', - isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', }, { actorName: 'compass/google-places-api', folder: 'actors/compass_google-places-api', - isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', }, { actorName: 'lukaskrivka/google-maps-with-contact-details', folder: 'actors/lukaskrivka_google-maps-with-contact-details', - isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: '', }, { actorName: 'natasha.lekh/gas-prices-scraper', folder: 'actors/natasha.lekh_gas-prices-scraper', - isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_NATASHA_LEKH', dockerContextDir: '', }, { actorName: 'natasha.lekh/vegan-places-finder', folder: 'actors/natasha.lekh_vegan-places-finder', - isStandalone: false, tokenEnvVar: 'APIFY_TOKEN_NATASHA_LEKH', dockerContextDir: '', }, { actorName: 'lukaskrivka/google-maps-scraper-orchestrator', folder: 'standalone-actors/lukaskrivka_google-maps-scraper-orchestrator', - isStandalone: true, tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: 'standalone-actors/lukaskrivka_google-maps-scraper-orchestrator', }, @@ -339,6 +327,6 @@ describe('Should build and test parser', () => { commits, }); - expect(actorsChanged).toEqual(ACTOR_CONFIGS_GOOGLE_MAPS.filter(({ isStandalone }) => !isStandalone)); + expect(actorsChanged).toEqual(ACTOR_CONFIGS_GOOGLE_MAPS.slice(0, 8)); }); }); From 6a68294cd098f3f0147b924453b1a0e9a198f22f Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Fri, 26 Jun 2026 15:36:02 +0100 Subject: [PATCH 15/33] Respect dockerignore to further skip non relevant paths --- bin/diff-changes.ts | 17 +++++++++++++---- bin/dockerignore.ts | 43 +++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 38 ++++++++++++++++++++++++-------------- package.json | 9 +++++---- 4 files changed, 85 insertions(+), 22 deletions(-) create mode 100644 bin/dockerignore.ts diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index 5bfe5bc..4878955 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -1,4 +1,5 @@ import { isCosmeticOnlyJsonSchemaChange } from './diff-json-schema.js'; +import { type DockerIgnoreMatcher, loadDockerIgnore } from './dockerignore.js'; import type { ActorConfig, Commit } from './types.js'; interface ShouldBuildAndTestOptions { @@ -47,15 +48,17 @@ const isFileInContext = (lowercaseFilePath: string, actor: ActorConfig): boolean * Steps (in order): * 1. Hardcoded ignore list (repo-level dev files) → ignored * 2. Context matching (dockerContextDir or overrideActorContext) → outside-context if no match - * 3. README/CHANGELOG by filename → cosmetic (not semantically verified) - * 4. .json inside the actor's own folder with only cosmetic schema diffs → cosmetic (semantically verified) - * 5. Everything else → functional + * 3. .dockerignore filtering (patterns relative to dockerContextDir) → ignored if matched + * 4. README/CHANGELOG by filename → cosmetic (not semantically verified) + * 5. .json inside the actor's own folder with only cosmetic schema diffs → cosmetic (semantically verified) + * 6. Everything else → functional */ const classifyFileChange = ( originalFilePath: string, actor: ActorConfig, commits: Commit[], cosmeticCache: Map, + dockerIgnoreMatcher: DockerIgnoreMatcher, ): FileChangeForActor => { const lowercaseFilePath = originalFilePath.toLowerCase(); @@ -67,6 +70,10 @@ const classifyFileChange = ( return { impact: 'outside-context' }; } + if (dockerIgnoreMatcher(originalFilePath)) { + return { impact: 'ignored' }; + } + if (lowercaseFilePath.endsWith('readme.md') || lowercaseFilePath.endsWith('changelog.md')) { return { impact: 'cosmetic', semanticallyVerified: false }; } @@ -135,6 +142,8 @@ export const getChangedActors = ({ const fileImpacts = new Map(); for (const actor of actorConfigs) { + const dockerIgnoreMatcher = loadDockerIgnore(actor.dockerContextDir); + for (const originalFilePath of filepathsChanged) { const lowercaseFilePath = originalFilePath.toLowerCase(); @@ -142,7 +151,7 @@ export const getChangedActors = ({ continue; } - const change = classifyFileChange(originalFilePath, actor, commits, cosmeticCache); + const change = classifyFileChange(originalFilePath, actor, commits, cosmeticCache, dockerIgnoreMatcher); updateFileImpact(fileImpacts, originalFilePath, change.impact); if (change.impact === 'ignored' || change.impact === 'outside-context') continue; diff --git a/bin/dockerignore.ts b/bin/dockerignore.ts new file mode 100644 index 0000000..09281e9 --- /dev/null +++ b/bin/dockerignore.ts @@ -0,0 +1,43 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import ignore from 'ignore'; + +export type DockerIgnoreMatcher = (repoRelativePath: string) => boolean; + +/** + * Load .dockerignore from the root of an actor's dockerContextDir and return a matcher + * that accepts repo-root-relative file paths. Patterns are resolved relative to + * dockerContextDir, matching Docker's own behavior. + * + * Returns a no-op matcher (always returns false) when the file is absent. + */ +export const loadDockerIgnore = (dockerContextDir: string): DockerIgnoreMatcher => { + const dockerignorePath = dockerContextDir ? path.join(dockerContextDir, '.dockerignore') : '.dockerignore'; + + let content: string; + try { + content = fs.readFileSync(dockerignorePath, 'utf-8'); + } catch { + return () => false; + } + + const matcher = ignore().add(content); + + return (repoRelativePath: string): boolean => { + const lowerPath = repoRelativePath.toLowerCase(); + const lowerContext = dockerContextDir.toLowerCase(); + + // Strip the dockerContextDir prefix to get a path relative to the context root + let relativePath: string; + if (lowerContext === '') { + relativePath = repoRelativePath; + } else if (lowerPath.startsWith(`${lowerContext}/`)) { + relativePath = repoRelativePath.slice(dockerContextDir.length + 1); + } else { + return false; + } + + return matcher.ignores(relativePath); + }; +}; diff --git a/package-lock.json b/package-lock.json index 46e3634..fb5784d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@apify/consts": "^2.43.0", "@slack/web-api": "^7.9.2", "apify-client": "^2.22.2", + "ignore": "^7.0.5", "yargs": "^18.0.0" }, "bin": { @@ -765,6 +766,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@eslint/eslintrc/node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/@eslint/eslintrc/node_modules/minimatch": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", @@ -3505,6 +3516,16 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/eslint/node_modules/minimatch": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", @@ -4252,10 +4273,9 @@ } }, "node_modules/ignore": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", - "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", - "dev": true, + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", "license": "MIT", "engines": { "node": ">= 4" @@ -7021,16 +7041,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/typescript-eslint/node_modules/ignore": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", - "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, "node_modules/typescript-eslint/node_modules/minimatch": { "version": "10.2.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", diff --git a/package.json b/package.json index d8f3f85..2ffbbea 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@apify/consts": "^2.43.0", "@slack/web-api": "^7.9.2", "apify-client": "^2.22.2", + "ignore": "^7.0.5", "yargs": "^18.0.0" }, "devDependencies": { @@ -29,14 +30,14 @@ "eslint": "^9.29.0", "eslint-config-prettier": "^10.1.5", "globals": "^17.0.0", + "husky": "^9.0.11", + "knip": "^5.65.0", + "lint-staged": "^15.2.2", "prettier": "^3.5.3", "tsx": "^4.20.3", "typescript": "^5.9.3", "typescript-eslint": "^8.34.1", - "vitest": "^3.2.4", - "knip": "^5.65.0", - "husky": "^9.0.11", - "lint-staged": "^15.2.2" + "vitest": "^3.2.4" }, "peerDependencies": { "vitest": ">=3.2.4" From a77a6381aea979f49ddc61ea08b7609cce956744 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Fri, 26 Jun 2026 15:57:39 +0100 Subject: [PATCH 16/33] update tests --- test/unit/bin/diff-changes.test.ts | 150 +++++++++++++++++++++++++++++ test/unit/bin/dockerignore.test.ts | 83 ++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 test/unit/bin/dockerignore.test.ts diff --git a/test/unit/bin/diff-changes.test.ts b/test/unit/bin/diff-changes.test.ts index f90cf89..06dcdb8 100644 --- a/test/unit/bin/diff-changes.test.ts +++ b/test/unit/bin/diff-changes.test.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { getChangedActors } from '../../../bin/diff-changes.js'; import * as DiffJsonSchema from '../../../bin/diff-json-schema.js'; +import * as Dockerignore from '../../../bin/dockerignore.js'; import type { ActorConfig } from '../../../bin/types.js'; const miniActor: ActorConfig = { @@ -196,4 +197,153 @@ describe('getChangedActors', () => { }); expect(result).toEqual([miniActor]); }); + + it('triggers actor with overrideActorContext when file matches an override path', () => { + const overrideActor: ActorConfig = { + actorName: 'team/override-actor', + folder: 'actors/override', + tokenEnvVar: 'APIFY_TOKEN_TEAM', + dockerContextDir: 'actors/override', + overrideActorContext: ['actors/override', 'packages'], + }; + const result = getChangedActors({ + filepathsChanged: ['packages/shared/utils.ts'], + actorConfigs: [overrideActor], + commits, + }); + expect(result).toEqual([overrideActor]); + }); + + it('does not trigger actor with overrideActorContext when file is outside all override paths', () => { + const overrideActor: ActorConfig = { + actorName: 'team/override-actor', + folder: 'actors/override', + tokenEnvVar: 'APIFY_TOKEN_TEAM', + dockerContextDir: 'actors/override', + overrideActorContext: ['actors/override', 'packages'], + }; + const result = getChangedActors({ + filepathsChanged: ['other-dir/file.ts'], + actorConfigs: [overrideActor], + commits, + }); + expect(result).toEqual([]); + }); + + it('broad-context actor skips files in sibling actor folders', () => { + const actorA: ActorConfig = { + actorName: 'team/actor-a', + folder: 'actors/a', + tokenEnvVar: 'APIFY_TOKEN_TEAM', + dockerContextDir: '', + }; + const actorB: ActorConfig = { + actorName: 'team/actor-b', + folder: 'actors/b', + tokenEnvVar: 'APIFY_TOKEN_TEAM', + dockerContextDir: '', + }; + const result = getChangedActors({ + filepathsChanged: ['actors/b/src/main.ts'], + actorConfigs: [actorA, actorB], + commits, + }); + expect(result).toEqual([actorB]); + }); + + it('root actor (folder="") is excluded from sibling actor folder files', () => { + const rootActor: ActorConfig = { + actorName: 'team/root', + folder: '', + tokenEnvVar: 'APIFY_TOKEN_TEAM', + dockerContextDir: '', + }; + const childActor: ActorConfig = { + actorName: 'team/child', + folder: 'actors/child', + tokenEnvVar: 'APIFY_TOKEN_TEAM', + dockerContextDir: 'actors/child', + }; + const result = getChangedActors({ + filepathsChanged: ['actors/child/src/main.ts'], + actorConfigs: [rootActor, childActor], + commits, + }); + expect(result).not.toContainEqual(rootActor); + expect(result).toContainEqual(childActor); + }); + + it('root actor (folder="") sees files outside any actor folder', () => { + const rootActor: ActorConfig = { + actorName: 'team/root', + folder: '', + tokenEnvVar: 'APIFY_TOKEN_TEAM', + dockerContextDir: '', + }; + const childActor: ActorConfig = { + actorName: 'team/child', + folder: 'actors/child', + tokenEnvVar: 'APIFY_TOKEN_TEAM', + dockerContextDir: 'actors/child', + }; + const result = getChangedActors({ + filepathsChanged: ['lib/shared-utils.ts'], + actorConfigs: [rootActor, childActor], + commits, + }); + expect(result).toContainEqual(rootActor); + expect(result).not.toContainEqual(childActor); + }); + + it('file matched by .dockerignore is treated as ignored', () => { + vi.spyOn(Dockerignore, 'loadDockerIgnore').mockReturnValue( + (filePath) => filePath === 'actors/foo_bar/node_modules/foo.js', + ); + const result = getChangedActors({ + filepathsChanged: ['actors/foo_bar/node_modules/foo.js'], + actorConfigs: [miniActor], + commits, + }); + expect(result).toEqual([]); + }); + + it('file not matched by .dockerignore is classified normally', () => { + vi.spyOn(Dockerignore, 'loadDockerIgnore').mockReturnValue((filePath) => filePath.includes('node_modules')); + const result = getChangedActors({ + filepathsChanged: ['actors/foo_bar/src/main.ts'], + actorConfigs: [miniActor], + commits, + }); + expect(result).toEqual([miniActor]); + }); + + it('JSON file in context but outside actor folder is functional (not checked for cosmetic)', () => { + vi.spyOn(DiffJsonSchema, 'isCosmeticOnlyJsonSchemaChange').mockReturnValue(true); + const result = getChangedActors({ + filepathsChanged: ['lib/config.json'], + actorConfigs: [miniActor], + commits, + }); + expect(result).toEqual([miniActor]); + }); + + it('README outside actor folder but inside context is cosmetic', () => { + const result = getChangedActors({ + filepathsChanged: ['docs/README.md'], + actorConfigs: [miniActor], + commits, + isLatest: true, + }); + expect(result).toEqual([miniActor]); + }); + + it('README outside actor folder but inside context is skipped when not isLatest', () => { + const result = getChangedActors({ + filepathsChanged: ['docs/README.md'], + actorConfigs: [miniActor], + commits, + isLatest: false, + }); + expect(result).toEqual([]); + }); }); diff --git a/test/unit/bin/dockerignore.test.ts b/test/unit/bin/dockerignore.test.ts new file mode 100644 index 0000000..bc8eac5 --- /dev/null +++ b/test/unit/bin/dockerignore.test.ts @@ -0,0 +1,83 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { loadDockerIgnore } from '../../../bin/dockerignore.js'; + +const { fsMock } = vi.hoisted(() => ({ + fsMock: { + readFileSync: vi.fn(), + }, +})); + +vi.mock('node:fs', () => ({ default: fsMock })); + +afterEach(() => vi.restoreAllMocks()); + +describe('loadDockerIgnore', () => { + it('returns no-op matcher when .dockerignore is absent', () => { + fsMock.readFileSync.mockImplementation(() => { + throw new Error('ENOENT'); + }); + const matcher = loadDockerIgnore('actors/my-actor'); + expect(matcher('actors/my-actor/src/main.ts')).toBe(false); + expect(matcher('actors/my-actor/node_modules/foo.js')).toBe(false); + }); + + it('matches files listed in .dockerignore', () => { + fsMock.readFileSync.mockReturnValue('node_modules\n*.log\n'); + const matcher = loadDockerIgnore(''); + expect(matcher('node_modules/foo/bar.js')).toBe(true); + expect(matcher('debug.log')).toBe(true); + }); + + it('does not match files not in .dockerignore', () => { + fsMock.readFileSync.mockReturnValue('node_modules\n'); + const matcher = loadDockerIgnore(''); + expect(matcher('src/main.ts')).toBe(false); + }); + + it('handles directory patterns', () => { + fsMock.readFileSync.mockReturnValue('dist/\n'); + const matcher = loadDockerIgnore(''); + expect(matcher('dist/bundle.js')).toBe(true); + expect(matcher('src/dist-utils.ts')).toBe(false); + }); + + it('handles negation patterns', () => { + fsMock.readFileSync.mockReturnValue('*.log\n!important.log\n'); + const matcher = loadDockerIgnore(''); + expect(matcher('debug.log')).toBe(true); + expect(matcher('important.log')).toBe(false); + }); + + it('strips dockerContextDir prefix before matching', () => { + fsMock.readFileSync.mockReturnValue('node_modules\n'); + const matcher = loadDockerIgnore('actors/shopify'); + expect(matcher('actors/shopify/node_modules/foo.js')).toBe(true); + expect(matcher('actors/shopify/src/main.ts')).toBe(false); + }); + + it('returns false for files outside dockerContextDir', () => { + fsMock.readFileSync.mockReturnValue('*\n'); + const matcher = loadDockerIgnore('actors/shopify'); + expect(matcher('other-actor/src/main.ts')).toBe(false); + }); + + it('reads .dockerignore from the dockerContextDir root', () => { + fsMock.readFileSync.mockReturnValue(''); + loadDockerIgnore('actors/shopify'); + expect(fsMock.readFileSync).toHaveBeenCalledWith('actors/shopify/.dockerignore', 'utf-8'); + }); + + it('reads .dockerignore from repo root when dockerContextDir is empty', () => { + fsMock.readFileSync.mockReturnValue(''); + loadDockerIgnore(''); + expect(fsMock.readFileSync).toHaveBeenCalledWith('.dockerignore', 'utf-8'); + }); + + it('handles comments and blank lines', () => { + fsMock.readFileSync.mockReturnValue('# this is a comment\n\nnode_modules\n'); + const matcher = loadDockerIgnore(''); + expect(matcher('node_modules/foo.js')).toBe(true); + expect(matcher('src/main.ts')).toBe(false); + }); +}); From 08476b2cc62216b99c7946248d4b72c94d4b4de1 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Fri, 26 Jun 2026 15:58:26 +0100 Subject: [PATCH 17/33] update README --- README.md | 57 +++++++++++++++++++++++++------------------------------ 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index df818c3..11fe041 100644 --- a/README.md +++ b/README.md @@ -15,35 +15,21 @@ npm i -D apify-test-tools ### 2. Create the config file -Every repo that uses `apify-test-tools` must have a `.test-tools-actors-config.json` file at the root. This file tells the tool which actors live in the repo, who owns them, and which token to use. - -You can generate a starter config automatically: - -```bash -npx apify-test-tools init-config -``` - -This scans the repo for `.actor/actor.json` files and creates `.test-tools-actors-config.json` with placeholder values. You can also pass defaults: - -```bash -npx apify-test-tools init-config --default-owner myteam --default-token-var APIFY_TOKEN_MYTEAM -``` - -The generated file looks like this: +Every repo that uses `apify-test-tools` must have a `.test-tools-actors-config.json` file at the root. This file tells the tool which actors live in the repo, how to identify them, and which token to use. ```json { "actors": [ { "folder": "actors/web-scraper", - "owner": "", - "tokenEnvVar": "" + "actorName": "myteam/web-scraper", + "tokenEnvVar": "APIFY_TOKEN_MYTEAM" }, { "folder": "actors/email-sender", - "owner": "", - "tokenEnvVar": "", - "isStandalone": true + "actorName": "myteam/email-sender", + "tokenEnvVar": "APIFY_TOKEN_MYTEAM", + "overrideActorContext": ["actors/email-sender", "packages/shared"] } ] } @@ -51,18 +37,16 @@ The generated file looks like this: Each entry has: -| Field | Required | Description | -|-------|----------|-------------| -| `folder` | yes | Relative path from repo root to the actor directory. Use `"."` for a single-actor repo where `.actor/` is at the root. | -| `owner` | yes | Apify username that owns the actor. Combined with the `name` from `/.actor/actor.json` to form the full actor name (`owner/name`). | -| `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. No fallback — if the env var is not set at build time, the build fails. | -| `isStandalone` | no | Defaults to `false`. Standalone actors are only built when their own folder changes, not when shared code changes. | - -The actor's `name` is always read from `/.actor/actor.json` — it is **not** duplicated in the config. +| Field | Required | Description | +| ---------------------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `folder` | yes | Relative path from repo root to the actor directory. Use `"."` for a single-actor repo where `.actor/` is at the root. | +| `actorName` | yes | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used. | +| `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. No fallback — if the env var is not set at build time, the build fails. | +| `overrideActorContext` | no | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. | ### 3. Set up actor folders -Each actor in the config must have a `.actor/actor.json` file with at least a `name` field: +Each actor in the config must have a `.actor/actor.json` file. The `dockerContextDir` field in `actor.json` defines the build context boundary — this is what the tool uses to determine which files can affect the actor's build. ``` my-repo @@ -70,11 +54,11 @@ my-repo ├── actors │ ├── web-scraper │ │ ├── .actor -│ │ │ └── actor.json <- { "name": "web-scraper" } +│ │ │ └── actor.json │ │ └── src/ │ └── email-sender │ ├── .actor -│ │ └── actor.json <- { "name": "email-sender" } +│ │ └── actor.json │ └── src/ └── test ├── unit @@ -87,6 +71,17 @@ my-repo For a single-actor repo, set `"folder": "."` in the config and place `.actor/actor.json` at the repo root. +### Change detection + +When a PR is opened or code is pushed, the tool determines which actors need to be built and tested based on the changed files. For each changed file, for each actor: + +1. **Hardcoded ignore list** — repo-level dev files (`.vscode/`, `.gitignore`, `.husky/`, `.eslintrc`, `eslint.config.mjs`, `.prettierrc`, `.editorconfig`, root `README.md`) are always ignored. +2. **Context matching** — the file must fall within the actor's build context (`dockerContextDir` from `actor.json`, or `overrideActorContext` from config if set). Files outside the context are skipped. +3. **`.dockerignore` filtering** — if a `.dockerignore` exists at the root of the actor's `dockerContextDir`, matching files are ignored. Patterns are resolved relative to `dockerContextDir`, matching Docker's own behavior. +4. **Sibling exclusion** — files inside another actor's `folder` are excluded. This prevents an actor with broad context from being triggered by changes that belong to a sibling actor. +5. **Cosmetic classification** — `README.md` and `CHANGELOG.md` files, and `.json` files inside the actor's folder with only cosmetic schema changes (whitespace, key ordering), only trigger a release build (not tests). +6. **Functional** — everything else triggers both build and tests. + ### 4. Create test directories ```bash From ae0d1dc1dc7f577c7cb7d3e35ed5f8fbf332562e Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Wed, 1 Jul 2026 13:41:37 +0100 Subject: [PATCH 18/33] rename variable --- bin/diff-changes.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index 4878955..b50a0be 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -55,7 +55,7 @@ const isFileInContext = (lowercaseFilePath: string, actor: ActorConfig): boolean */ const classifyFileChange = ( originalFilePath: string, - actor: ActorConfig, + actorConfig: ActorConfig, commits: Commit[], cosmeticCache: Map, dockerIgnoreMatcher: DockerIgnoreMatcher, @@ -66,7 +66,7 @@ const classifyFileChange = ( return { impact: 'ignored' }; } - if (!isFileInContext(lowercaseFilePath, actor)) { + if (!isFileInContext(lowercaseFilePath, actorConfig)) { return { impact: 'outside-context' }; } @@ -78,7 +78,7 @@ const classifyFileChange = ( return { impact: 'cosmetic', semanticallyVerified: false }; } - const lowerFolder = actor.folder.toLowerCase(); + const lowerFolder = actorConfig.folder.toLowerCase(); const isInActorFolder = lowerFolder === '' || lowercaseFilePath.startsWith(`${lowerFolder}/`); if (lowercaseFilePath.endsWith('.json') && isInActorFolder) { let isCosmetic = cosmeticCache.get(originalFilePath); @@ -141,23 +141,29 @@ export const getChangedActors = ({ const cosmeticCache = new Map(); const fileImpacts = new Map(); - for (const actor of actorConfigs) { - const dockerIgnoreMatcher = loadDockerIgnore(actor.dockerContextDir); + for (const actorConfig of actorConfigs) { + const dockerIgnoreMatcher = loadDockerIgnore(actorConfig.dockerContextDir); for (const originalFilePath of filepathsChanged) { const lowercaseFilePath = originalFilePath.toLowerCase(); - if (isExcludedBySibling(lowercaseFilePath, actor, actorConfigs)) { + if (isExcludedBySibling(lowercaseFilePath, actorConfig, actorConfigs)) { continue; } - const change = classifyFileChange(originalFilePath, actor, commits, cosmeticCache, dockerIgnoreMatcher); + const change = classifyFileChange( + originalFilePath, + actorConfig, + commits, + cosmeticCache, + dockerIgnoreMatcher, + ); updateFileImpact(fileImpacts, originalFilePath, change.impact); if (change.impact === 'ignored' || change.impact === 'outside-context') continue; if (change.impact === 'cosmetic' && !isLatest) continue; - actorsChangedMap.set(actor.folder, actor); + actorsChangedMap.set(actorConfig.folder, actorConfig); } } From 18cf03926a98a40e470eedd3557642d6c5c337eb Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Wed, 1 Jul 2026 14:14:33 +0100 Subject: [PATCH 19/33] move context-path resolution into readConfigFile --- bin/diff-changes.ts | 18 +++++++----------- bin/main.ts | 10 +++++----- bin/types.ts | 2 +- bin/utils.ts | 6 +----- test/unit/bin/diff-changes.test.ts | 18 ++++++++++++++---- test/unit/bin/utils.test.ts | 7 ++++--- test/unit/should-built-and-test.test.ts | 12 ++++++++++++ 7 files changed, 44 insertions(+), 29 deletions(-) diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index b50a0be..9b1ad40 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -31,15 +31,11 @@ type FileChangeForActor = | { impact: 'cosmetic'; semanticallyVerified: boolean } | { impact: 'functional' }; -const isFileInContext = (lowercaseFilePath: string, actor: ActorConfig): boolean => { - if (actor.overrideActorContext) { - return actor.overrideActorContext.some((contextPath) => { - const lowerContextPath = contextPath.toLowerCase(); - return lowerContextPath === '' || lowercaseFilePath.startsWith(`${lowerContextPath}/`); - }); - } - const lowerDockerContext = actor.dockerContextDir.toLowerCase(); - return lowerDockerContext === '' || lowercaseFilePath.startsWith(`${lowerDockerContext}/`); +const isFileInActorContext = (lowercaseFilePath: string, contextPaths: string[]): boolean => { + return contextPaths.some((contextPath) => { + const lowerContextPath = contextPath.toLowerCase(); + return lowerContextPath === '' || lowercaseFilePath.startsWith(`${lowerContextPath}/`); + }); }; /** @@ -47,7 +43,7 @@ const isFileInContext = (lowercaseFilePath: string, actor: ActorConfig): boolean * * Steps (in order): * 1. Hardcoded ignore list (repo-level dev files) → ignored - * 2. Context matching (dockerContextDir or overrideActorContext) → outside-context if no match + * 2. Context matching (actorConfig.contextPaths) → outside-context if no match * 3. .dockerignore filtering (patterns relative to dockerContextDir) → ignored if matched * 4. README/CHANGELOG by filename → cosmetic (not semantically verified) * 5. .json inside the actor's own folder with only cosmetic schema diffs → cosmetic (semantically verified) @@ -66,7 +62,7 @@ const classifyFileChange = ( return { impact: 'ignored' }; } - if (!isFileInContext(lowercaseFilePath, actorConfig)) { + if (!isFileInActorContext(lowercaseFilePath, actorConfig.contextPaths)) { return { impact: 'outside-context' }; } diff --git a/bin/main.ts b/bin/main.ts index 120ea1b..94778f6 100644 --- a/bin/main.ts +++ b/bin/main.ts @@ -13,7 +13,7 @@ import { getPushData } from './github.js'; import { notifyToSlack } from './slack.js'; import { reportTestResults } from './test-report.js'; import type { Config } from './types.js'; -import { getRepoActors, setCwd, spawnCommandInGhWorkspace } from './utils.js'; +import { readConfigFile, setCwd, spawnCommandInGhWorkspace } from './utils.js'; /** * Middlewares to be run before every command execution @@ -43,7 +43,7 @@ const resolveChangedActors = async ( { targetBranch, sourceBranch, baseCommit }: Config, { isLatest }: { isLatest: boolean }, ) => { - const actorConfigs = await getRepoActors(); + const actorConfigs = await readConfigFile(); // This is an optimization for the common case where a branch only has cosmetic changes but had to merge in // functional changes from master (being up-to-date is a CI requirement). Master is already validated, and @@ -107,7 +107,7 @@ await yargs() '', (_) => _, async () => { - const actorConfigs = await getRepoActors(); + const actorConfigs = await readConfigFile(); console.log(JSON.stringify(actorConfigs)); }, ) @@ -172,7 +172,7 @@ await yargs() args.pushEventPath, ); const isLatest = true; - const actorConfigs = await getRepoActors(); + const actorConfigs = await readConfigFile(); const actorsChanged = getChangedActors({ filepathsChanged: changedFiles, actorConfigs, @@ -207,7 +207,7 @@ await yargs() '', (_) => _, async () => { - const actorConfigs = await getRepoActors(); + const actorConfigs = await readConfigFile(); await deleteOldBuilds(actorConfigs); }, ) diff --git a/bin/types.ts b/bin/types.ts index 06d4bd5..94ff738 100644 --- a/bin/types.ts +++ b/bin/types.ts @@ -109,5 +109,5 @@ export interface ActorConfig { folder: string; tokenEnvVar: string; dockerContextDir: string; - overrideActorContext?: string[]; + contextPaths: string[]; } diff --git a/bin/utils.ts b/bin/utils.ts index 15089af..93a6abc 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -30,10 +30,6 @@ export const getEnvVar = (varName: string, defaultValue?: string): string => { return value; }; -export const getRepoActors = async (): Promise => { - return readConfigFile(); -}; - const CONFIG_FILE_NAME = '.test-tools-actors-config.json'; export const readConfigFile = async (): Promise => { @@ -122,7 +118,7 @@ export const readConfigFile = async (): Promise => { folder, tokenEnvVar: entry.tokenEnvVar, dockerContextDir: normalizedDockerContextDir, - overrideActorContext: entry.overrideActorContext, + contextPaths: entry.overrideActorContext ?? [normalizedDockerContextDir], }); } diff --git a/test/unit/bin/diff-changes.test.ts b/test/unit/bin/diff-changes.test.ts index 06dcdb8..ff5fc78 100644 --- a/test/unit/bin/diff-changes.test.ts +++ b/test/unit/bin/diff-changes.test.ts @@ -10,12 +10,14 @@ const miniActor: ActorConfig = { folder: 'actors/foo_bar', tokenEnvVar: 'APIFY_TOKEN_FOO', dockerContextDir: '', + contextPaths: [''], }; const standaloneActor: ActorConfig = { actorName: 'owner/standalone', folder: 'standalone-actors/standalone', tokenEnvVar: 'APIFY_TOKEN_OWNER', dockerContextDir: 'standalone-actors/standalone', + contextPaths: ['standalone-actors/standalone'], }; const actorConfigs = [miniActor, standaloneActor]; @@ -156,6 +158,7 @@ describe('getChangedActors', () => { folder: 'actors/shopify', tokenEnvVar: 'APIFY_TOKEN_MYTEAM', dockerContextDir: '', + contextPaths: [''], }; const result = getChangedActors({ filepathsChanged: ['actors/shopify/src/main.ts'], @@ -171,6 +174,7 @@ describe('getChangedActors', () => { folder: '', tokenEnvVar: 'BUILDER_APIFY_TOKEN', dockerContextDir: '', + contextPaths: [''], }; const result = getChangedActors({ filepathsChanged: ['.actor/actor.json'], @@ -198,13 +202,13 @@ describe('getChangedActors', () => { expect(result).toEqual([miniActor]); }); - it('triggers actor with overrideActorContext when file matches an override path', () => { + it('triggers actor with contextPaths override when file matches an override path', () => { const overrideActor: ActorConfig = { actorName: 'team/override-actor', folder: 'actors/override', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: 'actors/override', - overrideActorContext: ['actors/override', 'packages'], + contextPaths: ['actors/override', 'packages'], }; const result = getChangedActors({ filepathsChanged: ['packages/shared/utils.ts'], @@ -214,13 +218,13 @@ describe('getChangedActors', () => { expect(result).toEqual([overrideActor]); }); - it('does not trigger actor with overrideActorContext when file is outside all override paths', () => { + it('does not trigger actor with contextPaths override when file is outside all override paths', () => { const overrideActor: ActorConfig = { actorName: 'team/override-actor', folder: 'actors/override', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: 'actors/override', - overrideActorContext: ['actors/override', 'packages'], + contextPaths: ['actors/override', 'packages'], }; const result = getChangedActors({ filepathsChanged: ['other-dir/file.ts'], @@ -236,12 +240,14 @@ describe('getChangedActors', () => { folder: 'actors/a', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: '', + contextPaths: [''], }; const actorB: ActorConfig = { actorName: 'team/actor-b', folder: 'actors/b', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: '', + contextPaths: [''], }; const result = getChangedActors({ filepathsChanged: ['actors/b/src/main.ts'], @@ -257,12 +263,14 @@ describe('getChangedActors', () => { folder: '', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: '', + contextPaths: [''], }; const childActor: ActorConfig = { actorName: 'team/child', folder: 'actors/child', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: 'actors/child', + contextPaths: ['actors/child'], }; const result = getChangedActors({ filepathsChanged: ['actors/child/src/main.ts'], @@ -279,12 +287,14 @@ describe('getChangedActors', () => { folder: '', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: '', + contextPaths: [''], }; const childActor: ActorConfig = { actorName: 'team/child', folder: 'actors/child', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: 'actors/child', + contextPaths: ['actors/child'], }; const result = getChangedActors({ filepathsChanged: ['lib/shared-utils.ts'], diff --git a/test/unit/bin/utils.test.ts b/test/unit/bin/utils.test.ts index 8da1d35..9282e2b 100644 --- a/test/unit/bin/utils.test.ts +++ b/test/unit/bin/utils.test.ts @@ -43,7 +43,7 @@ describe('readConfigFile', () => { folder: 'actors/shopify', tokenEnvVar: 'APIFY_TOKEN_MYTEAM', dockerContextDir: '', - overrideActorContext: undefined, + contextPaths: [''], }, ]); }); @@ -71,6 +71,7 @@ describe('readConfigFile', () => { const result = await readConfigFile(); expect(result[0].dockerContextDir).toBe('actors/web-scraper'); + expect(result[0].contextPaths).toEqual(['actors/web-scraper']); }); it('resolves dockerContextDir relative to .actor/ folder', async () => { @@ -85,7 +86,7 @@ describe('readConfigFile', () => { expect(result[0].dockerContextDir).toBe(''); }); - it('passes through overrideActorContext', async () => { + it('resolves contextPaths from overrideActorContext', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ { @@ -99,7 +100,7 @@ describe('readConfigFile', () => { }); const result = await readConfigFile(); - expect(result[0].overrideActorContext).toEqual(['actors/shopify', 'packages']); + expect(result[0].contextPaths).toEqual(['actors/shopify', 'packages']); }); it('handles multiple actors', async () => { diff --git a/test/unit/should-built-and-test.test.ts b/test/unit/should-built-and-test.test.ts index 32b9bea..5492087 100644 --- a/test/unit/should-built-and-test.test.ts +++ b/test/unit/should-built-and-test.test.ts @@ -13,18 +13,21 @@ describe('Should build and test parser', () => { folder: 'actors/lukaskrivka_testing-github-integration-1', tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: '', + contextPaths: [''], }, { actorName: 'lukaskrivka/testing-github-integration-2', folder: 'actors/lukaskrivka_testing-github-integration-2', tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: '', + contextPaths: [''], }, { actorName: 'lukaskrivka/test-standalone', folder: 'standalone-actors/lukaskrivka_test-standalone', tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: 'standalone-actors/lukaskrivka_test-standalone', + contextPaths: ['standalone-actors/lukaskrivka_test-standalone'], }, ]; @@ -269,54 +272,63 @@ describe('Should build and test parser', () => { folder: 'actors/compass_Google-Maps-Reviews-Scraper', tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', + contextPaths: [''], }, { actorName: 'compass/crawler-google-places', folder: 'actors/compass_crawler-google-places', tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', + contextPaths: [''], }, { actorName: 'compass/easy-google-maps', folder: 'actors/compass_easy-google-maps', tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', + contextPaths: [''], }, { actorName: 'compass/google-maps-extractor', folder: 'actors/compass_google-maps-extractor', tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', + contextPaths: [''], }, { actorName: 'compass/google-places-api', folder: 'actors/compass_google-places-api', tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', + contextPaths: [''], }, { actorName: 'lukaskrivka/google-maps-with-contact-details', folder: 'actors/lukaskrivka_google-maps-with-contact-details', tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: '', + contextPaths: [''], }, { actorName: 'natasha.lekh/gas-prices-scraper', folder: 'actors/natasha.lekh_gas-prices-scraper', tokenEnvVar: 'APIFY_TOKEN_NATASHA_LEKH', dockerContextDir: '', + contextPaths: [''], }, { actorName: 'natasha.lekh/vegan-places-finder', folder: 'actors/natasha.lekh_vegan-places-finder', tokenEnvVar: 'APIFY_TOKEN_NATASHA_LEKH', dockerContextDir: '', + contextPaths: [''], }, { actorName: 'lukaskrivka/google-maps-scraper-orchestrator', folder: 'standalone-actors/lukaskrivka_google-maps-scraper-orchestrator', tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: 'standalone-actors/lukaskrivka_google-maps-scraper-orchestrator', + contextPaths: ['standalone-actors/lukaskrivka_google-maps-scraper-orchestrator'], }, ]; From faf4784da5a9446e1920e2b5c566f9c9db21b35b Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Thu, 2 Jul 2026 15:36:07 +0100 Subject: [PATCH 20/33] context aware IGNORED_TOP_LEVEL_FILES and updated readme-changelog logic --- README.md | 25 +++++----- bin/diff-changes.ts | 61 ++++++++++++----------- bin/utils.ts | 37 +++++++++++++- test/unit/bin/diff-changes.test.ts | 66 ++++++++++++++++++++++--- test/unit/bin/utils.test.ts | 65 ++++++++++++++++++++++++ test/unit/should-built-and-test.test.ts | 17 ++++++- 6 files changed, 221 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 11fe041..702809a 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,12 @@ Every repo that uses `apify-test-tools` must have a `.test-tools-actors-config.j Each entry has: -| Field | Required | Description | -| ---------------------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `folder` | yes | Relative path from repo root to the actor directory. Use `"."` for a single-actor repo where `.actor/` is at the root. | -| `actorName` | yes | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used. | -| `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. No fallback — if the env var is not set at build time, the build fails. | -| `overrideActorContext` | no | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. | +| Field | Required | Description | +| ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `folder` | yes | Relative path from repo root to the actor directory. Use `"."` for a single-actor repo where `.actor/` is at the root. | +| `actorName` | yes | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used. | +| `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. No fallback — if the env var is not set at build time, the build fails. | +| `overrideActorContext` | no | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. Entries must not be prefixes of one another (e.g. `["", "code"]` or `["actors", "actors/foo"]` are rejected), and the list must include a path that reaches the actor's own `folder` — otherwise the actor could never be detected as changed. | ### 3. Set up actor folders @@ -75,12 +75,13 @@ For a single-actor repo, set `"folder": "."` in the config and place `.actor/act When a PR is opened or code is pushed, the tool determines which actors need to be built and tested based on the changed files. For each changed file, for each actor: -1. **Hardcoded ignore list** — repo-level dev files (`.vscode/`, `.gitignore`, `.husky/`, `.eslintrc`, `eslint.config.mjs`, `.prettierrc`, `.editorconfig`, root `README.md`) are always ignored. -2. **Context matching** — the file must fall within the actor's build context (`dockerContextDir` from `actor.json`, or `overrideActorContext` from config if set). Files outside the context are skipped. -3. **`.dockerignore` filtering** — if a `.dockerignore` exists at the root of the actor's `dockerContextDir`, matching files are ignored. Patterns are resolved relative to `dockerContextDir`, matching Docker's own behavior. -4. **Sibling exclusion** — files inside another actor's `folder` are excluded. This prevents an actor with broad context from being triggered by changes that belong to a sibling actor. -5. **Cosmetic classification** — `README.md` and `CHANGELOG.md` files, and `.json` files inside the actor's folder with only cosmetic schema changes (whitespace, key ordering), only trigger a release build (not tests). -6. **Functional** — everything else triggers both build and tests. +1. **Sibling exclusion** — files inside another actor's `folder` are excluded first. This prevents an actor with broad context from being triggered by changes that belong to a sibling actor. +2. **Context matching** — the file must fall within one of the actor's context paths (`dockerContextDir` from `actor.json` by default, or `overrideActorContext` from config if set). Files outside every context path are skipped. +3. **Hardcoded ignore list, context-aware** — the file path is first "hoisted" relative to the context path it matched (e.g. a standalone actor's own `.eslintrc` is checked as just `.eslintrc`, not the full repo-root-relative path), then checked against repo-level dev file patterns (`.vscode/`, `.gitignore`, `.husky/`, `.eslintrc`, `eslint.config.mjs`, `.prettierrc`, `.editorconfig`). There's no hardcoded special-casing for legacy `code/`/`shared/` layouts — repos that need those directories treated as top-level must list them explicitly in `overrideActorContext`. +4. **`.dockerignore` filtering** — if a `.dockerignore` exists at the root of the actor's `dockerContextDir`, matching files are ignored. Patterns are resolved relative to `dockerContextDir`, matching Docker's own behavior. +5. **README/CHANGELOG classification** — a `README.md` or `CHANGELOG.md` file is `cosmetic` (only triggers a release build, not tests) if it lives inside the actor's own `folder`; otherwise it's ignored entirely, since it isn't documentation for this actor. +6. **Cosmetic JSON classification** — `.json` files inside the actor's own `folder` with only cosmetic schema changes (whitespace, key ordering) only trigger a release build. +7. **Functional** — everything else triggers both build and tests. ### 4. Create test directories diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index 9b1ad40..1803cf7 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -9,21 +9,19 @@ interface ShouldBuildAndTestOptions { commits: Commit[]; } -const isIgnoredTopLevelFile = (lowercaseFilePath: string) => { - const IGNORED_TOP_LEVEL_FILES = [ - '.vscode/', - '.gitignore', - 'readme.md', - '.husky/', - '.eslintrc', - 'eslint.config.mjs', - '.prettierrc', - '.editorconfig', - ]; - // Strip deprecated code/ and shared/ prefixes — repos like apify-store/amazon use these - const sanitized = lowercaseFilePath.replace(/^code\//, '').replace(/^shared\//, ''); - return IGNORED_TOP_LEVEL_FILES.some((pattern) => sanitized.startsWith(pattern)); -}; +const IGNORED_TOP_LEVEL_FILES = [ + '.vscode/', + '.gitignore', + '.husky/', + '.eslintrc', + 'eslint.config.mjs', + '.prettierrc', + '.editorconfig', +]; + +// Expects an already-hoisted path (relative to the matched context entry, see findMatchingContextPath). +const isIgnoredTopLevelFile = (hoistedLowercaseFilePath: string): boolean => + IGNORED_TOP_LEVEL_FILES.some((pattern) => hoistedLowercaseFilePath.startsWith(pattern)); type FileChangeForActor = | { impact: 'ignored' } @@ -31,21 +29,24 @@ type FileChangeForActor = | { impact: 'cosmetic'; semanticallyVerified: boolean } | { impact: 'functional' }; -const isFileInActorContext = (lowercaseFilePath: string, contextPaths: string[]): boolean => { - return contextPaths.some((contextPath) => { +/** + * Finds the single contextPaths entry a file falls under. `readConfigFile` validates that no entry is a + * path-prefix of another entry in the same actor's contextPaths, so at most one entry can ever match. + */ +const findMatchingContextPath = (lowercaseFilePath: string, contextPaths: string[]): string | undefined => + contextPaths.find((contextPath) => { const lowerContextPath = contextPath.toLowerCase(); return lowerContextPath === '' || lowercaseFilePath.startsWith(`${lowerContextPath}/`); }); -}; /** * Classify a single file change for a single actor. * * Steps (in order): - * 1. Hardcoded ignore list (repo-level dev files) → ignored - * 2. Context matching (actorConfig.contextPaths) → outside-context if no match + * 1. Context matching (actorConfig.contextPaths) → outside-context if no match + * 2. Hardcoded ignore list, checked against the path hoisted relative to the matched context entry → ignored * 3. .dockerignore filtering (patterns relative to dockerContextDir) → ignored if matched - * 4. README/CHANGELOG by filename → cosmetic (not semantically verified) + * 4. README/CHANGELOG by filename → cosmetic if inside the actor's own folder, otherwise ignored * 5. .json inside the actor's own folder with only cosmetic schema diffs → cosmetic (semantically verified) * 6. Everything else → functional */ @@ -58,24 +59,28 @@ const classifyFileChange = ( ): FileChangeForActor => { const lowercaseFilePath = originalFilePath.toLowerCase(); - if (isIgnoredTopLevelFile(lowercaseFilePath)) { - return { impact: 'ignored' }; + const matchedContext = findMatchingContextPath(lowercaseFilePath, actorConfig.contextPaths); + if (matchedContext === undefined) { + return { impact: 'outside-context' }; } - if (!isFileInActorContext(lowercaseFilePath, actorConfig.contextPaths)) { - return { impact: 'outside-context' }; + const hoistedFilePath = + matchedContext === '' ? lowercaseFilePath : lowercaseFilePath.slice(matchedContext.length + 1); + if (isIgnoredTopLevelFile(hoistedFilePath)) { + return { impact: 'ignored' }; } if (dockerIgnoreMatcher(originalFilePath)) { return { impact: 'ignored' }; } + const lowerFolder = actorConfig.folder.toLowerCase(); + const isInActorFolder = lowerFolder === '' || lowercaseFilePath.startsWith(`${lowerFolder}/`); + if (lowercaseFilePath.endsWith('readme.md') || lowercaseFilePath.endsWith('changelog.md')) { - return { impact: 'cosmetic', semanticallyVerified: false }; + return isInActorFolder ? { impact: 'cosmetic', semanticallyVerified: false } : { impact: 'ignored' }; } - const lowerFolder = actorConfig.folder.toLowerCase(); - const isInActorFolder = lowerFolder === '' || lowercaseFilePath.startsWith(`${lowerFolder}/`); if (lowercaseFilePath.endsWith('.json') && isInActorFolder) { let isCosmetic = cosmeticCache.get(originalFilePath); if (isCosmetic === undefined) { diff --git a/bin/utils.ts b/bin/utils.ts index 93a6abc..7193e67 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -32,6 +32,24 @@ export const getEnvVar = (varName: string, defaultValue?: string): string => { const CONFIG_FILE_NAME = '.test-tools-actors-config.json'; +// `prefix` and `target` are both repo-root-relative paths ("" means repo root, which is a prefix of everything). +const isPathPrefixOrEqual = (prefix: string, target: string): boolean => + prefix === '' || target === prefix || target.startsWith(`${prefix}/`); + +const findOverlappingContextPaths = (contextPaths: string[]): [string, string] | undefined => { + for (let i = 0; i < contextPaths.length; i++) { + for (let j = i + 1; j < contextPaths.length; j++) { + if ( + isPathPrefixOrEqual(contextPaths[i], contextPaths[j]) || + isPathPrefixOrEqual(contextPaths[j], contextPaths[i]) + ) { + return [contextPaths[i], contextPaths[j]]; + } + } + } + return undefined; +}; + export const readConfigFile = async (): Promise => { let raw: string; try { @@ -112,13 +130,30 @@ export const readConfigFile = async (): Promise => { } const normalizedDockerContextDir = dockerContextDir === '.' ? '' : dockerContextDir; + const contextPaths = entry.overrideActorContext ?? [normalizedDockerContextDir]; + + const overlap = findOverlappingContextPaths(contextPaths); + if (overlap) { + throw new Error( + `Invalid context paths for folder "${entry.folder}" in "${CONFIG_FILE_NAME}": ` + + `"${overlap[0]}" and "${overlap[1]}" overlap. Context paths must not be prefixes of one another.`, + ); + } + + if (!contextPaths.some((contextPath) => isPathPrefixOrEqual(contextPath, folder))) { + throw new Error( + `Actor folder "${entry.folder}" in "${CONFIG_FILE_NAME}" is not reachable through its own ` + + `context paths (${contextPaths.join(', ')}). Add the actor's own folder to ` + + `"overrideActorContext" or remove the override.`, + ); + } actorConfigs.push({ actorName: entry.actorName, folder, tokenEnvVar: entry.tokenEnvVar, dockerContextDir: normalizedDockerContextDir, - contextPaths: entry.overrideActorContext ?? [normalizedDockerContextDir], + contextPaths, }); } diff --git a/test/unit/bin/diff-changes.test.ts b/test/unit/bin/diff-changes.test.ts index ff5fc78..80700d0 100644 --- a/test/unit/bin/diff-changes.test.ts +++ b/test/unit/bin/diff-changes.test.ts @@ -20,6 +20,13 @@ const standaloneActor: ActorConfig = { contextPaths: ['standalone-actors/standalone'], }; const actorConfigs = [miniActor, standaloneActor]; +const amazonActor: ActorConfig = { + actorName: 'junglee/amazon-crawler', + folder: 'actors/junglee_Amazon-crawler', + tokenEnvVar: 'APIFY_TOKEN_JUNGLEE', + dockerContextDir: '', + contextPaths: ['actors/junglee_Amazon-crawler', 'code', 'shared'], +}; const commits = [{ sha: 'Commit1', author: '', date: '', message: '' }]; @@ -112,15 +119,14 @@ describe('getChangedActors', () => { expect(result).not.toContainEqual(standaloneActor); }); - it('does not trigger narrow-context actor from root changelog', () => { + it('root-level changelog outside any actor folder is ignored, not cosmetic', () => { const result = getChangedActors({ filepathsChanged: ['CHANGELOG.md'], actorConfigs, commits, isLatest: true, }); - expect(result).toContainEqual(miniActor); - expect(result).not.toContainEqual(standaloneActor); + expect(result).toEqual([]); }); it('triggers narrow-context actor when its own folder changes', () => { @@ -337,22 +343,68 @@ describe('getChangedActors', () => { expect(result).toEqual([miniActor]); }); - it('README outside actor folder but inside context is cosmetic', () => { + it('README outside actor folder but inside context is ignored (not cosmetic)', () => { const result = getChangedActors({ filepathsChanged: ['docs/README.md'], actorConfigs: [miniActor], commits, isLatest: true, }); + expect(result).toEqual([]); + }); + + it('README inside actor folder is cosmetic', () => { + const result = getChangedActors({ + filepathsChanged: ['actors/foo_bar/README.md'], + actorConfigs: [miniActor], + commits, + isLatest: true, + }); expect(result).toEqual([miniActor]); }); - it('README outside actor folder but inside context is skipped when not isLatest', () => { + it('hoists a standalone actor own top-level dev file relative to its context before checking the ignore list', () => { const result = getChangedActors({ - filepathsChanged: ['docs/README.md'], + filepathsChanged: ['standalone-actors/standalone/.eslintrc'], + actorConfigs, + commits, + }); + expect(result).toEqual([]); + }); + + it('does not special-case code/ and shared/ prefixes anymore — must be declared via overrideActorContext', () => { + const result = getChangedActors({ + filepathsChanged: ['code/.eslintrc'], actorConfigs: [miniActor], commits, - isLatest: false, + }); + expect(result).toEqual([miniActor]); + }); + + it('ignores code/.eslintrc when "code" is declared via overrideActorContext', () => { + const result = getChangedActors({ + filepathsChanged: ['code/.eslintrc'], + actorConfigs: [amazonActor], + commits, + }); + expect(result).toEqual([]); + }); + + it('shared/Dockerfile declared via overrideActorContext is functional', () => { + const result = getChangedActors({ + filepathsChanged: ['shared/Dockerfile'], + actorConfigs: [amazonActor], + commits, + }); + expect(result).toEqual([amazonActor]); + }); + + it('code/README.md declared via overrideActorContext is ignored (outside actor folder)', () => { + const result = getChangedActors({ + filepathsChanged: ['code/README.md'], + actorConfigs: [amazonActor], + commits, + isLatest: true, }); expect(result).toEqual([]); }); diff --git a/test/unit/bin/utils.test.ts b/test/unit/bin/utils.test.ts index 9282e2b..adf65ee 100644 --- a/test/unit/bin/utils.test.ts +++ b/test/unit/bin/utils.test.ts @@ -236,4 +236,69 @@ describe('readConfigFile', () => { await expect(readConfigFile()).rejects.toThrow('Invalid "overrideActorContext"'); }); + + it('throws when overrideActorContext entries overlap (one is a prefix of another)', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { + folder: 'actors/shopify', + actorName: 'myteam/shopify', + tokenEnvVar: 'APIFY_TOKEN', + overrideActorContext: ['actors/shopify', 'actors'], + }, + ]), + 'actors/shopify/.actor/actor.json': actorJson({}), + }); + + await expect(readConfigFile()).rejects.toThrow(/overlap/); + }); + + it('throws when overrideActorContext contains the repo root alongside another entry', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { + folder: 'actors/shopify', + actorName: 'myteam/shopify', + tokenEnvVar: 'APIFY_TOKEN', + overrideActorContext: ['', 'actors/shopify'], + }, + ]), + 'actors/shopify/.actor/actor.json': actorJson({}), + }); + + await expect(readConfigFile()).rejects.toThrow(/overlap/); + }); + + it('throws when overrideActorContext does not include the actor own folder', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { + folder: 'actors/shopify', + actorName: 'myteam/shopify', + tokenEnvVar: 'APIFY_TOKEN', + overrideActorContext: ['code', 'shared'], + }, + ]), + 'actors/shopify/.actor/actor.json': actorJson({}), + }); + + await expect(readConfigFile()).rejects.toThrow('not reachable through its own context paths'); + }); + + it('allows overrideActorContext with disjoint sibling paths that all reach the actor folder via one entry', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { + folder: 'actors/shopify', + actorName: 'myteam/shopify', + tokenEnvVar: 'APIFY_TOKEN', + overrideActorContext: ['actors/shopify', 'code', 'shared'], + }, + ]), + 'actors/shopify/.actor/actor.json': actorJson({}), + }); + + const result = await readConfigFile(); + expect(result[0].contextPaths).toEqual(['actors/shopify', 'code', 'shared']); + }); }); diff --git a/test/unit/should-built-and-test.test.ts b/test/unit/should-built-and-test.test.ts index 5492087..c8935f0 100644 --- a/test/unit/should-built-and-test.test.ts +++ b/test/unit/should-built-and-test.test.ts @@ -82,7 +82,7 @@ describe('Should build and test parser', () => { expect(actorsChanged).toEqual(ACTOR_CONFIGS.slice(0, 2)); }); - test('Only builds latest for all Actors', () => { + test('Root-level changelog outside any actor own folder is ignored, even on latest', () => { const FILES = ['shared/CHANGELOG.md', 'CHANGELOG.md']; const actorsChanged = getChangedActors({ @@ -92,7 +92,20 @@ describe('Should build and test parser', () => { commits, }); - expect(actorsChanged).toEqual(ACTOR_CONFIGS.slice(0, 2)); + expect(actorsChanged).toEqual([]); + }); + + test('Only builds latest for actor own changelog', () => { + const FILES = ['actors/lukaskrivka_testing-github-integration-1/CHANGELOG.md']; + + const actorsChanged = getChangedActors({ + actorConfigs: ACTOR_CONFIGS, + isLatest: true, + filepathsChanged: FILES, + commits, + }); + + expect(actorsChanged).toEqual([ACTOR_CONFIGS[0]]); }); test('Code updated, tests broad-context actors', () => { From d29eb167abe06caa178114f9ba071c8bdc6a3dcc Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Thu, 2 Jul 2026 16:36:51 +0100 Subject: [PATCH 21/33] define shared path-utils --- bin/diff-changes.ts | 23 +++++++---------------- bin/dockerignore.ts | 12 ++++-------- bin/path-utils.ts | 13 +++++++++++++ bin/utils.ts | 16 ++++++++-------- test/unit/bin/path-utils.test.ts | 17 +++++++++++++++++ test/unit/bin/utils.test.ts | 18 ++++++++++++++++++ 6 files changed, 67 insertions(+), 32 deletions(-) create mode 100644 bin/path-utils.ts create mode 100644 test/unit/bin/path-utils.test.ts diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index 1803cf7..362451a 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -1,5 +1,6 @@ import { isCosmeticOnlyJsonSchemaChange } from './diff-json-schema.js'; import { type DockerIgnoreMatcher, loadDockerIgnore } from './dockerignore.js'; +import { findContainingScope, hoistPath, isPathWithinScope } from './path-utils.js'; import type { ActorConfig, Commit } from './types.js'; interface ShouldBuildAndTestOptions { @@ -19,7 +20,7 @@ const IGNORED_TOP_LEVEL_FILES = [ '.editorconfig', ]; -// Expects an already-hoisted path (relative to the matched context entry, see findMatchingContextPath). +// Expects an already-hoisted path (relative to the matched context entry, see findContainingScope). const isIgnoredTopLevelFile = (hoistedLowercaseFilePath: string): boolean => IGNORED_TOP_LEVEL_FILES.some((pattern) => hoistedLowercaseFilePath.startsWith(pattern)); @@ -29,16 +30,6 @@ type FileChangeForActor = | { impact: 'cosmetic'; semanticallyVerified: boolean } | { impact: 'functional' }; -/** - * Finds the single contextPaths entry a file falls under. `readConfigFile` validates that no entry is a - * path-prefix of another entry in the same actor's contextPaths, so at most one entry can ever match. - */ -const findMatchingContextPath = (lowercaseFilePath: string, contextPaths: string[]): string | undefined => - contextPaths.find((contextPath) => { - const lowerContextPath = contextPath.toLowerCase(); - return lowerContextPath === '' || lowercaseFilePath.startsWith(`${lowerContextPath}/`); - }); - /** * Classify a single file change for a single actor. * @@ -58,14 +49,14 @@ const classifyFileChange = ( dockerIgnoreMatcher: DockerIgnoreMatcher, ): FileChangeForActor => { const lowercaseFilePath = originalFilePath.toLowerCase(); + const lowercaseContextPaths = actorConfig.contextPaths.map((contextPath) => contextPath.toLowerCase()); - const matchedContext = findMatchingContextPath(lowercaseFilePath, actorConfig.contextPaths); + const matchedContext = findContainingScope(lowercaseFilePath, lowercaseContextPaths); if (matchedContext === undefined) { return { impact: 'outside-context' }; } - const hoistedFilePath = - matchedContext === '' ? lowercaseFilePath : lowercaseFilePath.slice(matchedContext.length + 1); + const hoistedFilePath = hoistPath(lowercaseFilePath, matchedContext); if (isIgnoredTopLevelFile(hoistedFilePath)) { return { impact: 'ignored' }; } @@ -75,7 +66,7 @@ const classifyFileChange = ( } const lowerFolder = actorConfig.folder.toLowerCase(); - const isInActorFolder = lowerFolder === '' || lowercaseFilePath.startsWith(`${lowerFolder}/`); + const isInActorFolder = isPathWithinScope(lowercaseFilePath, lowerFolder); if (lowercaseFilePath.endsWith('readme.md') || lowercaseFilePath.endsWith('changelog.md')) { return isInActorFolder ? { impact: 'cosmetic', semanticallyVerified: false } : { impact: 'ignored' }; @@ -104,7 +95,7 @@ const isExcludedBySibling = (lowercaseFilePath: string, actor: ActorConfig, allA (other) => other.folder !== actor.folder && other.folder !== '' && - lowercaseFilePath.startsWith(`${other.folder.toLowerCase()}/`), + isPathWithinScope(lowercaseFilePath, other.folder.toLowerCase()), ); }; diff --git a/bin/dockerignore.ts b/bin/dockerignore.ts index 09281e9..c516571 100644 --- a/bin/dockerignore.ts +++ b/bin/dockerignore.ts @@ -3,6 +3,8 @@ import path from 'node:path'; import ignore from 'ignore'; +import { hoistPath, isPathWithinScope } from './path-utils.js'; + export type DockerIgnoreMatcher = (repoRelativePath: string) => boolean; /** @@ -28,16 +30,10 @@ export const loadDockerIgnore = (dockerContextDir: string): DockerIgnoreMatcher const lowerPath = repoRelativePath.toLowerCase(); const lowerContext = dockerContextDir.toLowerCase(); - // Strip the dockerContextDir prefix to get a path relative to the context root - let relativePath: string; - if (lowerContext === '') { - relativePath = repoRelativePath; - } else if (lowerPath.startsWith(`${lowerContext}/`)) { - relativePath = repoRelativePath.slice(dockerContextDir.length + 1); - } else { + if (!isPathWithinScope(lowerPath, lowerContext)) { return false; } - return matcher.ignores(relativePath); + return matcher.ignores(hoistPath(repoRelativePath, dockerContextDir)); }; }; diff --git a/bin/path-utils.ts b/bin/path-utils.ts new file mode 100644 index 0000000..a0bf961 --- /dev/null +++ b/bin/path-utils.ts @@ -0,0 +1,13 @@ +// filePath and scopePath are both repo-root-relative POSIX paths, already normalized (no trailing slashes). +// scopePath === '' means "matches everything" — the caller's own scope (a context path, an actor's folder, +// a sibling's folder, a docker context dir), not necessarily the repo root. +export const isPathWithinScope = (filePath: string, scopePath: string): boolean => + scopePath === '' || filePath === scopePath || filePath.startsWith(`${scopePath}/`); + +// Returns the single scopePaths entry filePath falls under, if any. +export const findContainingScope = (filePath: string, scopePaths: string[]): string | undefined => + scopePaths.find((scopePath) => isPathWithinScope(filePath, scopePath)); + +// Returns filePath relative to scopePath (assumes isPathWithinScope(filePath, scopePath) is already true). +export const hoistPath = (filePath: string, scopePath: string): string => + scopePath === '' ? filePath : filePath.slice(scopePath.length + 1); diff --git a/bin/utils.ts b/bin/utils.ts index 7193e67..d5a828a 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -2,6 +2,7 @@ import { spawnSync } from 'node:child_process'; import fs from 'node:fs/promises'; import path from 'node:path'; +import { isPathWithinScope } from './path-utils.js'; import type { ActorConfig, ActorConfigFile } from './types.js'; export const spawnCommandInGhWorkspace = (command: string, args: string[] = []) => { @@ -32,16 +33,15 @@ export const getEnvVar = (varName: string, defaultValue?: string): string => { const CONFIG_FILE_NAME = '.test-tools-actors-config.json'; -// `prefix` and `target` are both repo-root-relative paths ("" means repo root, which is a prefix of everything). -const isPathPrefixOrEqual = (prefix: string, target: string): boolean => - prefix === '' || target === prefix || target.startsWith(`${prefix}/`); +// Strips a trailing slash so config-declared paths ("actors/shopify/" vs "actors/shopify") compare equal. +const stripTrailingSlash = (pathValue: string): string => pathValue.replace(/\/+$/, ''); const findOverlappingContextPaths = (contextPaths: string[]): [string, string] | undefined => { for (let i = 0; i < contextPaths.length; i++) { for (let j = i + 1; j < contextPaths.length; j++) { if ( - isPathPrefixOrEqual(contextPaths[i], contextPaths[j]) || - isPathPrefixOrEqual(contextPaths[j], contextPaths[i]) + isPathWithinScope(contextPaths[i], contextPaths[j]) || + isPathWithinScope(contextPaths[j], contextPaths[i]) ) { return [contextPaths[i], contextPaths[j]]; } @@ -76,7 +76,7 @@ export const readConfigFile = async (): Promise => { const actorConfigs: ActorConfig[] = []; for (const entry of config.actors) { - const folder = entry.folder === '.' ? '' : entry.folder; + const folder = entry.folder === '.' ? '' : stripTrailingSlash(entry.folder); if (seenFolders.has(folder)) { throw new Error( @@ -130,7 +130,7 @@ export const readConfigFile = async (): Promise => { } const normalizedDockerContextDir = dockerContextDir === '.' ? '' : dockerContextDir; - const contextPaths = entry.overrideActorContext ?? [normalizedDockerContextDir]; + const contextPaths = (entry.overrideActorContext ?? [normalizedDockerContextDir]).map(stripTrailingSlash); const overlap = findOverlappingContextPaths(contextPaths); if (overlap) { @@ -140,7 +140,7 @@ export const readConfigFile = async (): Promise => { ); } - if (!contextPaths.some((contextPath) => isPathPrefixOrEqual(contextPath, folder))) { + if (!contextPaths.some((contextPath) => isPathWithinScope(folder, contextPath))) { throw new Error( `Actor folder "${entry.folder}" in "${CONFIG_FILE_NAME}" is not reachable through its own ` + `context paths (${contextPaths.join(', ')}). Add the actor's own folder to ` + diff --git a/test/unit/bin/path-utils.test.ts b/test/unit/bin/path-utils.test.ts new file mode 100644 index 0000000..2f46e96 --- /dev/null +++ b/test/unit/bin/path-utils.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; + +import { findContainingScope, hoistPath, isPathWithinScope } from '../../../bin/path-utils.js'; + +describe('path-utils', () => { + it('isPathWithinScope matches a scope entry that names an exact file, not just a directory prefix', () => { + expect(isPathWithinScope('shared/utils.ts', 'shared/utils.ts')).toBe(true); + }); + + it('findContainingScope picks the exact-file entry out of a list of scope paths', () => { + expect(findContainingScope('shared/utils.ts', ['actors/foo', 'shared/utils.ts'])).toBe('shared/utils.ts'); + }); + + it('hoistPath resolves an exact-file scope match to an empty string, not an out-of-bounds slice', () => { + expect(hoistPath('shared/utils.ts', 'shared/utils.ts')).toBe(''); + }); +}); diff --git a/test/unit/bin/utils.test.ts b/test/unit/bin/utils.test.ts index adf65ee..207c1c8 100644 --- a/test/unit/bin/utils.test.ts +++ b/test/unit/bin/utils.test.ts @@ -285,6 +285,24 @@ describe('readConfigFile', () => { await expect(readConfigFile()).rejects.toThrow('not reachable through its own context paths'); }); + it('strips trailing slashes from folder and overrideActorContext entries', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { + folder: 'actors/shopify/', + actorName: 'myteam/shopify', + tokenEnvVar: 'APIFY_TOKEN', + overrideActorContext: ['actors/shopify/', 'packages/'], + }, + ]), + 'actors/shopify/.actor/actor.json': actorJson({}), + }); + + const result = await readConfigFile(); + expect(result[0].folder).toBe('actors/shopify'); + expect(result[0].contextPaths).toEqual(['actors/shopify', 'packages']); + }); + it('allows overrideActorContext with disjoint sibling paths that all reach the actor folder via one entry', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ From 9a3ec87f9da6a32d7cad6908354122e7ef6418f2 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Fri, 3 Jul 2026 10:30:03 +0100 Subject: [PATCH 22/33] rework json classification rule --- bin/diff-changes.ts | 6 ++++-- test/unit/bin/diff-changes.test.ts | 29 ++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index 362451a..10c00d2 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -38,7 +38,7 @@ type FileChangeForActor = * 2. Hardcoded ignore list, checked against the path hoisted relative to the matched context entry → ignored * 3. .dockerignore filtering (patterns relative to dockerContextDir) → ignored if matched * 4. README/CHANGELOG by filename → cosmetic if inside the actor's own folder, otherwise ignored - * 5. .json inside the actor's own folder with only cosmetic schema diffs → cosmetic (semantically verified) + * 5. .json inside the actor's own `.actor/` dir with only cosmetic schema diffs → cosmetic (semantically verified) * 6. Everything else → functional */ const classifyFileChange = ( @@ -72,7 +72,9 @@ const classifyFileChange = ( return isInActorFolder ? { impact: 'cosmetic', semanticallyVerified: false } : { impact: 'ignored' }; } - if (lowercaseFilePath.endsWith('.json') && isInActorFolder) { + const actorDotDir = lowerFolder ? `${lowerFolder}/.actor` : '.actor'; + + if (lowercaseFilePath.endsWith('.json') && isPathWithinScope(lowercaseFilePath, actorDotDir)) { let isCosmetic = cosmeticCache.get(originalFilePath); if (isCosmetic === undefined) { isCosmetic = isCosmeticOnlyJsonSchemaChange(commits, originalFilePath); diff --git a/test/unit/bin/diff-changes.test.ts b/test/unit/bin/diff-changes.test.ts index 80700d0..4eccf30 100644 --- a/test/unit/bin/diff-changes.test.ts +++ b/test/unit/bin/diff-changes.test.ts @@ -80,7 +80,7 @@ describe('getChangedActors', () => { it('returns actor when isLatest and JSON file has only cosmetic changes', () => { vi.spyOn(DiffJsonSchema, 'isCosmeticOnlyJsonSchemaChange').mockReturnValue(true); const result = getChangedActors({ - filepathsChanged: ['actors/foo_bar/actor.json'], + filepathsChanged: ['actors/foo_bar/.actor/actor.json'], actorConfigs, commits, isLatest: true, @@ -91,7 +91,7 @@ describe('getChangedActors', () => { it('does not return actor when not isLatest and JSON file has only cosmetic changes', () => { vi.spyOn(DiffJsonSchema, 'isCosmeticOnlyJsonSchemaChange').mockReturnValue(true); const result = getChangedActors({ - filepathsChanged: ['actors/foo_bar/actor.json'], + filepathsChanged: ['actors/foo_bar/.actor/actor.json'], actorConfigs, commits, isLatest: false, @@ -102,13 +102,36 @@ describe('getChangedActors', () => { it('returns actor when JSON file has functional changes', () => { vi.spyOn(DiffJsonSchema, 'isCosmeticOnlyJsonSchemaChange').mockReturnValue(false); const result = getChangedActors({ - filepathsChanged: ['actors/foo_bar/actor.json'], + filepathsChanged: ['actors/foo_bar/.actor/actor.json'], actorConfigs, commits, }); expect(result).toEqual([miniActor]); }); + it('JSON file in actor folder but outside .actor/ is functional, not checked for cosmetic', () => { + vi.spyOn(DiffJsonSchema, 'isCosmeticOnlyJsonSchemaChange').mockReturnValue(true); + const result = getChangedActors({ + filepathsChanged: ['actors/foo_bar/package.json'], + actorConfigs, + commits, + isLatest: false, + }); + expect(result).toEqual([miniActor]); + expect(DiffJsonSchema.isCosmeticOnlyJsonSchemaChange).not.toHaveBeenCalled(); + }); + + it('JSON file under .actor/ inside actor folder is checked for cosmetic changes', () => { + vi.spyOn(DiffJsonSchema, 'isCosmeticOnlyJsonSchemaChange').mockReturnValue(true); + const result = getChangedActors({ + filepathsChanged: ['actors/foo_bar/.actor/input_schema.json'], + actorConfigs, + commits, + isLatest: true, + }); + expect(result).toEqual([miniActor]); + }); + it('does not trigger narrow-context actor when shared file changes', () => { const result = getChangedActors({ filepathsChanged: ['shared/utils.ts'], From ab37aeb4d96a8ca6db61381f0851dfe54a915b27 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Fri, 3 Jul 2026 10:30:20 +0100 Subject: [PATCH 23/33] update Readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 702809a..0fd1610 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Each entry has: | Field | Required | Description | | ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `folder` | yes | Relative path from repo root to the actor directory. Use `"."` for a single-actor repo where `.actor/` is at the root. | +| `folder` | yes | Relative path from repo root to the actor's own project directory — the folder that directly contains `.actor/actor.json` (i.e. `/.actor/actor.json`), the actor's README/CHANGELOG, and its source. Use `"."` for a single-actor repo where `.actor/` is at the root. | | `actorName` | yes | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used. | | `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. No fallback — if the env var is not set at build time, the build fails. | | `overrideActorContext` | no | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. Entries must not be prefixes of one another (e.g. `["", "code"]` or `["actors", "actors/foo"]` are rejected), and the list must include a path that reaches the actor's own `folder` — otherwise the actor could never be detected as changed. | @@ -80,7 +80,7 @@ When a PR is opened or code is pushed, the tool determines which actors need to 3. **Hardcoded ignore list, context-aware** — the file path is first "hoisted" relative to the context path it matched (e.g. a standalone actor's own `.eslintrc` is checked as just `.eslintrc`, not the full repo-root-relative path), then checked against repo-level dev file patterns (`.vscode/`, `.gitignore`, `.husky/`, `.eslintrc`, `eslint.config.mjs`, `.prettierrc`, `.editorconfig`). There's no hardcoded special-casing for legacy `code/`/`shared/` layouts — repos that need those directories treated as top-level must list them explicitly in `overrideActorContext`. 4. **`.dockerignore` filtering** — if a `.dockerignore` exists at the root of the actor's `dockerContextDir`, matching files are ignored. Patterns are resolved relative to `dockerContextDir`, matching Docker's own behavior. 5. **README/CHANGELOG classification** — a `README.md` or `CHANGELOG.md` file is `cosmetic` (only triggers a release build, not tests) if it lives inside the actor's own `folder`; otherwise it's ignored entirely, since it isn't documentation for this actor. -6. **Cosmetic JSON classification** — `.json` files inside the actor's own `folder` with only cosmetic schema changes (whitespace, key ordering) only trigger a release build. +6. **Cosmetic JSON classification** — `.json` files inside the actor's own `.actor/` directory with only cosmetic schema changes (whitespace, key ordering) only trigger a release build. 7. **Functional** — everything else triggers both build and tests. ### 4. Create test directories From b3df6fa8c26ec0cbfb6bb1f78e10893c9bac6405 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Fri, 3 Jul 2026 10:41:35 +0100 Subject: [PATCH 24/33] remove caching logic for cosmetic changes --- bin/diff-changes.ts | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index 10c00d2..dba75f0 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -45,7 +45,6 @@ const classifyFileChange = ( originalFilePath: string, actorConfig: ActorConfig, commits: Commit[], - cosmeticCache: Map, dockerIgnoreMatcher: DockerIgnoreMatcher, ): FileChangeForActor => { const lowercaseFilePath = originalFilePath.toLowerCase(); @@ -75,11 +74,7 @@ const classifyFileChange = ( const actorDotDir = lowerFolder ? `${lowerFolder}/.actor` : '.actor'; if (lowercaseFilePath.endsWith('.json') && isPathWithinScope(lowercaseFilePath, actorDotDir)) { - let isCosmetic = cosmeticCache.get(originalFilePath); - if (isCosmetic === undefined) { - isCosmetic = isCosmeticOnlyJsonSchemaChange(commits, originalFilePath); - cosmeticCache.set(originalFilePath, isCosmetic); - } + const isCosmetic = isCosmeticOnlyJsonSchemaChange(commits, originalFilePath); if (isCosmetic) { return { impact: 'cosmetic', semanticallyVerified: true }; } @@ -132,7 +127,6 @@ export const getChangedActors = ({ commits, }: ShouldBuildAndTestOptions): ActorConfig[] => { const actorsChangedMap = new Map(); - const cosmeticCache = new Map(); const fileImpacts = new Map(); for (const actorConfig of actorConfigs) { @@ -145,13 +139,7 @@ export const getChangedActors = ({ continue; } - const change = classifyFileChange( - originalFilePath, - actorConfig, - commits, - cosmeticCache, - dockerIgnoreMatcher, - ); + const change = classifyFileChange(originalFilePath, actorConfig, commits, dockerIgnoreMatcher); updateFileImpact(fileImpacts, originalFilePath, change.impact); if (change.impact === 'ignored' || change.impact === 'outside-context') continue; From 6980b49d02bd201b16fdc1cfe7035db26e57374e Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Fri, 3 Jul 2026 13:19:20 +0100 Subject: [PATCH 25/33] reworked logging logic --- bin/diff-changes.ts | 93 +++++++++++++++++++----------- test/unit/bin/diff-changes.test.ts | 93 ++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 35 deletions(-) diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index dba75f0..3406713 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -96,27 +96,58 @@ const isExcludedBySibling = (lowercaseFilePath: string, actor: ActorConfig, allA ); }; -type LoggableImpact = 'ignored' | 'cosmetic' | 'functional'; +type ActorChangeEntry = { + actorConfig: ActorConfig; + files: string[]; +}; -const IMPACT_PRIORITY: Record = { functional: 3, cosmetic: 2, ignored: 1 }; +type ChangeGroup = { actorNames: string[]; files: string[] }; /** - * A file can be classified differently by different actors (e.g. functional for a broad-context - * actor, outside-context for a narrow one). For logging we keep the most significant classification - * across all actors: functional > cosmetic > ignored. Files that are outside-context for every - * actor are treated as ignored. + * Maps each changed file to the set of actor names it triggered a change for. */ -const updateFileImpact = ( - fileImpacts: Map, - filePath: string, - impact: FileChangeForActor['impact'], -): void => { - if (impact === 'outside-context') return; - - const loggable = impact as LoggableImpact; - const current = fileImpacts.get(filePath); - if (!current || IMPACT_PRIORITY[loggable] > IMPACT_PRIORITY[current]) { - fileImpacts.set(filePath, loggable); +const buildFileToActorNamesMap = (actorsChangedMap: Map): Map> => { + const fileToActorNames = new Map>(); + for (const { actorConfig, files } of actorsChangedMap.values()) { + for (const file of files) { + const actorNames = fileToActorNames.get(file) ?? new Set(); + actorNames.add(actorConfig.actorName); + fileToActorNames.set(file, actorNames); + } + } + return fileToActorNames; +}; + +/** + * Groups files by their identical actor-set (files triggering a change for the exact same + * actors are grouped together), then orders the groups by descending actor-set size + * (most-shared groups first), breaking ties alphabetically by actor names. + */ +const groupFilesByActorSet = (fileToActorNames: Map>): ChangeGroup[] => { + const groupsByKey = new Map(); + for (const [file, actorNamesSet] of fileToActorNames) { + const actorNames = Array.from(actorNamesSet).sort(); + const key = actorNames.join(','); + const group = groupsByKey.get(key) ?? { actorNames, files: [] }; + group.files.push(file); + groupsByKey.set(key, group); + } + + return Array.from(groupsByKey.values()).sort((groupA, groupB) => { + if (groupB.actorNames.length !== groupA.actorNames.length) { + return groupB.actorNames.length - groupA.actorNames.length; + } + return groupA.actorNames.join(',').localeCompare(groupB.actorNames.join(',')); + }); +}; + +const logChangeGroups = (groups: ChangeGroup[]): void => { + for (const { actorNames, files } of groups) { + if (actorNames.length > 1) { + console.error(`[DIFF]: Shared changes for actors ${actorNames.join(', ')}: ${files.join(', ')}`); + } else { + console.error(`[DIFF]: Changes specific to actor ${actorNames[0]}: ${files.join(', ')}`); + } } }; @@ -126,8 +157,7 @@ export const getChangedActors = ({ isLatest = false, commits, }: ShouldBuildAndTestOptions): ActorConfig[] => { - const actorsChangedMap = new Map(); - const fileImpacts = new Map(); + const actorsChangedMap = new Map(); for (const actorConfig of actorConfigs) { const dockerIgnoreMatcher = loadDockerIgnore(actorConfig.dockerContextDir); @@ -140,30 +170,23 @@ export const getChangedActors = ({ } const change = classifyFileChange(originalFilePath, actorConfig, commits, dockerIgnoreMatcher); - updateFileImpact(fileImpacts, originalFilePath, change.impact); if (change.impact === 'ignored' || change.impact === 'outside-context') continue; if (change.impact === 'cosmetic' && !isLatest) continue; - actorsChangedMap.set(actorConfig.folder, actorConfig); + const entry = actorsChangedMap.get(actorConfig.folder) ?? { actorConfig, files: [] }; + entry.files.push(originalFilePath); + actorsChangedMap.set(actorConfig.folder, entry); } } - const actorsChanged = Array.from(actorsChangedMap.values()); - - // Logging - const formatFiles = (files: string[]) => (files.length > 0 ? files.join(', ') : ''); - - const ignoredFiles = filepathsChanged.filter((file) => { - const impact = fileImpacts.get(file); - return impact === 'ignored' || !impact; - }); - const cosmeticFiles = filepathsChanged.filter((file) => fileImpacts.get(file) === 'cosmetic'); - const functionalFiles = filepathsChanged.filter((file) => fileImpacts.get(file) === 'functional'); + const actorsChanged = Array.from(actorsChangedMap.values()).map((entry) => entry.actorConfig); - console.error(`[DIFF]: Ignored files (don't trigger test or build): ${formatFiles(ignoredFiles)}`); - console.error(`[DIFF]: Cosmetic files (only trigger release build): ${formatFiles(cosmeticFiles)}`); - console.error(`[DIFF]: Functional files (trigger test & release build): ${formatFiles(functionalFiles)}`); + // Log changes grouped by actor set, so changes shared across actors are logged once + // instead of being repeated per actor. + const fileToActorNames = buildFileToActorNamesMap(actorsChangedMap); + const groups = groupFilesByActorSet(fileToActorNames); + logChangeGroups(groups); if (actorsChanged.length > 0) { const actorNames = actorsChanged.map((config) => config.actorName); diff --git a/test/unit/bin/diff-changes.test.ts b/test/unit/bin/diff-changes.test.ts index 4eccf30..d6d87db 100644 --- a/test/unit/bin/diff-changes.test.ts +++ b/test/unit/bin/diff-changes.test.ts @@ -432,3 +432,96 @@ describe('getChangedActors', () => { expect(result).toEqual([]); }); }); + +describe('getChangedActors logging', () => { + beforeEach(() => { + vi.spyOn(DiffJsonSchema, 'isCosmeticOnlyJsonSchemaChange').mockReturnValue(false); + vi.spyOn(console, 'error').mockImplementation(() => undefined); + }); + + it('logs a single "specific" group for a single actor with one functional file', () => { + getChangedActors({ + filepathsChanged: ['actors/foo_bar/src/main.ts'], + actorConfigs: [miniActor], + commits, + }); + + expect(console.error).toHaveBeenCalledWith( + '[DIFF]: Changes specific to actor foo/bar: actors/foo_bar/src/main.ts', + ); + expect(console.error).toHaveBeenCalledWith('[DIFF]: Actors to be built and tested: foo/bar'); + }); + + it('logs a single "shared" group when two actors are triggered by the exact same file', () => { + const actorA: ActorConfig = { + actorName: 'team/actor-a', + folder: 'actors/a', + tokenEnvVar: 'APIFY_TOKEN_TEAM', + dockerContextDir: '', + contextPaths: ['', 'shared'], + }; + const actorB: ActorConfig = { + actorName: 'team/actor-b', + folder: 'actors/b', + tokenEnvVar: 'APIFY_TOKEN_TEAM', + dockerContextDir: '', + contextPaths: ['', 'shared'], + }; + + getChangedActors({ + filepathsChanged: ['shared/shared.ts'], + actorConfigs: [actorA, actorB], + commits, + }); + + expect(console.error).toHaveBeenCalledWith( + '[DIFF]: Shared changes for actors team/actor-a, team/actor-b: shared/shared.ts', + ); + expect(console.error).not.toHaveBeenCalledWith(expect.stringContaining('Changes specific to actor')); + expect(console.error).toHaveBeenCalledWith('[DIFF]: Actors to be built and tested: team/actor-a, team/actor-b'); + }); + + it('logs shared and specific groups in descending-size order for partial overlap across actors', () => { + const actorA: ActorConfig = { + actorName: 'team/actor-a', + folder: 'actors/a', + tokenEnvVar: 'APIFY_TOKEN_TEAM', + dockerContextDir: '', + contextPaths: [''], + }; + const actorB: ActorConfig = { + actorName: 'team/actor-b', + folder: 'actors/b', + tokenEnvVar: 'APIFY_TOKEN_TEAM', + dockerContextDir: '', + contextPaths: [''], + }; + + getChangedActors({ + filepathsChanged: ['shared.ts', 'actors/a/a-only.ts', 'actors/b/b-only.ts'], + actorConfigs: [actorA, actorB], + commits, + }); + + const errorCalls = (console.error as unknown as ReturnType).mock.calls.map((call) => call[0]); + + expect(errorCalls).toEqual([ + '[DIFF]: Shared changes for actors team/actor-a, team/actor-b: shared.ts', + '[DIFF]: Changes specific to actor team/actor-a: actors/a/a-only.ts', + '[DIFF]: Changes specific to actor team/actor-b: actors/b/b-only.ts', + '[DIFF]: Actors to be built and tested: team/actor-a, team/actor-b', + ]); + }); + + it('logs no group lines when zero actors changed', () => { + getChangedActors({ + filepathsChanged: ['.gitignore', 'README.md'], + actorConfigs, + commits, + }); + + expect(console.error).not.toHaveBeenCalledWith(expect.stringContaining('Shared changes')); + expect(console.error).not.toHaveBeenCalledWith(expect.stringContaining('Changes specific to')); + expect(console.error).toHaveBeenCalledWith('[DIFF]: No relevant files changed, skipping builds and tests'); + }); +}); From 73a2d8d1d95bedd9782eef46c9fc7893459c1fe3 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Fri, 3 Jul 2026 13:44:58 +0100 Subject: [PATCH 26/33] enforce folder field in config json --- bin/utils.ts | 9 ++++++++- test/unit/bin/utils.test.ts | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/bin/utils.ts b/bin/utils.ts index d5a828a..9fbf3d5 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -75,7 +75,14 @@ export const readConfigFile = async (): Promise => { const seenFolders = new Set(); const actorConfigs: ActorConfig[] = []; - for (const entry of config.actors) { + for (const [index, entry] of config.actors.entries()) { + if (typeof entry.folder !== 'string') { + throw new Error( + `Invalid "folder" for actor entry at index ${index} in "${CONFIG_FILE_NAME}". ` + + `Must be a string (use "." for a single-actor repo).`, + ); + } + const folder = entry.folder === '.' ? '' : stripTrailingSlash(entry.folder); if (seenFolders.has(folder)) { diff --git a/test/unit/bin/utils.test.ts b/test/unit/bin/utils.test.ts index 207c1c8..a290e40 100644 --- a/test/unit/bin/utils.test.ts +++ b/test/unit/bin/utils.test.ts @@ -172,6 +172,26 @@ describe('readConfigFile', () => { await expect(readConfigFile()).rejects.toThrow('Cannot read'); }); + it('throws when folder is missing', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { actorName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + ]), + }); + + await expect(readConfigFile()).rejects.toThrow(/Invalid "folder"/); + }); + + it('throws when folder is not a string', async () => { + mockFiles({ + '.test-tools-actors-config.json': validConfig([ + { folder: 123, actorName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + ]), + }); + + await expect(readConfigFile()).rejects.toThrow(/Invalid "folder"/); + }); + it('throws when actorName is missing', async () => { mockFiles({ '.test-tools-actors-config.json': validConfig([ From cbed5e218b6181d96780df36e635c06dbfc2f4a9 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Fri, 3 Jul 2026 15:43:10 +0100 Subject: [PATCH 27/33] rename config file --- README.md | 4 ++-- bin/utils.ts | 2 +- test/unit/bin/utils.test.ts | 48 +++++++++++++++++-------------------- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 0fd1610..74f9c16 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ npm i -D apify-test-tools ### 2. Create the config file -Every repo that uses `apify-test-tools` must have a `.test-tools-actors-config.json` file at the root. This file tells the tool which actors live in the repo, how to identify them, and which token to use. +Every repo that uses `apify-test-tools` must have an `apify-test-tools.config.json` file at the root. This file tells the tool which actors live in the repo, how to identify them, and which token to use. ```json { @@ -50,7 +50,7 @@ Each actor in the config must have a `.actor/actor.json` file. The `dockerContex ``` my-repo -├── .test-tools-actors-config.json +├── apify-test-tools.config.json ├── actors │ ├── web-scraper │ │ ├── .actor diff --git a/bin/utils.ts b/bin/utils.ts index 9fbf3d5..b496cf8 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -31,7 +31,7 @@ export const getEnvVar = (varName: string, defaultValue?: string): string => { return value; }; -const CONFIG_FILE_NAME = '.test-tools-actors-config.json'; +export const CONFIG_FILE_NAME = 'apify-test-tools.config.json'; // Strips a trailing slash so config-declared paths ("actors/shopify/" vs "actors/shopify") compare equal. const stripTrailingSlash = (pathValue: string): string => pathValue.replace(/\/+$/, ''); diff --git a/test/unit/bin/utils.test.ts b/test/unit/bin/utils.test.ts index a290e40..c8db4d3 100644 --- a/test/unit/bin/utils.test.ts +++ b/test/unit/bin/utils.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { readConfigFile } from '../../../bin/utils.js'; +import { CONFIG_FILE_NAME, readConfigFile } from '../../../bin/utils.js'; const { fsMock } = vi.hoisted(() => ({ fsMock: { @@ -29,7 +29,7 @@ const mockFiles = (files: Record) => { describe('readConfigFile', () => { it('returns correct ActorConfig[] for a valid config', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', actorName: 'myteam/shopify-scraper', tokenEnvVar: 'APIFY_TOKEN_MYTEAM' }, ]), 'actors/shopify/.actor/actor.json': actorJson({ dockerContextDir: '../../..' }), @@ -50,7 +50,7 @@ describe('readConfigFile', () => { it('normalizes folder "." to ""', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: '.', actorName: 'apify/my-actor', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), '.actor/actor.json': actorJson({}), @@ -63,7 +63,7 @@ describe('readConfigFile', () => { it('defaults dockerContextDir to actor folder when absent from actor.json', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/web-scraper', actorName: 'apify/web-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), 'actors/web-scraper/.actor/actor.json': actorJson({}), @@ -76,7 +76,7 @@ describe('readConfigFile', () => { it('resolves dockerContextDir relative to .actor/ folder', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', actorName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN' }, ]), 'actors/shopify/.actor/actor.json': actorJson({ dockerContextDir: '../../..' }), @@ -88,7 +88,7 @@ describe('readConfigFile', () => { it('resolves contextPaths from overrideActorContext', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', actorName: 'myteam/shopify', @@ -105,7 +105,7 @@ describe('readConfigFile', () => { it('handles multiple actors', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/web-scraper', actorName: 'apify/web-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, { folder: 'actors/email-sender', @@ -140,7 +140,7 @@ describe('readConfigFile', () => { it('throws on duplicate folders', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', actorName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, { folder: 'actors/shopify', actorName: 'other/shopify', tokenEnvVar: 'APIFY_TOKEN_OTHER' }, ]), @@ -152,7 +152,7 @@ describe('readConfigFile', () => { it('throws on duplicate folders after normalization ("." and "")', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: '.', actorName: 'apify/actor-a', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, { folder: '', actorName: 'other/actor-b', tokenEnvVar: 'APIFY_TOKEN_OTHER' }, ]), @@ -164,7 +164,7 @@ describe('readConfigFile', () => { it('throws when actor.json is missing', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', actorName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), }); @@ -174,9 +174,7 @@ describe('readConfigFile', () => { it('throws when folder is missing', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ - { actorName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, - ]), + [CONFIG_FILE_NAME]: validConfig([{ actorName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }]), }); await expect(readConfigFile()).rejects.toThrow(/Invalid "folder"/); @@ -184,7 +182,7 @@ describe('readConfigFile', () => { it('throws when folder is not a string', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 123, actorName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), }); @@ -194,9 +192,7 @@ describe('readConfigFile', () => { it('throws when actorName is missing', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ - { folder: 'actors/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, - ]), + [CONFIG_FILE_NAME]: validConfig([{ folder: 'actors/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }]), 'actors/shopify/.actor/actor.json': actorJson({}), }); @@ -205,7 +201,7 @@ describe('readConfigFile', () => { it('throws when actorName has no slash', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', actorName: 'shopify-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), 'actors/shopify/.actor/actor.json': actorJson({}), @@ -216,7 +212,7 @@ describe('readConfigFile', () => { it('throws when actorName has empty parts', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', actorName: '/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), 'actors/shopify/.actor/actor.json': actorJson({}), @@ -227,7 +223,7 @@ describe('readConfigFile', () => { it('throws when overrideActorContext is not an array', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', actorName: 'myteam/shopify', @@ -243,7 +239,7 @@ describe('readConfigFile', () => { it('throws when overrideActorContext contains non-strings', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', actorName: 'myteam/shopify', @@ -259,7 +255,7 @@ describe('readConfigFile', () => { it('throws when overrideActorContext entries overlap (one is a prefix of another)', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', actorName: 'myteam/shopify', @@ -275,7 +271,7 @@ describe('readConfigFile', () => { it('throws when overrideActorContext contains the repo root alongside another entry', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', actorName: 'myteam/shopify', @@ -291,7 +287,7 @@ describe('readConfigFile', () => { it('throws when overrideActorContext does not include the actor own folder', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', actorName: 'myteam/shopify', @@ -307,7 +303,7 @@ describe('readConfigFile', () => { it('strips trailing slashes from folder and overrideActorContext entries', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify/', actorName: 'myteam/shopify', @@ -325,7 +321,7 @@ describe('readConfigFile', () => { it('allows overrideActorContext with disjoint sibling paths that all reach the actor folder via one entry', async () => { mockFiles({ - '.test-tools-actors-config.json': validConfig([ + [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', actorName: 'myteam/shopify', From 5efdb134bb3d5c625605b5640c6514436754a05a Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Fri, 3 Jul 2026 15:50:40 +0100 Subject: [PATCH 28/33] disambiguate actor identifiers into actorFullName, actorRawId and actorId --- README.md | 8 +-- bin/build.ts | 52 +++++++++---------- bin/diff-changes.ts | 46 ++++++++--------- bin/test-report.ts | 10 ++-- bin/types.ts | 8 +-- bin/utils.ts | 6 +-- lib/lib.ts | 50 ++++++++++-------- lib/types.ts | 4 +- lib/utils.ts | 8 +-- test/unit/bin/diff-changes.test.ts | 34 ++++++------- test/unit/bin/utils.test.ts | 68 +++++++++++++------------ test/unit/should-built-and-test.test.ts | 24 ++++----- 12 files changed, 164 insertions(+), 154 deletions(-) diff --git a/README.md b/README.md index 74f9c16..1854b35 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,12 @@ Every repo that uses `apify-test-tools` must have an `apify-test-tools.config.js "actors": [ { "folder": "actors/web-scraper", - "actorName": "myteam/web-scraper", + "actorFullName": "myteam/web-scraper", "tokenEnvVar": "APIFY_TOKEN_MYTEAM" }, { "folder": "actors/email-sender", - "actorName": "myteam/email-sender", + "actorFullName": "myteam/email-sender", "tokenEnvVar": "APIFY_TOKEN_MYTEAM", "overrideActorContext": ["actors/email-sender", "packages/shared"] } @@ -40,7 +40,7 @@ Each entry has: | Field | Required | Description | | ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `folder` | yes | Relative path from repo root to the actor's own project directory — the folder that directly contains `.actor/actor.json` (i.e. `/.actor/actor.json`), the actor's README/CHANGELOG, and its source. Use `"."` for a single-actor repo where `.actor/` is at the root. | -| `actorName` | yes | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used. | +| `actorFullName` | yes | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used. | | `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. No fallback — if the env var is not set at build time, the build fails. | | `overrideActorContext` | no | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. Entries must not be prefixes of one another (e.g. `["", "code"]` or `["actors", "actors/foo"]` are rejected), and the list must include a path that reaches the actor's own `folder` — otherwise the actor could never be detected as changed. | @@ -412,7 +412,7 @@ GITHUB_WORKSPACE=. \ Remove `--dry-run` to actually trigger builds and update the branch names/ The command outputs a JSON array of build objects to stdout: ```json -[{ "buildId": "...", "actorId": "...", "buildNumber": "...", "actorName": "john.doe/my-actor" }] +[{ "buildId": "...", "actorRawId": "...", "buildNumber": "...", "actorFullName": "john.doe/my-actor" }] ``` #### 5. Run tests against the builds diff --git a/bin/build.ts b/bin/build.ts index f2fa80f..699dcb6 100644 --- a/bin/build.ts +++ b/bin/build.ts @@ -15,7 +15,7 @@ type BuildPrActorOptions = { class ApifyBuilder { private constructor( private readonly apifyClient: ApifyClient, - private readonly actorName: string, + private readonly actorFullName: string, ) {} // Usually 'latest' but not necessarily (can be e.g. 'version-0') @@ -24,29 +24,29 @@ class ApifyBuilder { defaultVersionNumber: string; defaultBuildTag: string; }> => { - const actorClient = this.apifyClient.actor(this.actorName); + const actorClient = this.apifyClient.actor(this.actorFullName); const actorInfo = await actorClient.get(); if (!actorInfo) { throw new Error( - `[${this.actorName}] not found. It is not published or we are missing token to access it privately or its name is misspelled`, + `[${this.actorFullName}] not found. It is not published or we are missing token to access it privately or its name is misspelled`, ); } const defaultBuildTag = actorInfo.defaultRunOptions.build; - console.error(`Default build tag for ${this.actorName} is ${defaultBuildTag}`); + console.error(`Default build tag for ${this.actorFullName} is ${defaultBuildTag}`); // We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) { throw new Error( - `[${this.actorName}] Default build is a build number, not a tag. While this could work, ` + + `[${this.actorFullName}] Default build is a build number, not a tag. While this could work, ` + `we want to have a default as tag so this is often an accidental misconfiguration from the dev`, ); } // I reported that buildNumber should probably not be optional const defaultBuildNumber = actorInfo.taggedBuilds![defaultBuildTag].buildNumber!; const defaultVersionNumber = defaultBuildNumber.match(/(\d+\.\d+)\.\d+/)![1]; - console.error(`Default version for ${this.actorName} is ${defaultVersionNumber}`); + console.error(`Default version for ${this.actorFullName} is ${defaultVersionNumber}`); return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag }; }; @@ -58,11 +58,11 @@ class ApifyBuilder { actorConfig, useDockerCache, }: BuildPrActorOptions): Promise => { - const actorClient = this.apifyClient.actor(this.actorName); + const actorClient = this.apifyClient.actor(this.actorFullName); const actorInfo = await actorClient.get(); if (!actorInfo) { throw new Error( - `No actor named '${this.actorName}' was found on the platform. If this` + + `No actor named '${this.actorFullName}' was found on the platform. If this` + ' is unexpected, make sure the actor you are targeting is spelled the' + ' same as the folder in the repository.', ); @@ -92,31 +92,31 @@ class ApifyBuilder { // We also get back actId so the testing actor can both match by actor ID and name const { id, actId, buildNumber } = await actorClient.build(versionNumber, { useCache: useDockerCache }); - console.error(`[${this.actorName}]: ${id} (${buildNumber})`); - return { buildId: id, actorId: actId, buildNumber, actorName: this.actorName, actorConfig }; + console.error(`[${this.actorFullName}]: ${id} (${buildNumber})`); + return { buildId: id, actorRawId: actId, buildNumber, actorFullName: this.actorFullName, actorConfig }; }; - waitForBuildToFinish = async (buildId: string, actorName: string): Promise => { + waitForBuildToFinish = async (buildId: string, actorFullName: string): Promise => { const build = await this.apifyClient.build(buildId).waitForFinish(); const versionNumber = build.buildNumber; if (build.status === 'FAILED' || build.status === 'TIMED-OUT') { const message = - `[BUILD][${actorName}]: Build ${buildId} (${versionNumber}) failed. ` + + `[BUILD][${actorFullName}]: Build ${buildId} (${versionNumber}) failed. ` + `Not continuing with other builds and tests.`; - console.error(`[${this.actorName}]: ${versionNumber}`); + console.error(`[${this.actorFullName}]: ${versionNumber}`); throw new Error(message); } - console.error(`[${this.actorName}]: ${versionNumber}`); + console.error(`[${this.actorFullName}]: ${versionNumber}`); return build; }; - static fromActorConfig = ({ actorName, tokenEnvVar }: ActorConfig): ApifyBuilder => { + static fromActorConfig = ({ actorFullName, tokenEnvVar }: ActorConfig): ApifyBuilder => { const token = process.env[tokenEnvVar]; if (!token) { - throw new Error(`Env var ${tokenEnvVar} is not set (needed for actor "${actorName}").`); + throw new Error(`Env var ${tokenEnvVar} is not set (needed for actor "${actorFullName}").`); } const apifyClient = new ApifyClient({ token }); - return new ApifyBuilder(apifyClient, actorName); + return new ApifyBuilder(apifyClient, actorFullName); }; /** @@ -148,7 +148,7 @@ class ApifyBuilder { const DEFAULT_DAYS_BACK_PROD_VERSIONS = 30; const DEFAULT_DAYS_BACK_DEVEL = 7; - const actorInfo = (await this.apifyClient.actor(this.actorName).get())!; + const actorInfo = (await this.apifyClient.actor(this.actorFullName).get())!; // 'devel' used to be hardcoded for testing version 0.99, once we get rid of this tag everywhere, we can remove this code const taggedDevelBuildNumber: string | undefined = actorInfo.taggedBuilds!.devel?.buildNumber; @@ -160,7 +160,7 @@ class ApifyBuilder { tag, })); - const { items } = await this.apifyClient.actor(this.actorName).builds().list(); + const { items } = await this.apifyClient.actor(this.actorFullName).builds().list(); // Deleting default build throws an error, so we skip it const { defaultBuildNumber, defaultBuildTag } = await this.getDefaultVersionAndTag(); @@ -173,7 +173,7 @@ class ApifyBuilder { const buildsToDelete = (items as CorrectBuildColletionItem[]).filter((build) => { if (build.buildNumber === defaultBuildNumber) { console.error( - `[DELETE OLD BUILDS][${this.actorName}]: Skipping default build ${defaultBuildNumber} (${defaultBuildTag}). ` + + `[DELETE OLD BUILDS][${this.actorFullName}]: Skipping default build ${defaultBuildNumber} (${defaultBuildTag}). ` + `We never delete default builds`, ); return false; @@ -184,7 +184,7 @@ class ApifyBuilder { ); if (protectedTagFound) { console.error( - `[DELETE OLD BUILDS][${this.actorName}]: Skipping protected build ${protectedTagFound.buildNumber} (${protectedTagFound.tag}).`, + `[DELETE OLD BUILDS][${this.actorFullName}]: Skipping protected build ${protectedTagFound.buildNumber} (${protectedTagFound.tag}).`, ); return false; } @@ -193,7 +193,7 @@ class ApifyBuilder { const shouldDeleteDevelBuild = build.startedAt.getTime() < daysAgoUnixDevel; if (shouldDeleteDevelBuild) { console.error( - `[DELETE OLD BUILDS][${this.actorName}]: Removing olf devel build ${taggedDevelBuildNumber}.`, + `[DELETE OLD BUILDS][${this.actorFullName}]: Removing olf devel build ${taggedDevelBuildNumber}.`, ); } return shouldDeleteDevelBuild; @@ -202,7 +202,7 @@ class ApifyBuilder { }); console.error( - `[DELETE OLD BUILDS][${this.actorName}]: Deleting ${buildsToDelete.length} old builds that are non-default and ` + + `[DELETE OLD BUILDS][${this.actorFullName}]: Deleting ${buildsToDelete.length} old builds that are non-default and ` + `older than 30 days from total ${items.length}`, ); for (const build of buildsToDelete) { @@ -268,13 +268,13 @@ export const runBuilds = async ({ await Promise.all( startedBuilds.map(async (buildData) => { const builder = ApifyBuilder.fromActorConfig(buildData.actorConfig); - await builder.waitForBuildToFinish(buildData.buildId, buildData.actorName); + await builder.waitForBuildToFinish(buildData.buildId, buildData.actorFullName); }), ); console.error('========================================='); console.error('SUMMARY:'); - for (const buildData of startedBuilds.sort((a, b) => a.actorName.localeCompare(b.actorName))) { - console.error(`[${buildData.actorName}]: ${buildData.buildNumber} `); + for (const buildData of startedBuilds.sort((a, b) => a.actorFullName.localeCompare(b.actorFullName))) { + console.error(`[${buildData.actorFullName}]: ${buildData.buildNumber} `); } console.error('========================================='); diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index 3406713..7872f2c 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -101,21 +101,21 @@ type ActorChangeEntry = { files: string[]; }; -type ChangeGroup = { actorNames: string[]; files: string[] }; +type ChangeGroup = { actors: string[]; files: string[] }; /** * Maps each changed file to the set of actor names it triggered a change for. */ -const buildFileToActorNamesMap = (actorsChangedMap: Map): Map> => { - const fileToActorNames = new Map>(); +const buildFileToActorsMap = (actorsChangedMap: Map): Map> => { + const fileToActors = new Map>(); for (const { actorConfig, files } of actorsChangedMap.values()) { for (const file of files) { - const actorNames = fileToActorNames.get(file) ?? new Set(); - actorNames.add(actorConfig.actorName); - fileToActorNames.set(file, actorNames); + const actors = fileToActors.get(file) ?? new Set(); + actors.add(actorConfig.actorFullName); + fileToActors.set(file, actors); } } - return fileToActorNames; + return fileToActors; }; /** @@ -123,30 +123,30 @@ const buildFileToActorNamesMap = (actorsChangedMap: Map>): ChangeGroup[] => { +const groupFilesByActorSet = (fileToActors: Map>): ChangeGroup[] => { const groupsByKey = new Map(); - for (const [file, actorNamesSet] of fileToActorNames) { - const actorNames = Array.from(actorNamesSet).sort(); - const key = actorNames.join(','); - const group = groupsByKey.get(key) ?? { actorNames, files: [] }; + for (const [file, actorsSet] of fileToActors) { + const actors = Array.from(actorsSet).sort(); + const key = actors.join(','); + const group = groupsByKey.get(key) ?? { actors, files: [] }; group.files.push(file); groupsByKey.set(key, group); } return Array.from(groupsByKey.values()).sort((groupA, groupB) => { - if (groupB.actorNames.length !== groupA.actorNames.length) { - return groupB.actorNames.length - groupA.actorNames.length; + if (groupB.actors.length !== groupA.actors.length) { + return groupB.actors.length - groupA.actors.length; } - return groupA.actorNames.join(',').localeCompare(groupB.actorNames.join(',')); + return groupA.actors.join(',').localeCompare(groupB.actors.join(',')); }); }; const logChangeGroups = (groups: ChangeGroup[]): void => { - for (const { actorNames, files } of groups) { - if (actorNames.length > 1) { - console.error(`[DIFF]: Shared changes for actors ${actorNames.join(', ')}: ${files.join(', ')}`); + for (const { actors, files } of groups) { + if (actors.length > 1) { + console.error(`[DIFF]: Shared changes for actors ${actors.join(', ')}: ${files.join(', ')}`); } else { - console.error(`[DIFF]: Changes specific to actor ${actorNames[0]}: ${files.join(', ')}`); + console.error(`[DIFF]: Changes specific to actor ${actors[0]}: ${files.join(', ')}`); } } }; @@ -184,13 +184,13 @@ export const getChangedActors = ({ // Log changes grouped by actor set, so changes shared across actors are logged once // instead of being repeated per actor. - const fileToActorNames = buildFileToActorNamesMap(actorsChangedMap); - const groups = groupFilesByActorSet(fileToActorNames); + const fileToActors = buildFileToActorsMap(actorsChangedMap); + const groups = groupFilesByActorSet(fileToActors); logChangeGroups(groups); if (actorsChanged.length > 0) { - const actorNames = actorsChanged.map((config) => config.actorName); - console.error(`[DIFF]: Actors to be built and tested: ${actorNames.join(', ')}`); + const actors = actorsChanged.map((config) => config.actorFullName); + console.error(`[DIFF]: Actors to be built and tested: ${actors.join(', ')}`); } else { console.error(`[DIFF]: No relevant files changed, skipping builds and tests`); } diff --git a/bin/test-report.ts b/bin/test-report.ts index 257f356..98e66c8 100644 --- a/bin/test-report.ts +++ b/bin/test-report.ts @@ -36,7 +36,7 @@ export const reportTestResults = async ({ } } - const failedAssertions: { message: string; runLink: string; actorName: string }[] = []; + const failedAssertions: { message: string; runLink: string; actorId: string }[] = []; console.error(); console.error(`PASSED: ${passed.length}, FAILED: ${failed.length}`); @@ -62,7 +62,7 @@ export const reportTestResults = async ({ ...failureMessages.map((message) => ({ message: message.split('\n')?.[0], runLink: meta.runLink, - actorName: meta.actorName, + actorId: meta.actorId, })), ); } @@ -89,10 +89,10 @@ export const reportTestResults = async ({ const jobLink = jobUrl ? ` Check <${jobUrl}|the job>.` : ''; let slackMessage = `\`${workflowName ?? '-'}\``; slackMessage += `: has ${failedAssertions.length} failed assertions. Failing test suites: ${failed.length}/${total}.${jobLink}`; - slackMessage += `\n\n${failedAssertions[0].message} --- <${failedAssertions[0].runLink}|${failedAssertions[0].actorName}>`; + slackMessage += `\n\n${failedAssertions[0].message} --- <${failedAssertions[0].runLink}|${failedAssertions[0].actorId}>`; const blocks = failedAssertions .slice(1) - .map(({ message, runLink, actorName }) => `• ${message} --- <${runLink}|${actorName}>`); + .map(({ message, runLink, actorId }) => `• ${message} --- <${runLink}|${actorId}>`); console.error('SLACK:', slackMessage); console.error('\tblocks:', blocks.join('\n\t\t')); @@ -122,7 +122,7 @@ interface JsonAssertionResult { meta: { runId: string; runLink: string; - actorName: string; + actorId: string; }; duration?: Milliseconds | null; failureMessages: string[] | null; diff --git a/bin/types.ts b/bin/types.ts index 94ff738..b10c8d0 100644 --- a/bin/types.ts +++ b/bin/types.ts @@ -87,7 +87,7 @@ export interface GithubCommit { export interface ActorConfigFileEntry { folder: string; - actorName: string; + actorFullName: string; tokenEnvVar: string; overrideActorContext?: string[]; } @@ -98,14 +98,14 @@ export interface ActorConfigFile { export interface BuildData { buildId: string; - actorId: string; - actorName: string; + actorRawId: string; + actorFullName: string; actorConfig: ActorConfig; buildNumber: string; } export interface ActorConfig { - actorName: string; + actorFullName: string; folder: string; tokenEnvVar: string; dockerContextDir: string; diff --git a/bin/utils.ts b/bin/utils.ts index b496cf8..97f88a5 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -92,10 +92,10 @@ export const readConfigFile = async (): Promise => { } seenFolders.add(folder); - const nameParts = entry.actorName?.split('/'); + const nameParts = entry.actorFullName?.split('/'); if (!nameParts || nameParts.length !== 2 || !nameParts[0] || !nameParts[1]) { throw new Error( - `Invalid "actorName" for folder "${entry.folder}" in "${CONFIG_FILE_NAME}". ` + + `Invalid "actorFullName" for folder "${entry.folder}" in "${CONFIG_FILE_NAME}". ` + `Must be in "owner/name" format (e.g. "apify/web-scraper").`, ); } @@ -156,7 +156,7 @@ export const readConfigFile = async (): Promise => { } actorConfigs.push({ - actorName: entry.actorName, + actorFullName: entry.actorFullName, folder, tokenEnvVar: entry.tokenEnvVar, dockerContextDir: normalizedDockerContextDir, diff --git a/lib/lib.ts b/lib/lib.ts index c4590b7..d65fdb1 100644 --- a/lib/lib.ts +++ b/lib/lib.ts @@ -24,8 +24,8 @@ try { } const config = actorBuilds.reduce>((map, cfg) => { - map.set(cfg.actorName, cfg); - map.set(cfg.actorId, cfg); + map.set(cfg.actorFullName, cfg); + map.set(cfg.actorRawId, cfg); return map; }, new Map()); @@ -51,8 +51,11 @@ const DEFAULT_TEST_ACTOR_OPTIONS: ActorTestOptions = { timeout: DEFAULT_TEST_RUN_DURATION_MS, }; +/** + * @param actorId - The actor's raw platform ID or its full name (`owner/name`, e.g. `"apify/web-scraper"`). + */ export const testActor = ( - actorName: string, + actorId: string, testName: string, fn: TestFunction<{ run: ReturnType> }>, testOptions?: ActorTestOptions, @@ -61,13 +64,13 @@ export const testActor = ( ...DEFAULT_TEST_ACTOR_OPTIONS, ...testOptions, }; - const name = `${actorName}: ${testName}`; - const shouldRun = !!RUN_ALL_PLATFORM_TESTS || config.has(actorName); + const name = `${actorId}: ${testName}`; + const shouldRun = !!RUN_ALL_PLATFORM_TESTS || config.has(actorId); vitestTest.runIf(shouldRun)(name, options, async (context: TYPE) => { const { expect, ...rest } = context; await fn({ expect: extendExpect(expect), - run: createStartRunFn(actorName, context), + run: createStartRunFn(actorId, context), ...rest, }); }); @@ -79,9 +82,12 @@ export const testActor = ( * * Using task is just current shortcoming of standby feature but ideally we would use Actor directly */ +/** + * @param actorId - The actor's raw platform ID or its full name (`owner/name`, e.g. `"apify/web-scraper"`). + */ // eslint-disable-next-line @typescript-eslint/no-explicit-any export const testStandbyActor = ( - actorName: string, + actorId: string, testName: string, fn: TestFunction<{ callStandby: ReturnType> }>, testOptions?: ActorTestOptions, @@ -90,11 +96,11 @@ export const testStandbyActor = ( ...DEFAULT_TEST_ACTOR_OPTIONS, ...testOptions, }; - const name = `${actorName}: ${testName}`; - const shouldRun = !!RUN_ALL_PLATFORM_TESTS || config.has(actorName); + const name = `${actorId}: ${testName}`; + const shouldRun = !!RUN_ALL_PLATFORM_TESTS || config.has(actorId); vitestTest.runIf(shouldRun)(name, options, async (context: T) => { - const standbyTask = await createStandbyTask(actorName, config.get(actorName)?.buildNumber); + const standbyTask = await createStandbyTask(actorId, config.get(actorId)?.buildNumber); const { expect, ...rest } = context; // NOTE: we wrap `fn` in try/finally so cleanup (deleting the task) always runs afterwards @@ -193,20 +199,20 @@ interface StandbyTask { * * @throws if actor doesn't exist or it doesn't support standby mode. */ -const createStandbyTask = async (actorNameOrId: string, buildNumber?: string): Promise => { - const actor = apifyClient.actor(actorNameOrId); +const createStandbyTask = async (actorId: string, buildNumber?: string): Promise => { + const actor = apifyClient.actor(actorId); const actorInfo = (await actor.get()) as Actor & { standbyUrl?: string }; if (!actorInfo) { - throw new Error(`Actor "${actorNameOrId}" not found`); + throw new Error(`Actor "${actorId}" not found`); } if (!actorInfo.standbyUrl) { - throw new Error(`Actor "${actorNameOrId}" doesn't support standby mode`); + throw new Error(`Actor "${actorId}" doesn't support standby mode`); } if (!actorInfo.actorStandby) { - throw new Error(`Actor "${actorNameOrId} doesn't contain actorStandby options`); + throw new Error(`Actor "${actorId} doesn't contain actorStandby options`); } const { isEnabled, ...defaultActorStandby } = actorInfo.actorStandby; delete defaultActorStandby.disableStandbyFieldsOverride; @@ -225,9 +231,9 @@ const createStandbyTask = async (actorNameOrId: string, buildNumber?: string): P const randomValue = Math.random().toString(10).slice(2).padEnd(randomValueLength, '0'); const name = `test-${randomValue.slice(0, randomValueLength)}`; const newTask = (await apifyClient.tasks().create({ - actId: actorNameOrId, + actId: actorId, actorStandby: actorStandbyOptions, - description: `Task for testing standby version ${build} of actor "${actorNameOrId}"`, + description: `Task for testing standby version ${build} of actor "${actorId}"`, title, name, })) as Task & { standbyUrl?: string }; @@ -247,9 +253,9 @@ const createStandbyTask = async (actorNameOrId: string, buildNumber?: string): P } }; -const createStartRunFn = (actorNameOrId: string, testContext: TestContext) => { +const createStartRunFn = (actorId: string, testContext: TestContext) => { const { annotate, task } = testContext; - const actorConfig = config.get(actorNameOrId); + const actorConfig = config.get(actorId); const build = actorConfig?.buildNumber; const buildId = actorConfig?.buildId; return async (runOptions: RunOptions) => { @@ -263,10 +269,10 @@ const createStartRunFn = (actorNameOrId: string, testContext: TestContext) => return new RunTestResult(apifyClient, run); } - const actor = apifyClient.actor(actorNameOrId); + const actor = apifyClient.actor(actorId); const actorInput = { - ...(prefilledInput && (await getActorPrefilledInput(apifyClient, actorNameOrId, buildId))), + ...(prefilledInput && (await getActorPrefilledInput(apifyClient, actorId, buildId))), ...input, }; const run = await actor.call(actorInput, { build, log: null, ...options }); @@ -277,7 +283,7 @@ const createStartRunFn = (actorNameOrId: string, testContext: TestContext) => task.meta = { runId: run.id, runLink, - actorName: actorNameOrId, + actorId: actorConfig?.actorFullName ?? actorId, }; // waiting for datasetItemCount and chargedEventCounts to sync diff --git a/lib/types.ts b/lib/types.ts index 6b301f3..d231bd5 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -3,9 +3,9 @@ import type { Assertion, TestOptions } from 'vitest'; export type ActorBuild = { buildId: string; - actorId: string; + actorRawId: string; buildNumber: string; - actorName: string; + actorFullName: string; }; export type RunOptions = { diff --git a/lib/utils.ts b/lib/utils.ts index 0034cb2..014299c 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -6,11 +6,11 @@ import type { ApifyClient } from 'apify-client'; */ export const getActorPrefilledInput = async ( apifyClient: ApifyClient, - actorNameOrId: string, + actorId: string, buildId: string | undefined, ) => { if (!buildId) { - const actorInfo = await apifyClient.actor(actorNameOrId).get(); + const actorInfo = await apifyClient.actor(actorId).get(); const defaultBuildTag = actorInfo?.defaultRunOptions.build; @@ -19,7 +19,7 @@ export const getActorPrefilledInput = async ( buildId = taggedBuild?.buildId; if (!buildId) { - console.error(`Coudn't find default build for actor ${actorNameOrId}. Prefilled values will not be used.`); + console.error(`Coudn't find default build for actor ${actorId}. Prefilled values will not be used.`); return {}; } } @@ -34,7 +34,7 @@ export const getActorPrefilledInput = async ( if (!inputSchema) { console.error( - `Coudn't find input schema definition for actor ${actorNameOrId}, build ${buildId}.`, + `Coudn't find input schema definition for actor ${actorId}, build ${buildId}.`, 'Prefilled values will not be used', ); return {}; diff --git a/test/unit/bin/diff-changes.test.ts b/test/unit/bin/diff-changes.test.ts index d6d87db..2379a1f 100644 --- a/test/unit/bin/diff-changes.test.ts +++ b/test/unit/bin/diff-changes.test.ts @@ -6,14 +6,14 @@ import * as Dockerignore from '../../../bin/dockerignore.js'; import type { ActorConfig } from '../../../bin/types.js'; const miniActor: ActorConfig = { - actorName: 'foo/bar', + actorFullName: 'foo/bar', folder: 'actors/foo_bar', tokenEnvVar: 'APIFY_TOKEN_FOO', dockerContextDir: '', contextPaths: [''], }; const standaloneActor: ActorConfig = { - actorName: 'owner/standalone', + actorFullName: 'owner/standalone', folder: 'standalone-actors/standalone', tokenEnvVar: 'APIFY_TOKEN_OWNER', dockerContextDir: 'standalone-actors/standalone', @@ -21,7 +21,7 @@ const standaloneActor: ActorConfig = { }; const actorConfigs = [miniActor, standaloneActor]; const amazonActor: ActorConfig = { - actorName: 'junglee/amazon-crawler', + actorFullName: 'junglee/amazon-crawler', folder: 'actors/junglee_Amazon-crawler', tokenEnvVar: 'APIFY_TOKEN_JUNGLEE', dockerContextDir: '', @@ -183,7 +183,7 @@ describe('getChangedActors', () => { it('matches folder where folder name differs from actor name', () => { const ownerlessActor: ActorConfig = { - actorName: 'myteam/shopify-scraper', + actorFullName: 'myteam/shopify-scraper', folder: 'actors/shopify', tokenEnvVar: 'APIFY_TOKEN_MYTEAM', dockerContextDir: '', @@ -199,7 +199,7 @@ describe('getChangedActors', () => { it('in single-actor repo, .actor/ changes trigger builds', () => { const rootActor: ActorConfig = { - actorName: 'myteam/my-actor', + actorFullName: 'myteam/my-actor', folder: '', tokenEnvVar: 'BUILDER_APIFY_TOKEN', dockerContextDir: '', @@ -233,7 +233,7 @@ describe('getChangedActors', () => { it('triggers actor with contextPaths override when file matches an override path', () => { const overrideActor: ActorConfig = { - actorName: 'team/override-actor', + actorFullName: 'team/override-actor', folder: 'actors/override', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: 'actors/override', @@ -249,7 +249,7 @@ describe('getChangedActors', () => { it('does not trigger actor with contextPaths override when file is outside all override paths', () => { const overrideActor: ActorConfig = { - actorName: 'team/override-actor', + actorFullName: 'team/override-actor', folder: 'actors/override', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: 'actors/override', @@ -265,14 +265,14 @@ describe('getChangedActors', () => { it('broad-context actor skips files in sibling actor folders', () => { const actorA: ActorConfig = { - actorName: 'team/actor-a', + actorFullName: 'team/actor-a', folder: 'actors/a', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: '', contextPaths: [''], }; const actorB: ActorConfig = { - actorName: 'team/actor-b', + actorFullName: 'team/actor-b', folder: 'actors/b', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: '', @@ -288,14 +288,14 @@ describe('getChangedActors', () => { it('root actor (folder="") is excluded from sibling actor folder files', () => { const rootActor: ActorConfig = { - actorName: 'team/root', + actorFullName: 'team/root', folder: '', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: '', contextPaths: [''], }; const childActor: ActorConfig = { - actorName: 'team/child', + actorFullName: 'team/child', folder: 'actors/child', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: 'actors/child', @@ -312,14 +312,14 @@ describe('getChangedActors', () => { it('root actor (folder="") sees files outside any actor folder', () => { const rootActor: ActorConfig = { - actorName: 'team/root', + actorFullName: 'team/root', folder: '', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: '', contextPaths: [''], }; const childActor: ActorConfig = { - actorName: 'team/child', + actorFullName: 'team/child', folder: 'actors/child', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: 'actors/child', @@ -454,14 +454,14 @@ describe('getChangedActors logging', () => { it('logs a single "shared" group when two actors are triggered by the exact same file', () => { const actorA: ActorConfig = { - actorName: 'team/actor-a', + actorFullName: 'team/actor-a', folder: 'actors/a', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: '', contextPaths: ['', 'shared'], }; const actorB: ActorConfig = { - actorName: 'team/actor-b', + actorFullName: 'team/actor-b', folder: 'actors/b', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: '', @@ -483,14 +483,14 @@ describe('getChangedActors logging', () => { it('logs shared and specific groups in descending-size order for partial overlap across actors', () => { const actorA: ActorConfig = { - actorName: 'team/actor-a', + actorFullName: 'team/actor-a', folder: 'actors/a', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: '', contextPaths: [''], }; const actorB: ActorConfig = { - actorName: 'team/actor-b', + actorFullName: 'team/actor-b', folder: 'actors/b', tokenEnvVar: 'APIFY_TOKEN_TEAM', dockerContextDir: '', diff --git a/test/unit/bin/utils.test.ts b/test/unit/bin/utils.test.ts index c8db4d3..4e65208 100644 --- a/test/unit/bin/utils.test.ts +++ b/test/unit/bin/utils.test.ts @@ -30,7 +30,11 @@ describe('readConfigFile', () => { it('returns correct ActorConfig[] for a valid config', async () => { mockFiles({ [CONFIG_FILE_NAME]: validConfig([ - { folder: 'actors/shopify', actorName: 'myteam/shopify-scraper', tokenEnvVar: 'APIFY_TOKEN_MYTEAM' }, + { + folder: 'actors/shopify', + actorFullName: 'myteam/shopify-scraper', + tokenEnvVar: 'APIFY_TOKEN_MYTEAM', + }, ]), 'actors/shopify/.actor/actor.json': actorJson({ dockerContextDir: '../../..' }), }); @@ -39,7 +43,7 @@ describe('readConfigFile', () => { expectFileRead('actors/shopify/.actor/actor.json'); expect(result).toEqual([ { - actorName: 'myteam/shopify-scraper', + actorFullName: 'myteam/shopify-scraper', folder: 'actors/shopify', tokenEnvVar: 'APIFY_TOKEN_MYTEAM', dockerContextDir: '', @@ -51,7 +55,7 @@ describe('readConfigFile', () => { it('normalizes folder "." to ""', async () => { mockFiles({ [CONFIG_FILE_NAME]: validConfig([ - { folder: '.', actorName: 'apify/my-actor', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: '.', actorFullName: 'apify/my-actor', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), '.actor/actor.json': actorJson({}), }); @@ -64,7 +68,7 @@ describe('readConfigFile', () => { it('defaults dockerContextDir to actor folder when absent from actor.json', async () => { mockFiles({ [CONFIG_FILE_NAME]: validConfig([ - { folder: 'actors/web-scraper', actorName: 'apify/web-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 'actors/web-scraper', actorFullName: 'apify/web-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), 'actors/web-scraper/.actor/actor.json': actorJson({}), }); @@ -77,7 +81,7 @@ describe('readConfigFile', () => { it('resolves dockerContextDir relative to .actor/ folder', async () => { mockFiles({ [CONFIG_FILE_NAME]: validConfig([ - { folder: 'actors/shopify', actorName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN' }, + { folder: 'actors/shopify', actorFullName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN' }, ]), 'actors/shopify/.actor/actor.json': actorJson({ dockerContextDir: '../../..' }), }); @@ -91,7 +95,7 @@ describe('readConfigFile', () => { [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', - actorName: 'myteam/shopify', + actorFullName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: ['actors/shopify', 'packages'], }, @@ -106,10 +110,10 @@ describe('readConfigFile', () => { it('handles multiple actors', async () => { mockFiles({ [CONFIG_FILE_NAME]: validConfig([ - { folder: 'actors/web-scraper', actorName: 'apify/web-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 'actors/web-scraper', actorFullName: 'apify/web-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, { folder: 'actors/email-sender', - actorName: 'other-team/email-sender', + actorFullName: 'other-team/email-sender', tokenEnvVar: 'APIFY_TOKEN_OTHER_TEAM', }, ]), @@ -119,8 +123,8 @@ describe('readConfigFile', () => { const result = await readConfigFile(); expect(result).toHaveLength(2); - expect(result[0].actorName).toBe('apify/web-scraper'); - expect(result[1].actorName).toBe('other-team/email-sender'); + expect(result[0].actorFullName).toBe('apify/web-scraper'); + expect(result[1].actorFullName).toBe('other-team/email-sender'); }); it('throws when config file is missing', async () => { @@ -141,8 +145,8 @@ describe('readConfigFile', () => { it('throws on duplicate folders', async () => { mockFiles({ [CONFIG_FILE_NAME]: validConfig([ - { folder: 'actors/shopify', actorName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, - { folder: 'actors/shopify', actorName: 'other/shopify', tokenEnvVar: 'APIFY_TOKEN_OTHER' }, + { folder: 'actors/shopify', actorFullName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 'actors/shopify', actorFullName: 'other/shopify', tokenEnvVar: 'APIFY_TOKEN_OTHER' }, ]), 'actors/shopify/.actor/actor.json': actorJson({}), }); @@ -153,8 +157,8 @@ describe('readConfigFile', () => { it('throws on duplicate folders after normalization ("." and "")', async () => { mockFiles({ [CONFIG_FILE_NAME]: validConfig([ - { folder: '.', actorName: 'apify/actor-a', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, - { folder: '', actorName: 'other/actor-b', tokenEnvVar: 'APIFY_TOKEN_OTHER' }, + { folder: '.', actorFullName: 'apify/actor-a', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: '', actorFullName: 'other/actor-b', tokenEnvVar: 'APIFY_TOKEN_OTHER' }, ]), '.actor/actor.json': actorJson({}), }); @@ -165,7 +169,7 @@ describe('readConfigFile', () => { it('throws when actor.json is missing', async () => { mockFiles({ [CONFIG_FILE_NAME]: validConfig([ - { folder: 'actors/shopify', actorName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 'actors/shopify', actorFullName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), }); @@ -174,7 +178,7 @@ describe('readConfigFile', () => { it('throws when folder is missing', async () => { mockFiles({ - [CONFIG_FILE_NAME]: validConfig([{ actorName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }]), + [CONFIG_FILE_NAME]: validConfig([{ actorFullName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }]), }); await expect(readConfigFile()).rejects.toThrow(/Invalid "folder"/); @@ -183,42 +187,42 @@ describe('readConfigFile', () => { it('throws when folder is not a string', async () => { mockFiles({ [CONFIG_FILE_NAME]: validConfig([ - { folder: 123, actorName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 123, actorFullName: 'apify/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), }); await expect(readConfigFile()).rejects.toThrow(/Invalid "folder"/); }); - it('throws when actorName is missing', async () => { + it('throws when actorFullName is missing', async () => { mockFiles({ [CONFIG_FILE_NAME]: validConfig([{ folder: 'actors/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }]), 'actors/shopify/.actor/actor.json': actorJson({}), }); - await expect(readConfigFile()).rejects.toThrow('Invalid "actorName"'); + await expect(readConfigFile()).rejects.toThrow('Invalid "actorFullName"'); }); - it('throws when actorName has no slash', async () => { + it('throws when actorFullName has no slash', async () => { mockFiles({ [CONFIG_FILE_NAME]: validConfig([ - { folder: 'actors/shopify', actorName: 'shopify-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 'actors/shopify', actorFullName: 'shopify-scraper', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), 'actors/shopify/.actor/actor.json': actorJson({}), }); - await expect(readConfigFile()).rejects.toThrow('Invalid "actorName"'); + await expect(readConfigFile()).rejects.toThrow('Invalid "actorFullName"'); }); - it('throws when actorName has empty parts', async () => { + it('throws when actorFullName has empty parts', async () => { mockFiles({ [CONFIG_FILE_NAME]: validConfig([ - { folder: 'actors/shopify', actorName: '/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, + { folder: 'actors/shopify', actorFullName: '/shopify', tokenEnvVar: 'APIFY_TOKEN_APIFY' }, ]), 'actors/shopify/.actor/actor.json': actorJson({}), }); - await expect(readConfigFile()).rejects.toThrow('Invalid "actorName"'); + await expect(readConfigFile()).rejects.toThrow('Invalid "actorFullName"'); }); it('throws when overrideActorContext is not an array', async () => { @@ -226,7 +230,7 @@ describe('readConfigFile', () => { [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', - actorName: 'myteam/shopify', + actorFullName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: 'packages', }, @@ -242,7 +246,7 @@ describe('readConfigFile', () => { [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', - actorName: 'myteam/shopify', + actorFullName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: [123], }, @@ -258,7 +262,7 @@ describe('readConfigFile', () => { [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', - actorName: 'myteam/shopify', + actorFullName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: ['actors/shopify', 'actors'], }, @@ -274,7 +278,7 @@ describe('readConfigFile', () => { [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', - actorName: 'myteam/shopify', + actorFullName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: ['', 'actors/shopify'], }, @@ -290,7 +294,7 @@ describe('readConfigFile', () => { [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', - actorName: 'myteam/shopify', + actorFullName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: ['code', 'shared'], }, @@ -306,7 +310,7 @@ describe('readConfigFile', () => { [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify/', - actorName: 'myteam/shopify', + actorFullName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: ['actors/shopify/', 'packages/'], }, @@ -324,7 +328,7 @@ describe('readConfigFile', () => { [CONFIG_FILE_NAME]: validConfig([ { folder: 'actors/shopify', - actorName: 'myteam/shopify', + actorFullName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN', overrideActorContext: ['actors/shopify', 'code', 'shared'], }, diff --git a/test/unit/should-built-and-test.test.ts b/test/unit/should-built-and-test.test.ts index c8935f0..a93083b 100644 --- a/test/unit/should-built-and-test.test.ts +++ b/test/unit/should-built-and-test.test.ts @@ -9,21 +9,21 @@ describe('Should build and test parser', () => { // From https://github.com/apify-store/testing-repo-for-github-actions const ACTOR_CONFIGS: ActorConfig[] = [ { - actorName: 'lukaskrivka/testing-github-integration-1', + actorFullName: 'lukaskrivka/testing-github-integration-1', folder: 'actors/lukaskrivka_testing-github-integration-1', tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: '', contextPaths: [''], }, { - actorName: 'lukaskrivka/testing-github-integration-2', + actorFullName: 'lukaskrivka/testing-github-integration-2', folder: 'actors/lukaskrivka_testing-github-integration-2', tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: '', contextPaths: [''], }, { - actorName: 'lukaskrivka/test-standalone', + actorFullName: 'lukaskrivka/test-standalone', folder: 'standalone-actors/lukaskrivka_test-standalone', tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: 'standalone-actors/lukaskrivka_test-standalone', @@ -281,63 +281,63 @@ describe('Should build and test parser', () => { const ACTOR_CONFIGS_GOOGLE_MAPS: ActorConfig[] = [ { // Edge case of capitals in actor name :) - actorName: 'compass/Google-Maps-Reviews-Scraper', + actorFullName: 'compass/Google-Maps-Reviews-Scraper', folder: 'actors/compass_Google-Maps-Reviews-Scraper', tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', contextPaths: [''], }, { - actorName: 'compass/crawler-google-places', + actorFullName: 'compass/crawler-google-places', folder: 'actors/compass_crawler-google-places', tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', contextPaths: [''], }, { - actorName: 'compass/easy-google-maps', + actorFullName: 'compass/easy-google-maps', folder: 'actors/compass_easy-google-maps', tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', contextPaths: [''], }, { - actorName: 'compass/google-maps-extractor', + actorFullName: 'compass/google-maps-extractor', folder: 'actors/compass_google-maps-extractor', tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', contextPaths: [''], }, { - actorName: 'compass/google-places-api', + actorFullName: 'compass/google-places-api', folder: 'actors/compass_google-places-api', tokenEnvVar: 'APIFY_TOKEN_COMPASS', dockerContextDir: '', contextPaths: [''], }, { - actorName: 'lukaskrivka/google-maps-with-contact-details', + actorFullName: 'lukaskrivka/google-maps-with-contact-details', folder: 'actors/lukaskrivka_google-maps-with-contact-details', tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: '', contextPaths: [''], }, { - actorName: 'natasha.lekh/gas-prices-scraper', + actorFullName: 'natasha.lekh/gas-prices-scraper', folder: 'actors/natasha.lekh_gas-prices-scraper', tokenEnvVar: 'APIFY_TOKEN_NATASHA_LEKH', dockerContextDir: '', contextPaths: [''], }, { - actorName: 'natasha.lekh/vegan-places-finder', + actorFullName: 'natasha.lekh/vegan-places-finder', folder: 'actors/natasha.lekh_vegan-places-finder', tokenEnvVar: 'APIFY_TOKEN_NATASHA_LEKH', dockerContextDir: '', contextPaths: [''], }, { - actorName: 'lukaskrivka/google-maps-scraper-orchestrator', + actorFullName: 'lukaskrivka/google-maps-scraper-orchestrator', folder: 'standalone-actors/lukaskrivka_google-maps-scraper-orchestrator', tokenEnvVar: 'APIFY_TOKEN_LUKASKRIVKA', dockerContextDir: 'standalone-actors/lukaskrivka_google-maps-scraper-orchestrator', From 11eb3b210847c534ea23d0ff7e50aee29cce3724 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Thu, 16 Jul 2026 16:22:13 +0100 Subject: [PATCH 29/33] simpler apifyBuilder --- bin/build.ts | 20 ++++++++++++-------- bin/types.ts | 1 - 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/bin/build.ts b/bin/build.ts index 699dcb6..3dd960b 100644 --- a/bin/build.ts +++ b/bin/build.ts @@ -55,7 +55,6 @@ class ApifyBuilder { buildTag, versionNumber, gitRepoUrl, - actorConfig, useDockerCache, }: BuildPrActorOptions): Promise => { const actorClient = this.apifyClient.actor(this.actorFullName); @@ -93,15 +92,15 @@ class ApifyBuilder { const { id, actId, buildNumber } = await actorClient.build(versionNumber, { useCache: useDockerCache }); console.error(`[${this.actorFullName}]: ${id} (${buildNumber})`); - return { buildId: id, actorRawId: actId, buildNumber, actorFullName: this.actorFullName, actorConfig }; + return { buildId: id, actorRawId: actId, buildNumber, actorFullName: this.actorFullName }; }; - waitForBuildToFinish = async (buildId: string, actorFullName: string): Promise => { + waitForBuildToFinish = async (buildId: string): Promise => { const build = await this.apifyClient.build(buildId).waitForFinish(); const versionNumber = build.buildNumber; if (build.status === 'FAILED' || build.status === 'TIMED-OUT') { const message = - `[BUILD][${actorFullName}]: Build ${buildId} (${versionNumber}) failed. ` + + `[BUILD][${this.actorFullName}]: Build ${buildId} (${versionNumber}) failed. ` + `Not continuing with other builds and tests.`; console.error(`[${this.actorFullName}]: ${versionNumber}`); throw new Error(message); @@ -110,7 +109,8 @@ class ApifyBuilder { return build; }; - static fromActorConfig = ({ actorFullName, tokenEnvVar }: ActorConfig): ApifyBuilder => { + static fromActorConfig = (actorConfig: ActorConfig): ApifyBuilder => { + const { actorFullName, tokenEnvVar } = actorConfig; const token = process.env[tokenEnvVar]; if (!token) { throw new Error(`Env var ${tokenEnvVar} is not set (needed for actor "${actorFullName}").`); @@ -254,11 +254,15 @@ export const runBuilds = async ({ if (dryRun) { return buildConfigs; } + + const buildersByActorFullName = new Map( + actorConfigs.map((actorConfig) => [actorConfig.actorFullName, ApifyBuilder.fromActorConfig(actorConfig)]), + ); console.error('========================================='); console.error('STARTED BUILDS:'); const startedBuilds = await Promise.all( buildConfigs.map(async (buildConfig) => { - const builder = ApifyBuilder.fromActorConfig(buildConfig.actorConfig); + const builder = buildersByActorFullName.get(buildConfig.actorConfig.actorFullName)!; const buildData = await builder.startActorBuild(buildConfig); return buildData; }), @@ -267,8 +271,8 @@ export const runBuilds = async ({ console.error('FINISHED BUILDS:'); await Promise.all( startedBuilds.map(async (buildData) => { - const builder = ApifyBuilder.fromActorConfig(buildData.actorConfig); - await builder.waitForBuildToFinish(buildData.buildId, buildData.actorFullName); + const builder = buildersByActorFullName.get(buildData.actorFullName)!; + await builder.waitForBuildToFinish(buildData.buildId); }), ); console.error('========================================='); diff --git a/bin/types.ts b/bin/types.ts index b10c8d0..7baa85c 100644 --- a/bin/types.ts +++ b/bin/types.ts @@ -100,7 +100,6 @@ export interface BuildData { buildId: string; actorRawId: string; actorFullName: string; - actorConfig: ActorConfig; buildNumber: string; } From 48b7847455a2acf4f935c26b200320270c935a29 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Mon, 20 Jul 2026 10:44:48 +0100 Subject: [PATCH 30/33] dockerignore reconciliation --- bin/build-from-local.ts | 48 ++++--- bin/diff-changes.ts | 19 ++- bin/dockerignore.ts | 39 ++++-- bin/utils.ts | 61 --------- test/unit/bin/build-from-local.test.ts | 183 +++++++++---------------- test/unit/bin/dockerignore.test.ts | 54 +++++++- 6 files changed, 169 insertions(+), 235 deletions(-) diff --git a/bin/build-from-local.ts b/bin/build-from-local.ts index df74b54..cdb26a0 100644 --- a/bin/build-from-local.ts +++ b/bin/build-from-local.ts @@ -5,14 +5,10 @@ import path from 'node:path'; import type { ActorVersionSourceFile } from 'apify-client'; import { ApifyBuilder, waitAndSummarizeBuilds } from './build.js'; +import { buildDockerIgnoreMatcher } from './dockerignore.js'; +import { isPathWithinScope } from './path-utils.js'; import type { ActorConfig, BuildData } from './types.js'; -import { - getDockerignoredPaths, - getGitignoredPaths, - isOutsideDir, - listRepoFilePaths, - toActorVersionSourceFile, -} from './utils.js'; +import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, toActorVersionSourceFile } from './utils.js'; // JUST IN CASE. File patterns that commonly hold credentials — never ship these into a build, regardless // of sourceType or of whether the repo's .gitignore happens to list them. Everything else that should be @@ -34,11 +30,13 @@ export const collectSourceFiles = async (actorName: string, actorDir: string): P const contextAbsDir = rawContextDir ? path.resolve(absActorDir, '.actor', rawContextDir) : undefined; const isMonorepoActor = !!contextAbsDir && isOutsideDir(contextAbsDir, absActorDir); - const collectRootDir = isMonorepoActor ? contextAbsDir! : absActorDir; - const keptFilePaths = collectNonIgnoredFiles(collectRootDir, repoRoot); + const dockerContextDirAbs = isMonorepoActor ? contextAbsDir! : absActorDir; + const keptFilePaths = collectNonIgnoredFiles(dockerContextDirAbs, repoRoot); if (!isMonorepoActor) { - return Promise.all(keptFilePaths.map(async (filePath) => toActorVersionSourceFile(filePath, collectRootDir))); + return Promise.all( + keptFilePaths.map(async (filePath) => toActorVersionSourceFile(filePath, dockerContextDirAbs)), + ); } const { tempDir, filePaths } = await flattenMonorepoContext( @@ -47,7 +45,6 @@ export const collectSourceFiles = async (actorName: string, actorDir: string): P contextAbsDir!, actorJson, keptFilePaths, - repoRoot, ); try { return await Promise.all(filePaths.map(async (filePath) => toActorVersionSourceFile(filePath, tempDir))); @@ -61,21 +58,21 @@ export const collectSourceFiles = async (actorName: string, actorDir: string): P // manual directory walk — nested .gitignore files, `.git/info/exclude`, and global excludes are // all honored since this delegates to git itself instead of re-implementing gitignore matching, // and .git/ is never walked because git never lists its own internals here. `.dockerignore` at -// `rootDir` (the Docker build context) is honored the same way, since those files would never -// reach a real Docker build either. `.actor/` (the Actor specification folder) is always kept +// `dockerContextDir` (the Docker build context) is honored the same way, since those files would +// never reach a real Docker build either. `.actor/` (the Actor specification folder) is always kept // regardless of .gitignore/.dockerignore, matching Apify CLI's own behavior. Files matching the // hardcoded secret-pattern backstop (keys, certs, .env variants) are dropped unconditionally, // .actor/ included, since those should never ship regardless of what the ignore files say. -export const collectNonIgnoredFiles = (rootDir: string, repoRoot: string): string[] => { - const relativePaths = listRepoFilePaths(repoRoot, rootDir); +export const collectNonIgnoredFiles = (dockerContextDir: string, repoRoot: string): string[] => { + const relativePaths = listRepoFilePaths(repoRoot, dockerContextDir); const ignoredPaths = getGitignoredPaths(relativePaths); const rootRelativePaths = new Map( relativePaths.map((relPath) => [ relPath, - path.relative(rootDir, path.join(repoRoot, relPath)).split(path.sep).join('/'), + path.relative(dockerContextDir, path.join(repoRoot, relPath)).split(path.sep).join('/'), ]), ); - const dockerIgnoredPaths = getDockerignoredPaths(rootDir, [...rootRelativePaths.values()]); + const isDockerIgnored = buildDockerIgnoreMatcher(dockerContextDir); return relativePaths .filter((relPath) => { @@ -83,7 +80,7 @@ export const collectNonIgnoredFiles = (rootDir: string, repoRoot: string): strin const isUnderActorDir = relPath.split('/').includes('.actor'); if (isUnderActorDir) return true; if (ignoredPaths.has(relPath)) return false; - return !dockerIgnoredPaths.has(rootRelativePaths.get(relPath)!); + return !isDockerIgnored(rootRelativePaths.get(relPath)!); }) .map((relPath) => path.join(repoRoot, relPath)); }; @@ -92,8 +89,8 @@ export const collectNonIgnoredFiles = (rootDir: string, repoRoot: string): strin // collect the actor directory of a monorepo actor — the platform would reject any path // escaping it. Fix: create a temporary "flattened" directory where: // - the Docker context's non-ignored files (repo root) are copied to the temp dir root -// - the actor's .actor/ directory is overlaid at the temp dir root (through the same -// gitignore/secret-pattern filter as the rest of the context — see collectNonIgnoredFiles) +// - the actor's own .actor/ directory (already present within those same non-ignored files) +// is overlaid at the temp dir root instead of its original nested position // - actor.json path fields are rewritten to be relative to the new location // // Result: the collected root IS the Docker context, .actor/ is at that root, and @@ -105,7 +102,6 @@ export const flattenMonorepoContext = async ( contextAbsDir: string, actorJson: Record, keptContextFiles: string[], - repoRoot: string, ): Promise<{ tempDir: string; filePaths: string[] }> => { console.error(`[${actorName}]: monorepo actor detected — flattening from Docker context`); @@ -124,11 +120,13 @@ export const flattenMonorepoContext = async ( }), ); - // Step 2: overlay the actor's .actor/ directory at the temp dir root. collectNonIgnoredFiles - // always keeps .actor/ paths regardless of .gitignore, but still drops the hardcoded secret - // patterns — so this isn't a raw copy, a stray secret file living inside .actor/ is still dropped. + // Step 2: overlay the actor's own .actor/ directory at the temp dir root. Its files are already + // present in keptContextFiles (collectNonIgnoredFiles keeps .actor/ paths unconditionally — see + // step 1 above) — pick out this actor's own subset (ignoring any sibling actors' .actor/ folders + // that might also appear in the broader context) and hoist each to its position relative to + // .actor/ itself, so it lands under tempDir/.actor/ instead of its original nested location. const actorMetaDir = path.join(absActorDir, '.actor'); - const keptActorFiles = collectNonIgnoredFiles(actorMetaDir, repoRoot); + const keptActorFiles = keptContextFiles.filter((absFilePath) => isPathWithinScope(absFilePath, actorMetaDir)); await Promise.all( keptActorFiles.map(async (absFilePath) => { const relPath = path.relative(actorMetaDir, absFilePath); diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index 7872f2c..8930688 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -36,7 +36,8 @@ type FileChangeForActor = * Steps (in order): * 1. Context matching (actorConfig.contextPaths) → outside-context if no match * 2. Hardcoded ignore list, checked against the path hoisted relative to the matched context entry → ignored - * 3. .dockerignore filtering (patterns relative to dockerContextDir) → ignored if matched + * 3. .dockerignore filtering (patterns relative to dockerContextDir), skipped for the actor's own `.actor/` + * dir → ignored if matched * 4. README/CHANGELOG by filename → cosmetic if inside the actor's own folder, otherwise ignored * 5. .json inside the actor's own `.actor/` dir with only cosmetic schema diffs → cosmetic (semantically verified) * 6. Everything else → functional @@ -60,20 +61,24 @@ const classifyFileChange = ( return { impact: 'ignored' }; } - if (dockerIgnoreMatcher(originalFilePath)) { + const lowercaseFolder = actorConfig.folder.toLowerCase(); + const actorDotDir = lowercaseFolder ? `${lowercaseFolder}/.actor` : '.actor'; + const isUnderActorDotDir = isPathWithinScope(lowercaseFilePath, actorDotDir); + + // .actor/ can legitimately be listed in .dockerignore (the Apify platform evaluates it before + // the Docker build, so excluding it from the build context is a valid caching optimization) — + // that shouldn't cause changes to .actor/ itself to be ignored here. + if (!isUnderActorDotDir && dockerIgnoreMatcher(originalFilePath)) { return { impact: 'ignored' }; } - const lowerFolder = actorConfig.folder.toLowerCase(); - const isInActorFolder = isPathWithinScope(lowercaseFilePath, lowerFolder); + const isInActorFolder = isPathWithinScope(lowercaseFilePath, lowercaseFolder); if (lowercaseFilePath.endsWith('readme.md') || lowercaseFilePath.endsWith('changelog.md')) { return isInActorFolder ? { impact: 'cosmetic', semanticallyVerified: false } : { impact: 'ignored' }; } - const actorDotDir = lowerFolder ? `${lowerFolder}/.actor` : '.actor'; - - if (lowercaseFilePath.endsWith('.json') && isPathWithinScope(lowercaseFilePath, actorDotDir)) { + if (lowercaseFilePath.endsWith('.json') && isUnderActorDotDir) { const isCosmetic = isCosmeticOnlyJsonSchemaChange(commits, originalFilePath); if (isCosmetic) { return { impact: 'cosmetic', semanticallyVerified: true }; diff --git a/bin/dockerignore.ts b/bin/dockerignore.ts index c516571..d12b310 100644 --- a/bin/dockerignore.ts +++ b/bin/dockerignore.ts @@ -7,33 +7,44 @@ import { hoistPath, isPathWithinScope } from './path-utils.js'; export type DockerIgnoreMatcher = (repoRelativePath: string) => boolean; +// Docker normalizes each pattern before matching, so a leading "./" (as in the common "./node_modules" +// style) is a no-op for Docker. The `ignore` package has no such normalization — it treats "./" as +// literal pattern text that can never match a real path, so a .dockerignore written in that style +// would otherwise silently match nothing. Strip it here (after any negation prefix) so the pattern +// behaves the way Docker itself would apply it. +const normalizeDockerignorePattern = (line: string): string => line.replace(/^(!?)(?:\.\/)+/, '$1'); + /** - * Load .dockerignore from the root of an actor's dockerContextDir and return a matcher - * that accepts repo-root-relative file paths. Patterns are resolved relative to - * dockerContextDir, matching Docker's own behavior. + * Reads `.dockerignore` from `absoluteRootDir` and returns a matcher for paths relative to + * `hoistFrom`. `hoistFrom` defaults to '', meaning callers already pass paths relative to + * `absoluteRootDir` directly — isPathWithinScope/hoistPath both treat '' as "matches everything" / + * identity, so the scope-check and hoist collapse to a no-op in that case, not via a branch. * * Returns a no-op matcher (always returns false) when the file is absent. */ -export const loadDockerIgnore = (dockerContextDir: string): DockerIgnoreMatcher => { - const dockerignorePath = dockerContextDir ? path.join(dockerContextDir, '.dockerignore') : '.dockerignore'; - +export const buildDockerIgnoreMatcher = (absoluteRootDir: string, hoistFrom = ''): DockerIgnoreMatcher => { let content: string; try { - content = fs.readFileSync(dockerignorePath, 'utf-8'); + content = fs.readFileSync(path.join(absoluteRootDir, '.dockerignore'), 'utf-8'); } catch { return () => false; } - const matcher = ignore().add(content); - - return (repoRelativePath: string): boolean => { - const lowerPath = repoRelativePath.toLowerCase(); - const lowerContext = dockerContextDir.toLowerCase(); + const matcher = ignore().add(content.split('\n').map(normalizeDockerignorePattern).join('\n')); - if (!isPathWithinScope(lowerPath, lowerContext)) { + return (filePath: string): boolean => { + if (!isPathWithinScope(filePath.toLowerCase(), hoistFrom.toLowerCase())) { return false; } - return matcher.ignores(hoistPath(repoRelativePath, dockerContextDir)); + return matcher.ignores(hoistPath(filePath, hoistFrom)); }; }; + +/** + * Load .dockerignore from the root of an actor's dockerContextDir and return a matcher + * that accepts repo-root-relative file paths. Patterns are resolved relative to + * dockerContextDir, matching Docker's own behavior. + */ +export const loadDockerIgnore = (dockerContextDir: string): DockerIgnoreMatcher => + buildDockerIgnoreMatcher(path.resolve(dockerContextDir), dockerContextDir); diff --git a/bin/utils.ts b/bin/utils.ts index db54b2a..b6fa1f5 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -1,7 +1,5 @@ import { spawnSync } from 'node:child_process'; -import fsSync from 'node:fs'; import fs from 'node:fs/promises'; -import os from 'node:os'; import path from 'node:path'; import type { ActorVersionSourceFile } from 'apify-client'; @@ -60,65 +58,6 @@ export const getGitignoredPaths = (relativePaths: string[]): Set => { return new Set(result.stdout.toString().split('\n').filter(Boolean)); }; -// Docker normalizes each pattern before matching, so a leading "./" (as in the common "./node_modules" -// style) is a no-op for Docker. git's ignore engine has no such normalization — it treats "./" as -// literal pattern text that can never match a real path, so a .dockerignore written in that style -// would otherwise silently match nothing under git. Strip it here (after any negation prefix) so -// git sees the same effective pattern Docker would. -const normalizeDockerignorePattern = (line: string): string => line.replace(/^(!?)(?:\.\/)+/, '$1'); - -/** - * Given paths relative to `rootDir`, returns the subset that a `.dockerignore` file sitting at - * `rootDir` would exclude — mirroring getGitignoredPaths, but for Docker's own ignore file. `git - * check-ignore` has no built-in notion of .dockerignore, but pointing `core.excludesFile` at a - * normalized copy of it for a single invocation makes git apply its patterns exactly like a - * .gitignore, without re-implementing gitignore-style pattern matching ourselves. A missing - * .dockerignore is a no-op (empty set), so there's no need to check for its existence first. - * - * Crucially this passes `--no-index`: by default git never reports an already-tracked file as - * ignored (that's how real .gitignore semantics work — tracked files aren't affected by ignore - * rules), but Docker excludes a matching path from the build context unconditionally, regardless - * of git tracking. `--no-index` makes git apply the patterns uniformly, matching Docker's behavior. - */ -export const getDockerignoredPaths = (rootDir: string, relativePaths: string[]): Set => { - if (relativePaths.length === 0) return new Set(); - - let dockerignoreContent: string; - try { - dockerignoreContent = fsSync.readFileSync(path.join(rootDir, '.dockerignore'), 'utf8'); - } catch { - return new Set(); - } - - const normalizedContent = dockerignoreContent.split('\n').map(normalizeDockerignorePattern).join('\n'); - - const tempDir = fsSync.mkdtempSync(path.join(os.tmpdir(), 'apify-dockerignore-')); - try { - const normalizedPath = path.join(tempDir, '.dockerignore'); - fsSync.writeFileSync(normalizedPath, normalizedContent); - - const result = spawnSync( - 'git', - ['-c', `core.excludesFile=${normalizedPath}`, 'check-ignore', '--no-index', '--stdin'], - { - cwd: rootDir, - input: relativePaths.join('\n'), - maxBuffer: 100 * 1024 * 1024, - }, - ); - - // Exit code 1 means none of the given paths are ignored - not an error. Anything else - // (e.g. 128 for "not a git repository") is a real failure. - if (result.status !== 0 && result.status !== 1) { - throw new Error(`[Command failed]: git check-ignore (dockerignore)\n${result.stderr.toString()}`); - } - - return new Set(result.stdout.toString().split('\n').filter(Boolean)); - } finally { - fsSync.rmSync(tempDir, { recursive: true, force: true }); - } -}; - const isBinary = (buffer: Buffer): boolean => buffer.includes(0); export const toActorVersionSourceFile = async (absPath: string, rootDir: string): Promise => { diff --git a/test/unit/bin/build-from-local.test.ts b/test/unit/bin/build-from-local.test.ts index 31aa0ac..ade282d 100644 --- a/test/unit/bin/build-from-local.test.ts +++ b/test/unit/bin/build-from-local.test.ts @@ -8,6 +8,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { collectNonIgnoredFiles, + collectSourceFiles, flattenMonorepoContext, rewriteActorJsonPaths, // eslint-disable-next-line @typescript-eslint/ban-ts-comment @@ -62,86 +63,76 @@ describe('build-from-local helpers', () => { }); describe('flattenMonorepoContext', () => { - it('filters the .actor/ overlay through the same secret-pattern check as the rest of the context', async () => { + it("overlays the actor's own .actor/ files at the flattened root", async () => { + // Simulates flattening from a monorepo with an actors//.actor structure. const repoRoot = await mkTempDir('apify-test-tools-repo-'); tempDirs.push(repoRoot); - initGitRepo(repoRoot); const absActorDir = path.join(repoRoot, 'actors', 'owner_actor'); - await fs.mkdir(path.join(absActorDir, '.actor'), { recursive: true }); - await fs.writeFile( - path.join(absActorDir, '.actor', 'actor.json'), - JSON.stringify({ actorSpecification: 1, name: 'actor' }), - ); - // A stray secret file inside .actor/ must never survive into the build. - await fs.writeFile(path.join(absActorDir, '.actor', '.env'), 'SECRET=leaked'); - await fs.writeFile(path.join(absActorDir, '.actor', 'INPUT_SCHEMA.json'), '{}'); + const actorJsonPath = path.join(absActorDir, '.actor', 'actor.json'); + const inputSchemaPath = path.join(absActorDir, '.actor', 'INPUT_SCHEMA.json'); + const mainSrcPath = path.join(absActorDir, 'src', 'main.ts'); + const packageJsonPath = path.join(repoRoot, 'package.json'); - const keptContextFile = path.join(repoRoot, 'package.json'); - await fs.writeFile(keptContextFile, '{}'); + await fs.mkdir(path.join(absActorDir, '.actor'), { recursive: true }); + await fs.mkdir(path.join(absActorDir, 'src'), { recursive: true }); + await fs.writeFile(actorJsonPath, JSON.stringify({ actorSpecification: 1, name: 'actor' })); + await fs.writeFile(inputSchemaPath, '{}'); + await fs.writeFile(mainSrcPath, 'console.log(1)'); + await fs.writeFile(packageJsonPath, '{}'); - // Nothing is gitignored here — isolates the assertion to the secret-pattern filter. - vi.spyOn(Utils, 'getGitignoredPaths').mockReturnValue(new Set()); + const keptContextFiles = [packageJsonPath, mainSrcPath, actorJsonPath, inputSchemaPath]; - const actorJson = JSON.parse( - await fs.readFile(path.join(absActorDir, '.actor', 'actor.json'), 'utf8'), - ) as Record; + const actorJson = JSON.parse(await fs.readFile(actorJsonPath, 'utf8')) as Record; const { tempDir: flattenedDir, filePaths } = await flattenMonorepoContext( 'test/actor', absActorDir, repoRoot, actorJson, - [keptContextFile], - repoRoot, + keptContextFiles, ); tempDirs.push(flattenedDir); - await expect(fs.access(path.join(flattenedDir, '.actor', '.env'))).rejects.toThrow(); - await expect(fs.access(path.join(flattenedDir, '.actor', 'INPUT_SCHEMA.json'))).resolves.toBeUndefined(); + const flattenedMainSrcPath = path.join(flattenedDir, 'actors', 'owner_actor', 'src', 'main.ts'); await expect(fs.access(path.join(flattenedDir, '.actor', 'actor.json'))).resolves.toBeUndefined(); + await expect(fs.access(path.join(flattenedDir, '.actor', 'INPUT_SCHEMA.json'))).resolves.toBeUndefined(); await expect(fs.access(path.join(flattenedDir, 'package.json'))).resolves.toBeUndefined(); - - // The returned filePaths must match what actually landed on disk — no .env, everything else present. - expect(new Set(filePaths)).toStrictEqual( - new Set([ - path.join(flattenedDir, 'package.json'), - path.join(flattenedDir, '.actor', 'INPUT_SCHEMA.json'), - path.join(flattenedDir, '.actor', 'actor.json'), - ]), - ); + await expect(fs.access(flattenedMainSrcPath)).resolves.toBeUndefined(); + expect(filePaths).toContain(path.join(flattenedDir, '.actor', 'actor.json')); + expect(filePaths).toContain(path.join(flattenedDir, '.actor', 'INPUT_SCHEMA.json')); + expect(filePaths).toContain(flattenedMainSrcPath); }); it("keeps another actor's .actor/ directory intact at its original nested path", async () => { + // Simulates flattening from a monorepo with an actors//.actor structure — two + // sibling actors share the same context, but only one is being flattened here. const repoRoot = await mkTempDir('apify-test-tools-repo-'); tempDirs.push(repoRoot); - initGitRepo(repoRoot); const absActorDir = path.join(repoRoot, 'actors', 'owner_actor'); - await fs.mkdir(path.join(absActorDir, '.actor'), { recursive: true }); - await fs.writeFile( - path.join(absActorDir, '.actor', 'actor.json'), - JSON.stringify({ actorSpecification: 1, name: 'actor' }), - ); - + const actorJsonPath = path.join(absActorDir, '.actor', 'actor.json'); const otherActorDir = path.join(repoRoot, 'actors', 'owner_other-actor'); - await fs.mkdir(path.join(otherActorDir, '.actor'), { recursive: true }); const otherActorSchemaFile = path.join(otherActorDir, '.actor', 'input_schema.json'); + + await fs.mkdir(path.join(absActorDir, '.actor'), { recursive: true }); + await fs.writeFile(actorJsonPath, JSON.stringify({ actorSpecification: 1, name: 'actor' })); + await fs.mkdir(path.join(otherActorDir, '.actor'), { recursive: true }); await fs.writeFile(otherActorSchemaFile, JSON.stringify({ schema: 'other' })); - vi.spyOn(Utils, 'getGitignoredPaths').mockReturnValue(new Set()); + // Stands in for collectNonIgnoredFiles's already-filtered output: the current actor's own + // actor.json (always kept by collectNonIgnoredFiles's .actor/ bypass in the real flow — + // see the collectSourceFiles test below) plus the other actor's file. + const keptContextFiles = [actorJsonPath, otherActorSchemaFile]; - const actorJson = JSON.parse( - await fs.readFile(path.join(absActorDir, '.actor', 'actor.json'), 'utf8'), - ) as Record; + const actorJson = JSON.parse(await fs.readFile(actorJsonPath, 'utf8')) as Record; const { tempDir: flattenedDir, filePaths } = await flattenMonorepoContext( 'test/actor', absActorDir, repoRoot, actorJson, - [otherActorSchemaFile], - repoRoot, + keptContextFiles, ); tempDirs.push(flattenedDir); @@ -151,6 +142,33 @@ describe('build-from-local helpers', () => { }); }); + describe('collectSourceFiles', () => { + it("always collects the actor's own .actor/actor.json for a monorepo actor, since flattenMonorepoContext depends on it", async () => { + const repoRoot = await mkTempDir('apify-test-tools-collect-source-'); + tempDirs.push(repoRoot); + initGitRepo(repoRoot); + + const originalCwd = process.cwd(); + // We simulate the working directory being the repo root, since collectSourceFiles uses relative paths to the repo root. + Utils.setCwd({ workspace: repoRoot }); + try { + const cwd = process.cwd(); + const actorDir = path.join(cwd, 'actors', 'owner_actor'); + await fs.mkdir(path.join(actorDir, '.actor'), { recursive: true }); + await fs.writeFile( + path.join(actorDir, '.actor', 'actor.json'), + JSON.stringify({ actorSpecification: 1, name: 'actor', dockerContextDir: '../../..' }), + ); + await fs.writeFile(path.join(cwd, 'package.json'), '{}'); + + const sourceFiles = await collectSourceFiles('owner/actor', actorDir); + expect(sourceFiles.map((file) => file.name)).toContain('.actor/actor.json'); + } finally { + process.chdir(originalCwd); + } + }); + }); + describe('rewriteActorJsonPaths', () => { it('rewrites path fields that escape .actor/ relative to the new flattened location', async () => { const repoRoot = await mkTempDir('apify-test-tools-rewrite-'); @@ -182,83 +200,6 @@ describe('build-from-local helpers', () => { }); }); -describe('getDockerignoredPaths', () => { - const tempDirs: string[] = []; - - afterEach(async () => { - vi.mocked(spawnSync).mockClear(); - await Promise.all(tempDirs.splice(0).map(async (dir) => fs.rm(dir, { recursive: true, force: true }))); - }); - - it('returns an empty set without calling git when given no paths', () => { - const result = Utils.getDockerignoredPaths('/does/not/matter', []); - - expect(result).toStrictEqual(new Set()); - expect(spawnSync).not.toHaveBeenCalled(); - }); - - it('returns an empty set when rootDir has no .dockerignore', async () => { - const rootDir = await mkTempDir('apify-test-tools-dockerignore-'); - tempDirs.push(rootDir); - initGitRepo(rootDir); - - expect(Utils.getDockerignoredPaths(rootDir, ['main.js'])).toStrictEqual(new Set()); - }); - - it('returns the paths that match .dockerignore patterns rooted at rootDir', async () => { - const rootDir = await mkTempDir('apify-test-tools-dockerignore-'); - tempDirs.push(rootDir); - initGitRepo(rootDir); - - await fs.writeFile(path.join(rootDir, '.dockerignore'), 'test/\n*.log\n'); - - const result = Utils.getDockerignoredPaths(rootDir, ['main.js', 'test/fixture.json', 'debug.log']); - - expect(result).toStrictEqual(new Set(['test/fixture.json', 'debug.log'])); - }); - - it('matches "./"-prefixed patterns, which Docker treats as a no-op but git treats as literal text', async () => { - const rootDir = await mkTempDir('apify-test-tools-dockerignore-'); - tempDirs.push(rootDir); - initGitRepo(rootDir); - - // The common real-world .dockerignore style: every pattern prefixed with "./", - // including a negation re-including one of the otherwise-matched files. - await fs.writeFile(path.join(rootDir, '.dockerignore'), './node_modules\n./*.log\n!./keep.log\n'); - - const result = Utils.getDockerignoredPaths(rootDir, [ - 'node_modules/foo.js', - 'debug.log', - 'keep.log', - 'main.js', - ]); - - expect(result).toStrictEqual(new Set(['node_modules/foo.js', 'debug.log'])); - }); - - it('matches already-tracked files too, since Docker excludes them regardless of git tracking', async () => { - const rootDir = await mkTempDir('apify-test-tools-dockerignore-'); - tempDirs.push(rootDir); - initGitRepo(rootDir); - spawnSync('git', ['config', 'user.email', 'test@example.com'], { cwd: rootDir }); - spawnSync('git', ['config', 'user.name', 'Test'], { cwd: rootDir }); - - // Husky hooks are committed to the repo, so this path is tracked by git — by default - // `git check-ignore` never reports a tracked file as ignored, which would otherwise mask - // this exact pattern from matching, unlike a real Docker build which excludes it anyway. - await fs.mkdir(path.join(rootDir, '.husky')); - await fs.writeFile(path.join(rootDir, '.husky', 'pre-commit'), '#!/bin/sh\n'); - spawnSync('git', ['add', '.husky/pre-commit'], { cwd: rootDir }); - spawnSync('git', ['commit', '-q', '-m', 'add husky hook'], { cwd: rootDir }); - - await fs.writeFile(path.join(rootDir, '.dockerignore'), './.husky\n'); - - const result = Utils.getDockerignoredPaths(rootDir, ['.husky/pre-commit', 'main.js']); - - expect(result).toStrictEqual(new Set(['.husky/pre-commit'])); - }); -}); - describe('getGitignoredPaths', () => { beforeEach(() => { vi.mocked(spawnSync).mockReset(); diff --git a/test/unit/bin/dockerignore.test.ts b/test/unit/bin/dockerignore.test.ts index bc8eac5..f1ff6a6 100644 --- a/test/unit/bin/dockerignore.test.ts +++ b/test/unit/bin/dockerignore.test.ts @@ -1,6 +1,8 @@ +import path from 'node:path'; + import { afterEach, describe, expect, it, vi } from 'vitest'; -import { loadDockerIgnore } from '../../../bin/dockerignore.js'; +import { buildDockerIgnoreMatcher, loadDockerIgnore } from '../../../bin/dockerignore.js'; const { fsMock } = vi.hoisted(() => ({ fsMock: { @@ -35,8 +37,8 @@ describe('loadDockerIgnore', () => { expect(matcher('src/main.ts')).toBe(false); }); - it('handles directory patterns', () => { - fsMock.readFileSync.mockReturnValue('dist/\n'); + it.each(['dist', 'dist/'])('handles directory pattern "%s"', (pattern) => { + fsMock.readFileSync.mockReturnValue(`${pattern}\n`); const matcher = loadDockerIgnore(''); expect(matcher('dist/bundle.js')).toBe(true); expect(matcher('src/dist-utils.ts')).toBe(false); @@ -62,16 +64,19 @@ describe('loadDockerIgnore', () => { expect(matcher('other-actor/src/main.ts')).toBe(false); }); - it('reads .dockerignore from the dockerContextDir root', () => { + it('reads .dockerignore from the resolved, absolute dockerContextDir root', () => { fsMock.readFileSync.mockReturnValue(''); loadDockerIgnore('actors/shopify'); - expect(fsMock.readFileSync).toHaveBeenCalledWith('actors/shopify/.dockerignore', 'utf-8'); + expect(fsMock.readFileSync).toHaveBeenCalledWith( + path.join(path.resolve('actors/shopify'), '.dockerignore'), + 'utf-8', + ); }); - it('reads .dockerignore from repo root when dockerContextDir is empty', () => { + it('reads .dockerignore from the resolved repo root when dockerContextDir is empty', () => { fsMock.readFileSync.mockReturnValue(''); loadDockerIgnore(''); - expect(fsMock.readFileSync).toHaveBeenCalledWith('.dockerignore', 'utf-8'); + expect(fsMock.readFileSync).toHaveBeenCalledWith(path.join(path.resolve(''), '.dockerignore'), 'utf-8'); }); it('handles comments and blank lines', () => { @@ -80,4 +85,39 @@ describe('loadDockerIgnore', () => { expect(matcher('node_modules/foo.js')).toBe(true); expect(matcher('src/main.ts')).toBe(false); }); + + it('normalizes a leading "./" in patterns, which Docker treats as a no-op', () => { + // The common real-world .dockerignore style: every pattern prefixed with "./", + // including a negation re-including one of the otherwise-matched files. + fsMock.readFileSync.mockReturnValue('./node_modules\n./*.log\n!./keep.log\n'); + const matcher = loadDockerIgnore(''); + expect(matcher('node_modules/foo.js')).toBe(true); + expect(matcher('debug.log')).toBe(true); + expect(matcher('keep.log')).toBe(false); + expect(matcher('main.js')).toBe(false); + }); +}); + +describe('buildDockerIgnoreMatcher', () => { + it('treats paths as already relative to absoluteRootDir when hoistFrom is left at its default', () => { + fsMock.readFileSync.mockReturnValue('test/\n*.log\n'); + const matcher = buildDockerIgnoreMatcher('/repo/actors/shopify'); + expect(matcher('test/fixture.json')).toBe(true); + expect(matcher('debug.log')).toBe(true); + expect(matcher('main.js')).toBe(false); + }); + + it('reads .dockerignore from the given absoluteRootDir', () => { + fsMock.readFileSync.mockReturnValue(''); + buildDockerIgnoreMatcher('/repo/actors/shopify'); + expect(fsMock.readFileSync).toHaveBeenCalledWith('/repo/actors/shopify/.dockerignore', 'utf-8'); + }); + + it('returns no-op matcher when .dockerignore is absent', () => { + fsMock.readFileSync.mockImplementation(() => { + throw new Error('ENOENT'); + }); + const matcher = buildDockerIgnoreMatcher('/repo/actors/shopify'); + expect(matcher('main.js')).toBe(false); + }); }); From 98c610bcf404b30b206029ce691098aa1e92bb42 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Mon, 20 Jul 2026 11:11:18 +0100 Subject: [PATCH 31/33] revert logic for changelog classification and add TODO --- README.md | 13 +++++++------ bin/diff-changes.ts | 24 +++++++++++++++++------- test/unit/bin/diff-changes.test.ts | 13 ++++++++++++- test/unit/should-built-and-test.test.ts | 17 ++++++++++++----- 4 files changed, 48 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index a598bdf..9da7da1 100644 --- a/README.md +++ b/README.md @@ -76,12 +76,13 @@ For a single-actor repo, set `"folder": "."` in the config and place `.actor/act When a PR is opened or code is pushed, the tool determines which actors need to be built and tested based on the changed files. For each changed file, for each actor: 1. **Sibling exclusion** — files inside another actor's `folder` are excluded first. This prevents an actor with broad context from being triggered by changes that belong to a sibling actor. -2. **Context matching** — the file must fall within one of the actor's context paths (`dockerContextDir` from `actor.json` by default, or `overrideActorContext` from config if set). Files outside every context path are skipped. -3. **Hardcoded ignore list, context-aware** — the file path is first "hoisted" relative to the context path it matched (e.g. a standalone actor's own `.eslintrc` is checked as just `.eslintrc`, not the full repo-root-relative path), then checked against repo-level dev file patterns (`.vscode/`, `.gitignore`, `.husky/`, `.eslintrc`, `eslint.config.mjs`, `.prettierrc`, `.editorconfig`). There's no hardcoded special-casing for legacy `code/`/`shared/` layouts — repos that need those directories treated as top-level must list them explicitly in `overrideActorContext`. -4. **`.dockerignore` filtering** — if a `.dockerignore` exists at the root of the actor's `dockerContextDir`, matching files are ignored. Patterns are resolved relative to `dockerContextDir`, matching Docker's own behavior. -5. **README/CHANGELOG classification** — a `README.md` or `CHANGELOG.md` file is `cosmetic` (only triggers a release build, not tests) if it lives inside the actor's own `folder`; otherwise it's ignored entirely, since it isn't documentation for this actor. -6. **Cosmetic JSON classification** — `.json` files inside the actor's own `.actor/` directory with only cosmetic schema changes (whitespace, key ordering) only trigger a release build. -7. **Functional** — everything else triggers both build and tests. +2. **CHANGELOG classification** — a `CHANGELOG.md` file is always `cosmetic` (only triggers a release build, not tests), for every actor, regardless of context or folder. (See [issue #106](https://github.com/apify/apify-test-tools/issues/106).) +3. **Context matching** — the file must fall within one of the actor's context paths (`dockerContextDir` from `actor.json` by default, or `overrideActorContext` from config if set). Files outside every context path are skipped. +4. **Hardcoded ignore list, context-aware** — the file path is first "hoisted" relative to the context path it matched (e.g. a standalone actor's own `.eslintrc` is checked as just `.eslintrc`, not the full repo-root-relative path), then checked against repo-level dev file patterns (`.vscode/`, `.gitignore`, `.husky/`, `.eslintrc`, `eslint.config.mjs`, `.prettierrc`, `.editorconfig`). There's no hardcoded special-casing for legacy `code/`/`shared/` layouts — repos that need those directories treated as top-level must list them explicitly in `overrideActorContext`. +5. **`.dockerignore` filtering** — if a `.dockerignore` exists at the root of the actor's `dockerContextDir`, matching files are ignored. Patterns are resolved relative to `dockerContextDir`, matching Docker's own behavior. +6. **README classification** — a `README.md` file is `cosmetic` (only triggers a release build, not tests) if it lives inside the actor's own `folder`; otherwise it's ignored entirely, since it isn't documentation for this actor. +7. **Cosmetic JSON classification** — `.json` files inside the actor's own `.actor/` directory with only cosmetic schema changes (whitespace, key ordering) only trigger a release build. +8. **Functional** — everything else triggers both build and tests. ### 4. Create test directories diff --git a/bin/diff-changes.ts b/bin/diff-changes.ts index 8930688..47e73d8 100644 --- a/bin/diff-changes.ts +++ b/bin/diff-changes.ts @@ -34,13 +34,15 @@ type FileChangeForActor = * Classify a single file change for a single actor. * * Steps (in order): - * 1. Context matching (actorConfig.contextPaths) → outside-context if no match - * 2. Hardcoded ignore list, checked against the path hoisted relative to the matched context entry → ignored - * 3. .dockerignore filtering (patterns relative to dockerContextDir), skipped for the actor's own `.actor/` + * 1. CHANGELOG.md, by filename, anywhere → cosmetic. There is a single repo-wide shared changelog, + * not one per actor, so it applies to every actor regardless of context/folder. + * 2. Context matching (actorConfig.contextPaths) → outside-context if no match + * 3. Hardcoded ignore list, checked against the path hoisted relative to the matched context entry → ignored + * 4. .dockerignore filtering (patterns relative to dockerContextDir), skipped for the actor's own `.actor/` * dir → ignored if matched - * 4. README/CHANGELOG by filename → cosmetic if inside the actor's own folder, otherwise ignored - * 5. .json inside the actor's own `.actor/` dir with only cosmetic schema diffs → cosmetic (semantically verified) - * 6. Everything else → functional + * 5. README.md by filename → cosmetic if inside the actor's own folder, otherwise ignored + * 6. .json inside the actor's own `.actor/` dir with only cosmetic schema diffs → cosmetic (semantically verified) + * 7. Everything else → functional */ const classifyFileChange = ( originalFilePath: string, @@ -49,6 +51,14 @@ const classifyFileChange = ( dockerIgnoreMatcher: DockerIgnoreMatcher, ): FileChangeForActor => { const lowercaseFilePath = originalFilePath.toLowerCase(); + + // TODO: hardcodes that there's a single repo-wide changelog belonging to every actor. Should instead + // be derived from parsing actor.json (readme, changelog, schema paths), see + // https://github.com/apify/apify-test-tools/issues/106 + if (lowercaseFilePath.endsWith('changelog.md')) { + return { impact: 'cosmetic', semanticallyVerified: false }; + } + const lowercaseContextPaths = actorConfig.contextPaths.map((contextPath) => contextPath.toLowerCase()); const matchedContext = findContainingScope(lowercaseFilePath, lowercaseContextPaths); @@ -74,7 +84,7 @@ const classifyFileChange = ( const isInActorFolder = isPathWithinScope(lowercaseFilePath, lowercaseFolder); - if (lowercaseFilePath.endsWith('readme.md') || lowercaseFilePath.endsWith('changelog.md')) { + if (lowercaseFilePath.endsWith('readme.md')) { return isInActorFolder ? { impact: 'cosmetic', semanticallyVerified: false } : { impact: 'ignored' }; } diff --git a/test/unit/bin/diff-changes.test.ts b/test/unit/bin/diff-changes.test.ts index 2379a1f..cb09142 100644 --- a/test/unit/bin/diff-changes.test.ts +++ b/test/unit/bin/diff-changes.test.ts @@ -142,13 +142,24 @@ describe('getChangedActors', () => { expect(result).not.toContainEqual(standaloneActor); }); - it('root-level changelog outside any actor folder is ignored, not cosmetic', () => { + it('root-level changelog outside any actor folder is cosmetic for every actor when isLatest', () => { const result = getChangedActors({ filepathsChanged: ['CHANGELOG.md'], actorConfigs, commits, isLatest: true, }); + expect(result).toEqual(expect.arrayContaining([miniActor, standaloneActor])); + expect(result).toHaveLength(2); + }); + + it('root-level changelog is not cosmetic-triggered when not isLatest', () => { + const result = getChangedActors({ + filepathsChanged: ['CHANGELOG.md'], + actorConfigs, + commits, + isLatest: false, + }); expect(result).toEqual([]); }); diff --git a/test/unit/should-built-and-test.test.ts b/test/unit/should-built-and-test.test.ts index a93083b..f68ab17 100644 --- a/test/unit/should-built-and-test.test.ts +++ b/test/unit/should-built-and-test.test.ts @@ -82,20 +82,27 @@ describe('Should build and test parser', () => { expect(actorsChanged).toEqual(ACTOR_CONFIGS.slice(0, 2)); }); - test('Root-level changelog outside any actor own folder is ignored, even on latest', () => { + test('Root-level changelog is cosmetic for every actor, only on latest', () => { const FILES = ['shared/CHANGELOG.md', 'CHANGELOG.md']; - const actorsChanged = getChangedActors({ + const actorsChangedNotLatest = getChangedActors({ actorConfigs: ACTOR_CONFIGS, - isLatest: true, + isLatest: false, filepathsChanged: FILES, commits, }); + expect(actorsChangedNotLatest).toEqual([]); - expect(actorsChanged).toEqual([]); + const actorsChangedLatest = getChangedActors({ + actorConfigs: ACTOR_CONFIGS, + isLatest: true, + filepathsChanged: FILES, + commits, + }); + expect(actorsChangedLatest).toEqual(ACTOR_CONFIGS); }); - test('Only builds latest for actor own changelog', () => { + test('A changelog nested inside one actor own folder is excluded for sibling actors, only triggers that actor', () => { const FILES = ['actors/lukaskrivka_testing-github-integration-1/CHANGELOG.md']; const actorsChanged = getChangedActors({ From 397f3ed7980ed96815d7a84af7e814450df027e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20K=C5=99ivka?= Date: Tue, 21 Jul 2026 16:35:24 +0200 Subject: [PATCH 32/33] Bump version from 0.8.6 to 0.9.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f686094..9f7e58d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "apify-test-tools", - "version": "0.8.6", + "version": "0.9.0", "type": "module", "description": "TBD", "repository": { From ec44cae9aee941f5e032e6dd20cf37568725d051 Mon Sep 17 00:00:00 2001 From: Luigi Ruocco Date: Mon, 27 Jul 2026 13:42:59 +0100 Subject: [PATCH 33/33] add actor folder path if missing in overriden context --- README.md | 12 ++++++------ bin/utils.ts | 14 ++++++-------- test/unit/bin/utils.test.ts | 5 +++-- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 9da7da1..84ceeb9 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,12 @@ Every repo that uses `apify-test-tools` must have an `apify-test-tools.config.js Each entry has: -| Field | Required | Description | -| ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `folder` | yes | Relative path from repo root to the actor's own project directory — the folder that directly contains `.actor/actor.json` (i.e. `/.actor/actor.json`), the actor's README/CHANGELOG, and its source. Use `"."` for a single-actor repo where `.actor/` is at the root. | -| `actorFullName` | yes | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used. | -| `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. No fallback — if the env var is not set at build time, the build fails. | -| `overrideActorContext` | no | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. Entries must not be prefixes of one another (e.g. `["", "code"]` or `["actors", "actors/foo"]` are rejected), and the list must include a path that reaches the actor's own `folder` — otherwise the actor could never be detected as changed. | +| Field | Required | Description | +| ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `folder` | yes | Relative path from repo root to the actor's own project directory — the folder that directly contains `.actor/actor.json` (i.e. `/.actor/actor.json`), the actor's README/CHANGELOG, and its source. Use `"."` for a single-actor repo where `.actor/` is at the root. | +| `actorFullName` | yes | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used. | +| `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. No fallback — if the env var is not set at build time, the build fails. | +| `overrideActorContext` | no | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. Entries must not be prefixes of one another (e.g. `["", "code"]` or `["actors", "actors/foo"]` are rejected). The actor's own `folder` is always part of its context — if none of the listed entries reach it, it's added automatically. | ### 3. Set up actor folders diff --git a/bin/utils.ts b/bin/utils.ts index b6fa1f5..ea400ad 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -202,6 +202,12 @@ export const readConfigFile = async (): Promise => { const normalizedDockerContextDir = dockerContextDir === '.' ? '' : dockerContextDir; const contextPaths = (entry.overrideActorContext ?? [normalizedDockerContextDir]).map(stripTrailingSlash); + // The actor's own folder is always part of its context. When an explicit "overrideActorContext" + // doesn't already cover it, add it automatically instead of failing the workflow. + if (!contextPaths.some((contextPath) => isPathWithinScope(folder, contextPath))) { + contextPaths.push(folder); + } + const overlap = findOverlappingContextPaths(contextPaths); if (overlap) { throw new Error( @@ -210,14 +216,6 @@ export const readConfigFile = async (): Promise => { ); } - if (!contextPaths.some((contextPath) => isPathWithinScope(folder, contextPath))) { - throw new Error( - `Actor folder "${entry.folder}" in "${CONFIG_FILE_NAME}" is not reachable through its own ` + - `context paths (${contextPaths.join(', ')}). Add the actor's own folder to ` + - `"overrideActorContext" or remove the override.`, - ); - } - actorConfigs.push({ actorFullName: entry.actorFullName, folder, diff --git a/test/unit/bin/utils.test.ts b/test/unit/bin/utils.test.ts index 4e65208..5fa2149 100644 --- a/test/unit/bin/utils.test.ts +++ b/test/unit/bin/utils.test.ts @@ -289,7 +289,7 @@ describe('readConfigFile', () => { await expect(readConfigFile()).rejects.toThrow(/overlap/); }); - it('throws when overrideActorContext does not include the actor own folder', async () => { + it('adds the actor own folder automatically when overrideActorContext does not cover it', async () => { mockFiles({ [CONFIG_FILE_NAME]: validConfig([ { @@ -302,7 +302,8 @@ describe('readConfigFile', () => { 'actors/shopify/.actor/actor.json': actorJson({}), }); - await expect(readConfigFile()).rejects.toThrow('not reachable through its own context paths'); + const result = await readConfigFile(); + expect(result[0].contextPaths).toEqual(['code', 'shared', 'actors/shopify']); }); it('strips trailing slashes from folder and overrideActorContext entries', async () => {