From 85efa624e8dd1191846219d382a2c5c72a9f0c1b Mon Sep 17 00:00:00 2001 From: Gytis Scipokas Date: Tue, 7 Jul 2026 13:36:01 +0200 Subject: [PATCH 1/5] feat: build zip from local source --- README.md | 22 ++++- bin/build.ts | 254 ++++++++++++++++++++++++++++++++++++++++++++++++++- bin/main.ts | 27 +++++- bin/utils.ts | 37 ++++++++ 4 files changed, 333 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8eb6b25..f9b7601 100644 --- a/README.md +++ b/README.md @@ -346,6 +346,26 @@ Remove `--dry-run` to actually trigger builds and update the branch names/ The c [{ "buildId": "...", "actorId": "...", "buildNumber": "...", "actorName": "john.doe/my-actor" }] ``` +#### Build from local source (no push needed) + +If you don't want to push a dummy branch just to test a change and wait for all the tests to finish, `build-zip` builds Actors directly from your local files (zipped and uploaded as `SOURCE_FILES`), skipping steps 1-4 above. + +```bash +APIFY_TOKEN_JOHN_DOE= \ +GITHUB_WORKSPACE=. \ + npx apify-test-tools build-zip --actors john.doe/my-actor +``` + +Pass a hardcoded actor name via `--actors` to build only that Actor (comma-separate multiple names). Omit `--actors` to build all Actors in the repo, or add `--dry-run` to preview without building. It outputs the same JSON build array as `build`, so you run tests against it the same way as in step 5 below: + +````bash +# Build from local source and capture output +BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \ + GITHUB_WORKSPACE=. \ + npx apify-test-tools build-zip --actors apify/my-actor) + +Since you already scoped the build to just the Actor(s) you care about, point vitest at a specific test file (or a `-t` name filter) instead of the whole `test/platform` directory — you get feedback on that one test without waiting for the full suite to run. + #### 5. Run tests against the builds Pass the build output as `ACTOR_BUILDS` and provide `TESTER_APIFY_TOKEN`. The token can point to your own account (if you have enough memory) or you can use the testing account (xRGg9iAfJSymqartk). @@ -357,7 +377,7 @@ ACTOR_BUILDS='' \ TESTER_APIFY_TOKEN= \ RUN_PLATFORM_TESTS=1 \ npx vitest --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100 test/platform -``` +```` #### Full example diff --git a/bin/build.ts b/bin/build.ts index eefb7c9..f45c99b 100644 --- a/bin/build.ts +++ b/bin/build.ts @@ -1,9 +1,38 @@ -import type { Build } from 'apify-client'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import type { ActorVersionSourceFile, Build } from 'apify-client'; import { ApifyClient } from 'apify-client'; import { ACTOR_SOURCE_TYPES } from '@apify/consts'; import type { ActorConfig, BuildData } from './types.js'; +import { collectFilePaths, isOutsideDir, toSourceFile } from './utils.js'; + +const SKIP_DIRS = new Set(['node_modules', '.git', 'apify_storage', 'dist', 'build', 'out', '.next', '.cache']); + +// JUST IN CASE. Filenames that commonly hold credentials — never ship these into a build, regardless of sourceType. +const SKIP_FILE_NAMES = new Set([ + '.env', + '.env.local', + '.env.development', + '.env.testing', + '.env.production', + '.npmrc', + '.netrc', + '.pgpass', + 'credentials.json', + 'id_rsa', + 'id_dsa', + 'id_ecdsa', + 'id_ed25519', +]); +const SKIP_FILE_PATTERNS = [/^\.env(\..+)?$/, /\.pem$/, /\.key$/, /\.pfx$/, /\.p12$/]; +const isSecretFile = (fileName: string): boolean => + SKIP_FILE_NAMES.has(fileName) || SKIP_FILE_PATTERNS.some((pattern) => pattern.test(fileName)); + +const MAX_SOURCE_FILES_BYTES = 3 * 1024 * 1024; type BuildPrActorOptions = { buildTag?: string; @@ -101,15 +130,179 @@ class ApifyBuilder { return { buildId: id, actorId: actId, buildNumber, actorName: this.actorName }; }; + startActorBuildFromSourceFiles = async (sourceFiles: ActorVersionSourceFile[]): Promise => { + const ZIP_VERSION = '0.98'; + const actorClient = this.apifyClient.actor(this.actorName); + const actorInfo = await actorClient.get(); + if (!actorInfo) { + throw new Error( + `No actor named '${this.actorName}' was found on the platform. If this` + + ' is unexpected, make sure the actor you are targeting is spelled the' + + ' same as the folder in the repository.', + ); + } + + type ActorVersion = Parameters['update']>[0]; + const actorVersion: ActorVersion = { + versionNumber: ZIP_VERSION, + sourceFiles, + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore: couldn't find this type :( + sourceType: ACTOR_SOURCE_TYPES.SOURCE_FILES, + }; + + const versionExists = !actorInfo.versions.find((v) => v.versionNumber === ZIP_VERSION); + if (versionExists) { + await actorClient.versions().create(actorVersion); + } else { + await actorClient.version(ZIP_VERSION).update(actorVersion); + } + + const { id, actId, buildNumber } = await actorClient.build(ZIP_VERSION, { useCache: false }); + console.error(`[${this.actorName}]: ${id} (${buildNumber})`); + return { buildId: id, actorId: actId, buildNumber, actorName: this.actorName }; + }; + + collectSourceFiles = async (actorDir: string): Promise => { + const absActorDir = path.resolve(actorDir); + + // Read actor.json to check if this is a monorepo actor with an external dockerContextDir. + // Monorepo actors point their dockerContextDir to a parent directory (e.g. "../../.."), + // which means the Docker build context is the repo root, not the actor directory itself. + const actorJsonPath = path.join(absActorDir, '.actor', 'actor.json'); + const actorJson = JSON.parse(await fs.readFile(actorJsonPath, 'utf8')) as Record; + const rawContextDir = actorJson.dockerContextDir as string | undefined; + const contextAbsDir = rawContextDir ? path.resolve(absActorDir, '.actor', rawContextDir) : undefined; + const isMonorepoActor = !!contextAbsDir && isOutsideDir(contextAbsDir, absActorDir); + + const sourceRootDir = isMonorepoActor + ? await this.flattenMonorepoContext(absActorDir, contextAbsDir!, actorJson) + : absActorDir; + + try { + const filePaths = await collectFilePaths(sourceRootDir, SKIP_DIRS, isSecretFile); + const sourceFiles = await Promise.all( + filePaths.map(async (filePath) => toSourceFile(filePath, sourceRootDir)), + ); + + this.assertWithinSizeLimit(sourceFiles); + return sourceFiles; + } finally { + // Only the flattened copy is temporary — never delete the actor's own directory. + if (isMonorepoActor) { + await fs.rm(sourceRootDir, { recursive: true, force: true }); + } + } + }; + + // SOURCE_FILES always treats the collected root as the actor root, so we cannot simply + // collect the actor directory of a monorepo actor — the platform would reject any path + // escaping it. Fix: create a temporary "flattened" directory where: + // - the Docker context contents (repo root) are copied to the temp dir root + // - the actor's .actor/ directory is overlaid at the temp dir root + // - actor.json path fields are rewritten to be relative to the new location + // + // Result: the collected root IS the Docker context, .actor/ is at that root, and + // all relative paths (dockerfile, dockerContextDir, changelog) are exactly one + // level up ("..") instead of three ("../../.."). + flattenMonorepoContext = async ( + absActorDir: string, + contextAbsDir: string, + actorJson: Record, + ): Promise => { + console.error(`[${this.actorName}]: monorepo actor detected — flattening from Docker context`); + + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), `apify-build-${this.actorName.replace('/', '_')}-`)); + + // Step 1: copy the Docker context (repo root) into the temp dir, skipping + // generated/large directories and secret files so they never touch disk here. + await fs.cp(contextAbsDir, tempDir, { + recursive: true, + filter: (src) => !SKIP_DIRS.has(path.basename(src)) && !isSecretFile(path.basename(src)), + }); + + // Step 2: overlay the actor's .actor/ directory at the temp dir root, + // overwriting anything that was copied from the context (unlikely but safe). + await fs.cp(path.join(absActorDir, '.actor'), path.join(tempDir, '.actor'), { + recursive: true, + force: true, + }); + + // Step 3: rewrite actor.json path fields so they resolve correctly from the new location. + await this.rewriteActorJsonPaths(absActorDir, contextAbsDir, tempDir, actorJson); + + return tempDir; + }; + + // Rewrites actor.json path fields so they resolve correctly from the new .actor/ location + // (one level below the root) instead of the original three-levels-deep location. + // + // Algorithm for each path field: + // 1. Resolve the original value to an absolute path on disk. + // 2. Compute its position relative to the Docker context root (e.g. repo root). + // That relative position is exactly where the file landed inside tempDir, + // because we copied contextAbsDir → tempDir in flattenMonorepoContext's step 1. + // 3. Build the new path from newActorDir to that file in tempDir. + // + // Local paths (e.g. "./dataset_schema.json") point inside .actor/ and are left + // unchanged — .actor/ was copied intact so those paths still resolve correctly. + rewriteActorJsonPaths = async ( + absActorDir: string, + contextAbsDir: string, + tempDir: string, + actorJson: Record, + ): Promise => { + const originalActorDir = path.join(absActorDir, '.actor'); + const newActorDir = path.join(tempDir, '.actor'); + const pathFields = ['dockerfile', 'dockerContextDir', 'changelog', 'readme'] as const; + const rewritten = { ...actorJson }; + for (const field of pathFields) { + const value = rewritten[field]; + if (typeof value !== 'string') continue; + + const absPath = path.resolve(originalActorDir, value); + + // Skip paths that stay inside .actor/ — they don't need rewriting. + if (!isOutsideDir(absPath, originalActorDir)) continue; + + // Where does this file live inside the Docker context? That's also where + // it lives inside tempDir after the copy in flattenMonorepoContext's step 1. + const relativeToContext = path.relative(contextAbsDir, absPath); + const newAbsPath = path.join(tempDir, relativeToContext); + rewritten[field] = path.relative(newActorDir, newAbsPath); + } + await fs.writeFile(path.join(newActorDir, 'actor.json'), JSON.stringify(rewritten, null, 4)); + }; + + // The Apify API caps combined SOURCE_FILES content at MAX_SOURCE_FILES_BYTES — fail fast + // with a clear message instead of letting the platform reject an opaque, oversized payload. + assertWithinSizeLimit = (sourceFiles: ActorVersionSourceFile[]): void => { + const totalBytes = sourceFiles.reduce((sum, file) => sum + Buffer.byteLength(file.content), 0); + if (totalBytes <= MAX_SOURCE_FILES_BYTES) return; + + throw new Error( + `[${this.actorName}]: Actor source is ${(totalBytes / 1024 / 1024).toFixed(2)} MiB, which exceeds ` + + `the ${MAX_SOURCE_FILES_BYTES / 1024 / 1024} MiB limit for SOURCE_FILES builds. Exclude more files ` + + 'or use a git-based build instead.', + ); + }; + waitForBuildToFinish = async (buildId: string, actorName: string): Promise => { const build = await this.apifyClient.build(buildId).waitForFinish(); const versionNumber = build.buildNumber; if (build.status === 'FAILED' || build.status === 'TIMED-OUT') { - const message = - `[BUILD][${actorName}]: Build ${buildId} (${versionNumber}) failed. ` + - `Not continuing with other builds and tests.`; console.error(`[${this.actorName}]: ${versionNumber}`); - throw new Error(message); + try { + const log = await this.apifyClient.build(buildId).log().get(); + const logTail = log?.split('\n').slice(-40).join('\n'); + console.error(`\n--- BUILD LOG (last 40 lines) ---\n${logTail}\n---`); + } catch (err) { + console.error(`[${this.actorName}]: Failed to fetch build log: ${err}`); + } + throw new Error( + `[BUILD][${actorName}]: Build ${buildId} (${versionNumber}) failed. ` + + `Not continuing with other builds and tests.`, + ); } console.error(`[${this.actorName}]: ${versionNumber}`); return build; @@ -304,3 +497,54 @@ export const deleteOldBuilds = async (actorConfigs: ActorConfig[]) => { await ApifyBuilder.fromActorName(actorName).deleteOldBuilds(); } }; + +export const runZipBuilds = async ({ + actorConfigs, + dryRun, +}: { + actorConfigs: ActorConfig[]; + dryRun: boolean; +}): Promise => { + if (dryRun) { + console.error('[DRY RUN] Would build from local source:'); + for (const { actorName, folder } of actorConfigs) { + console.error(` ${actorName} (${folder})`); + } + return actorConfigs.map(({ actorName }) => ({ + buildId: 'dry-run', + actorId: 'dry-run', + buildNumber: '0.98.0', + actorName, + })); + } + + console.error('========================================='); + console.error('STARTED ZIP BUILDS:'); + const startedBuilds = await Promise.all( + actorConfigs.map(async ({ actorName, folder }) => { + const builder = ApifyBuilder.fromActorName(actorName); + const sourceFiles = await builder.collectSourceFiles(folder); + return builder.startActorBuildFromSourceFiles(sourceFiles); + }), + ); + + console.error('========================================='); + console.error('FINISHED ZIP BUILDS:'); + await Promise.all( + startedBuilds.map(async (buildData: BuildData) => { + const builder = ApifyBuilder.fromActorName(buildData.actorName); + await builder.waitForBuildToFinish(buildData.buildId, buildData.actorName); + }), + ); + + console.error('========================================='); + console.error('SUMMARY:'); + for (const buildData of startedBuilds.sort((a: BuildData, b: BuildData) => + a.actorName.localeCompare(b.actorName), + )) { + console.error(`[${buildData.actorName}]: ${buildData.buildNumber}`); + } + console.error('========================================='); + + return startedBuilds; +}; diff --git a/bin/main.ts b/bin/main.ts index 120ea1b..6809fac 100644 --- a/bin/main.ts +++ b/bin/main.ts @@ -6,7 +6,7 @@ import yargs, { type Argv } from 'yargs'; // eslint-disable-next-line import/extensions --- With .js, it cannot find types import { hideBin } from 'yargs/helpers'; -import { deleteOldBuilds, runBuilds } from './build.js'; +import { deleteOldBuilds, runBuilds, runZipBuilds } from './build.js'; import { getChangedActors } from './diff-changes.js'; import { getBranchOnlyChangedFiles, getChangedFiles, getCommits, hasMergeFromTarget } from './git.js'; import { getPushData } from './github.js'; @@ -202,6 +202,31 @@ await yargs() }); }, ) + .command( + 'build-zip', + '', + (args) => + args + .option('actors', { + type: 'string', + description: + 'Comma-separated actor names (owner/name) to build. Defaults to all actors in the repo.', + }) + .option('dry-run', { type: 'boolean', default: false }), + async ({ actors, dryRun }) => { + const allActorConfigs = await getRepoActors(); + const actorConfigs = actors + ? actors.split(',').map((name) => { + const trimmed = name.trim(); + const config = allActorConfigs.find((c) => c.actorName === trimmed); + if (!config) throw new Error(`Actor "${trimmed}" not found in repo`); + return config; + }) + : allActorConfigs; + const builds = await runZipBuilds({ actorConfigs, dryRun }); + console.log(JSON.stringify(builds)); + }, + ) .command( 'delete-old-builds', '', diff --git a/bin/utils.ts b/bin/utils.ts index 477a6b5..c0f6c39 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -1,8 +1,45 @@ import { spawnSync } from 'node:child_process'; import fs from 'node:fs/promises'; +import path from 'node:path'; + +import type { ActorVersionSourceFile } from 'apify-client'; import type { ActorConfig } from './types.js'; +// Returns true when `childPath` is not inside `parentPath`. +// Used to detect monorepo actors whose dockerContextDir escapes the actor directory. +export const isOutsideDir = (childPath: string, parentPath: string): boolean => + path.relative(parentPath, childPath).startsWith('..'); + +export const collectFilePaths = async ( + rootDir: string, + skipDirs: Set, + isSecretFile: (fileName: string) => boolean, +): Promise => { + const entries = await fs.readdir(rootDir, { withFileTypes: true }); + const filePaths: string[] = []; + for (const entry of entries) { + if (entry.isDirectory()) { + if (skipDirs.has(entry.name)) continue; + filePaths.push(...(await collectFilePaths(path.join(rootDir, entry.name), skipDirs, isSecretFile))); + } else if (entry.isFile()) { + if (isSecretFile(entry.name)) continue; + filePaths.push(path.join(rootDir, entry.name)); + } + } + return filePaths; +}; + +const isBinary = (buffer: Buffer): boolean => buffer.includes(0); + +export const toSourceFile = async (absPath: string, rootDir: string): Promise => { + const buffer = await fs.readFile(absPath); + const name = path.relative(rootDir, absPath).split(path.sep).join('/'); + return isBinary(buffer) + ? { name, format: 'BASE64', content: buffer.toString('base64') } + : { name, format: 'TEXT', content: buffer.toString('utf8') }; +}; + export const spawnCommandInGhWorkspace = (command: string, args: string[] = []) => { console.error(command, args.join(' ')); const commandResult = spawnSync(command, args, { shell: true, maxBuffer: 100 * 1024 * 1024 }); From 816aa98ec537194c0407178ed5e7696baae1635e Mon Sep 17 00:00:00 2001 From: Gytis Scipokas Date: Tue, 7 Jul 2026 14:49:23 +0200 Subject: [PATCH 2/5] fix: gitignored files --- bin/build.ts | 72 +++++++++++++++++++++++---------------- bin/utils.ts | 32 +++++++++++++---- test/unit/bin/git.test.ts | 56 ++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 36 deletions(-) diff --git a/bin/build.ts b/bin/build.ts index f45c99b..6b4492d 100644 --- a/bin/build.ts +++ b/bin/build.ts @@ -8,29 +8,16 @@ import { ApifyClient } from 'apify-client'; import { ACTOR_SOURCE_TYPES } from '@apify/consts'; import type { ActorConfig, BuildData } from './types.js'; -import { collectFilePaths, isOutsideDir, toSourceFile } from './utils.js'; +import { collectFilePaths, getGitignoredPaths, isOutsideDir, toSourceFile } from './utils.js'; const SKIP_DIRS = new Set(['node_modules', '.git', 'apify_storage', 'dist', 'build', 'out', '.next', '.cache']); -// JUST IN CASE. Filenames that commonly hold credentials — never ship these into a build, regardless of sourceType. -const SKIP_FILE_NAMES = new Set([ - '.env', - '.env.local', - '.env.development', - '.env.testing', - '.env.production', - '.npmrc', - '.netrc', - '.pgpass', - 'credentials.json', - 'id_rsa', - 'id_dsa', - 'id_ecdsa', - 'id_ed25519', -]); +// JUST IN CASE. File patterns that commonly hold credentials — never ship these into a build, regardless +// of sourceType or of whether the repo's .gitignore happens to list them. Everything else that should be +// excluded (build output, local overrides, project-specific secret files, ...) is expected to already be +// in the repo's .gitignore — see getGitignoredPaths. const SKIP_FILE_PATTERNS = [/^\.env(\..+)?$/, /\.pem$/, /\.key$/, /\.pfx$/, /\.p12$/]; -const isSecretFile = (fileName: string): boolean => - SKIP_FILE_NAMES.has(fileName) || SKIP_FILE_PATTERNS.some((pattern) => pattern.test(fileName)); +const isSecretFile = (fileName: string): boolean => SKIP_FILE_PATTERNS.some((pattern) => pattern.test(fileName)); const MAX_SOURCE_FILES_BYTES = 3 * 1024 * 1024; @@ -164,6 +151,7 @@ class ApifyBuilder { }; collectSourceFiles = async (actorDir: string): Promise => { + const repoRoot = process.cwd(); const absActorDir = path.resolve(actorDir); // Read actor.json to check if this is a monorepo actor with an external dockerContextDir. @@ -175,12 +163,17 @@ class ApifyBuilder { const contextAbsDir = rawContextDir ? path.resolve(absActorDir, '.actor', rawContextDir) : undefined; const isMonorepoActor = !!contextAbsDir && isOutsideDir(contextAbsDir, absActorDir); + const collectRootDir = isMonorepoActor ? contextAbsDir! : absActorDir; + const keptFilePaths = await this.collectNonIgnoredFiles(collectRootDir, repoRoot); + const sourceRootDir = isMonorepoActor - ? await this.flattenMonorepoContext(absActorDir, contextAbsDir!, actorJson) - : absActorDir; + ? await this.flattenMonorepoContext(absActorDir, contextAbsDir!, actorJson, keptFilePaths) + : collectRootDir; try { - const filePaths = await collectFilePaths(sourceRootDir, SKIP_DIRS, isSecretFile); + // The flattened temp dir already contains exactly the files we want (kept files + + // the .actor/ overlay), so it's walked fresh; the non-monorepo case reuses keptFilePaths directly. + const filePaths = isMonorepoActor ? await collectFilePaths(sourceRootDir, SKIP_DIRS) : keptFilePaths; const sourceFiles = await Promise.all( filePaths.map(async (filePath) => toSourceFile(filePath, sourceRootDir)), ); @@ -195,10 +188,26 @@ class ApifyBuilder { } }; + // Walks `rootDir` and drops anything the repo's .gitignore excludes — nested .gitignore files, + // `.git/info/exclude`, and global excludes are all honored since this delegates to `git check-ignore` + // instead of re-implementing gitignore matching. Also drops files matching the hardcoded secret-pattern + // backstop (keys, certs, .env variants), which we never ship regardless of what .gitignore says. + collectNonIgnoredFiles = async (rootDir: string, repoRoot: string): Promise => { + const candidatePaths = await collectFilePaths(rootDir, SKIP_DIRS); + const relativePaths = candidatePaths.map((absPath) => + path.relative(repoRoot, absPath).split(path.sep).join('/'), + ); + const ignoredPaths = getGitignoredPaths(relativePaths); + + return candidatePaths.filter( + (absPath, i) => !ignoredPaths.has(relativePaths[i]) && !isSecretFile(path.basename(absPath)), + ); + }; + // SOURCE_FILES always treats the collected root as the actor root, so we cannot simply // collect the actor directory of a monorepo actor — the platform would reject any path // escaping it. Fix: create a temporary "flattened" directory where: - // - the Docker context contents (repo root) are copied to the temp dir root + // - the Docker context's non-ignored files (repo root) are copied to the temp dir root // - the actor's .actor/ directory is overlaid at the temp dir root // - actor.json path fields are rewritten to be relative to the new location // @@ -209,17 +218,22 @@ class ApifyBuilder { absActorDir: string, contextAbsDir: string, actorJson: Record, + keptContextFiles: string[], ): Promise => { console.error(`[${this.actorName}]: monorepo actor detected — flattening from Docker context`); const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), `apify-build-${this.actorName.replace('/', '_')}-`)); - // Step 1: copy the Docker context (repo root) into the temp dir, skipping - // generated/large directories and secret files so they never touch disk here. - await fs.cp(contextAbsDir, tempDir, { - recursive: true, - filter: (src) => !SKIP_DIRS.has(path.basename(src)) && !isSecretFile(path.basename(src)), - }); + // Step 1: copy only the files that survived gitignore/secret filtering, preserving their + // position relative to the Docker context root. + await Promise.all( + keptContextFiles.map(async (absFilePath) => { + const relPath = path.relative(contextAbsDir, absFilePath); + const destPath = path.join(tempDir, relPath); + await fs.mkdir(path.dirname(destPath), { recursive: true }); + await fs.copyFile(absFilePath, destPath); + }), + ); // Step 2: overlay the actor's .actor/ directory at the temp dir root, // overwriting anything that was copied from the context (unlikely but safe). diff --git a/bin/utils.ts b/bin/utils.ts index c0f6c39..d9f2ed1 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -11,25 +11,43 @@ import type { ActorConfig } from './types.js'; export const isOutsideDir = (childPath: string, parentPath: string): boolean => path.relative(parentPath, childPath).startsWith('..'); -export const collectFilePaths = async ( - rootDir: string, - skipDirs: Set, - isSecretFile: (fileName: string) => boolean, -): Promise => { +export const collectFilePaths = async (rootDir: string, skipDirs: Set): Promise => { const entries = await fs.readdir(rootDir, { withFileTypes: true }); const filePaths: string[] = []; for (const entry of entries) { if (entry.isDirectory()) { if (skipDirs.has(entry.name)) continue; - filePaths.push(...(await collectFilePaths(path.join(rootDir, entry.name), skipDirs, isSecretFile))); + filePaths.push(...(await collectFilePaths(path.join(rootDir, entry.name), skipDirs))); } else if (entry.isFile()) { - if (isSecretFile(entry.name)) continue; filePaths.push(path.join(rootDir, entry.name)); } } return filePaths; }; +/** + * Given paths relative to the repo root, returns the subset that `git` would exclude because of + * .gitignore rules (including nested .gitignore files, `.git/info/exclude`, and global excludes — + * anything `git` itself respects). Delegating to `git check-ignore` avoids re-implementing gitignore + * pattern matching. + */ +export const getGitignoredPaths = (relativePaths: string[]): Set => { + if (relativePaths.length === 0) return new Set(); + + const result = spawnSync('git', ['check-ignore', '--stdin'], { + input: relativePaths.join('\n'), + maxBuffer: 100 * 1024 * 1024, + }); + + // Exit code 1 means none of the given paths are ignored - not an error. Anything else + // (e.g. 128 for "not a git repository") is a real failure. + if (result.status !== 0 && result.status !== 1) { + throw new Error(`[Command failed]: git check-ignore\n${result.stderr.toString()}`); + } + + return new Set(result.stdout.toString().split('\n').filter(Boolean)); +}; + const isBinary = (buffer: Buffer): boolean => buffer.includes(0); export const toSourceFile = async (absPath: string, rootDir: string): Promise => { diff --git a/test/unit/bin/git.test.ts b/test/unit/bin/git.test.ts index 84fec14..e3bb7fe 100644 --- a/test/unit/bin/git.test.ts +++ b/test/unit/bin/git.test.ts @@ -1,3 +1,5 @@ +import { spawnSync } from 'node:child_process'; + import type { MockInstance } from 'vitest'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -10,6 +12,10 @@ import { } from '../../../bin/git.js'; import * as Utils from '../../../bin/utils.js'; +vi.mock('node:child_process', () => ({ + spawnSync: vi.fn(), +})); + describe('getCommits', () => { const sourceBranch = 'feature-branch'; const targetBranch = 'main'; @@ -250,3 +256,53 @@ describe('parseBaseCommit', () => { expect(() => parseBaseCommit(badJson)).toThrow('Invalid base commit SHA'); }); }); + +describe('getGitignoredPaths', () => { + beforeEach(() => { + vi.mocked(spawnSync).mockReset(); + }); + + it('returns an empty set without calling git when given no paths', () => { + const result = Utils.getGitignoredPaths([]); + + expect(result).toStrictEqual(new Set()); + expect(spawnSync).not.toHaveBeenCalled(); + }); + + it('returns the paths git reports as ignored, feeding all candidates via stdin', () => { + vi.mocked(spawnSync).mockReturnValue({ + status: 0, + stdout: 'node_modules/foo.js\n.env\n', + stderr: '', + } as unknown as ReturnType); + + const result = Utils.getGitignoredPaths(['node_modules/foo.js', 'bin/build.ts', '.env']); + + expect(result).toStrictEqual(new Set(['node_modules/foo.js', '.env'])); + expect(spawnSync).toHaveBeenCalledWith( + 'git', + ['check-ignore', '--stdin'], + expect.objectContaining({ input: 'node_modules/foo.js\nbin/build.ts\n.env' }), + ); + }); + + it('returns an empty set when git reports nothing is ignored (exit code 1)', () => { + vi.mocked(spawnSync).mockReturnValue({ + status: 1, + stdout: '', + stderr: '', + } as unknown as ReturnType); + + expect(Utils.getGitignoredPaths(['bin/build.ts'])).toStrictEqual(new Set()); + }); + + it('throws on an unexpected git failure instead of silently including/excluding files', () => { + vi.mocked(spawnSync).mockReturnValue({ + status: 128, + stdout: '', + stderr: 'fatal: not a git repository', + } as unknown as ReturnType); + + expect(() => Utils.getGitignoredPaths(['bin/build.ts'])).toThrow('git check-ignore'); + }); +}); From d821aa586e6dc4e942c41c6a66bc75c3c88dc7a3 Mon Sep 17 00:00:00 2001 From: Gytis Scipokas Date: Tue, 7 Jul 2026 15:20:27 +0200 Subject: [PATCH 3/5] fix: build-local unit tests --- README.md | 5 +- bin/build.ts | 27 +++-- test/unit/bin/build-local.test.ts | 179 ++++++++++++++++++++++++++++++ test/unit/bin/git.test.ts | 56 ---------- 4 files changed, 200 insertions(+), 67 deletions(-) create mode 100644 test/unit/bin/build-local.test.ts diff --git a/README.md b/README.md index f9b7601..b16823b 100644 --- a/README.md +++ b/README.md @@ -358,11 +358,12 @@ GITHUB_WORKSPACE=. \ Pass a hardcoded actor name via `--actors` to build only that Actor (comma-separate multiple names). Omit `--actors` to build all Actors in the repo, or add `--dry-run` to preview without building. It outputs the same JSON build array as `build`, so you run tests against it the same way as in step 5 below: -````bash +```bash # Build from local source and capture output BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \ GITHUB_WORKSPACE=. \ npx apify-test-tools build-zip --actors apify/my-actor) +``` Since you already scoped the build to just the Actor(s) you care about, point vitest at a specific test file (or a `-t` name filter) instead of the whole `test/platform` directory — you get feedback on that one test without waiting for the full suite to run. @@ -377,7 +378,7 @@ ACTOR_BUILDS='' \ TESTER_APIFY_TOKEN= \ RUN_PLATFORM_TESTS=1 \ npx vitest --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100 test/platform -```` +``` #### Full example diff --git a/bin/build.ts b/bin/build.ts index 6b4492d..a66f81a 100644 --- a/bin/build.ts +++ b/bin/build.ts @@ -28,7 +28,7 @@ type BuildPrActorOptions = { actorName: string; useDockerCache: boolean; }; -class ApifyBuilder { +export class ApifyBuilder { private constructor( private readonly apifyClient: ApifyClient, private readonly actorName: string, @@ -167,7 +167,7 @@ class ApifyBuilder { const keptFilePaths = await this.collectNonIgnoredFiles(collectRootDir, repoRoot); const sourceRootDir = isMonorepoActor - ? await this.flattenMonorepoContext(absActorDir, contextAbsDir!, actorJson, keptFilePaths) + ? await this.flattenMonorepoContext(absActorDir, contextAbsDir!, actorJson, keptFilePaths, repoRoot) : collectRootDir; try { @@ -208,7 +208,8 @@ class ApifyBuilder { // collect the actor directory of a monorepo actor — the platform would reject any path // escaping it. Fix: create a temporary "flattened" directory where: // - the Docker context's non-ignored files (repo root) are copied to the temp dir root - // - the actor's .actor/ directory is overlaid at the temp dir root + // - the actor's .actor/ directory is overlaid at the temp dir root (through the same + // gitignore/secret-pattern filter as the rest of the context — see collectNonIgnoredFiles) // - actor.json path fields are rewritten to be relative to the new location // // Result: the collected root IS the Docker context, .actor/ is at that root, and @@ -219,6 +220,7 @@ class ApifyBuilder { contextAbsDir: string, actorJson: Record, keptContextFiles: string[], + repoRoot: string, ): Promise => { console.error(`[${this.actorName}]: monorepo actor detected — flattening from Docker context`); @@ -235,12 +237,19 @@ class ApifyBuilder { }), ); - // Step 2: overlay the actor's .actor/ directory at the temp dir root, - // overwriting anything that was copied from the context (unlikely but safe). - await fs.cp(path.join(absActorDir, '.actor'), path.join(tempDir, '.actor'), { - recursive: true, - force: true, - }); + // Step 2: overlay the actor's .actor/ directory at the temp dir root, applying the same + // gitignore/secret-pattern filtering as step 1 instead of a raw copy — otherwise a stray + // secret file living inside .actor/ would ship unfiltered. + const actorMetaDir = path.join(absActorDir, '.actor'); + const keptActorFiles = await this.collectNonIgnoredFiles(actorMetaDir, repoRoot); + await Promise.all( + keptActorFiles.map(async (absFilePath) => { + const relPath = path.relative(actorMetaDir, absFilePath); + const destPath = path.join(tempDir, '.actor', relPath); + await fs.mkdir(path.dirname(destPath), { recursive: true }); + await fs.copyFile(absFilePath, destPath); + }), + ); // Step 3: rewrite actor.json path fields so they resolve correctly from the new location. await this.rewriteActorJsonPaths(absActorDir, contextAbsDir, tempDir, actorJson); diff --git a/test/unit/bin/build-local.test.ts b/test/unit/bin/build-local.test.ts new file mode 100644 index 0000000..3fac2d0 --- /dev/null +++ b/test/unit/bin/build-local.test.ts @@ -0,0 +1,179 @@ +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore: editor-only TS6059 — test/tsconfig.json's rootDir doesn't span bin/, but the root +// tsconfig (used for the real build and for eslint's type-aware linting) has no such restriction. +import { ApifyBuilder } from '../../../bin/build.js'; +import * as Utils from '../../../bin/utils.js'; + +vi.mock('node:child_process', () => ({ + spawnSync: vi.fn(), +})); + +const APIFY_TOKEN_ENV_VAR = 'APIFY_TOKEN_TEST'; + +const mkTempDir = async (prefix: string) => fs.mkdtemp(path.join(os.tmpdir(), prefix)); + +describe('ApifyBuilder', () => { + let builder: ApifyBuilder; + const tempDirs: string[] = []; + + beforeEach(() => { + process.env[APIFY_TOKEN_ENV_VAR] = 'dummy-token'; + builder = ApifyBuilder.fromActorName('test/actor'); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + delete process.env[APIFY_TOKEN_ENV_VAR]; + await Promise.all(tempDirs.splice(0).map(async (dir) => fs.rm(dir, { recursive: true, force: true }))); + }); + + describe('collectNonIgnoredFiles', () => { + it('drops secret-pattern files and gitignored files, keeps everything else', async () => { + const rootDir = await mkTempDir('apify-test-tools-collect-'); + tempDirs.push(rootDir); + + await fs.writeFile(path.join(rootDir, 'main.js'), 'console.log(1)'); + await fs.writeFile(path.join(rootDir, '.env'), 'SECRET=1'); + await fs.mkdir(path.join(rootDir, 'sub')); + await fs.writeFile(path.join(rootDir, 'sub', 'ignored.log'), 'log'); + + // Only the gitignore side is mocked — isSecretFile runs for real, so this also + // proves the secret-pattern backstop applies independently of .gitignore. + vi.spyOn(Utils, 'getGitignoredPaths').mockImplementation( + (relativePaths) => new Set(relativePaths.filter((p) => p.endsWith('.log'))), + ); + + const result = await builder.collectNonIgnoredFiles(rootDir, rootDir); + + expect(result).toStrictEqual([path.join(rootDir, 'main.js')]); + }); + }); + + describe('flattenMonorepoContext', () => { + it('filters the .actor/ overlay through the same secret-pattern check as the rest of the context', async () => { + const repoRoot = await mkTempDir('apify-test-tools-repo-'); + tempDirs.push(repoRoot); + + const absActorDir = path.join(repoRoot, 'actors', 'owner_actor'); + await fs.mkdir(path.join(absActorDir, '.actor'), { recursive: true }); + await fs.writeFile( + path.join(absActorDir, '.actor', 'actor.json'), + JSON.stringify({ actorSpecification: 1, name: 'actor' }), + ); + // A stray secret file inside .actor/ must never survive into the build. + await fs.writeFile(path.join(absActorDir, '.actor', '.env'), 'SECRET=leaked'); + await fs.writeFile(path.join(absActorDir, '.actor', 'INPUT_SCHEMA.json'), '{}'); + + const keptContextFile = path.join(repoRoot, 'package.json'); + await fs.writeFile(keptContextFile, '{}'); + + // Nothing is gitignored here — isolates the assertion to the secret-pattern filter. + vi.spyOn(Utils, 'getGitignoredPaths').mockReturnValue(new Set()); + + const actorJson = JSON.parse( + await fs.readFile(path.join(absActorDir, '.actor', 'actor.json'), 'utf8'), + ) as Record; + + const flattenedDir = await builder.flattenMonorepoContext( + absActorDir, + repoRoot, + actorJson, + [keptContextFile], + repoRoot, + ); + tempDirs.push(flattenedDir); + + await expect(fs.access(path.join(flattenedDir, '.actor', '.env'))).rejects.toThrow(); + await expect(fs.access(path.join(flattenedDir, '.actor', 'INPUT_SCHEMA.json'))).resolves.toBeUndefined(); + await expect(fs.access(path.join(flattenedDir, '.actor', 'actor.json'))).resolves.toBeUndefined(); + await expect(fs.access(path.join(flattenedDir, 'package.json'))).resolves.toBeUndefined(); + }); + }); + + describe('rewriteActorJsonPaths', () => { + it('rewrites path fields that escape .actor/ relative to the new flattened location', async () => { + const repoRoot = await mkTempDir('apify-test-tools-rewrite-'); + tempDirs.push(repoRoot); + const flattenedDir = await mkTempDir('apify-test-tools-flattened-'); + tempDirs.push(flattenedDir); + + const absActorDir = path.join(repoRoot, 'actors', 'owner_actor'); + await fs.mkdir(path.join(flattenedDir, '.actor'), { recursive: true }); + + const actorJson = { + actorSpecification: 1, + name: 'my-actor', + dockerfile: '../../../Dockerfile', // repo root, outside .actor/ + dockerContextDir: '../../..', // repo root itself + changelog: './CHANGELOG.md', // stays inside .actor/ + }; + + await builder.rewriteActorJsonPaths(absActorDir, repoRoot, flattenedDir, actorJson); + + const rewritten = JSON.parse( + await fs.readFile(path.join(flattenedDir, '.actor', 'actor.json'), 'utf8'), + ) as Record; + + expect(rewritten.dockerfile).toBe('../Dockerfile'); + expect(rewritten.dockerContextDir).toBe('..'); + expect(rewritten.changelog).toBe('./CHANGELOG.md'); + }); + }); +}); + +describe('getGitignoredPaths', () => { + beforeEach(() => { + vi.mocked(spawnSync).mockReset(); + }); + + it('returns an empty set without calling git when given no paths', () => { + const result = Utils.getGitignoredPaths([]); + + expect(result).toStrictEqual(new Set()); + expect(spawnSync).not.toHaveBeenCalled(); + }); + + it('returns the paths git reports as ignored, feeding all candidates via stdin', () => { + vi.mocked(spawnSync).mockReturnValue({ + status: 0, + stdout: 'node_modules/foo.js\n.env\n', + stderr: '', + } as unknown as ReturnType); + + const result = Utils.getGitignoredPaths(['node_modules/foo.js', 'bin/build.ts', '.env']); + + expect(result).toStrictEqual(new Set(['node_modules/foo.js', '.env'])); + expect(spawnSync).toHaveBeenCalledWith( + 'git', + ['check-ignore', '--stdin'], + expect.objectContaining({ input: 'node_modules/foo.js\nbin/build.ts\n.env' }), + ); + }); + + it('returns an empty set when git reports nothing is ignored (exit code 1)', () => { + vi.mocked(spawnSync).mockReturnValue({ + status: 1, + stdout: '', + stderr: '', + } as unknown as ReturnType); + + expect(Utils.getGitignoredPaths(['bin/build.ts'])).toStrictEqual(new Set()); + }); + + it('throws on an unexpected git failure instead of silently including/excluding files', () => { + vi.mocked(spawnSync).mockReturnValue({ + status: 128, + stdout: '', + stderr: 'fatal: not a git repository', + } as unknown as ReturnType); + + expect(() => Utils.getGitignoredPaths(['bin/build.ts'])).toThrow('git check-ignore'); + }); +}); diff --git a/test/unit/bin/git.test.ts b/test/unit/bin/git.test.ts index e3bb7fe..84fec14 100644 --- a/test/unit/bin/git.test.ts +++ b/test/unit/bin/git.test.ts @@ -1,5 +1,3 @@ -import { spawnSync } from 'node:child_process'; - import type { MockInstance } from 'vitest'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -12,10 +10,6 @@ import { } from '../../../bin/git.js'; import * as Utils from '../../../bin/utils.js'; -vi.mock('node:child_process', () => ({ - spawnSync: vi.fn(), -})); - describe('getCommits', () => { const sourceBranch = 'feature-branch'; const targetBranch = 'main'; @@ -256,53 +250,3 @@ describe('parseBaseCommit', () => { expect(() => parseBaseCommit(badJson)).toThrow('Invalid base commit SHA'); }); }); - -describe('getGitignoredPaths', () => { - beforeEach(() => { - vi.mocked(spawnSync).mockReset(); - }); - - it('returns an empty set without calling git when given no paths', () => { - const result = Utils.getGitignoredPaths([]); - - expect(result).toStrictEqual(new Set()); - expect(spawnSync).not.toHaveBeenCalled(); - }); - - it('returns the paths git reports as ignored, feeding all candidates via stdin', () => { - vi.mocked(spawnSync).mockReturnValue({ - status: 0, - stdout: 'node_modules/foo.js\n.env\n', - stderr: '', - } as unknown as ReturnType); - - const result = Utils.getGitignoredPaths(['node_modules/foo.js', 'bin/build.ts', '.env']); - - expect(result).toStrictEqual(new Set(['node_modules/foo.js', '.env'])); - expect(spawnSync).toHaveBeenCalledWith( - 'git', - ['check-ignore', '--stdin'], - expect.objectContaining({ input: 'node_modules/foo.js\nbin/build.ts\n.env' }), - ); - }); - - it('returns an empty set when git reports nothing is ignored (exit code 1)', () => { - vi.mocked(spawnSync).mockReturnValue({ - status: 1, - stdout: '', - stderr: '', - } as unknown as ReturnType); - - expect(Utils.getGitignoredPaths(['bin/build.ts'])).toStrictEqual(new Set()); - }); - - it('throws on an unexpected git failure instead of silently including/excluding files', () => { - vi.mocked(spawnSync).mockReturnValue({ - status: 128, - stdout: '', - stderr: 'fatal: not a git repository', - } as unknown as ReturnType); - - expect(() => Utils.getGitignoredPaths(['bin/build.ts'])).toThrow('git check-ignore'); - }); -}); From 8471b7f4f7557cfda40d886ddaa4aa4f29da1e26 Mon Sep 17 00:00:00 2001 From: Gytis Scipokas Date: Tue, 7 Jul 2026 16:49:29 +0200 Subject: [PATCH 4/5] fix: remove source file limit and other small improvements --- bin/build.ts | 49 ++++++++++++++++++++++++------------------------- bin/utils.ts | 6 ++++-- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/bin/build.ts b/bin/build.ts index a66f81a..aecf08e 100644 --- a/bin/build.ts +++ b/bin/build.ts @@ -10,7 +10,18 @@ import { ACTOR_SOURCE_TYPES } from '@apify/consts'; import type { ActorConfig, BuildData } from './types.js'; import { collectFilePaths, getGitignoredPaths, isOutsideDir, toSourceFile } from './utils.js'; -const SKIP_DIRS = new Set(['node_modules', '.git', 'apify_storage', 'dist', 'build', 'out', '.next', '.cache']); +const SKIP_DIRS = new Set([ + 'node_modules', + '.git', + 'apify_storage', + 'storage', + 'crawlee_storage', + 'dist', + 'build', + 'out', + '.next', + '.cache', +]); // JUST IN CASE. File patterns that commonly hold credentials — never ship these into a build, regardless // of sourceType or of whether the repo's .gitignore happens to list them. Everything else that should be @@ -19,8 +30,6 @@ const SKIP_DIRS = new Set(['node_modules', '.git', 'apify_storage', 'dist', 'bui const SKIP_FILE_PATTERNS = [/^\.env(\..+)?$/, /\.pem$/, /\.key$/, /\.pfx$/, /\.p12$/]; const isSecretFile = (fileName: string): boolean => SKIP_FILE_PATTERNS.some((pattern) => pattern.test(fileName)); -const MAX_SOURCE_FILES_BYTES = 3 * 1024 * 1024; - type BuildPrActorOptions = { buildTag?: string; versionNumber: string; @@ -178,7 +187,6 @@ export class ApifyBuilder { filePaths.map(async (filePath) => toSourceFile(filePath, sourceRootDir)), ); - this.assertWithinSizeLimit(sourceFiles); return sourceFiles; } finally { // Only the flattened copy is temporary — never delete the actor's own directory. @@ -190,8 +198,10 @@ export class ApifyBuilder { // Walks `rootDir` and drops anything the repo's .gitignore excludes — nested .gitignore files, // `.git/info/exclude`, and global excludes are all honored since this delegates to `git check-ignore` - // instead of re-implementing gitignore matching. Also drops files matching the hardcoded secret-pattern - // backstop (keys, certs, .env variants), which we never ship regardless of what .gitignore says. + // instead of re-implementing gitignore matching. `.actor/` (the Actor specification folder) is always + // kept regardless of .gitignore, matching Apify CLI's own behavior. Files matching the hardcoded + // secret-pattern backstop (keys, certs, .env variants) are dropped unconditionally, .actor/ included, + // since those should never ship regardless of what .gitignore says. collectNonIgnoredFiles = async (rootDir: string, repoRoot: string): Promise => { const candidatePaths = await collectFilePaths(rootDir, SKIP_DIRS); const relativePaths = candidatePaths.map((absPath) => @@ -199,9 +209,11 @@ export class ApifyBuilder { ); const ignoredPaths = getGitignoredPaths(relativePaths); - return candidatePaths.filter( - (absPath, i) => !ignoredPaths.has(relativePaths[i]) && !isSecretFile(path.basename(absPath)), - ); + return candidatePaths.filter((absPath, i) => { + if (isSecretFile(path.basename(absPath))) return false; + const isUnderActorDir = relativePaths[i].split('/').includes('.actor'); + return isUnderActorDir || !ignoredPaths.has(relativePaths[i]); + }); }; // SOURCE_FILES always treats the collected root as the actor root, so we cannot simply @@ -237,9 +249,9 @@ export class ApifyBuilder { }), ); - // Step 2: overlay the actor's .actor/ directory at the temp dir root, applying the same - // gitignore/secret-pattern filtering as step 1 instead of a raw copy — otherwise a stray - // secret file living inside .actor/ would ship unfiltered. + // Step 2: overlay the actor's .actor/ directory at the temp dir root. collectNonIgnoredFiles + // always keeps .actor/ paths regardless of .gitignore, but still drops the hardcoded secret + // patterns — so this isn't a raw copy, a stray secret file living inside .actor/ is still dropped. const actorMetaDir = path.join(absActorDir, '.actor'); const keptActorFiles = await this.collectNonIgnoredFiles(actorMetaDir, repoRoot); await Promise.all( @@ -297,19 +309,6 @@ export class ApifyBuilder { await fs.writeFile(path.join(newActorDir, 'actor.json'), JSON.stringify(rewritten, null, 4)); }; - // The Apify API caps combined SOURCE_FILES content at MAX_SOURCE_FILES_BYTES — fail fast - // with a clear message instead of letting the platform reject an opaque, oversized payload. - assertWithinSizeLimit = (sourceFiles: ActorVersionSourceFile[]): void => { - const totalBytes = sourceFiles.reduce((sum, file) => sum + Buffer.byteLength(file.content), 0); - if (totalBytes <= MAX_SOURCE_FILES_BYTES) return; - - throw new Error( - `[${this.actorName}]: Actor source is ${(totalBytes / 1024 / 1024).toFixed(2)} MiB, which exceeds ` + - `the ${MAX_SOURCE_FILES_BYTES / 1024 / 1024} MiB limit for SOURCE_FILES builds. Exclude more files ` + - 'or use a git-based build instead.', - ); - }; - waitForBuildToFinish = async (buildId: string, actorName: string): Promise => { const build = await this.apifyClient.build(buildId).waitForFinish(); const versionNumber = build.buildNumber; diff --git a/bin/utils.ts b/bin/utils.ts index d9f2ed1..8733794 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -4,6 +4,8 @@ import path from 'node:path'; import type { ActorVersionSourceFile } from 'apify-client'; +import { SOURCE_FILE_FORMATS } from '@apify/consts'; + import type { ActorConfig } from './types.js'; // Returns true when `childPath` is not inside `parentPath`. @@ -54,8 +56,8 @@ export const toSourceFile = async (absPath: string, rootDir: string): Promise { From 74289521ba654b19fe0022ef9b0a30896a524119 Mon Sep 17 00:00:00 2001 From: Gytis Scipokas Date: Fri, 10 Jul 2026 09:53:15 +0200 Subject: [PATCH 5/5] fix: applied revisions --- README.md | 6 +- bin/build-from-local.ts | 204 ++++++++++++++ bin/build.ts | 262 ++---------------- bin/main.ts | 7 +- bin/utils.ts | 31 ++- ...local.test.ts => build-from-local.test.ts} | 88 ++++-- 6 files changed, 322 insertions(+), 276 deletions(-) create mode 100644 bin/build-from-local.ts rename test/unit/bin/{build-local.test.ts => build-from-local.test.ts} (67%) diff --git a/README.md b/README.md index b16823b..cee23f2 100644 --- a/README.md +++ b/README.md @@ -348,12 +348,12 @@ Remove `--dry-run` to actually trigger builds and update the branch names/ The c #### Build from local source (no push needed) -If you don't want to push a dummy branch just to test a change and wait for all the tests to finish, `build-zip` builds Actors directly from your local files (zipped and uploaded as `SOURCE_FILES`), skipping steps 1-4 above. +If you don't want to push a dummy branch just to test a change and wait for all the tests to finish, `build-from-local` builds Actors directly from your local files (zipped and uploaded as `SOURCE_FILES`), skipping steps 1-4 above. ```bash APIFY_TOKEN_JOHN_DOE= \ GITHUB_WORKSPACE=. \ - npx apify-test-tools build-zip --actors john.doe/my-actor + npx apify-test-tools build-from-local --actors john.doe/my-actor ``` Pass a hardcoded actor name via `--actors` to build only that Actor (comma-separate multiple names). Omit `--actors` to build all Actors in the repo, or add `--dry-run` to preview without building. It outputs the same JSON build array as `build`, so you run tests against it the same way as in step 5 below: @@ -362,7 +362,7 @@ Pass a hardcoded actor name via `--actors` to build only that Actor (comma-separ # Build from local source and capture output BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \ GITHUB_WORKSPACE=. \ - npx apify-test-tools build-zip --actors apify/my-actor) + npx apify-test-tools build-from-local --actors apify/my-actor) ``` Since you already scoped the build to just the Actor(s) you care about, point vitest at a specific test file (or a `-t` name filter) instead of the whole `test/platform` directory — you get feedback on that one test without waiting for the full suite to run. diff --git a/bin/build-from-local.ts b/bin/build-from-local.ts new file mode 100644 index 0000000..e131264 --- /dev/null +++ b/bin/build-from-local.ts @@ -0,0 +1,204 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import type { ActorVersionSourceFile } from 'apify-client'; + +import { ApifyBuilder, waitAndSummarizeBuilds } from './build.js'; +import type { ActorConfig, BuildData } from './types.js'; +import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, toActorVersionSourceFile } from './utils.js'; + +// JUST IN CASE. File patterns that commonly hold credentials — never ship these into a build, regardless +// of sourceType or of whether the repo's .gitignore happens to list them. Everything else that should be +// excluded (build output, local overrides, project-specific secret files, ...) is expected to already be +// in the repo's .gitignore — see collectNonIgnoredFiles. +const SKIP_FILE_PATTERNS = [/^\.env(\..+)?$/, /\.pem$/, /\.key$/, /\.pfx$/, /\.p12$/]; +const isSecretFile = (fileName: string): boolean => SKIP_FILE_PATTERNS.some((pattern) => pattern.test(fileName)); + +export const collectSourceFiles = async (actorName: string, actorDir: string): Promise => { + const repoRoot = process.cwd(); + const absActorDir = path.resolve(actorDir); + + // Read actor.json to check if this is a monorepo actor with an external dockerContextDir. + // Monorepo actors point their dockerContextDir to a parent directory (e.g. "../../.."), + // which means the Docker build context is the repo root, not the actor directory itself. + const actorJsonPath = path.join(absActorDir, '.actor', 'actor.json'); + const actorJson = JSON.parse(await fs.readFile(actorJsonPath, 'utf8')) as Record; + const rawContextDir = actorJson.dockerContextDir as string | undefined; + const contextAbsDir = rawContextDir ? path.resolve(absActorDir, '.actor', rawContextDir) : undefined; + const isMonorepoActor = !!contextAbsDir && isOutsideDir(contextAbsDir, absActorDir); + + const collectRootDir = isMonorepoActor ? contextAbsDir! : absActorDir; + const keptFilePaths = collectNonIgnoredFiles(collectRootDir, repoRoot); + + if (!isMonorepoActor) { + return Promise.all(keptFilePaths.map(async (filePath) => toActorVersionSourceFile(filePath, collectRootDir))); + } + + const { tempDir, filePaths } = await flattenMonorepoContext( + actorName, + absActorDir, + contextAbsDir!, + actorJson, + keptFilePaths, + repoRoot, + ); + try { + return await Promise.all(filePaths.map(async (filePath) => toActorVersionSourceFile(filePath, tempDir))); + } finally { + // Only the flattened copy is temporary — never delete the actor's own directory. + await fs.rm(tempDir, { recursive: true, force: true }); + } +}; + +// Candidates come from `git ls-files` (tracked + untracked, gitignored included) rather than a +// manual directory walk — nested .gitignore files, `.git/info/exclude`, and global excludes are +// all honored since this delegates to git itself instead of re-implementing gitignore matching, +// and .git/ is never walked because git never lists its own internals here. `.actor/` (the Actor +// specification folder) is always kept regardless of .gitignore, matching Apify CLI's own behavior. +// Files matching the hardcoded secret-pattern backstop (keys, certs, .env variants) are dropped +// unconditionally, .actor/ included, since those should never ship regardless of what .gitignore says. +export const collectNonIgnoredFiles = (rootDir: string, repoRoot: string): string[] => { + const relativePaths = listRepoFilePaths(repoRoot, rootDir); + const ignoredPaths = getGitignoredPaths(relativePaths); + + return relativePaths + .filter((relPath) => { + if (isSecretFile(path.basename(relPath))) return false; + const isUnderActorDir = relPath.split('/').includes('.actor'); + return isUnderActorDir || !ignoredPaths.has(relPath); + }) + .map((relPath) => path.join(repoRoot, relPath)); +}; + +// SOURCE_FILES always treats the collected root as the actor root, so we cannot simply +// collect the actor directory of a monorepo actor — the platform would reject any path +// escaping it. Fix: create a temporary "flattened" directory where: +// - the Docker context's non-ignored files (repo root) are copied to the temp dir root +// - the actor's .actor/ directory is overlaid at the temp dir root (through the same +// gitignore/secret-pattern filter as the rest of the context — see collectNonIgnoredFiles) +// - actor.json path fields are rewritten to be relative to the new location +// +// Result: the collected root IS the Docker context, .actor/ is at that root, and +// all relative paths (dockerfile, dockerContextDir, changelog) are exactly one +// level up ("..") instead of three ("../../.."). +export const flattenMonorepoContext = async ( + actorName: string, + absActorDir: string, + contextAbsDir: string, + actorJson: Record, + keptContextFiles: string[], + repoRoot: string, +): Promise<{ tempDir: string; filePaths: string[] }> => { + console.error(`[${actorName}]: monorepo actor detected — flattening from Docker context`); + + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), `apify-build-${actorName.replace('/', '_')}-`)); + const filePaths: string[] = []; + + // Step 1: copy only the files that survived gitignore/secret filtering, preserving their + // position relative to the Docker context root. + await Promise.all( + keptContextFiles.map(async (absFilePath) => { + const relPath = path.relative(contextAbsDir, absFilePath); + const destPath = path.join(tempDir, relPath); + await fs.mkdir(path.dirname(destPath), { recursive: true }); + await fs.copyFile(absFilePath, destPath); + filePaths.push(destPath); + }), + ); + + // Step 2: overlay the actor's .actor/ directory at the temp dir root. collectNonIgnoredFiles + // always keeps .actor/ paths regardless of .gitignore, but still drops the hardcoded secret + // patterns — so this isn't a raw copy, a stray secret file living inside .actor/ is still dropped. + const actorMetaDir = path.join(absActorDir, '.actor'); + const keptActorFiles = collectNonIgnoredFiles(actorMetaDir, repoRoot); + await Promise.all( + keptActorFiles.map(async (absFilePath) => { + const relPath = path.relative(actorMetaDir, absFilePath); + const destPath = path.join(tempDir, '.actor', relPath); + await fs.mkdir(path.dirname(destPath), { recursive: true }); + await fs.copyFile(absFilePath, destPath); + filePaths.push(destPath); + }), + ); + + // Step 3: rewrite actor.json path fields so they resolve correctly from the new location. + // This overwrites the actor.json already copied in step 2 in place, so its path is already + // accounted for in filePaths — no need to add it again. + await rewriteActorJsonPaths(absActorDir, contextAbsDir, tempDir, actorJson); + + return { tempDir, filePaths }; +}; + +// Rewrites actor.json path fields so they resolve correctly from the new .actor/ location +// (one level below the root) instead of the original three-levels-deep location. +// +// Algorithm for each path field: +// 1. Resolve the original value to an absolute path on disk. +// 2. Compute its position relative to the Docker context root (e.g. repo root). +// That relative position is exactly where the file landed inside tempDir, +// because we copied contextAbsDir → tempDir in flattenMonorepoContext's step 1. +// 3. Build the new path from newActorDir to that file in tempDir. +// +// Local paths (e.g. "./dataset_schema.json") point inside .actor/ and are left +// unchanged — .actor/ was copied intact so those paths still resolve correctly. +export const rewriteActorJsonPaths = async ( + absActorDir: string, + contextAbsDir: string, + tempDir: string, + actorJson: Record, +): Promise => { + const originalActorDir = path.join(absActorDir, '.actor'); + const newActorDir = path.join(tempDir, '.actor'); + const pathFields = ['dockerfile', 'dockerContextDir', 'changelog', 'readme'] as const; + const rewritten = { ...actorJson }; + for (const field of pathFields) { + const value = rewritten[field]; + if (typeof value !== 'string') continue; + + const absPath = path.resolve(originalActorDir, value); + + // Skip paths that stay inside .actor/ — they don't need rewriting. + if (!isOutsideDir(absPath, originalActorDir)) continue; + + // Where does this file live inside the Docker context? That's also where + // it lives inside tempDir after the copy in flattenMonorepoContext's step 1. + const relativeToContext = path.relative(contextAbsDir, absPath); + const newAbsPath = path.join(tempDir, relativeToContext); + rewritten[field] = path.relative(newActorDir, newAbsPath); + } + await fs.writeFile(path.join(newActorDir, 'actor.json'), JSON.stringify(rewritten, null, 4)); +}; + +export const runBuildsFromLocal = async ({ + actorConfigs, + dryRun, +}: { + actorConfigs: ActorConfig[]; + dryRun: boolean; +}): Promise => { + if (dryRun) { + console.error('[DRY RUN] Would build from local source:'); + for (const { actorName, folder } of actorConfigs) { + console.error(` ${actorName} (${folder})`); + } + return actorConfigs.map(({ actorName }) => ({ + buildId: 'dry-run', + actorId: 'dry-run', + buildNumber: '0.98.0', + actorName, + })); + } + + console.error('========================================='); + console.error('STARTED LOCAL BUILDS:'); + const startedBuilds = await Promise.all( + actorConfigs.map(async ({ actorName, folder }) => { + const builder = ApifyBuilder.fromActorName(actorName); + const sourceFiles = await collectSourceFiles(actorName, folder); + return builder.startActorBuildFromSourceFiles(sourceFiles); + }), + ); + + return waitAndSummarizeBuilds(startedBuilds, 'LOCAL BUILDS'); +}; diff --git a/bin/build.ts b/bin/build.ts index aecf08e..8cf9f16 100644 --- a/bin/build.ts +++ b/bin/build.ts @@ -1,34 +1,9 @@ -import fs from 'node:fs/promises'; -import os from 'node:os'; -import path from 'node:path'; - import type { ActorVersionSourceFile, Build } from 'apify-client'; import { ApifyClient } from 'apify-client'; import { ACTOR_SOURCE_TYPES } from '@apify/consts'; import type { ActorConfig, BuildData } from './types.js'; -import { collectFilePaths, getGitignoredPaths, isOutsideDir, toSourceFile } from './utils.js'; - -const SKIP_DIRS = new Set([ - 'node_modules', - '.git', - 'apify_storage', - 'storage', - 'crawlee_storage', - 'dist', - 'build', - 'out', - '.next', - '.cache', -]); - -// JUST IN CASE. File patterns that commonly hold credentials — never ship these into a build, regardless -// of sourceType or of whether the repo's .gitignore happens to list them. Everything else that should be -// excluded (build output, local overrides, project-specific secret files, ...) is expected to already be -// in the repo's .gitignore — see getGitignoredPaths. -const SKIP_FILE_PATTERNS = [/^\.env(\..+)?$/, /\.pem$/, /\.key$/, /\.pfx$/, /\.p12$/]; -const isSecretFile = (fileName: string): boolean => SKIP_FILE_PATTERNS.some((pattern) => pattern.test(fileName)); type BuildPrActorOptions = { buildTag?: string; @@ -159,156 +134,6 @@ export class ApifyBuilder { return { buildId: id, actorId: actId, buildNumber, actorName: this.actorName }; }; - collectSourceFiles = async (actorDir: string): Promise => { - const repoRoot = process.cwd(); - const absActorDir = path.resolve(actorDir); - - // Read actor.json to check if this is a monorepo actor with an external dockerContextDir. - // Monorepo actors point their dockerContextDir to a parent directory (e.g. "../../.."), - // which means the Docker build context is the repo root, not the actor directory itself. - const actorJsonPath = path.join(absActorDir, '.actor', 'actor.json'); - const actorJson = JSON.parse(await fs.readFile(actorJsonPath, 'utf8')) as Record; - const rawContextDir = actorJson.dockerContextDir as string | undefined; - const contextAbsDir = rawContextDir ? path.resolve(absActorDir, '.actor', rawContextDir) : undefined; - const isMonorepoActor = !!contextAbsDir && isOutsideDir(contextAbsDir, absActorDir); - - const collectRootDir = isMonorepoActor ? contextAbsDir! : absActorDir; - const keptFilePaths = await this.collectNonIgnoredFiles(collectRootDir, repoRoot); - - const sourceRootDir = isMonorepoActor - ? await this.flattenMonorepoContext(absActorDir, contextAbsDir!, actorJson, keptFilePaths, repoRoot) - : collectRootDir; - - try { - // The flattened temp dir already contains exactly the files we want (kept files + - // the .actor/ overlay), so it's walked fresh; the non-monorepo case reuses keptFilePaths directly. - const filePaths = isMonorepoActor ? await collectFilePaths(sourceRootDir, SKIP_DIRS) : keptFilePaths; - const sourceFiles = await Promise.all( - filePaths.map(async (filePath) => toSourceFile(filePath, sourceRootDir)), - ); - - return sourceFiles; - } finally { - // Only the flattened copy is temporary — never delete the actor's own directory. - if (isMonorepoActor) { - await fs.rm(sourceRootDir, { recursive: true, force: true }); - } - } - }; - - // Walks `rootDir` and drops anything the repo's .gitignore excludes — nested .gitignore files, - // `.git/info/exclude`, and global excludes are all honored since this delegates to `git check-ignore` - // instead of re-implementing gitignore matching. `.actor/` (the Actor specification folder) is always - // kept regardless of .gitignore, matching Apify CLI's own behavior. Files matching the hardcoded - // secret-pattern backstop (keys, certs, .env variants) are dropped unconditionally, .actor/ included, - // since those should never ship regardless of what .gitignore says. - collectNonIgnoredFiles = async (rootDir: string, repoRoot: string): Promise => { - const candidatePaths = await collectFilePaths(rootDir, SKIP_DIRS); - const relativePaths = candidatePaths.map((absPath) => - path.relative(repoRoot, absPath).split(path.sep).join('/'), - ); - const ignoredPaths = getGitignoredPaths(relativePaths); - - return candidatePaths.filter((absPath, i) => { - if (isSecretFile(path.basename(absPath))) return false; - const isUnderActorDir = relativePaths[i].split('/').includes('.actor'); - return isUnderActorDir || !ignoredPaths.has(relativePaths[i]); - }); - }; - - // SOURCE_FILES always treats the collected root as the actor root, so we cannot simply - // collect the actor directory of a monorepo actor — the platform would reject any path - // escaping it. Fix: create a temporary "flattened" directory where: - // - the Docker context's non-ignored files (repo root) are copied to the temp dir root - // - the actor's .actor/ directory is overlaid at the temp dir root (through the same - // gitignore/secret-pattern filter as the rest of the context — see collectNonIgnoredFiles) - // - actor.json path fields are rewritten to be relative to the new location - // - // Result: the collected root IS the Docker context, .actor/ is at that root, and - // all relative paths (dockerfile, dockerContextDir, changelog) are exactly one - // level up ("..") instead of three ("../../.."). - flattenMonorepoContext = async ( - absActorDir: string, - contextAbsDir: string, - actorJson: Record, - keptContextFiles: string[], - repoRoot: string, - ): Promise => { - console.error(`[${this.actorName}]: monorepo actor detected — flattening from Docker context`); - - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), `apify-build-${this.actorName.replace('/', '_')}-`)); - - // Step 1: copy only the files that survived gitignore/secret filtering, preserving their - // position relative to the Docker context root. - await Promise.all( - keptContextFiles.map(async (absFilePath) => { - const relPath = path.relative(contextAbsDir, absFilePath); - const destPath = path.join(tempDir, relPath); - await fs.mkdir(path.dirname(destPath), { recursive: true }); - await fs.copyFile(absFilePath, destPath); - }), - ); - - // Step 2: overlay the actor's .actor/ directory at the temp dir root. collectNonIgnoredFiles - // always keeps .actor/ paths regardless of .gitignore, but still drops the hardcoded secret - // patterns — so this isn't a raw copy, a stray secret file living inside .actor/ is still dropped. - const actorMetaDir = path.join(absActorDir, '.actor'); - const keptActorFiles = await this.collectNonIgnoredFiles(actorMetaDir, repoRoot); - await Promise.all( - keptActorFiles.map(async (absFilePath) => { - const relPath = path.relative(actorMetaDir, absFilePath); - const destPath = path.join(tempDir, '.actor', relPath); - await fs.mkdir(path.dirname(destPath), { recursive: true }); - await fs.copyFile(absFilePath, destPath); - }), - ); - - // Step 3: rewrite actor.json path fields so they resolve correctly from the new location. - await this.rewriteActorJsonPaths(absActorDir, contextAbsDir, tempDir, actorJson); - - return tempDir; - }; - - // Rewrites actor.json path fields so they resolve correctly from the new .actor/ location - // (one level below the root) instead of the original three-levels-deep location. - // - // Algorithm for each path field: - // 1. Resolve the original value to an absolute path on disk. - // 2. Compute its position relative to the Docker context root (e.g. repo root). - // That relative position is exactly where the file landed inside tempDir, - // because we copied contextAbsDir → tempDir in flattenMonorepoContext's step 1. - // 3. Build the new path from newActorDir to that file in tempDir. - // - // Local paths (e.g. "./dataset_schema.json") point inside .actor/ and are left - // unchanged — .actor/ was copied intact so those paths still resolve correctly. - rewriteActorJsonPaths = async ( - absActorDir: string, - contextAbsDir: string, - tempDir: string, - actorJson: Record, - ): Promise => { - const originalActorDir = path.join(absActorDir, '.actor'); - const newActorDir = path.join(tempDir, '.actor'); - const pathFields = ['dockerfile', 'dockerContextDir', 'changelog', 'readme'] as const; - const rewritten = { ...actorJson }; - for (const field of pathFields) { - const value = rewritten[field]; - if (typeof value !== 'string') continue; - - const absPath = path.resolve(originalActorDir, value); - - // Skip paths that stay inside .actor/ — they don't need rewriting. - if (!isOutsideDir(absPath, originalActorDir)) continue; - - // Where does this file live inside the Docker context? That's also where - // it lives inside tempDir after the copy in flattenMonorepoContext's step 1. - const relativeToContext = path.relative(contextAbsDir, absPath); - const newAbsPath = path.join(tempDir, relativeToContext); - rewritten[field] = path.relative(newActorDir, newAbsPath); - } - await fs.writeFile(path.join(newActorDir, 'actor.json'), JSON.stringify(rewritten, null, 4)); - }; - waitForBuildToFinish = async (buildId: string, actorName: string): Promise => { const build = await this.apifyClient.build(buildId).waitForFinish(); const versionNumber = build.buildNumber; @@ -444,6 +269,26 @@ export class ApifyBuilder { } } +export const waitAndSummarizeBuilds = async (startedBuilds: BuildData[], label: string): Promise => { + console.error('========================================='); + console.error(`FINISHED ${label}:`); + await Promise.all( + startedBuilds.map(async (buildData) => { + const builder = ApifyBuilder.fromActorName(buildData.actorName); + await builder.waitForBuildToFinish(buildData.buildId, buildData.actorName); + }), + ); + + console.error('========================================='); + console.error('SUMMARY:'); + for (const buildData of startedBuilds.sort((a, b) => a.actorName.localeCompare(b.actorName))) { + console.error(`[${buildData.actorName}]: ${buildData.buildNumber}`); + } + console.error('========================================='); + + return startedBuilds; +}; + type RunBuildsOptions = { actorConfigs: ActorConfig[]; isLatest?: boolean; @@ -496,22 +341,8 @@ export const runBuilds = async ({ return buildData; }), ); - console.error('========================================='); - console.error('FINISHED BUILDS:'); - await Promise.all( - startedBuilds.map(async (buildData) => { - const builder = ApifyBuilder.fromActorName(buildData.actorName); - await builder.waitForBuildToFinish(buildData.buildId, buildData.actorName); - }), - ); - console.error('========================================='); - console.error('SUMMARY:'); - for (const buildData of startedBuilds.sort((a, b) => a.actorName.localeCompare(b.actorName))) { - console.error(`[${buildData.actorName}]: ${buildData.buildNumber} `); - } - console.error('========================================='); - return startedBuilds; + return waitAndSummarizeBuilds(startedBuilds, 'BUILDS'); }; export const deleteOldBuilds = async (actorConfigs: ActorConfig[]) => { @@ -519,54 +350,3 @@ export const deleteOldBuilds = async (actorConfigs: ActorConfig[]) => { await ApifyBuilder.fromActorName(actorName).deleteOldBuilds(); } }; - -export const runZipBuilds = async ({ - actorConfigs, - dryRun, -}: { - actorConfigs: ActorConfig[]; - dryRun: boolean; -}): Promise => { - if (dryRun) { - console.error('[DRY RUN] Would build from local source:'); - for (const { actorName, folder } of actorConfigs) { - console.error(` ${actorName} (${folder})`); - } - return actorConfigs.map(({ actorName }) => ({ - buildId: 'dry-run', - actorId: 'dry-run', - buildNumber: '0.98.0', - actorName, - })); - } - - console.error('========================================='); - console.error('STARTED ZIP BUILDS:'); - const startedBuilds = await Promise.all( - actorConfigs.map(async ({ actorName, folder }) => { - const builder = ApifyBuilder.fromActorName(actorName); - const sourceFiles = await builder.collectSourceFiles(folder); - return builder.startActorBuildFromSourceFiles(sourceFiles); - }), - ); - - console.error('========================================='); - console.error('FINISHED ZIP BUILDS:'); - await Promise.all( - startedBuilds.map(async (buildData: BuildData) => { - const builder = ApifyBuilder.fromActorName(buildData.actorName); - await builder.waitForBuildToFinish(buildData.buildId, buildData.actorName); - }), - ); - - console.error('========================================='); - console.error('SUMMARY:'); - for (const buildData of startedBuilds.sort((a: BuildData, b: BuildData) => - a.actorName.localeCompare(b.actorName), - )) { - console.error(`[${buildData.actorName}]: ${buildData.buildNumber}`); - } - console.error('========================================='); - - return startedBuilds; -}; diff --git a/bin/main.ts b/bin/main.ts index 6809fac..44d80bd 100644 --- a/bin/main.ts +++ b/bin/main.ts @@ -6,7 +6,8 @@ import yargs, { type Argv } from 'yargs'; // eslint-disable-next-line import/extensions --- With .js, it cannot find types import { hideBin } from 'yargs/helpers'; -import { deleteOldBuilds, runBuilds, runZipBuilds } from './build.js'; +import { deleteOldBuilds, runBuilds } from './build.js'; +import { runBuildsFromLocal } from './build-from-local.js'; import { getChangedActors } from './diff-changes.js'; import { getBranchOnlyChangedFiles, getChangedFiles, getCommits, hasMergeFromTarget } from './git.js'; import { getPushData } from './github.js'; @@ -203,7 +204,7 @@ await yargs() }, ) .command( - 'build-zip', + 'build-from-local', '', (args) => args @@ -223,7 +224,7 @@ await yargs() return config; }) : allActorConfigs; - const builds = await runZipBuilds({ actorConfigs, dryRun }); + const builds = await runBuildsFromLocal({ actorConfigs, dryRun }); console.log(JSON.stringify(builds)); }, ) diff --git a/bin/utils.ts b/bin/utils.ts index 8733794..1c12086 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -13,18 +13,25 @@ import type { ActorConfig } from './types.js'; export const isOutsideDir = (childPath: string, parentPath: string): boolean => path.relative(parentPath, childPath).startsWith('..'); -export const collectFilePaths = async (rootDir: string, skipDirs: Set): Promise => { - const entries = await fs.readdir(rootDir, { withFileTypes: true }); - const filePaths: string[] = []; - for (const entry of entries) { - if (entry.isDirectory()) { - if (skipDirs.has(entry.name)) continue; - filePaths.push(...(await collectFilePaths(path.join(rootDir, entry.name), skipDirs))); - } else if (entry.isFile()) { - filePaths.push(path.join(rootDir, entry.name)); - } +/** + * Lists every file under `subDir` (paths relative to `repoRoot`) that's either tracked by git or + * present but untracked in the working tree — deliberately omitting `--exclude-standard`, so + * gitignored files are included too. Callers combine this with getGitignoredPaths to decide what + * to keep, e.g. because .actor/ must survive even if .gitignore would otherwise exclude it. + * This also means .git/ itself is never walked, since git never lists its own internals here. + */ +export const listRepoFilePaths = (repoRoot: string, subDir: string): string[] => { + const relSubDir = path.relative(repoRoot, subDir).split(path.sep).join('/') || '.'; + const result = spawnSync('git', ['ls-files', '--cached', '--others', '-z', '--', relSubDir], { + cwd: repoRoot, + maxBuffer: 100 * 1024 * 1024, + }); + + if (result.status !== 0) { + throw new Error(`[Command failed]: git ls-files\n${result.stderr.toString()}`); } - return filePaths; + + return result.stdout.toString().split('\0').filter(Boolean); }; /** @@ -52,7 +59,7 @@ export const getGitignoredPaths = (relativePaths: string[]): Set => { const isBinary = (buffer: Buffer): boolean => buffer.includes(0); -export const toSourceFile = async (absPath: string, rootDir: string): Promise => { +export const toActorVersionSourceFile = async (absPath: string, rootDir: string): Promise => { const buffer = await fs.readFile(absPath); const name = path.relative(rootDir, absPath).split(path.sep).join('/'); return isBinary(buffer) diff --git a/test/unit/bin/build-local.test.ts b/test/unit/bin/build-from-local.test.ts similarity index 67% rename from test/unit/bin/build-local.test.ts rename to test/unit/bin/build-from-local.test.ts index 3fac2d0..744c95f 100644 --- a/test/unit/bin/build-local.test.ts +++ b/test/unit/bin/build-from-local.test.ts @@ -1,3 +1,4 @@ +import type * as ChildProcessModule from 'node:child_process'; import { spawnSync } from 'node:child_process'; import fs from 'node:fs/promises'; import os from 'node:os'; @@ -8,29 +9,32 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore: editor-only TS6059 — test/tsconfig.json's rootDir doesn't span bin/, but the root // tsconfig (used for the real build and for eslint's type-aware linting) has no such restriction. -import { ApifyBuilder } from '../../../bin/build.js'; +import { + collectNonIgnoredFiles, + flattenMonorepoContext, + rewriteActorJsonPaths, +} from '../../../bin/build-from-local.js'; import * as Utils from '../../../bin/utils.js'; -vi.mock('node:child_process', () => ({ - spawnSync: vi.fn(), -})); - -const APIFY_TOKEN_ENV_VAR = 'APIFY_TOKEN_TEST'; +// Defaults to the real spawnSync so `git init`/`git ls-files` calls made by the code under test +// (and by test setup below) actually run — individual tests override this via mockReturnValue +// where they need to fake git's output, and vi.restoreAllMocks() reverts back to this passthrough. +vi.mock('node:child_process', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, spawnSync: vi.fn(actual.spawnSync) }; +}); const mkTempDir = async (prefix: string) => fs.mkdtemp(path.join(os.tmpdir(), prefix)); -describe('ApifyBuilder', () => { - let builder: ApifyBuilder; - const tempDirs: string[] = []; +const initGitRepo = (dir: string) => { + spawnSync('git', ['init', '-q'], { cwd: dir }); +}; - beforeEach(() => { - process.env[APIFY_TOKEN_ENV_VAR] = 'dummy-token'; - builder = ApifyBuilder.fromActorName('test/actor'); - }); +describe('build-from-local helpers', () => { + const tempDirs: string[] = []; afterEach(async () => { vi.restoreAllMocks(); - delete process.env[APIFY_TOKEN_ENV_VAR]; await Promise.all(tempDirs.splice(0).map(async (dir) => fs.rm(dir, { recursive: true, force: true }))); }); @@ -38,6 +42,7 @@ describe('ApifyBuilder', () => { it('drops secret-pattern files and gitignored files, keeps everything else', async () => { const rootDir = await mkTempDir('apify-test-tools-collect-'); tempDirs.push(rootDir); + initGitRepo(rootDir); await fs.writeFile(path.join(rootDir, 'main.js'), 'console.log(1)'); await fs.writeFile(path.join(rootDir, '.env'), 'SECRET=1'); @@ -50,7 +55,7 @@ describe('ApifyBuilder', () => { (relativePaths) => new Set(relativePaths.filter((p) => p.endsWith('.log'))), ); - const result = await builder.collectNonIgnoredFiles(rootDir, rootDir); + const result = collectNonIgnoredFiles(rootDir, rootDir); expect(result).toStrictEqual([path.join(rootDir, 'main.js')]); }); @@ -60,6 +65,7 @@ describe('ApifyBuilder', () => { it('filters the .actor/ overlay through the same secret-pattern check as the rest of the context', async () => { const repoRoot = await mkTempDir('apify-test-tools-repo-'); tempDirs.push(repoRoot); + initGitRepo(repoRoot); const absActorDir = path.join(repoRoot, 'actors', 'owner_actor'); await fs.mkdir(path.join(absActorDir, '.actor'), { recursive: true }); @@ -81,7 +87,8 @@ describe('ApifyBuilder', () => { await fs.readFile(path.join(absActorDir, '.actor', 'actor.json'), 'utf8'), ) as Record; - const flattenedDir = await builder.flattenMonorepoContext( + const { tempDir: flattenedDir, filePaths } = await flattenMonorepoContext( + 'test/actor', absActorDir, repoRoot, actorJson, @@ -94,6 +101,53 @@ describe('ApifyBuilder', () => { await expect(fs.access(path.join(flattenedDir, '.actor', 'INPUT_SCHEMA.json'))).resolves.toBeUndefined(); await expect(fs.access(path.join(flattenedDir, '.actor', 'actor.json'))).resolves.toBeUndefined(); await expect(fs.access(path.join(flattenedDir, 'package.json'))).resolves.toBeUndefined(); + + // The returned filePaths must match what actually landed on disk — no .env, everything else present. + expect(new Set(filePaths)).toStrictEqual( + new Set([ + path.join(flattenedDir, 'package.json'), + path.join(flattenedDir, '.actor', 'INPUT_SCHEMA.json'), + path.join(flattenedDir, '.actor', 'actor.json'), + ]), + ); + }); + + it("keeps another actor's .actor/ directory intact at its original nested path", async () => { + const repoRoot = await mkTempDir('apify-test-tools-repo-'); + tempDirs.push(repoRoot); + initGitRepo(repoRoot); + + const absActorDir = path.join(repoRoot, 'actors', 'owner_actor'); + await fs.mkdir(path.join(absActorDir, '.actor'), { recursive: true }); + await fs.writeFile( + path.join(absActorDir, '.actor', 'actor.json'), + JSON.stringify({ actorSpecification: 1, name: 'actor' }), + ); + + const otherActorDir = path.join(repoRoot, 'actors', 'owner_other-actor'); + await fs.mkdir(path.join(otherActorDir, '.actor'), { recursive: true }); + const otherActorSchemaFile = path.join(otherActorDir, '.actor', 'input_schema.json'); + await fs.writeFile(otherActorSchemaFile, JSON.stringify({ schema: 'other' })); + + vi.spyOn(Utils, 'getGitignoredPaths').mockReturnValue(new Set()); + + const actorJson = JSON.parse( + await fs.readFile(path.join(absActorDir, '.actor', 'actor.json'), 'utf8'), + ) as Record; + + const { tempDir: flattenedDir, filePaths } = await flattenMonorepoContext( + 'test/actor', + absActorDir, + repoRoot, + actorJson, + [otherActorSchemaFile], + repoRoot, + ); + tempDirs.push(flattenedDir); + + const preservedPath = path.join(flattenedDir, 'actors', 'owner_other-actor', '.actor', 'input_schema.json'); + await expect(fs.access(preservedPath)).resolves.toBeUndefined(); + expect(filePaths).toContain(preservedPath); }); }); @@ -115,7 +169,7 @@ describe('ApifyBuilder', () => { changelog: './CHANGELOG.md', // stays inside .actor/ }; - await builder.rewriteActorJsonPaths(absActorDir, repoRoot, flattenedDir, actorJson); + await rewriteActorJsonPaths(absActorDir, repoRoot, flattenedDir, actorJson); const rewritten = JSON.parse( await fs.readFile(path.join(flattenedDir, '.actor', 'actor.json'), 'utf8'),