diff --git a/bin/build-from-local.ts b/bin/build-from-local.ts index e131264..0c3d3f2 100644 --- a/bin/build-from-local.ts +++ b/bin/build-from-local.ts @@ -6,7 +6,13 @@ import type { ActorVersionSourceFile } from 'apify-client'; import { ApifyBuilder, waitAndSummarizeBuilds } from './build.js'; import type { ActorConfig, BuildData } from './types.js'; -import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, toActorVersionSourceFile } from './utils.js'; +import { + getDockerignoredPaths, + getGitignoredPaths, + isOutsideDir, + listRepoFilePaths, + toActorVersionSourceFile, +} from './utils.js'; // JUST IN CASE. File patterns that commonly hold credentials — never ship these into a build, regardless // of sourceType or of whether the repo's .gitignore happens to list them. Everything else that should be @@ -54,19 +60,30 @@ export const collectSourceFiles = async (actorName: string, actorDir: string): P // Candidates come from `git ls-files` (tracked + untracked, gitignored included) rather than a // manual directory walk — nested .gitignore files, `.git/info/exclude`, and global excludes are // all honored since this delegates to git itself instead of re-implementing gitignore matching, -// and .git/ is never walked because git never lists its own internals here. `.actor/` (the Actor -// specification folder) is always kept regardless of .gitignore, matching Apify CLI's own behavior. -// Files matching the hardcoded secret-pattern backstop (keys, certs, .env variants) are dropped -// unconditionally, .actor/ included, since those should never ship regardless of what .gitignore says. +// and .git/ is never walked because git never lists its own internals here. `.dockerignore` at +// `rootDir` (the Docker build context) is honored the same way, since those files would never +// reach a real Docker build either. `.actor/` (the Actor specification folder) is always kept +// regardless of .gitignore/.dockerignore, matching Apify CLI's own behavior. Files matching the +// hardcoded secret-pattern backstop (keys, certs, .env variants) are dropped unconditionally, +// .actor/ included, since those should never ship regardless of what the ignore files say. export const collectNonIgnoredFiles = (rootDir: string, repoRoot: string): string[] => { const relativePaths = listRepoFilePaths(repoRoot, rootDir); const ignoredPaths = getGitignoredPaths(relativePaths); + const rootRelativePaths = new Map( + relativePaths.map((relPath) => [ + relPath, + path.relative(rootDir, path.join(repoRoot, relPath)).split(path.sep).join('/'), + ]), + ); + const dockerIgnoredPaths = getDockerignoredPaths(rootDir, [...rootRelativePaths.values()]); return relativePaths .filter((relPath) => { if (isSecretFile(path.basename(relPath))) return false; const isUnderActorDir = relPath.split('/').includes('.actor'); - return isUnderActorDir || !ignoredPaths.has(relPath); + if (isUnderActorDir) return true; + if (ignoredPaths.has(relPath)) return false; + return !dockerIgnoredPaths.has(rootRelativePaths.get(relPath)!); }) .map((relPath) => path.join(repoRoot, relPath)); }; diff --git a/bin/utils.ts b/bin/utils.ts index 1c12086..841b6aa 100644 --- a/bin/utils.ts +++ b/bin/utils.ts @@ -1,5 +1,7 @@ import { spawnSync } from 'node:child_process'; +import fsSync from 'node:fs'; import fs from 'node:fs/promises'; +import os from 'node:os'; import path from 'node:path'; import type { ActorVersionSourceFile } from 'apify-client'; @@ -57,6 +59,65 @@ export const getGitignoredPaths = (relativePaths: string[]): Set => { return new Set(result.stdout.toString().split('\n').filter(Boolean)); }; +// Docker normalizes each pattern before matching, so a leading "./" (as in the common "./node_modules" +// style) is a no-op for Docker. git's ignore engine has no such normalization — it treats "./" as +// literal pattern text that can never match a real path, so a .dockerignore written in that style +// would otherwise silently match nothing under git. Strip it here (after any negation prefix) so +// git sees the same effective pattern Docker would. +const normalizeDockerignorePattern = (line: string): string => line.replace(/^(!?)(?:\.\/)+/, '$1'); + +/** + * Given paths relative to `rootDir`, returns the subset that a `.dockerignore` file sitting at + * `rootDir` would exclude — mirroring getGitignoredPaths, but for Docker's own ignore file. `git + * check-ignore` has no built-in notion of .dockerignore, but pointing `core.excludesFile` at a + * normalized copy of it for a single invocation makes git apply its patterns exactly like a + * .gitignore, without re-implementing gitignore-style pattern matching ourselves. A missing + * .dockerignore is a no-op (empty set), so there's no need to check for its existence first. + * + * Crucially this passes `--no-index`: by default git never reports an already-tracked file as + * ignored (that's how real .gitignore semantics work — tracked files aren't affected by ignore + * rules), but Docker excludes a matching path from the build context unconditionally, regardless + * of git tracking. `--no-index` makes git apply the patterns uniformly, matching Docker's behavior. + */ +export const getDockerignoredPaths = (rootDir: string, relativePaths: string[]): Set => { + if (relativePaths.length === 0) return new Set(); + + let dockerignoreContent: string; + try { + dockerignoreContent = fsSync.readFileSync(path.join(rootDir, '.dockerignore'), 'utf8'); + } catch { + return new Set(); + } + + const normalizedContent = dockerignoreContent.split('\n').map(normalizeDockerignorePattern).join('\n'); + + const tempDir = fsSync.mkdtempSync(path.join(os.tmpdir(), 'apify-dockerignore-')); + try { + const normalizedPath = path.join(tempDir, '.dockerignore'); + fsSync.writeFileSync(normalizedPath, normalizedContent); + + const result = spawnSync( + 'git', + ['-c', `core.excludesFile=${normalizedPath}`, 'check-ignore', '--no-index', '--stdin'], + { + cwd: rootDir, + input: relativePaths.join('\n'), + maxBuffer: 100 * 1024 * 1024, + }, + ); + + // Exit code 1 means none of the given paths are ignored - not an error. Anything else + // (e.g. 128 for "not a git repository") is a real failure. + if (result.status !== 0 && result.status !== 1) { + throw new Error(`[Command failed]: git check-ignore (dockerignore)\n${result.stderr.toString()}`); + } + + return new Set(result.stdout.toString().split('\n').filter(Boolean)); + } finally { + fsSync.rmSync(tempDir, { recursive: true, force: true }); + } +}; + const isBinary = (buffer: Buffer): boolean => buffer.includes(0); export const toActorVersionSourceFile = async (absPath: string, rootDir: string): Promise => { diff --git a/test/unit/bin/build-from-local.test.ts b/test/unit/bin/build-from-local.test.ts index 744c95f..31aa0ac 100644 --- a/test/unit/bin/build-from-local.test.ts +++ b/test/unit/bin/build-from-local.test.ts @@ -6,13 +6,13 @@ import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore: editor-only TS6059 — test/tsconfig.json's rootDir doesn't span bin/, but the root -// tsconfig (used for the real build and for eslint's type-aware linting) has no such restriction. import { collectNonIgnoredFiles, flattenMonorepoContext, rewriteActorJsonPaths, + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore: editor-only TS6059 — test/tsconfig.json's rootDir doesn't span bin/, but the root + // tsconfig (used for the real build and for eslint's type-aware linting) has no such restriction. } from '../../../bin/build-from-local.js'; import * as Utils from '../../../bin/utils.js'; @@ -182,6 +182,83 @@ describe('build-from-local helpers', () => { }); }); +describe('getDockerignoredPaths', () => { + const tempDirs: string[] = []; + + afterEach(async () => { + vi.mocked(spawnSync).mockClear(); + await Promise.all(tempDirs.splice(0).map(async (dir) => fs.rm(dir, { recursive: true, force: true }))); + }); + + it('returns an empty set without calling git when given no paths', () => { + const result = Utils.getDockerignoredPaths('/does/not/matter', []); + + expect(result).toStrictEqual(new Set()); + expect(spawnSync).not.toHaveBeenCalled(); + }); + + it('returns an empty set when rootDir has no .dockerignore', async () => { + const rootDir = await mkTempDir('apify-test-tools-dockerignore-'); + tempDirs.push(rootDir); + initGitRepo(rootDir); + + expect(Utils.getDockerignoredPaths(rootDir, ['main.js'])).toStrictEqual(new Set()); + }); + + it('returns the paths that match .dockerignore patterns rooted at rootDir', async () => { + const rootDir = await mkTempDir('apify-test-tools-dockerignore-'); + tempDirs.push(rootDir); + initGitRepo(rootDir); + + await fs.writeFile(path.join(rootDir, '.dockerignore'), 'test/\n*.log\n'); + + const result = Utils.getDockerignoredPaths(rootDir, ['main.js', 'test/fixture.json', 'debug.log']); + + expect(result).toStrictEqual(new Set(['test/fixture.json', 'debug.log'])); + }); + + it('matches "./"-prefixed patterns, which Docker treats as a no-op but git treats as literal text', async () => { + const rootDir = await mkTempDir('apify-test-tools-dockerignore-'); + tempDirs.push(rootDir); + initGitRepo(rootDir); + + // The common real-world .dockerignore style: every pattern prefixed with "./", + // including a negation re-including one of the otherwise-matched files. + await fs.writeFile(path.join(rootDir, '.dockerignore'), './node_modules\n./*.log\n!./keep.log\n'); + + const result = Utils.getDockerignoredPaths(rootDir, [ + 'node_modules/foo.js', + 'debug.log', + 'keep.log', + 'main.js', + ]); + + expect(result).toStrictEqual(new Set(['node_modules/foo.js', 'debug.log'])); + }); + + it('matches already-tracked files too, since Docker excludes them regardless of git tracking', async () => { + const rootDir = await mkTempDir('apify-test-tools-dockerignore-'); + tempDirs.push(rootDir); + initGitRepo(rootDir); + spawnSync('git', ['config', 'user.email', 'test@example.com'], { cwd: rootDir }); + spawnSync('git', ['config', 'user.name', 'Test'], { cwd: rootDir }); + + // Husky hooks are committed to the repo, so this path is tracked by git — by default + // `git check-ignore` never reports a tracked file as ignored, which would otherwise mask + // this exact pattern from matching, unlike a real Docker build which excludes it anyway. + await fs.mkdir(path.join(rootDir, '.husky')); + await fs.writeFile(path.join(rootDir, '.husky', 'pre-commit'), '#!/bin/sh\n'); + spawnSync('git', ['add', '.husky/pre-commit'], { cwd: rootDir }); + spawnSync('git', ['commit', '-q', '-m', 'add husky hook'], { cwd: rootDir }); + + await fs.writeFile(path.join(rootDir, '.dockerignore'), './.husky\n'); + + const result = Utils.getDockerignoredPaths(rootDir, ['.husky/pre-commit', 'main.js']); + + expect(result).toStrictEqual(new Set(['.husky/pre-commit'])); + }); +}); + describe('getGitignoredPaths', () => { beforeEach(() => { vi.mocked(spawnSync).mockReset();