Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions bin/copytree.js
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,25 @@ program
.option('--no-instructions', 'Disable including instructions in output')
.option('--instructions <name>', 'Use custom instructions set (default: default)')
.option('--no-validate', 'Disable configuration validation')
.option(
'--ext <extensions>',
'Filter by file extensions, comma-separated (e.g., .js,.ts,.tsx or js,ts)',
)
.option(
'--max-depth <n>',
'Maximum directory traversal depth (0 = root files only, 1 = one level deep, etc.)',
(val) => {
const n = parseInt(val, 10);
if (isNaN(n) || n < 0) {
throw new InvalidArgumentError(
`'${val}' is not a valid depth. Must be a non-negative integer.`,
);
}
return n;
},
)
.option('--min-size <size>', 'Exclude files smaller than this size (e.g., 1KB, 500B, 10MB)')
.option('--max-size <size>', 'Exclude files larger than this size (e.g., 10MB, 1GB)')
.option('--secrets-guard', 'Enable automatic secret detection and redaction (default: enabled)')
.option('--no-secrets-guard', 'Disable secret detection and redaction')
.option(
Expand Down
49 changes: 48 additions & 1 deletion src/commands/copy.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { fileURLToPath } from 'url';
import { summarize as getFsErrorSummary, reset as resetFsErrors } from '../utils/fsErrorReport.js';
import FolderProfileLoader from '../config/FolderProfileLoader.js';
import { Profiler, writeProfilingReport } from '../utils/profiler.js';
import { parseSize } from '../utils/helpers.js';

// Lazy initialization for Jest compatibility
let __filename, __dirname, pkg;
Expand Down Expand Up @@ -237,8 +238,44 @@ async function copyCommand(targetPath = '.', options = {}) {
}

/**
* Load and prepare profile
* Parse comma-separated file extensions into a normalized array.
* Accepts formats like ".js,.ts" or "js,ts" (with or without leading dot).
* Returns an array of lowercase extensions with leading dots, e.g., ['.js', '.ts'].
*
* @param {string} extStr - Comma-separated extension string from --ext flag
* @returns {string[]} Normalized extensions array
*/
function parseExtensions(extStr) {
const exts = extStr
.split(',')
.map((e) => e.trim())
.filter(Boolean)
.map((e) => (e.startsWith('.') ? e.toLowerCase() : `.${e.toLowerCase()}`));
if (exts.length === 0) {
throw new CommandError(
`Invalid --ext value '${extStr}'. Provide at least one extension, e.g., .js,.ts`,
);
}
return exts;
}

/**
* Parse a human-readable size string to bytes, throwing a CommandError on invalid input.
*
* @param {string} sizeStr - Size string (e.g., '1KB', '10MB')
* @param {string} flagName - Flag name for error messages
* @returns {number} Size in bytes
*/
function parseSizeOption(sizeStr, flagName) {
try {
return parseSize(sizeStr);
} catch {
throw new CommandError(
`Invalid --${flagName} value '${sizeStr}'. Use a format like 1KB, 500B, 10MB, 1GB.`,
);
}
}

/**
* Build profile configuration from CLI options, folder profiles, and config defaults
* Integrates the new FolderProfileLoader system
Expand Down Expand Up @@ -320,6 +357,11 @@ async function buildProfileFromCliOptions(options) {
maxFileSize: options.maxFileSize ?? copytreeConfig.maxFileSize,
maxTotalSize: options.maxTotalSize ?? copytreeConfig.maxTotalSize,
maxFileCount: options.maxFileCount ?? copytreeConfig.maxFileCount,
// Convenience filter flags
extFilter: options.ext ? parseExtensions(options.ext) : null,
maxDepth: options.maxDepth !== undefined ? options.maxDepth : null,
minSizeBytes: options.minSize ? parseSizeOption(options.minSize, 'min-size') : null,
maxSizeBytes: options.maxSize ? parseSizeOption(options.maxSize, 'max-size') : null,
},

// Transformer configuration
Expand Down Expand Up @@ -380,6 +422,11 @@ async function setupPipelineStages(basePath, profile, options) {
maxTotalSize: profile.options?.maxTotalSize,
maxFileCount: profile.options?.maxFileCount,
forceInclude: mergedAlways,
// Convenience filter flags
extFilter: profile.options?.extFilter ?? null,
maxDepth: profile.options?.maxDepth ?? null,
minSizeBytes: profile.options?.minSizeBytes ?? null,
maxSizeBytes: profile.options?.maxSizeBytes ?? null,
}),
);

Expand Down
39 changes: 36 additions & 3 deletions src/pipeline/stages/FileDiscoveryStage.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,18 @@ class FileDiscoveryStage extends Stage {
this.respectGitignore = options.respectGitignore !== false;
this.forceInclude = options.forceInclude || [];
this.excludes = options.excludes || [];
// Convenience filter options
this.extFilter = options.extFilter || null; // e.g. ['.js', '.ts']
this.maxDepth =
options.maxDepth !== undefined && options.maxDepth !== null ? options.maxDepth : null;
this.minSizeBytes =
options.minSizeBytes !== undefined && options.minSizeBytes !== null
? options.minSizeBytes
: null;
this.maxSizeBytes =
options.maxSizeBytes !== undefined && options.maxSizeBytes !== null
? options.maxSizeBytes
: null;
}

async process(input) {
Expand Down Expand Up @@ -158,6 +170,7 @@ class FileDiscoveryStage extends Stage {
explain: this.options.explain || false,
initialLayers,
config: this.config.all(), // Pass full config for retry settings
maxDepth: this.maxDepth !== null ? this.maxDepth : undefined,
};

// Add parallel-specific options if enabled
Expand All @@ -183,10 +196,21 @@ class FileDiscoveryStage extends Stage {
if (!matches) continue;
}

// Apply extension filter (--ext)
if (this.extFilter && this.extFilter.length > 0) {
const ext = path.extname(relativePath).toLowerCase();
if (!this.extFilter.includes(ext)) continue;
}

// Apply size filters (--min-size, --max-size)
const fileSize = fileInfo.stats.size;
if (this.minSizeBytes !== null && fileSize < this.minSizeBytes) continue;
if (this.maxSizeBytes !== null && fileSize > this.maxSizeBytes) continue;

discoveredFiles.push({
path: relativePath,
absolutePath: fileInfo.path,
size: fileInfo.stats.size,
size: fileSize,
modified: fileInfo.stats.mtime,
stats: fileInfo.stats,
});
Expand Down Expand Up @@ -232,8 +256,17 @@ class FileDiscoveryStage extends Stage {
const byPath = new Map(merged.map((f) => [f.path, f]));
const finalFiles = [...byPath.values()];

const filterDesc = [
this.extFilter ? `ext: ${this.extFilter.join(',')}` : null,
this.maxDepth !== null ? `max-depth: ${this.maxDepth}` : null,
this.minSizeBytes !== null ? `min-size: ${this.minSizeBytes}B` : null,
this.maxSizeBytes !== null ? `max-size: ${this.maxSizeBytes}B` : null,
]
.filter(Boolean)
.join(', ');

this.log(
`Discovered ${finalFiles.length} files (${forcedEntries.length} force-included) in ${this.getElapsedTime(startTime)}`,
`Discovered ${finalFiles.length} files (${forcedEntries.length} force-included)${filterDesc ? ` [filters: ${filterDesc}]` : ''} in ${this.getElapsedTime(startTime)}`,
'info',
);

Expand All @@ -243,7 +276,7 @@ class FileDiscoveryStage extends Stage {
files: finalFiles,
stats: {
totalFiles: discoveredFiles.length,
filteredFiles: discoveredFiles.length, // No post-filtering needed
filteredFiles: discoveredFiles.length,
forcedFiles: forcedEntries.length,
excludedFiles: 0, // Not tracked with walker approach
},
Expand Down
9 changes: 6 additions & 3 deletions src/utils/ignoreWalker.js
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ export async function* walkWithIgnore(root, options = {}) {
initialLayers = [],
config = {},
cache = false,
maxDepth = undefined,
} = options;

// Extract retry configuration with defaults
Expand All @@ -162,7 +163,7 @@ export async function* walkWithIgnore(root, options = {}) {
stats.directoriesPruned = 0;
stats.filesExcluded = 0;

async function* walk(dir, layers) {
async function* walk(dir, layers, depth = 0) {
stats.directoriesScanned++;

// Load ignore rules at this level
Expand Down Expand Up @@ -262,8 +263,10 @@ export async function* walkWithIgnore(root, options = {}) {
yield result;
}

// Recurse into subdirectory
yield* walk(absPath, nextLayers);
// Recurse into subdirectory (respect maxDepth if set)
if (maxDepth === undefined || depth < maxDepth) {
yield* walk(absPath, nextLayers, depth + 1);
}
} else {
stats.filesScanned++;

Expand Down
21 changes: 13 additions & 8 deletions src/utils/parallelWalker.js
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ export async function* walkParallel(root, options = {}) {
concurrency = 5,
highWaterMark = concurrency * 2,
signal,
maxDepth = undefined,
} = options;

// Extract retry configuration with defaults
Expand Down Expand Up @@ -213,8 +214,9 @@ export async function* walkParallel(root, options = {}) {
* @param {string} dir - Parent directory path
* @param {fs.Dirent} entry - Directory entry
* @param {Array} layers - Current ignore layers
* @param {number} depth - Current traversal depth (0 = root)
*/
async function processEntry(dir, entry, layers) {
async function processEntry(dir, entry, layers, depth) {
if (signal?.aborted) {
throw new Error('Traversal aborted');
}
Expand Down Expand Up @@ -288,8 +290,10 @@ export async function* walkParallel(root, options = {}) {
queuedResult = result;
}

// Add subdirectory to queue for processing
queue.push({ dir: absPath, layers });
// Add subdirectory to queue for processing (respect maxDepth)
if (maxDepth === undefined || depth < maxDepth) {
queue.push({ dir: absPath, layers, depth: depth + 1 });
}
return queuedResult;
} else {
stats.filesScanned++;
Expand Down Expand Up @@ -329,8 +333,9 @@ export async function* walkParallel(root, options = {}) {
* Process a single directory: read entries and schedule them
* @param {string} dir - Directory path
* @param {Array} layers - Current ignore layers
* @param {number} depth - Current traversal depth (0 = root)
*/
async function processDirectory(dir, layers) {
async function processDirectory(dir, layers, depth) {
stats.directoriesScanned++;

// Load ignore rules at this level
Expand Down Expand Up @@ -368,13 +373,13 @@ export async function* walkParallel(root, options = {}) {

// Process all entries in this directory in parallel (bounded by limit)
const entryResults = await Promise.all(
entries.map((entry) => limit(() => processEntry(dir, entry, nextLayers))),
entries.map((entry) => limit(() => processEntry(dir, entry, nextLayers, depth))),
);
return entryResults.filter(Boolean);
}

// Initialize queue with root directory
queue.push({ dir: root, layers: initialLayers });
queue.push({ dir: root, layers: initialLayers, depth: 0 });

// Main traversal loop
try {
Expand All @@ -386,8 +391,8 @@ export async function* walkParallel(root, options = {}) {
}

while (queue.length > 0 && running.size < concurrency) {
const { dir, layers } = queue.shift();
const task = processDirectory(dir, layers).then(async (results) => {
const { dir, layers, depth } = queue.shift();
const task = processDirectory(dir, layers, depth).then(async (results) => {
// Guard against undefined results (when readdir fails)
if (results) {
for (const result of results) {
Expand Down
Loading
Loading