Skip to content

Commit 1e8fbdf

Browse files
naman-contentstacknaman-contentstack
authored
Merge pull request #88 from contentstack/feat/DX-5836
feat: add concurrency in export and update logs in AM
2 parents 248ce74 + e7657a5 commit 1e8fbdf

25 files changed

Lines changed: 389 additions & 145 deletions

File tree

‎.talismanrc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
fileignoreconfig:
22
- filename: pnpm-lock.yaml
3-
checksum: 8b5a2f43585d3191cdc71ad611f50c94b6d13fb7442cf4218ee0851a068af178
3+
checksum: 7a2d08a029dd995917883504dd816fc7a579aca7d3e39fc5368959f0e766c7b2
44
version: '1.0'

‎packages/contentstack-asset-management/src/export/asset-types.ts‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,17 @@ export default class ExportAssetTypes extends AssetManagementExportAdapter {
1313

1414
async start(spaceUid: string): Promise<void> {
1515
await this.init();
16+
17+
log.debug('Starting shared asset types export process...', this.exportContext.context);
18+
1619
const assetTypesData = await this.getWorkspaceAssetTypes(spaceUid);
1720
const items = getArrayFromResponse(assetTypesData, 'asset_types');
1821
const dir = this.getAssetTypesDir();
19-
log.debug(
20-
items.length === 0
21-
? 'No asset types, wrote empty asset-types'
22-
: `Writing ${items.length} shared asset types`,
23-
this.exportContext.context,
24-
);
22+
if (items.length === 0) {
23+
log.info('No asset types to export, writing empty asset-types', this.exportContext.context);
24+
} else {
25+
log.debug(`Writing ${items.length} shared asset types`, this.exportContext.context);
26+
}
2527
await this.writeItemsToChunkedJson(dir, 'asset-types.json', 'asset_types', ['uid', 'title', 'category', 'file_extension'], items);
2628
this.tick(true, PROCESS_NAMES.AM_ASSET_TYPES, null);
2729
}

‎packages/contentstack-asset-management/src/export/assets.ts‎

Lines changed: 52 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import type { AssetManagementAPIConfig, LinkedWorkspace } from '../types/asset-m
77
import type { ExportContext } from '../types/export-types';
88
import { AssetManagementExportAdapter } from './base';
99
import { getAssetItems, writeStreamToFile } from '../utils/export-helpers';
10+
import { runInBatches } from '../utils/concurrent-batch';
1011
import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index';
1112

1213
export default class ExportAssets extends AssetManagementExportAdapter {
@@ -16,8 +17,14 @@ export default class ExportAssets extends AssetManagementExportAdapter {
1617

1718
async start(workspace: LinkedWorkspace, spaceDir: string): Promise<void> {
1819
await this.init();
20+
21+
log.debug(`Starting assets export for space ${workspace.space_uid}`, this.exportContext.context);
22+
log.info(`Exporting asset folders, metadata, and files for space ${workspace.space_uid}`, this.exportContext.context);
23+
1924
const assetsDir = pResolve(spaceDir, 'assets');
2025
await mkdir(assetsDir, { recursive: true });
26+
log.debug(`Assets directory ready: ${assetsDir}`, this.exportContext.context);
27+
2128
log.debug(`Fetching folders and assets for space ${workspace.space_uid}`, this.exportContext.context);
2229

2330
const [folders, assetsData] = await Promise.all([
@@ -43,37 +50,61 @@ export default class ExportAssets extends AssetManagementExportAdapter {
4350
['uid', 'url', 'filename', 'file_name', 'parent_uid'],
4451
assetItems,
4552
);
53+
log.debug(
54+
`Finished writing chunked assets metadata (${assetItems.length} item(s)) under ${assetsDir}`,
55+
this.exportContext.context,
56+
);
57+
log.info(
58+
assetItems.length === 0
59+
? `Wrote empty asset metadata for space ${workspace.space_uid}`
60+
: `Wrote ${assetItems.length} asset metadata record(s) for space ${workspace.space_uid}`,
61+
this.exportContext.context,
62+
);
4663
this.tick(true, `assets: ${workspace.space_uid} (${assetItems.length})`, null);
4764

65+
log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context);
4866
await this.downloadWorkspaceAssets(assetsData, assetsDir, workspace.space_uid);
4967
}
5068

51-
private async downloadWorkspaceAssets(
52-
assetsData: unknown,
53-
assetsDir: string,
54-
spaceUid: string,
55-
): Promise<void> {
69+
private async downloadWorkspaceAssets(assetsData: unknown, assetsDir: string, spaceUid: string): Promise<void> {
5670
const items = getAssetItems(assetsData);
5771
if (items.length === 0) {
72+
log.info(`No asset files to download for space ${spaceUid}`, this.exportContext.context);
5873
log.debug('No assets to download', this.exportContext.context);
5974
return;
6075
}
6176

6277
this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].DOWNLOADING);
78+
log.info(`Downloading asset files for space ${spaceUid} (${items.length} in metadata)`, this.exportContext.context);
6379
log.debug(`Downloading ${items.length} asset file(s) for space ${spaceUid}...`, this.exportContext.context);
6480
const filesDir = pResolve(assetsDir, 'files');
6581
await mkdir(filesDir, { recursive: true });
82+
log.debug(`Asset files directory ready: ${filesDir}`, this.exportContext.context);
6683

6784
const securedAssets = this.exportContext.securedAssets ?? false;
6885
const authtoken = securedAssets ? configHandler.get('authtoken') : null;
86+
log.debug(
87+
`Asset downloads: securedAssets=${securedAssets}, concurrency=${this.downloadAssetsBatchConcurrency}`,
88+
this.exportContext.context,
89+
);
6990
let lastError: string | null = null;
7091
let allSuccess = true;
92+
let downloadOk = 0;
93+
let downloadFail = 0;
7194

72-
for (const asset of items) {
95+
const validItems = items.filter((asset) => Boolean(asset.url && (asset.uid ?? asset._uid)));
96+
const skipped = items.length - validItems.length;
97+
if (skipped > 0) {
98+
log.debug(
99+
`Skipping ${skipped} asset row(s) without url or uid (${validItems.length} file download(s) scheduled)`,
100+
this.exportContext.context,
101+
);
102+
}
103+
await runInBatches(validItems, this.downloadAssetsBatchConcurrency, async (asset) => {
73104
const uid = asset.uid ?? asset._uid;
74105
const url = asset.url;
75106
const filename = asset.filename ?? asset.file_name ?? 'asset';
76-
if (!url || !uid) continue;
107+
if (!url || !uid) return;
77108
try {
78109
const separator = url.includes('?') ? '&' : '?';
79110
const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url;
@@ -86,15 +117,26 @@ export default class ExportAssets extends AssetManagementExportAdapter {
86117
await mkdir(assetFolderPath, { recursive: true });
87118
const filePath = pResolve(assetFolderPath, filename);
88119
await writeStreamToFile(nodeStream, filePath);
89-
log.debug(`Downloaded asset ${uid}`, this.exportContext.context);
120+
downloadOk += 1;
121+
log.debug(`Downloaded asset ${uid} → ${filePath}`, this.exportContext.context);
90122
} catch (e) {
91123
allSuccess = false;
124+
downloadFail += 1;
92125
lastError = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED;
93126
log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context);
94127
}
95-
}
128+
});
96129

97130
this.tick(allSuccess, `downloads: ${spaceUid}`, lastError);
98-
log.debug('Asset downloads completed', this.exportContext.context);
131+
log.info(
132+
allSuccess
133+
? `Finished downloading ${downloadOk} asset file(s) for space ${spaceUid}`
134+
: `Asset downloads for space ${spaceUid} completed with errors: ${downloadOk} succeeded, ${downloadFail} failed`,
135+
this.exportContext.context,
136+
);
137+
log.debug(
138+
`Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}, allSuccess=${allSuccess}`,
139+
this.exportContext.context,
140+
);
99141
}
100142
}

‎packages/contentstack-asset-management/src/export/base.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack
55
import type { AssetManagementAPIConfig } from '../types/asset-management-api';
66
import type { ExportContext } from '../types/export-types';
77
import { AssetManagementAdapter } from '../utils/asset-management-api-adapter';
8-
import { AM_MAIN_PROCESS_NAME, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index';
8+
import { AM_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index';
99

1010
export type { ExportContext };
1111

@@ -63,6 +63,16 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter {
6363
return this.exportContext.spacesRootPath;
6464
}
6565

66+
/** Parallel AM export limit for bootstrap and default batch operations. */
67+
protected get apiConcurrency(): number {
68+
return this.exportContext.apiConcurrency ?? FALLBACK_AM_API_CONCURRENCY;
69+
}
70+
71+
/** Asset download batch size; falls back to {@link apiConcurrency}. */
72+
protected get downloadAssetsBatchConcurrency(): number {
73+
return this.exportContext.downloadAssetsConcurrency ?? this.apiConcurrency;
74+
}
75+
6676
protected getAssetTypesDir(): string {
6777
return pResolve(this.exportContext.spacesRootPath, 'asset_types');
6878
}

‎packages/contentstack-asset-management/src/export/fields.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,17 @@ export default class ExportFields extends AssetManagementExportAdapter {
1313

1414
async start(spaceUid: string): Promise<void> {
1515
await this.init();
16+
17+
log.debug('Starting shared fields export process...', this.exportContext.context);
18+
1619
const fieldsData = await this.getWorkspaceFields(spaceUid);
1720
const items = getArrayFromResponse(fieldsData, 'fields');
1821
const dir = this.getFieldsDir();
19-
log.debug(
20-
items.length === 0 ? 'No field items, wrote empty fields' : `Writing ${items.length} shared fields`,
21-
this.exportContext.context,
22-
);
22+
if (items.length === 0) {
23+
log.info('No field items to export, writing empty fields', this.exportContext.context);
24+
} else {
25+
log.debug(`Writing ${items.length} shared fields`, this.exportContext.context);
26+
}
2327
await this.writeItemsToChunkedJson(dir, 'fields.json', 'fields', ['uid', 'title', 'display_type'], items);
2428
this.tick(true, PROCESS_NAMES.AM_FIELDS, null);
2529
}

‎packages/contentstack-asset-management/src/export/spaces.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { resolve as pResolve } from 'node:path';
22
import { mkdir } from 'node:fs/promises';
3-
import { log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities';
3+
import { log, CLIProgressManager, configHandler, handleAndLogError } from '@contentstack/cli-utilities';
44

55
import type { AssetManagementExportOptions, AssetManagementAPIConfig } from '../types/asset-management-api';
66
import type { ExportContext } from '../types/export-types';
@@ -46,6 +46,8 @@ export class ExportSpaces {
4646
return;
4747
}
4848

49+
log.debug('Starting Asset Management export process...', context);
50+
log.info('Started Asset Management export', context);
4951
log.debug(`Exporting Asset Management 2.0 (${linkedWorkspaces.length} space(s))`, context);
5052
log.debug(`Spaces: ${linkedWorkspaces.map((ws) => ws.space_uid).join(', ')}`, context);
5153

@@ -70,6 +72,8 @@ export class ExportSpaces {
7072
context,
7173
securedAssets,
7274
chunkFileSizeMb,
75+
apiConcurrency: this.options.apiConcurrency,
76+
downloadAssetsConcurrency: this.options.downloadAssetsConcurrency,
7377
};
7478

7579
const sharedFieldsDir = pResolve(spacesRootPath, 'fields');
@@ -81,11 +85,9 @@ export class ExportSpaces {
8185
try {
8286
const exportAssetTypes = new ExportAssetTypes(apiConfig, exportContext);
8387
exportAssetTypes.setParentProgressManager(progress);
84-
await exportAssetTypes.start(firstSpaceUid);
85-
8688
const exportFields = new ExportFields(apiConfig, exportContext);
8789
exportFields.setParentProgressManager(progress);
88-
await exportFields.start(firstSpaceUid);
90+
await Promise.all([exportAssetTypes.start(firstSpaceUid), exportFields.start(firstSpaceUid)]);
8991

9092
for (const ws of linkedWorkspaces) {
9193
progress.updateStatus(`Exporting space: ${ws.space_uid}...`, AM_MAIN_PROCESS_NAME);
@@ -109,9 +111,11 @@ export class ExportSpaces {
109111
}
110112

111113
progress.completeProcess(AM_MAIN_PROCESS_NAME, true);
114+
log.info('Asset Management export completed successfully', context);
112115
log.debug('Asset Management 2.0 export completed', context);
113116
} catch (err) {
114117
progress.completeProcess(AM_MAIN_PROCESS_NAME, false);
118+
handleAndLogError(err, { ...(context as Record<string, unknown>) }, 'Asset Management export failed');
115119
throw err;
116120
}
117121
}

‎packages/contentstack-asset-management/src/export/workspaces.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ export default class ExportWorkspace extends AssetManagementExportAdapter {
1515

1616
async start(workspace: LinkedWorkspace, spaceDir: string, branchName: string): Promise<void> {
1717
await this.init();
18+
19+
log.debug(`Starting export for AM space ${workspace.space_uid}`, this.exportContext.context);
20+
1821
const spaceResponse = await this.getSpace(workspace.space_uid);
1922
const space = spaceResponse.space;
2023
await mkdir(spaceDir, { recursive: true });
@@ -25,7 +28,13 @@ export default class ExportWorkspace extends AssetManagementExportAdapter {
2528
is_default: workspace.is_default,
2629
branch: branchName || 'main',
2730
};
28-
await writeFile(pResolve(spaceDir, 'metadata.json'), JSON.stringify(metadata, null, 2));
31+
const metadataPath = pResolve(spaceDir, 'metadata.json');
32+
try {
33+
await writeFile(metadataPath, JSON.stringify(metadata, null, 2));
34+
} catch (e) {
35+
log.warn(`Could not write ${metadataPath}: ${e}`, this.exportContext.context);
36+
throw e;
37+
}
2938
this.tick(true, `space: ${workspace.space_uid}`, null);
3039
log.debug(`Space metadata written for ${workspace.space_uid}`, this.exportContext.context);
3140

‎packages/contentstack-asset-management/src/import/asset-types.ts‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,30 +31,33 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter {
3131
async start(): Promise<void> {
3232
await this.init();
3333

34+
log.debug('Starting shared asset types import process...', this.importContext.context);
35+
3436
const stripKeys = this.importContext.assetTypesImportInvalidKeys ?? [...FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS];
3537
const dir = this.getAssetTypesDir();
3638
const indexName = this.importContext.assetTypesFileName ?? 'asset-types.json';
3739
const indexPath = join(dir, indexName);
3840

3941
if (!existsSync(indexPath)) {
40-
log.debug('No shared asset types to import (index missing)', this.importContext.context);
42+
log.info('No shared asset types to import (index missing)', this.importContext.context);
4143
return;
4244
}
4345

4446
const existingByUid = await this.loadExistingAssetTypesMap();
4547

46-
this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES);
48+
this.updateStatus(
49+
PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].IMPORTING,
50+
PROCESS_NAMES.AM_IMPORT_ASSET_TYPES,
51+
);
4752

4853
await forEachChunkedJsonStore<Record<string, unknown>>(
4954
dir,
5055
indexName,
5156
{
5257
context: this.importContext.context,
5358
chunkReadLogLabel: 'asset-types',
54-
onOpenError: (e) =>
55-
log.debug(`Could not open chunked asset-types index: ${e}`, this.importContext.context),
56-
onEmptyIndexer: () =>
57-
log.debug('No shared asset types to import (empty indexer)', this.importContext.context),
59+
onOpenError: (e) => log.warn(`Could not open chunked asset-types index: ${e}`, this.importContext.context),
60+
onEmptyIndexer: () => log.debug('No shared asset types to import (empty indexer)', this.importContext.context),
5861
},
5962
async (records) => {
6063
const toCreate = this.buildAssetTypesToCreate(records, existingByUid, stripKeys);
@@ -103,7 +106,10 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter {
103106
this.importContext.context,
104107
);
105108
} else {
106-
log.debug(`Asset type "${uid}" already exists with matching definition, skipping`, this.importContext.context);
109+
log.debug(
110+
`Asset type "${uid}" already exists with matching definition, skipping`,
111+
this.importContext.context,
112+
);
107113
}
108114
this.tick(true, `asset-type: ${uid} (skipped, already exists)`, null, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES);
109115
continue;

0 commit comments

Comments
 (0)