From 9371734b4b4215200b0e521b2602640d2df6d72f Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Wed, 18 Mar 2026 12:23:15 +0800 Subject: [PATCH 1/4] chunk datavalue import --- src/data/InstanceDhisRepository.ts | 98 ++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 13 deletions(-) diff --git a/src/data/InstanceDhisRepository.ts b/src/data/InstanceDhisRepository.ts index 8b6b7b7f..7009d449 100644 --- a/src/data/InstanceDhisRepository.ts +++ b/src/data/InstanceDhisRepository.ts @@ -28,7 +28,11 @@ import { DhisInstance } from "../domain/entities/DhisInstance"; import { Locale } from "../domain/entities/Locale"; import { OrgUnit } from "../domain/entities/OrgUnit"; import { NamedRef, Ref } from "../domain/entities/ReferenceObject"; -import { SynchronizationResult } from "../domain/entities/SynchronizationResult"; +import { + SynchronizationResult, + SynchronizationStats, + SynchronizationStatus, +} from "../domain/entities/SynchronizationResult"; import { Program, TrackedEntityInstance } from "../domain/entities/TrackedEntityInstance"; import { BuilderMetadata, @@ -46,6 +50,7 @@ import { D2TrackedEntityType, DataStore, DataValueSetsGetResponse, + DataValueSetsPostResponse, Id, SelectedPick, D2SharingSchema, @@ -56,6 +61,7 @@ import { postEvents } from "./Dhis2Events"; import { getProgram, getTrackedEntityInstances, updateTrackedEntityInstances } from "./Dhis2TrackedEntityInstances"; import { Sharing } from "../domain/entities/Sharing"; import { getMetadataDetailsFromErrors } from "./Dhis2Import"; +import { Maybe } from "../types/utils"; export class InstanceDhisRepository implements InstanceRepository { private api: D2Api; @@ -458,13 +464,28 @@ export class InstanceDhisRepository implements InstanceRepository { const title = importStrategy === "DELETE" ? i18n.t("Data values - Delete") : i18n.t("Data values - Create/update"); - const { response } = await this.api.dataValues - .postSetAsync({ importStrategy }, { dataSet: dataSetId, dataValues }) - .getData(); + if (dataValues.length === 0) { + return { + title, + status: "SUCCESS", + message: i18n.t("No data values to import"), + stats: [{ imported: 0, deleted: 0, updated: 0, ignored: 0 }], + errors: [], + rawResponse: {}, + }; + } - const importSummary = await this.api.system.waitFor(response.jobType, response.id).getData(); + const chunks = _.chunk(dataValues, 3000); - if (!importSummary) { + const chunkResults = await promiseMap(chunks, async chunk => { + const { response } = await this.api.dataValues + .postSetAsync({ importStrategy }, { dataSet: dataSetId, dataValues: chunk }) + .getData(); + + return this.api.system.waitFor(response.jobType, response.id).getData(); + }); + + if (chunkResults.every(r => !r)) { return { title, status: "ERROR", @@ -475,21 +496,72 @@ export class InstanceDhisRepository implements InstanceRepository { }; } - const { status, description, conflicts, importCount } = importSummary; - const { imported, deleted, updated, ignored } = importCount; - const errors = conflicts?.map(({ object, value }) => ({ id: object, message: value, details: "" })) ?? []; + const { mergedStatus, mergedDescription, mergedImportCount, nullChunkStats, summaries } = + this.mergeChunkResults(chunks, chunkResults); + + const allConflicts = _.flatMap(summaries, s => s.conflicts ?? []); + const errors = allConflicts.map(({ object, value }) => ({ id: object, message: value, details: "" })); const errorDetails = await getMetadataDetailsFromErrors(this.api, errors); return { title, - status, - message: description, - stats: [{ imported, deleted, updated, ignored }], + status: mergedStatus, + message: mergedDescription, + stats: [mergedImportCount, ...nullChunkStats], errors: errorDetails, - rawResponse: importSummary, + rawResponse: summaries, }; } + private mergeChunkResults(chunks: AggregatedDataValue[][], chunkResults: Array) { + const emptyChunkCount = chunkResults.filter(r => !r).length; + const hasEmptySummaries = emptyChunkCount > 0; + const summaries = _.compact(chunkResults); + + const uniqueStatuses = _.uniq(summaries.map(s => s.status)); + const mergedStatus: SynchronizationStatus = + hasEmptySummaries || uniqueStatuses.length !== 1 ? "WARNING" : uniqueStatuses[0] ?? "WARNING"; + + const mergedDescription = [ + ..._.uniq(summaries.map(s => s.description).filter(Boolean)), + ...(hasEmptySummaries + ? [ + i18n.t("{{count}} chunk(s) returned no summary — import result unknown for those records.", { + count: emptyChunkCount, + }), + ] + : []), + ].join(" / "); + + const mergedImportCount = summaries.reduce( + (acc, s) => ({ + imported: acc.imported + s.importCount.imported, + deleted: acc.deleted + s.importCount.deleted, + updated: acc.updated + s.importCount.updated, + ignored: acc.ignored + s.importCount.ignored, + }), + { imported: 0, deleted: 0, updated: 0, ignored: 0 } + ); + + // Add a dedicated stat row per null chunk so the UI surfaces unknown outcomes explicitly. + // The total is set to the chunk size so the user knows how many records have an unknown outcome. + const nullChunkStats: SynchronizationStats[] = hasEmptySummaries + ? chunkResults + .map((result, i) => ({ result, chunk: chunks[i] })) + .filter(({ result }) => !result) + .map(({ chunk }) => ({ + type: i18n.t("Chunk (unknown outcome)"), + imported: 0, + deleted: 0, + updated: 0, + ignored: 0, + total: chunk?.length ?? 0, + })) + : []; + + return { mergedStatus, mergedDescription, mergedImportCount, nullChunkStats, summaries }; + } + // TODO: Review when data validation comes in private async validateAggregateImportPackage(dataValues: AggregatedDataValue[]) { const dataElements = _.uniq(dataValues.map(({ dataElement }) => dataElement)); From eaa67730be5566e15fa2bfe7ecb5e41c08229b35 Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:35:08 +0800 Subject: [PATCH 2/4] clean up dependency --- src/data/InstanceDhisRepository.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/data/InstanceDhisRepository.ts b/src/data/InstanceDhisRepository.ts index 7009d449..71488513 100644 --- a/src/data/InstanceDhisRepository.ts +++ b/src/data/InstanceDhisRepository.ts @@ -61,7 +61,6 @@ import { postEvents } from "./Dhis2Events"; import { getProgram, getTrackedEntityInstances, updateTrackedEntityInstances } from "./Dhis2TrackedEntityInstances"; import { Sharing } from "../domain/entities/Sharing"; import { getMetadataDetailsFromErrors } from "./Dhis2Import"; -import { Maybe } from "../types/utils"; export class InstanceDhisRepository implements InstanceRepository { private api: D2Api; From 42fbf64765065758ff7697b107ab953df890d30b Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:37:21 +0800 Subject: [PATCH 3/4] update comment --- src/data/InstanceDhisRepository.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/data/InstanceDhisRepository.ts b/src/data/InstanceDhisRepository.ts index 71488513..6f0c852c 100644 --- a/src/data/InstanceDhisRepository.ts +++ b/src/data/InstanceDhisRepository.ts @@ -542,8 +542,7 @@ export class InstanceDhisRepository implements InstanceRepository { { imported: 0, deleted: 0, updated: 0, ignored: 0 } ); - // Add a dedicated stat row per null chunk so the UI surfaces unknown outcomes explicitly. - // The total is set to the chunk size so the user knows how many records have an unknown outcome. + // Add a dedicated stat row per null chunk with total to unknown outcomes are explicitly. const nullChunkStats: SynchronizationStats[] = hasEmptySummaries ? chunkResults .map((result, i) => ({ result, chunk: chunks[i] })) From 52767ff76607e88964266e456f20c00fdcf298f0 Mon Sep 17 00:00:00 2001 From: gqcorneby <185756521+gqcorneby@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:37:27 +0800 Subject: [PATCH 4/4] translations --- i18n/en.pot | 19 +++++++++++++++++-- i18n/es.po | 18 +++++++++++++++++- i18n/fr.po | 18 +++++++++++++++++- i18n/pt.po | 18 +++++++++++++++++- i18n/ru.po | 19 ++++++++++++++++++- 5 files changed, 86 insertions(+), 6 deletions(-) diff --git a/i18n/en.pot b/i18n/en.pot index 56b0abcf..dfa46fae 100644 --- a/i18n/en.pot +++ b/i18n/en.pot @@ -5,8 +5,8 @@ msgstr "" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1)\n" -"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" -"PO-Revision-Date: 2026-03-11T19:21:15.859Z\n" +"POT-Creation-Date: 2026-03-18T04:23:33.926Z\n" +"PO-Revision-Date: 2026-03-18T04:23:33.926Z\n" msgid "Events - Create/update" msgstr "" @@ -38,9 +38,24 @@ msgstr "" msgid "Data values - Create/update" msgstr "" +msgid "No data values to import" +msgstr "" + msgid "Failed to import data values" msgstr "" +msgid "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgid_plural "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgstr[0] "" +msgstr[1] "" + +msgid "Chunk (unknown outcome)" +msgstr "" + msgid "Service" msgstr "" diff --git a/i18n/es.po b/i18n/es.po index be274f85..74a6780a 100644 --- a/i18n/es.po +++ b/i18n/es.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" +"POT-Creation-Date: 2026-03-18T04:23:33.926Z\n" "Language: es\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -41,9 +41,25 @@ msgstr "Valores de los datos - Borrar" msgid "Data values - Create/update" msgstr "Valores de los datos - Crear/actualizar" +#, fuzzy +msgid "No data values to import" +msgstr "valores de datos" + msgid "Failed to import data values" msgstr "" +msgid "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgid_plural "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgstr[0] "" +msgstr[1] "" + +msgid "Chunk (unknown outcome)" +msgstr "" + msgid "Service" msgstr "" diff --git a/i18n/fr.po b/i18n/fr.po index 38648ff4..05f98fdf 100644 --- a/i18n/fr.po +++ b/i18n/fr.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load App\n" -"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" +"POT-Creation-Date: 2026-03-18T04:23:33.926Z\n" "Language: fr\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -41,9 +41,25 @@ msgstr "" msgid "Data values - Create/update" msgstr "" +#, fuzzy +msgid "No data values to import" +msgstr "valeurs de données" + msgid "Failed to import data values" msgstr "" +msgid "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgid_plural "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgstr[0] "" +msgstr[1] "" + +msgid "Chunk (unknown outcome)" +msgstr "" + msgid "Service" msgstr "" diff --git a/i18n/pt.po b/i18n/pt.po index 926b8a67..39ff4dea 100644 --- a/i18n/pt.po +++ b/i18n/pt.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" +"POT-Creation-Date: 2026-03-18T04:23:33.926Z\n" "Language: pt\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -41,9 +41,25 @@ msgstr "Valores de dados - Eliminar" msgid "Data values - Create/update" msgstr "Valores dos dados - Criar/actualizar" +#, fuzzy +msgid "No data values to import" +msgstr "valores de dados" + msgid "Failed to import data values" msgstr "" +msgid "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgid_plural "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgstr[0] "" +msgstr[1] "" + +msgid "Chunk (unknown outcome)" +msgstr "" + msgid "Service" msgstr "" diff --git a/i18n/ru.po b/i18n/ru.po index 4aa1643f..ce368f6d 100644 --- a/i18n/ru.po +++ b/i18n/ru.po @@ -1,7 +1,7 @@ msgid "" msgstr "" "Project-Id-Version: Bulk Load\n" -"POT-Creation-Date: 2026-03-11T19:21:15.859Z\n" +"POT-Creation-Date: 2026-03-18T04:23:33.926Z\n" "Language: ru\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -42,9 +42,26 @@ msgstr "Значения данных - Удалить" msgid "Data values - Create/update" msgstr "Значения данных - создание/обновление" +#, fuzzy +msgid "No data values to import" +msgstr "значения данных" + msgid "Failed to import data values" msgstr "" +msgid "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgid_plural "" +"{{count}} chunk(s) returned no summary — import result unknown for those " +"records." +msgstr[0] "" +msgstr[1] "" +msgstr[2] "" + +msgid "Chunk (unknown outcome)" +msgstr "" + msgid "Service" msgstr ""